diff options
Diffstat (limited to 'lib')
397 files changed, 103639 insertions, 57868 deletions
diff --git a/lib/arch/arch.nim b/lib/arch/arch.nim deleted file mode 100644 index a11bfb21f..000000000 --- a/lib/arch/arch.nim +++ /dev/null @@ -1,59 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Rokas Kupstys -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -when defined(windows): - const - ABI* = "ms" -elif defined(unix): - const - ABI* = "unix" -else: - {.error: "Unsupported ABI".} - -when defined(amd64): - when defined(unix): - # unix (sysv) ABI - type - JmpBufReg* {.pure.} = enum - BX, BP, R12, R13, R14, R15, SP, IP, TOTAL - elif defined(windows): - # ms ABI - type - JmpBufReg* {.pure.} = enum - BX, BP, R12, R13, R14, R15, SP, IP, SI, DI, TOTAL - type - Reg* {.pure.} = enum - AX, BX, CX, DX, SI, DI, BP, SP, IP, R8, R9, R10, R11, R12, R13, R14, R15, TOTAL - -elif defined(i386): - # identical fastcall calling convention on all x86 OS - type - JmpBufReg* {.pure.} = enum - BX, SI, DI, BP, SP, IP, TOTAL - - Reg* {.pure.} = enum - AX, BX, CX, BP, SP, DI, SI, TOTAL - -else: - {.error: "Unsupported architecture".} - -{.compile: "./" & ABI & "_" & hostCPU & ".asm"} - -type - JmpBuf* = array[JmpBufReg.TOTAL, pointer] - Registers* = array[Reg.TOTAL, pointer] - - -proc getRegisters*(ctx: var Registers) {.importc: "narch_$1", fastcall.} - -proc setjmp*(ctx: var JmpBuf): int {.importc: "narch_$1", fastcall.} -proc longjmp*(ctx: JmpBuf, ret=1) {.importc: "narch_$1", fastcall.} - -proc coroSwitchStack*(sp: pointer) {.importc: "narch_$1", fastcall.} -proc coroRestoreStack*() {.importc: "narch_$1", fastcall.} diff --git a/lib/arch/i386.asm b/lib/arch/i386.asm deleted file mode 100644 index 61f6fdda7..000000000 --- a/lib/arch/i386.asm +++ /dev/null @@ -1,79 +0,0 @@ -; -; -; Nim's Runtime Library -; (c) Copyright 2015 Rokas Kupstys -; -; See the file "copying.txt", included in this -; distribution, for details about the copyright. -; - -section ".text" executable -public narch_getRegisters -public @narch_getRegisters@4 -public narch_setjmp -public @narch_setjmp@4 -public narch_longjmp -public @narch_longjmp@8 -public narch_coroSwitchStack -public @narch_coroSwitchStack@4 -public narch_coroRestoreStack -public @narch_coroRestoreStack@0 - -@narch_getRegisters@4: -narch_getRegisters: - mov [ecx], eax - mov [ecx+4], ebx - mov [ecx+8], ecx - mov [ecx+0Ch], ebp - mov [ecx+10h], esp - mov [ecx+14h], edi - mov [ecx+18h], esi - ret - - -@narch_setjmp@4: -narch_setjmp: - ; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. - mov [ecx], ebx - mov [ecx+4], esi - mov [ecx+8], edi - mov [ecx+0Ch], ebp - lea eax, [esp+4] - mov [ecx+10h], eax - mov eax, [esp] - mov [ecx+14h], eax - xor eax, eax - ret - - -@narch_longjmp@8: -narch_longjmp: - ; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. - mov eax, edx - test eax, eax - jnz @F - inc eax -@@: - mov ebx, [ecx] - mov esi, [ecx+4] - mov edi, [ecx+8] - mov ebp, [ecx+0Ch] - mov esp, [ecx+10h] - mov edx, [ecx+14h] - jmp edx - - -@narch_coroSwitchStack@4: -narch_coroSwitchStack: - pop eax ; return address - mov edx, esp ; old esp for saving - mov esp, ecx ; swap stack with one passed to func - push edx ; store old stack pointer on newly switched stack - jmp eax ; return - - -@narch_coroRestoreStack@0: -narch_coroRestoreStack: - pop eax ; return address - pop esp ; resture old stack pointer - jmp eax ; return diff --git a/lib/arch/ms_amd64.asm b/lib/arch/ms_amd64.asm deleted file mode 100644 index 0503b31c9..000000000 --- a/lib/arch/ms_amd64.asm +++ /dev/null @@ -1,90 +0,0 @@ -; -; -; Nim's Runtime Library -; (c) Copyright 2015 Rokas Kupstys -; -; See the file "copying.txt", included in this -; distribution, for details about the copyright. -; - -format MS64 COFF - -section ".text" executable align 16 -public narch_getRegisters -public narch_setjmp -public narch_longjmp -public narch_coroSwitchStack -public narch_coroRestoreStack - - -narch_getRegisters: - mov [rcx], rax - mov [rcx+8], rbx - mov [rcx+10h], rcx - mov [rcx+18h], rdx - mov [rcx+20h], rsi - mov [rcx+28h], rdi - mov [rcx+30h], rbp - mov [rcx+38h], rsp - mov rax, [rsp] - mov [rcx+40h], rax ; rip - mov [rcx+48h], r8 - mov [rcx+50h], r9 - mov [rcx+58h], r10 - mov [rcx+60h], r11 - mov [rcx+68h], r12 - mov [rcx+70h], r13 - mov [rcx+78h], r14 - mov [rcx+80h], r15 - ret - - -narch_setjmp: - ; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. - mov [rcx], rbx ; rcx is jmp_buf, move registers onto it - mov [rcx+8], rbp - mov [rcx+10h], r12 - mov [rcx+18h], r13 - mov [rcx+20h], r14 - mov [rcx+28h], r15 - lea rdx, [rsp+8] ; this is our rsp WITHOUT current ret addr - mov [rcx+30h], rdx - mov rdx, [rsp] ; save return addr ptr for new rip - mov [rcx+38h], rdx - mov [rcx+40h], rsi - mov [rcx+48h], rdi - xor rax, rax ; always return 0 - ret - -narch_longjmp: - ; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. - mov rax, rdx ; val will be longjmp return - test rax, rax - jnz @F - inc rax ; if val==0, val=1 per longjmp semantics -@@: - mov rbx, [rcx] ; rax is the jmp_buf, restore regs from it - mov rbp, [rcx+8] - mov r12, [rcx+10h] - mov r13, [rcx+18h] - mov r14, [rcx+20h] - mov r15, [rcx+28h] - mov rsp, [rcx+30h] ; this ends up being the stack pointer - mov rdx, [rcx+38h] ; this is the instruction pointer - jmp rdx ; goto saved address without altering rsp - - -narch_coroSwitchStack: - pop rax ; return address - mov rdx, rsp ; old rsp for saving - mov rsp, rcx ; swap stack with one passed to func - push rdx ; store old stack pointer on newly switched stack - sub rsp, 28h ; stack alignment + shadow space - jmp rax ; return - - -narch_coroRestoreStack: - pop rax ; return address - add rsp, 28h ; stack alignment + shadow space - pop rsp ; resture old stack pointer - jmp rax ; return diff --git a/lib/arch/ms_i386.asm b/lib/arch/ms_i386.asm deleted file mode 100644 index a31a698d1..000000000 --- a/lib/arch/ms_i386.asm +++ /dev/null @@ -1,12 +0,0 @@ -; -; -; Nim's Runtime Library -; (c) Copyright 2015 Rokas Kupstys -; -; See the file "copying.txt", included in this -; distribution, for details about the copyright. -; - -format MS COFF - -include 'i386.asm' diff --git a/lib/arch/unix_amd64.asm b/lib/arch/unix_amd64.asm deleted file mode 100644 index 3005c150c..000000000 --- a/lib/arch/unix_amd64.asm +++ /dev/null @@ -1,89 +0,0 @@ -; -; -; Nim's Runtime Library -; (c) Copyright 2015 Rokas Kupstys -; -; See the file "copying.txt", included in this -; distribution, for details about the copyright. -; - -format ELF64 - -section ".text" executable align 16 -public narch_getRegisters -public narch_setjmp -public narch_longjmp -public narch_coroSwitchStack -public narch_coroRestoreStack - - -narch_getRegisters: - mov [rdi], rax - mov [rdi+8], rbx - mov [rdi+10h], rcx - mov [rdi+18h], rdx - mov [rdi+20h], rsi - mov [rdi+28h], rdi - mov [rdi+30h], rbp - mov [rdi+38h], rsp - mov rax, [rsp] - mov [rdi+40h], rax ; rip - mov [rdi+48h], r8 - mov [rdi+50h], r9 - mov [rdi+58h], r10 - mov [rdi+60h], r11 - mov [rdi+68h], r12 - mov [rdi+70h], r13 - mov [rdi+78h], r14 - mov [rdi+80h], r15 - ret - - -narch_setjmp: - ; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. - mov [rdi], rbx ; rdi is jmp_buf, move registers onto it - mov [rdi+8], rbp - mov [rdi+10h], r12 - mov [rdi+18h], r13 - mov [rdi+20h], r14 - mov [rdi+28h], r15 - lea rdx, [rsp+8] ; this is our rsp WITHOUT current ret addr - mov [rdi+30h], rdx - mov rdx, [rsp] ; save return addr ptr for new rip - mov [rdi+38h], rdx - xor rax, rax ; always return 0 - ret - - -narch_longjmp: - ; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. - mov rax, rsi ; val will be longjmp return - test rax, rax - jnz @F - inc rax ; if val==0, val=1 per longjmp semantics -@@: - mov rbx, [rdi] ; rdi is the jmp_buf, restore regs from it - mov rbp, [rdi+8] - mov r12, [rdi+10h] - mov r13, [rdi+18h] - mov r14, [rdi+20h] - mov r15, [rdi+28h] - mov rsp, [rdi+30h] ; this ends up being the stack pointer - mov rdx, [rdi+38h] ; this is the instruction pointer - jmp rdx ; goto saved address without altering rsp - - -narch_coroSwitchStack: - pop rsi ; return address - mov rdx, rsp ; old rsp for saving - mov rsp, rdi ; swap stack with one passed to func - push rdx ; store old stack pointer on newly switched stack - sub rsp, 8h ; stack alignment - jmp rsi ; return - - -narch_coroRestoreStack: - pop rsi ; return address - add rsp, 8h ; stack alignment - pop rsp ; resture old stack pointer - jmp rsi ; return diff --git a/lib/arch/unix_i386.asm b/lib/arch/unix_i386.asm deleted file mode 100644 index 278679067..000000000 --- a/lib/arch/unix_i386.asm +++ /dev/null @@ -1,12 +0,0 @@ -; -; -; Nim's Runtime Library -; (c) Copyright 2015 Rokas Kupstys -; -; See the file "copying.txt", included in this -; distribution, for details about the copyright. -; - -format ELF - -include 'i386.asm' diff --git a/lib/arch/x86/amd64.S b/lib/arch/x86/amd64.S new file mode 100644 index 000000000..47a26f627 --- /dev/null +++ b/lib/arch/x86/amd64.S @@ -0,0 +1,96 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Rokas Kupstys +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +# Partially based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. + +.globl narch_coroExecWithStack +.globl narch_setjmp +.globl narch_longjmp +.text + + +# SysV ABI - first argument is rdi. +# MS ABI - first argument is rcx. +#if defined(__MINGW32__) || defined(__MINGW64__) + #define REG_ARG1 rcx + #define REG_ARG2 rdx +#else + #define REG_ARG1 rdi + #define REG_ARG2 rsi +#endif + + +narch_coroExecWithStack: + mov %REG_ARG2, %rsp # swap stack with one passed to func + sub $0x30, %rsp # shadow space (for ms ABI) 0x20 + 0x10 for possible misalignment + and $-0x10, %rsp # 16-byte stack alignment + call *%REG_ARG1 + + +narch_setjmp: + add $0x10, %REG_ARG1 # 16-byte alignment + and $-0x10, %REG_ARG1 + mov %rbx, 0x00(%REG_ARG1) # jmp_buf, move registers onto it + mov %rbp, 0x08(%REG_ARG1) + mov %r12, 0x10(%REG_ARG1) + mov %r13, 0x18(%REG_ARG1) + mov %r14, 0x20(%REG_ARG1) + mov %r15, 0x28(%REG_ARG1) + lea 0x08(%rsp), %rdx # this is our rsp WITHOUT current ret addr + mov %rdx, 0x30(%REG_ARG1) + mov (%rsp), %rdx # save return addr ptr for new rip + mov %rdx, 0x38(%REG_ARG1) + mov %rsi, 0x40(%REG_ARG1) + mov %rdi, 0x48(%REG_ARG1) +#if defined(__MINGW32__) || defined(__MINGW64__) + movaps %xmm6, 0x50(%REG_ARG1) + movaps %xmm7, 0x60(%REG_ARG1) + movaps %xmm8, 0x70(%REG_ARG1) + movaps %xmm9, 0x80(%REG_ARG1) + movaps %xmm10, 0x90(%REG_ARG1) + movaps %xmm11, 0xA0(%REG_ARG1) + movaps %xmm12, 0xB0(%REG_ARG1) + movaps %xmm13, 0xC0(%REG_ARG1) + movaps %xmm14, 0xD0(%REG_ARG1) + movaps %xmm15, 0xE0(%REG_ARG1) +#endif + xor %rax, %rax # always return 0 + ret + + +narch_longjmp: + add $0x10, %REG_ARG1 # 16-byte alignment + and $-0x10, %REG_ARG1 # + mov %REG_ARG2, %rax # val will be longjmp return + test %rax, %rax + jnz narch_longjmp_1 + inc %rax # if val==0, val=1 per longjmp semantics +narch_longjmp_1: + mov 0x00(%REG_ARG1), %rbx # jmp_buf, restore regs from it + mov 0x08(%REG_ARG1), %rbp + mov 0x10(%REG_ARG1), %r12 + mov 0x18(%REG_ARG1), %r13 + mov 0x20(%REG_ARG1), %r14 + mov 0x28(%REG_ARG1), %r15 + mov 0x30(%REG_ARG1), %rsp # this ends up being the stack pointer + mov 0x38(%REG_ARG1), %rdx # this is the instruction pointer + mov 0x40(%REG_ARG1), %rsi + mov 0x48(%REG_ARG1), %rdi +#if defined(__MINGW32__) || defined(__MINGW64__) + movaps 0x50(%REG_ARG1), %xmm6 + movaps 0x60(%REG_ARG1), %xmm7 + movaps 0x70(%REG_ARG1), %xmm8 + movaps 0x80(%REG_ARG1), %xmm9 + movaps 0x90(%REG_ARG1), %xmm10 + movaps 0xA0(%REG_ARG1), %xmm11 + movaps 0xB0(%REG_ARG1), %xmm12 + movaps 0xC0(%REG_ARG1), %xmm13 + movaps 0xD0(%REG_ARG1), %xmm14 + movaps 0xE0(%REG_ARG1), %xmm15 +#endif + jmp *%rdx # goto saved address without altering rsp diff --git a/lib/arch/x86/i386.S b/lib/arch/x86/i386.S new file mode 100644 index 000000000..d7de4a4c3 --- /dev/null +++ b/lib/arch/x86/i386.S @@ -0,0 +1,64 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Rokas Kupstys +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +# Partially based on code from musl libc Copyright © 2005-2014 Rich Felker, et al. + +.globl narch_coroExecWithStack +.globl narch_setjmp +.globl narch_longjmp +#if defined(__MINGW32__) || defined(__MINGW64__) +.globl @narch_coroExecWithStack@8 +.globl @narch_setjmp@4 +.globl @narch_longjmp@8 +#endif +.text + + +#if defined(__MINGW32__) || defined(__MINGW64__) +@narch_coroExecWithStack@8: +#endif +narch_coroExecWithStack: + mov %edx, %esp # swap stack with one passed to func + sub $0x10, %esp # 16-byte alignment + and $-0x10, %esp # + sub $4, %esp # Simulate misalignment caused by return addr + jmp *%ecx + + +#if defined(__MINGW32__) || defined(__MINGW64__) +@narch_setjmp@4: +#endif +narch_setjmp: + mov %ebx, (%ecx) + mov %esi, 0x04(%ecx) + mov %edi, 0x08(%ecx) + mov %ebp, 0x0C(%ecx) + lea 0x04(%esp), %eax + mov %eax, 0x10(%ecx) + mov (%esp), %eax + mov %eax, 0x14(%ecx) + xor %eax, %eax + ret + + +#if defined(__MINGW32__) || defined(__MINGW64__) +@narch_longjmp@8: +#endif +narch_longjmp: + mov %edx, %eax + test %eax, %eax + jnz narch_longjmp_1 + inc %eax +narch_longjmp_1: + mov (%ecx), %ebx + mov 0x04(%ecx), %esi + mov 0x08(%ecx), %edi + mov 0x0C(%ecx), %ebp + mov 0x10(%ecx), %esp + mov 0x14(%ecx), %edx + jmp *%edx diff --git a/lib/core/hotcodereloading.nim b/lib/core/hotcodereloading.nim new file mode 100644 index 000000000..3a876885c --- /dev/null +++ b/lib/core/hotcodereloading.nim @@ -0,0 +1,41 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Unstable API. + +when defined(hotcodereloading): + import + std/macros + + template beforeCodeReload*(body: untyped) = + hcrAddEventHandler(true, proc = body) {.executeOnReload.} + + template afterCodeReload*(body: untyped) = + hcrAddEventHandler(false, proc = body) {.executeOnReload.} + + macro hasModuleChanged*(module: typed): untyped = + if module.kind != nnkSym or module.symKind != nskModule: + error "hasModuleChanged expects a module symbol", module + return newCall(bindSym"hcrHasModuleChanged", newLit(module.signatureHash)) + + proc hasAnyModuleChanged*(): bool = hcrReloadNeeded() + + when not defined(js): + template performCodeReload* = + when isMainModule: + {.warning: "Code residing in the main module will not be changed from calling a code-reload".} + hcrPerformCodeReload() + else: + template performCodeReload* = discard +else: + template beforeCodeReload*(body: untyped) = discard + template afterCodeReload*(body: untyped) = discard + template hasModuleChanged*(module: typed): bool = false + proc hasAnyModuleChanged*(): bool = false + template performCodeReload*() = discard diff --git a/lib/core/locks.nim b/lib/core/locks.nim index f1a74876e..523727479 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,48 +9,82 @@ ## This module contains Nim's support for locks and condition vars. -include "system/syslocks" +#[ +for js, for now we treat locks as noop's to avoid pushing `when defined(js)` +in client code that uses locks. +]# + +when not compileOption("threads") and not defined(nimdoc): + when false: # fix #12330 + {.error: "Locks requires --threads:on option.".} + +import std/private/syslocks type Lock* = SysLock ## Nim lock; whether this is re-entrant ## or not is unspecified! Cond* = SysCond ## Nim condition variable -{.deprecated: [TLock: Lock, TCond: Cond].} +{.push stackTrace: off.} + + +proc `$`*(lock: Lock): string = + # workaround bug #14873 + result = "()" proc initLock*(lock: var Lock) {.inline.} = ## Initializes the given lock. - initSysLock(lock) + when not defined(js): + initSysLock(lock) -proc deinitLock*(lock: var Lock) {.inline.} = +proc deinitLock*(lock: Lock) {.inline.} = ## Frees the resources associated with the lock. deinitSys(lock) -proc tryAcquire*(lock: var Lock): bool = +proc tryAcquire*(lock: var Lock): bool {.inline.} = ## Tries to acquire the given lock. Returns `true` on success. result = tryAcquireSys(lock) -proc acquire*(lock: var Lock) = +proc acquire*(lock: var Lock) {.inline.} = ## Acquires the given lock. - acquireSys(lock) + when not defined(js): + acquireSys(lock) -proc release*(lock: var Lock) = +proc release*(lock: var Lock) {.inline.} = ## Releases the given lock. - releaseSys(lock) + when not defined(js): + releaseSys(lock) proc initCond*(cond: var Cond) {.inline.} = ## Initializes the given condition variable. initSysCond(cond) -proc deinitCond*(cond: var Cond) {.inline.} = - ## Frees the resources associated with the lock. +proc deinitCond*(cond: Cond) {.inline.} = + ## Frees the resources associated with the condition variable. deinitSysCond(cond) proc wait*(cond: var Cond, lock: var Lock) {.inline.} = - ## waits on the condition variable `cond`. + ## Waits on the condition variable `cond`. waitSysCond(cond, lock) proc signal*(cond: var Cond) {.inline.} = - ## sends a signal to the condition variable `cond`. + ## Sends a signal to the condition variable `cond`. signalSysCond(cond) + +proc broadcast*(cond: var Cond) {.inline.} = + ## Unblocks all threads currently blocked on the + ## specified condition variable `cond`. + broadcastSysCond(cond) + +template withLock*(a: Lock, body: untyped) = + ## Acquires the given lock, executes the statements in body and + ## releases the lock after the statements finish executing. + acquire(a) + {.locks: [a].}: + try: + body + finally: + release(a) + +{.pop.} diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim new file mode 100644 index 000000000..39999fa11 --- /dev/null +++ b/lib/core/macrocache.nim @@ -0,0 +1,246 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides an API for macros to collect compile-time information +## across module boundaries. It should be used instead of global `{.compileTime.}` +## variables as those break incremental compilation. +## +## The main feature of this module is that if you create `CacheTable`s or +## any other `Cache` types with the same name in different modules, their +## content will be shared, meaning that you can fill a `CacheTable` in +## one module, and iterate over its contents in another. + +runnableExamples: + import std/macros + + const mcTable = CacheTable"myTable" + const mcSeq = CacheSeq"mySeq" + const mcCounter = CacheCounter"myCounter" + + static: + # add new key "val" with the value `myval` + let myval = newLit("hello ic") + mcTable["val"] = myval + assert mcTable["val"].kind == nnkStrLit + + # Can access the same cache from different static contexts + # All the information is retained + static: + # get value from `mcTable` and add it to `mcSeq` + mcSeq.add(mcTable["val"]) + assert mcSeq.len == 1 + + static: + assert mcSeq[0].strVal == "hello ic" + + # increase `mcCounter` by 3 + mcCounter.inc(3) + assert mcCounter.value == 3 + + +type + CacheSeq* = distinct string + ## Compile-time sequence of `NimNode`s. + CacheTable* = distinct string + ## Compile-time table of key-value pairs. + ## + ## Keys are `string`s and values are `NimNode`s. + CacheCounter* = distinct string + ## Compile-time counter, uses `int` for storing the count. + +proc value*(c: CacheCounter): int {.magic: "NccValue".} = + ## Returns the value of a counter `c`. + runnableExamples: + static: + let counter = CacheCounter"valTest" + # default value is 0 + assert counter.value == 0 + + inc counter + assert counter.value == 1 + +proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".} = + ## Increments the counter `c` with the value `by`. + runnableExamples: + static: + let counter = CacheCounter"incTest" + inc counter + inc counter, 5 + + assert counter.value == 6 + +proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} = + ## Adds `value` to `s`. + runnableExamples: + import std/macros + const mySeq = CacheSeq"addTest" + + static: + mySeq.add(newLit(5)) + mySeq.add(newLit("hello ic")) + + assert mySeq.len == 2 + assert mySeq[1].strVal == "hello ic" + +proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} = + ## Adds `value` to `s`. + ## + ## .. hint:: This doesn't do anything if `value` is already in `s`. + runnableExamples: + import std/macros + const mySeq = CacheSeq"inclTest" + + static: + mySeq.incl(newLit(5)) + mySeq.incl(newLit(5)) + + # still one element + assert mySeq.len == 1 + +proc len*(s: CacheSeq): int {.magic: "NcsLen".} = + ## Returns the length of `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"lenTest" + static: + let val = newLit("helper") + mySeq.add(val) + assert mySeq.len == 1 + + mySeq.add(val) + assert mySeq.len == 2 + +proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".} = + ## Returns the `i`th value from `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"subTest" + static: + mySeq.add(newLit(42)) + assert mySeq[0].intVal == 42 + +proc `[]`*(s: CacheSeq; i: BackwardsIndex): NimNode = + ## Returns the `i`th last value from `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"backTest" + static: + mySeq &= newLit(42) + mySeq &= newLit(7) + assert mySeq[^1].intVal == 7 # Last item + assert mySeq[^2].intVal == 42 # Second last item + s[s.len - int(i)] + +iterator items*(s: CacheSeq): NimNode = + ## Iterates over each item in `s`. + runnableExamples: + import std/macros + const myseq = CacheSeq"itemsTest" + + static: + myseq.add(newLit(5)) + myseq.add(newLit(42)) + + for val in myseq: + # check that all values in `myseq` are int literals + assert val.kind == nnkIntLit + + for i in 0 ..< len(s): yield s[i] + +proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} = + ## Inserts a `(key, value)` pair into `t`. + ## + ## .. warning:: `key` has to be unique! Assigning `value` to a `key` that is already + ## in the table will result in a compiler error. + runnableExamples: + import std/macros + + const mcTable = CacheTable"subTest" + static: + # assign newLit(5) to the key "value" + mcTable["value"] = newLit(5) + + # check that we can get the value back + assert mcTable["value"].kind == nnkIntLit + +proc len*(t: CacheTable): int {.magic: "NctLen".} = + ## Returns the number of elements in `t`. + runnableExamples: + import std/macros + + const dataTable = CacheTable"lenTest" + static: + dataTable["key"] = newLit(5) + assert dataTable.len == 1 + +proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} = + ## Retrieves the `NimNode` value at `t[key]`. + runnableExamples: + import std/macros + + const mcTable = CacheTable"subTest" + static: + mcTable["toAdd"] = newStmtList() + + # get the NimNode back + assert mcTable["toAdd"].kind == nnkStmtList + +proc hasKey*(t: CacheTable; key: string): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * [contains proc][contains(CacheTable, string)] for use with the `in` operator + runnableExamples: + import std/macros + const mcTable = CacheTable"hasKeyEx" + static: + assert not mcTable.hasKey("foo") + mcTable["foo"] = newEmptyNode() + # Will now be true since we inserted a value + assert mcTable.hasKey("foo") + discard "Implemented in vmops" + +proc contains*(t: CacheTable; key: string): bool {.inline.} = + ## Alias of [hasKey][hasKey(CacheTable, string)] for use with the `in` operator. + runnableExamples: + import std/macros + const mcTable = CacheTable"containsEx" + static: + mcTable["foo"] = newEmptyNode() + # Will be true since we gave it a value before + assert "foo" in mcTable + t.hasKey(key) + +proc hasNext(t: CacheTable; iter: int): bool {.magic: "NctHasNext".} +proc next(t: CacheTable; iter: int): (string, NimNode, int) {.magic: "NctNext".} + +iterator pairs*(t: CacheTable): (string, NimNode) = + ## Iterates over all `(key, value)` pairs in `t`. + runnableExamples: + import std/macros + const mytabl = CacheTable"values" + + static: + mytabl["intVal"] = newLit(5) + mytabl["otherVal"] = newLit(6) + for key, val in mytabl: + # make sure that we actually get the same keys + assert key in ["intVal", "otherVal"] + + # all vals are int literals + assert val.kind == nnkIntLit + + var h = 0 + while hasNext(t, h): + let (a, b, h2) = next(t, h) + yield (a, b) + h = h2 diff --git a/lib/core/macros.nim b/lib/core/macros.nim index c8cb3d199..7646b165c 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -8,11 +8,25 @@ # include "system/inclrtl" +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. +## +## See also: +## * `macros tutorial <tut3.html>`_ +## * `macros section in Nim manual <manual.html#macros>`_ + +## .. include:: ../../doc/astspec.txt + +## .. importdoc:: system.nim -## .. include:: ../doc/astspec.txt +# If you look for the implementation of the magic symbol +# ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`. type NimNodeKind* = enum @@ -21,7 +35,7 @@ type nnkInt16Lit, nnkInt32Lit, nnkInt64Lit, nnkUIntLit, nnkUInt8Lit, nnkUInt16Lit, nnkUInt32Lit, nnkUInt64Lit, nnkFloatLit, nnkFloat32Lit, nnkFloat64Lit, nnkFloat128Lit, nnkStrLit, nnkRStrLit, - nnkTripleStrLit, nnkNilLit, nnkMetaNode, nnkDotCall, + nnkTripleStrLit, nnkNilLit, nnkComesFrom, nnkDotCall, nnkCommand, nnkCall, nnkCallStrLit, nnkInfix, nnkPrefix, nnkPostfix, nnkHiddenCallConv, nnkExprEqExpr, @@ -63,19 +77,29 @@ type nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkRefTy, nnkPtrTy, nnkVarTy, - nnkConstTy, nnkMutableTy, + nnkConstTy, nnkOutTy, nnkDistinctTy, nnkProcTy, nnkIteratorTy, # iterator type - nnkSharedTy, # 'shared T' + nnkSinkAsgn, nnkEnumTy, nnkEnumFieldDef, - nnkArglist, nnkPattern - nnkReturnToken + nnkArgList, nnkPattern + nnkHiddenTryStmt, + nnkClosure, + nnkGotoState, + nnkState, + nnkBreakState, + nnkFuncDef, + nnkTupleConstr, + nnkError, ## erroneous AST node + nnkModuleRef, nnkReplayAction, nnkNilRodNode ## internal IC nodes + nnkOpenSym + NimNodeKinds* = set[NimNodeKind] - NimTypeKind* = enum + NimTypeKind* = enum # some types are no longer used, see ast.nim ntyNone, ntyBool, ntyChar, ntyEmpty, - ntyArrayConstr, ntyNil, ntyExpr, ntyStmt, + ntyAlias, ntyNil, ntyExpr, ntyStmt, ntyTypeDesc, ntyGenericInvocation, ntyGenericBody, ntyGenericInst, ntyGenericParam, ntyDistinct, ntyEnum, ntyOrdinal, ntyArray, ntyObject, ntyTuple, ntySet, @@ -85,97 +109,194 @@ type ntyInt8, ntyInt16, ntyInt32, ntyInt64, ntyFloat, ntyFloat32, ntyFloat64, ntyFloat128, ntyUInt, ntyUInt8, ntyUInt16, ntyUInt32, ntyUInt64, - ntyBigNum, - ntyConst, ntyMutable, ntyVarargs, - ntyIter, + ntyUnused0, ntyUnused1, ntyUnused2, + ntyVarargs, + ntyUncheckedArray, ntyError, - ntyBuiltinTypeClass, ntyConcept, ntyConceptInst, ntyComposite, - ntyAnd, ntyOr, ntyNot + ntyBuiltinTypeClass, ntyUserTypeClass, ntyUserTypeClassInst, + ntyCompositeTypeClass, ntyInferred, ntyAnd, ntyOr, ntyNot, + ntyAnything, ntyStatic, ntyFromExpr, ntyOptDeprecated, ntyVoid TNimTypeKinds* {.deprecated.} = set[NimTypeKind] NimSymKind* = enum nskUnknown, nskConditional, nskDynLib, nskParam, nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet, nskConst, nskResult, - nskProc, nskMethod, nskIterator, nskClosureIterator, + nskProc, nskFunc, nskMethod, nskIterator, nskConverter, nskMacro, nskTemplate, nskField, nskEnumField, nskForVar, nskLabel, nskStub TNimSymKinds* {.deprecated.} = set[NimSymKind] +const + nnkMutableTy* {.deprecated.} = nnkOutTy + nnkSharedTy* {.deprecated.} = nnkSinkAsgn + type - NimIdent* = object of RootObj - ## represents a Nim identifier in the AST + NimIdent* {.deprecated.} = object of RootObj + ## Represents a Nim identifier in the AST. **Note**: This is only + ## rarely useful, for identifier construction from a string + ## use `ident"abc"`. NimSymObj = object # hidden - NimSym* = ref NimSymObj - ## represents a Nim *symbol* in the compiler; a *symbol* is a looked-up + NimSym* {.deprecated.} = ref NimSymObj + ## Represents a Nim *symbol* in the compiler; a *symbol* is a looked-up ## *ident*. -{.deprecated: [TNimrodNodeKind: NimNodeKind, TNimNodeKinds: NimNodeKinds, - TNimrodTypeKind: NimTypeKind, TNimrodSymKind: NimSymKind, - TNimrodIdent: NimIdent, PNimrodSymbol: NimSym].} const nnkLiterals* = {nnkCharLit..nnkNilLit} + # see matching set CallNodes below nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, - nnkCallStrLit} + nnkCallStrLit, nnkHiddenCallConv} + nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit} -proc `[]`*(n: NimNode, i: int): NimNode {.magic: "NChild", noSideEffect.} - ## get `n`'s `i`'th child. - -proc `[]=`*(n: NimNode, i: int, child: NimNode) {.magic: "NSetChild", - noSideEffect.} - ## set `n`'s `i`'th child to `child`. +{.push warnings: off.} -proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect.} - ## constructs an identifier from the string `s` +proc toNimIdent*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated: + "Deprecated since version 0.18.0: Use 'ident' or 'newIdentNode' instead.".} + ## Constructs an identifier from the string `s`. -proc `$`*(i: NimIdent): string {.magic: "IdentToStr", noSideEffect.} - ## converts a Nim identifier to a string +proc `==`*(a, b: NimIdent): bool {.magic: "EqIdent", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Use '==' on 'NimNode' instead.".} + ## Compares two Nim identifiers. -proc `$`*(s: NimSym): string {.magic: "IdentToStr", noSideEffect.} - ## converts a Nim symbol to a string +proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.} + ## Compare two Nim nodes. Return true if nodes are structurally + ## equivalent. This means two independently created nodes can be equal. -proc `==`*(a, b: NimIdent): bool {.magic: "EqIdent", noSideEffect.} - ## compares two Nim identifiers +proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Use '==(NimNode, NimNode)' instead.".} + ## Compares two Nim symbols. -proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.} - ## compares two Nim nodes +{.pop.} proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} = - ## compares two Nim nodes' types. Return true if the types are the same, - ## eg. true when comparing alias with original type. + ## Compares two Nim nodes' types. Return true if the types are the same, + ## e.g. true when comparing alias with original type. discard proc len*(n: NimNode): int {.magic: "NLen", noSideEffect.} - ## returns the number of children of `n`. + ## Returns the number of children of `n`. + +proc `[]`*(n: NimNode, i: int): NimNode {.magic: "NChild", noSideEffect.} + ## Get `n`'s `i`'th child. + +proc `[]`*(n: NimNode, i: BackwardsIndex): NimNode = n[n.len - i.int] + ## Get `n`'s `i`'th child. + +template `^^`(n: NimNode, i: untyped): untyped = + (when i is BackwardsIndex: n.len - int(i) else: int(i)) + +proc `[]`*[T, U: Ordinal](n: NimNode, x: HSlice[T, U]): seq[NimNode] = + ## Slice operation for NimNode. + ## Returns a seq of child of `n` who inclusive range `[n[x.a], n[x.b]]`. + let xa = n ^^ x.a + let L = (n ^^ x.b) - xa + 1 + result = newSeq[NimNode](L) + for i in 0..<L: + result[i] = n[i + xa] + +proc `[]=`*(n: NimNode, i: int, child: NimNode) {.magic: "NSetChild", + noSideEffect.} + ## Set `n`'s `i`'th child to `child`. + +proc `[]=`*(n: NimNode, i: BackwardsIndex, child: NimNode) = + ## Set `n`'s `i`'th child to `child`. + n[n.len - i.int] = child + +template `or`*(x, y: NimNode): NimNode = + ## Evaluate `x` and when it is not an empty node, return + ## it. Otherwise evaluate to `y`. Can be used to chain several + ## expressions to get the first expression that is not empty. + ## ```nim + ## let node = mightBeEmpty() or mightAlsoBeEmpty() or fallbackNode + ## ``` + + let arg = x + if arg != nil and arg.kind != nnkEmpty: + arg + else: + y proc add*(father, child: NimNode): NimNode {.magic: "NAdd", discardable, - noSideEffect, locks: 0.} + noSideEffect.} ## Adds the `child` to the `father` node. Returns the ## father node so that calls can be nested. proc add*(father: NimNode, children: varargs[NimNode]): NimNode {. - magic: "NAddMultiple", discardable, noSideEffect, locks: 0.} + magic: "NAddMultiple", discardable, noSideEffect.} ## Adds each child of `children` to the `father` node. ## Returns the `father` node so that calls can be nested. proc del*(father: NimNode, idx = 0, n = 1) {.magic: "NDel", noSideEffect.} - ## deletes `n` children of `father` starting at index `idx`. + ## Deletes `n` children of `father` starting at index `idx`. proc kind*(n: NimNode): NimNodeKind {.magic: "NKind", noSideEffect.} - ## returns the `kind` of the node `n`. + ## Returns the `kind` of the node `n`. proc intVal*(n: NimNode): BiggestInt {.magic: "NIntVal", noSideEffect.} + ## Returns an integer value from any integer literal or enum field symbol. proc floatVal*(n: NimNode): BiggestFloat {.magic: "NFloatVal", noSideEffect.} -proc symbol*(n: NimNode): NimSym {.magic: "NSymbol", noSideEffect.} -proc ident*(n: NimNode): NimIdent {.magic: "NIdent", noSideEffect.} + ## Returns a float from any floating point literal. + + +proc symKind*(symbol: NimNode): NimSymKind {.magic: "NSymKind", noSideEffect.} +proc getImpl*(symbol: NimNode): NimNode {.magic: "GetImpl", noSideEffect.} + ## Returns a copy of the declaration of a symbol or `nil`. +proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} + ## Returns the string value of an identifier, symbol, comment, or string literal. + ## + ## See also: + ## * `strVal= proc<#strVal=,NimNode,string>`_ for setting the string value. + +{.push warnings: off.} # silence `deprecated` + +proc ident*(n: NimNode): NimIdent {.magic: "NIdent", noSideEffect, deprecated: + "Deprecated since version 0.18.1; All functionality is defined on 'NimNode'.".} + +proc symbol*(n: NimNode): NimSym {.magic: "NSymbol", noSideEffect, deprecated: + "Deprecated since version 0.18.1; All functionality is defined on 'NimNode'.".} + +proc getImpl*(s: NimSym): NimNode {.magic: "GetImpl", noSideEffect, deprecated: "use `getImpl: NimNode -> NimNode` instead".} + +proc `$`*(i: NimIdent): string {.magic: "NStrVal", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Use 'strVal' instead.".} + ## Converts a Nim identifier to a string. + +proc `$`*(s: NimSym): string {.magic: "NStrVal", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Use 'strVal' instead.".} + ## Converts a Nim symbol to a string. + +{.pop.} + +when (NimMajor, NimMinor, NimPatch) >= (1, 3, 5) or defined(nimSymImplTransform): + proc getImplTransformed*(symbol: NimNode): NimNode {.magic: "GetImplTransf", noSideEffect.} + ## For a typed proc returns the AST after transformation pass; this is useful + ## for debugging how the compiler transforms code (e.g.: `defer`, `for`) but + ## note that code transformations are implementation dependent and subject to change. + ## See an example in `tests/macros/tmacros_various.nim`. + +proc owner*(sym: NimNode): NimNode {.magic: "SymOwner", noSideEffect, deprecated.} + ## Accepts a node of kind `nnkSym` and returns its owner's symbol. + ## The meaning of 'owner' depends on `sym`'s `NimSymKind` and declaration + ## context. For top level declarations this is an `nskModule` symbol, + ## for proc local variables an `nskProc` symbol, for enum/object fields an + ## `nskType` symbol, etc. For symbols without an owner, `nil` is returned. + ## + ## See also: + ## * `symKind proc<#symKind,NimNode>`_ to get the kind of a symbol + ## * `getImpl proc<#getImpl,NimNode>`_ to get the declaration of a symbol + +proc isInstantiationOf*(instanceProcSym, genProcSym: NimNode): bool {.magic: "SymIsInstantiationOf", noSideEffect.} + ## Checks if a proc symbol is an instance of the generic proc symbol. + ## Useful to check proc symbols against generic symbols + ## returned by `bindSym`. proc getType*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} - ## with 'getType' you can access the node's `type`:idx:. A Nim type is + ## With 'getType' you can access the node's `type`:idx:. A Nim type is ## mapped to a Nim AST too, so it's slightly confusing but it means the same ## API can be used to traverse types. Recursive types are flattened for you ## so there is no danger of infinite recursions during traversal. To @@ -183,503 +304,1073 @@ proc getType*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} ## kind of type it is, call `typeKind` on getType's result. proc getType*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} - ## Returns the Nim type node for given type. This can be used to turn macro - ## typedesc parameter into proper NimNode representing type, since typedesc - ## are an exception in macro calls - they are not mapped implicitly to - ## NimNode like any other arguments. + ## Version of `getType` which takes a `typedesc`. proc typeKind*(n: NimNode): NimTypeKind {.magic: "NGetType", noSideEffect.} ## Returns the type kind of the node 'n' that should represent a type, that ## means the node should have been obtained via `getType`. -proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} +proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = + ## Returns the `type`:idx: of a node in a form matching the way the + ## type instance was declared in the code. + runnableExamples: + type + Vec[N: static[int], T] = object + arr: array[N, T] + Vec4[T] = Vec[4, T] + Vec4f = Vec4[float32] + var a: Vec4f + var b: Vec4[float32] + var c: Vec[4, float32] + macro dumpTypeInst(x: typed): untyped = + newLit(x.getTypeInst.repr) + doAssert(dumpTypeInst(a) == "Vec4f") + doAssert(dumpTypeInst(b) == "Vec4[float32]") + doAssert(dumpTypeInst(c) == "Vec[4, float32]") + +proc getTypeInst*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} + ## Version of `getTypeInst` which takes a `typedesc`. + +proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = + ## Returns the `type`:idx: of a node in a form matching the implementation + ## of the type. Any intermediate aliases are expanded to arrive at the final + ## type implementation. You can instead use `getImpl` on a symbol if you + ## want to find the intermediate aliases. + runnableExamples: + type + Vec[N: static[int], T] = object + arr: array[N, T] + Vec4[T] = Vec[4, T] + Vec4f = Vec4[float32] + var a: Vec4f + var b: Vec4[float32] + var c: Vec[4, float32] + macro dumpTypeImpl(x: typed): untyped = + newLit(x.getTypeImpl.repr) + let t = """ +object + arr: array[0 .. 3, float32]""" + doAssert(dumpTypeImpl(a) == t) + doAssert(dumpTypeImpl(b) == t) + doAssert(dumpTypeImpl(c) == t) + +proc signatureHash*(n: NimNode): string {.magic: "NSigHash", noSideEffect.} + ## Returns a stable identifier derived from the signature of a symbol. + ## The signature combines many factors such as the type of the symbol, + ## the owning module of the symbol and others. The same identifier is + ## used in the back-end to produce the mangled symbol name. + +proc symBodyHash*(s: NimNode): string {.noSideEffect.} = + ## Returns a stable digest for symbols derived not only from type signature + ## and owning module, but also implementation body. All procs/variables used in + ## the implementation of this symbol are hashed recursively as well, including + ## magics from system module. + discard + +proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} + ## Version of `getTypeImpl` which takes a `typedesc`. proc `intVal=`*(n: NimNode, val: BiggestInt) {.magic: "NSetIntVal", noSideEffect.} proc `floatVal=`*(n: NimNode, val: BiggestFloat) {.magic: "NSetFloatVal", noSideEffect.} -proc `symbol=`*(n: NimNode, val: NimSym) {.magic: "NSetSymbol", noSideEffect.} -proc `ident=`*(n: NimNode, val: NimIdent) {.magic: "NSetIdent", noSideEffect.} -#proc `typ=`*(n: NimNode, typ: typedesc) {.magic: "NSetType".} -# this is not sound! Unfortunately forbidding 'typ=' is not enough, as you -# can easily do: -# let bracket = semCheck([1, 2]) -# let fake = semCheck(2.0) -# bracket[0] = fake # constructs a mixed array with ints and floats! - -proc `strVal=`*(n: NimNode, val: string) {.magic: "NSetStrVal", noSideEffect.} - -proc newNimNode*(kind: NimNodeKind, - n: NimNode=nil): NimNode {.magic: "NNewNimNode", noSideEffect.} -proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} -proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} +{.push warnings: off.} -proc getImpl*(s: NimSym): NimNode {.magic: "GetImpl", noSideEffect.} = - ## retrieve the implementation of a symbol `s`. `s` can be a routine or a - ## const. - discard +proc `symbol=`*(n: NimNode, val: NimSym) {.magic: "NSetSymbol", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Generate a new 'NimNode' with 'genSym' instead.".} -proc error*(msg: string) {.magic: "NError", benign.} - ## writes an error message at compile time +proc `ident=`*(n: NimNode, val: NimIdent) {.magic: "NSetIdent", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Generate a new 'NimNode' with 'ident(string)' instead.".} -proc warning*(msg: string) {.magic: "NWarning", benign.} - ## writes a warning message at compile time +{.pop.} -proc hint*(msg: string) {.magic: "NHint", benign.} - ## writes a hint message at compile time +proc `strVal=`*(n: NimNode, val: string) {.magic: "NSetStrVal", noSideEffect.} + ## Sets the string value of a string literal or comment. + ## Setting `strVal` is disallowed for `nnkIdent` and `nnkSym` nodes; a new node + ## must be created using `ident` or `bindSym` instead. + ## + ## See also: + ## * `strVal proc<#strVal,NimNode>`_ for getting the string value. + ## * `ident proc<#ident,string>`_ for creating an identifier. + ## * `bindSym proc<#bindSym%2C%2CBindSymRule>`_ for binding a symbol. -proc newStrLitNode*(s: string): NimNode {.compileTime, noSideEffect.} = - ## creates a string literal node from `s` +proc newNimNode*(kind: NimNodeKind, + lineInfoFrom: NimNode = nil): NimNode + {.magic: "NNewNimNode", noSideEffect.} + ## Creates a new AST node of the specified kind. + ## + ## The `lineInfoFrom` parameter is used for line information when the + ## produced code crashes. You should ensure that it is set to a node that + ## you are transforming. + +proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} = + ## Creates a new AST node by copying the node `n`. Note that unlike `copyNimTree`, + ## child nodes of `n` are not copied. + runnableExamples: + macro foo(x: typed) = + var s = copyNimNode(x) + doAssert s.len == 0 + doAssert s.kind == nnkStmtList + + foo: + let x = 12 + echo x + +proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} = + ## Creates a new AST node by recursively copying the node `n`. Note that + ## unlike `copyNimNode`, this copies `n`, the children of `n`, etc. + runnableExamples: + macro foo(x: typed) = + var s = copyNimTree(x) + doAssert s.len == 2 + doAssert s.kind == nnkStmtList + + foo: + let x = 12 + echo x + +when defined(nimHasNoReturnError): + {.pragma: errorNoReturn, noreturn.} +else: + {.pragma: errorNoReturn.} + +proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign, errorNoReturn.} + ## Writes an error message at compile time. The optional `n: NimNode` + ## parameter is used as the source for file and line number information in + ## the compilation error message. + +proc warning*(msg: string, n: NimNode = nil) {.magic: "NWarning", benign.} + ## Writes a warning message at compile time. + +proc hint*(msg: string, n: NimNode = nil) {.magic: "NHint", benign.} + ## Writes a hint message at compile time. + +proc newStrLitNode*(s: string): NimNode {.noSideEffect.} = + ## Creates a string literal node from `s`. result = newNimNode(nnkStrLit) result.strVal = s -proc newIntLitNode*(i: BiggestInt): NimNode {.compileTime.} = - ## creates a int literal node from `i` +proc newCommentStmtNode*(s: string): NimNode {.noSideEffect.} = + ## Creates a comment statement node. + result = newNimNode(nnkCommentStmt) + result.strVal = s + +proc newIntLitNode*(i: BiggestInt): NimNode = + ## Creates an int literal node from `i`. result = newNimNode(nnkIntLit) result.intVal = i -proc newFloatLitNode*(f: BiggestFloat): NimNode {.compileTime.} = - ## creates a float literal node from `f` +proc newFloatLitNode*(f: BiggestFloat): NimNode = + ## Creates a float literal node from `f`. result = newNimNode(nnkFloatLit) result.floatVal = f -proc newIdentNode*(i: NimIdent): NimNode {.compileTime.} = - ## creates an identifier node from `i` +{.push warnings: off.} + +proc newIdentNode*(i: NimIdent): NimNode {.deprecated: "use ident(string)".} = + ## Creates an identifier node from `i`. result = newNimNode(nnkIdent) result.ident = i -proc newIdentNode*(i: string): NimNode {.compileTime.} = - ## creates an identifier node from `i` - result = newNimNode(nnkIdent) - result.ident = !i +{.pop.} + +proc newIdentNode*(i: string): NimNode {.magic: "StrToIdent", noSideEffect.} + ## Creates an identifier node from `i`. It is simply an alias for + ## `ident(string)`. Use that, it's shorter. + +proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.} + ## Create a new ident node from a string. type - BindSymRule* = enum ## specifies how ``bindSym`` behaves + BindSymRule* = enum ## Specifies how `bindSym` behaves. The difference + ## between open and closed symbols can be found in + ## `<manual.html#symbol-lookup-in-generics-open-and-closed-symbols>`_ brClosed, ## only the symbols in current scope are bound - brOpen, ## open wrt overloaded symbols, but may be a single + brOpen, ## open for overloaded symbols, but may be a single ## symbol if not ambiguous (the rules match that of ## binding in generics) brForceOpen ## same as brOpen, but it will always be open even ## if not ambiguous (this cannot be achieved with ## any other means in the language currently) -{.deprecated: [TBindSymRule: BindSymRule].} - -proc bindSym*(ident: string, rule: BindSymRule = brClosed): NimNode {. +proc bindSym*(ident: string | NimNode, rule: BindSymRule = brClosed): NimNode {. magic: "NBindSym", noSideEffect.} - ## creates a node that binds `ident` to a symbol node. The bound symbol + ## Creates a node that binds `ident` to a symbol node. The bound symbol ## may be an overloaded symbol. - ## If ``rule == brClosed`` either an ``nkClosedSymChoice`` tree is - ## returned or ``nkSym`` if the symbol is not ambiguous. - ## If ``rule == brOpen`` either an ``nkOpenSymChoice`` tree is - ## returned or ``nkSym`` if the symbol is not ambiguous. - ## If ``rule == brForceOpen`` always an ``nkOpenSymChoice`` tree is + ## if `ident` is a NimNode, it must have `nnkIdent` kind. + ## If `rule == brClosed` either an `nnkClosedSymChoice` tree is + ## returned or `nnkSym` if the symbol is not ambiguous. + ## If `rule == brOpen` either an `nnkOpenSymChoice` tree is + ## returned or `nnkSym` if the symbol is not ambiguous. + ## If `rule == brForceOpen` always an `nnkOpenSymChoice` tree is ## returned even if the symbol is not ambiguous. + ## + ## See the `manual <manual.html#macros-bindsym>`_ for more details. proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. magic: "NGenSym", noSideEffect.} - ## generates a fresh symbol that is guaranteed to be unique. The symbol + ## Generates a fresh symbol that is guaranteed to be unique. The symbol ## needs to occur in a declaration context. -proc callsite*(): NimNode {.magic: "NCallSite", benign.} - ## returns the AST of the invocation expression that invoked this macro. +proc callsite*(): NimNode {.magic: "NCallSite", benign, deprecated: + "Deprecated since v0.18.1; use `varargs[untyped]` in the macro prototype instead".} + ## Returns the AST of the invocation expression that invoked this macro. + # see https://github.com/nim-lang/RFCs/issues/387 as candidate replacement. -proc toStrLit*(n: NimNode): NimNode {.compileTime.} = - ## converts the AST `n` to the concrete Nim code and wraps that - ## in a string literal node +proc toStrLit*(n: NimNode): NimNode = + ## Converts the AST `n` to the concrete Nim code and wraps that + ## in a string literal node. return newStrLitNode(repr(n)) -proc lineinfo*(n: NimNode): string {.magic: "NLineInfo", noSideEffect.} - ## returns the position the node appears in the original source file - ## in the form filename(line, col) - -proc internalParseExpr(s: string): NimNode {. +type + LineInfo* = object + filename*: string + line*,column*: int + +proc `$`*(arg: LineInfo): string = + ## Return a string representation in the form `filepath(line, column)`. + # BUG: without `result = `, gives compile error + result = arg.filename & "(" & $arg.line & ", " & $arg.column & ")" + +#proc lineinfo*(n: NimNode): LineInfo {.magic: "NLineInfo", noSideEffect.} +# ## returns the position the node appears in the original source file +# ## in the form filename(line, col) + +proc getLine(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.} +proc getColumn(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.} +proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.} + +proc copyLineInfo*(arg: NimNode, info: NimNode) {.magic: "NLineInfo", noSideEffect.} + ## Copy lineinfo from `info`. + +proc setLine(arg: NimNode, line: uint16) {.magic: "NLineInfo", noSideEffect.} +proc setColumn(arg: NimNode, column: int16) {.magic: "NLineInfo", noSideEffect.} +proc setFile(arg: NimNode, file: string) {.magic: "NLineInfo", noSideEffect.} + +proc setLineInfo*(arg: NimNode, file: string, line: int, column: int) = + ## Sets the line info on the NimNode. The file needs to exists, but can be a + ## relative path. If you want to attach line info to a block using `quote` + ## you'll need to add the line information after the quote block. + arg.setFile(file) + arg.setLine(line.uint16) + arg.setColumn(column.int16) + +proc setLineInfo*(arg: NimNode, lineInfo: LineInfo) = + ## See `setLineInfo proc<#setLineInfo,NimNode,string,int,int>`_ + setLineInfo(arg, lineInfo.filename, lineInfo.line, lineInfo.column) + +proc lineInfoObj*(n: NimNode): LineInfo = + ## Returns `LineInfo` of `n`, using absolute path for `filename`. + result = LineInfo(filename: n.getFile, line: n.getLine, column: n.getColumn) + +proc lineInfo*(arg: NimNode): string = + ## Return line info in the form `filepath(line, column)`. + $arg.lineInfoObj + +proc internalParseExpr(s, filename: string): NimNode {. magic: "ParseExprToAst", noSideEffect.} -proc internalParseStmt(s: string): NimNode {. +proc internalParseStmt(s, filename: string): NimNode {. magic: "ParseStmtToAst", noSideEffect.} proc internalErrorFlag*(): string {.magic: "NError", noSideEffect.} ## Some builtins set an error flag. This is then turned into a proper ## exception. **Note**: Ordinary application code should not call this. -proc parseExpr*(s: string): NimNode {.noSideEffect, compileTime.} = +proc parseExpr*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. - ## Expects a single expression. Raises ``ValueError`` for parsing errors. - result = internalParseExpr(s) + ## Expects a single expression. Raises `ValueError` for parsing errors. + ## A filename can be given for more informative errors. + result = internalParseExpr(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc parseStmt*(s: string): NimNode {.noSideEffect, compileTime.} = +proc parseStmt*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. - ## Expects one or more statements. Raises ``ValueError`` for parsing errors. - result = internalParseStmt(s) + ## Expects one or more statements. Raises `ValueError` for parsing errors. + ## A filename can be given for more informative errors. + result = internalParseStmt(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc getAst*(macroOrTemplate: expr): NimNode {.magic: "ExpandToAst", noSideEffect.} +proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.} ## Obtains the AST nodes returned from a macro or template invocation. + ## See also `genasts.genAst`. ## Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## macro FooMacro() = ## var ast = getAst(BarTemplate()) + ## ``` -proc quote*(bl: stmt, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} +proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} = ## Quasi-quoting operator. ## Accepts an expression or a block and returns the AST that represents it. ## Within the quoted AST, you are able to interpolate NimNode expressions ## from the surrounding scope. If no operator is given, quoting is done using ## backticks. Otherwise, the given operator must be used as a prefix operator ## for any interpolated expression. The original meaning of the interpolation - ## operator may be obtained by escaping it (by prefixing it with itself): - ## e.g. `@` is escaped as `@@`, `@@` is escaped as `@@@` and so on. + ## operator may be obtained by escaping it (by prefixing it with itself) when used + ## as a unary operator: + ## e.g. `@` is escaped as `@@`, `&%` is escaped as `&%&%` and so on; see examples. ## - ## Example: + ## A custom operator interpolation needs accent quoted (``) whenever it resolves + ## to a symbol. ## - ## .. code-block:: nim - ## - ## macro check(ex: expr): stmt = - ## # this is a simplified version of the check macro from the - ## # unittest module. - ## - ## # If there is a failed check, we want to make it easy for - ## # the user to jump to the faulty line in the code, so we - ## # get the line info here: - ## var info = ex.lineinfo - ## - ## # We will also display the code string of the failed check: - ## var expString = ex.toStrLit - ## - ## # Finally we compose the code to implement the check: - ## result = quote do: - ## if not `ex`: - ## echo `info` & ": Check failed: " & `expString` - -proc expectKind*(n: NimNode, k: NimNodeKind) {.compileTime.} = - ## checks that `n` is of kind `k`. If this is not the case, + ## See also `genasts <genasts.html>`_ which avoids some issues with `quote`. + runnableExamples: + macro check(ex: untyped) = + # this is a simplified version of the check macro from the + # unittest module. + + # If there is a failed check, we want to make it easy for + # the user to jump to the faulty line in the code, so we + # get the line info here: + var info = ex.lineinfo + + # We will also display the code string of the failed check: + var expString = ex.toStrLit + + # Finally we compose the code to implement the check: + result = quote do: + if not `ex`: + echo `info` & ": Check failed: " & `expString` + check 1 + 1 == 2 + + runnableExamples: + # example showing how to define a symbol that requires backtick without + # quoting it. + var destroyCalled = false + macro bar() = + let s = newTree(nnkAccQuoted, ident"=destroy") + # let s = ident"`=destroy`" # this would not work + result = quote do: + type Foo = object + # proc `=destroy`(a: var Foo) = destroyCalled = true # this would not work + proc `s`(a: var Foo) = destroyCalled = true + block: + let a = Foo() + bar() + doAssert destroyCalled + + runnableExamples: + # custom `op` + var destroyCalled = false + macro bar(ident) = + var x = 1.5 + result = quote("@") do: + type Foo = object + let `@ident` = 0 # custom op interpolated symbols need quoted (``) + proc `=destroy`(a: var Foo) = + doAssert @x == 1.5 + doAssert compiles(@x == 1.5) + let b1 = @[1,2] + let b2 = @@[1,2] + doAssert $b1 == "[1, 2]" + doAssert $b2 == "@[1, 2]" + destroyCalled = true + block: + let a = Foo() + bar(someident) + doAssert destroyCalled + + proc `&%`(x: int): int = 1 + proc `&%`(x, y: int): int = 2 + + macro bar2() = + var x = 3 + result = quote("&%") do: + var y = &%x # quoting operator + doAssert &%&%y == 1 # unary operator => need to escape + doAssert y &% y == 2 # binary operator => no need to escape + doAssert y == 3 + bar2() + +proc expectKind*(n: NimNode, k: NimNodeKind) = + ## Checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check the AST that is passed to them. - if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind) + if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind, n) -proc expectMinLen*(n: NimNode, min: int) {.compileTime.} = - ## checks that `n` has at least `min` children. If this is not the case, +proc expectMinLen*(n: NimNode, min: int) = + ## Checks that `n` has at least `min` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. - if n.len < min: error("macro expects a node with " & $min & " children") + if n.len < min: error("Expected a node with at least " & $min & " children, got " & $n.len, n) -proc expectLen*(n: NimNode, len: int) {.compileTime.} = - ## checks that `n` has exactly `len` children. If this is not the case, +proc expectLen*(n: NimNode, len: int) = + ## Checks that `n` has exactly `len` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. - if n.len != len: error("macro expects a node with " & $len & " children") + if n.len != len: error("Expected a node with " & $len & " children, got " & $n.len, n) + +proc expectLen*(n: NimNode, min, max: int) = + ## Checks that `n` has a number of children in the range `min..max`. + ## If this is not the case, compilation aborts with an error message. + ## This is useful for writing macros that check its number of arguments. + if n.len < min or n.len > max: + error("Expected a node with " & $min & ".." & $max & " children, got " & $n.len, n) proc newTree*(kind: NimNodeKind, - children: varargs[NimNode]): NimNode {.compileTime.} = - ## produces a new node with children. + children: varargs[NimNode]): NimNode = + ## Produces a new node with children. result = newNimNode(kind) result.add(children) -proc newCall*(theProc: NimNode, - args: varargs[NimNode]): NimNode {.compileTime.} = - ## produces a new call node. `theProc` is the proc that is called with - ## the arguments ``args[0..]``. +proc newCall*(theProc: NimNode, args: varargs[NimNode]): NimNode = + ## Produces a new call node. `theProc` is the proc that is called with + ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(theProc) result.add(args) -proc newCall*(theProc: NimIdent, - args: varargs[NimNode]): NimNode {.compileTime.} = - ## produces a new call node. `theProc` is the proc that is called with - ## the arguments ``args[0..]``. +{.push warnings: off.} + +proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.deprecated: + "Deprecated since v0.18.1; use 'newCall(string, ...)' or 'newCall(NimNode, ...)' instead".} = + ## Produces a new call node. `theProc` is the proc that is called with + ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(newIdentNode(theProc)) result.add(args) +{.pop.} + proc newCall*(theProc: string, - args: varargs[NimNode]): NimNode {.compileTime.} = - ## produces a new call node. `theProc` is the proc that is called with - ## the arguments ``args[0..]``. + args: varargs[NimNode]): NimNode = + ## Produces a new call node. `theProc` is the proc that is called with + ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(newIdentNode(theProc)) result.add(args) -proc newLit*(c: char): NimNode {.compileTime.} = - ## produces a new character literal node. +proc newLit*(c: char): NimNode = + ## Produces a new character literal node. result = newNimNode(nnkCharLit) result.intVal = ord(c) -proc newLit*(i: BiggestInt): NimNode {.compileTime.} = - ## produces a new integer literal node. +proc newLit*(i: int): NimNode = + ## Produces a new integer literal node. result = newNimNode(nnkIntLit) result.intVal = i -proc newLit*(b: bool): NimNode {.compileTime.} = - ## produces a new boolean literal node. - result = if b: bindSym"true" else: bindSym"false" +proc newLit*(i: int8): NimNode = + ## Produces a new integer literal node. + result = newNimNode(nnkInt8Lit) + result.intVal = i -proc newLit*(f: BiggestFloat): NimNode {.compileTime.} = - ## produces a new float literal node. - result = newNimNode(nnkFloatLit) - result.floatVal = f +proc newLit*(i: int16): NimNode = + ## Produces a new integer literal node. + result = newNimNode(nnkInt16Lit) + result.intVal = i + +proc newLit*(i: int32): NimNode = + ## Produces a new integer literal node. + result = newNimNode(nnkInt32Lit) + result.intVal = i -proc newLit*(s: string): NimNode {.compileTime.} = - ## produces a new string literal node. +proc newLit*(i: int64): NimNode = + ## Produces a new integer literal node. + result = newNimNode(nnkInt64Lit) + result.intVal = i + +proc newLit*(i: uint): NimNode = + ## Produces a new unsigned integer literal node. + result = newNimNode(nnkUIntLit) + result.intVal = BiggestInt(i) + +proc newLit*(i: uint8): NimNode = + ## Produces a new unsigned integer literal node. + result = newNimNode(nnkUInt8Lit) + result.intVal = BiggestInt(i) + +proc newLit*(i: uint16): NimNode = + ## Produces a new unsigned integer literal node. + result = newNimNode(nnkUInt16Lit) + result.intVal = BiggestInt(i) + +proc newLit*(i: uint32): NimNode = + ## Produces a new unsigned integer literal node. + result = newNimNode(nnkUInt32Lit) + result.intVal = BiggestInt(i) + +proc newLit*(i: uint64): NimNode = + ## Produces a new unsigned integer literal node. + result = newNimNode(nnkUInt64Lit) + result.intVal = BiggestInt(i) + +proc newLit*(b: bool): NimNode = + ## Produces a new boolean literal node. + result = if b: bindSym"true" else: bindSym"false" + +proc newLit*(s: string): NimNode = + ## Produces a new string literal node. result = newNimNode(nnkStrLit) result.strVal = s -proc nestList*(theProc: NimIdent, - x: NimNode): NimNode {.compileTime.} = - ## nests the list `x` into a tree of call expressions: - ## ``[a, b, c]`` is transformed into ``theProc(a, theProc(c, d))``. - var L = x.len - result = newCall(theProc, x[L-2], x[L-1]) - for i in countdown(L-3, 0): - # XXX the 'copyNimTree' here is necessary due to a bug in the evaluation - # engine that would otherwise create an endless loop here. :-( - # This could easily user code and so should be fixed in evals.nim somehow. - result = newCall(theProc, x[i], copyNimTree(result)) - -proc treeRepr*(n: NimNode): string {.compileTime, benign.} = +when false: + # the float type is not really a distinct type as described in https://github.com/nim-lang/Nim/issues/5875 + proc newLit*(f: float): NimNode = + ## Produces a new float literal node. + result = newNimNode(nnkFloatLit) + result.floatVal = f + +proc newLit*(f: float32): NimNode = + ## Produces a new float literal node. + result = newNimNode(nnkFloat32Lit) + result.floatVal = f + +proc newLit*(f: float64): NimNode = + ## Produces a new float literal node. + result = newNimNode(nnkFloat64Lit) + result.floatVal = f + +when declared(float128): + proc newLit*(f: float128): NimNode = + ## Produces a new float literal node. + result = newNimNode(nnkFloat128Lit) + result.floatVal = f + +proc newLit*(arg: enum): NimNode = + result = newCall( + arg.typeof.getTypeInst[1], + newLit(int(arg)) + ) + +proc newLit*[N,T](arg: array[N,T]): NimNode +proc newLit*[T](arg: seq[T]): NimNode +proc newLit*[T](s: set[T]): NimNode +proc newLit*[T: tuple](arg: T): NimNode + +proc newLit*(arg: object): NimNode = + result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1]) + for a, b in arg.fieldPairs: + result.add nnkExprColonExpr.newTree( newIdentNode(a), newLit(b) ) + +proc newLit*(arg: ref object): NimNode = + ## produces a new ref type literal node. + result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1]) + for a, b in fieldPairs(arg[]): + result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b)) + +proc newLit*[N,T](arg: array[N,T]): NimNode = + result = nnkBracket.newTree + for x in arg: + result.add newLit(x) + +proc newLit*[T](arg: seq[T]): NimNode = + let bracket = nnkBracket.newTree + for x in arg: + bracket.add newLit(x) + result = nnkPrefix.newTree( + bindSym"@", + bracket + ) + if arg.len == 0: + # add type cast for empty seq + var typ = getTypeInst(typeof(arg))[1] + result = newCall(typ,result) + +proc newLit*[T](s: set[T]): NimNode = + result = nnkCurly.newTree + for x in s: + result.add newLit(x) + if result.len == 0: + # add type cast for empty set + var typ = getTypeInst(typeof(s))[1] + result = newCall(typ,result) + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + ## See `typetraits.isNamedTuple` + +proc newLit*[T: tuple](arg: T): NimNode = + ## use -d:nimHasWorkaround14720 to restore behavior prior to PR, forcing + ## a named tuple even when `arg` is unnamed. + result = nnkTupleConstr.newTree + when defined(nimHasWorkaround14720) or isNamedTuple(T): + for a, b in arg.fieldPairs: + result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b)) + else: + for b in arg.fields: + result.add newLit(b) + +proc nestList*(op: NimNode; pack: NimNode): NimNode = + ## Nests the list `pack` into a tree of call expressions: + ## `[a, b, c]` is transformed into `op(a, op(c, d))`. + ## This is also known as fold expression. + if pack.len < 1: + error("`nestList` expects a node with at least 1 child") + result = pack[^1] + for i in countdown(pack.len - 2, 0): + result = newCall(op, pack[i], result) + +proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode = + ## Nests the list `pack` into a tree of call expressions: + ## `[a, b, c]` is transformed into `op(a, op(c, d))`. + ## This is also known as fold expression. + result = init + for i in countdown(pack.len - 1, 0): + result = newCall(op, pack[i], result) + +proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + +proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` can be an identifier or a + ## symbol. `a` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `b` can be an identifier or a + ## symbol. `b` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` and `b` can be an + ## identifier or a symbol. Both may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +const collapseSymChoice = not defined(nimLegacyMacrosCollapseSymChoice) + +proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indented = false) {.benign.} = + if level > 0: + if indented: + res.add("\n") + for i in 0 .. level-1: + if isLisp: + res.add(" ") # dumpLisp indentation + else: + res.add(" ") # dumpTree indentation + else: + res.add(" ") + + if isLisp: + res.add("(") + res.add(($n.kind).substr(3)) + + case n.kind + of nnkEmpty, nnkNilLit: + discard # same as nil node in this representation + of nnkCharLit .. nnkInt64Lit: + res.add(" " & $n.intVal) + of nnkUIntLit .. nnkUInt64Lit: + res.add(" " & $cast[uint64](n.intVal)) + of nnkFloatLit .. nnkFloat64Lit: + res.add(" " & $n.floatVal) + of nnkStrLit .. nnkTripleStrLit, nnkCommentStmt, nnkIdent, nnkSym: + res.add(" " & $n.strVal.newLit.repr) + of nnkNone: + assert false + elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice: + res.add(" " & $n.len) + if n.len > 0: + var allSameSymName = true + for i in 0..<n.len: + if n[i].kind != nnkSym or not eqIdent(n[i], n[0]): + allSameSymName = false + break + if allSameSymName: + res.add(" " & $n[0].strVal.newLit.repr) + else: + for j in 0 ..< n.len: + n[j].treeTraverse(res, level+1, isLisp, indented) + else: + for j in 0 ..< n.len: + n[j].treeTraverse(res, level+1, isLisp, indented) + + if isLisp: + res.add(")") + +proc treeRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to a human-readable tree-like string. ## - ## See also `repr` and `lispRepr`. + ## See also `repr`, `lispRepr`_, and `astGenRepr`_. + result = "" + n.treeTraverse(result, isLisp = false, indented = true) + +proc lispRepr*(n: NimNode; indented = false): string {.benign.} = + ## Convert the AST `n` to a human-readable lisp-like string. + ## + ## See also `repr`, `treeRepr`_, and `astGenRepr`_. + result = "" + n.treeTraverse(result, isLisp = true, indented = indented) + +proc astGenRepr*(n: NimNode): string {.benign.} = + ## Convert the AST `n` to the code required to generate that AST. + ## + ## See also `repr`_, `treeRepr`_, and `lispRepr`_. + + const + NodeKinds = {nnkEmpty, nnkIdent, nnkSym, nnkNone, nnkCommentStmt} + LitKinds = {nnkCharLit..nnkInt64Lit, nnkFloatLit..nnkFloat64Lit, nnkStrLit..nnkTripleStrLit} + proc traverse(res: var string, level: int, n: NimNode) {.benign.} = for i in 0..level-1: res.add " " - res.add(($n.kind).substr(3)) + if n.kind in NodeKinds: + res.add("new" & ($n.kind).substr(3) & "Node(") + elif n.kind in LitKinds: + res.add("newLit(") + elif n.kind == nnkNilLit: + res.add("newNilLit()") + else: + res.add($n.kind) case n.kind - of nnkEmpty: discard # same as nil node in this representation - of nnkNilLit: res.add(" nil") - of nnkCharLit..nnkInt64Lit: res.add(" " & $n.intVal) - of nnkFloatLit..nnkFloat64Lit: res.add(" " & $n.floatVal) - of nnkStrLit..nnkTripleStrLit: res.add(" " & $n.strVal) - of nnkIdent: res.add(" !\"" & $n.ident & '"') - of nnkSym: res.add(" \"" & $n.symbol & '"') + of nnkEmpty, nnkNilLit: discard + of nnkCharLit: res.add("'" & $chr(n.intVal) & "'") + of nnkIntLit..nnkInt64Lit: res.add($n.intVal) + of nnkFloatLit..nnkFloat64Lit: res.add($n.floatVal) + of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkIdent, nnkSym: + res.add(n.strVal.newLit.repr) of nnkNone: assert false + elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice: + res.add(", # unrepresentable symbols: " & $n.len) + if n.len > 0: + res.add(" " & n[0].strVal.newLit.repr) else: - for j in 0..n.len-1: + res.add(".newTree(") + for j in 0..<n.len: res.add "\n" traverse(res, level + 1, n[j]) + if j != n.len-1: + res.add(",") + + res.add("\n") + for i in 0..level-1: res.add " " + res.add(")") + + if n.kind in NodeKinds+LitKinds: + res.add(")") result = "" traverse(result, 0, n) -proc lispRepr*(n: NimNode): string {.compileTime, benign.} = - ## Convert the AST `n` to a human-readable lisp-like string, +macro dumpTree*(s: untyped): untyped = echo s.treeRepr + ## Accepts a block of nim code and prints the parsed abstract syntax + ## tree using the `treeRepr` proc. Printing is done *at compile time*. ## - ## See also `repr` and `treeRepr`. - - result = ($n.kind).substr(3) - add(result, "(") - - case n.kind - of nnkEmpty: discard # same as nil node in this representation - of nnkNilLit: add(result, "nil") - of nnkCharLit..nnkInt64Lit: add(result, $n.intVal) - of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal) - of nnkStrLit..nnkTripleStrLit: add(result, $n.strVal) - of nnkIdent: add(result, "!\"" & $n.ident & '"') - of nnkSym: add(result, $n.symbol) - of nnkNone: assert false - else: - if n.len > 0: - add(result, lispRepr(n[0])) - for j in 1..n.len-1: - add(result, ", ") - add(result, lispRepr(n[j])) - - add(result, ")") + ## You can use this as a tool to explore the Nim's abstract syntax + ## tree and to discover what kind of nodes must be created to represent + ## a certain expression/statement. + ## + ## For example: + ## ```nim + ## dumpTree: + ## echo "Hello, World!" + ## ``` + ## + ## Outputs: + ## ``` + ## StmtList + ## Command + ## Ident "echo" + ## StrLit "Hello, World!" + ## ``` + ## + ## Also see `dumpAstGen` and `dumpLisp`. -macro dumpTree*(s: stmt): stmt {.immediate.} = echo s.treeRepr +macro dumpLisp*(s: untyped): untyped = echo s.lispRepr(indented = true) ## Accepts a block of nim code and prints the parsed abstract syntax - ## tree using the `toTree` function. Printing is done *at compile time*. + ## tree using the `lispRepr` proc. Printing is done *at compile time*. ## ## You can use this as a tool to explore the Nim's abstract syntax ## tree and to discover what kind of nodes must be created to represent ## a certain expression/statement. + ## + ## For example: + ## ```nim + ## dumpLisp: + ## echo "Hello, World!" + ## ``` + ## + ## Outputs: + ## ``` + ## (StmtList + ## (Command + ## (Ident "echo") + ## (StrLit "Hello, World!"))) + ## ``` + ## + ## Also see `dumpAstGen` and `dumpTree`. -macro dumpLisp*(s: stmt): stmt {.immediate.} = echo s.lispRepr +macro dumpAstGen*(s: untyped): untyped = echo s.astGenRepr ## Accepts a block of nim code and prints the parsed abstract syntax - ## tree using the `toLisp` function. Printing is done *at compile time*. + ## tree using the `astGenRepr` proc. Printing is done *at compile time*. ## - ## See `dumpTree`. - -macro dumpTreeImm*(s: stmt): stmt {.immediate, deprecated.} = echo s.treeRepr - ## The ``immediate`` version of `dumpTree`. - -macro dumpLispImm*(s: stmt): stmt {.immediate, deprecated.} = echo s.lispRepr - ## The ``immediate`` version of `dumpLisp`. - + ## You can use this as a tool to write macros quicker by writing example + ## outputs and then copying the snippets into the macro for modification. + ## + ## For example: + ## ```nim + ## dumpAstGen: + ## echo "Hello, World!" + ## ``` + ## + ## Outputs: + ## ``` + ## nnkStmtList.newTree( + ## nnkCommand.newTree( + ## newIdentNode("echo"), + ## newLit("Hello, World!") + ## ) + ## ) + ## ``` + ## + ## Also see `dumpTree` and `dumpLisp`. -proc newEmptyNode*(): NimNode {.compileTime, noSideEffect.} = - ## Create a new empty node +proc newEmptyNode*(): NimNode {.noSideEffect.} = + ## Create a new empty node. result = newNimNode(nnkEmpty) -proc newStmtList*(stmts: varargs[NimNode]): NimNode {.compileTime.}= - ## Create a new statement list +proc newStmtList*(stmts: varargs[NimNode]): NimNode = + ## Create a new statement list. result = newNimNode(nnkStmtList).add(stmts) -proc newPar*(exprs: varargs[NimNode]): NimNode {.compileTime.}= - ## Create a new parentheses-enclosed expression +proc newPar*(exprs: NimNode): NimNode = + ## Create a new parentheses-enclosed expression. + newNimNode(nnkPar).add(exprs) + +proc newPar*(exprs: varargs[NimNode]): NimNode {.deprecated: + "don't use newPar/nnkPar to construct tuple expressions; use nnkTupleConstr instead".} = + ## Create a new parentheses-enclosed expression. newNimNode(nnkPar).add(exprs) -proc newBlockStmt*(label, body: NimNode): NimNode {.compileTime.} = - ## Create a new block statement with label +proc newBlockStmt*(label, body: NimNode): NimNode = + ## Create a new block statement with label. return newNimNode(nnkBlockStmt).add(label, body) -proc newBlockStmt*(body: NimNode): NimNode {.compiletime.} = - ## Create a new block: stmt +proc newBlockStmt*(body: NimNode): NimNode = + ## Create a new block: stmt. return newNimNode(nnkBlockStmt).add(newEmptyNode(), body) -proc newVarStmt*(name, value: NimNode): NimNode {.compiletime.} = - ## Create a new var stmt +proc newVarStmt*(name, value: NimNode): NimNode = + ## Create a new var stmt. return newNimNode(nnkVarSection).add( newNimNode(nnkIdentDefs).add(name, newNimNode(nnkEmpty), value)) -proc newLetStmt*(name, value: NimNode): NimNode {.compiletime.} = - ## Create a new let stmt +proc newLetStmt*(name, value: NimNode): NimNode = + ## Create a new let stmt. return newNimNode(nnkLetSection).add( newNimNode(nnkIdentDefs).add(name, newNimNode(nnkEmpty), value)) -proc newConstStmt*(name, value: NimNode): NimNode {.compileTime.} = - ## Create a new const stmt +proc newConstStmt*(name, value: NimNode): NimNode = + ## Create a new const stmt. newNimNode(nnkConstSection).add( newNimNode(nnkConstDef).add(name, newNimNode(nnkEmpty), value)) -proc newAssignment*(lhs, rhs: NimNode): NimNode {.compileTime.} = +proc newAssignment*(lhs, rhs: NimNode): NimNode = return newNimNode(nnkAsgn).add(lhs, rhs) -proc newDotExpr*(a, b: NimNode): NimNode {.compileTime.} = - ## Create new dot expression - ## a.dot(b) -> `a.b` +proc newDotExpr*(a, b: NimNode): NimNode = + ## Create new dot expression. + ## a.dot(b) -> `a.b` return newNimNode(nnkDotExpr).add(a, b) -proc newColonExpr*(a, b: NimNode): NimNode {.compileTime.} = - ## Create new colon expression - ## newColonExpr(a, b) -> `a: b` +proc newColonExpr*(a, b: NimNode): NimNode = + ## Create new colon expression. + ## newColonExpr(a, b) -> `a: b` newNimNode(nnkExprColonExpr).add(a, b) proc newIdentDefs*(name, kind: NimNode; - default = newEmptyNode()): NimNode {.compileTime.} = - ## Creates a new ``nnkIdentDefs`` node of a specific kind and value. + default = newEmptyNode()): NimNode = + ## Creates a new `nnkIdentDefs` node of a specific kind and value. ## - ## ``nnkIdentDefs`` need to have at least three children, but they can have + ## `nnkIdentDefs` need to have at least three children, but they can have ## more: first comes a list of identifiers followed by a type and value ## nodes. This helper proc creates a three node subtree, the first subnode - ## being a single identifier name. Both the ``kind`` node and ``default`` - ## (value) nodes may be empty depending on where the ``nnkIdentDefs`` - ## appears: tuple or object definitions will have an empty ``default`` node, - ## ``let`` or ``var`` blocks may have an empty ``kind`` node if the + ## being a single identifier name. Both the `kind` node and `default` + ## (value) nodes may be empty depending on where the `nnkIdentDefs` + ## appears: tuple or object definitions will have an empty `default` node, + ## `let` or `var` blocks may have an empty `kind` node if the ## identifier is being assigned a value. Example: ## - ## .. code-block:: nim - ## + ## ```nim ## var varSection = newNimNode(nnkVarSection).add( ## newIdentDefs(ident("a"), ident("string")), ## newIdentDefs(ident("b"), newEmptyNode(), newLit(3))) ## # --> var ## # a: string ## # b = 3 + ## ``` ## ## If you need to create multiple identifiers you need to use the lower level - ## ``newNimNode``: - ## - ## .. code-block:: nim - ## + ## `newNimNode`: + ## ```nim ## result = newNimNode(nnkIdentDefs).add( ## ident("a"), ident("b"), ident("c"), ident("string"), ## newStrLitNode("Hello")) + ## ``` newNimNode(nnkIdentDefs).add(name, kind, default) -proc newNilLit*(): NimNode {.compileTime.} = - ## New nil literal shortcut +proc newNilLit*(): NimNode = + ## New nil literal shortcut. result = newNimNode(nnkNilLit) -proc last*(node: NimNode): NimNode {.compileTime.} = node[<node.len] - ## Return the last item in nodes children. Same as `node[^1]` +proc last*(node: NimNode): NimNode = node[node.len-1] + ## Return the last item in nodes children. Same as `node[^1]`. const - RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda, nnkIteratorDef} + RoutineNodes* = {nnkProcDef, nnkFuncDef, nnkMethodDef, nnkDo, nnkLambda, + nnkIteratorDef, nnkTemplateDef, nnkConverterDef, nnkMacroDef} AtomicNodes* = {nnkNone..nnkNilLit} - CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, - nnkCallStrLit, nnkHiddenCallConv} + # see matching set nnkCallKinds above + CallNodes* = nnkCallKinds -proc expectKind*(n: NimNode; k: set[NimNodeKind]) {.compileTime.} = - assert n.kind in k, "Expected one of " & $k & ", got " & $n.kind - -proc newProc*(name = newEmptyNode(); params: openArray[NimNode] = [newEmptyNode()]; - body: NimNode = newStmtList(), procType = nnkProcDef): NimNode {.compileTime.} = - ## shortcut for creating a new proc +proc expectKind*(n: NimNode; k: set[NimNodeKind]) = + ## Checks that `n` is of kind `k`. If this is not the case, + ## compilation aborts with an error message. This is useful for writing + ## macros that check the AST that is passed to them. + if n.kind notin k: error("Expected one of " & $k & ", got " & $n.kind, n) + +proc newProc*(name = newEmptyNode(); + params: openArray[NimNode] = [newEmptyNode()]; + body: NimNode = newStmtList(); + procType = nnkProcDef; + pragmas: NimNode = newEmptyNode()): NimNode = + ## Shortcut for creating a new proc. ## - ## The ``params`` array must start with the return type of the proc, + ## The `params` array must start with the return type of the proc, ## followed by a list of IdentDefs which specify the params. - assert procType in RoutineNodes + if procType notin RoutineNodes: + error("Expected one of " & $RoutineNodes & ", got " & $procType) + pragmas.expectKind({nnkEmpty, nnkPragma}) result = newNimNode(procType).add( name, newEmptyNode(), newEmptyNode(), - newNimNode(nnkFormalParams).add(params), ##params - newEmptyNode(), ## pragmas + newNimNode(nnkFormalParams).add(params), + pragmas, newEmptyNode(), body) -proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): - NimNode {.compiletime.} = - ## Constructor for ``if`` statements. - ## - ## .. code-block:: nim - ## - ## newIfStmt( - ## (Ident, StmtList), - ## ... - ## ) - ## +proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): NimNode = + ## Constructor for `if` statements. + ## ```nim + ## newIfStmt( + ## (Ident, StmtList), + ## ... + ## ) + ## ``` result = newNimNode(nnkIfStmt) + if len(branches) < 1: + error("If statement must have at least one branch") for i in branches: - result.add(newNimNode(nnkElifBranch).add(i.cond, i.body)) + result.add(newTree(nnkElifBranch, i.cond, i.body)) + +proc newEnum*(name: NimNode, fields: openArray[NimNode], + public, pure: bool): NimNode = + + ## Creates a new enum. `name` must be an ident. Fields are allowed to be + ## either idents or EnumFieldDef: + ## ```nim + ## newEnum( + ## name = ident("Colors"), + ## fields = [ident("Blue"), ident("Red")], + ## public = true, pure = false) + ## + ## # type Colors* = Blue Red + ## ``` + + expectKind name, nnkIdent + if len(fields) < 1: + error("Enum must contain at least one field") + for field in fields: + expectKind field, {nnkIdent, nnkEnumFieldDef} + + let enumBody = newNimNode(nnkEnumTy).add(newEmptyNode()).add(fields) + var typeDefArgs = [name, newEmptyNode(), enumBody] + if public: + let postNode = newNimNode(nnkPostfix).add( + newIdentNode("*"), typeDefArgs[0]) -proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}= - ## Copy all children from `src` to `dest` - for i in 0 .. < src.len: + typeDefArgs[0] = postNode + + if pure: + let pragmaNode = newNimNode(nnkPragmaExpr).add( + typeDefArgs[0], + add(newNimNode(nnkPragma), newIdentNode("pure"))) + + typeDefArgs[0] = pragmaNode + + let + typeDef = add(newNimNode(nnkTypeDef), typeDefArgs) + typeSect = add(newNimNode(nnkTypeSection), typeDef) + + return typeSect + +proc copyChildrenTo*(src, dest: NimNode) = + ## Copy all children from `src` to `dest`. + for i in 0 ..< src.len: dest.add src[i].copyNimTree -template expectRoutine(node: NimNode): stmt = +template expectRoutine(node: NimNode) = expectKind(node, RoutineNodes) -proc name*(someProc: NimNode): NimNode {.compileTime.} = +proc name*(someProc: NimNode): NimNode = someProc.expectRoutine result = someProc[0] -proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} = - someProc.expectRoutine - someProc[0] = val + if result.kind == nnkPostfix: + if result[1].kind == nnkAccQuoted: + result = result[1][0] + else: + result = result[1] + elif result.kind == nnkAccQuoted: + result = result[0] -proc params*(someProc: NimNode): NimNode {.compileTime.} = - someProc.expectRoutine - result = someProc[3] -proc `params=`* (someProc: NimNode; params: NimNode) {.compileTime.}= +proc `name=`*(someProc: NimNode; val: NimNode) = someProc.expectRoutine - assert params.kind == nnkFormalParams - someProc[3] = params + if someProc[0].kind == nnkPostfix: + someProc[0][1] = val + else: someProc[0] = val -proc pragma*(someProc: NimNode): NimNode {.compileTime.} = - ## Get the pragma of a proc type - ## These will be expanded - someProc.expectRoutine - result = someProc[4] -proc `pragma=`*(someProc: NimNode; val: NimNode){.compileTime.}= - ## Set the pragma of a proc type - someProc.expectRoutine - assert val.kind in {nnkEmpty, nnkPragma} - someProc[4] = val - - -template badNodeKind(k; f): stmt{.immediate.} = - assert false, "Invalid node kind " & $k & " for macros.`" & $f & "`" +proc params*(someProc: NimNode): NimNode = + if someProc.kind == nnkProcTy: + someProc[0] + else: + someProc.expectRoutine + someProc[3] -proc body*(someProc: NimNode): NimNode {.compileTime.} = +proc `params=`* (someProc: NimNode; params: NimNode) = + expectKind(params, nnkFormalParams) + if someProc.kind == nnkProcTy: + someProc[0] = params + else: + someProc.expectRoutine + someProc[3] = params + +proc pragma*(someProc: NimNode): NimNode = + ## Get the pragma of a proc type. + ## These will be expanded. + if someProc.kind == nnkProcTy: + result = someProc[1] + else: + someProc.expectRoutine + result = someProc[4] +proc `pragma=`*(someProc: NimNode; val: NimNode) = + ## Set the pragma of a proc type. + expectKind(val, {nnkEmpty, nnkPragma}) + if someProc.kind == nnkProcTy: + someProc[1] = val + else: + someProc.expectRoutine + someProc[4] = val + +proc addPragma*(someProc, pragma: NimNode) = + ## Adds pragma to routine definition. + someProc.expectKind(RoutineNodes + {nnkProcTy}) + var pragmaNode = someProc.pragma + if pragmaNode.isNil or pragmaNode.kind == nnkEmpty: + pragmaNode = newNimNode(nnkPragma) + someProc.pragma = pragmaNode + pragmaNode.add(pragma) + +template badNodeKind(n, f) = + error("Invalid node kind " & $n.kind & " for macros.`" & $f & "`", n) + +proc body*(someProc: NimNode): NimNode = case someProc.kind: of RoutineNodes: return someProc[6] @@ -688,9 +1379,9 @@ proc body*(someProc: NimNode): NimNode {.compileTime.} = of nnkForStmt: return someProc.last else: - badNodeKind someProc.kind, "body" + badNodeKind someProc, "body" -proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = +proc `body=`*(someProc: NimNode, val: NimNode) = case someProc.kind of RoutineNodes: someProc[6] = val @@ -699,143 +1390,132 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = of nnkForStmt: someProc[len(someProc)-1] = val else: - badNodeKind someProc.kind, "body=" - -proc basename*(a: NimNode): NimNode {.compiletime, benign.} + badNodeKind someProc, "body=" +proc basename*(a: NimNode): NimNode = + ## Pull an identifier from prefix/postfix expressions. + case a.kind + of nnkIdent: result = a + of nnkPostfix, nnkPrefix: result = a[1] + of nnkPragmaExpr: result = basename(a[0]) + else: + error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & + repr(a), a) -proc `$`*(node: NimNode): string {.compileTime.} = - ## Get the string of an identifier node +proc `$`*(node: NimNode): string = + ## Get the string of an identifier node. case node.kind - of nnkIdent: - result = $node.ident of nnkPostfix: - result = $node.basename.ident & "*" - of nnkStrLit..nnkTripleStrLit: + result = node.basename.strVal & "*" + of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkSym, nnkIdent: result = node.strVal - of nnkSym: - result = $node.symbol - of nnkOpenSymChoice, nnkClosedSymChoice: + of nnkOpenSymChoice, nnkClosedSymChoice, nnkOpenSym: result = $node[0] + of nnkAccQuoted: + result = "" + for i in 0 ..< node.len: + result.add(repr(node[i])) else: - badNodeKind node.kind, "$" - -proc ident*(name: string): NimNode {.compileTime,inline.} = newIdentNode(name) - ## Create a new ident node from a string + badNodeKind node, "$" iterator items*(n: NimNode): NimNode {.inline.} = - ## Iterates over the children of the NimNode ``n``. + ## Iterates over the children of the NimNode `n`. for i in 0 ..< n.len: yield n[i] +iterator pairs*(n: NimNode): (int, NimNode) {.inline.} = + ## Iterates over the children of the NimNode `n` and its indices. + for i in 0 ..< n.len: + yield (i, n[i]) + iterator children*(n: NimNode): NimNode {.inline.} = - ## Iterates over the children of the NimNode ``n``. + ## Iterates over the children of the NimNode `n`. for i in 0 ..< n.len: yield n[i] -template findChild*(n: NimNode; cond: expr): NimNode {. - immediate, dirty.} = +template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = ## Find the first child node matching condition (or nil). - ## - ## .. code-block:: nim + ## ```nim ## var res = findChild(n, it.kind == nnkPostfix and - ## it.basename.ident == !"foo") + ## it.basename.ident == ident"foo") + ## ``` block: - var result: NimNode + var res: NimNode for it in n.children: if cond: - result = it + res = it break - result + res -proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = - ## Insert node B into A at pos +proc insert*(a: NimNode; pos: int; b: NimNode) = + ## Insert node `b` into node `a` at `pos`. if len(a)-1 < pos: - ## add some empty nodes first + # add some empty nodes first for i in len(a)-1..pos-2: a.add newEmptyNode() a.add b else: - ## push the last item onto the list again - ## and shift each item down to pos up one + # push the last item onto the list again + # and shift each item down to pos up one a.add(a[a.len-1]) for i in countdown(len(a) - 3, pos): a[i + 1] = a[i] a[pos] = b -proc basename*(a: NimNode): NimNode = - ## Pull an identifier from prefix/postfix expressions - case a.kind - of nnkIdent: return a - of nnkPostfix, nnkPrefix: return a[1] - else: - quit "Do not know how to get basename of (" & treeRepr(a) & ")\n" & repr(a) - -proc `basename=`*(a: NimNode; val: string) {.compileTime.}= +proc `basename=`*(a: NimNode; val: string) = case a.kind - of nnkIdent: macros.`ident=`(a, !val) - of nnkPostfix, nnkPrefix: a[1] = ident(val) + of nnkIdent: + a.strVal = val + of nnkPostfix, nnkPrefix: + a[1] = ident(val) + of nnkPragmaExpr: `basename=`(a[0], val) else: - quit "Do not know how to get basename of (" & treeRepr(a) & ")\n" & repr(a) + error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & + repr(a), a) -proc postfix*(node: NimNode; op: string): NimNode {.compileTime.} = +proc postfix*(node: NimNode; op: string): NimNode = newNimNode(nnkPostfix).add(ident(op), node) -proc prefix*(node: NimNode; op: string): NimNode {.compileTime.} = +proc prefix*(node: NimNode; op: string): NimNode = newNimNode(nnkPrefix).add(ident(op), node) proc infix*(a: NimNode; op: string; - b: NimNode): NimNode {.compileTime.} = + b: NimNode): NimNode = newNimNode(nnkInfix).add(ident(op), a, b) -proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. - compileTime.} = +proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] = node.expectKind nnkPostfix - result = (node[0], $node[1]) + result = (node[1], $node[0]) -proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. - compileTime.} = +proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] = node.expectKind nnkPrefix - result = (node[0], $node[1]) + result = (node[1], $node[0]) -proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; - right: NimNode] {.compileTime.} = - assert node.kind == nnkInfix - result = (node[0], $node[1], node[2]) +proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] = + expectKind(node, nnkInfix) + result = (node[1], $node[0], node[2]) -proc copy*(node: NimNode): NimNode {.compileTime.} = - ## An alias for copyNimTree(). +proc copy*(node: NimNode): NimNode = + ## An alias for `copyNimTree<#copyNimTree,NimNode>`_. return node.copyNimTree() -proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = - proc toLower(c: char): char {.inline.} = - if c in {'A'..'Z'}: result = chr(ord(c) + (ord('a') - ord('A'))) - else: result = c - var i = 0 - var j = 0 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - -proc eqIdent* (a, b: string): bool = cmpIgnoreStyle(a, b) == 0 - ## Check if two idents are identical. - -proc hasArgOfName* (params: NimNode; name: string): bool {.compiletime.}= - ## Search nnkFormalParams for an argument. - assert params.kind == nnkFormalParams - for i in 1 .. <params.len: - template node: expr = params[i] - if name.eqIdent( $ node[0]): - return true - -proc addIdentIfAbsent*(dest: NimNode, ident: string) {.compiletime.} = - ## Add ident to dest if it is not present. This is intended for use +proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} = + ## Check that `eqIdent(n,name)` holds true. If this is not the + ## case, compilation aborts with an error message. This is useful + ## for writing macros that check the AST that is passed to them. + if not eqIdent(n, name): + error("Expected identifier to be `" & name & "` here", n) + +proc hasArgOfName*(params: NimNode; name: string): bool = + ## Search `nnkFormalParams` for an argument. + expectKind(params, nnkFormalParams) + for i in 1..<params.len: + for j in 0..<params[i].len-2: + if name.eqIdent($params[i][j]): + return true + +proc addIdentIfAbsent*(dest: NimNode, ident: string) = + ## Add `ident` to `dest` if it is not present. This is intended for use ## with pragmas. for node in dest.children: case node.kind @@ -846,19 +1526,287 @@ proc addIdentIfAbsent*(dest: NimNode, ident: string) {.compiletime.} = else: discard dest.add(ident(ident)) -proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = +proc boolVal*(n: NimNode): bool {.noSideEffect.} = if n.kind == nnkIntLit: n.intVal != 0 else: n == bindSym"true" # hacky solution for now -when not defined(booting): - template emit*(e: static[string]): stmt = - ## accepts a single string argument and treats it as nim code - ## that should be inserted verbatim in the program - ## Example: - ## - ## .. code-block:: nim - ## emit("echo " & '"' & "hello world".toUpper & '"') - ## - macro payload: stmt {.gensym.} = - result = parseStmt(e) - payload() +proc nodeID*(n: NimNode): int {.magic: "NodeId".} + ## Returns the id of `n`, when the compiler has been compiled + ## with the flag `-d:useNodeids`, otherwise returns `-1`. This + ## proc is for the purpose to debug the compiler only. + +macro expandMacros*(body: typed): untyped = + ## Expands one level of macro - useful for debugging. + ## Can be used to inspect what happens when a macro call is expanded, + ## without altering its result. + ## + ## For instance, + ## + ## ```nim + ## import std/[sugar, macros] + ## + ## let + ## x = 10 + ## y = 20 + ## expandMacros: + ## dump(x + y) + ## ``` + ## + ## will actually dump `x + y`, but at the same time will print at + ## compile time the expansion of the `dump` macro, which in this + ## case is `debugEcho ["x + y", " = ", x + y]`. + echo body.toStrLit + result = body + +proc extractTypeImpl(n: NimNode): NimNode = + ## attempts to extract the type definition of the given symbol + case n.kind + of nnkSym: # can extract an impl + result = n.getImpl.extractTypeImpl() + of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n + of nnkBracketExpr: + if n.typeKind == ntyTypeDesc: + result = n[1].extractTypeImpl() + else: + doAssert n.typeKind == ntyGenericInst + result = n[0].getImpl() + of nnkTypeDef: + result = n[2] + else: error("Invalid node to retrieve type implementation of: " & $n.kind) + +proc customPragmaNode(n: NimNode): NimNode = + expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkType, nnkCheckedFieldExpr}) + let + typ = n.getTypeInst() + + if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy: + return typ[1][1] + elif typ.typeKind == ntyTypeDesc: + let impl = getImpl( + if kind(typ[1]) == nnkBracketExpr: typ[1][0] + else: typ[1] + ) + if impl.kind == nnkNilLit: + return impl + elif impl[0].kind == nnkPragmaExpr: + return impl[0][1] + else: + return impl[0] # handle types which don't have macro at all + + if n.kind == nnkSym: # either an variable or a proc + let impl = n.getImpl() + if impl.kind in RoutineNodes: + return impl.pragma + elif impl.kind in {nnkIdentDefs, nnkConstDef} and impl[0].kind == nnkPragmaExpr: + return impl[0][1] + else: + let timpl = getImpl(if typ.kind == nnkBracketExpr: typ[0] else: typ) + if timpl.len>0 and timpl[0].len>1: + return timpl[0][1] + else: + return timpl + + if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: + let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) + var typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) + while typInst.kind in {nnkVarTy, nnkBracketExpr}: typInst = typInst[0] + var typDef = getImpl(typInst) + while typDef != nil: + typDef.expectKind(nnkTypeDef) + let typ = typDef[2].extractTypeImpl() + if typ.kind notin {nnkRefTy, nnkPtrTy, nnkObjectTy}: break + let isRef = typ.kind in {nnkRefTy, nnkPtrTy} + if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) + typDef = getImpl(typ[0]) + else: # object definition, maybe an object directly defined as a ref type + let + obj = (if isRef: typ[0] else: typ) + var identDefsStack = newSeq[NimNode](obj[2].len) + for i in 0..<identDefsStack.len: identDefsStack[i] = obj[2][i] + while identDefsStack.len > 0: + var identDefs = identDefsStack.pop() + + case identDefs.kind + of nnkRecList: + for child in identDefs.children: + identDefsStack.add(child) + of nnkRecCase: + # Add condition definition + identDefsStack.add(identDefs[0]) + # Add branches + for i in 1 ..< identDefs.len: + identDefsStack.add(identDefs[i].last) + else: + for i in 0 .. identDefs.len - 3: + let varNode = identDefs[i] + if varNode.kind == nnkPragmaExpr: + var varName = varNode[0] + if varName.kind == nnkPostfix: + # This is a public field. We are skipping the postfix * + varName = varName[1] + if eqIdent($varName, name): + return varNode[1] + + if obj[1].kind == nnkOfInherit: # explore the parent object + typDef = getImpl(obj[1][0]) + else: + typDef = nil + +macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = + ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr` + ## (if checking a field), a proc or a type has custom pragma `cp`. + ## + ## See also `getCustomPragmaVal`_. + ## + ## ```nim + ## template myAttr() {.pragma.} + ## type + ## MyObj = object + ## myField {.myAttr.}: int + ## + ## proc myProc() {.myAttr.} = discard + ## + ## var o: MyObj + ## assert(o.myField.hasCustomPragma(myAttr)) + ## assert(myProc.hasCustomPragma(myAttr)) + ## ``` + let pragmaNode = customPragmaNode(n) + for p in pragmaNode: + if (p.kind == nnkSym and p == cp) or + (p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp): + return newLit(true) + return newLit(false) + +macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = + ## Expands to value of custom pragma `cp` of expression `n` which is expected + ## to be `nnkDotExpr`, a proc or a type. + ## + ## See also `hasCustomPragma`_. + ## + ## ```nim + ## template serializationKey(key: string) {.pragma.} + ## type + ## MyObj {.serializationKey: "mo".} = object + ## myField {.serializationKey: "mf".}: int + ## var o: MyObj + ## assert(o.myField.getCustomPragmaVal(serializationKey) == "mf") + ## assert(o.getCustomPragmaVal(serializationKey) == "mo") + ## assert(MyObj.getCustomPragmaVal(serializationKey) == "mo") + ## ``` + result = nil + let pragmaNode = customPragmaNode(n) + for p in pragmaNode: + if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: + if p.len == 2 or (p.len == 3 and p[1].kind == nnkSym and p[1].symKind == nskType): + result = p[1] + else: + let def = p[0].getImpl[3] + result = newTree(nnkPar) + for i in 1 ..< def.len: + let key = def[i][0] + let val = p[i] + result.add newTree(nnkExprColonExpr, key, val) + break + if result.kind == nnkEmpty: + error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error, + +macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped = + ## Calls `callee` with `args` unpacked as individual arguments. + ## This is useful in 2 cases: + ## * when forwarding `varargs[T]` for some typed `T` + ## * when forwarding `varargs[untyped]` when `args` can potentially be empty, + ## due to a compiler limitation + runnableExamples: + template call1(fun: typed; args: varargs[untyped]): untyped = + unpackVarargs(fun, args) + # when varargsLen(args) > 0: fun(args) else: fun() # this would also work + template call2(fun: typed; args: varargs[typed]): untyped = + unpackVarargs(fun, args) + proc fn1(a = 0, b = 1) = discard (a, b) + call1(fn1, 10, 11) + call1(fn1) # `args` is empty in this case + if false: call2(echo, 10, 11) # would print 1011 + result = newCall(callee) + for i in 0 ..< args.len: + result.add args[i] + +proc getProjectPath*(): string = discard + ## Returns the path to the currently compiling project. + ## + ## This is not to be confused with `system.currentSourcePath <system.html#currentSourcePath.t>`_ + ## which returns the path of the source file containing that template + ## call. + ## + ## For example, assume a `dir1/foo.nim` that imports a `dir2/bar.nim`, + ## have the `bar.nim` print out both `getProjectPath` and + ## `currentSourcePath` outputs. + ## + ## Now when `foo.nim` is compiled, the `getProjectPath` from + ## `bar.nim` will return the `dir1/` path, while the `currentSourcePath` + ## will return the path to the `bar.nim` source file. + ## + ## Now when `bar.nim` is compiled directly, the `getProjectPath` + ## will now return the `dir2/` path, and the `currentSourcePath` + ## will still return the same path, the path to the `bar.nim` source + ## file. + ## + ## The path returned by this proc is set at compile time. + ## + ## See also: + ## * `getCurrentDir proc <os.html#getCurrentDir>`_ + +proc getSize*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = + ## Returns the same result as `system.sizeof` if the size is + ## known by the Nim compiler. Returns a negative value if the Nim + ## compiler does not know the size. +proc getAlign*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = + ## Returns the same result as `system.alignof` if the alignment + ## is known by the Nim compiler. It works on `NimNode` for use + ## in macro context. Returns a negative value if the Nim compiler + ## does not know the alignment. +proc getOffset*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = + ## Returns the same result as `system.offsetof` if the offset is + ## known by the Nim compiler. It expects a resolved symbol node + ## from a field of a type. Therefore it only requires one argument + ## instead of two. Returns a negative value if the Nim compiler + ## does not know the offset. + +proc isExported*(n: NimNode): bool {.noSideEffect.} = + ## Returns whether the symbol is exported or not. + +proc extractDocCommentsAndRunnables*(n: NimNode): NimNode = + ## returns a `nnkStmtList` containing the top-level doc comments and + ## runnableExamples in `a`, stopping at the first child that is neither. + ## Example: + ## + ## ```nim + ## import std/macros + ## macro transf(a): untyped = + ## result = quote do: + ## proc fun2*() = discard + ## let header = extractDocCommentsAndRunnables(a.body) + ## # correct usage: rest is appended + ## result.body = header + ## result.body.add quote do: discard # just an example + ## # incorrect usage: nesting inside a nnkStmtList: + ## # result.body = quote do: (`header`; discard) + ## + ## proc fun*() {.transf.} = + ## ## first comment + ## runnableExamples: discard + ## runnableExamples: discard + ## ## last comment + ## discard # first statement after doc comments + runnableExamples + ## ## not docgen'd + ## ``` + + result = newStmtList() + for ni in n: + case ni.kind + of nnkCommentStmt: + result.add ni + of nnkCall, nnkCommand: + if ni[0].kind == nnkIdent and ni[0].eqIdent "runnableExamples": + result.add ni + else: break + else: break diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim new file mode 100644 index 000000000..8cb0cef05 --- /dev/null +++ b/lib/core/rlocks.nim @@ -0,0 +1,57 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Anatoly Galiulin +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains Nim's support for reentrant locks. + + +when not compileOption("threads") and not defined(nimdoc): + when false: + # make rlocks modlue consistent with locks module, + # so they can replace each other seamlessly. + {.error: "Rlocks requires --threads:on option.".} + +import std/private/syslocks + +type + RLock* = SysLock ## Nim lock, re-entrant + +proc initRLock*(lock: var RLock) {.inline.} = + ## Initializes the given lock. + when defined(posix): + var a: SysLockAttr + initSysLockAttr(a) + setSysLockType(a, SysLockType_Reentrant) + initSysLock(lock, a.addr) + else: + initSysLock(lock) + +proc deinitRLock*(lock: RLock) {.inline.} = + ## Frees the resources associated with the lock. + deinitSys(lock) + +proc tryAcquire*(lock: var RLock): bool {.inline.} = + ## Tries to acquire the given lock. Returns `true` on success. + result = tryAcquireSys(lock) + +proc acquire*(lock: var RLock) {.inline.} = + ## Acquires the given lock. + acquireSys(lock) + +proc release*(lock: var RLock) {.inline.} = + ## Releases the given lock. + releaseSys(lock) + +template withRLock*(lock: RLock, code: untyped) = + ## Acquires the given lock and then executes the code. + acquire(lock) + {.locks: [lock].}: + try: + code + finally: + release(lock) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index db5a83755..f2fee91c4 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -8,12 +8,30 @@ # ## This module implements an interface to Nim's `runtime type information`:idx: -## (`RTTI`:idx:). -## Note that even though ``Any`` and its operations hide the nasty low level -## details from its clients, it remains inherently unsafe! +## (`RTTI`:idx:). See the `marshal <marshal.html>`_ module for an example of +## what this allows you to do. ## -## See the `marshal <marshal.html>`_ module for what this module allows you -## to do. +## .. note:: Even though `Any` and its operations hide the nasty low level +## details from its users, it remains inherently unsafe! Also, Nim's +## runtime type information will evolve and may eventually be deprecated. +## As an alternative approach to programmatically understanding and +## manipulating types, consider using the `macros <macros.html>`_ module to +## work with the types' AST representation at compile time. See for example +## the `getTypeImpl proc <macros.html#getTypeImpl,NimNode>`_. As an alternative +## approach to storing arbitrary types at runtime, consider using generics. + +runnableExamples: + var x: Any + + var i = 42 + x = i.toAny + assert x.kind == akInt + assert x.getInt == 42 + + var s = @[1, 2, 3] + x = s.toAny + assert x.kind == akSequence + assert x.len == 3 {.push hints: off.} @@ -22,44 +40,50 @@ include "system/hti.nim" {.pop.} +when defined(nimPreviewSlimSystem): + import std/assertions + + type - AnyKind* = enum ## what kind of ``any`` it is - akNone = 0, ## invalid any - akBool = 1, ## any represents a ``bool`` - akChar = 2, ## any represents a ``char`` - akEnum = 14, ## any represents an enum - akArray = 16, ## any represents an array - akObject = 17, ## any represents an object - akTuple = 18, ## any represents a tuple - akSet = 19, ## any represents a set - akRange = 20, ## any represents a range - akPtr = 21, ## any represents a ptr - akRef = 22, ## any represents a ref - akSequence = 24, ## any represents a sequence - akProc = 25, ## any represents a proc - akPointer = 26, ## any represents a pointer - akString = 28, ## any represents a string - akCString = 29, ## any represents a cstring - akInt = 31, ## any represents an int - akInt8 = 32, ## any represents an int8 - akInt16 = 33, ## any represents an int16 - akInt32 = 34, ## any represents an int32 - akInt64 = 35, ## any represents an int64 - akFloat = 36, ## any represents a float - akFloat32 = 37, ## any represents a float32 - akFloat64 = 38, ## any represents a float64 - akFloat128 = 39, ## any represents a float128 - akUInt = 40, ## any represents an unsigned int - akUInt8 = 41, ## any represents an unsigned int8 - akUInt16 = 42, ## any represents an unsigned in16 - akUInt32 = 43, ## any represents an unsigned int32 - akUInt64 = 44, ## any represents an unsigned int64 - - Any* = object ## can represent any nim value; NOTE: the wrapped - ## value can be modified with its wrapper! This means - ## that ``Any`` keeps a non-traced pointer to its - ## wrapped value and **must not** live longer than - ## its wrapped value. + AnyKind* = enum ## The kind of `Any`. + akNone = 0, ## invalid + akBool = 1, ## bool + akChar = 2, ## char + akEnum = 14, ## enum + akArray = 16, ## array + akObject = 17, ## object + akTuple = 18, ## tuple + akSet = 19, ## set + akRange = 20, ## range + akPtr = 21, ## ptr + akRef = 22, ## ref + akSequence = 24, ## sequence + akProc = 25, ## proc + akPointer = 26, ## pointer + akString = 28, ## string + akCString = 29, ## cstring + akInt = 31, ## int + akInt8 = 32, ## int8 + akInt16 = 33, ## int16 + akInt32 = 34, ## int32 + akInt64 = 35, ## int64 + akFloat = 36, ## float + akFloat32 = 37, ## float32 + akFloat64 = 38, ## float64 + akFloat128 = 39, ## float128 + akUInt = 40, ## uint + akUInt8 = 41, ## uint8 + akUInt16 = 42, ## uin16 + akUInt32 = 43, ## uint32 + akUInt64 = 44, ## uint64 +# akOpt = 44+18 ## the builtin 'opt' type. + + Any* = object + ## A type that can represent any nim value. + ## + ## .. danger:: The wrapped value can be modified with its wrapper! This means + ## that `Any` keeps a non-traced pointer to its wrapped value and + ## **must not** live longer than its wrapped value. value: pointer when defined(js): rawType: PNimType @@ -67,13 +91,25 @@ type rawTypePtr: pointer ppointer = ptr pointer - pbyteArray = ptr array[0.. 0xffff, int8] + pbyteArray = ptr array[0xffff, uint8] + +when not defined(gcDestructors): + type + TGenericSeq {.importc.} = object + len, space: int + when defined(gogc): + elemSize: int + PGenSeq = ptr TGenericSeq + + when defined(gogc): + const GenericSeqSize = 3 * sizeof(int) + else: + const GenericSeqSize = 2 * sizeof(int) - TGenericSeq {.importc.} = object - len, space: int - when defined(gogc): - elemSize: int - PGenSeq = ptr TGenericSeq +else: + include system/seqs_v2_reimpl + +from std/private/strimpl import cmpNimIdentifier when not defined(js): template rawType(x: Any): PNimType = @@ -82,40 +118,42 @@ when not defined(js): template `rawType=`(x: var Any, p: PNimType) = x.rawTypePtr = cast[pointer](p) -{.deprecated: [TAny: Any, TAnyKind: AnyKind].} +proc genericAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} -when defined(gogc): - const GenericSeqSize = (3 * sizeof(int)) +when not defined(gcDestructors): + proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} + proc incrSeq(seq: PGenSeq, elemSize, elemAlign: int): PGenSeq {.importCompilerProc.} + proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.} + proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.} + proc objectInit(dest: pointer, typ: PNimType) {.importCompilerProc.} else: - const GenericSeqSize = (2 * sizeof(int)) - -proc genericAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} -proc genericShallowAssign(dest, src: pointer, mt: PNimType) {. - importCompilerProc.} -proc incrSeq(seq: PGenSeq, elemSize: int): PGenSeq {.importCompilerProc.} -proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.} -proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.} -proc objectInit(dest: pointer, typ: PNimType) {.importCompilerProc.} + proc nimNewObj(size, align: int): pointer {.importCompilerProc.} + proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.importCompilerProc.} + proc prepareSeqAddUninit(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + importCompilerProc.} + proc zeroNewElements(len: int; p: pointer; addlen, elemSize, elemAlign: int) {. + importCompilerProc.} -template `+!!`(a, b: expr): expr = cast[pointer](cast[ByteAddress](a) + b) +template `+!!`(a, b): untyped = cast[pointer](cast[int](a) + b) proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = assert(n.kind == nkCase) var d: int - var a = cast[ByteAddress](aa) + let a = cast[int](aa) case n.typ.size - of 1: d = ze(cast[ptr int8](a +% n.offset)[]) - of 2: d = ze(cast[ptr int16](a +% n.offset)[]) - of 4: d = int(cast[ptr int32](a +% n.offset)[]) + of 1: d = int(cast[ptr uint8](a +% n.offset)[]) + of 2: d = int(cast[ptr uint16](a +% n.offset)[]) + of 4: d = int(cast[ptr uint32](a +% n.offset)[]) + of 8: d = int(cast[ptr uint64](a +% n.offset)[]) else: assert(false) return d proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = - var discr = getDiscriminant(aa, n) + let discr = getDiscriminant(aa, n) if discr <% n.len: result = n.sons[discr] if result == nil: result = n.sons[n.len] - # n.sons[n.len] contains the ``else`` part (but may be nil) + # n.sons[n.len] contains the `else` part (but may be nil) else: result = n.sons[n.len] @@ -123,143 +161,188 @@ proc newAny(value: pointer, rawType: PNimType): Any {.inline.} = result.value = value result.rawType = rawType -when declared(system.VarSlot): - proc toAny*(x: VarSlot): Any {.inline.} = - ## constructs a ``Any`` object from a variable slot ``x``. - ## This captures `x`'s address, so `x` can be modified with its - ## ``Any`` wrapper! The client needs to ensure that the wrapper - ## **does not** live longer than `x`! - ## This is provided for easier reflection capabilities of a debugger. - result.value = x.address - result.rawType = x.typ - proc toAny*[T](x: var T): Any {.inline.} = - ## constructs a ``Any`` object from `x`. This captures `x`'s address, so - ## `x` can be modified with its ``Any`` wrapper! The client needs to ensure + ## Constructs an `Any` object from `x`. This captures `x`'s address, so + ## `x` can be modified with its `Any` wrapper! The caller needs to ensure ## that the wrapper **does not** live longer than `x`! newAny(addr(x), cast[PNimType](getTypeInfo(x))) proc kind*(x: Any): AnyKind {.inline.} = - ## get the type kind + ## Gets the type kind. result = AnyKind(ord(x.rawType.kind)) proc size*(x: Any): int {.inline.} = - ## returns the size of `x`'s type. + ## Returns the size of `x`'s type. result = x.rawType.size proc baseTypeKind*(x: Any): AnyKind {.inline.} = - ## get the base type's kind; ``akNone`` is returned if `x` has no base type. + ## Gets the base type's kind. If `x` has no base type, `akNone` is returned. if x.rawType.base != nil: result = AnyKind(ord(x.rawType.base.kind)) proc baseTypeSize*(x: Any): int {.inline.} = - ## returns the size of `x`'s basetype. + ## Returns the size of `x`'s base type. If `x` has no base type, 0 is returned. if x.rawType.base != nil: result = x.rawType.base.size proc invokeNew*(x: Any) = - ## performs ``new(x)``. `x` needs to represent a ``ref``. + ## Performs `new(x)`. `x` needs to represent a `ref`. assert x.rawType.kind == tyRef - var z = newObj(x.rawType, x.rawType.base.size) - genericAssign(x.value, addr(z), x.rawType) + when defined(gcDestructors): + cast[ppointer](x.value)[] = nimNewObj(x.rawType.base.size, x.rawType.base.align) + else: + var z = newObj(x.rawType, x.rawType.base.size) + genericAssign(x.value, addr(z), x.rawType) proc invokeNewSeq*(x: Any, len: int) = - ## performs ``newSeq(x, len)``. `x` needs to represent a ``seq``. + ## Performs `newSeq(x, len)`. `x` needs to represent a `seq`. assert x.rawType.kind == tySequence - var z = newSeq(x.rawType, len) - genericShallowAssign(x.value, addr(z), x.rawType) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + s.len = len + let elem = x.rawType.base + s.p = cast[ptr NimSeqPayloadReimpl](newSeqPayload(len, elem.size, elem.align)) + else: + var z = newSeq(x.rawType, len) + genericShallowAssign(x.value, addr(z), x.rawType) proc extendSeq*(x: Any) = - ## performs ``setLen(x, x.len+1)``. `x` needs to represent a ``seq``. + ## Performs `setLen(x, x.len+1)`. `x` needs to represent a `seq`. assert x.rawType.kind == tySequence - var y = cast[ptr PGenSeq](x.value)[] - var z = incrSeq(y, x.rawType.base.size) - # 'incrSeq' already freed the memory for us and copied over the RC! - # So we simply copy the raw pointer into 'x.value': - cast[ppointer](x.value)[] = z - #genericShallowAssign(x.value, addr(z), x.rawType) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + let elem = x.rawType.base + if s.p == nil or s.p.cap < s.len+1: + s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAddUninit(s.len, s.p, 1, elem.size, elem.align)) + zeroNewElements(s.len, s.p, 1, elem.size, elem.align) + inc s.len + else: + var y = cast[ptr PGenSeq](x.value)[] + var z = incrSeq(y, x.rawType.base.size, x.rawType.base.align) + # 'incrSeq' already freed the memory for us and copied over the RC! + # So we simply copy the raw pointer into 'x.value': + cast[ppointer](x.value)[] = z + #genericShallowAssign(x.value, addr(z), x.rawType) proc setObjectRuntimeType*(x: Any) = - ## this needs to be called to set `x`'s runtime object type field. + ## This needs to be called to set `x`'s runtime object type field. assert x.rawType.kind == tyObject - objectInit(x.value, x.rawType) + when defined(gcDestructors): + cast[ppointer](x.value)[] = x.rawType.typeInfoV2 + else: + objectInit(x.value, x.rawType) proc skipRange(x: PNimType): PNimType {.inline.} = result = x if result.kind == tyRange: result = result.base +proc align(address, alignment: int): int = + result = (address + (alignment - 1)) and not (alignment - 1) + proc `[]`*(x: Any, i: int): Any = - ## accessor for an any `x` that represents an array or a sequence. + ## Accessor for an any `x` that represents an array or a sequence. case x.rawType.kind of tyArray: - var bs = x.rawType.base.size + let bs = x.rawType.base.size if i >=% x.rawType.size div bs: - raise newException(IndexError, "index out of bounds") + raise newException(IndexDefect, formatErrorIndexBound(i, x.rawType.size div bs)) return newAny(x.value +!! i*bs, x.rawType.base) of tySequence: - var s = cast[ppointer](x.value)[] - if s == nil: raise newException(ValueError, "sequence is nil") - var bs = x.rawType.base.size - if i >=% cast[PGenSeq](s).len: - raise newException(IndexError, "index out of bounds") - return newAny(s +!! (GenericSeqSize+i*bs), x.rawType.base) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + if i >=% s.len: + raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) + let bs = x.rawType.base.size + let ba = x.rawType.base.align + let headerSize = align(sizeof(int), ba) + return newAny(s.p +!! (headerSize+i*bs), x.rawType.base) + else: + var s = cast[ppointer](x.value)[] + if s == nil: raise newException(ValueError, "sequence is nil") + let bs = x.rawType.base.size + if i >=% cast[PGenSeq](s).len: + raise newException(IndexDefect, formatErrorIndexBound(i, cast[PGenSeq](s).len-1)) + return newAny(s +!! (align(GenericSeqSize, x.rawType.base.align)+i*bs), x.rawType.base) else: assert false proc `[]=`*(x: Any, i: int, y: Any) = - ## accessor for an any `x` that represents an array or a sequence. + ## Accessor for an any `x` that represents an array or a sequence. case x.rawType.kind of tyArray: var bs = x.rawType.base.size if i >=% x.rawType.size div bs: - raise newException(IndexError, "index out of bounds") + raise newException(IndexDefect, formatErrorIndexBound(i, x.rawType.size div bs)) assert y.rawType == x.rawType.base genericAssign(x.value +!! i*bs, y.value, y.rawType) of tySequence: - var s = cast[ppointer](x.value)[] - if s == nil: raise newException(ValueError, "sequence is nil") - var bs = x.rawType.base.size - if i >=% cast[PGenSeq](s).len: - raise newException(IndexError, "index out of bounds") - assert y.rawType == x.rawType.base - genericAssign(s +!! (GenericSeqSize+i*bs), y.value, y.rawType) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + if i >=% s.len: + raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) + let bs = x.rawType.base.size + let ba = x.rawType.base.align + let headerSize = align(sizeof(int), ba) + assert y.rawType == x.rawType.base + genericAssign(s.p +!! (headerSize+i*bs), y.value, y.rawType) + else: + var s = cast[ppointer](x.value)[] + if s == nil: raise newException(ValueError, "sequence is nil") + var bs = x.rawType.base.size + if i >=% cast[PGenSeq](s).len: + raise newException(IndexDefect, formatErrorIndexBound(i, cast[PGenSeq](s).len-1)) + assert y.rawType == x.rawType.base + genericAssign(s +!! (align(GenericSeqSize, x.rawType.base.align)+i*bs), y.value, y.rawType) else: assert false proc len*(x: Any): int = - ## len for an any `x` that represents an array or a sequence. + ## `len` for an any `x` that represents an array or a sequence. case x.rawType.kind - of tyArray: result = x.rawType.size div x.rawType.base.size - of tySequence: result = cast[PGenSeq](cast[ppointer](x.value)[]).len + of tyArray: + result = x.rawType.size div x.rawType.base.size + of tySequence: + when defined(gcDestructors): + result = cast[ptr NimSeqV2Reimpl](x.value).len + else: + let pgenSeq = cast[PGenSeq](cast[ppointer](x.value)[]) + if isNil(pgenSeq): + result = 0 + else: + result = pgenSeq.len else: assert false proc base*(x: Any): Any = - ## returns base Any (useful for inherited object types). + ## Returns the base type of `x` (useful for inherited object types). result.rawType = x.rawType.base result.value = x.value proc isNil*(x: Any): bool = - ## `isNil` for an any `x` that represents a sequence, string, cstring, - ## proc or some pointer type. - assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer, - tySequence, tyProc} + ## `isNil` for an `x` that represents a cstring, proc or + ## some pointer type. + assert x.rawType.kind in {tyCstring, tyRef, tyPtr, tyPointer, tyProc} result = isNil(cast[ppointer](x.value)[]) +const pointerLike = + when defined(gcDestructors): {tyCstring, tyRef, tyPtr, tyPointer, tyProc} + else: {tyString, tyCstring, tyRef, tyPtr, tyPointer, tySequence, tyProc} + proc getPointer*(x: Any): pointer = - ## retrieve the pointer value out of `x`. ``x`` needs to be of kind - ## ``akString``, ``akCString``, ``akProc``, ``akRef``, ``akPtr``, - ## ``akPointer``, ``akSequence``. - assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer, - tySequence, tyProc} + ## Retrieves the pointer value out of `x`. `x` needs to be of kind + ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`, + ## `akPointer` or `akSequence`. + assert x.rawType.kind in pointerLike result = cast[ppointer](x.value)[] proc setPointer*(x: Any, y: pointer) = - ## sets the pointer value of `x`. ``x`` needs to be of kind - ## ``akString``, ``akCString``, ``akProc``, ``akRef``, ``akPtr``, - ## ``akPointer``, ``akSequence``. - assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer, - tySequence, tyProc} - cast[ppointer](x.value)[] = y + ## Sets the pointer value of `x`. `x` needs to be of kind + ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`, + ## `akPointer` or `akSequence`. + assert x.rawType.kind in pointerLike + if y != nil and x.rawType.kind != tyPointer: + genericAssign(x.value, y, x.rawType) + else: + cast[ppointer](x.value)[] = y proc fieldsAux(p: pointer, n: ptr TNimNode, ret: var seq[tuple[name: cstring, any: Any]]) = @@ -276,15 +359,15 @@ proc fieldsAux(p: pointer, n: ptr TNimNode, if m != nil: fieldsAux(p, m, ret) iterator fields*(x: Any): tuple[name: string, any: Any] = - ## iterates over every active field of the any `x` that represents an object + ## Iterates over every active field of `x`. `x` needs to represent an object ## or a tuple. assert x.rawType.kind in {tyTuple, tyObject} - var p = x.value + let p = x.value var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] - var ret: seq[tuple[name: cstring, any: Any]] = @[] + var ret: seq[tuple[name: cstring, any: Any]] if t.kind == tyObject: while true: fieldsAux(p, t.node, ret) @@ -295,48 +378,31 @@ iterator fields*(x: Any): tuple[name: string, any: Any] = for name, any in items(ret): yield ($name, any) -proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = - proc toLower(c: char): char {.inline.} = - if c in {'A'..'Z'}: result = chr(ord(c) + (ord('a') - ord('A'))) - else: result = c - var i = 0 - var j = 0 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - -proc getFieldNode(p: pointer, n: ptr TNimNode, - name: cstring): ptr TNimNode = +proc getFieldNode(p: pointer, n: ptr TNimNode, name: cstring): ptr TNimNode = case n.kind of nkNone: assert(false) of nkSlot: - if cmpIgnoreStyle(n.name, name) == 0: + if cmpNimIdentifier(n.name, name) == 0: result = n of nkList: for i in 0..n.len-1: result = getFieldNode(p, n.sons[i], name) if result != nil: break of nkCase: - if cmpIgnoreStyle(n.name, name) == 0: + if cmpNimIdentifier(n.name, name) == 0: result = n else: - var m = selectBranch(p, n) + let m = selectBranch(p, n) if m != nil: result = getFieldNode(p, m, name) proc `[]=`*(x: Any, fieldName: string, value: Any) = - ## sets a field of `x`; `x` represents an object or a tuple. + ## Sets a field of `x`. `x` needs to represent an object or a tuple. var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] assert x.rawType.kind in {tyTuple, tyObject} - var n = getFieldNode(x.value, t.node, fieldName) + let n = getFieldNode(x.value, t.node, fieldName) if n != nil: assert n.typ == value.rawType genericAssign(x.value +!! n.offset, value.value, value.rawType) @@ -344,13 +410,13 @@ proc `[]=`*(x: Any, fieldName: string, value: Any) = raise newException(ValueError, "invalid field name: " & fieldName) proc `[]`*(x: Any, fieldName: string): Any = - ## gets a field of `x`; `x` represents an object or a tuple. + ## Gets a field of `x`. `x` needs to represent an object or a tuple. var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] assert x.rawType.kind in {tyTuple, tyObject} - var n = getFieldNode(x.value, t.node, fieldName) + let n = getFieldNode(x.value, t.node, fieldName) if n != nil: result.value = x.value +!! n.offset result.rawType = n.typ @@ -360,47 +426,47 @@ proc `[]`*(x: Any, fieldName: string): Any = raise newException(ValueError, "invalid field name: " & fieldName) proc `[]`*(x: Any): Any = - ## dereference operation for the any `x` that represents a ptr or a ref. + ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref. assert x.rawType.kind in {tyRef, tyPtr} result.value = cast[ppointer](x.value)[] result.rawType = x.rawType.base proc `[]=`*(x, y: Any) = - ## dereference operation for the any `x` that represents a ptr or a ref. + ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref. assert x.rawType.kind in {tyRef, tyPtr} assert y.rawType == x.rawType.base genericAssign(cast[ppointer](x.value)[], y.value, y.rawType) proc getInt*(x: Any): int = - ## retrieve the int value out of `x`. `x` needs to represent an int. + ## Retrieves the `int` value out of `x`. `x` needs to represent an `int`. assert skipRange(x.rawType).kind == tyInt result = cast[ptr int](x.value)[] proc getInt8*(x: Any): int8 = - ## retrieve the int8 value out of `x`. `x` needs to represent an int8. + ## Retrieves the `int8` value out of `x`. `x` needs to represent an `int8`. assert skipRange(x.rawType).kind == tyInt8 result = cast[ptr int8](x.value)[] proc getInt16*(x: Any): int16 = - ## retrieve the int16 value out of `x`. `x` needs to represent an int16. + ## Retrieves the `int16` value out of `x`. `x` needs to represent an `int16`. assert skipRange(x.rawType).kind == tyInt16 result = cast[ptr int16](x.value)[] proc getInt32*(x: Any): int32 = - ## retrieve the int32 value out of `x`. `x` needs to represent an int32. + ## Retrieves the `int32` value out of `x`. `x` needs to represent an `int32`. assert skipRange(x.rawType).kind == tyInt32 result = cast[ptr int32](x.value)[] proc getInt64*(x: Any): int64 = - ## retrieve the int64 value out of `x`. `x` needs to represent an int64. + ## Retrieves the `int64` value out of `x`. `x` needs to represent an `int64`. assert skipRange(x.rawType).kind == tyInt64 result = cast[ptr int64](x.value)[] proc getBiggestInt*(x: Any): BiggestInt = - ## retrieve the integer value out of `x`. `x` needs to represent + ## Retrieves the integer value out of `x`. `x` needs to represent ## some integer, a bool, a char, an enum or a small enough bit set. - ## The value might be sign-extended to ``BiggestInt``. - var t = skipRange(x.rawType) + ## The value might be sign-extended to `BiggestInt`. + let t = skipRange(x.rawType) case t.kind of tyInt: result = BiggestInt(cast[ptr int](x.value)[]) of tyInt8: result = BiggestInt(cast[ptr int8](x.value)[]) @@ -411,8 +477,8 @@ proc getBiggestInt*(x: Any): BiggestInt = of tyChar: result = BiggestInt(cast[ptr char](x.value)[]) of tyEnum, tySet: case t.size - of 1: result = ze64(cast[ptr int8](x.value)[]) - of 2: result = ze64(cast[ptr int16](x.value)[]) + of 1: result = int64(cast[ptr uint8](x.value)[]) + of 2: result = int64(cast[ptr uint16](x.value)[]) of 4: result = BiggestInt(cast[ptr int32](x.value)[]) of 8: result = BiggestInt(cast[ptr int64](x.value)[]) else: assert false @@ -423,9 +489,9 @@ proc getBiggestInt*(x: Any): BiggestInt = else: assert false proc setBiggestInt*(x: Any, y: BiggestInt) = - ## sets the integer value of `x`. `x` needs to represent + ## Sets the integer value of `x`. `x` needs to represent ## some integer, a bool, a char, an enum or a small enough bit set. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyInt: cast[ptr int](x.value)[] = int(y) of tyInt8: cast[ptr int8](x.value)[] = int8(y) @@ -436,8 +502,8 @@ proc setBiggestInt*(x: Any, y: BiggestInt) = of tyChar: cast[ptr char](x.value)[] = chr(y.int) of tyEnum, tySet: case t.size - of 1: cast[ptr int8](x.value)[] = toU8(y.int) - of 2: cast[ptr int16](x.value)[] = toU16(y.int) + of 1: cast[ptr uint8](x.value)[] = uint8(y.int) + of 2: cast[ptr uint16](x.value)[] = uint16(y.int) of 4: cast[ptr int32](x.value)[] = int32(y) of 8: cast[ptr int64](x.value)[] = y else: assert false @@ -448,38 +514,34 @@ proc setBiggestInt*(x: Any, y: BiggestInt) = else: assert false proc getUInt*(x: Any): uint = - ## retrieve the uint value out of `x`, `x` needs to represent an uint. + ## Retrieves the `uint` value out of `x`. `x` needs to represent a `uint`. assert skipRange(x.rawType).kind == tyUInt result = cast[ptr uint](x.value)[] proc getUInt8*(x: Any): uint8 = - ## retrieve the uint8 value out of `x`, `x` needs to represent an - ## uint8. + ## Retrieves the `uint8` value out of `x`. `x` needs to represent a `uint8`. assert skipRange(x.rawType).kind == tyUInt8 result = cast[ptr uint8](x.value)[] proc getUInt16*(x: Any): uint16 = - ## retrieve the uint16 value out of `x`, `x` needs to represent an - ## uint16. + ## Retrieves the `uint16` value out of `x`. `x` needs to represent a `uint16`. assert skipRange(x.rawType).kind == tyUInt16 result = cast[ptr uint16](x.value)[] proc getUInt32*(x: Any): uint32 = - ## retrieve the uint32 value out of `x`, `x` needs to represent an - ## uint32. + ## Retrieves the `uint32` value out of `x`. `x` needs to represent a `uint32`. assert skipRange(x.rawType).kind == tyUInt32 result = cast[ptr uint32](x.value)[] proc getUInt64*(x: Any): uint64 = - ## retrieve the uint64 value out of `x`, `x` needs to represent an - ## uint64. + ## Retrieves the `uint64` value out of `x`. `x` needs to represent a `uint64`. assert skipRange(x.rawType).kind == tyUInt64 result = cast[ptr uint64](x.value)[] proc getBiggestUint*(x: Any): uint64 = - ## retrieve the unsigned integer value out of `x`. `x` needs to + ## Retrieves the unsigned integer value out of `x`. `x` needs to ## represent an unsigned integer. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyUInt: result = uint64(cast[ptr uint](x.value)[]) of tyUInt8: result = uint64(cast[ptr uint8](x.value)[]) @@ -489,9 +551,9 @@ proc getBiggestUint*(x: Any): uint64 = else: assert false proc setBiggestUint*(x: Any; y: uint64) = - ## sets the unsigned integer value of `c`. `c` needs to represent an + ## Sets the unsigned integer value of `x`. `x` needs to represent an ## unsigned integer. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind: of tyUInt: cast[ptr uint](x.value)[] = uint(y) of tyUInt8: cast[ptr uint8](x.value)[] = uint8(y) @@ -501,33 +563,33 @@ proc setBiggestUint*(x: Any; y: uint64) = else: assert false proc getChar*(x: Any): char = - ## retrieve the char value out of `x`. `x` needs to represent a char. - var t = skipRange(x.rawType) + ## Retrieves the `char` value out of `x`. `x` needs to represent a `char`. + let t = skipRange(x.rawType) assert t.kind == tyChar result = cast[ptr char](x.value)[] proc getBool*(x: Any): bool = - ## retrieve the bool value out of `x`. `x` needs to represent a bool. - var t = skipRange(x.rawType) + ## Retrieves the `bool` value out of `x`. `x` needs to represent a `bool`. + let t = skipRange(x.rawType) assert t.kind == tyBool result = cast[ptr bool](x.value)[] proc skipRange*(x: Any): Any = - ## skips the range information of `x`. + ## Skips the range information of `x`. assert x.rawType.kind == tyRange result.rawType = x.rawType.base result.value = x.value proc getEnumOrdinal*(x: Any, name: string): int = - ## gets the enum field ordinal from `name`. `x` needs to represent an enum + ## Gets the enum field ordinal from `name`. `x` needs to represent an enum ## but is only used to access the type information. In case of an error - ## ``low(int)`` is returned. - var typ = skipRange(x.rawType) + ## `low(int)` is returned. + let typ = skipRange(x.rawType) assert typ.kind == tyEnum - var n = typ.node - var s = n.sons + let n = typ.node + let s = n.sons for i in 0 .. n.len-1: - if cmpIgnoreStyle($s[i].name, name) == 0: + if cmpNimIdentifier($s[i].name, name) == 0: if ntfEnumHole notin typ.flags: return i else: @@ -535,45 +597,45 @@ proc getEnumOrdinal*(x: Any, name: string): int = result = low(int) proc getEnumField*(x: Any, ordinalValue: int): string = - ## gets the enum field name as a string. `x` needs to represent an enum + ## Gets the enum field name as a string. `x` needs to represent an enum ## but is only used to access the type information. The field name of ## `ordinalValue` is returned. - var typ = skipRange(x.rawType) + let typ = skipRange(x.rawType) assert typ.kind == tyEnum - var e = ordinalValue + let e = ordinalValue if ntfEnumHole notin typ.flags: if e <% typ.node.len: return $typ.node.sons[e].name else: # ugh we need a slow linear search: - var n = typ.node - var s = n.sons + let n = typ.node + let s = n.sons for i in 0 .. n.len-1: if s[i].offset == e: return $s[i].name result = $e proc getEnumField*(x: Any): string = - ## gets the enum field name as a string. `x` needs to represent an enum. + ## Gets the enum field name as a string. `x` needs to represent an enum. result = getEnumField(x, getBiggestInt(x).int) proc getFloat*(x: Any): float = - ## retrieve the float value out of `x`. `x` needs to represent an float. + ## Retrieves the `float` value out of `x`. `x` needs to represent a `float`. assert skipRange(x.rawType).kind == tyFloat result = cast[ptr float](x.value)[] proc getFloat32*(x: Any): float32 = - ## retrieve the float32 value out of `x`. `x` needs to represent an float32. + ## Retrieves the `float32` value out of `x`. `x` needs to represent a `float32`. assert skipRange(x.rawType).kind == tyFloat32 result = cast[ptr float32](x.value)[] proc getFloat64*(x: Any): float64 = - ## retrieve the float64 value out of `x`. `x` needs to represent an float64. + ## Retrieves the `float64` value out of `x`. `x` needs to represent a `float64`. assert skipRange(x.rawType).kind == tyFloat64 result = cast[ptr float64](x.value)[] proc getBiggestFloat*(x: Any): BiggestFloat = - ## retrieve the float value out of `x`. `x` needs to represent - ## some float. The value is extended to ``BiggestFloat``. + ## Retrieves the float value out of `x`. `x` needs to represent + ## some float. The value is extended to `BiggestFloat`. case skipRange(x.rawType).kind of tyFloat: result = BiggestFloat(cast[ptr float](x.value)[]) of tyFloat32: result = BiggestFloat(cast[ptr float32](x.value)[]) @@ -581,7 +643,7 @@ proc getBiggestFloat*(x: Any): BiggestFloat = else: assert false proc setBiggestFloat*(x: Any, y: BiggestFloat) = - ## sets the float value of `x`. `x` needs to represent + ## Sets the float value of `x`. `x` needs to represent ## some float. case skipRange(x.rawType).kind of tyFloat: cast[ptr float](x.value)[] = y @@ -590,56 +652,59 @@ proc setBiggestFloat*(x: Any, y: BiggestFloat) = else: assert false proc getString*(x: Any): string = - ## retrieve the string value out of `x`. `x` needs to represent a string. + ## Retrieves the `string` value out of `x`. `x` needs to represent a `string`. assert x.rawType.kind == tyString - if not isNil(cast[ptr pointer](x.value)[]): + when defined(gcDestructors): result = cast[ptr string](x.value)[] + else: + if not isNil(cast[ptr pointer](x.value)[]): + result = cast[ptr string](x.value)[] proc setString*(x: Any, y: string) = - ## sets the string value of `x`. `x` needs to represent a string. + ## Sets the `string` value of `x`. `x` needs to represent a `string`. assert x.rawType.kind == tyString - cast[ptr string](x.value)[] = y + cast[ptr string](x.value)[] = y # also correct for gcDestructors proc getCString*(x: Any): cstring = - ## retrieve the cstring value out of `x`. `x` needs to represent a cstring. - assert x.rawType.kind == tyCString + ## Retrieves the `cstring` value out of `x`. `x` needs to represent a `cstring`. + assert x.rawType.kind == tyCstring result = cast[ptr cstring](x.value)[] proc assign*(x, y: Any) = - ## copies the value of `y` to `x`. The assignment operator for ``Any`` + ## Copies the value of `y` to `x`. The assignment operator for `Any` ## does NOT do this; it performs a shallow copy instead! assert y.rawType == x.rawType genericAssign(x.value, y.value, y.rawType) iterator elements*(x: Any): int = - ## iterates over every element of `x` that represents a Nim bitset. + ## Iterates over every element of `x`. `x` needs to represent a `set`. assert x.rawType.kind == tySet - var typ = x.rawType - var p = x.value + let typ = x.rawType + let p = x.value # "typ.slots.len" field is for sets the "first" field var u: int64 case typ.size - of 1: u = ze64(cast[ptr int8](p)[]) - of 2: u = ze64(cast[ptr int16](p)[]) - of 4: u = ze64(cast[ptr int32](p)[]) + of 1: u = int64(cast[ptr uint8](p)[]) + of 2: u = int64(cast[ptr uint16](p)[]) + of 4: u = int64(cast[ptr uint32](p)[]) of 8: u = cast[ptr int64](p)[] else: - var a = cast[pbyteArray](p) + let a = cast[pbyteArray](p) for i in 0 .. typ.size*8-1: - if (ze(a[i div 8]) and (1 shl (i mod 8))) != 0: - yield i+typ.node.len + if (int(a[i div 8]) and (1 shl (i mod 8))) != 0: + yield i + typ.node.len if typ.size <= 8: for i in 0..sizeof(int64)*8-1: if (u and (1'i64 shl int64(i))) != 0'i64: - yield i+typ.node.len + yield i + typ.node.len proc inclSetElement*(x: Any, elem: int) = - ## includes an element `elem` in `x`. `x` needs to represent a Nim bitset. + ## Includes an element `elem` in `x`. `x` needs to represent a Nim bitset. assert x.rawType.kind == tySet - var typ = x.rawType - var p = x.value + let typ = x.rawType + let p = x.value # "typ.slots.len" field is for sets the "first" field - var e = elem - typ.node.len + let e = elem - typ.node.len case typ.size of 1: var a = cast[ptr int8](p) @@ -655,65 +720,4 @@ proc inclSetElement*(x: Any, elem: int) = a[] = a[] or (1'i64 shl e) else: var a = cast[pbyteArray](p) - a[e shr 3] = toU8(a[e shr 3] or (1 shl (e and 7))) - -when isMainModule: - type - TE = enum - blah, blah2 - - TestObj = object - test, asd: int - case test2: TE - of blah: - help: string - else: - nil - - var test = @[0,1,2,3,4] - var x = toAny(test) - var y = 78 - x[4] = toAny(y) - assert cast[ptr int](x[2].value)[] == 2 - - var test2: tuple[name: string, s: int] = ("test", 56) - var x2 = toAny(test2) - var i = 0 - for n, a in fields(x2): - case i - of 0: assert n == "name" and $a.kind == "akString" - of 1: assert n == "s" and $a.kind == "akInt" - else: assert false - inc i - - var test3: TestObj - test3.test = 42 - test3.test2 = blah2 - var x3 = toAny(test3) - i = 0 - for n, a in fields(x3): - case i - of 0: assert n == "test" and $a.kind == "akInt" - of 1: assert n == "asd" and $a.kind == "akInt" - of 2: assert n == "test2" and $a.kind == "akEnum" - else: assert false - inc i - - var test4: ref string - new(test4) - test4[] = "test" - var x4 = toAny(test4) - assert($x4[].kind() == "akString") - - block: - # gimme a new scope dammit - var myarr: array[0..4, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"]] - var m = toAny(myArr) - for i in 0 .. m.len-1: - for j in 0 .. m[i].len-1: - echo getString(m[i][j]) - - + a[e shr 3] = a[e shr 3] or uint8(1 shl (e and 7)) diff --git a/lib/deprecated/core/unsigned.nim b/lib/deprecated/core/unsigned.nim deleted file mode 100644 index 93a29e1c9..000000000 --- a/lib/deprecated/core/unsigned.nim +++ /dev/null @@ -1,18 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warning:** Since version 0.11.4 this module is deprecated. -## -## This module implemented basic arithmetic operators for unsigned integers. -## These operators are now available in the ``system`` module directly. - -{.deprecated.} - -export `shr`, `shl`, `and`, `or`, `xor`, `==`, `+`, `-`, `*`, `div`, `mod`, - `<=`, `<` diff --git a/lib/deprecated/pure/actors.nim b/lib/deprecated/pure/actors.nim deleted file mode 100644 index f0791f954..000000000 --- a/lib/deprecated/pure/actors.nim +++ /dev/null @@ -1,241 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## `Actor`:idx: support for Nim. An actor is implemented as a thread with -## a channel as its inbox. This module requires the ``--threads:on`` -## command line switch. -## -## Example: -## -## .. code-block:: nim -## -## var -## a: ActorPool[int, void] -## createActorPool(a) -## for i in 0 .. < 300: -## a.spawn(i, proc (x: int) {.thread.} = echo x) -## a.join() -## -## **Note**: This whole module is deprecated. Use `threadpool` and ``spawn`` -## instead. - -{.deprecated.} - -from os import sleep - -type - Task*[In, Out] = object{.pure, final.} ## a task - when Out isnot void: - receiver*: ptr Channel[Out] ## the receiver channel of the response - action*: proc (x: In): Out {.thread.} ## action to execute; - ## sometimes useful - shutDown*: bool ## set to tell an actor to shut-down - data*: In ## the data to process - - Actor[In, Out] = object{.pure, final.} - i: Channel[Task[In, Out]] - t: TThread[ptr Actor[In, Out]] - - PActor*[In, Out] = ptr Actor[In, Out] ## an actor -{.deprecated: [TTask: Task, TActor: Actor].} - -proc spawn*[In, Out](action: proc( - self: PActor[In, Out]){.thread.}): PActor[In, Out] = - ## creates an actor; that is a thread with an inbox. The caller MUST call - ## ``join`` because that also frees the actor's associated resources. - result = cast[PActor[In, Out]](allocShared0(sizeof(result[]))) - open(result.i) - createThread(result.t, action, result) - -proc inbox*[In, Out](self: PActor[In, Out]): ptr Channel[In] = - ## gets a pointer to the associated inbox of the actor `self`. - result = addr(self.i) - -proc running*[In, Out](a: PActor[In, Out]): bool = - ## returns true if the actor `a` is running. - result = running(a.t) - -proc ready*[In, Out](a: PActor[In, Out]): bool = - ## returns true if the actor `a` is ready to process new messages. - result = ready(a.i) - -proc join*[In, Out](a: PActor[In, Out]) = - ## joins an actor. - joinThread(a.t) - close(a.i) - deallocShared(a) - -proc recv*[In, Out](a: PActor[In, Out]): Task[In, Out] = - ## receives a task from `a`'s inbox. - result = recv(a.i) - -proc send*[In, Out, X, Y](receiver: PActor[In, Out], msg: In, - sender: PActor[X, Y]) = - ## sends a message to `a`'s inbox. - var t: Task[In, Out] - t.receiver = addr(sender.i) - shallowCopy(t.data, msg) - send(receiver.i, t) - -proc send*[In, Out](receiver: PActor[In, Out], msg: In, - sender: ptr Channel[Out] = nil) = - ## sends a message to `receiver`'s inbox. - var t: Task[In, Out] - t.receiver = sender - shallowCopy(t.data, msg) - send(receiver.i, t) - -proc sendShutdown*[In, Out](receiver: PActor[In, Out]) = - ## send a shutdown message to `receiver`. - var t: Task[In, Out] - t.shutdown = true - send(receiver.i, t) - -proc reply*[In, Out](t: Task[In, Out], m: Out) = - ## sends a message to io's output message box. - when Out is void: - {.error: "you cannot reply to a void outbox".} - assert t.receiver != nil - send(t.receiver[], m) - - -# ----------------- actor pools ---------------------------------------------- - -type - ActorPool*[In, Out] = object{.pure, final.} ## an actor pool - actors: seq[PActor[In, Out]] - when Out isnot void: - outputs: Channel[Out] -{.deprecated: [TActorPool: ActorPool].} - -proc `^`*[T](f: ptr Channel[T]): T = - ## alias for 'recv'. - result = recv(f[]) - -proc poolWorker[In, Out](self: PActor[In, Out]) {.thread.} = - while true: - var m = self.recv - if m.shutDown: break - when Out is void: - m.action(m.data) - else: - send(m.receiver[], m.action(m.data)) - #self.reply() - -proc createActorPool*[In, Out](a: var ActorPool[In, Out], poolSize = 4) = - ## creates an actor pool. - newSeq(a.actors, poolSize) - when Out isnot void: - open(a.outputs) - for i in 0 .. < a.actors.len: - a.actors[i] = spawn(poolWorker[In, Out]) - -proc sync*[In, Out](a: var ActorPool[In, Out], polling=50) = - ## waits for every actor of `a` to finish with its work. Currently this is - ## implemented as polling every `polling` ms and has a slight chance - ## of failing since we check for every actor to be in `ready` state and not - ## for messages still in ether. This will change in a later - ## version, however. - var allReadyCount = 0 - while true: - var wait = false - for i in 0..high(a.actors): - if not a.actors[i].i.ready: - wait = true - allReadyCount = 0 - break - if not wait: - # it's possible that some actor sent a message to some other actor but - # both appeared to be non-working as the message takes some time to - # arrive. We assume that this won't take longer than `polling` and - # simply attempt a second time and declare victory then. ;-) - inc allReadyCount - if allReadyCount > 1: break - sleep(polling) - -proc terminate*[In, Out](a: var ActorPool[In, Out]) = - ## terminates each actor in the actor pool `a` and frees the - ## resources attached to `a`. - var t: Task[In, Out] - t.shutdown = true - for i in 0.. <a.actors.len: send(a.actors[i].i, t) - for i in 0.. <a.actors.len: join(a.actors[i]) - when Out isnot void: - close(a.outputs) - a.actors = nil - -proc join*[In, Out](a: var ActorPool[In, Out]) = - ## short-cut for `sync` and then `terminate`. - sync(a) - terminate(a) - -template setupTask = - t.action = action - shallowCopy(t.data, input) - -template schedule = - # extremely simple scheduler: We always try the first thread first, so that - # it remains 'hot' ;-). Round-robin hurts for keeping threads hot. - for i in 0..high(p.actors): - if p.actors[i].i.ready: - p.actors[i].i.send(t) - return - # no thread ready :-( --> send message to the thread which has the least - # messages pending: - var minIdx = -1 - var minVal = high(int) - for i in 0..high(p.actors): - var curr = p.actors[i].i.peek - if curr == 0: - # ok, is ready now: - p.actors[i].i.send(t) - return - if curr < minVal and curr >= 0: - minVal = curr - minIdx = i - if minIdx >= 0: - p.actors[minIdx].i.send(t) - else: - raise newException(DeadThreadError, "cannot send message; thread died") - -proc spawn*[In, Out](p: var ActorPool[In, Out], input: In, - action: proc (input: In): Out {.thread.} - ): ptr Channel[Out] = - ## uses the actor pool to run ``action(input)`` concurrently. - ## `spawn` is guaranteed to not block. - var t: Task[In, Out] - setupTask() - result = addr(p.outputs) - t.receiver = result - schedule() - -proc spawn*[In](p: var ActorPool[In, void], input: In, - action: proc (input: In) {.thread.}) = - ## uses the actor pool to run ``action(input)`` concurrently. - ## `spawn` is guaranteed to not block. - var t: Task[In, void] - setupTask() - schedule() - -when not defined(testing) and isMainModule: - var - a: ActorPool[int, void] - createActorPool(a) - for i in 0 .. < 300: - a.spawn(i, proc (x: int) {.thread.} = echo x) - - when false: - proc treeDepth(n: PNode): int {.thread.} = - var x = a.spawn(treeDepth, n.le) - var y = a.spawn(treeDepth, n.ri) - result = max(^x, ^y) + 1 - - a.join() - - diff --git a/lib/deprecated/pure/actors.nim.cfg b/lib/deprecated/pure/actors.nim.cfg deleted file mode 100644 index c6bb9c545..000000000 --- a/lib/deprecated/pure/actors.nim.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# to shut up the tester: ---threads:on - diff --git a/lib/deprecated/pure/asyncio.nim b/lib/deprecated/pure/asyncio.nim deleted file mode 100644 index 5fd45b215..000000000 --- a/lib/deprecated/pure/asyncio.nim +++ /dev/null @@ -1,712 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf, Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import sockets, os - -## -## **Warning:** This module is deprecated since version 0.10.2. -## Use the brand new `asyncdispatch <asyncdispatch.html>`_ module together -## with the `asyncnet <asyncnet.html>`_ module. - -## This module implements an asynchronous event loop together with asynchronous -## sockets which use this event loop. -## It is akin to Python's asyncore module. Many modules that use sockets -## have an implementation for this module, those modules should all have a -## ``register`` function which you should use to add the desired objects to a -## dispatcher which you created so -## that you can receive the events associated with that module's object. -## -## Once everything is registered in a dispatcher, you need to call the ``poll`` -## function in a while loop. -## -## **Note:** Most modules have tasks which need to be ran regularly, this is -## why you should not call ``poll`` with a infinite timeout, or even a -## very long one. In most cases the default timeout is fine. -## -## **Note:** This module currently only supports select(), this is limited by -## FD_SETSIZE, which is usually 1024. So you may only be able to use 1024 -## sockets at a time. -## -## Most (if not all) modules that use asyncio provide a userArg which is passed -## on with the events. The type that you set userArg to must be inheriting from -## ``RootObj``! -## -## **Note:** If you want to provide async ability to your module please do not -## use the ``Delegate`` object, instead use ``AsyncSocket``. It is possible -## that in the future this type's fields will not be exported therefore breaking -## your code. -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. -## -## Asynchronous sockets -## ==================== -## -## For most purposes you do not need to worry about the ``Delegate`` type. The -## ``AsyncSocket`` is what you are after. It's a reference to -## the ``AsyncSocketObj`` object. This object defines events which you should -## overwrite by your own procedures. -## -## For server sockets the only event you need to worry about is the ``handleAccept`` -## event, in your handleAccept proc you should call ``accept`` on the server -## socket which will give you the client which is connecting. You should then -## set any events that you want to use on that client and add it to your dispatcher -## using the ``register`` procedure. -## -## An example ``handleAccept`` follows: -## -## .. code-block:: nim -## -## var disp = newDispatcher() -## ... -## proc handleAccept(s: AsyncSocket) = -## echo("Accepted client.") -## var client: AsyncSocket -## new(client) -## s.accept(client) -## client.handleRead = ... -## disp.register(client) -## ... -## -## For client sockets you should only be interested in the ``handleRead`` and -## ``handleConnect`` events. The former gets called whenever the socket has -## received messages and can be read from and the latter gets called whenever -## the socket has established a connection to a server socket; from that point -## it can be safely written to. -## -## Getting a blocking client from an AsyncSocket -## ============================================= -## -## If you need a asynchronous server socket but you wish to process the clients -## synchronously then you can use the ``getSocket`` converter to get -## a ``Socket`` from the ``AsyncSocket`` object, this can then be combined -## with ``accept`` like so: -## -## .. code-block:: nim -## -## proc handleAccept(s: AsyncSocket) = -## var client: Socket -## getSocket(s).accept(client) - -{.deprecated.} - -when defined(windows): - from winlean import TimeVal, SocketHandle, FD_SET, FD_ZERO, TFdSet, - FD_ISSET, select -else: - from posix import TimeVal, SocketHandle, FD_SET, FD_ZERO, TFdSet, - FD_ISSET, select - -type - DelegateObj* = object - fd*: SocketHandle - deleVal*: RootRef - - handleRead*: proc (h: RootRef) {.nimcall, gcsafe.} - handleWrite*: proc (h: RootRef) {.nimcall, gcsafe.} - handleError*: proc (h: RootRef) {.nimcall, gcsafe.} - hasDataBuffered*: proc (h: RootRef): bool {.nimcall, gcsafe.} - - open*: bool - task*: proc (h: RootRef) {.nimcall, gcsafe.} - mode*: FileMode - - Delegate* = ref DelegateObj - - Dispatcher* = ref DispatcherObj - DispatcherObj = object - delegates: seq[Delegate] - - AsyncSocket* = ref AsyncSocketObj - AsyncSocketObj* = object of RootObj - socket: Socket - info: SocketStatus - - handleRead*: proc (s: AsyncSocket) {.closure, gcsafe.} - handleWrite: proc (s: AsyncSocket) {.closure, gcsafe.} - handleConnect*: proc (s: AsyncSocket) {.closure, gcsafe.} - - handleAccept*: proc (s: AsyncSocket) {.closure, gcsafe.} - - handleTask*: proc (s: AsyncSocket) {.closure, gcsafe.} - - lineBuffer: TaintedString ## Temporary storage for ``readLine`` - sendBuffer: string ## Temporary storage for ``send`` - sslNeedAccept: bool - proto: Protocol - deleg: Delegate - - SocketStatus* = enum - SockIdle, SockConnecting, SockConnected, SockListening, SockClosed, - SockUDPBound - -{.deprecated: [TDelegate: DelegateObj, PDelegate: Delegate, - TInfo: SocketStatus, PAsyncSocket: AsyncSocket, TAsyncSocket: AsyncSocketObj, - TDispatcher: DispatcherObj, PDispatcher: Dispatcher, - ].} - - -proc newDelegate*(): Delegate = - ## Creates a new delegate. - new(result) - result.handleRead = (proc (h: RootRef) = discard) - result.handleWrite = (proc (h: RootRef) = discard) - result.handleError = (proc (h: RootRef) = discard) - result.hasDataBuffered = (proc (h: RootRef): bool = return false) - result.task = (proc (h: RootRef) = discard) - result.mode = fmRead - -proc newAsyncSocket(): AsyncSocket = - new(result) - result.info = SockIdle - - result.handleRead = (proc (s: AsyncSocket) = discard) - result.handleWrite = nil - result.handleConnect = (proc (s: AsyncSocket) = discard) - result.handleAccept = (proc (s: AsyncSocket) = discard) - result.handleTask = (proc (s: AsyncSocket) = discard) - - result.lineBuffer = "".TaintedString - result.sendBuffer = "" - -proc asyncSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, - buffered = true): AsyncSocket = - ## Initialises an AsyncSocket object. If a socket cannot be initialised - ## EOS is raised. - result = newAsyncSocket() - result.socket = socket(domain, typ, protocol, buffered) - result.proto = protocol - if result.socket == invalidSocket: raiseOSError(osLastError()) - result.socket.setBlocking(false) - -proc toAsyncSocket*(sock: Socket, state: SocketStatus = SockConnected): AsyncSocket = - ## Wraps an already initialized ``Socket`` into a AsyncSocket. - ## This is useful if you want to use an already connected Socket as an - ## asynchronous AsyncSocket in asyncio's event loop. - ## - ## ``state`` may be overriden, i.e. if ``sock`` is not connected it should be - ## adjusted properly. By default it will be assumed that the socket is - ## connected. Please note this is only applicable to TCP client sockets, if - ## ``sock`` is a different type of socket ``state`` needs to be adjusted!!! - ## - ## ================ ================================================================ - ## Value Meaning - ## ================ ================================================================ - ## SockIdle Socket has only just been initialised, not connected or closed. - ## SockConnected Socket is connected to a server. - ## SockConnecting Socket is in the process of connecting to a server. - ## SockListening Socket is a server socket and is listening for connections. - ## SockClosed Socket has been closed. - ## SockUDPBound Socket is a UDP socket which is listening for data. - ## ================ ================================================================ - ## - ## **Warning**: If ``state`` is set incorrectly the resulting ``AsyncSocket`` - ## object may not work properly. - ## - ## **Note**: This will set ``sock`` to be non-blocking. - result = newAsyncSocket() - result.socket = sock - result.proto = if state == SockUDPBound: IPPROTO_UDP else: IPPROTO_TCP - result.socket.setBlocking(false) - result.info = state - -proc asyncSockHandleRead(h: RootRef) = - when defined(ssl): - if AsyncSocket(h).socket.isSSL and not - AsyncSocket(h).socket.gotHandshake: - return - - if AsyncSocket(h).info != SockListening: - if AsyncSocket(h).info != SockConnecting: - AsyncSocket(h).handleRead(AsyncSocket(h)) - else: - AsyncSocket(h).handleAccept(AsyncSocket(h)) - -proc close*(sock: AsyncSocket) {.gcsafe.} -proc asyncSockHandleWrite(h: RootRef) = - when defined(ssl): - if AsyncSocket(h).socket.isSSL and not - AsyncSocket(h).socket.gotHandshake: - return - - if AsyncSocket(h).info == SockConnecting: - AsyncSocket(h).handleConnect(AsyncSocket(h)) - AsyncSocket(h).info = SockConnected - # Stop receiving write events if there is no handleWrite event. - if AsyncSocket(h).handleWrite == nil: - AsyncSocket(h).deleg.mode = fmRead - else: - AsyncSocket(h).deleg.mode = fmReadWrite - else: - if AsyncSocket(h).sendBuffer != "": - let sock = AsyncSocket(h) - try: - let bytesSent = sock.socket.sendAsync(sock.sendBuffer) - if bytesSent == 0: - # Apparently the socket cannot be written to. Even though select - # just told us that it can be... This used to be an assert. Just - # do nothing instead. - discard - elif bytesSent != sock.sendBuffer.len: - sock.sendBuffer = sock.sendBuffer[bytesSent .. ^1] - elif bytesSent == sock.sendBuffer.len: - sock.sendBuffer = "" - - if AsyncSocket(h).handleWrite != nil: - AsyncSocket(h).handleWrite(AsyncSocket(h)) - except OSError: - # Most likely the socket closed before the full buffer could be sent to it. - sock.close() # TODO: Provide a handleError for users? - else: - if AsyncSocket(h).handleWrite != nil: - AsyncSocket(h).handleWrite(AsyncSocket(h)) - else: - AsyncSocket(h).deleg.mode = fmRead - -when defined(ssl): - proc asyncSockDoHandshake(h: PObject) {.gcsafe.} = - if AsyncSocket(h).socket.isSSL and not - AsyncSocket(h).socket.gotHandshake: - if AsyncSocket(h).sslNeedAccept: - var d = "" - let ret = AsyncSocket(h).socket.acceptAddrSSL(AsyncSocket(h).socket, d) - assert ret != AcceptNoClient - if ret == AcceptSuccess: - AsyncSocket(h).info = SockConnected - else: - # handshake will set socket's ``sslNoHandshake`` field. - discard AsyncSocket(h).socket.handshake() - - -proc asyncSockTask(h: RootRef) = - when defined(ssl): - h.asyncSockDoHandshake() - - AsyncSocket(h).handleTask(AsyncSocket(h)) - -proc toDelegate(sock: AsyncSocket): Delegate = - result = newDelegate() - result.deleVal = sock - result.fd = getFD(sock.socket) - # We need this to get write events, just to know when the socket connects. - result.mode = fmReadWrite - result.handleRead = asyncSockHandleRead - result.handleWrite = asyncSockHandleWrite - result.task = asyncSockTask - # TODO: Errors? - #result.handleError = (proc (h: PObject) = assert(false)) - - result.hasDataBuffered = - proc (h: RootRef): bool {.nimcall.} = - return AsyncSocket(h).socket.hasDataBuffered() - - sock.deleg = result - if sock.info notin {SockIdle, SockClosed}: - sock.deleg.open = true - else: - sock.deleg.open = false - -proc connect*(sock: AsyncSocket, name: string, port = Port(0), - af: Domain = AF_INET) = - ## Begins connecting ``sock`` to ``name``:``port``. - sock.socket.connectAsync(name, port, af) - sock.info = SockConnecting - if sock.deleg != nil: - sock.deleg.open = true - -proc close*(sock: AsyncSocket) = - ## Closes ``sock``. Terminates any current connections. - sock.socket.close() - sock.info = SockClosed - if sock.deleg != nil: - sock.deleg.open = false - -proc bindAddr*(sock: AsyncSocket, port = Port(0), address = "") = - ## Equivalent to ``sockets.bindAddr``. - sock.socket.bindAddr(port, address) - if sock.proto == IPPROTO_UDP: - sock.info = SockUDPBound - if sock.deleg != nil: - sock.deleg.open = true - -proc listen*(sock: AsyncSocket) = - ## Equivalent to ``sockets.listen``. - sock.socket.listen() - sock.info = SockListening - if sock.deleg != nil: - sock.deleg.open = true - -proc acceptAddr*(server: AsyncSocket, client: var AsyncSocket, - address: var string) = - ## Equivalent to ``sockets.acceptAddr``. This procedure should be called in - ## a ``handleAccept`` event handler **only** once. - ## - ## **Note**: ``client`` needs to be initialised. - assert(client != nil) - client = newAsyncSocket() - var c: Socket - new(c) - when defined(ssl): - if server.socket.isSSL: - var ret = server.socket.acceptAddrSSL(c, address) - # The following shouldn't happen because when this function is called - # it is guaranteed that there is a client waiting. - # (This should be called in handleAccept) - assert(ret != AcceptNoClient) - if ret == AcceptNoHandshake: - client.sslNeedAccept = true - else: - client.sslNeedAccept = false - client.info = SockConnected - else: - server.socket.acceptAddr(c, address) - client.sslNeedAccept = false - client.info = SockConnected - else: - server.socket.acceptAddr(c, address) - client.sslNeedAccept = false - client.info = SockConnected - - if c == invalidSocket: raiseSocketError(server.socket) - c.setBlocking(false) # TODO: Needs to be tested. - - # deleg.open is set in ``toDelegate``. - - client.socket = c - client.lineBuffer = "".TaintedString - client.sendBuffer = "" - client.info = SockConnected - -proc accept*(server: AsyncSocket, client: var AsyncSocket) = - ## Equivalent to ``sockets.accept``. - var dummyAddr = "" - server.acceptAddr(client, dummyAddr) - -proc acceptAddr*(server: AsyncSocket): tuple[sock: AsyncSocket, - address: string] {.deprecated.} = - ## Equivalent to ``sockets.acceptAddr``. - ## - ## **Deprecated since version 0.9.0:** Please use the function above. - var client = newAsyncSocket() - var address: string = "" - acceptAddr(server, client, address) - return (client, address) - -proc accept*(server: AsyncSocket): AsyncSocket {.deprecated.} = - ## Equivalent to ``sockets.accept``. - ## - ## **Deprecated since version 0.9.0:** Please use the function above. - new(result) - var address = "" - server.acceptAddr(result, address) - -proc newDispatcher*(): Dispatcher = - new(result) - result.delegates = @[] - -proc register*(d: Dispatcher, deleg: Delegate) = - ## Registers delegate ``deleg`` with dispatcher ``d``. - d.delegates.add(deleg) - -proc register*(d: Dispatcher, sock: AsyncSocket): Delegate {.discardable.} = - ## Registers async socket ``sock`` with dispatcher ``d``. - result = sock.toDelegate() - d.register(result) - -proc unregister*(d: Dispatcher, deleg: Delegate) = - ## Unregisters deleg ``deleg`` from dispatcher ``d``. - for i in 0..len(d.delegates)-1: - if d.delegates[i] == deleg: - d.delegates.del(i) - return - raise newException(IndexError, "Could not find delegate.") - -proc isWriteable*(s: AsyncSocket): bool = - ## Determines whether socket ``s`` is ready to be written to. - var writeSock = @[s.socket] - return selectWrite(writeSock, 1) != 0 and s.socket notin writeSock - -converter getSocket*(s: AsyncSocket): Socket = - return s.socket - -proc isConnected*(s: AsyncSocket): bool = - ## Determines whether ``s`` is connected. - return s.info == SockConnected -proc isListening*(s: AsyncSocket): bool = - ## Determines whether ``s`` is listening for incoming connections. - return s.info == SockListening -proc isConnecting*(s: AsyncSocket): bool = - ## Determines whether ``s`` is connecting. - return s.info == SockConnecting -proc isClosed*(s: AsyncSocket): bool = - ## Determines whether ``s`` has been closed. - return s.info == SockClosed -proc isSendDataBuffered*(s: AsyncSocket): bool = - ## Determines whether ``s`` has data waiting to be sent, i.e. whether this - ## socket's sendBuffer contains data. - return s.sendBuffer.len != 0 - -proc setHandleWrite*(s: AsyncSocket, - handleWrite: proc (s: AsyncSocket) {.closure, gcsafe.}) = - ## Setter for the ``handleWrite`` event. - ## - ## To remove this event you should use the ``delHandleWrite`` function. - ## It is advised to use that function instead of just setting the event to - ## ``proc (s: AsyncSocket) = nil`` as that would mean that that function - ## would be called constantly. - s.deleg.mode = fmReadWrite - s.handleWrite = handleWrite - -proc delHandleWrite*(s: AsyncSocket) = - ## Removes the ``handleWrite`` event handler on ``s``. - s.handleWrite = nil - -{.push warning[deprecated]: off.} -proc recvLine*(s: AsyncSocket, line: var TaintedString): bool {.deprecated.} = - ## Behaves similar to ``sockets.recvLine``, however it handles non-blocking - ## sockets properly. This function guarantees that ``line`` is a full line, - ## if this function can only retrieve some data; it will save this data and - ## add it to the result when a full line is retrieved. - ## - ## Unlike ``sockets.recvLine`` this function will raise an EOS or ESSL - ## exception if an error occurs. - ## - ## **Deprecated since version 0.9.2**: This function has been deprecated in - ## favour of readLine. - setLen(line.string, 0) - var dataReceived = "".TaintedString - var ret = s.socket.recvLineAsync(dataReceived) - case ret - of RecvFullLine: - if s.lineBuffer.len > 0: - string(line).add(s.lineBuffer.string) - setLen(s.lineBuffer.string, 0) - string(line).add(dataReceived.string) - if string(line) == "": - line = "\c\L".TaintedString - result = true - of RecvPartialLine: - string(s.lineBuffer).add(dataReceived.string) - result = false - of RecvDisconnected: - result = true - of RecvFail: - s.raiseSocketError(async = true) - result = false -{.pop.} - -proc readLine*(s: AsyncSocket, line: var TaintedString): bool = - ## Behaves similar to ``sockets.readLine``, however it handles non-blocking - ## sockets properly. This function guarantees that ``line`` is a full line, - ## if this function can only retrieve some data; it will save this data and - ## add it to the result when a full line is retrieved, when this happens - ## False will be returned. True will only be returned if a full line has been - ## retrieved or the socket has been disconnected in which case ``line`` will - ## be set to "". - ## - ## This function will raise an EOS exception when a socket error occurs. - setLen(line.string, 0) - var dataReceived = "".TaintedString - var ret = s.socket.readLineAsync(dataReceived) - case ret - of ReadFullLine: - if s.lineBuffer.len > 0: - string(line).add(s.lineBuffer.string) - setLen(s.lineBuffer.string, 0) - string(line).add(dataReceived.string) - if string(line) == "": - line = "\c\L".TaintedString - result = true - of ReadPartialLine: - string(s.lineBuffer).add(dataReceived.string) - result = false - of ReadNone: - result = false - of ReadDisconnected: - result = true - -proc send*(sock: AsyncSocket, data: string) = - ## Sends ``data`` to socket ``sock``. This is basically a nicer implementation - ## of ``sockets.sendAsync``. - ## - ## If ``data`` cannot be sent immediately it will be buffered and sent - ## when ``sock`` becomes writeable (during the ``handleWrite`` event). - ## It's possible that only a part of ``data`` will be sent immediately, while - ## the rest of it will be buffered and sent later. - if sock.sendBuffer.len != 0: - sock.sendBuffer.add(data) - return - let bytesSent = sock.socket.sendAsync(data) - assert bytesSent >= 0 - if bytesSent == 0: - sock.sendBuffer.add(data) - sock.deleg.mode = fmReadWrite - elif bytesSent != data.len: - sock.sendBuffer.add(data[bytesSent .. ^1]) - sock.deleg.mode = fmReadWrite - -proc timeValFromMilliseconds(timeout = 500): Timeval = - if timeout != -1: - var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - -proc createFdSet(fd: var TFdSet, s: seq[Delegate], m: var int) = - FD_ZERO(fd) - for i in items(s): - m = max(m, int(i.fd)) - FD_SET(i.fd, fd) - -proc pruneSocketSet(s: var seq[Delegate], fd: var TFdSet) = - var i = 0 - var L = s.len - while i < L: - if FD_ISSET(s[i].fd, fd) != 0'i32: - s[i] = s[L-1] - dec(L) - else: - inc(i) - setLen(s, L) - -proc select(readfds, writefds, exceptfds: var seq[Delegate], - timeout = 500): int = - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd, wr, ex: TFdSet - var m = 0 - createFdSet(rd, readfds, m) - createFdSet(wr, writefds, m) - createFdSet(ex, exceptfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), nil)) - - pruneSocketSet(readfds, (rd)) - pruneSocketSet(writefds, (wr)) - pruneSocketSet(exceptfds, (ex)) - -proc poll*(d: Dispatcher, timeout: int = 500): bool = - ## This function checks for events on all the delegates in the `PDispatcher`. - ## It then proceeds to call the correct event handler. - ## - ## This function returns ``True`` if there are file descriptors that are still - ## open, otherwise ``False``. File descriptors that have been - ## closed are immediately removed from the dispatcher automatically. - ## - ## **Note:** Each delegate has a task associated with it. This gets called - ## after each select() call, if you set timeout to ``-1`` the tasks will - ## only be executed after one or more file descriptors becomes readable or - ## writeable. - result = true - var readDg, writeDg, errorDg: seq[Delegate] = @[] - var len = d.delegates.len - var dc = 0 - - while dc < len: - let deleg = d.delegates[dc] - if (deleg.mode != fmWrite or deleg.mode != fmAppend) and deleg.open: - readDg.add(deleg) - if (deleg.mode != fmRead) and deleg.open: - writeDg.add(deleg) - if deleg.open: - errorDg.add(deleg) - inc dc - else: - # File/socket has been closed. Remove it from dispatcher. - d.delegates[dc] = d.delegates[len-1] - dec len - - d.delegates.setLen(len) - - var hasDataBufferedCount = 0 - for d in d.delegates: - if d.hasDataBuffered(d.deleVal): - hasDataBufferedCount.inc() - d.handleRead(d.deleVal) - if hasDataBufferedCount > 0: return true - - if readDg.len() == 0 and writeDg.len() == 0: - ## TODO: Perhaps this shouldn't return if errorDg has something? - return false - - if select(readDg, writeDg, errorDg, timeout) != 0: - for i in 0..len(d.delegates)-1: - if i > len(d.delegates)-1: break # One delegate might've been removed. - let deleg = d.delegates[i] - if not deleg.open: continue # This delegate might've been closed. - if (deleg.mode != fmWrite or deleg.mode != fmAppend) and - deleg notin readDg: - deleg.handleRead(deleg.deleVal) - if (deleg.mode != fmRead) and deleg notin writeDg: - deleg.handleWrite(deleg.deleVal) - if deleg notin errorDg: - deleg.handleError(deleg.deleVal) - - # Execute tasks - for i in items(d.delegates): - i.task(i.deleVal) - -proc len*(disp: Dispatcher): int = - ## Retrieves the amount of delegates in ``disp``. - return disp.delegates.len - -when not defined(testing) and isMainModule: - - proc testConnect(s: AsyncSocket, no: int) = - echo("Connected! " & $no) - - proc testRead(s: AsyncSocket, no: int) = - echo("Reading! " & $no) - var data = "" - if not s.readLine(data): return - if data == "": - echo("Closing connection. " & $no) - s.close() - echo(data) - echo("Finished reading! " & $no) - - proc testAccept(s: AsyncSocket, disp: Dispatcher, no: int) = - echo("Accepting client! " & $no) - var client: AsyncSocket - new(client) - var address = "" - s.acceptAddr(client, address) - echo("Accepted ", address) - client.handleRead = - proc (s: AsyncSocket) = - testRead(s, 2) - disp.register(client) - - proc main = - var d = newDispatcher() - - var s = asyncSocket() - s.connect("amber.tenthbit.net", Port(6667)) - s.handleConnect = - proc (s: AsyncSocket) = - testConnect(s, 1) - s.handleRead = - proc (s: AsyncSocket) = - testRead(s, 1) - d.register(s) - - var server = asyncSocket() - server.handleAccept = - proc (s: AsyncSocket) = - testAccept(s, d, 78) - server.bindAddr(Port(5555)) - server.listen() - d.register(server) - - while d.poll(-1): discard - main() diff --git a/lib/deprecated/pure/ftpclient.nim b/lib/deprecated/pure/ftpclient.nim deleted file mode 100644 index 1188c0795..000000000 --- a/lib/deprecated/pure/ftpclient.nim +++ /dev/null @@ -1,675 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import sockets, strutils, parseutils, times, os, asyncio - -from asyncnet import nil -from nativesockets import nil -from asyncdispatch import PFuture -## **Note**: This module is deprecated since version 0.11.3. -## You should use the async version of this module -## `asyncftpclient <asyncftpclient.html>`_. -## -## ---- -## -## This module **partially** implements an FTP client as specified -## by `RFC 959 <http://tools.ietf.org/html/rfc959>`_. -## -## This module provides both a synchronous and asynchronous implementation. -## The asynchronous implementation requires you to use the ``asyncFTPClient`` -## function. You are then required to register the ``AsyncFTPClient`` with a -## asyncio dispatcher using the ``register`` function. Take a look at the -## asyncio module documentation for more information. -## -## **Note**: The asynchronous implementation is only asynchronous for long -## file transfers, calls to functions which use the command socket will block. -## -## Here is some example usage of this module: -## -## .. code-block:: Nim -## var ftp = ftpClient("example.org", user = "user", pass = "pass") -## ftp.connect() -## ftp.retrFile("file.ext", "file.ext") -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. - -{.deprecated.} - -type - FtpBase*[SockType] = ref FtpBaseObj[SockType] - FtpBaseObj*[SockType] = object - csock*: SockType - dsock*: SockType - when SockType is asyncio.AsyncSocket: - handleEvent*: proc (ftp: AsyncFTPClient, ev: FTPEvent){.closure,gcsafe.} - disp: Dispatcher - asyncDSockID: Delegate - user*, pass*: string - address*: string - when SockType is asyncnet.AsyncSocket: - port*: nativesockets.Port - else: - port*: Port - - jobInProgress*: bool - job*: FTPJob[SockType] - - dsockConnected*: bool - - FTPJobType* = enum - JRetrText, JRetr, JStore - - FtpJob[T] = ref FtpJobObj[T] - FTPJobObj[T] = object - prc: proc (ftp: FTPBase[T], async: bool): bool {.nimcall, gcsafe.} - case typ*: FTPJobType - of JRetrText: - lines: string - of JRetr, JStore: - file: File - filename: string - total: BiggestInt # In bytes. - progress: BiggestInt # In bytes. - oneSecond: BiggestInt # Bytes transferred in one second. - lastProgressReport: float # Time - toStore: string # Data left to upload (Only used with async) - else: nil - - FtpClientObj* = FtpBaseObj[Socket] - FtpClient* = ref FtpClientObj - - AsyncFtpClient* = ref AsyncFtpClientObj ## Async alternative to TFTPClient. - AsyncFtpClientObj* = FtpBaseObj[asyncio.AsyncSocket] - - FTPEventType* = enum - EvTransferProgress, EvLines, EvRetr, EvStore - - FTPEvent* = object ## Event - filename*: string - case typ*: FTPEventType - of EvLines: - lines*: string ## Lines that have been transferred. - of EvRetr, EvStore: ## Retr/Store operation finished. - nil - of EvTransferProgress: - bytesTotal*: BiggestInt ## Bytes total. - bytesFinished*: BiggestInt ## Bytes transferred. - speed*: BiggestInt ## Speed in bytes/s - currentJob*: FTPJobType ## The current job being performed. - - ReplyError* = object of IOError - FTPError* = object of IOError - -{.deprecated: [ - TFTPClient: FTPClientObj, TFTPJob: FTPJob, PAsyncFTPClient: AsyncFTPClient, - TAsyncFTPClient: AsyncFTPClientObj, TFTPEvent: FTPEvent, - EInvalidReply: ReplyError, EFTP: FTPError -].} - -const multiLineLimit = 10000 - -proc ftpClient*(address: string, port = Port(21), - user, pass = ""): FtpClient = - ## Create a ``FtpClient`` object. - new(result) - result.user = user - result.pass = pass - result.address = address - result.port = port - - result.dsockConnected = false - result.csock = socket() - if result.csock == invalidSocket: raiseOSError(osLastError()) - -template blockingOperation(sock: Socket, body: stmt) {.immediate.} = - body - -template blockingOperation(sock: asyncio.AsyncSocket, body: stmt) {.immediate.} = - sock.setBlocking(true) - body - sock.setBlocking(false) - -proc expectReply[T](ftp: FtpBase[T]): TaintedString = - result = TaintedString"" - blockingOperation(ftp.csock): - when T is Socket: - ftp.csock.readLine(result) - else: - discard ftp.csock.readLine(result) - var count = 0 - while result[3] == '-': - ## Multi-line reply. - var line = TaintedString"" - when T is Socket: - ftp.csock.readLine(line) - else: - discard ftp.csock.readLine(line) - result.add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*[T](ftp: FtpBase[T], m: string): TaintedString = - ## Send a message to the server, and wait for a primary reply. - ## ``\c\L`` is added for you. - ## - ## **Note:** The server may return multiple lines of coded replies. - blockingOperation(ftp.csock): - ftp.csock.send(m & "\c\L") - return ftp.expectReply() - -proc assertReply(received: TaintedString, expected: string) = - if not received.string.startsWith(expected): - raise newException(ReplyError, - "Expected reply '$1' got: $2" % [ - expected, received.string]) - -proc assertReply(received: TaintedString, expected: varargs[string]) = - for i in items(expected): - if received.string.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received.string]) - -proc createJob[T](ftp: FtpBase[T], - prc: proc (ftp: FtpBase[T], async: bool): bool {. - nimcall,gcsafe.}, - cmd: FTPJobType) = - if ftp.jobInProgress: - raise newException(FTPError, "Unable to do two jobs at once.") - ftp.jobInProgress = true - new(ftp.job) - ftp.job.prc = prc - ftp.job.typ = cmd - case cmd - of JRetrText: - ftp.job.lines = "" - of JRetr, JStore: - ftp.job.toStore = "" - -proc deleteJob[T](ftp: FtpBase[T]) = - assert ftp.jobInProgress - ftp.jobInProgress = false - case ftp.job.typ - of JRetrText: - ftp.job.lines = "" - of JRetr, JStore: - ftp.job.file.close() - ftp.dsock.close() - -proc handleTask(s: AsyncSocket, ftp: AsyncFTPClient) = - if ftp.jobInProgress: - if ftp.job.typ in {JRetr, JStore}: - if epochTime() - ftp.job.lastProgressReport >= 1.0: - var r: FTPEvent - ftp.job.lastProgressReport = epochTime() - r.typ = EvTransferProgress - r.bytesTotal = ftp.job.total - r.bytesFinished = ftp.job.progress - r.speed = ftp.job.oneSecond - r.filename = ftp.job.filename - r.currentJob = ftp.job.typ - ftp.job.oneSecond = 0 - ftp.handleEvent(ftp, r) - -proc handleWrite(s: AsyncSocket, ftp: AsyncFTPClient) = - if ftp.jobInProgress: - if ftp.job.typ == JStore: - assert (not ftp.job.prc(ftp, true)) - -proc handleConnect(s: AsyncSocket, ftp: AsyncFTPClient) = - ftp.dsockConnected = true - assert(ftp.jobInProgress) - if ftp.job.typ == JStore: - s.setHandleWrite(proc (s: AsyncSocket) = handleWrite(s, ftp)) - else: - s.delHandleWrite() - -proc handleRead(s: AsyncSocket, ftp: AsyncFTPClient) = - assert ftp.jobInProgress - assert ftp.job.typ != JStore - # This can never return true, because it shouldn't check for code - # 226 from csock. - assert(not ftp.job.prc(ftp, true)) - -proc pasv[T](ftp: FtpBase[T]) = - ## Negotiate a data connection. - when T is Socket: - ftp.dsock = socket() - if ftp.dsock == invalidSocket: raiseOSError(osLastError()) - elif T is AsyncSocket: - ftp.dsock = asyncSocket() - ftp.dsock.handleRead = - proc (s: AsyncSocket) = - handleRead(s, ftp) - ftp.dsock.handleConnect = - proc (s: AsyncSocket) = - handleConnect(s, ftp) - ftp.dsock.handleTask = - proc (s: AsyncSocket) = - handleTask(s, ftp) - ftp.disp.register(ftp.dsock) - else: - {.fatal: "Incorrect socket instantiation".} - - var pasvMsg = ftp.send("PASV").string.strip.TaintedString - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg.string, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0.. ^3] - var port = nums[^2.. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - ftp.dsock.connect(ip.join("."), Port(properPort.toU16)) - when T is AsyncSocket: - ftp.dsockConnected = false - else: - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*[T](ftp: FtpBase[T]) = - ## Connect to the FTP server specified by ``ftp``. - when T is AsyncSocket: - blockingOperation(ftp.csock): - ftp.csock.connect(ftp.address, ftp.port) - elif T is Socket: - ftp.csock.connect(ftp.address, ftp.port) - else: - {.fatal: "Incorrect socket instantiation".} - - var reply = ftp.expectReply() - if reply.startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = ftp.expectReply() - - # Handle 220 messages from the server - assertReply ftp.expectReply(), "220" - - if ftp.user != "": - assertReply(ftp.send("USER " & ftp.user), "230", "331") - - if ftp.pass != "": - assertReply ftp.send("PASS " & ftp.pass), "230" - -proc pwd*[T](ftp: FtpBase[T]): string = - ## Returns the current working directory. - var wd = ftp.send("PWD") - assertReply wd, "257" - return wd.string.captureBetween('"') # " - -proc cd*[T](ftp: FtpBase[T], dir: string) = - ## Changes the current directory on the remote FTP server to ``dir``. - assertReply ftp.send("CWD " & dir.normalizePathSep), "250" - -proc cdup*[T](ftp: FtpBase[T]) = - ## Changes the current directory to the parent of the current directory. - assertReply ftp.send("CDUP"), "200" - -proc getLines[T](ftp: FtpBase[T], async: bool = false): bool = - ## Downloads text data in ASCII mode - ## Returns true if the download is complete. - ## It doesn't if `async` is true, because it doesn't check for 226 then. - if ftp.dsockConnected: - var r = TaintedString"" - when T is AsyncSocket: - if ftp.asyncDSock.readLine(r): - if r.string == "": - ftp.dsockConnected = false - else: - ftp.job.lines.add(r.string & "\n") - elif T is Socket: - assert(not async) - ftp.dsock.readLine(r) - if r.string == "": - ftp.dsockConnected = false - else: - ftp.job.lines.add(r.string & "\n") - else: - {.fatal: "Incorrect socket instantiation".} - - if not async: - var readSocks: seq[Socket] = @[ftp.csock] - # This is only needed here. Asyncio gets this socket... - blockingOperation(ftp.csock): - if readSocks.select(1) != 0 and ftp.csock in readSocks: - assertReply ftp.expectReply(), "226" - return true - -proc listDirs*[T](ftp: FtpBase[T], dir: string = "", - async = false): seq[string] = - ## Returns a list of filenames in the given directory. If ``dir`` is "", - ## the current directory is used. If ``async`` is true, this - ## function will return immediately and it will be your job to - ## use asyncio's ``poll`` to progress this operation. - - ftp.createJob(getLines[T], JRetrText) - ftp.pasv() - - assertReply ftp.send("NLST " & dir.normalizePathSep), ["125", "150"] - - if not async: - while not ftp.job.prc(ftp, false): discard - result = splitLines(ftp.job.lines) - ftp.deleteJob() - else: return @[] - -proc fileExists*(ftp: FtpClient, file: string): bool {.deprecated.} = - ## **Deprecated since version 0.9.0:** Please use ``existsFile``. - ## - ## Determines whether ``file`` exists. - ## - ## Warning: This function may block. Especially on directories with many - ## files, because a full list of file names must be retrieved. - var files = ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc existsFile*(ftp: FtpClient, file: string): bool = - ## Determines whether ``file`` exists. - ## - ## Warning: This function may block. Especially on directories with many - ## files, because a full list of file names must be retrieved. - var files = ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*[T](ftp: FtpBase[T], dir: string, recursive: bool = false) = - ## Creates a directory ``dir``. If ``recursive`` is true, the topmost - ## subdirectory of ``dir`` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the ``dir`` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply ftp.send("MKD " & dir.normalizePathSep), "257" - else: - var reply = TaintedString"" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*[T](ftp: FtpBase[T], path: string, - permissions: set[FilePermission]) = - ## Changes permission of ``path`` to ``permissions``. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep), "200" - -proc list*[T](ftp: FtpBase[T], dir: string = "", async = false): string = - ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current - ## working directory. If ``async`` is true, this function will return - ## immediately and it will be your job to call asyncio's - ## ``poll`` to progress this operation. - ftp.createJob(getLines[T], JRetrText) - ftp.pasv() - - assertReply(ftp.send("LIST" & " " & dir.normalizePathSep), ["125", "150"]) - - if not async: - while not ftp.job.prc(ftp, false): discard - result = ftp.job.lines - ftp.deleteJob() - else: - return "" - -proc retrText*[T](ftp: FtpBase[T], file: string, async = false): string = - ## Retrieves ``file``. File must be ASCII text. - ## If ``async`` is true, this function will return immediately and - ## it will be your job to call asyncio's ``poll`` to progress this operation. - ftp.createJob(getLines[T], JRetrText) - ftp.pasv() - assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"] - - if not async: - while not ftp.job.prc(ftp, false): discard - result = ftp.job.lines - ftp.deleteJob() - else: - return "" - -proc getFile[T](ftp: FtpBase[T], async = false): bool = - if ftp.dsockConnected: - var r = "".TaintedString - var bytesRead = 0 - var returned = false - if async: - when T is Socket: - raise newException(FTPError, "FTPClient must be async.") - else: - bytesRead = ftp.dsock.recvAsync(r, BufferSize) - returned = bytesRead != -1 - else: - bytesRead = ftp.dsock.recv(r, BufferSize) - returned = true - let r2 = r.string - if r2 != "": - ftp.job.progress.inc(r2.len) - ftp.job.oneSecond.inc(r2.len) - ftp.job.file.write(r2) - elif returned and r2 == "": - ftp.dsockConnected = false - - when T is Socket: - if not async: - var readSocks: seq[Socket] = @[ftp.csock] - blockingOperation(ftp.csock): - if readSocks.select(1) != 0 and ftp.csock in readSocks: - assertReply ftp.expectReply(), "226" - return true - -proc retrFile*[T](ftp: FtpBase[T], file, dest: string, async = false) = - ## Downloads ``file`` and saves it to ``dest``. Usage of this function - ## asynchronously is recommended to view the progress of the download. - ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function - ## when the download is finished, and the ``filename`` field will be equal - ## to ``file``. - ftp.createJob(getFile[T], JRetr) - ftp.job.file = open(dest, mode = fmWrite) - ftp.pasv() - var reply = ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply.string: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - ftp.job.total = fileSize - ftp.job.lastProgressReport = epochTime() - ftp.job.filename = file.normalizePathSep - - if not async: - while not ftp.job.prc(ftp, false): discard - ftp.deleteJob() - -proc doUpload[T](ftp: FtpBase[T], async = false): bool = - if ftp.dsockConnected: - if ftp.job.toStore.len() > 0: - assert(async) - let bytesSent = ftp.dsock.sendAsync(ftp.job.toStore) - if bytesSent == ftp.job.toStore.len: - ftp.job.toStore = "" - elif bytesSent != ftp.job.toStore.len and bytesSent != 0: - ftp.job.toStore = ftp.job.toStore[bytesSent .. ^1] - ftp.job.progress.inc(bytesSent) - ftp.job.oneSecond.inc(bytesSent) - else: - var s = newStringOfCap(4000) - var len = ftp.job.file.readBuffer(addr(s[0]), 4000) - setLen(s, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - if not async: - assertReply ftp.expectReply(), "226" - return true - return false - - if not async: - ftp.dsock.send(s) - else: - let bytesSent = ftp.dsock.sendAsync(s) - if bytesSent == 0: - ftp.job.toStore.add(s) - elif bytesSent != s.len: - ftp.job.toStore.add(s[bytesSent .. ^1]) - len = bytesSent - - ftp.job.progress.inc(len) - ftp.job.oneSecond.inc(len) - -proc store*[T](ftp: FtpBase[T], file, dest: string, async = false) = - ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The ``EvStore`` event is passed to the specified ``handleEvent`` function - ## when the upload is finished, and the ``filename`` field will be - ## equal to ``file``. - ftp.createJob(doUpload[T], JStore) - ftp.job.file = open(file) - ftp.job.total = ftp.job.file.getFileSize() - ftp.job.lastProgressReport = epochTime() - ftp.job.filename = file - ftp.pasv() - - assertReply ftp.send("STOR " & dest.normalizePathSep), ["125", "150"] - - if not async: - while not ftp.job.prc(ftp, false): discard - ftp.deleteJob() - -proc close*[T](ftp: FtpBase[T]) = - ## Terminates the connection to the server. - assertReply ftp.send("QUIT"), "221" - if ftp.jobInProgress: ftp.deleteJob() - ftp.csock.close() - ftp.dsock.close() - -proc csockHandleRead(s: AsyncSocket, ftp: AsyncFTPClient) = - if ftp.jobInProgress: - assertReply ftp.expectReply(), "226" # Make sure the transfer completed. - var r: FTPEvent - case ftp.job.typ - of JRetrText: - r.typ = EvLines - r.lines = ftp.job.lines - of JRetr: - r.typ = EvRetr - r.filename = ftp.job.filename - if ftp.job.progress != ftp.job.total: - raise newException(FTPError, "Didn't download full file.") - of JStore: - r.typ = EvStore - r.filename = ftp.job.filename - if ftp.job.progress != ftp.job.total: - raise newException(FTPError, "Didn't upload full file.") - ftp.deleteJob() - - ftp.handleEvent(ftp, r) - -proc asyncFTPClient*(address: string, port = Port(21), - user, pass = "", - handleEvent: proc (ftp: AsyncFTPClient, ev: FTPEvent) {.closure,gcsafe.} = - (proc (ftp: AsyncFTPClient, ev: FTPEvent) = discard)): AsyncFTPClient = - ## Create a ``AsyncFTPClient`` object. - ## - ## Use this if you want to use asyncio's dispatcher. - var dres: AsyncFtpClient - new(dres) - dres.user = user - dres.pass = pass - dres.address = address - dres.port = port - dres.dsockConnected = false - dres.handleEvent = handleEvent - dres.csock = asyncSocket() - dres.csock.handleRead = - proc (s: AsyncSocket) = - csockHandleRead(s, dres) - result = dres - -proc register*(d: Dispatcher, ftp: AsyncFTPClient): Delegate {.discardable.} = - ## Registers ``ftp`` with dispatcher ``d``. - ftp.disp = d - return ftp.disp.register(ftp.csock) - -when not defined(testing) and isMainModule: - proc main = - var d = newDispatcher() - let hev = - proc (ftp: AsyncFTPClient, event: FTPEvent) = - case event.typ - of EvStore: - echo("Upload finished!") - ftp.retrFile("payload.jpg", "payload2.jpg", async = true) - of EvTransferProgress: - var time: int64 = -1 - if event.speed != 0: - time = (event.bytesTotal - event.bytesFinished) div event.speed - echo(event.currentJob) - echo(event.speed div 1000, " kb/s. - ", - event.bytesFinished, "/", event.bytesTotal, - " - ", time, " seconds") - echo(d.len) - of EvRetr: - echo("Download finished!") - ftp.close() - echo d.len - else: assert(false) - var ftp = asyncFTPClient("example.com", user = "foo", pass = "bar", handleEvent = hev) - - d.register(ftp) - d.len.echo() - ftp.connect() - echo "connected" - ftp.store("payload.jpg", "payload.jpg", async = true) - d.len.echo() - echo "uploading..." - while true: - if not d.poll(): break - main() - -when not defined(testing) and isMainModule: - var ftp = ftpClient("example.com", user = "foo", pass = "bar") - ftp.connect() - echo ftp.pwd() - echo ftp.list() - echo("uploading") - ftp.store("payload.jpg", "payload.jpg", async = false) - - echo("Upload complete") - ftp.retrFile("payload.jpg", "payload2.jpg", async = false) - - echo("Download complete") - sleep(5000) - ftp.close() - sleep(200) diff --git a/lib/deprecated/pure/future.nim b/lib/deprecated/pure/future.nim new file mode 100644 index 000000000..0e06161f2 --- /dev/null +++ b/lib/deprecated/pure/future.nim @@ -0,0 +1,6 @@ +## This module is a deprecated alias for the `sugar` module. Deprecated since 0.19.0. + +{.deprecated: "Use the new 'sugar' module instead".} + +import std/sugar +export sugar diff --git a/lib/deprecated/pure/mersenne.nim b/lib/deprecated/pure/mersenne.nim new file mode 100644 index 000000000..37c5085b1 --- /dev/null +++ b/lib/deprecated/pure/mersenne.nim @@ -0,0 +1,51 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister) +## random number generator. +## .. note:: The procs in this module work at compile-time. + +{.deprecated: "use `std/random` instead".} + +runnableExamples: + var rand = newMersenneTwister(uint32.high) ## must be "var" + doAssert rand.getNum() != rand.getNum() ## pseudorandom number +## See also +## ======== +## * `random module<random.html>`_ for Nim's standard random number generator +type + MersenneTwister* = object + ## The Mersenne Twister. + mt: array[0..623, uint32] + index: int + +proc newMersenneTwister*(seed: uint32): MersenneTwister = + ## Creates a new `MersenneTwister` with seed `seed`. + result.index = 0 + result.mt[0] = seed + for i in 1'u32 .. 623'u32: + result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor + (result.mt[i-1] shr 30'u32)) + i) + +proc generateNumbers(m: var MersenneTwister) = + + for i in 0..623: + var y = (m.mt[i] and 0x80000000'u32) + + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) + m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) + if (y mod 2'u32) != 0: + m.mt[i] = m.mt[i] xor 0x9908b0df'u32 + +proc getNum*(m: var MersenneTwister): uint32 = + ## Returns the next pseudorandom `uint32`. + if m.index == 0: + generateNumbers(m) + result = m.mt[m.index] + m.index = (m.index + 1) mod m.mt.len + result = result xor (result shr 11'u32) + result = result xor ((result shl 7'u32) and 0x9d2c5680'u32) + result = result xor ((result shl 15'u32) and 0xefc60000'u32) + result = result xor (result shr 18'u32) diff --git a/lib/deprecated/pure/ospaths.nim b/lib/deprecated/pure/ospaths.nim new file mode 100644 index 000000000..43fcb17cc --- /dev/null +++ b/lib/deprecated/pure/ospaths.nim @@ -0,0 +1,22 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module is deprecated since 0.20.0, `import std/os` instead. + +{.deprecated: "use `std/os` instead".} + +import std/os +export ReadEnvEffect, WriteEnvEffect, ReadDirEffect, WriteDirEffect, OSErrorCode, + doslikeFileSystem, CurDir, ParDir, DirSep, AltSep, PathSep, FileSystemCaseSensitive, + ExeExt, ScriptExt, DynlibFormat, ExtSep, joinPath, `/`, splitPath, parentDir, + tailDir, isRootDir, parentDirs, `/../`, searchExtPos, splitFile, extractFilename, + lastPathPart, changeFileExt, addFileExt, cmpPaths, isAbsolute, unixToNativePath, + `==`, `$`, osErrorMsg, raiseOSError, osLastError, getEnv, existsEnv, putEnv, + getHomeDir, getConfigDir, getTempDir, expandTilde, quoteShellWindows, + quoteShellPosix, quoteShell, quoteShellCommand diff --git a/lib/deprecated/pure/oswalkdir.nim b/lib/deprecated/pure/oswalkdir.nim new file mode 100644 index 000000000..57a2cb81d --- /dev/null +++ b/lib/deprecated/pure/oswalkdir.nim @@ -0,0 +1,13 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module is deprecated, `import std/os` instead. +{.deprecated: "import 'std/os' instead".} +import std/os +export PathComponent, walkDir, walkDirRec diff --git a/lib/deprecated/pure/parseurl.nim b/lib/deprecated/pure/parseurl.nim deleted file mode 100644 index 6d58e8a73..000000000 --- a/lib/deprecated/pure/parseurl.nim +++ /dev/null @@ -1,114 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warnings:** This module is deprecated since version 0.10.2. -## Use the `uri <uri.html>`_ module instead. -## -## Parses & constructs URLs. - -{.deprecated.} - -import strutils - -type - Url* = tuple[ ## represents a *Uniform Resource Locator* (URL) - ## any optional component is "" if it does not exist - scheme, username, password, - hostname, port, path, query, anchor: string] - -{.deprecated: [TUrl: Url].} - -proc parseUrl*(url: string): Url {.deprecated.} = - var i = 0 - - var scheme, username, password: string = "" - var hostname, port, path, query, anchor: string = "" - - var temp = "" - - if url[i] != '/': # url isn't a relative path - while true: - # Scheme - if url[i] == ':': - if url[i+1] == '/' and url[i+2] == '/': - scheme = temp - temp.setLen(0) - inc(i, 3) # Skip the // - # Authority(username, password) - if url[i] == '@': - username = temp - let colon = username.find(':') - if colon >= 0: - password = username.substr(colon+1) - username = username.substr(0, colon-1) - temp.setLen(0) - inc(i) #Skip the @ - # hostname(subdomain, domain, port) - if url[i] == '/' or url[i] == '\0': - hostname = temp - let colon = hostname.find(':') - if colon >= 0: - port = hostname.substr(colon+1) - hostname = hostname.substr(0, colon-1) - - temp.setLen(0) - break - - temp.add(url[i]) - inc(i) - - if url[i] == '/': inc(i) # Skip the '/' - # Path - while true: - if url[i] == '?': - path = temp - temp.setLen(0) - if url[i] == '#': - if temp[0] == '?': - query = temp - else: - path = temp - temp.setLen(0) - - if url[i] == '\0': - if temp[0] == '?': - query = temp - elif temp[0] == '#': - anchor = temp - else: - path = temp - break - - temp.add(url[i]) - inc(i) - - return (scheme, username, password, hostname, port, path, query, anchor) - -proc `$`*(u: Url): string {.deprecated.} = - ## turns the URL `u` into its string representation. - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - result.add("/") - result.add(u.path) - result.add(u.query) - result.add(u.anchor) - diff --git a/lib/deprecated/pure/rawsockets.nim b/lib/deprecated/pure/rawsockets.nim deleted file mode 100644 index ee77b232e..000000000 --- a/lib/deprecated/pure/rawsockets.nim +++ /dev/null @@ -1,14 +0,0 @@ -import nativesockets -export nativesockets - -{.warning: "rawsockets module is deprecated, use nativesockets instead".} - -template newRawSocket*(domain, sockType, protocol: cint): expr = - {.warning: "newRawSocket is deprecated, use newNativeSocket instead".} - newNativeSocket(domain, sockType, protocol) - -template newRawSocket*(domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): expr = - {.warning: "newRawSocket is deprecated, use newNativeSocket instead".} - newNativeSocket(domain, sockType, protocol) diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim deleted file mode 100644 index 5d6fa0078..000000000 --- a/lib/deprecated/pure/sockets.nim +++ /dev/null @@ -1,1738 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf, Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## **Warning:** Since version 0.10.2 this module is deprecated. -## Use the `net <net.html>`_ or the -## `nativesockets <nativesockets.html>`_ module instead. -## -## This module implements portable sockets, it supports a mix of different types -## of sockets. Sockets are buffered by default meaning that data will be -## received in ``BufferSize`` (4000) sized chunks, buffering -## behaviour can be disabled by setting the ``buffered`` parameter when calling -## the ``socket`` function to `false`. Be aware that some functions may not yet -## support buffered sockets (mainly the recvFrom function). -## -## Most procedures raise OSError on error, but some may return ``-1`` or a -## boolean ``false``. -## -## SSL is supported through the OpenSSL library. This support can be activated -## by compiling with the ``-d:ssl`` switch. When an SSL socket is used it will -## raise ESSL exceptions when SSL errors occur. -## -## Asynchronous sockets are supported, however a better alternative is to use -## the `asyncio <asyncio.html>`_ module. - -{.deprecated.} - -include "system/inclrtl" - -{.deadCodeElim: on.} - -when hostOS == "solaris": - {.passl: "-lsocket -lnsl".} - -import os, parseutils -from times import epochTime -import unsigned - -when defined(ssl): - import openssl - -when defined(Windows): - import winlean -else: - import posix - -# Note: The enumerations are mapped to Window's constants. - -when defined(ssl): - - type - SSLError* = object of Exception - - SSLCVerifyMode* = enum - CVerifyNone, CVerifyPeer - - SSLProtVersion* = enum - protSSLv2, protSSLv3, protTLSv1, protSSLv23 - - SSLContext* = distinct SSLCTX - - SSLAcceptResult* = enum - AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess - - {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, - TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, - TSSLAcceptResult: SSLAcceptResult].} - -const - BufferSize*: int = 4000 ## size of a buffered socket's buffer - -type - SocketImpl = object ## socket type - fd: SocketHandle - case isBuffered: bool # determines whether this socket is buffered. - of true: - buffer: array[0..BufferSize, char] - currPos: int # current index in buffer - bufLen: int # current length of buffer - of false: nil - when defined(ssl): - case isSsl: bool - of true: - sslHandle: SSLPtr - sslContext: SSLContext - sslNoHandshake: bool # True if needs handshake. - sslHasPeekChar: bool - sslPeekChar: char - of false: nil - nonblocking: bool - - Socket* = ref SocketImpl - - Port* = distinct uint16 ## port type - - Domain* = enum ## domain, which specifies the protocol family of the - ## created socket. Other domains than those that are listed - ## here are unsupported. - AF_UNIX, ## for local socket (using a file). Unsupported on Windows. - AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = 23 ## for network protocol IPv6. - - SockType* = enum ## second argument to `socket` proc - SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets - SOCK_DGRAM = 2, ## datagram service or Datagram Sockets - SOCK_RAW = 3, ## raw protocols atop the network layer. - SOCK_SEQPACKET = 5 ## reliable sequenced packet service - - Protocol* = enum ## third argument to `socket` proc - IPPROTO_TCP = 6, ## Transmission control protocol. - IPPROTO_UDP = 17, ## User datagram protocol. - IPPROTO_IP, ## Internet protocol. Unsupported on Windows. - IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. - IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. - IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. - - Servent* = object ## information about a service - name*: string - aliases*: seq[string] - port*: Port - proto*: string - - Hostent* = object ## information about a given host - name*: string - aliases*: seq[string] - addrtype*: Domain - length*: int - addrList*: seq[string] - - SOBool* = enum ## Boolean socket options. - OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr - - RecvLineResult* = enum ## result for recvLineAsync - RecvFullLine, RecvPartialLine, RecvDisconnected, RecvFail - - ReadLineResult* = enum ## result for readLineAsync - ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone - - TimeoutError* = object of Exception - -{.deprecated: [TSocket: Socket, TType: SockType, TPort: Port, TDomain: Domain, - TProtocol: Protocol, TServent: Servent, THostent: Hostent, - TSOBool: SOBool, TRecvLineResult: RecvLineResult, - TReadLineResult: ReadLineResult, ETimeout: TimeoutError, - TSocketImpl: SocketImpl].} - -when defined(booting): - let invalidSocket*: Socket = nil ## invalid socket -else: - const invalidSocket*: Socket = nil ## invalid socket - -when defined(windows): - let - osInvalidSocket = winlean.INVALID_SOCKET -else: - let - osInvalidSocket = posix.INVALID_SOCKET - -proc newTSocket(fd: SocketHandle, isBuff: bool): Socket = - if fd == osInvalidSocket: - return nil - new(result) - result.fd = fd - result.isBuffered = isBuff - if isBuff: - result.currPos = 0 - result.nonblocking = false - -proc `==`*(a, b: Port): bool {.borrow.} - ## ``==`` for ports. - -proc `$`*(p: Port): string {.borrow.} - ## returns the port number as a string - -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. - ## On machines where the host byte order is the same as network byte order, - ## this is a no-op; otherwise, it performs a 4-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) - -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) - -proc htonl*(x: int32): int32 = - ## Converts 32-bit integers from host to network byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 4-byte swap operation. - result = sockets.ntohl(x) - -proc htons*(x: int16): int16 = - ## Converts 16-bit positive integers from host to network byte order. - ## On machines where the host byte order is the same as network byte - ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. - result = sockets.ntohs(x) - -when defined(Posix): - proc toInt(domain: Domain): cint = - case domain - of AF_UNIX: result = posix.AF_UNIX - of AF_INET: result = posix.AF_INET - of AF_INET6: result = posix.AF_INET6 - else: discard - - proc toInt(typ: SockType): cint = - case typ - of SOCK_STREAM: result = posix.SOCK_STREAM - of SOCK_DGRAM: result = posix.SOCK_DGRAM - of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET - of SOCK_RAW: result = posix.SOCK_RAW - else: discard - - proc toInt(p: Protocol): cint = - case p - of IPPROTO_TCP: result = posix.IPPROTO_TCP - of IPPROTO_UDP: result = posix.IPPROTO_UDP - of IPPROTO_IP: result = posix.IPPROTO_IP - of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 - of IPPROTO_RAW: result = posix.IPPROTO_RAW - of IPPROTO_ICMP: result = posix.IPPROTO_ICMP - else: discard - -else: - proc toInt(domain: Domain): cint = - result = toU16(ord(domain)) - - proc toInt(typ: SockType): cint = - result = cint(ord(typ)) - - proc toInt(p: Protocol): cint = - result = cint(ord(p)) - -proc socket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): Socket = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. - - # TODO: Perhaps this should just raise EOS when an error occurs. - when defined(Windows): - result = newTSocket(winlean.socket(ord(domain), ord(typ), ord(protocol)), buffered) - else: - result = newTSocket(posix.socket(toInt(domain), toInt(typ), toInt(protocol)), buffered) - -when defined(ssl): - CRYPTO_malloc_init() - SslLibraryInit() - SslLoadErrorStrings() - ErrLoadBioStrings() - OpenSSL_add_all_algorithms() - - proc raiseSSLError(s = "") = - if s != "": - raise newException(SSLError, s) - let err = ErrPeekLastError() - if err == 0: - raise newException(SSLError, "No error reported.") - if err == -1: - raiseOSError(osLastError()) - var errStr = ErrErrorString(err, nil) - raise newException(SSLError, $errStr) - - # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html - proc loadCertificates(ctx: SSL_CTX, certFile, keyFile: string) = - if certFile != "" and not existsFile(certFile): - raise newException(system.IOError, "Certificate file could not be found: " & certFile) - if keyFile != "" and not existsFile(keyFile): - raise newException(system.IOError, "Key file could not be found: " & keyFile) - - if certFile != "": - var ret = SSLCTXUseCertificateChainFile(ctx, certFile) - if ret != 1: - raiseSslError() - - # TODO: Password? www.rtfm.com/openssl-examples/part1.pdf - if keyFile != "": - if SSL_CTX_use_PrivateKey_file(ctx, keyFile, - SSL_FILETYPE_PEM) != 1: - raiseSslError() - - if SSL_CTX_check_private_key(ctx) != 1: - raiseSslError("Verification of private key file failed.") - - proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, - certFile = "", keyFile = ""): SSLContext = - ## Creates an SSL context. - ## - ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 are - ## are available with the addition of ``ProtSSLv23`` which allows for - ## compatibility with all of them. - ## - ## There are currently only two options for verify mode; - ## one is ``CVerifyNone`` and with it certificates will not be verified - ## the other is ``CVerifyPeer`` and certificates will be verified for - ## it, ``CVerifyPeer`` is the safest choice. - ## - ## The last two parameters specify the certificate file path and the key file - ## path, a server socket will most likely not work without these. - ## Certificates can be generated using the following command: - ## ``openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.pem -out mycert.pem``. - var newCTX: SSL_CTX - case protVersion - of protSSLv23: - newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. - of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") - of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) - of protTLSv1: - newCTX = SSL_CTX_new(TLSv1_method()) - - if newCTX.SSLCTXSetCipherList("ALL") != 1: - raiseSslError() - case verifyMode - of CVerifyPeer: - newCTX.SSLCTXSetVerify(SSLVerifyPeer, nil) - of CVerifyNone: - newCTX.SSLCTXSetVerify(SSLVerifyNone, nil) - if newCTX == nil: - raiseSslError() - - discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) - newCTX.loadCertificates(certFile, keyFile) - return SSLContext(newCTX) - - proc wrapSocket*(ctx: SSLContext, socket: Socket) = - ## Wraps a socket in an SSL context. This function effectively turns - ## ``socket`` into an SSL socket. - ## - ## **Disclaimer**: This code is not well tested, may be very unsafe and - ## prone to security vulnerabilities. - - socket.isSSL = true - socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) - socket.sslNoHandshake = false - socket.sslHasPeekChar = false - if socket.sslHandle == nil: - raiseSslError() - - if SSLSetFd(socket.sslHandle, socket.fd) != 1: - raiseSslError() - -proc raiseSocketError*(socket: Socket, err: int = -1, async = false) = - ## Raises proper errors based on return values of ``recv`` functions. - ## - ## If ``async`` is ``True`` no error will be thrown in the case when the - ## error was caused by no data being available to be read. - ## - ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): - if socket.isSSL: - if err <= 0: - var ret = SSLGetError(socket.sslHandle, err.cint) - case ret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - if async: - return - else: raiseSslError("Not enough data on socket.") - of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - if async: - return - else: raiseSslError("Not enough data on socket.") - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: raiseSslError("Unknown Error") - - if err == -1 and not (when defined(ssl): socket.isSSL else: false): - let lastError = osLastError() - if async: - when defined(windows): - if lastError.int32 == WSAEWOULDBLOCK: - return - else: raiseOSError(lastError) - else: - if lastError.int32 == EAGAIN or lastError.int32 == EWOULDBLOCK: - return - else: raiseOSError(lastError) - else: raiseOSError(lastError) - -proc listen*(socket: Socket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the - ## queue of pending connections. - if listen(socket.fd, cint(backlog)) < 0'i32: raiseOSError(osLastError()) - -proc invalidIp4(s: string) {.noreturn, noinline.} = - raise newException(ValueError, "invalid ip4 address: " & s) - -proc parseIp4*(s: string): BiggestInt = - ## parses an IP version 4 in dotted decimal form like "a.b.c.d". - ## - ## This is equivalent to `inet_ntoa`:idx:. - ## - ## Raises EInvalidValue in case of an error. - var a, b, c, d: int - var i = 0 - var j = parseInt(s, a, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] == '.': inc(i) - else: invalidIp4(s) - j = parseInt(s, b, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] == '.': inc(i) - else: invalidIp4(s) - j = parseInt(s, c, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] == '.': inc(i) - else: invalidIp4(s) - j = parseInt(s, d, i) - if j <= 0: invalidIp4(s) - inc(i, j) - if s[i] != '\0': invalidIp4(s) - result = BiggestInt(a shl 24 or b shl 16 or c shl 8 or d) - -template gaiNim(a, p, h, list: expr): stmt = - block: - var gaiResult = getaddrinfo(a, $p, addr(h), list) - if gaiResult != 0'i32: - when defined(windows): - raiseOSError(osLastError()) - else: - raiseOSError(osLastError(), $gai_strerror(gaiResult)) - -proc bindAddr*(socket: Socket, port = Port(0), address = "") {. - tags: [ReadIOEffect].} = - ## binds an address/port number to a socket. - ## Use address string in dotted decimal form like "a.b.c.d" - ## or leave "" for any address. - - if address == "": - var name: Sockaddr_in - when defined(Windows): - name.sin_family = int16(ord(AF_INET)) - else: - name.sin_family = posix.AF_INET - name.sin_port = sockets.htons(int16(port)) - name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) - if bindSocket(socket.fd, cast[ptr SockAddr](addr(name)), - sizeof(name).SockLen) < 0'i32: - raiseOSError(osLastError()) - else: - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(AF_INET) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(address, port, hints, aiList) - if bindSocket(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc getSockName*(socket: Socket): Port = - ## returns the socket's associated port number. - var name: Sockaddr_in - when defined(Windows): - name.sin_family = int16(ord(AF_INET)) - else: - name.sin_family = posix.AF_INET - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket.fd, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(sockets.ntohs(name.sin_port)) - -template acceptAddrPlain(noClientRet, successRet: expr, - sslImplementation: stmt): stmt {.immediate.} = - assert(client != nil) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) - - if sock == osInvalidSocket: - let err = osLastError() - when defined(windows): - if err.int32 == WSAEINPROGRESS: - client = invalidSocket - address = "" - when noClientRet.int == -1: - return - else: - return noClientRet - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - client = invalidSocket - address = "" - when noClientRet.int == -1: - return - else: - return noClientRet - else: raiseOSError(err) - else: - client.fd = sock - client.isBuffered = server.isBuffered - sslImplementation - # Client socket is set above. - address = $inet_ntoa(sockAddress.sin_addr) - when successRet.int == -1: - return - else: - return successRet - -proc acceptAddr*(server: Socket, client: var Socket, address: var string) {. - tags: [ReadIOEffect].} = - ## Blocks until a connection is being made from a client. When a connection - ## is made sets ``client`` to the client socket and ``address`` to the address - ## of the connecting client. - ## If ``server`` is non-blocking then this function returns immediately, and - ## if there are no connections queued the returned socket will be - ## ``InvalidSocket``. - ## This function will raise EOS if an error occurs. - ## - ## The resulting client will inherit any properties of the server socket. For - ## example: whether the socket is buffered or not. - ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. - ## - ## **Warning:** When using SSL with non-blocking sockets, it is best to use - ## the acceptAddrSSL procedure as this procedure will most likely block. - acceptAddrPlain(-1, -1): - when defined(ssl): - if server.isSSL: - # We must wrap the client sock in a ssl context. - - server.sslContext.wrapSocket(client) - let ret = SSLAccept(client.sslHandle) - while ret <= 0: - let err = SSLGetError(client.sslHandle, ret) - if err != SSL_ERROR_WANT_ACCEPT: - case err - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, - SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("acceptAddrSSL should be used for non-blocking SSL sockets.") - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown error") - -proc setBlocking*(s: Socket, blocking: bool) {.tags: [], gcsafe.} - ## Sets blocking mode on socket - -when defined(ssl): - proc acceptAddrSSL*(server: Socket, client: var Socket, - address: var string): SSLAcceptResult {. - tags: [ReadIOEffect].} = - ## This procedure should only be used for non-blocking **SSL** sockets. - ## It will immediately return with one of the following values: - ## - ## ``AcceptSuccess`` will be returned when a client has been successfully - ## accepted and the handshake has been successfully performed between - ## ``server`` and the newly connected client. - ## - ## ``AcceptNoHandshake`` will be returned when a client has been accepted - ## but no handshake could be performed. This can happen when the client - ## connects but does not yet initiate a handshake. In this case - ## ``acceptAddrSSL`` should be called again with the same parameters. - ## - ## ``AcceptNoClient`` will be returned when no client is currently attempting - ## to connect. - template doHandshake(): stmt = - when defined(ssl): - if server.isSSL: - client.setBlocking(false) - # We must wrap the client sock in a ssl context. - - if not client.isSSL or client.sslHandle == nil: - server.sslContext.wrapSocket(client) - let ret = SSLAccept(client.sslHandle) - while ret <= 0: - let err = SSLGetError(client.sslHandle, ret) - if err != SSL_ERROR_WANT_ACCEPT: - case err - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, - SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - client.sslNoHandshake = true - return AcceptNoHandshake - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown error") - client.sslNoHandshake = false - - if client.isSSL and client.sslNoHandshake: - doHandshake() - return AcceptSuccess - else: - acceptAddrPlain(AcceptNoClient, AcceptSuccess): - doHandshake() - -proc accept*(server: Socket, client: var Socket) {.tags: [ReadIOEffect].} = - ## Equivalent to ``acceptAddr`` but doesn't return the address, only the - ## socket. - ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. - - var addrDummy = "" - acceptAddr(server, client, addrDummy) - -proc acceptAddr*(server: Socket): tuple[client: Socket, address: string] {. - deprecated, tags: [ReadIOEffect].} = - ## Slightly different version of ``acceptAddr``. - ## - ## **Deprecated since version 0.9.0:** Please use the function above. - var client: Socket - new(client) - var address = "" - acceptAddr(server, client, address) - return (client, address) - -proc accept*(server: Socket): Socket {.deprecated, tags: [ReadIOEffect].} = - ## **Deprecated since version 0.9.0:** Please use the function above. - new(result) - var address = "" - acceptAddr(server, result, address) - -proc close*(socket: Socket) = - ## closes a socket. - when defined(windows): - discard winlean.closesocket(socket.fd) - else: - discard posix.close(socket.fd) - # TODO: These values should not be discarded. An EOS should be raised. - # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times - when defined(ssl): - if socket.isSSL: - discard SSLShutdown(socket.sslHandle) - SSLFree(socket.sslHandle) - socket.sslHandle = nil - -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by ``name`` matches the s_name member - ## and the protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when defined(Windows): - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by ``port`` matches the s_port member and the - ## protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when defined(Windows): - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when defined(windows): - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(sockets.AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when defined(windows): - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when defined(Windows): - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when defined(windows): - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getSockOptInt*(socket: Socket, level, optname: int): int {. - tags: [ReadIOEffect].} = - ## getsockopt for integer options. - var res: cint - var size = sizeof(res).SockLen - if getsockopt(socket.fd, cint(level), cint(optname), - addr(res), addr(size)) < 0'i32: - raiseOSError(osLastError()) - result = int(res) - -proc setSockOptInt*(socket: Socket, level, optname, optval: int) {. - tags: [WriteIOEffect].} = - ## setsockopt for integer options. - var value = cint(optval) - if setsockopt(socket.fd, cint(level), cint(optname), addr(value), - sizeof(value).SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc toCInt(opt: SOBool): cint = - case opt - of OptAcceptConn: SO_ACCEPTCONN - of OptBroadcast: SO_BROADCAST - of OptDebug: SO_DEBUG - of OptDontRoute: SO_DONTROUTE - of OptKeepAlive: SO_KEEPALIVE - of OptOOBInline: SO_OOBINLINE - of OptReuseAddr: SO_REUSEADDR - -proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. - tags: [ReadIOEffect].} = - ## Retrieves option ``opt`` as a boolean value. - var res: cint - var size = sizeof(res).SockLen - if getsockopt(socket.fd, cint(level), toCInt(opt), - addr(res), addr(size)) < 0'i32: - raiseOSError(osLastError()) - result = res != 0 - -proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {. - tags: [WriteIOEffect].} = - ## Sets option ``opt`` to a boolean value specified by ``value``. - var valuei = cint(if value: 1 else: 0) - if setsockopt(socket.fd, cint(level), toCInt(opt), addr(valuei), - sizeof(valuei).SockLen) < 0'i32: - raiseOSError(osLastError()) - -proc connect*(socket: Socket, address: string, port = Port(0), - af: Domain = AF_INET) {.tags: [ReadIOEffect].} = - ## Connects socket to ``address``:``port``. ``Address`` can be an IP address or a - ## host name. If ``address`` is a host name, this function will try each IP - ## of that host name. ``htons`` is already performed on ``port`` so you must - ## not do it. - ## - ## If ``socket`` is an SSL socket a handshake will be automatically performed. - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(address, port, hints, aiList) - # try all possibilities: - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - if connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) == 0'i32: - success = true - break - else: lastError = osLastError() - it = it.ai_next - - freeaddrinfo(aiList) - if not success: raiseOSError(lastError) - - when defined(ssl): - if socket.isSSL: - let ret = SSLConnect(socket.sslHandle) - if ret <= 0: - let err = SSLGetError(socket.sslHandle, ret) - case err - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_CONNECT, - SSL_ERROR_WANT_ACCEPT: - raiseSslError("The operation did not complete. Perhaps you should use connectAsync?") - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown error") - - when false: - var s: TSockAddrIn - s.sin_addr.s_addr = inet_addr(address) - s.sin_port = sockets.htons(int16(port)) - when defined(windows): - s.sin_family = toU16(ord(af)) - else: - case af - of AF_UNIX: s.sin_family = posix.AF_UNIX - of AF_INET: s.sin_family = posix.AF_INET - of AF_INET6: s.sin_family = posix.AF_INET6 - else: nil - if connect(socket.fd, cast[ptr TSockAddr](addr(s)), sizeof(s).cint) < 0'i32: - OSError() - -proc connectAsync*(socket: Socket, name: string, port = Port(0), - af: Domain = AF_INET) {.tags: [ReadIOEffect].} = - ## A variant of ``connect`` for non-blocking sockets. - ## - ## This procedure will immediately return, it will not block until a connection - ## is made. It is up to the caller to make sure the connection has been established - ## by checking (using ``select``) whether the socket is writeable. - ## - ## **Note**: For SSL sockets, the ``handshake`` procedure must be called - ## whenever the socket successfully connects to a server. - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(name, port, hints, aiList) - # try all possibilities: - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - var ret = connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) - if ret == 0'i32: - success = true - break - else: - lastError = osLastError() - when defined(windows): - # Windows EINTR doesn't behave same as POSIX. - if lastError.int32 == WSAEWOULDBLOCK: - success = true - break - else: - if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: - success = true - break - - it = it.ai_next - - freeaddrinfo(aiList) - if not success: raiseOSError(lastError) - when defined(ssl): - if socket.isSSL: - socket.sslNoHandshake = true - -when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = - ## This proc needs to be called on a socket after it connects. This is - ## only applicable when using ``connectAsync``. - ## This proc performs the SSL handshake. - ## - ## Returns ``False`` whenever the socket is not yet ready for a handshake, - ## ``True`` whenever handshake completed successfully. - ## - ## A ESSL error is raised on any other errors. - result = true - if socket.isSSL: - var ret = SSLConnect(socket.sslHandle) - if ret <= 0: - var errret = SSLGetError(socket.sslHandle, ret) - case errret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT, - SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE: - return false - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: - raiseSslError("Unknown Error") - socket.sslNoHandshake = false - else: - raiseSslError("Socket is not an SSL socket.") - - proc gotHandshake*(socket: Socket): bool = - ## Determines whether a handshake has occurred between a client (``socket``) - ## and the server that ``socket`` is connected to. - ## - ## Throws ESSL if ``socket`` is not an SSL socket. - if socket.isSSL: - return not socket.sslNoHandshake - else: - raiseSslError("Socket is not an SSL socket.") - -proc timeValFromMilliseconds(timeout = 500): Timeval = - if timeout != -1: - var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - -proc createFdSet(fd: var TFdSet, s: seq[Socket], m: var int) = - FD_ZERO(fd) - for i in items(s): - m = max(m, int(i.fd)) - FD_SET(i.fd, fd) - -proc pruneSocketSet(s: var seq[Socket], fd: var TFdSet) = - var i = 0 - var L = s.len - while i < L: - if FD_ISSET(s[i].fd, fd) == 0'i32: - # not set. - s[i] = s[L-1] - dec(L) - else: - inc(i) - setLen(s, L) - -proc hasDataBuffered*(s: Socket): bool = - ## Determines whether a socket has data buffered. - result = false - if s.isBuffered: - result = s.bufLen > 0 and s.currPos != s.bufLen - - when defined(ssl): - if s.isSSL and not result: - result = s.sslHasPeekChar - -proc checkBuffer(readfds: var seq[Socket]): int = - ## Checks the buffer of each socket in ``readfds`` to see whether there is data. - ## Removes the sockets from ``readfds`` and returns the count of removed sockets. - var res: seq[Socket] = @[] - result = 0 - for s in readfds: - if hasDataBuffered(s): - inc(result) - res.add(s) - if result > 0: - readfds = res - -proc select*(readfds, writefds, exceptfds: var seq[Socket], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## Traditional select function. This function will return the number of - ## sockets that are ready to be read from, written to, or which have errors. - ## If there are none; 0 is returned. - ## ``Timeout`` is in milliseconds and -1 can be specified for no timeout. - ## - ## Sockets which are **not** ready for reading, writing or which don't have - ## errors waiting on them are removed from the ``readfds``, ``writefds``, - ## ``exceptfds`` sequences respectively. - let buffersFilled = checkBuffer(readfds) - if buffersFilled > 0: - return buffersFilled - - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd, wr, ex: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - createFdSet((wr), writefds, m) - createFdSet((ex), exceptfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), nil)) - - pruneSocketSet(readfds, (rd)) - pruneSocketSet(writefds, (wr)) - pruneSocketSet(exceptfds, (ex)) - -proc select*(readfds, writefds: var seq[Socket], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## Variant of select with only a read and write list. - let buffersFilled = checkBuffer(readfds) - if buffersFilled > 0: - return buffersFilled - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd, wr: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - createFdSet((wr), writefds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), addr(wr), nil, nil)) - - pruneSocketSet(readfds, (rd)) - pruneSocketSet(writefds, (wr)) - -proc selectWrite*(writefds: var seq[Socket], - timeout = 500): int {.tags: [ReadIOEffect].} = - ## When a socket in ``writefds`` is ready to be written to then a non-zero - ## value will be returned specifying the count of the sockets which can be - ## written to. The sockets which **cannot** be written to will also be removed - ## from ``writefds``. - ## - ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for - ## an unlimited time. - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var wr: TFdSet - var m = 0 - createFdSet((wr), writefds, m) - - if timeout != -1: - result = int(select(cint(m+1), nil, addr(wr), nil, addr(tv))) - else: - result = int(select(cint(m+1), nil, addr(wr), nil, nil)) - - pruneSocketSet(writefds, (wr)) - -proc select*(readfds: var seq[Socket], timeout = 500): int = - ## variant of select with a read list only - let buffersFilled = checkBuffer(readfds) - if buffersFilled > 0: - return buffersFilled - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) - - var rd: TFdSet - var m = 0 - createFdSet((rd), readfds, m) - - if timeout != -1: - result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) - else: - result = int(select(cint(m+1), addr(rd), nil, nil, nil)) - - pruneSocketSet(readfds, (rd)) - -proc readIntoBuf(socket: Socket, flags: int32): int = - result = 0 - when defined(ssl): - if socket.isSSL: - result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - if result <= 0: - socket.bufLen = 0 - socket.currPos = 0 - return result - socket.bufLen = result - socket.currPos = 0 - -template retRead(flags, readBytes: int) {.dirty.} = - let res = socket.readIntoBuf(flags.int32) - if res <= 0: - if readBytes > 0: - return readBytes - else: - return res - -proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect].} = - ## Receives data from a socket. - ## - ## **Note**: This is a low-level function, you may be interested in the higher - ## level versions of this function which are also named ``recv``. - if size == 0: return - if socket.isBuffered: - if socket.bufLen == 0: - retRead(0'i32, 0) - - var read = 0 - while read < size: - if socket.currPos >= socket.bufLen: - retRead(0'i32, read) - - let chunk = min(socket.bufLen-socket.currPos, size-read) - var d = cast[cstring](data) - copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk) - read.inc(chunk) - socket.currPos.inc(chunk) - - result = read - else: - when defined(ssl): - if socket.isSSL: - if socket.sslHasPeekChar: - copyMem(data, addr(socket.sslPeekChar), 1) - socket.sslHasPeekChar = false - if size-1 > 0: - var d = cast[cstring](data) - result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 - else: - result = 1 - else: - result = SSLRead(socket.sslHandle, data, size) - else: - result = recv(socket.fd, data, size.cint, 0'i32) - else: - result = recv(socket.fd, data, size.cint, 0'i32) - -proc waitFor(socket: Socket, waited: var float, timeout, size: int, - funcName: string): int {.tags: [TimeEffect].} = - ## determines the amount of characters that can be read. Result will never - ## be larger than ``size``. For unbuffered sockets this will be ``1``. - ## For buffered sockets it can be as big as ``BufferSize``. - ## - ## If this function does not determine that there is data on the socket - ## within ``timeout`` ms, an ETimeout error will be raised. - result = 1 - if size <= 0: assert false - if timeout == -1: return size - if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos: - result = socket.bufLen - socket.currPos - result = min(result, size) - else: - if timeout - int(waited * 1000.0) < 1: - raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - - when defined(ssl): - if socket.isSSL: - if socket.hasDataBuffered: - # sslPeekChar is present. - return 1 - let sslPending = SSLPending(socket.sslHandle) - if sslPending != 0: - return sslPending - - var s = @[socket] - var startTime = epochTime() - let selRet = select(s, timeout - int(waited * 1000.0)) - if selRet < 0: raiseOSError(osLastError()) - if selRet != 1: - raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - waited += (epochTime() - startTime) - -proc recv*(socket: Socket, data: pointer, size: int, timeout: int): int {. - tags: [ReadIOEffect, TimeEffect].} = - ## overload with a ``timeout`` parameter in milliseconds. - var waited = 0.0 # number of seconds already waited - - var read = 0 - while read < size: - let avail = waitFor(socket, waited, timeout, size-read, "recv") - var d = cast[cstring](data) - result = recv(socket, addr(d[read]), avail) - if result == 0: break - if result < 0: - return result - inc(read, result) - - result = read - -proc recv*(socket: Socket, data: var string, size: int, timeout = -1): int = - ## Higher-level version of ``recv``. - ## - ## When 0 is returned the socket's connection has been closed. - ## - ## This function will throw an EOS exception when an error occurs. A value - ## lower than 0 is never returned. - ## - ## A timeout may be specified in milliseconds, if enough data is not received - ## within the time specified an ETimeout exception will be raised. - ## - ## **Note**: ``data`` must be initialised. - data.setLen(size) - result = recv(socket, cstring(data), size, timeout) - if result < 0: - data.setLen(0) - socket.raiseSocketError(result) - data.setLen(result) - -proc recvAsync*(socket: Socket, data: var string, size: int): int = - ## Async version of ``recv``. - ## - ## When socket is non-blocking and no data is available on the socket, - ## ``-1`` will be returned and ``data`` will be ``""``. - ## - ## **Note**: ``data`` must be initialised. - data.setLen(size) - result = recv(socket, cstring(data), size) - if result < 0: - data.setLen(0) - socket.raiseSocketError(async = true) - result = -1 - data.setLen(result) - -proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = - if socket.isBuffered: - result = 1 - if socket.bufLen == 0 or socket.currPos > socket.bufLen-1: - var res = socket.readIntoBuf(0'i32) - if res <= 0: - result = res - - c = socket.buffer[socket.currPos] - else: - when defined(ssl): - if socket.isSSL: - if not socket.sslHasPeekChar: - result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) - socket.sslHasPeekChar = true - - c = socket.sslPeekChar - return - result = recv(socket.fd, addr(c), 1, MSG_PEEK) - -proc recvLine*(socket: Socket, line: var TaintedString, timeout = -1): bool {. - tags: [ReadIOEffect, TimeEffect], deprecated.} = - ## Receive a line of data from ``socket``. - ## - ## If a full line is received ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is received then ``line`` - ## will be set to it. - ## - ## ``True`` is returned if data is available. ``False`` suggests an - ## error, EOS exceptions are not raised and ``False`` is simply returned - ## instead. - ## - ## If the socket is disconnected, ``line`` will be set to ``""`` and ``True`` - ## will be returned. - ## - ## A timeout can be specified in milliseconds, if data is not received within - ## the specified time an ETimeout exception will be raised. - ## - ## **Deprecated since version 0.9.2**: This function has been deprecated in - ## favour of readLine. - - template addNLIfEmpty(): stmt = - if line.len == 0: - line.add("\c\L") - - var waited = 0.0 - - setLen(line.string, 0) - while true: - var c: char - discard waitFor(socket, waited, timeout, 1, "recvLine") - var n = recv(socket, addr(c), 1) - if n < 0: return - elif n == 0: return true - if c == '\r': - discard waitFor(socket, waited, timeout, 1, "recvLine") - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: return false - addNLIfEmpty() - return true - elif c == '\L': - addNLIfEmpty() - return true - add(line.string, c) - -proc readLine*(socket: Socket, line: var TaintedString, timeout = -1) {. - tags: [ReadIOEffect, TimeEffect].} = - ## Reads a line of data from ``socket``. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## An EOS exception will be raised in the case of a socket error. - ## - ## A timeout can be specified in milliseconds, if data is not received within - ## the specified time an ETimeout exception will be raised. - - template addNLIfEmpty(): stmt = - if line.len == 0: - line.add("\c\L") - - var waited = 0.0 - - setLen(line.string, 0) - while true: - var c: char - discard waitFor(socket, waited, timeout, 1, "readLine") - var n = recv(socket, addr(c), 1) - if n < 0: socket.raiseSocketError() - elif n == 0: return - if c == '\r': - discard waitFor(socket, waited, timeout, 1, "readLine") - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: socket.raiseSocketError() - addNLIfEmpty() - return - elif c == '\L': - addNLIfEmpty() - return - add(line.string, c) - -proc recvLineAsync*(socket: Socket, - line: var TaintedString): RecvLineResult {.tags: [ReadIOEffect], deprecated.} = - ## Similar to ``recvLine`` but designed for non-blocking sockets. - ## - ## The values of the returned enum should be pretty self explanatory: - ## - ## * If a full line has been retrieved; ``RecvFullLine`` is returned. - ## * If some data has been retrieved; ``RecvPartialLine`` is returned. - ## * If the socket has been disconnected; ``RecvDisconnected`` is returned. - ## * If call to ``recv`` failed; ``RecvFail`` is returned. - ## - ## **Deprecated since version 0.9.2**: This function has been deprecated in - ## favour of readLineAsync. - - setLen(line.string, 0) - while true: - var c: char - var n = recv(socket, addr(c), 1) - if n < 0: - return (if line.len == 0: RecvFail else: RecvPartialLine) - elif n == 0: - return (if line.len == 0: RecvDisconnected else: RecvPartialLine) - if c == '\r': - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: - return (if line.len == 0: RecvFail else: RecvPartialLine) - return RecvFullLine - elif c == '\L': return RecvFullLine - add(line.string, c) - -proc readLineAsync*(socket: Socket, - line: var TaintedString): ReadLineResult {.tags: [ReadIOEffect].} = - ## Similar to ``recvLine`` but designed for non-blocking sockets. - ## - ## The values of the returned enum should be pretty self explanatory: - ## - ## * If a full line has been retrieved; ``ReadFullLine`` is returned. - ## * If some data has been retrieved; ``ReadPartialLine`` is returned. - ## * If the socket has been disconnected; ``ReadDisconnected`` is returned. - ## * If no data could be retrieved; ``ReadNone`` is returned. - ## * If call to ``recv`` failed; **an EOS exception is raised.** - setLen(line.string, 0) - - template errorOrNone = - socket.raiseSocketError(async = true) - return ReadNone - - while true: - var c: char - var n = recv(socket, addr(c), 1) - #echo(n) - if n < 0: - if line.len == 0: errorOrNone else: return ReadPartialLine - elif n == 0: - return (if line.len == 0: ReadDisconnected else: ReadPartialLine) - if c == '\r': - n = peekChar(socket, c) - if n > 0 and c == '\L': - discard recv(socket, addr(c), 1) - elif n <= 0: - if line.len == 0: errorOrNone else: return ReadPartialLine - return ReadFullLine - elif c == '\L': return ReadFullLine - add(line.string, c) - -proc recv*(socket: Socket): TaintedString {.tags: [ReadIOEffect], deprecated.} = - ## receives all the available data from the socket. - ## Socket errors will result in an ``EOS`` error. - ## If socket is not a connectionless socket and socket is not connected - ## ``""`` will be returned. - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - const bufSize = 4000 - result = newStringOfCap(bufSize).TaintedString - var pos = 0 - while true: - var bytesRead = recv(socket, addr(string(result)[pos]), bufSize-1) - if bytesRead == -1: raiseOSError(osLastError()) - setLen(result.string, pos + bytesRead) - if bytesRead != bufSize-1: break - # increase capacity: - setLen(result.string, result.string.len + bufSize) - inc(pos, bytesRead) - when false: - var buf = newString(bufSize) - result = TaintedString"" - while true: - var bytesRead = recv(socket, cstring(buf), bufSize-1) - # Error - if bytesRead == -1: OSError(osLastError()) - - buf[bytesRead] = '\0' # might not be necessary - setLen(buf, bytesRead) - add(result.string, buf) - if bytesRead != bufSize-1: break - -{.push warning[deprecated]: off.} -proc recvTimeout*(socket: Socket, timeout: int): TaintedString {. - tags: [ReadIOEffect], deprecated.} = - ## overloaded variant to support a ``timeout`` parameter, the ``timeout`` - ## parameter specifies the amount of milliseconds to wait for data on the - ## socket. - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - if socket.bufLen == 0: - var s = @[socket] - if s.select(timeout) != 1: - raise newException(TimeoutError, "Call to recv() timed out.") - - return socket.recv -{.pop.} - -proc recvAsync*(socket: Socket, s: var TaintedString): bool {. - tags: [ReadIOEffect], deprecated.} = - ## receives all the data from a non-blocking socket. If socket is non-blocking - ## and there are no messages available, `False` will be returned. - ## Other socket errors will result in an ``EOS`` error. - ## If socket is not a connectionless socket and socket is not connected - ## ``s`` will be set to ``""``. - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - const bufSize = 1000 - # ensure bufSize capacity: - setLen(s.string, bufSize) - setLen(s.string, 0) - var pos = 0 - while true: - var bytesRead = recv(socket, addr(string(s)[pos]), bufSize-1) - when defined(ssl): - if socket.isSSL: - if bytesRead <= 0: - var ret = SSLGetError(socket.sslHandle, bytesRead.cint) - case ret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("Unexpected error occurred.") # This should just not happen. - of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - return false - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: raiseSslError("Unknown Error") - - if bytesRead == -1 and not (when defined(ssl): socket.isSSL else: false): - let err = osLastError() - when defined(windows): - if err.int32 == WSAEWOULDBLOCK: - return false - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - return false - else: raiseOSError(err) - - setLen(s.string, pos + bytesRead) - if bytesRead != bufSize-1: break - # increase capacity: - setLen(s.string, s.string.len + bufSize) - inc(pos, bytesRead) - result = true - -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. - tags: [ReadIOEffect].} = - ## Receives data from ``socket``. This function should normally be used with - ## connection-less sockets (UDP sockets). - ## - ## If an error occurs the return value will be ``-1``. Otherwise the return - ## value will be the length of data received. - ## - ## **Warning:** This function does not yet have a buffered implementation, - ## so when ``socket`` is buffered the non-buffered implementation will be - ## used. Therefore if ``socket`` contains something in its buffer this - ## function will make no effort to return it. - - # TODO: Buffered sockets - data.setLen(length) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - - if result != -1: - data.setLen(result) - address = $inet_ntoa(sockAddress.sin_addr) - port = ntohs(sockAddress.sin_port).Port - -proc recvFromAsync*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, - flags = 0'i32): bool {.tags: [ReadIOEffect].} = - ## Variant of ``recvFrom`` for non-blocking sockets. Unlike ``recvFrom``, - ## this function will raise an EOS error whenever a socket error occurs. - ## - ## If there is no data to be read from the socket ``False`` will be returned. - result = true - var callRes = recvFrom(socket, data, length, address, port, flags) - if callRes < 0: - let err = osLastError() - when defined(windows): - if err.int32 == WSAEWOULDBLOCK: - return false - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - return false - else: raiseOSError(err) - -proc skip*(socket: Socket) {.tags: [ReadIOEffect], deprecated.} = - ## skips all the data that is pending for the socket - ## - ## **Deprecated since version 0.9.2**: This function is not safe for use. - const bufSize = 1000 - var buf = alloc(bufSize) - while recv(socket, buf, bufSize) == bufSize: discard - dealloc(buf) - -proc skip*(socket: Socket, size: int, timeout = -1) = - ## Skips ``size`` amount of bytes. - ## - ## An optional timeout can be specified in milliseconds, if skipping the - ## bytes takes longer than specified an ETimeout exception will be raised. - ## - ## Returns the number of skipped bytes. - var waited = 0.0 - var dummy = alloc(size) - var bytesSkipped = 0 - while bytesSkipped != size: - let avail = waitFor(socket, waited, timeout, size-bytesSkipped, "skip") - bytesSkipped += recv(socket, dummy, avail) - dealloc(dummy) - -proc send*(socket: Socket, data: pointer, size: int): int {. - tags: [WriteIOEffect].} = - ## sends data to a socket. - when defined(ssl): - if socket.isSSL: - return SSLWrite(socket.sslHandle, cast[cstring](data), size) - - when defined(windows) or defined(macosx): - result = send(socket.fd, data, size.cint, 0'i32) - else: - when defined(solaris): - const MSG_NOSIGNAL = 0 - result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) - -proc send*(socket: Socket, data: string) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - if socket.nonblocking: - raise newException(ValueError, "This function cannot be used on non-blocking sockets.") - let sent = send(socket, cstring(data), data.len) - if sent < 0: - when defined(ssl): - if socket.isSSL: - raiseSslError() - - raiseOSError(osLastError()) - - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") - -proc sendAsync*(socket: Socket, data: string): int {.tags: [WriteIOEffect].} = - ## sends data to a non-blocking socket. - ## Returns ``0`` if no data could be sent, if data has been sent - ## returns the amount of bytes of ``data`` that was successfully sent. This - ## number may not always be the length of ``data`` but typically is. - ## - ## An EOS (or ESSL if socket is an SSL socket) exception is raised if an error - ## occurs. - result = send(socket, cstring(data), data.len) - when defined(ssl): - if socket.isSSL: - if result <= 0: - let ret = SSLGetError(socket.sslHandle, result.cint) - case ret - of SSL_ERROR_ZERO_RETURN: - raiseSslError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: - raiseSslError("Unexpected error occurred.") # This should just not happen. - of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - return 0 - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSslError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSslError() - else: raiseSslError("Unknown Error") - else: - return - if result == -1: - let err = osLastError() - when defined(windows): - if err.int32 == WSAEINPROGRESS: - return 0 - else: raiseOSError(err) - else: - if err.int32 == EAGAIN or err.int32 == EWOULDBLOCK: - return 0 - else: raiseOSError(err) - - -proc trySend*(socket: Socket, data: string): bool {.tags: [WriteIOEffect].} = - ## safe alternative to ``send``. Does not raise an EOS when an error occurs, - ## and instead returns ``false`` on failure. - result = send(socket, cstring(data), data.len) == data.len - -proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, - size: int, af: Domain = AF_INET, flags = 0'i32): int {. - tags: [WriteIOEffect].} = - ## low-level sendTo proc. This proc sends ``data`` to the specified ``address``, - ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. - ## - ## **Note:** This proc is not available for SSL sockets. - var hints: AddrInfo - var aiList: ptr AddrInfo = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(SOCK_STREAM) - hints.ai_protocol = toInt(IPPROTO_TCP) - gaiNim(address, port, hints, aiList) - - # try all possibilities: - var success = false - var it = aiList - while it != nil: - result = sendto(socket.fd, data, size.cint, flags.cint, it.ai_addr, - it.ai_addrlen.SockLen) - if result != -1'i32: - success = true - break - it = it.ai_next - - freeaddrinfo(aiList) - -proc sendTo*(socket: Socket, address: string, port: Port, - data: string): int {.tags: [WriteIOEffect].} = - ## Friendlier version of the low-level ``sendTo``. - result = socket.sendTo(address, port, cstring(data), data.len) - -when defined(Windows): - const - IOCPARM_MASK = 127 - IOC_IN = int(-2147483648) - FIONBIO = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or - (102 shl 8) or 126 - - proc ioctlsocket(s: SocketHandle, cmd: clong, - argptr: ptr clong): cint {. - stdcall, importc:"ioctlsocket", dynlib: "ws2_32.dll".} - -proc setBlocking(s: Socket, blocking: bool) = - when defined(Windows): - var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking - if ioctlsocket(s.fd, FIONBIO, addr(mode)) == -1: - raiseOSError(osLastError()) - else: # BSD sockets - var x: int = fcntl(s.fd, F_GETFL, 0) - if x == -1: - raiseOSError(osLastError()) - else: - var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK - if fcntl(s.fd, F_SETFL, mode) == -1: - raiseOSError(osLastError()) - s.nonblocking = not blocking - -discard """ proc setReuseAddr*(s: Socket) = - var blah: int = 1 - var mode = SO_REUSEADDR - if setsockopt(s.fd, SOL_SOCKET, mode, addr blah, TSOcklen(sizeof(int))) == -1: - raiseOSError(osLastError()) """ - -proc connect*(socket: Socket, address: string, port = Port(0), timeout: int, - af: Domain = AF_INET) {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Connects to server as specified by ``address`` on port specified by ``port``. - ## - ## The ``timeout`` paremeter specifies the time in milliseconds to allow for - ## the connection to the server to be made. - let originalStatus = not socket.nonblocking - socket.setBlocking(false) - - socket.connectAsync(address, port, af) - var s: seq[Socket] = @[socket] - if selectWrite(s, timeout) != 1: - raise newException(TimeoutError, "Call to 'connect' timed out.") - else: - when defined(ssl): - if socket.isSSL: - socket.setBlocking(true) - doAssert socket.handshake() - socket.setBlocking(originalStatus) - -proc isSSL*(socket: Socket): bool = return socket.isSSL - ## Determines whether ``socket`` is a SSL socket. - -proc getFD*(socket: Socket): SocketHandle = return socket.fd - ## Returns the socket's file descriptor - -proc isBlocking*(socket: Socket): bool = not socket.nonblocking - ## Determines whether ``socket`` is blocking. - -when defined(Windows): - var wsa: WSAData - if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) - - diff --git a/lib/deprecated/pure/sums.nim b/lib/deprecated/pure/sums.nim new file mode 100644 index 000000000..a6ce1b85d --- /dev/null +++ b/lib/deprecated/pure/sums.nim @@ -0,0 +1,80 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 b3liever +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## Accurate summation functions. + +{.deprecated: "use the nimble package `sums` instead.".} + +runnableExamples: + import std/math + + template `~=`(x, y: float): bool = abs(x - y) < 1e-4 + + let + n = 1_000_000 + first = 1e10 + small = 0.1 + var data = @[first] + for _ in 1 .. n: + data.add(small) + + let result = first + small * n.float + + doAssert abs(sum(data) - result) > 0.3 + doAssert sumKbn(data) ~= result + doAssert sumPairs(data) ~= result + +## See also +## ======== +## * `math module <math.html>`_ for a standard `sum proc <math.html#sum,openArray[T]>`_ + +func sumKbn*[T](x: openArray[T]): T = + ## Kahan-Babuška-Neumaier summation: O(1) error growth, at the expense + ## of a considerable increase in computational cost. + ## + ## See: + ## * https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements + if len(x) == 0: return + var sum = x[0] + var c = T(0) + for i in 1 ..< len(x): + let xi = x[i] + let t = sum + xi + if abs(sum) >= abs(xi): + c += (sum - t) + xi + else: + c += (xi - t) + sum + sum = t + result = sum + c + +func sumPairwise[T](x: openArray[T], i0, n: int): T = + if n < 128: + result = x[i0] + for i in i0 + 1 ..< i0 + n: + result += x[i] + else: + let n2 = n div 2 + result = sumPairwise(x, i0, n2) + sumPairwise(x, i0 + n2, n - n2) + +func sumPairs*[T](x: openArray[T]): T = + ## Pairwise (cascade) summation of `x[i0:i0+n-1]`, with O(log n) error growth + ## (vs O(n) for a simple loop) with negligible performance cost if + ## the base case is large enough. + ## + ## See, e.g.: + ## * https://en.wikipedia.org/wiki/Pairwise_summation + ## * Higham, Nicholas J. (1993), "The accuracy of floating point + ## summation", SIAM Journal on Scientific Computing 14 (4): 783–799. + ## + ## In fact, the root-mean-square error growth, assuming random roundoff + ## errors, is only O(sqrt(log n)), which is nearly indistinguishable from O(1) + ## in practice. See: + ## * Manfred Tasche and Hansmartin Zeuner, Handbook of + ## Analytic-Computational Methods in Applied Mathematics (2000). + let n = len(x) + if n == 0: T(0) else: sumPairwise(x, 0, n) diff --git a/lib/deps.txt b/lib/deps.txt new file mode 100644 index 000000000..2a44b5fe2 --- /dev/null +++ b/lib/deps.txt @@ -0,0 +1,14 @@ +# Dragonbox.nim and Schubfach.nim + +"Dragonbox" is Nim's "float64 to string" algorithm. +"Schubfach" is Nim's "float32 to string" algorithm. These are based on + +https://github.com/abolz/Drachennest/blob/master/src/dragonbox.cc +https://github.com/abolz/Drachennest/blob/master/src/schubfach_32.cc + +commit e6714a39ad331b4489d0b6aaf3968635bff4eb5e + +The `.cc` files were translated by c2nim via `--cpp --keepBodies --nep1` +and then modified to remove the unsafe code. + +We used c2nim as of commit f0469c909d9e2e28d59687e394bf5ac862f561b6. diff --git a/lib/experimental/diff.nim b/lib/experimental/diff.nim new file mode 100644 index 000000000..669e9f613 --- /dev/null +++ b/lib/experimental/diff.nim @@ -0,0 +1,339 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements an algorithm to compute the +## `diff`:idx: between two sequences of lines. +## +## - To learn more see `Diff on Wikipedia. <http://wikipedia.org/wiki/Diff>`_ + +runnableExamples: + assert diffInt( + [0, 1, 2, 3, 4, 5, 6, 7, 8], + [-1, 1, 2, 3, 4, 5, 666, 7, 42]) == + @[Item(startA: 0, startB: 0, deletedA: 1, insertedB: 1), + Item(startA: 6, startB: 6, deletedA: 1, insertedB: 1), + Item(startA: 8, startB: 8, deletedA: 1, insertedB: 1)] + +runnableExamples: + # 2 samples of text (from "The Call of Cthulhu" by Lovecraft) + let txt0 = """ +abc +def ghi +jkl2""" + let txt1 = """ +bacx +abc +def ghi +jkl""" + assert diffText(txt0, txt1) == + @[Item(startA: 0, startB: 0, deletedA: 0, insertedB: 1), + Item(startA: 2, startB: 3, deletedA: 1, insertedB: 1)] + +# code owner: Arne Döring +# +# This is based on C# code written by Matthias Hertel, http://www.mathertel.de +# +# This Class implements the Difference Algorithm published in +# "An O(ND) Difference Algorithm and its Variations" by Eugene Myers +# Algorithmica Vol. 1 No. 2, 1986, p 251. + +import std/[tables, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions + +type + Item* = object ## An Item in the list of differences. + startA*: int ## Start Line number in Data A. + startB*: int ## Start Line number in Data B. + deletedA*: int ## Number of changes in Data A. + insertedB*: int ## Number of changes in Data B. + + DiffData = object ## Data on one input file being compared. + data: seq[int] ## Buffer of numbers that will be compared. + modified: seq[bool] ## Array of booleans that flag for modified + ## data. This is the result of the diff. + ## This means deletedA in the first Data or + ## inserted in the second Data. + + Smsrd = object + x, y: int + +# template to avoid a seq copy. Required until `sink` parameters are ready. +template newDiffData(initData: seq[int]; L: int): DiffData = + DiffData( + data: initData, + modified: newSeq[bool](L + 2) + ) + +proc len(d: DiffData): int {.inline.} = d.data.len + +proc diffCodes(aText: string; h: var Table[string, int]): DiffData = + ## This function converts all textlines of the text into unique numbers for every unique textline + ## so further work can work only with simple numbers. + ## `aText` the input text + ## `h` This extern initialized hashtable is used for storing all ever used textlines. + ## `trimSpace` ignore leading and trailing space characters + ## Returns a array of integers. + var lastUsedCode = h.len + result.data = newSeq[int]() + for s in aText.splitLines: + if h.contains s: + result.data.add h[s] + else: + inc lastUsedCode + h[s] = lastUsedCode + result.data.add lastUsedCode + result.modified = newSeq[bool](result.data.len + 2) + +proc optimize(data: var DiffData) = + ## If a sequence of modified lines starts with a line that contains the same content + ## as the line that appends the changes, the difference sequence is modified so that the + ## appended line and not the starting line is marked as modified. + ## This leads to more readable diff sequences when comparing text files. + var startPos = 0 + while startPos < data.len: + while startPos < data.len and not data.modified[startPos]: + inc startPos + var endPos = startPos + while endPos < data.len and data.modified[endPos]: + inc endPos + + if endPos < data.len and data.data[startPos] == data.data[endPos]: + data.modified[startPos] = false + data.modified[endPos] = true + else: + startPos = endPos + +proc sms(dataA: var DiffData; lowerA, upperA: int; dataB: DiffData; lowerB, upperB: int; + downVector, upVector: var openArray[int]): Smsrd = + ## This is the algorithm to find the Shortest Middle Snake (sms). + ## `dataA` sequence A + ## `lowerA` lower bound of the actual range in dataA + ## `upperA` upper bound of the actual range in dataA (exclusive) + ## `dataB` sequence B + ## `lowerB` lower bound of the actual range in dataB + ## `upperB` upper bound of the actual range in dataB (exclusive) + ## `downVector` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons. + ## `upVector` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons. + ## Returns a MiddleSnakeData record containing x,y and u,v. + + let max = dataA.len + dataB.len + 1 + + let downK = lowerA - lowerB # the k-line to start the forward search + let upK = upperA - upperB # the k-line to start the reverse search + + let delta = (upperA - lowerA) - (upperB - lowerB) + let oddDelta = (delta and 1) != 0 + + # The vectors in the publication accepts negative indexes. the vectors implemented here are 0-based + # and are access using a specific offset: upOffset upVector and downOffset for downVector + let downOffset = max - downK + let upOffset = max - upK + + let maxD = ((upperA - lowerA + upperB - lowerB) div 2) + 1 + + downVector[downOffset + downK + 1] = lowerA + upVector[upOffset + upK - 1] = upperA + + for D in 0 .. maxD: + # Extend the forward path. + for k in countup(downK - D, downK + D, 2): + # find the only or better starting point + var x: int + if k == downK - D: + x = downVector[downOffset + k + 1] # down + else: + x = downVector[downOffset + k - 1] + 1 # a step to the right + if k < downK + D and downVector[downOffset + k + 1] >= x: + x = downVector[downOffset + k + 1] # down + + var y = x - k + + # find the end of the furthest reaching forward D-path in diagonal k. + while x < upperA and y < upperB and dataA.data[x] == dataB.data[y]: + inc x + inc y + + downVector[downOffset + k] = x + + # overlap ? + if oddDelta and upK - D < k and k < upK + D: + if upVector[upOffset + k] <= downVector[downOffset + k]: + return Smsrd(x: downVector[downOffset + k], + y: downVector[downOffset + k] - k) + + # Extend the reverse path. + for k in countup(upK - D, upK + D, 2): + # find the only or better starting point + var x: int + if k == upK + D: + x = upVector[upOffset + k - 1] # up + else: + x = upVector[upOffset + k + 1] - 1 # left + if k > upK - D and upVector[upOffset + k - 1] < x: + x = upVector[upOffset + k - 1] # up + + var y = x - k + while x > lowerA and y > lowerB and dataA.data[x - 1] == dataB.data[y - 1]: + dec x + dec y + + upVector[upOffset + k] = x + + # overlap ? + if not oddDelta and downK-D <= k and k <= downK+D: + if upVector[upOffset + k] <= downVector[downOffset + k]: + return Smsrd(x: downVector[downOffset + k], + y: downVector[downOffset + k] - k) + + assert false, "the algorithm should never come here." + +proc lcs(dataA: var DiffData; lowerA, upperA: int; dataB: var DiffData; lowerB, upperB: int; + downVector, upVector: var openArray[int]) = + ## This is the divide-and-conquer implementation of the longes common-subsequence (lcs) + ## algorithm. + ## The published algorithm passes recursively parts of the A and B sequences. + ## To avoid copying these arrays the lower and upper bounds are passed while the sequences stay constant. + ## `dataA` sequence A + ## `lowerA` lower bound of the actual range in dataA + ## `upperA` upper bound of the actual range in dataA (exclusive) + ## `dataB` sequence B + ## `lowerB` lower bound of the actual range in dataB + ## `upperB` upper bound of the actual range in dataB (exclusive) + ## `downVector` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons. + ## `upVector` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons. + + # make mutable copy: + var lowerA = lowerA + var lowerB = lowerB + var upperA = upperA + var upperB = upperB + + # Fast walkthrough equal lines at the start + while lowerA < upperA and lowerB < upperB and dataA.data[lowerA] == dataB.data[lowerB]: + inc lowerA + inc lowerB + + # Fast walkthrough equal lines at the end + while lowerA < upperA and lowerB < upperB and dataA.data[upperA - 1] == dataB.data[upperB - 1]: + dec upperA + dec upperB + + if lowerA == upperA: + # mark as inserted lines. + while lowerB < upperB: + dataB.modified[lowerB] = true + inc lowerB + + elif lowerB == upperB: + # mark as deleted lines. + while lowerA < upperA: + dataA.modified[lowerA] = true + inc lowerA + + else: + # Find the middle snake and length of an optimal path for A and B + let smsrd = sms(dataA, lowerA, upperA, dataB, lowerB, upperB, downVector, upVector) + # Debug.Write(2, "MiddleSnakeData", String.Format("{0},{1}", smsrd.x, smsrd.y)) + + # The path is from LowerX to (x,y) and (x,y) to UpperX + lcs(dataA, lowerA, smsrd.x, dataB, lowerB, smsrd.y, downVector, upVector) + lcs(dataA, smsrd.x, upperA, dataB, smsrd.y, upperB, downVector, upVector) # 2002.09.20: no need for 2 points + +proc createDiffs(dataA, dataB: DiffData): seq[Item] = + ## Scan the tables of which lines are inserted and deleted, + ## producing an edit script in forward order. + var startA = 0 + var startB = 0 + var lineA = 0 + var lineB = 0 + while lineA < dataA.len or lineB < dataB.len: + if lineA < dataA.len and not dataA.modified[lineA] and + lineB < dataB.len and not dataB.modified[lineB]: + # equal lines + inc lineA + inc lineB + else: + # maybe deleted and/or inserted lines + startA = lineA + startB = lineB + + while lineA < dataA.len and (lineB >= dataB.len or dataA.modified[lineA]): + inc lineA + + while lineB < dataB.len and (lineA >= dataA.len or dataB.modified[lineB]): + inc lineB + + if (startA < lineA) or (startB < lineB): + result.add Item(startA: startA, + startB: startB, + deletedA: lineA - startA, + insertedB: lineB - startB) + + +proc diffInt*(arrayA, arrayB: openArray[int]): seq[Item] = + ## Find the difference in 2 arrays of integers. + ## + ## `arrayA` A-version of the numbers (usually the old one) + ## + ## `arrayB` B-version of the numbers (usually the new one) + ## + ## Returns a sequence of Items that describe the differences. + + # The A-Version of the data (original data) to be compared. + var dataA = newDiffData(@arrayA, arrayA.len) + + # The B-Version of the data (modified data) to be compared. + var dataB = newDiffData(@arrayB, arrayB.len) + + let max = dataA.len + dataB.len + 1 + # vector for the (0,0) to (x,y) search + var downVector = newSeq[int](2 * max + 2) + # vector for the (u,v) to (N,M) search + var upVector = newSeq[int](2 * max + 2) + + lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector) + result = createDiffs(dataA, dataB) + +proc diffText*(textA, textB: string): seq[Item] = + ## Find the difference in 2 text documents, comparing by textlines. + ## + ## The algorithm itself is comparing 2 arrays of numbers so when comparing 2 text documents + ## each line is converted into a (hash) number. This hash-value is computed by storing all + ## textlines into a common hashtable so i can find duplicates in there, and generating a + ## new number each time a new textline is inserted. + ## + ## `textA` A-version of the text (usually the old one) + ## + ## `textB` B-version of the text (usually the new one) + ## + ## Returns a seq of Items that describe the differences. + # See also `gitutils.diffStrings`. + # prepare the input-text and convert to comparable numbers. + var h = initTable[string, int]() # TextA.len + TextB.len <- probably wrong initial size + # The A-Version of the data (original data) to be compared. + var dataA = diffCodes(textA, h) + + # The B-Version of the data (modified data) to be compared. + var dataB = diffCodes(textB, h) + + h.clear # free up hashtable memory (maybe) + + let max = dataA.len + dataB.len + 1 + # vector for the (0,0) to (x,y) search + var downVector = newSeq[int](2 * max + 2) + # vector for the (u,v) to (N,M) search + var upVector = newSeq[int](2 * max + 2) + + lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector) + + optimize(dataA) + optimize(dataB) + result = createDiffs(dataA, dataB) diff --git a/lib/genode/alloc.nim b/lib/genode/alloc.nim new file mode 100644 index 000000000..24fb9954e --- /dev/null +++ b/lib/genode/alloc.nim @@ -0,0 +1,119 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level dataspace allocator for Genode. +# For interacting with dataspaces outside of the +# standard library see the Genode Nimble package. + +when not defined(genode): + {.error: "Genode only module".} + +when not declared(GenodeEnv): + import genode/env + +type RamDataspaceCapability {. + importcpp: "Genode::Ram_dataspace_capability", pure.} = object + +type + Map = object + attachment: pointer + size: int + ds: RamDataspaceCapability + + SlabMeta = object + next: ptr MapSlab + ds: RamDataspaceCapability + + MapSlab = object + meta: SlabMeta + maps: array[1,Map] + +const SlabBackendSize = 4096 + +proc ramAvail(env: GenodeEnv): int {. + importcpp: "#->pd().avail_ram().value".} + ## Return number of bytes available for allocation. + +proc capsAvail(env: GenodeEnv): int {. + importcpp: "#->pd().avail_caps().value".} + ## Return the number of available capabilities. + ## Each dataspace allocation consumes a capability. + +proc allocDataspace(env: GenodeEnv; size: int): RamDataspaceCapability {. + importcpp: "#->pd().alloc(@)".} + ## Allocate a dataspace and its capability. + +proc attachDataspace(env: GenodeEnv; ds: RamDataspaceCapability): pointer {. + importcpp: "#->rm().attach(@)".} + ## Attach a dataspace into the component address-space. + +proc detachAddress(env: GenodeEnv; p: pointer) {. + importcpp: "#->rm().detach(@)".} + ## Detach a dataspace from the component address-space. + +proc freeDataspace(env: GenodeEnv; ds: RamDataspaceCapability) {. + importcpp: "#->pd().free(@)".} + ## Free a dataspace. + +proc newMapSlab(): ptr MapSlab = + let + ds = runtimeEnv.allocDataspace SlabBackendSize + p = runtimeEnv.attachDataspace ds + result = cast[ptr MapSlab](p) + result.meta.ds = ds + +iterator items(s: ptr MapSlab): ptr Map = + let mapCount = (SlabBackendSize - sizeof(SlabMeta)) div sizeof(Map) + for i in 0 ..< mapCount: + yield s.maps[i].addr + +var slabs: ptr MapSlab + +proc osAllocPages(size: int): pointer = + if slabs.isNil: + slabs = newMapSlab() + var + slab = slabs + map: ptr Map + let mapCount = (SlabBackendSize - sizeof(SlabMeta)) div sizeof(Map) + block findFreeMap: + while true: + # lookup first free spot in slabs + for m in slab.items: + if m.attachment.isNil: + map = m + break findFreeMap + if slab.meta.next.isNil: + slab.meta.next = newMapSlab() + # tack a new slab on the tail + slab = slab.meta.next + # move to next slab in linked list + map.ds = runtimeEnv.allocDataspace size + map.size = size + map.attachment = runtimeEnv.attachDataspace map.ds + result = map.attachment + +proc osTryAllocPages(size: int): pointer = + if runtimeEnv.ramAvail() >= size and runtimeEnv.capsAvail() > 4: + result = osAllocPages size + +proc osDeallocPages(p: pointer; size: int) = + var slab = slabs + while not slab.isNil: + # lookup first free spot in slabs + for m in slab.items: + if m.attachment == p: + if m.size != size: + echo "cannot partially detach dataspace" + rawQuit -1 + runtimeEnv.detachAddress m.attachment + runtimeEnv.freeDataspace m.ds + m[] = Map() + return + slab = slab.meta.next diff --git a/lib/genode/constructibles.nim b/lib/genode/constructibles.nim new file mode 100644 index 000000000..3a4a646e0 --- /dev/null +++ b/lib/genode/constructibles.nim @@ -0,0 +1,21 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +type Constructible*[T] {. + importcpp: "Genode::Constructible", + header: "<util/reconstructible.h>", byref, pure.} = object + +proc construct*[T](x: Constructible[T]) {.importcpp.} + ## Construct a constructible C++ object. + +proc destruct*[T](x: Constructible[T]) {.importcpp.} + ## Destruct a constructible C++ object. + +proc constructed*[T](x: Constructible[T]): bool {.importcpp.} + ## Test if an object is constructed. diff --git a/lib/genode/entrypoints.nim b/lib/genode/entrypoints.nim new file mode 100644 index 000000000..0bf5e0e0e --- /dev/null +++ b/lib/genode/entrypoints.nim @@ -0,0 +1,22 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## See `Genode Foundations - Entrypoint <https://genode.org/documentation/genode-foundations/21.05/functional_specification/Entrypoint.html>` +## for a description of Entrypoints. + +type + EntrypointObj {. + importcpp: "Genode::Entrypoint", + header: "<base/entrypoint.h>", + pure.} = object + Entrypoint* = ptr EntrypointObj + ## Opaque Entrypoint object. + +proc ep*(env: GenodeEnv): Entrypoint {.importcpp: "(&#->ep())".} + ## Access the entrypoint associated with `env`. diff --git a/lib/genode/env.nim b/lib/genode/env.nim new file mode 100644 index 000000000..babe2a8a0 --- /dev/null +++ b/lib/genode/env.nim @@ -0,0 +1,29 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# +# This file contains the minimum required definitions +# for interacting with the initial Genode environment. +# It is reserved for use only within the standard +# library. See `componentConstructHook` in the system +# module for accessing the Genode environment after the +# standard library has finished initializating. +# + +when not defined(genode): + {.error: "Genode only module".} + +type + GenodeEnvObj* {.importcpp: "Genode::Env", header: "<base/env.h>", pure.} = object + GenodeEnvPtr* = ptr GenodeEnvObj + +const runtimeEnvSym* = "nim_runtime_env" + +when not defined(nimscript): + var runtimeEnv* {.importcpp: runtimeEnvSym.}: GenodeEnvPtr diff --git a/lib/genode/signals.nim b/lib/genode/signals.nim new file mode 100644 index 000000000..7d1875730 --- /dev/null +++ b/lib/genode/signals.nim @@ -0,0 +1,77 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## See `Genode Foundations - Asynchronous notifications <https://genode.org/documentation/genode-foundations/21.05/architecture/Inter-component_communication.html#Asynchronous_notifications>` +## for a description of Genode signals. + +when not defined(genode) or defined(nimdoc): + {.error: "Genode only module".} + +import ./entrypoints, ./constructibles + +export ep # Entrypoint accessor on GenodeEnv + +type + SignalContextCapability* {. + importcpp: "Genode::Signal_context_capability", + header: "<base/signal.h>", pure.} = object + ## Capability to an asynchronous signal context. + +proc isValid*(cap: SignalContextCapability): bool {.importcpp: "#.valid()".} + ## Call the Genode core to check if this `SignalContextCapability` is valid. + # TODO: RpcEffect + +type + HandlerProc = proc () {.closure, gcsafe.} + + SignalHandlerBase {. + importcpp: "Nim::SignalHandler", + header: "genode_cpp/signals.h", + pure.} = object + + SignalHandlerCpp = Constructible[SignalHandlerBase] + + SignalHandlerObj = object + cpp: SignalHandlerCpp + cb: HandlerProc + ## Signal handling procedure called during dispatch. + + SignalHandler* = ref SignalHandlerObj + ## Nim object enclosing a Genode signal handler. + +proc construct(cpp: SignalHandlerCpp; ep: Entrypoint; sh: SignalHandler) {.importcpp.} + +proc cap(cpp: SignalHandlerCpp): SignalContextCapability {.importcpp: "#->cap()".} + +proc newSignalHandler*(ep: Entrypoint; cb: HandlerProc): SignalHandler = + ## Create a new signal handler. A label is recommended for + ## debugging purposes. A signal handler will not be garbage + ## collected until after it has been dissolved. + result = SignalHandler(cb: cb) + result.cpp.construct(ep, result) + GCref result + +proc dissolve*(sig: SignalHandler) = + ## Dissolve signal dispatcher from entrypoint. + # TODO: =destroy? + destruct sig.cpp + sig.cb = nil # lose the callback + GCunref sig + +proc cap*(sig: SignalHandler): SignalContextCapability = + ## Signal context capability. Can be delegated to external components. + sig.cpp.cap + +proc submit*(cap: SignalContextCapability) {. + importcpp: "Genode::Signal_transmitter(#).submit()".} + ## Submit a signal to a context capability. + +proc nimHandleSignal(p: pointer) {.exportc.} = + ## C symbol invoked by entrypoint during signal dispatch. + cast[SignalHandler](p).cb() diff --git a/lib/genode_cpp/signals.h b/lib/genode_cpp/signals.h new file mode 100644 index 000000000..fa3975d38 --- /dev/null +++ b/lib/genode_cpp/signals.h @@ -0,0 +1,39 @@ +/* + * + * Nim's Runtime Library + * (c) Copyright 2022 Emery Hemingway + * + * See the file "copying.txt", included in this + * distribution, for details about the copyright. + * + */ + +#ifndef _NIM_SIGNALS_H_ +#define _NIM_SIGNALS_H_ + +#include <libc/component.h> +#include <base/signal.h> +#include <util/reconstructible.h> + +// Symbol for calling back into Nim +extern "C" void nimHandleSignal(void *arg); + +namespace Nim { struct SignalHandler; } + +struct Nim::SignalHandler +{ + // Pointer to the Nim handler object. + void *arg; + + void handle_signal() { + Libc::with_libc([this] () { nimHandleSignal(arg); }); } + + Genode::Signal_handler<SignalHandler> handler; + + SignalHandler(Genode::Entrypoint *ep, void *arg) + : arg(arg), handler(*ep, *this, &SignalHandler::handle_signal) { } + + Genode::Signal_context_capability cap() { return handler; } +}; + +#endif diff --git a/lib/genode_cpp/syslocks.h b/lib/genode_cpp/syslocks.h new file mode 100644 index 000000000..b5d5ae694 --- /dev/null +++ b/lib/genode_cpp/syslocks.h @@ -0,0 +1,81 @@ +/* + * + * Nim's Runtime Library + * (c) Copyright 2017 Emery Hemingway + * + * See the file "copying.txt", included in this + * distribution, for details about the copyright. + * + */ + +#ifndef _GENODE_CPP__SYSLOCKS_H_ +#define _GENODE_CPP__SYSLOCKS_H_ + +/* Genode includes */ +#include <base/semaphore.h> +#include <base/mutex.h> + +namespace Nim { + struct SysLock; + struct SysCond; +} + +struct Nim::SysLock +{ + Genode::Mutex _mutex_a, _mutex_b; + bool _locked; + + void acquireSys() + { + Genode::Mutex::Guard guard(_mutex_a); + _locked = true; + _mutex_b.acquire(); + } + + bool tryAcquireSys() + { + if (_locked) + return false; + + Genode::Mutex::Guard guard(_mutex_a); + + if (_locked) { + return false; + } else { + _locked = true; + _mutex_b.acquire(); + return true; + } + } + + void releaseSys() + { + Genode::Mutex::Guard guard(_mutex_a); + _locked = false; + _mutex_b.release(); + } +}; + +struct Nim::SysCond +{ + Genode::Semaphore _semaphore; + + void waitSysCond(SysLock &syslock) + { + syslock.releaseSys(); + _semaphore.down(); + syslock.acquireSys(); + } + + void signalSysCond() + { + _semaphore.up(); + } + + void broadcastSysCond() + { + _semaphore.up(); + } +}; + +#endif diff --git a/lib/genode_cpp/threads.h b/lib/genode_cpp/threads.h new file mode 100644 index 000000000..c901efb45 --- /dev/null +++ b/lib/genode_cpp/threads.h @@ -0,0 +1,73 @@ +/* + * + * Nim's Runtime Library + * (c) Copyright 2017 Emery Hemingway + * + * See the file "copying.txt", included in this + * distribution, for details about the copyright. + * + */ + + +#ifndef _GENODE_CPP__THREAD_H_ +#define _GENODE_CPP__THREAD_H_ + +#include <base/thread.h> +#include <base/env.h> +#include <util/reconstructible.h> + +namespace Nim { struct SysThread; } + +struct Nim::SysThread +{ + typedef void (Entry)(void*); + + struct Thread : Genode::Thread + { + void *_tls; + + Entry *_func; + void *_arg; + + void entry() override { + (_func)(_arg); } + + Thread(Genode::Env &env, Genode::size_t stack_size, Entry func, void *arg, int affinity) + : Genode::Thread(env, "nim-thread", stack_size, + env.cpu().affinity_space().location_of_index(affinity), + Genode::Cpu_session::Weight(Genode::Cpu_session::Weight::DEFAULT_WEIGHT-1), + env.cpu()), + _func(func), _arg(arg) + { + Genode::Thread::start(); + } + }; + + Genode::Constructible<Thread> _thread; + + void initThread(Genode::Env *env, Genode::size_t stack_size, Entry func, void *arg, int aff) { + _thread.construct(*env, stack_size, func, arg, aff); } + + void joinThread() { + _thread->join(); } + + static bool offMainThread() { + return dynamic_cast<SysThread::Thread*>(Genode::Thread::myself()); } + + static void *threadVarGetValue() + { + SysThread::Thread *thr = + static_cast<SysThread::Thread*>(Genode::Thread::myself()); + return thr->_tls; + } + + static void threadVarSetValue(void *value) + { + SysThread::Thread *thr = + static_cast<SysThread::Thread*>(Genode::Thread::myself()); + thr->_tls = value; + } + +}; + +#endif diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim deleted file mode 100644 index 170fee8b8..000000000 --- a/lib/impure/db_mysql.nim +++ /dev/null @@ -1,360 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `mySQL`:idx: database wrapper. The same interface is -## implemented for other databases too. -## -## Example: -## -## .. code-block:: Nim -## -## import db_mysql, math -## -## let theDb = open("localhost", "nim", "nim", "test") -## -## theDb.exec(sql"Drop table if exists myTestTbl") -## theDb.exec(sql("create table myTestTbl (" & -## " Id INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY, " & -## " Name VARCHAR(50) NOT NULL, " & -## " i INT(11), " & -## " f DECIMAL(18,10))")) -## -## theDb.exec(sql"START TRANSACTION") -## for i in 1..1000: -## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#" & $i, i, sqrt(i.float)) -## theDb.exec(sql"COMMIT") -## -## for x in theDb.fastRows(sql"select * from myTestTbl"): -## echo x -## -## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) -## -## theDb.close() - - -import strutils, mysql - -import db_common -export db_common - -type - DbConn* = PMySQL ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = object ## a handle that can be used to get a row's - ## column text on demand - row: cstringArray - len: int -{.deprecated: [TRow: Row, TDbConn: DbConn].} - -proc dbError*(db: DbConn) {.noreturn.} = - ## raises a DbError exception. - var e: ref DbError - new(e) - e.msg = $mysql.error(db) - raise e - -when false: - proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) = - var stmt = mysql_stmt_init(db) - if stmt == nil: dbError(db) - if mysql_stmt_prepare(stmt, query, len(query)) != 0: - dbError(db) - var - binding: seq[MYSQL_BIND] - discard mysql_stmt_close(stmt) - -proc dbQuote*(s: string): string = - ## DB quotes the string. - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - if args[a] == nil: - add(result, "NULL") - else: - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var q = dbFormat(query, args) - return mysql.realQuery(db, q, q.len) == 0'i32 - -proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = - var q = dbFormat(query, args) - if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) - -proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query and raises EDB if not successful. - var q = dbFormat(query, args) - if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) - -proc newRow(L: int): Row = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = - if row != nil: - while mysql.fetchRow(sqlres) != nil: discard - mysql.freeResult(sqlres) - -iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the query and iterates over the result dataset. - ## - ## This is very fast, but potentially dangerous. Use this iterator only - ## if you require **ALL** the rows. - ## - ## Breaking the fastRows() iterator during a loop will cause the next - ## database query to raise an [EDb] exception ``Commands out of sync``. - rawExec(db, query, args) - var sqlres = mysql.useResult(db) - if sqlres != nil: - var L = int(mysql.numFields(sqlres)) - var result = newRow(L) - var row: cstringArray - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - for i in 0..L-1: - setLen(result[i], 0) - if row[i] == nil: - result[i] = nil - else: - add(result[i], row[i]) - yield result - properFreeResult(sqlres, row) - -iterator instantRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## Same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - rawExec(db, query, args) - var sqlres = mysql.useResult(db) - if sqlres != nil: - let L = int(mysql.numFields(sqlres)) - var row: cstringArray - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - yield InstantRow(row: row, len: L) - properFreeResult(sqlres, row) - -proc setTypeName(t: var DbType; f: PFIELD) = - shallowCopy(t.name, $f.name) - t.maxReprLen = Natural(f.max_length) - if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true - case f.ftype - of TYPE_DECIMAL: - t.kind = dbDecimal - of TYPE_TINY: - t.kind = dbInt - t.size = 1 - of TYPE_SHORT: - t.kind = dbInt - t.size = 2 - of TYPE_LONG: - t.kind = dbInt - t.size = 4 - of TYPE_FLOAT: - t.kind = dbFloat - t.size = 4 - of TYPE_DOUBLE: - t.kind = dbFloat - t.size = 8 - of TYPE_NULL: - t.kind = dbNull - of TYPE_TIMESTAMP: - t.kind = dbTimestamp - of TYPE_LONGLONG: - t.kind = dbInt - t.size = 8 - of TYPE_INT24: - t.kind = dbInt - t.size = 3 - of TYPE_DATE: - t.kind = dbDate - of TYPE_TIME: - t.kind = dbTime - of TYPE_DATETIME: - t.kind = dbDatetime - of TYPE_YEAR: - t.kind = dbDate - of TYPE_NEWDATE: - t.kind = dbDate - of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING: - t.kind = dbVarchar - of TYPE_BIT: - t.kind = dbBit - of TYPE_NEWDECIMAL: - t.kind = dbDecimal - of TYPE_ENUM: t.kind = dbEnum - of TYPE_SET: t.kind = dbSet - of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB, - TYPE_BLOB: t.kind = dbBlob - of TYPE_GEOMETRY: - t.kind = dbGeometry - -proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) = - setLen(columns, L) - for i in 0..<L: - let fp = mysql.fetch_field_direct(res, cint(i)) - setTypeName(columns[i].typ, fp) - columns[i].name = $fp.name - columns[i].tableName = $fp.table - columns[i].primaryKey = (fp.flags and PRI_KEY_FLAG) != 0 - #columns[i].foreignKey = there is no such thing in mysql - -iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; - args: varargs[string, `$`]): InstantRow = - ## Same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - rawExec(db, query, args) - var sqlres = mysql.useResult(db) - if sqlres != nil: - let L = int(mysql.numFields(sqlres)) - setColumnInfo(columns, sqlres, L) - var row: cstringArray - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - yield InstantRow(row: row, len: L) - properFreeResult(sqlres, row) - - -proc `[]`*(row: InstantRow, col: int): string {.inline.} = - ## Returns text for given column of the row. - $row.row[col] - -proc len*(row: InstantRow): int {.inline.} = - ## Returns number of columns in the row. - row.len - -proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## Retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - rawExec(db, query, args) - var sqlres = mysql.useResult(db) - if sqlres != nil: - var L = int(mysql.numFields(sqlres)) - result = newRow(L) - var row = mysql.fetchRow(sqlres) - if row != nil: - for i in 0..L-1: - setLen(result[i], 0) - if row[i] == nil: - result[i] = nil - else: - add(result[i], row[i]) - properFreeResult(sqlres, row) - -proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = - ## executes the query and returns the whole result dataset. - result = @[] - rawExec(db, query, args) - var sqlres = mysql.useResult(db) - if sqlres != nil: - var L = int(mysql.numFields(sqlres)) - var row: cstringArray - var j = 0 - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - setLen(result, j+1) - newSeq(result[j], L) - for i in 0..L-1: - if row[i] == nil: - result[j][i] = nil - else: - result[j][i] = $row[i] - inc(j) - mysql.freeResult(sqlres) - -iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `fastRows`, but slower and safe. - for r in items(getAllRows(db, query, args)): yield r - -proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - result = getRow(db, query, args)[0] - -proc tryInsertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. - var q = dbFormat(query, args) - if mysql.realQuery(db, q, q.len) != 0'i32: - result = -1'i64 - else: - result = mysql.insertId(db) - -proc insertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## runs the query (typically "UPDATE") and returns the - ## number of affected rows - rawExec(db, query, args) - result = mysql.affectedRows(db) - -proc close*(db: DbConn) {.tags: [DbEffect].} = - ## closes the database connection. - if db != nil: mysql.close(db) - -proc open*(connection, user, password, database: string): DbConn {. - tags: [DbEffect].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. - result = mysql.init(nil) - if result == nil: dbError("could not open database connection") - let - colonPos = connection.find(':') - host = if colonPos < 0: connection - else: substr(connection, 0, colonPos-1) - port: int32 = if colonPos < 0: 0'i32 - else: substr(connection, colonPos+1).parseInt.int32 - if mysql.realConnect(result, host, user, password, database, - port, nil, 0) == nil: - var errmsg = $mysql.error(result) - db_mysql.close(result) - dbError(errmsg) - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [DbEffect].} = - ## sets the encoding of a database connection, returns true for - ## success, false for failure. - result = mysql.set_character_set(connection, encoding) == 0 diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim deleted file mode 100644 index 6af69d842..000000000 --- a/lib/impure/db_odbc.nim +++ /dev/null @@ -1,463 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `ODBC` database wrapper. -## -## This is the same interface that is implemented for other databases. -## -## This has NOT yet been (extensively) tested agains ODBC drivers for -## Teradata, Oracle, Sybase, MSSqlvSvr, et. al. databases -## -## Currently all queries are ANSI calls, not Unicode. -## -## Example: -## -## .. code-block:: Nim -## -## import db_odbc, math -## -## let theDb = open("localhost", "nim", "nim", "test") -## -## theDb.exec(sql"Drop table if exists myTestTbl") -## theDb.exec(sql("create table myTestTbl (" & -## " Id INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY, " & -## " Name VARCHAR(50) NOT NULL, " & -## " i INT(11), " & -## " f DECIMAL(18,10))")) -## -## theDb.exec(sql"START TRANSACTION") -## for i in 1..1000: -## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#" & $i, i, sqrt(i.float)) -## theDb.exec(sql"COMMIT") -## -## for x in theDb.fastRows(sql"select * from myTestTbl"): -## echo x -## -## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) -## -## theDb.close() - - -import strutils, odbcsql - -import db_common -export db_common - -type - OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt] - DbConn* = OdbcConnTyp ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = tuple[row: seq[string], len: int] ## a handle that can be - ## used to get a row's - ## column text on demand - -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} - -var - buf: array[0..4096, char] - -proc properFreeResult(hType: int, sqlres: var SqlHandle) {. - tags: [WriteDbEffect], raises: [].} = - try: - discard SQLFreeHandle(hType.TSqlSmallInt, sqlres) - sqlres = nil - except: discard - -proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. - tags: [ReadDbEffect], raises: [].} = - ## Returns ODBC error information - var - sqlState: array[0..512, char] - nativeErr: array[0..512, char] - errMsg: array[0..512, char] - retSz: TSqlSmallInt = 0 - res: TSqlSmallInt = 0 - try: - sqlState[0] = '\0' - nativeErr[0] = '\0' - errMsg[0] = '\0' - res = SQLErr(db.env, db.hDb, db.stmt, - cast[PSQLCHAR](sqlState.addr), - cast[PSQLCHAR](nativeErr.addr), - cast[PSQLCHAR](errMsg.addr), - 511.TSqlSmallInt, retSz.addr.PSQLSMALLINT) - except: - discard - return (res.int, $sqlState, $nativeErr, $errMsg) - -proc dbError*(db: var DbConn) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} = - ## Raises an `[DbError]` exception with ODBC error information - var - e: ref DbError - ss, ne, msg: string = "" - isAnError = false - res: int = 0 - prevSs = "" - while true: - prevSs = ss - (res, ss, ne, msg) = db.getErrInfo() - if prevSs == ss: - break - # sqlState of 00000 is not an error - elif ss == "00000": - break - elif ss == "01000": - echo "\nWarning: ", ss, " ", msg - continue - else: - isAnError = true - echo "\nError: ", ss, " ", msg - if isAnError: - new(e) - e.msg = "ODBC Error" - if db.stmt != nil: - properFreeResult(SQL_HANDLE_STMT, db.stmt) - properFreeResult(SQL_HANDLE_DBC, db.hDb) - properFreeResult(SQL_HANDLE_ENV, db.env) - raise e - -proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} = - ## Wrapper that checks if ``resVal`` is not SQL_SUCCESS and if so, raises [EDb] - if resVal != SQL_SUCCESS: dbError(db) - -proc SqlGetDBMS(db: var DbConn): string {. - tags: [ReadDbEffect, WriteDbEffect], raises: [] .} = - ## Returns the ODBC SQL_DBMS_NAME string - const - SQL_DBMS_NAME = 17.SqlUSmallInt - var - sz: TSqlSmallInt = 0 - buf[0] = '\0' - try: - db.SqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr), - 4095.TSqlSmallInt, sz.addr)) - except: discard - return $buf.cstring - -proc dbQuote*(s: string): string {.noSideEffect.} = - ## DB quotes the string. - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. - noSideEffect.} = - ## Replace any ``?`` placeholders with `args`, - ## and quotes the arguments - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - if args[a] == nil: - add(result, "NULL") - else: - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc prepareFetch(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - # Prepare a statement, execute it and fetch the data to the driver - # ready for retrieval of the data - # Used internally by iterators and retrieval procs - # requires calling - # properFreeResult(SQL_HANDLE_STMT, db.stmt) - # when finished - db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) - var q = dbFormat(query, args) - db.SqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) - db.SqlCheck(SQLExecute(db.stmt)) - db.SqlCheck(SQLFetch(db.stmt)) - -proc prepareFetchDirect(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - # Prepare a statement, execute it and fetch the data to the driver - # ready for retrieval of the data - # Used internally by iterators and retrieval procs - # requires calling - # properFreeResult(SQL_HANDLE_STMT, db.stmt) - # when finished - db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) - var q = dbFormat(query, args) - db.SqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) - db.SqlCheck(SQLFetch(db.stmt)) - -proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [ReadDbEffect, WriteDbEffect], raises: [].} = - ## Tries to execute the query and returns true if successful, false otherwise. - var - res:TSqlSmallInt = -1 - try: - db.prepareFetchDirect(query, args) - var - rCnt = -1 - res = SQLRowCount(db.stmt, rCnt) - if res != SQL_SUCCESS: dbError(db) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - except: discard - return res == SQL_SUCCESS - -proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - db.prepareFetchDirect(query, args) - -proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query and raises EDB if not successful. - db.prepareFetchDirect(query, args) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - -proc newRow(L: int): Row {.noSideEFfect.} = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -iterator fastRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query and iterates over the result dataset. - ## - ## This is very fast, but potentially dangerous. Use this iterator only - ## if you require **ALL** the rows. - ## - ## Breaking the fastRows() iterator during a loop may cause a driver error - ## for subsequenct queries - ## - ## Rows are retrieved from the server at each iteration. - var - rowRes: Row - sz: TSqlSmallInt = 0 - cCnt: TSqlSmallInt = 0.TSqlSmallInt - rCnt = -1 - - db.prepareFetch(query, args) - db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) - db.SqlCheck(SQLRowCount(db.stmt, rCnt)) - rowRes = newRow(cCnt) - for rNr in 1..rCnt: - for colId in 1..cCnt: - buf[0] = '\0' - db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) - rowRes[colId-1] = $buf.cstring - db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) - yield rowRes - properFreeResult(SQL_HANDLE_STMT, db.stmt) - -iterator instantRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect, WriteDbEffect].} = - ## Same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the interator body. - var - rowRes: Row - sz: TSqlSmallInt = 0 - cCnt: TSqlSmallInt = 0.TSqlSmallInt - rCnt = -1 - db.prepareFetch(query, args) - db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) - db.SqlCheck(SQLRowCount(db.stmt, rCnt)) - rowRes = newRow(cCnt) - for rNr in 1..rCnt: - for colId in 1..cCnt: - buf[0] = '\0' - db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) - rowRes[colId-1] = $buf.cstring - db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) - yield (row: rowRes, len: cCnt.int) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - -proc `[]`*(row: InstantRow, col: int): string {.inline.} = - ## Returns text for given column of the row - row.row[col] - -proc len*(row: InstantRow): int {.inline.} = - ## Returns number of columns in the row - row.len - -proc getRow*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - var - sz: TSqlSmallInt = 0.TSqlSmallInt - cCnt: TSqlSmallInt = 0.TSqlSmallInt - rCnt = -1 - result = @[] - db.prepareFetch(query, args) - db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) - - db.SqlCheck(SQLRowCount(db.stmt, rCnt)) - for colId in 1..cCnt: - db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) - result.add($buf.cstring) - db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - -proc getAllRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query and returns the whole result dataset. - var - rowRes: Row - sz: TSqlSmallInt = 0 - cCnt: TSqlSmallInt = 0.TSqlSmallInt - rCnt = -1 - db.prepareFetch(query, args) - db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) - db.SqlCheck(SQLRowCount(db.stmt, rCnt)) - result = @[] - for rNr in 1..rCnt: - rowRes = @[] - buf[0] = '\0' - for colId in 1..cCnt: - db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[SqlPointer](buf.addr), 4095.TSqlSmallInt, sz.addr)) - rowRes.add($buf.cstring) - db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) - result.add(rowRes) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - -iterator rows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Same as `fastRows`, but slower and safe. - ## - ## This retrieves ALL rows into memory before - ## iterating through the rows. - ## Large dataset queries will impact on memory usage. - for r in items(getAllRows(db, query, args)): yield r - -proc getValue*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {. - tags: [ReadDbEffect, WriteDbEffect], raises: [].} = - ## Executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - result = "" - try: - result = getRow(db, query, args)[0] - except: discard - -proc tryInsertId*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect], raises: [].} = - ## Executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. - if not tryExec(db, query, args): - result = -1'i64 - else: - echo "DBMS: ",SqlGetDBMS(db).toLower() - result = -1'i64 - try: - case SqlGetDBMS(db).toLower(): - of "postgresql": - result = getValue(db, sql"SELECT LASTVAL();", []).parseInt - of "mysql": - result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt - of "sqlite": - result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt - of "microsoft sql server": - result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt - of "oracle": - result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt - else: result = -1'i64 - except: discard - -proc insertId*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query (typically "INSERT") and returns the - ## generated ID for the row. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Runs the query (typically "UPDATE") and returns the - ## number of affected rows - result = -1 - var res = SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle) - if res != SQL_SUCCESS: dbError(db) - var q = dbFormat(query, args) - res = SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt) - if res != SQL_SUCCESS: dbError(db) - rawExec(db, query, args) - var rCnt = -1 - result = SQLRowCount(db.hDb, rCnt) - if res != SQL_SUCCESS: dbError(db) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - result = rCnt - -proc close*(db: var DbConn) {. - tags: [WriteDbEffect], raises: [].} = - ## Closes the database connection. - if db.hDb != nil: - try: - var res = SQLDisconnect(db.hDb) - if db.stmt != nil: - res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt) - res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb) - res = SQLFreeHandle(SQL_HANDLE_ENV, db.env) - db = (hDb: nil, env: nil, stmt: nil) - except: - discard - -proc open*(connection, user, password, database: string): DbConn {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Opens a database connection. - ## - ## Raises `EDb` if the connection could not be established. - ## - ## Currently the database parameter is ignored, - ## but included to match ``open()`` in the other db_xxxxx library modules. - var - val: TSqlInteger = SQL_OV_ODBC3 - resLen = 0 - result = (hDb: nil, env: nil, stmt: nil) - # allocate environment handle - var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env) - if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.") - res = SQLSetEnvAttr(result.env, - SQL_ATTR_ODBC_VERSION.TSqlInteger, - val, resLen.TSqlInteger) - if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.") - # allocate hDb handle - res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb) - if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.") - - # Connect: connection = dsn str, - res = SQLConnect(result.hDb, - connection.PSQLCHAR , connection.len.TSqlSmallInt, - user.PSQLCHAR, user.len.TSqlSmallInt, - password.PSQLCHAR, password.len.TSqlSmallInt) - if res != SQL_SUCCESS: - result.dbError() - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Currently not implemented for ODBC. - ## - ## Sets the encoding of a database connection, returns true for - ## success, false for failure. - #result = set_character_set(connection, encoding) == 0 - dbError("setEncoding() is currently not implemented by the db_odbc module") diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim deleted file mode 100644 index 9bdbae4c2..000000000 --- a/lib/impure/db_postgres.nim +++ /dev/null @@ -1,366 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `PostgreSQL`:idx: database wrapper. This interface -## is implemented for other databases also. -## -## Parameter substitution -## ---------------------- -## -## All ``db_*`` modules support the same form of parameter substitution. -## That is, using the ``?`` (question mark) to signify the place where a -## value should be placed. For example: -## -## .. code-block:: Nim -## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" -## -## **Note**: There are two approaches to parameter substitution support by -## this module. -## -## 1. ``SqlQuery`` using ``?, ?, ?, ...`` (same as all the ``db_*`` modules) -## -## 2. ``SqlPrepared`` using ``$1, $2, $3, ...`` -## -## .. code-block:: Nim -## prepare(db, "myExampleInsert", -## sql"""INSERT INTO myTable -## (colA, colB, colC) -## VALUES ($1, $2, $3)""", -## 3) -## -## Examples -## -------- -## -## Opening a connection to a database -## ================================== -## -## .. code-block:: Nim -## import db_postgres -## let db = open("localhost", "user", "password", "dbname") -## db.close() -## -## Creating a table -## ================ -## -## .. code-block:: Nim -## db.exec(sql"DROP TABLE IF EXISTS myTable") -## db.exec(sql("""CREATE TABLE myTable ( -## id integer, -## name varchar(50) not null)""")) -## -## Inserting data -## ============== -## -## .. code-block:: Nim -## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", -## "Dominik") -import strutils, postgres - -import db_common -export db_common - -type - DbConn* = PPGconn ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = tuple[res: PPGresult, line: int32] ## a handle that can be - ## used to get a row's - ## column text on demand - SqlPrepared* = distinct string ## a identifier for the prepared queries - -{.deprecated: [TRow: Row, TDbConn: DbConn, - TSqlPrepared: SqlPrepared].} - -proc dbError*(db: DbConn) {.noreturn.} = - ## raises a DbError exception. - var e: ref DbError - new(e) - e.msg = $pqErrorMessage(db) - raise e - -proc dbQuote*(s: string): string = - ## DB quotes the string. - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - if args.len > 0 and not string(formatstr).contains("?"): - dbError("""parameter substitution expects "?" """) - for c in items(string(formatstr)): - if c == '?': - if args[a] == nil: - add(result, "NULL") - else: - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, - nil, nil, 0) - result = pqresultStatus(res) == PGRES_COMMAND_OK - pqclear(res) - -proc tryExec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): bool {.tags: [ - ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var arr = allocCStringArray(args) - var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - result = pqresultStatus(res) == PGRES_COMMAND_OK - pqclear(res) - -proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query and raises EDB if not successful. - var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, - nil, nil, 0) - if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db) - pqclear(res) - -proc exec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} = - var arr = allocCStringArray(args) - var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db) - pqclear(res) - -proc newRow(L: int): Row = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -proc setupQuery(db: DbConn, query: SqlQuery, - args: varargs[string]): PPGresult = - result = pqexec(db, dbFormat(query, args)) - if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) - -proc setupQuery(db: DbConn, stmtName: SqlPrepared, - args: varargs[string]): PPGresult = - var arr = allocCStringArray(args) - result = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) - -proc prepare*(db: DbConn; stmtName: string, query: SqlQuery; - nParams: int): SqlPrepared = - if nParams > 0 and not string(query).contains("$1"): - dbError("""parameter substitution expects "$1" """) - var res = pqprepare(db, stmtName, query.string, int32(nParams), nil) - if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db) - return SqlPrepared(stmtName) - -proc setRow(res: PPGresult, r: var Row, line, cols: int32) = - for col in 0..cols-1: - setLen(r[col], 0) - let x = pqgetvalue(res, line, col) - if x.isNil: - r[col] = nil - else: - add(r[col], x) - -iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the query and iterates over the result dataset. This is very - ## fast, but potenially dangerous: If the for-loop-body executes another - ## query, the results can be undefined. For Postgres it is safe though. - var res = setupQuery(db, query, args) - var L = pqnfields(res) - var result = newRow(L) - for i in 0..pqntuples(res)-1: - setRow(res, result, i, L) - yield result - pqclear(res) - -iterator fastRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the prepared query and iterates over the result dataset. - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - var result = newRow(L) - for i in 0..pqNtuples(res)-1: - setRow(res, result, i, L) - yield result - pqClear(res) - -iterator instantRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, query, args) - for i in 0..pqNtuples(res)-1: - yield (res: res, line: i) - pqClear(res) - -iterator instantRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, stmtName, args) - for i in 0..pqNtuples(res)-1: - yield (res: res, line: i) - pqClear(res) - -proc `[]`*(row: InstantRow, col: int32): string {.inline.} = - ## returns text for given column of the row - $pqgetvalue(row.res, row.line, col) - -proc len*(row: InstantRow): int32 {.inline.} = - ## returns number of columns in the row - pqNfields(row.res) - -proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - var res = setupQuery(db, query, args) - var L = pqnfields(res) - result = newRow(L) - setRow(res, result, 0, L) - pqclear(res) - -proc getRow*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - result = newRow(L) - setRow(res, result, 0, L) - pqClear(res) - -proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {. - tags: [ReadDbEffect].} = - ## executes the query and returns the whole result dataset. - result = @[] - for r in fastRows(db, query, args): - result.add(r) - -proc getAllRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): seq[Row] {.tags: - [ReadDbEffect].} = - ## executes the prepared query and returns the whole result dataset. - result = @[] - for r in fastRows(db, stmtName, args): - result.add(r) - -iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `fastRows`, but slower and safe. - for r in items(getAllRows(db, query, args)): yield r - -iterator rows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `fastRows`, but slower and safe. - for r in items(getAllRows(db, stmtName, args)): yield r - -proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {. - tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - var x = pqgetvalue(setupQuery(db, query, args), 0, 0) - result = if isNil(x): "" else: $x - -proc tryInsertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [WriteDbEffect].}= - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. For Postgre this adds - ## ``RETURNING id`` to the query, so it only works if your primary key is - ## named ``id``. - var x = pqgetvalue(setupQuery(db, SqlQuery(string(query) & " RETURNING id"), - args), 0, 0) - if not isNil(x): - result = parseBiggestInt($x) - else: - result = -1 - -proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row. For Postgre this adds - ## ``RETURNING id`` to the query, so it only works if your primary key is - ## named ``id``. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [ - ReadDbEffect, WriteDbEffect].} = - ## executes the query (typically "UPDATE") and returns the - ## number of affected rows. - var q = dbFormat(query, args) - var res = pqExec(db, q) - if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db) - result = parseBiggestInt($pqcmdTuples(res)) - pqclear(res) - -proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): int64 {.tags: [ - ReadDbEffect, WriteDbEffect].} = - ## executes the query (typically "UPDATE") and returns the - ## number of affected rows. - var arr = allocCStringArray(args) - var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db) - result = parseBiggestInt($pqcmdTuples(res)) - pqclear(res) - -proc close*(db: DbConn) {.tags: [DbEffect].} = - ## closes the database connection. - if db != nil: pqfinish(db) - -proc open*(connection, user, password, database: string): DbConn {. - tags: [DbEffect].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. - ## - ## Clients can also use Postgres keyword/value connection strings to - ## connect. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## con = open("", "", "", "host=localhost port=5432 dbname=mydb") - ## - ## See http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING - ## for more information. - ## - ## Note that the connection parameter is not used but exists to maintain - ## the nim db api. - result = pqsetdbLogin(nil, nil, nil, nil, database, user, password) - if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [DbEffect].} = - ## sets the encoding of a database connection, returns true for - ## success, false for failure. - return pqsetClientEncoding(connection, encoding) == 0 - - -# Tests are in ../../tests/untestable/tpostgres. diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim deleted file mode 100644 index 5cfed1eba..000000000 --- a/lib/impure/db_sqlite.nim +++ /dev/null @@ -1,291 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `SQLite`:idx: database wrapper. This interface -## is implemented for other databases too. -## -## Example: -## -## .. code-block:: nim -## -## import db_sqlite, math -## -## let theDb = open("mytest.db", nil, nil, nil) -## -## theDb.exec(sql"Drop table if exists myTestTbl") -## theDb.exec(sql("""create table myTestTbl ( -## Id INTEGER PRIMARY KEY, -## Name VARCHAR(50) NOT NULL, -## i INT(11), -## f DECIMAL(18,10))""")) -## -## theDb.exec(sql"BEGIN") -## for i in 1..1000: -## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#" & $i, i, sqrt(i.float)) -## theDb.exec(sql"COMMIT") -## -## for x in theDb.fastRows(sql"select * from myTestTbl"): -## echo x -## -## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) -## -## theDb.close() - -import strutils, sqlite3 - -import db_common -export db_common - -type - DbConn* = PSqlite3 ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = Pstmt ## a handle that can be used to get a row's column - ## text on demand -{.deprecated: [TRow: Row, TDbConn: DbConn].} - -proc dbError*(db: DbConn) {.noreturn.} = - ## raises a DbError exception. - var e: ref DbError - new(e) - e.msg = $sqlite3.errmsg(db) - raise e - -proc dbQuote*(s: string): string = - ## DB quotes the string. - if s.isNil: return "NULL" - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var q = dbFormat(query, args) - var stmt: sqlite3.Pstmt - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = finalize(stmt) == SQLITE_OK - -proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query and raises DbError if not successful. - if not tryExec(db, query, args): dbError(db) - -proc newRow(L: int): Row = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -proc setupQuery(db: DbConn, query: SqlQuery, - args: varargs[string]): Pstmt = - var q = dbFormat(query, args) - if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db) - -proc setRow(stmt: Pstmt, r: var Row, cols: cint) = - for col in 0..cols-1: - setLen(r[col], column_bytes(stmt, col)) # set capacity - setLen(r[col], 0) - let x = column_text(stmt, col) - if not isNil(x): add(r[col], x) - -iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## Executes the query and iterates over the result dataset. - ## - ## This is very fast, but potentially dangerous. Use this iterator only - ## if you require **ALL** the rows. - ## - ## Breaking the fastRows() iterator during a loop will cause the next - ## database query to raise a DbError exception ``unable to close due to ...``. - var stmt = setupQuery(db, query, args) - var L = (column_count(stmt)) - var result = newRow(L) - while step(stmt) == SQLITE_ROW: - setRow(stmt, result, L) - yield result - if finalize(stmt) != SQLITE_OK: dbError(db) - -iterator instantRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - var stmt = setupQuery(db, query, args) - while step(stmt) == SQLITE_ROW: - yield stmt - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc toTypeKind(t: var DbType; x: int32) = - case x - of SQLITE_INTEGER: - t.kind = dbInt - t.size = 8 - of SQLITE_FLOAT: - t.kind = dbFloat - t.size = 8 - of SQLITE_BLOB: t.kind = dbBlob - of SQLITE_NULL: t.kind = dbNull - of SQLITE_TEXT: t.kind = dbVarchar - else: t.kind = dbUnknown - -proc setColumns(columns: var DbColumns; x: PStmt) = - let L = column_count(x) - setLen(columns, L) - for i in 0'i32 ..< L: - columns[i].name = $column_name(x, i) - columns[i].typ.name = $column_decltype(x, i) - toTypeKind(columns[i].typ, column_type(x, i)) - columns[i].tableName = $column_table_name(x, i) - -iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - var stmt = setupQuery(db, query, args) - setColumns(columns, stmt) - while step(stmt) == SQLITE_ROW: - yield stmt - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc `[]`*(row: InstantRow, col: int32): string {.inline.} = - ## returns text for given column of the row - $column_text(row, col) - -proc len*(row: InstantRow): int32 {.inline.} = - ## returns number of columns in the row - column_count(row) - -proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - var stmt = setupQuery(db, query, args) - var L = (column_count(stmt)) - result = newRow(L) - if step(stmt) == SQLITE_ROW: - setRow(stmt, result, L) - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = - ## executes the query and returns the whole result dataset. - result = @[] - for r in fastRows(db, query, args): - result.add(r) - -iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `FastRows`, but slower and safe. - for r in fastRows(db, query, args): yield r - -proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - var stmt = setupQuery(db, query, args) - if step(stmt) == SQLITE_ROW: - let cb = column_bytes(stmt, 0) - if cb == 0: - result = "" - else: - result = newStringOfCap(cb) - add(result, column_text(stmt, 0)) - else: - result = "" - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc tryInsertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. - var q = dbFormat(query, args) - var stmt: sqlite3.Pstmt - result = -1 - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: - result = -1 - -proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row. For Postgre this adds - ## ``RETURNING id`` to the query, so it only works if your primary key is - ## named ``id``. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query (typically "UPDATE") and returns the - ## number of affected rows. - exec(db, query, args) - result = changes(db) - -proc close*(db: DbConn) {.tags: [DbEffect].} = - ## closes the database connection. - if sqlite3.close(db) != SQLITE_OK: dbError(db) - -proc open*(connection, user, password, database: string): DbConn {. - tags: [DbEffect].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. Only the ``connection`` parameter is used for ``sqlite``. - var db: DbConn - if sqlite3.open(connection, db) == SQLITE_OK: - result = db - else: - dbError(db) - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [DbEffect].} = - ## sets the encoding of a database connection, returns true for - ## success, false for failure. - ## - ## Note that the encoding cannot be changed once it's been set. - ## According to SQLite3 documentation, any attempt to change - ## the encoding after the database is created will be silently - ## ignored. - exec(connection, sql"PRAGMA encoding = ?", [encoding]) - result = connection.getValue(sql"PRAGMA encoding") == encoding - -when not defined(testing) and isMainModule: - var db = open("db.sql", "", "", "") - exec(db, sql"create table tbl1(one varchar(10), two smallint)", []) - exec(db, sql"insert into tbl1 values('hello!',10)", []) - exec(db, sql"insert into tbl1 values('goodbye', 20)", []) - #db.query("create table tbl1(one varchar(10), two smallint)") - #db.query("insert into tbl1 values('hello!',10)") - #db.query("insert into tbl1 values('goodbye', 20)") - for r in db.rows(sql"select * from tbl1", []): - echo(r[0], r[1]) - for r in db.instantRows(sql"select * from tbl1", []): - echo(r[0], r[1]) - - db_sqlite.close(db) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 973f1f2ee..39d238055 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -1,87 +1,95 @@ # # Nim's Runtime Library -# (c) Copyright 2015 Nim Contributers +# (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # - -from pcre import nil -import nre.private.util -import tables -import unsigned -from strutils import toLower, `%` -from math import ceil -import options -from unicode import runeLenAt - +when defined(js): + {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE; See jsre for JS backend.".} ## What is NRE? ## ============ ## ## A regular expression library for Nim using PCRE to do the hard work. ## +## For documentation on how to write patterns, there exists `the official PCRE +## pattern documentation +## <https://www.pcre.org/original/doc/html/pcrepattern.html>`_. You can also +## search the internet for a wide variety of third-party documentation and +## tools. +## +## .. warning:: If you love `sequtils.toSeq` we have bad news for you. This +## library doesn't work with it due to documented compiler limitations. As +## a workaround, use this: +runnableExamples: + # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`: + import std/sequtils + iterator iota(n: int): int = + for i in 0..<n: yield i + assert sequtils.toSeq(iota(3)) == @[0, 1, 2] +## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre) +## and [regex](https://github.com/nitely/nim-regex). ## Licencing ## --------- ## -## PCRE has some additional terms that you must comply with if you use this module.:: +## PCRE has `some additional terms`_ that you must agree to in order to use +## this module. ## -## > Copyright (c) 1997-2001 University of Cambridge -## > -## > Permission is granted to anyone to use this software for any purpose on any -## > computer system, and to redistribute it freely, subject to the following -## > restrictions: -## > -## > 1. This software is distributed in the hope that it will be useful, -## > but WITHOUT ANY WARRANTY; without even the implied warranty of -## > MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -## > -## > 2. The origin of this software must not be misrepresented, either by -## > explicit claim or by omission. In practice, this means that if you use -## > PCRE in software that you distribute to others, commercially or -## > otherwise, you must put a sentence like this -## > -## > Regular expression support is provided by the PCRE library package, -## > which is open source software, written by Philip Hazel, and copyright -## > by the University of Cambridge, England. -## > -## > somewhere reasonably visible in your documentation and in any relevant -## > files or online help data or similar. A reference to the ftp site for -## > the source, that is, to -## > -## > ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/ -## > -## > should also be given in the documentation. However, this condition is not -## > intended to apply to whole chains of software. If package A includes PCRE, -## > it must acknowledge it, but if package B is software that includes package -## > A, the condition is not imposed on package B (unless it uses PCRE -## > independently). -## > -## > 3. Altered versions must be plainly marked as such, and must not be -## > misrepresented as being the original software. -## > -## > 4. If PCRE is embedded in any software that is released under the GNU -## > General Purpose Licence (GPL), or Lesser General Purpose Licence (LGPL), -## > then the terms of that licence shall supersede any condition above with -## > which it is incompatible. - - -# Type definitions {{{ +## .. _`some additional terms`: http://pcre.sourceforge.net/license.txt +runnableExamples: + import std/sugar + let vowels = re"[aeoui]" + let bounds = collect: + for match in "moiga".findIter(vowels): match.matchBounds + assert bounds == @[1 .. 1, 2 .. 2, 4 .. 4] + from std/sequtils import toSeq + let s = sequtils.toSeq("moiga".findIter(vowels)) + # fully qualified to avoid confusion with nre.toSeq + assert s.len == 3 + + let firstVowel = "foo".find(vowels) + let hasVowel = firstVowel.isSome() + assert hasVowel + let matchBounds = firstVowel.get().captureBounds[-1] + assert matchBounds.a == 1 + + # as with module `re`, unless specified otherwise, `start` parameter in each + # proc indicates where the scan starts, but outputs are relative to the start + # of the input string, not to `start`: + assert find("uxabc", re"(?<=x|y)ab", start = 1).get.captures[-1] == "ab" + assert find("uxabc", re"ab", start = 3).isNone + +from std/pcre import nil +import nre/private/util +import std/tables +from std/strutils import `%` +import std/options +from std/unicode import runeLenAt + +when defined(nimPreviewSlimSystem): + import std/assertions + +export options + type Regex* = ref object ## Represents the pattern that things are matched against, constructed with - ## ``re(string)``. Examples: ``re"foo"``, ``re(r"(*ANYCRLF)(?x)foo # - ## comment".`` + ## `re(string)`. Examples: `re"foo"`, `re(r"(*ANYCRLF)(?x)foo # + ## comment".` ## - ## ``pattern: string`` - ## the string that was used to create the pattern. + ## `pattern: string` + ## : the string that was used to create the pattern. For details on how + ## to write a pattern, please see `the official PCRE pattern + ## documentation. + ## <https://www.pcre.org/original/doc/html/pcrepattern.html>`_ ## - ## ``captureCount: int`` - ## the number of captures that the pattern has. + ## `captureCount: int` + ## : the number of captures that the pattern has. ## - ## ``captureNameId: Table[string, int]`` - ## a table from the capture names to their numeric id. + ## `captureNameId: Table[string, int]` + ## : a table from the capture names to their numeric id. ## ## ## Options @@ -90,30 +98,30 @@ type ## The following options may appear anywhere in the pattern, and they affect ## the rest of it. ## - ## - ``(?i)`` - case insensitive - ## - ``(?m)`` - multi-line: ``^`` and ``$`` match the beginning and end of + ## - `(?i)` - case insensitive + ## - `(?m)` - multi-line: `^` and `$` match the beginning and end of ## lines, not of the subject string - ## - ``(?s)`` - ``.`` also matches newline (*dotall*) - ## - ``(?U)`` - expressions are not greedy by default. ``?`` can be added + ## - `(?s)` - `.` also matches newline (*dotall*) + ## - `(?U)` - expressions are not greedy by default. `?` can be added ## to a qualifier to make it greedy - ## - ``(?x)`` - whitespace and comments (``#``) are ignored (*extended*) - ## - ``(?X)`` - character escapes without special meaning (``\w`` vs. - ## ``\a``) are errors (*extra*) + ## - `(?x)` - whitespace and comments (`#`) are ignored (*extended*) + ## - `(?X)` - character escapes without special meaning (`\w` vs. + ## `\a`) are errors (*extra*) ## ## One or a combination of these options may appear only at the beginning ## of the pattern: ## - ## - ``(*UTF8)`` - treat both the pattern and subject as UTF-8 - ## - ``(*UCP)`` - Unicode character properties; ``\w`` matches ``я`` - ## - ``(*U)`` - a combination of the two options above - ## - ``(*FIRSTLINE*)`` - fails if there is not a match on the first line - ## - ``(*NO_AUTO_CAPTURE)`` - turn off auto-capture for groups; - ## ``(?<name>...)`` can be used to capture - ## - ``(*CR)`` - newlines are separated by ``\r`` - ## - ``(*LF)`` - newlines are separated by ``\n`` (UNIX default) - ## - ``(*CRLF)`` - newlines are separated by ``\r\n`` (Windows default) - ## - ``(*ANYCRLF)`` - newlines are separated by any of the above - ## - ``(*ANY)`` - newlines are separated by any of the above and Unicode + ## - `(*UTF8)` - treat both the pattern and subject as UTF-8 + ## - `(*UCP)` - Unicode character properties; `\w` matches `я` + ## - `(*U)` - a combination of the two options above + ## - `(*FIRSTLINE*)` - fails if there is not a match on the first line + ## - `(*NO_AUTO_CAPTURE)` - turn off auto-capture for groups; + ## `(?<name>...)` can be used to capture + ## - `(*CR)` - newlines are separated by `\r` + ## - `(*LF)` - newlines are separated by `\n` (UNIX default) + ## - `(*CRLF)` - newlines are separated by `\r\n` (Windows default) + ## - `(*ANYCRLF)` - newlines are separated by any of the above + ## - `(*ANY)` - newlines are separated by any of the above and Unicode ## newlines: ## ## single characters VT (vertical tab, U+000B), FF (form feed, U+000C), @@ -122,16 +130,22 @@ type ## are recognized only in UTF-8 mode. ## — man pcre ## - ## - ``(*JAVASCRIPT_COMPAT)`` - JavaScript compatibility - ## - ``(*NO_STUDY)`` - turn off studying; study is enabled by default + ## - `(*JAVASCRIPT_COMPAT)` - JavaScript compatibility + ## - `(*NO_STUDY)` - turn off studying; study is enabled by default ## ## For more details on the leading option groups, see the `Option - ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`__ + ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`_ ## and the `Newline - ## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`__ + ## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`_ ## sections of the `PCRE syntax - ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`__. - pattern*: string ## not nil + ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`_. + ## + ## Some of these options are not part of PCRE and are converted by nre + ## into PCRE flags. These include `NEVER_UTF`, `ANCHORED`, + ## `DOLLAR_ENDONLY`, `FIRSTLINE`, `NO_AUTO_CAPTURE`, + ## `JAVASCRIPT_COMPAT`, `U`, `NO_STUDY`. In other PCRE wrappers, you + ## will need to pass these as separate flags to PCRE. + pattern*: string pcreObj: ptr pcre.Pcre ## not nil pcreExtra: ptr pcre.ExtraData ## nil @@ -141,57 +155,48 @@ type ## Usually seen as Option[RegexMatch], it represents the result of an ## execution. On failure, it is none, on success, it is some. ## - ## ``pattern: Regex`` - ## the pattern that is being matched + ## `pattern: Regex` + ## : the pattern that is being matched ## - ## ``str: string`` - ## the string that was matched against + ## `str: string` + ## : the string that was matched against ## - ## ``captures[]: string`` - ## the string value of whatever was captured at that id. If the value - ## is invalid, then behavior is undefined. If the id is ``-1``, then + ## `captures[]: string` + ## : the string value of whatever was captured at that id. If the value + ## is invalid, then behavior is undefined. If the id is `-1`, then ## the whole match is returned. If the given capture was not matched, - ## ``nil`` is returned. + ## `nil` is returned. See examples for `match`. ## - ## - ``"abc".match(re"(\w)").captures[0] == "a"`` - ## - ``"abc".match(re"(?<letter>\w)").captures["letter"] == "a"`` - ## - ``"abc".match(re"(\w)\w").captures[-1] == "ab"`` + ## `captureBounds[]: HSlice[int, int]` + ## : gets the bounds of the given capture according to the same rules as + ## the above. If the capture is not filled, then `None` is returned. + ## The bounds are both inclusive. See examples for `match`. ## - ## ``captureBounds[]: Option[Slice[int]]`` - ## gets the bounds of the given capture according to the same rules as - ## the above. If the capture is not filled, then ``None`` is returned. - ## The bounds are both inclusive. + ## `match: string` + ## : the full text of the match. ## - ## - ``"abc".match(re"(\w)").captureBounds[0] == 0 .. 0`` - ## - ``"abc".match(re"").captureBounds[-1] == 0 .. -1`` - ## - ``"abc".match(re"abc").captureBounds[-1] == 0 .. 2`` + ## `matchBounds: HSlice[int, int]` + ## : the bounds of the match, as in `captureBounds[]` ## - ## ``match: string`` - ## the full text of the match. + ## `(captureBounds|captures).toTable` + ## : returns a table with each named capture as a key. ## - ## ``matchBounds: Slice[int]`` - ## the bounds of the match, as in ``captureBounds[]`` + ## `(captureBounds|captures).toSeq` + ## : returns all the captures by their number. ## - ## ``(captureBounds|captures).toTable`` - ## returns a table with each named capture as a key. - ## - ## ``(captureBounds|captures).toSeq`` - ## returns all the captures by their number. - ## - ## ``$: string`` - ## same as ``match`` + ## `$: string` + ## : same as `match` pattern*: Regex ## The regex doing the matching. ## Not nil. str*: string ## The string that was matched against. - ## Not nil. - pcreMatchBounds: seq[Slice[cint]] ## First item is the bounds of the match - ## Other items are the captures - ## `a` is inclusive start, `b` is exclusive end + pcreMatchBounds: seq[HSlice[cint, cint]] ## First item is the bounds of the match + ## Other items are the captures + ## `a` is inclusive start, `b` is exclusive end Captures* = distinct RegexMatch CaptureBounds* = distinct RegexMatch - RegexError* = ref object of Exception + RegexError* = ref object of CatchableError RegexInternalError* = ref object of RegexError ## Internal error in the module, this probably means that there is a bug @@ -207,19 +212,71 @@ type pattern*: string ## the pattern that caused the problem StudyError* = ref object of RegexError - ## Thrown when studying the regular expression failes + ## Thrown when studying the regular expression fails ## for whatever reason. The message contains the error ## code. -# }}} + +proc destroyRegex(pattern: Regex) = + `=destroy`(pattern.pattern) + pcre.free_substring(cast[cstring](pattern.pcreObj)) + if pattern.pcreExtra != nil: + pcre.free_study(pattern.pcreExtra) + `=destroy`(pattern.captureNameToId) proc getinfo[T](pattern: Regex, opt: cint): T = let retcode = pcre.fullinfo(pattern.pcreObj, pattern.pcreExtra, opt, addr result) if retcode < 0: # XXX Error message that doesn't expose implementation details - raise newException(FieldError, "Invalid getinfo for $1, errno $2" % [$opt, $retcode]) + raise newException(FieldDefect, "Invalid getinfo for $1, errno $2" % [$opt, $retcode]) + +proc getNameToNumberTable(pattern: Regex): Table[string, int] = + let entryCount = getinfo[cint](pattern, pcre.INFO_NAMECOUNT) + let entrySize = getinfo[cint](pattern, pcre.INFO_NAMEENTRYSIZE) + let table = cast[ptr UncheckedArray[uint8]]( + getinfo[int](pattern, pcre.INFO_NAMETABLE)) + + result = initTable[string, int]() + + for i in 0 ..< entryCount: + let pos = i * entrySize + let num = (int(table[pos]) shl 8) or int(table[pos + 1]) - 1 + var name = "" + + var idx = 2 + while table[pos + idx] != 0: + name.add(char(table[pos + idx])) + idx += 1 + + result[name] = num + +proc initRegex(pattern: string, flags: int, study = true): Regex = + new(result, destroyRegex) + result.pattern = pattern + + var errorMsg: cstring + var errOffset: cint + + result.pcreObj = pcre.compile(cstring(pattern), + # better hope int is at least 4 bytes.. + cint(flags), addr errorMsg, + addr errOffset, nil) + if result.pcreObj == nil: + # failed to compile + raise SyntaxError(msg: $errorMsg, pos: errOffset, pattern: pattern) + + if study: + var options: cint = 0 + var hasJit: cint + if pcre.config(pcre.CONFIG_JIT, addr hasJit) == 0: + if hasJit == 1'i32: + options = pcre.STUDY_JIT_COMPILE + result.pcreExtra = pcre.study(result.pcreObj, options, addr errorMsg) + if errorMsg != nil: + raise StudyError(msg: $errorMsg) + + result.captureNameToId = result.getNameToNumberTable() -# Regex accessors {{{ proc captureCount*(pattern: Regex): int = return getinfo[cint](pattern, pcre.INFO_CAPTURECOUNT) @@ -231,7 +288,7 @@ proc matchesCrLf(pattern: Regex): bool = let newlineFlags = flags and (pcre.NEWLINE_CRLF or pcre.NEWLINE_ANY or pcre.NEWLINE_ANYCRLF) - if newLineFlags > 0u32: + if newlineFlags > 0u32: return true # get flags from build config @@ -246,101 +303,122 @@ proc matchesCrLf(pattern: Regex): bool = of -2: return true of -1: return true else: return false -# }}} -# Capture accessors {{{ -proc captureBounds*(pattern: RegexMatch): CaptureBounds = return CaptureBounds(pattern) -proc captures*(pattern: RegexMatch): Captures = return Captures(pattern) +func captureBounds*(pattern: RegexMatch): CaptureBounds = return CaptureBounds(pattern) + +func captures*(pattern: RegexMatch): Captures = return Captures(pattern) -proc `[]`*(pattern: CaptureBounds, i: int): Option[Slice[int]] = +func contains*(pattern: CaptureBounds, i: int): bool = let pattern = RegexMatch(pattern) - if pattern.pcreMatchBounds[i + 1].a != -1: - let bounds = pattern.pcreMatchBounds[i + 1] - return some(int(bounds.a) .. int(bounds.b-1)) - else: - return none(Slice[int]) + pattern.pcreMatchBounds[i + 1].a != -1 + +func contains*(pattern: Captures, i: int): bool = + i in CaptureBounds(pattern) -proc `[]`*(pattern: Captures, i: int): string = +func `[]`*(pattern: CaptureBounds, i: int): HSlice[int, int] = + let pattern = RegexMatch(pattern) + if not (i in pattern.captureBounds): + raise newException(IndexDefect, "Group '" & $i & "' was not captured") + + let bounds = pattern.pcreMatchBounds[i + 1] + int(bounds.a)..int(bounds.b-1) + +func `[]`*(pattern: Captures, i: int): string = let pattern = RegexMatch(pattern) let bounds = pattern.captureBounds[i] - if bounds.isSome: - let bounds = bounds.get - return pattern.str.substr(bounds.a, bounds.b) - else: - return nil + pattern.str.substr(bounds.a, bounds.b) -proc match*(pattern: RegexMatch): string = +func match*(pattern: RegexMatch): string = return pattern.captures[-1] -proc matchBounds*(pattern: RegexMatch): Slice[int] = - return pattern.captureBounds[-1].get +func matchBounds*(pattern: RegexMatch): HSlice[int, int] = + return pattern.captureBounds[-1] -proc `[]`*(pattern: CaptureBounds, name: string): Option[Slice[int]] = +func contains*(pattern: CaptureBounds, name: string): bool = let pattern = RegexMatch(pattern) - return pattern.captureBounds[pattern.pattern.captureNameToId.fget(name)] + let nameToId = pattern.pattern.captureNameToId + if not (name in nameToId): + return false + nameToId[name] in pattern.captureBounds -proc `[]`*(pattern: Captures, name: string): string = +func contains*(pattern: Captures, name: string): bool = + name in CaptureBounds(pattern) + +func checkNamedCaptured(pattern: RegexMatch, name: string) = + if not (name in pattern.captureBounds): + raise newException(KeyError, "Group '" & name & "' was not captured") + +func `[]`*(pattern: CaptureBounds, name: string): HSlice[int, int] = + let pattern = RegexMatch(pattern) + checkNamedCaptured(pattern, name) + {.noSideEffect.}: + result = pattern.captureBounds[pattern.pattern.captureNameToId[name]] + +func `[]`*(pattern: Captures, name: string): string = let pattern = RegexMatch(pattern) - return pattern.captures[pattern.pattern.captureNameToId.fget(name)] + checkNamedCaptured(pattern, name) + {.noSideEffect.}: + result = pattern.captures[pattern.pattern.captureNameToId[name]] -template toTableImpl(cond: bool): stmt {.immediate, dirty.} = +template toTableImpl() {.dirty.} = for key in RegexMatch(pattern).pattern.captureNameId.keys: - let nextVal = pattern[key] - if cond: - result[key] = default - else: - result[key] = nextVal + if key in pattern: + result[key] = pattern[key] -proc toTable*(pattern: Captures, default: string = nil): Table[string, string] = +func toTable*(pattern: Captures): Table[string, string] = result = initTable[string, string]() - toTableImpl(nextVal == nil) + toTableImpl() -proc toTable*(pattern: CaptureBounds, default = none(Slice[int])): - Table[string, Option[Slice[int]]] = - result = initTable[string, Option[Slice[int]]]() - toTableImpl(nextVal.isNone) +func toTable*(pattern: CaptureBounds): Table[string, HSlice[int, int]] = + result = initTable[string, HSlice[int, int]]() + toTableImpl() -template itemsImpl(cond: bool): stmt {.immediate, dirty.} = - for i in 0 .. <RegexMatch(pattern).pattern.captureCount: - let nextVal = pattern[i] +template itemsImpl() {.dirty.} = + for i in 0 ..< RegexMatch(pattern).pattern.captureCount: # done in this roundabout way to avoid multiple yields (potential code # bloat) - let nextYieldVal = if cond: default else: nextVal - yield nextYieldVal + let nextYieldVal = if i in pattern: + some(pattern[i]) + else: + default + yield nextYieldVal -iterator items*(pattern: CaptureBounds, default = none(Slice[int])): Option[Slice[int]] = - itemsImpl(nextVal.isNone) +iterator items*(pattern: CaptureBounds, + default = none(HSlice[int, int])): Option[HSlice[int, int]] = + itemsImpl() -iterator items*(pattern: Captures, default: string = nil): string = - itemsImpl(nextVal == nil) +iterator items*(pattern: Captures, + default: Option[string] = none(string)): Option[string] = + itemsImpl() -proc toSeq*(pattern: CaptureBounds, default = none(Slice[int])): seq[Option[Slice[int]]] = - accumulateResult(pattern.items(default)) +proc toSeq*(pattern: CaptureBounds, + default = none(HSlice[int, int])): seq[Option[HSlice[int, int]]] = + result = @[] + for it in pattern.items(default): result.add it -proc toSeq*(pattern: Captures, default: string = nil): seq[string] = - accumulateResult(pattern.items(default)) +proc toSeq*(pattern: Captures, + default: Option[string] = none(string)): seq[Option[string]] = + result = @[] + for it in pattern.items(default): result.add it proc `$`*(pattern: RegexMatch): string = return pattern.captures[-1] proc `==`*(a, b: Regex): bool = if not a.isNil and not b.isNil: - return a.pattern == b.pattern and - a.pcreObj == b.pcreObj and + return a.pattern == b.pattern and + a.pcreObj == b.pcreObj and a.pcreExtra == b.pcreExtra else: return system.`==`(a, b) proc `==`*(a, b: RegexMatch): bool = return a.pattern == b.pattern and - a.str == b.str -# }}} + a.str == b.str -# Creation & Destruction {{{ -# PCRE Options {{{ const PcreOptions = { "NEVER_UTF": pcre.NEVER_UTF, "ANCHORED": pcre.ANCHORED, @@ -396,73 +474,19 @@ proc extractOptions(pattern: string): tuple[pattern: string, flags: int, study: result.pattern.add pattern[optionStart .. pattern.high] -# }}} - -type UncheckedArray {.unchecked.}[T] = array[0 .. 0, T] - -proc destroyRegex(pattern: Regex) = - pcre.free_substring(cast[cstring](pattern.pcreObj)) - pattern.pcreObj = nil - if pattern.pcreExtra != nil: - pcre.free_study(pattern.pcreExtra) - -proc getNameToNumberTable(pattern: Regex): Table[string, int] = - let entryCount = getinfo[cint](pattern, pcre.INFO_NAMECOUNT) - let entrySize = getinfo[cint](pattern, pcre.INFO_NAMEENTRYSIZE) - let table = cast[ptr UncheckedArray[uint8]]( - getinfo[int](pattern, pcre.INFO_NAMETABLE)) - - result = initTable[string, int]() - - for i in 0 .. <entryCount: - let pos = i * entrySize - let num = (int(table[pos]) shl 8) or int(table[pos + 1]) - 1 - var name = "" - - var idx = 2 - while table[pos + idx] != 0: - name.add(char(table[pos + idx])) - idx += 1 - - result[name] = num - -proc initRegex(pattern: string, flags: int, study = true): Regex = - new(result, destroyRegex) - result.pattern = pattern - - var errorMsg: cstring - var errOffset: cint - - result.pcreObj = pcre.compile(cstring(pattern), - # better hope int is at least 4 bytes.. - cint(flags), addr errorMsg, - addr errOffset, nil) - if result.pcreObj == nil: - # failed to compile - raise SyntaxError(msg: $errorMsg, pos: errOffset, pattern: pattern) - - if study: - # XXX investigate JIT - result.pcreExtra = pcre.study(result.pcreObj, 0x0, addr errorMsg) - if errorMsg != nil: - raise StudyError(msg: $errorMsg) - - result.captureNameToId = result.getNameToNumberTable() - proc re*(pattern: string): Regex = let (pattern, flags, study) = extractOptions(pattern) initRegex(pattern, flags, study) -# }}} -# Operations {{{ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Option[RegexMatch] = - var myResult = RegexMatch(pattern : pattern, str : str) + var myResult = RegexMatch(pattern: pattern, str: str) # See PCRE man pages. # 2x capture count to make room for start-end pairs # 1x capture count as slack space for PCRE let vecsize = (pattern.captureCount() + 1) * 3 # div 2 because each element is 2 cints long - myResult.pcreMatchBounds = newSeq[Slice[cint]](ceil(vecsize / 2).int) + # plus 1 because we need the ceiling, not the floor + myResult.pcreMatchBounds = newSeq[HSlice[cint, cint]]((vecsize + 1) div 2) myResult.pcreMatchBounds.setLen(vecsize div 3) let strlen = if endpos == int.high: str.len else: endpos+1 @@ -483,54 +507,67 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt of pcre.ERROR_NOMATCH: return none(RegexMatch) of pcre.ERROR_NULL: - raise newException(AccessViolationError, "Expected non-null parameters") + raise newException(AccessViolationDefect, "Expected non-null parameters") of pcre.ERROR_BADOPTION: - raise RegexInternalError(msg : "Unknown pattern flag. Either a bug or " & + raise RegexInternalError(msg: "Unknown pattern flag. Either a bug or " & "outdated PCRE.") of pcre.ERROR_BADUTF8, pcre.ERROR_SHORTUTF8, pcre.ERROR_BADUTF8_OFFSET: - raise InvalidUnicodeError(msg : "Invalid unicode byte sequence", - pos : myResult.pcreMatchBounds[0].a) + raise InvalidUnicodeError(msg: "Invalid unicode byte sequence", + pos: myResult.pcreMatchBounds[0].a) else: - raise RegexInternalError(msg : "Unknown internal error: " & $execRet) + raise RegexInternalError(msg: "Unknown internal error: " & $execRet) proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = - ## Like ```find(...)`` <#proc-find>`__, but anchored to the start of the - ## string. This means that ``"foo".match(re"f") == true``, but - ## ``"foo".match(re"o") == false``. + ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the + ## string. + runnableExamples: + assert "foo".match(re"f").isSome + assert "foo".match(re"o").isNone + + assert "abc".match(re"(\w)").get.captures[0] == "a" + assert "abc".match(re"(?<letter>\w)").get.captures["letter"] == "a" + assert "abc".match(re"(\w)\w").get.captures[-1] == "ab" + + assert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 + assert 0 in "abc".match(re"(\w)").get.captureBounds + assert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 + assert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = - ## Works the same as ```find(...)`` <#proc-find>`__, but finds every - ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not - ## ``"22", "22", "22"``. - ## - ## Arguments are the same as ```find(...)`` <#proc-find>`__ + ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every + ## non-overlapping match: + runnableExamples: + import std/sugar + assert collect(for a in "2222".findIter(re"22"): a.match) == @["22", "22"] + # not @["22", "22", "22"] + ## Arguments are the same as `find(...)<#find,string,Regex,int>`_ ## ## Variants: ## - ## - ``proc findAll(...)`` returns a ``seq[string]`` - # see pcredemo for explaination + ## - `proc findAll(...)` returns a `seq[string]` + # see pcredemo for explanation => https://www.pcre.org/original/doc/html/pcredemo.html let matchesCrLf = pattern.matchesCrLf() let unicode = uint32(getinfo[culong](pattern, pcre.INFO_OPTIONS) and pcre.UTF8) > 0u32 let strlen = if endpos == int.high: str.len else: endpos+1 - var offset = start var match: Option[RegexMatch] + var neverMatched = true + while true: var flags = 0 - if match.isSome and match.get.matchBounds.a > match.get.matchBounds.b: # 0-len match flags = pcre.NOTEMPTY_ATSTART - match = str.matchImpl(pattern, offset, endpos, flags) if match.isNone: # either the end of the input or the string - # cannot be split here - if offset >= strlen: + # cannot be split here - we also need to bail + # if we've never matched and we've already tried to... + if flags == 0 or offset >= strlen or neverMatched: # All matches found break if matchesCrLf and offset < (str.len - 1) and @@ -544,21 +581,20 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R else: offset += 1 else: + neverMatched = false offset = match.get.matchBounds.b + 1 - yield match.get - proc find*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = ## Finds the given pattern in the string between the end and start ## positions. ## - ## ``start`` - ## The start point at which to start matching. ``|abc`` is ``0``; - ## ``a|bc`` is ``1`` + ## `start` + ## : The start point at which to start matching. `|abc` is `0`; + ## `a|bc` is `1` ## - ## ``endpos`` - ## The maximum index for a match; ``int.high`` means the end of the + ## `endpos` + ## : The maximum index for a match; `int.high` means the end of the ## string, otherwise it’s an inclusive upper bound. return str.matchImpl(pattern, start, endpos, 0) @@ -567,22 +603,36 @@ proc findAll*(str: string, pattern: Regex, start = 0, endpos = int.high): seq[st for match in str.findIter(pattern, start, endpos): result.add(match.match) +proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool = + ## Determine if the string contains the given pattern between the end and + ## start positions: + ## This function is equivalent to `isSome(str.find(pattern, start, endpos))`. + runnableExamples: + assert "abc".contains(re"bc") + assert not "abc".contains(re"cd") + assert not "abc".contains(re"a", start = 1) + + return isSome(str.find(pattern, start, endpos)) + proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] = ## Splits the string with the given regex. This works according to the - ## rules that Perl and Javascript use: + ## rules that Perl and Javascript use. ## - ## - If the match is zero-width, then the string is still split: - ## ``"123".split(r"") == @["1", "2", "3"]``. + ## `start` behaves the same as in `find(...)<#find,string,Regex,int>`_. ## - ## - If the pattern has a capture in it, it is added after the string - ## split: ``"12".split(re"(\d)") == @["", "1", "", "2", ""]``. - ## - ## - If ``maxsplit != -1``, then the string will only be split - ## ``maxsplit - 1`` times. This means that there will be ``maxsplit`` - ## strings in the output seq. - ## ``"1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"]`` - ## - ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`__. + runnableExamples: + # - If the match is zero-width, then the string is still split: + assert "123".split(re"") == @["1", "2", "3"] + + # - If the pattern has a capture in it, it is added after the string + # split: + assert "12".split(re"(\d)") == @["", "1", "", "2", ""] + + # - If `maxsplit != -1`, then the string will only be split + # `maxsplit - 1` times. This means that there will be `maxsplit` + # strings in the output seq. + assert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] + result = @[] var lastIdx = start var splits = 0 @@ -610,7 +660,8 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] for cap in match.captures: # if there are captures, include them in the result - result.add(cap) + if cap.isSome: + result.add(cap.get) if splits == maxSplit - 1: break @@ -626,7 +677,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] result.add(str.substr(bounds.b + 1, str.high)) template replaceImpl(str: string, pattern: Regex, - replacement: expr): stmt {.immediate, dirty.} = + replacement: untyped) {.dirty.} = # XXX seems very similar to split, maybe I can reduce code duplication # somehow? result = "" @@ -635,7 +686,6 @@ template replaceImpl(str: string, pattern: Regex, let bounds = match.matchBounds result.add(str.substr(lastIdx, bounds.a - 1)) let nextVal = replacement - assert(nextVal != nil) result.add(nextVal) lastIdx = bounds.b + 1 @@ -645,27 +695,27 @@ template replaceImpl(str: string, pattern: Regex, proc replace*(str: string, pattern: Regex, subproc: proc (match: RegexMatch): string): string = - ## Replaces each match of Regex in the string with ``sub``, which should - ## never be or return ``nil``. + ## Replaces each match of Regex in the string with `subproc`, which should + ## never be or return `nil`. ## - ## If ``sub`` is a ``proc (RegexMatch): string``, then it is executed with + ## If `subproc` is a `proc (RegexMatch): string`, then it is executed with ## each match and the return value is the replacement value. ## - ## If ``sub`` is a ``proc (string): string``, then it is executed with the - ## full text of the match and and the return value is the replacement - ## value. + ## If `subproc` is a `proc (string): string`, then it is executed with the + ## full text of the match and the return value is the replacement value. ## - ## If ``sub`` is a string, the syntax is as follows: + ## If `subproc` is a string, the syntax is as follows: ## - ## - ``$$`` - literal ``$`` - ## - ``$123`` - capture number ``123`` - ## - ``$foo`` - named capture ``foo`` - ## - ``${foo}`` - same as above - ## - ``$1$#`` - first and second captures - ## - ``$#`` - first capture - ## - ``$0`` - full match + ## - `$$` - literal `$` + ## - `$123` - capture number `123` + ## - `$foo` - named capture `foo` + ## - `${foo}` - same as above + ## - `$1$#` - first and second captures + ## - `$#` - first capture + ## - `$0` - full match ## - ## If a given capture is missing, a ``ValueError`` exception is thrown. + ## If a given capture is missing, `IndexDefect` thrown for un-named captures + ## and `KeyError` for named captures. replaceImpl(str, pattern, subproc(match)) proc replace*(str: string, pattern: Regex, @@ -677,10 +727,25 @@ proc replace*(str: string, pattern: Regex, sub: string): string = replaceImpl(str, pattern, formatStr(sub, match.captures[name], match.captures[id - 1])) -# }}} - -let SpecialCharMatcher = re"([\\+*?[^\]$(){}=!<>|:-])" -proc escapeRe*(str: string): string = - ## Escapes the string so it doesn’t match any special characters. - ## Incompatible with the Extra flag (``X``). - str.replace(SpecialCharMatcher, "\\$1") +proc escapeRe*(str: string): string {.gcsafe.} = + ## Escapes the string so it doesn't match any special characters. + ## Incompatible with the Extra flag (`X`). + ## + ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -` + runnableExamples: + assert escapeRe("fly+wind") == "fly\\+wind" + assert escapeRe("!") == "\\!" + assert escapeRe("nim*") == "nim\\*" + + #([\\+*?[^\]$(){}=!<>|:-]) + const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(', + ')', '{', '}', '=', '!', '<', '>', '|', ':', + '-'} + + for c in items(str): + case c + of SpecialCharMatcher: + result.add("\\") + result.add(c) + else: + result.add(c) diff --git a/lib/impure/nre/private/util.nim b/lib/impure/nre/private/util.nim index 253bfada7..ed8420776 100644 --- a/lib/impure/nre/private/util.nim +++ b/lib/impure/nre/private/util.nim @@ -1,22 +1,10 @@ ## INTERNAL FILE FOR USE ONLY BY nre.nim. -import tables - -proc fget*[K, V](self: Table[K, V], key: K): V = - if self.hasKey(key): - return self[key] - else: - raise newException(KeyError, "Key does not exist in table: " & $key) +import std/tables const Ident = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} const StartIdent = Ident - {'0'..'9'} -proc checkNil(arg: string): string = - if arg == nil: - raise newException(ValueError, "Cannot use nil capture") - else: - return arg - -template formatStr*(howExpr, namegetter, idgetter: expr): expr = +template formatStr*(howExpr, namegetter, idgetter): untyped = let how = howExpr var val = newStringOfCap(how.len) var i = 0 @@ -32,7 +20,7 @@ template formatStr*(howExpr, namegetter, idgetter: expr): expr = i += 2 elif how[i + 1] == '#': var id {.inject.} = lastNum - val.add(checkNil(idgetter)) + val.add(idgetter) lastNum += 1 i += 2 elif how[i + 1] in {'0'..'9'}: @@ -41,7 +29,7 @@ template formatStr*(howExpr, namegetter, idgetter: expr): expr = while i < how.len and how[i] in {'0'..'9'}: id += (id * 10) + (ord(how[i]) - ord('0')) i += 1 - val.add(checkNil(idgetter)) + val.add(idgetter) lastNum = id + 1 elif how[i + 1] in StartIdent: i += 1 @@ -49,7 +37,7 @@ template formatStr*(howExpr, namegetter, idgetter: expr): expr = while i < how.len and how[i] in Ident: name.add(how[i]) i += 1 - val.add(checkNil(namegetter)) + val.add(namegetter) elif how[i + 1] == '{': i += 2 var name {.inject.} = "" @@ -57,7 +45,7 @@ template formatStr*(howExpr, namegetter, idgetter: expr): expr = name.add(how[i]) i += 1 i += 1 - val.add(checkNil(namegetter)) + val.add(namegetter) else: - raise newException(Exception, "Syntax error in format string at " & $i) + raise newException(ValueError, "Syntax error in format string at " & $i) val diff --git a/lib/impure/osinfo_posix.nim b/lib/impure/osinfo_posix.nim deleted file mode 100644 index 0362fca12..000000000 --- a/lib/impure/osinfo_posix.nim +++ /dev/null @@ -1,10 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.error: "This module has been moved to the 'osinfo' nimble package.".} diff --git a/lib/impure/osinfo_win.nim b/lib/impure/osinfo_win.nim deleted file mode 100644 index 0362fca12..000000000 --- a/lib/impure/osinfo_win.nim +++ /dev/null @@ -1,10 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.error: "This module has been moved to the 'osinfo' nimble package.".} diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index b373859f4..f4fc26380 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -9,134 +9,66 @@ ## This module contains code for reading from `stdin`:idx:. On UNIX the ## linenoise library is wrapped and set up to provide default key bindings -## (e.g. you can navigate with the arrow keys). On Windows ``system.readLine`` +## (e.g. you can navigate with the arrow keys). On Windows `system.readLine` ## is used. This suffices because Windows' console already provides the ## wanted functionality. -{.deadCodeElim: on.} +runnableExamples("-r:off"): + echo readLineFromStdin("Is Nim awesome? (Y/n): ") + var line: string + while true: + let ok = readLineFromStdin("How are you? ", line) + if not ok: break # ctrl-C or ctrl-D will cause a break + if line.len > 0: echo line + echo "exiting" -when defined(Windows): - proc readLineFromStdin*(prompt: string): TaintedString {. + +when defined(windows): + when defined(nimPreviewSlimSystem): + import std/syncio + + proc readLineFromStdin*(prompt: string): string {. tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a line from stdin. stdout.write(prompt) + stdout.flushFile() result = readLine(stdin) - proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. + proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a `line` from stdin. `line` must not be - ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ``CR``, ``LF`` or - ## ``CRLF``. The newline character(s) are not part of the returned string. - ## Returns ``false`` if the end of the file has been reached, ``true`` - ## otherwise. If ``false`` is returned `line` contains no new data. + ## `nil`! May throw an IO exception. + ## A line of text may be delimited by `CR`, `LF` or + ## `CRLF`. The newline character(s) are not part of the returned string. + ## Returns `false` if the end of the file has been reached, `true` + ## otherwise. If `false` is returned `line` contains no new data. stdout.write(prompt) result = readLine(stdin, line) - import winlean - - const - VK_SHIFT* = 16 - VK_CONTROL* = 17 - VK_MENU* = 18 - KEY_EVENT* = 1 - - type - KEY_EVENT_RECORD = object - bKeyDown: WinBool - wRepeatCount: uint16 - wVirtualKeyCode: uint16 - wVirtualScanCode: uint16 - unicodeChar: uint16 - dwControlKeyState: uint32 - INPUT_RECORD = object - eventType*: int16 - reserved*: int16 - event*: KEY_EVENT_RECORD - safetyBuffer: array[0..5, DWORD] - - proc readConsoleInputW*(hConsoleInput: THANDLE, lpBuffer: var INPUTRECORD, - nLength: uint32, - lpNumberOfEventsRead: var uint32): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} - - proc getch(): uint16 = - let hStdin = getStdHandle(STD_INPUT_HANDLE) - var - irInputRecord: INPUT_RECORD - dwEventsRead: uint32 - - while readConsoleInputW(hStdin, irInputRecord, 1, dwEventsRead) != 0: - if irInputRecord.eventType == KEY_EVENT and - irInputRecord.event.wVirtualKeyCode notin {VK_SHIFT, VK_MENU, VK_CONTROL}: - result = irInputRecord.event.unicodeChar - discard readConsoleInputW(hStdin, irInputRecord, 1, dwEventsRead) - return result - - from unicode import toUTF8, Rune, runeLenAt +elif defined(genode): + proc readLineFromStdin*(prompt: string): string {. + tags: [ReadIOEffect, WriteIOEffect].} = + stdin.readLine() - proc readPasswordFromStdin*(prompt: string, password: var TaintedString): - bool {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Reads a `password` from stdin without printing it. `password` must not - ## be ``nil``! Returns ``false`` if the end of the file has been reached, - ## ``true`` otherwise. - password.setLen(0) - stdout.write(prompt) - while true: - let c = getch() - case c.char - of '\r', chr(0xA): - break - of '\b': - # ensure we delete the whole UTF-8 character: - var i = 0 - var x = 1 - while i < password.len: - x = runeLenAt(password, i) - inc i, x - password.setLen(max(password.len - x, 0)) - else: - password.add(toUTF8(c.Rune)) - stdout.write "\n" + proc readLineFromStdin*(prompt: string, line: var string): bool {. + tags: [ReadIOEffect, WriteIOEffect].} = + stdin.readLine(line) else: - import linenoise, termios, unsigned - - proc readLineFromStdin*(prompt: string): TaintedString {. - tags: [ReadIOEffect, WriteIOEffect].} = - var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) - result = TaintedString($buffer) - if result.string.len > 0: - historyAdd(buffer) - linenoise.free(buffer) + import std/linenoise - proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. + proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) - line = TaintedString($buffer) - if line.string.len > 0: + if isNil(buffer): + line.setLen(0) + return false + line = $buffer + if line.len > 0: historyAdd(buffer) linenoise.free(buffer) - # XXX how to determine CTRL+D? result = true - proc readPasswordFromStdin*(prompt: string, password: var TaintedString): - bool {.tags: [ReadIOEffect, WriteIOEffect].} = - password.setLen(0) - let fd = stdin.getFileHandle() - var cur, old: Termios - discard fd.tcgetattr(cur.addr) - old = cur - cur.c_lflag = cur.c_lflag and not Cflag(ECHO) - discard fd.tcsetattr(TCSADRAIN, cur.addr) - stdout.write prompt - result = stdin.readLine(password) - stdout.write "\n" - discard fd.tcsetattr(TCSADRAIN, old.addr) - -proc readPasswordFromStdin*(prompt: string): TaintedString = - ## Reads a password from stdin without printing it. - result = TaintedString("") - discard readPasswordFromStdin(prompt, result) + proc readLineFromStdin*(prompt: string): string {.inline.} = + if not readLineFromStdin(prompt, result): + raise newException(IOError, "Linenoise returned nil") diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 60bb6c77f..053c6ab55 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -7,42 +7,53 @@ # distribution, for details about the copyright. # -## Regular expression support for Nim. Deprecated. Consider using the ``nre`` -## or ``pegs`` modules instead. -## -## **Note:** The 're' proc defaults to the **extended regular expression -## syntax** which lets you use whitespace freely to make your regexes readable. -## However, this means to match whitespace ``\s`` or something similar has -## to be used. +when defined(js): + {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE; See jsre for JS backend.".} + +## Regular expression support for Nim. ## ## This module is implemented by providing a wrapper around the -## `PRCE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_ -## C library. This means that your application will depend on the PRCE +## `PCRE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_ +## C library. This means that your application will depend on the PCRE ## library's licence when using this module, which should not be a problem ## though. -## PRCE's licence follows: ## -## .. include:: ../doc/regexprs.txt +## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre) +## and [regex](https://github.com/nitely/nim-regex). +## +## PCRE's licence follows: ## +## .. include:: ../../doc/regexprs.txt +## + +runnableExamples: + ## Unless specified otherwise, `start` parameter in each proc indicates + ## where the scan starts, but outputs are relative to the start of the input + ## string, not to `start`: + doAssert find("uxabc", re"(?<=x|y)ab", start = 1) == 2 # lookbehind assertion + doAssert find("uxabc", re"ab", start = 3) == -1 # we're past `start` => not found + doAssert not match("xabc", re"^abc$", start = 1) + # can't match start of string since we're starting at 1 import - pcre, strutils, rtarrays + std/[pcre, strutils, rtarrays] -{.deprecated.} +when defined(nimPreviewSlimSystem): + import std/syncio const MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that can be captured. - ## This limit still exists for ``replacef`` and ``parallelReplace``. + ## This limit still exists for `replacef` and `parallelReplace`. type RegexFlag* = enum ## options for regular expressions - reIgnoreCase = 0, ## do caseless matching - reMultiLine = 1, ## ``^`` and ``$`` match newlines within data - reDotAll = 2, ## ``.`` matches anything including NL - reExtended = 3, ## ignore whitespace and ``#`` comments - reStudy = 4 ## study the expression (may be omitted if the - ## expression will be used only once) + reIgnoreCase = 0, ## do caseless matching + reMultiLine = 1, ## `^` and `$` match newlines within data + reDotAll = 2, ## `.` matches anything including NL + reExtended = 3, ## ignore whitespace and `#` comments + reStudy = 4 ## study the expression (may be omitted if the + ## expression will be used only once) RegexDesc = object h: ptr Pcre @@ -53,8 +64,17 @@ type RegexError* = object of ValueError ## is raised if the pattern is no valid regular expression. -{.deprecated: [TRegexFlag: RegexFlag, TRegexDesc: RegexDesc, TRegex: Regex, - EInvalidRegEx: RegexError].} +when defined(gcDestructors): + when defined(nimAllowNonVarDestructor): + proc `=destroy`(x: RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) + else: + proc `=destroy`(x: var RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = var e: ref RegexError @@ -64,76 +84,134 @@ proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = proc rawCompile(pattern: string, flags: cint): ptr Pcre = var - msg: cstring - offset: cint + msg: cstring = "" + offset: cint = 0 result = pcre.compile(pattern, flags, addr(msg), addr(offset), nil) if result == nil: raiseInvalidRegex($msg & "\n" & pattern & "\n" & spaces(offset) & "^\n") proc finalizeRegEx(x: Regex) = # XXX This is a hack, but PCRE does not export its "free" function properly. - # Sigh. The hack relies on PCRE's implementation (see ``pcre_get.c``). + # Sigh. The hack relies on PCRE's implementation (see `pcre_get.c`). # Fortunately the implementation is unlikely to change. pcre.free_substring(cast[cstring](x.h)) if not isNil(x.e): - pcre.free_substring(cast[cstring](x.e)) + pcre.free_study(x.e) -proc re*(s: string, flags = {reExtended, reStudy}): Regex {.deprecated.} = - ## Constructor of regular expressions. Note that Nim's - ## extended raw string literals support this syntax ``re"[abc]"`` as - ## a short form for ``re(r"[abc]")``. - new(result, finalizeRegEx) +proc re*(s: string, flags = {reStudy}): Regex = + ## Constructor of regular expressions. + ## + ## Note that Nim's + ## extended raw string literals support the syntax `re"[abc]"` as + ## a short form for `re(r"[abc]")`. Also note that since this + ## compiles the regular expression, which is expensive, you should + ## avoid putting it directly in the arguments of the functions like + ## the examples show below if you plan to use it a lot of times, as + ## this will hurt performance immensely. (e.g. outside a loop, ...) + when defined(gcDestructors): + result = Regex() + else: + new(result, finalizeRegEx) result.h = rawCompile(s, cast[cint](flags - {reStudy})) if reStudy in flags: - var msg: cstring - result.e = pcre.study(result.h, 0, addr msg) + var msg: cstring = "" + var options: cint = 0 + var hasJit: cint = 0 + if pcre.config(pcre.CONFIG_JIT, addr hasJit) == 0: + if hasJit == 1'i32: + options = pcre.STUDY_JIT_COMPILE + result.e = pcre.study(result.h, options, addr msg) if not isNil(msg): raiseInvalidRegex($msg) -proc matchOrFind(s: string, pattern: Regex, matches: var openArray[string], - start, flags: cint): cint = +proc rex*(s: string, flags = {reStudy, reExtended}): Regex = + ## Constructor for extended regular expressions. + ## + ## The extended means that comments starting with `#` and + ## whitespace are ignored. + result = re(s, flags) + +proc bufSubstr(b: cstring, sPos, ePos: int): string {.inline.} = + ## Return a Nim string built from a slice of a cstring buffer. + ## Don't assume cstring is '\0' terminated + let sz = ePos - sPos + result = newString(sz+1) + copyMem(addr(result[0]), unsafeAddr(b[sPos]), sz) + result.setLen(sz) + +proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string], + start, bufSize, flags: cint): cint = var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start, flags, + res = pcre.exec(pattern.h, pattern.e, buf, bufSize, start, flags, cast[ptr cint](rawMatches), (matches.len+1).cint*3) if res < 0'i32: return res for i in 1..int(res)-1: var a = rawMatches[i * 2] var b = rawMatches[i * 2 + 1] - if a >= 0'i32: matches[i-1] = substr(s, int(a), int(b)-1) - else: matches[i-1] = nil + if a >= 0'i32: + matches[i-1] = bufSubstr(buf, int(a), int(b)) + else: matches[i-1] = "" return rawMatches[1] - rawMatches[0] -proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], - start = 0): tuple[first, last: int] = - ## returns the starting position and end position of `pattern` in `s` +const MaxReBufSize* = high(cint) + ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even + ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1. + +proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), ## and the captured ## substrings in the array `matches`. If it does not match, nothing - ## is written into `matches` and ``(-1,0)`` is returned. + ## is written into `matches` and `(-1,0)` is returned. + ## + ## Note: The memory for `matches` needs to be allocated before this function is + ## called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32, + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, cast[ptr cint](rawMatches), (matches.len+1).cint*3) if res < 0'i32: return (-1, 0) for i in 1..int(res)-1: var a = rawMatches[i * 2] var b = rawMatches[i * 2 + 1] - if a >= 0'i32: matches[i-1] = substr(s, int(a), int(b)-1) - else: matches[i-1] = nil + if a >= 0'i32: matches[i-1] = bufSubstr(buf, int(a), int(b)) + else: matches[i-1] = "" return (rawMatches[0].int, rawMatches[1].int - 1) -proc findBounds*(s: string, pattern: Regex, +proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): tuple[first, last: int] {.inline.} = + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing + ## is written into `matches` and `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[string](1) + let (first, last) = findBounds("Hello World", re"(W\w+)", matches) + doAssert first == 6 + doAssert last == 10 + doAssert matches[0] == "World" + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[tuple[first, last: int]], - start = 0): tuple[first, last: int] = - ## returns the starting position and end position of ``pattern`` in ``s`` + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and - ## ``(-1,0)`` is returned. + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32, + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, cast[ptr cint](rawMatches), (matches.len+1).cint*3) if res < 0'i32: return (-1, 0) for i in 1..int(res)-1: @@ -144,80 +222,188 @@ proc findBounds*(s: string, pattern: Regex, return (rawMatches[0].int, rawMatches[1].int - 1) proc findBounds*(s: string, pattern: Regex, - start = 0): tuple[first, last: int] = - ## returns the starting position and end position of ``pattern`` in ``s``. - ## If it does not match, ``(-1,0)`` is returned. + matches: var openArray[tuple[first, last: int]], + start = 0): tuple[first, last: int] {.inline.} = + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[tuple[first, last: int]](1) + let (first, last) = findBounds("Hello World", re"(\w+)", matches) + doAssert first == 0 + doAssert last == 4 + doAssert matches[0] == (0, 4) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc findBoundsImpl(buf: cstring, pattern: Regex, + start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = + var rtarray = initRtArray[cint](3) + let rawMatches = rtarray.getRawData + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32, + cast[ptr cint](rawMatches), 3) + + if res < 0'i32: + result = (-1, 0) + else: + result = (int(rawMatches[0]), int(rawMatches[1]-1)) + +proc findBounds*(buf: cstring, pattern: Regex, + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the `first` and `last` position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `(-1,0)` is returned. var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, nil, s, len(s).cint, start.cint, 0'i32, + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, cast[ptr cint](rawMatches), 3) if res < 0'i32: return (int(res), 0) return (int(rawMatches[0]), int(rawMatches[1]-1)) -proc matchOrFind(s: string, pattern: Regex, start, flags: cint): cint = +proc findBounds*(s: string, pattern: Regex, + start = 0): tuple[first, last: int] {.inline.} = + ## returns the `first` and `last` position of `pattern` in `s`. + ## If it does not match, `(-1,0)` is returned. + ## + ## Note: there is a speed improvement if the matches do not need to be captured. + runnableExamples: + assert findBounds("01234abc89", re"abc") == (5,7) + result = findBounds(cstring(s), pattern, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint = var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData - result = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start, flags, + result = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags, cast[ptr cint](rawMatches), 3) if result >= 0'i32: result = rawMatches[1] - rawMatches[0] proc matchLen*(s: string, pattern: Regex, matches: var openArray[string], - start = 0): int = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, -1 is returned. Note that a match length + start = 0): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. - return matchOrFind(s, pattern, matches, start.cint, pcre.ANCHORED) + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED) -proc matchLen*(s: string, pattern: Regex, start = 0): int = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, -1 is returned. Note that a match length +proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length + ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED) + +proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length + ## of zero can happen. + ## + runnableExamples: + doAssert matchLen("abcdefg", re"cde", 2) == 3 + doAssert matchLen("abcdefg", re"abcde") == 5 + doAssert matchLen("abcdefg", re"cde") == -1 + result = matchOrFind(cstring(s), pattern, start.cint, s.len.cint, pcre.ANCHORED) + +proc matchLen*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. - return matchOrFind(s, pattern, start.cint, pcre.ANCHORED) + result = matchOrFind(buf, pattern, start.cint, bufSize, pcre.ANCHORED) -proc match*(s: string, pattern: Regex, start = 0): bool = - ## returns ``true`` if ``s[start..]`` matches the ``pattern``. - result = matchLen(s, pattern, start) != -1 +proc match*(s: string, pattern: Regex, start = 0): bool {.inline.} = + ## returns `true` if `s[start..]` matches the `pattern`. + result = matchLen(cstring(s), pattern, start, s.len) != -1 proc match*(s: string, pattern: Regex, matches: var openArray[string], - start = 0): bool = - ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is + start = 0): bool {.inline.} = + ## returns `true` if `s[start..]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is + ## returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + import std/sequtils + var matches: array[2, string] + if match("abcdefg", re"c(d)ef(g)", matches, 2): + doAssert toSeq(matches) == @["d", "g"] + result = matchLen(cstring(s), pattern, matches, start, s.len) != -1 + +proc match*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): bool {.inline.} = + ## returns `true` if `buf[start..<bufSize]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is ## returned. - result = matchLen(s, pattern, matches, start) != -1 + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + result = matchLen(buf, pattern, matches, start, bufSize) != -1 -proc find*(s: string, pattern: Regex, matches: var openArray[string], - start = 0): int = - ## returns the starting position of ``pattern`` in ``s`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and -1 is returned. +proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): int = + ## returns the starting position of `pattern` in `buf` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32, + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, cast[ptr cint](rawMatches), (matches.len+1).cint*3) if res < 0'i32: return res for i in 1..int(res)-1: var a = rawMatches[i * 2] var b = rawMatches[i * 2 + 1] - if a >= 0'i32: matches[i-1] = substr(s, int(a), int(b)-1) - else: matches[i-1] = nil + if a >= 0'i32: matches[i-1] = bufSubstr(buf, int(a), int(b)) + else: matches[i-1] = "" return rawMatches[0] -proc find*(s: string, pattern: Regex, start = 0): int = - ## returns the starting position of ``pattern`` in ``s``. If it does not - ## match, -1 is returned. +proc find*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): int {.inline.} = + ## returns the starting position of `pattern` in `s` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + result = find(cstring(s), pattern, matches, start, s.len) + +proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = + ## returns the starting position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `-1` is returned. var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, nil, s, len(s).cint, start.cint, 0'i32, + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, cast[ptr cint](rawMatches), 3) if res < 0'i32: return res return rawMatches[0] +proc find*(s: string, pattern: Regex, start = 0): int {.inline.} = + ## returns the starting position of `pattern` in `s`. If it does not + ## match, `-1` is returned. We start the scan at `start`. + runnableExamples: + doAssert find("abcdefg", re"cde") == 2 + doAssert find("abcdefg", re"abc") == 0 + doAssert find("abcdefg", re"zz") == -1 # not found + doAssert find("abcdefg", re"cde", start = 2) == 2 # still 2 + doAssert find("abcdefg", re"cde", start = 3) == -1 # we're past the start position + doAssert find("xabc", re"(?<=x|y)abc", start = 1) == 1 + # lookbehind assertion `(?<=x|y)` can look behind `start` + result = find(cstring(s), pattern, start, s.len) + iterator findAll*(s: string, pattern: Regex, start = 0): string = ## Yields all matching *substrings* of `s` that match `pattern`. ## @@ -237,32 +423,47 @@ iterator findAll*(s: string, pattern: Regex, start = 0): string = yield substr(s, int(a), int(b)-1) i = b -proc findAll*(s: string, pattern: Regex, start = 0): seq[string] = - ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. - accumulateResult(findAll(s, pattern, start)) - -when not defined(nimhygiene): - {.pragma: inject.} - -template `=~` *(s: string, pattern: Regex): expr = - ## This calls ``match`` with an implicit declared ``matches`` array that - ## can be used in the scope of the ``=~`` call: - ## - ## .. code-block:: nim - ## - ## if line =~ re"\s*(\w+)\s*\=\s*(\w+)": - ## # matches a key=value pair: - ## echo("Key: ", matches[0]) - ## echo("Value: ", matches[1]) - ## elif line =~ re"\s*(\#.*)": - ## # matches a comment - ## # note that the implicit ``matches`` array is different from the - ## # ``matches`` array of the first branch - ## echo("comment: ", matches[0]) - ## else: - ## echo("syntax error") +iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string = + ## Yields all matching `substrings` of `s` that match `pattern`. ## + ## Note that since this is an iterator you should not modify the string you + ## are iterating over: bad things could happen. + var + i = int32(start) + rtarray = initRtArray[cint](3) + rawMatches = rtarray.getRawData + while true: + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, i, 0'i32, + cast[ptr cint](rawMatches), 3) + if res < 0'i32: break + let a = rawMatches[0] + let b = rawMatches[1] + if a == b and a == i: break + var str = newString(b-a) + copyMem(str[0].addr, unsafeAddr(buf[a]), b-a) + yield str + i = b + +proc findAll*(s: string, pattern: Regex, start = 0): seq[string] {.inline.} = + ## returns all matching `substrings` of `s` that match `pattern`. + ## If it does not match, `@[]` is returned. + result = @[] + for x in findAll(s, pattern, start): result.add x + +template `=~` *(s: string, pattern: Regex): untyped = + ## This calls `match` with an implicit declared `matches` array that + ## can be used in the scope of the `=~` call: + runnableExamples: + proc parse(line: string): string = + if line =~ re"\s*(\w+)\s*\=\s*(\w+)": # matches a key=value pair: + result = $(matches[0], matches[1]) + elif line =~ re"\s*(\#.*)": # matches a comment + # note that the implicit `matches` array is different from 1st branch + result = $(matches[0],) + else: raiseAssert "unreachable" + doAssert not declared(matches) + doAssert parse("NAME = LENA") == """("NAME", "LENA")""" + doAssert parse(" # comment ... ") == """("# comment ... ",)""" bind MaxSubpatterns when not declaredInScope(matches): var matches {.inject.}: array[MaxSubpatterns, string] @@ -270,73 +471,67 @@ template `=~` *(s: string, pattern: Regex): expr = # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Regex, start = 0): bool = - ## same as ``find(s, pattern, start) >= 0`` +proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} = + ## same as `find(s, pattern, start) >= 0` return find(s, pattern, start) >= 0 proc contains*(s: string, pattern: Regex, matches: var openArray[string], - start = 0): bool = - ## same as ``find(s, pattern, matches, start) >= 0`` + start = 0): bool {.inline.} = + ## same as `find(s, pattern, matches, start) >= 0` + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Regex): bool = +proc startsWith*(s: string, prefix: Regex): bool {.inline.} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix) >= 0 -proc endsWith*(s: string, suffix: Regex): bool = - ## returns true if `s` ends with the pattern `prefix` +proc endsWith*(s: string, suffix: Regex): bool {.inline.} = + ## returns true if `s` ends with the pattern `suffix` for i in 0 .. s.len-1: if matchLen(s, suffix, i) == s.len - i: return true proc replace*(s: string, sub: Regex, by = ""): string = ## Replaces `sub` in `s` by the string `by`. Captures cannot be - ## accessed in `by`. Examples: - ## - ## .. code-block:: nim - ## "var1=key; var2=key2".replace(re"(\w+)=(\w+)") - ## - ## Results in: - ## - ## .. code-block:: nim - ## - ## "; " + ## accessed in `by`. + runnableExamples: + doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)") == "; " + doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?" result = "" var prev = 0 - while true: - var match = findBounds(s, sub, prev) + var flags = int32(0) + while prev < s.len: + var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags) + flags = 0 if match.first < 0: break add(result, substr(s, prev, match.first-1)) add(result, by) + if match.first > match.last: + # 0-len match + flags = pcre.NOTEMPTY_ATSTART prev = match.last + 1 add(result, substr(s, prev)) proc replacef*(s: string, sub: Regex, by: string): string = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` - ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: - ## - ## .. code-block:: nim - ## "var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") - ## - ## Results in: - ## - ## .. code-block:: nim - ## - ## "var1<-keykey; val2<-key2key2" + ## with the notation `$i` and `$#` (see strutils.\`%\`). + runnableExamples: + doAssert "var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") == + "var1<-keykey; var2<-key2key2" result = "" var caps: array[MaxSubpatterns, string] var prev = 0 - while true: + while prev < s.len: var match = findBounds(s, sub, caps, prev) if match.first < 0: break - assert result != nil - assert s != nil add(result, substr(s, prev, match.first-1)) addf(result, by, caps) + if match.last + 1 == prev: break prev = match.last + 1 add(result, substr(s, prev)) -proc parallelReplace*(s: string, subs: openArray[ - tuple[pattern: Regex, repl: string]]): string = +proc multiReplace*(s: string, subs: openArray[ + tuple[pattern: Regex, repl: string]]): string = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -358,49 +553,50 @@ proc parallelReplace*(s: string, subs: openArray[ proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## `parallelReplace`) and writes back to `outfile`. Raises `IOError` if an ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) + var x = readFile(infile) + writeFile(outfile, x.multiReplace(subs)) -iterator split*(s: string, sep: Regex): string = +iterator split*(s: string, sep: Regex; maxsplit = -1): string = ## Splits the string `s` into substrings. ## - ## Substrings are separated by the regular expression `sep`. - ## Examples: - ## - ## .. code-block:: nim - ## for word in split("00232this02939is39an22example111", re"\d+"): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: nim - ## "" - ## "this" - ## "is" - ## "an" - ## "example" - ## "" - ## - var - first = -1 - last = -1 - while last < len(s): - var x = matchLen(s, sep, last) - if x > 0: inc(last, x) - first = last - if x == 0: inc(last) + ## Substrings are separated by the regular expression `sep` + ## (and the portion matched by `sep` is not returned). + runnableExamples: + import std/sequtils + doAssert toSeq(split("00232this02939is39an22example111", re"\d+")) == + @["", "this", "is", "an", "example", ""] + var last = 0 + var splits = maxsplit + var x = -1 + if len(s) == 0: + last = 1 + if matchLen(s, sep, 0) == 0: + x = 0 + while last <= len(s): + var first = last + var sepLen = 1 + if x == 0: + inc(last) while last < len(s): x = matchLen(s, sep, last) - if x >= 0: break + if x >= 0: + sepLen = x + break inc(last) - if first <= last: - yield substr(s, first, last-1) - -proc split*(s: string, sep: Regex): seq[string] = - ## Splits the string `s` into substrings. - accumulateResult(split(s, sep)) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + +proc split*(s: string, sep: Regex, maxsplit = -1): seq[string] {.inline.} = + ## Splits the string `s` into a seq of substrings. + ## + ## The portion matched by `sep` is not returned. + result = @[] + for x in split(s, sep, maxsplit): result.add x proc escapeRe*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a regular @@ -413,97 +609,3 @@ proc escapeRe*(s: string): string = else: result.add("\\x") result.add(toHex(ord(c), 2)) - -const ## common regular expressions - reIdentifier* {.deprecated.} = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b" - ## describes an identifier - reNatural* {.deprecated.} = r"\b\d+\b" - ## describes a natural number - reInteger* {.deprecated.} = r"\b[-+]?\d+\b" - ## describes an integer - reHex* {.deprecated.} = r"\b0[xX][0-9a-fA-F]+\b" - ## describes a hexadecimal number - reBinary* {.deprecated.} = r"\b0[bB][01]+\b" - ## describes a binary number (example: 0b11101) - reOctal* {.deprecated.} = r"\b0[oO][0-7]+\b" - ## describes an octal number (example: 0o777) - reFloat* {.deprecated.} = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b" - ## describes a floating point number - reEmail* {.deprecated.} = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" & - r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@" & - r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" & - r"(?:[a-zA-Z]{2}|com|org|net|gov|mil|biz|" & - r"info|mobi|name|aero|jobs|museum)\b" - ## describes a common email address - reURL* {.deprecated.} = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms-help)" & - r":((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b" - ## describes an URL - -when isMainModule: - assert match("(a b c)", re"\( .* \)") - assert match("WHiLe", re("while", {reIgnoreCase})) - - assert "0158787".match(re"\d+") - assert "ABC 0232".match(re"\w+\s+\d+") - assert "ABC".match(re"\d+ | \w+") - - assert matchLen("key", re(reIdentifier)) == 3 - - var pattern = re"[a-z0-9]+\s*=\s*[a-z0-9]+" - assert matchLen("key1= cal9", pattern) == 11 - - assert find("_____abc_______", re"abc") == 5 - - var matches: array[6, string] - if match("abcdefg", re"c(d)ef(g)", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - if "abc" =~ re"(a)bcxyz|(\w+)": - assert matches[1] == "abc" - else: - assert false - - if "abc" =~ re"(cba)?.*": - assert matches[0] == nil - else: assert false - - if "abc" =~ re"().*": - assert matches[0] == "" - else: assert false - - assert "var1=key; var2=key2".endsWith(re"\w+=\w+") - assert("var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(re"(\w+)=(\w+)", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", re"\d+"): - accum.add(word) - assert(accum == @["", "this", "is", "an", "example", ""]) - - accum = @[] - for word in split("AAA : : BBB", re"\s*:\s*"): - accum.add(word) - assert(accum == @["AAA", "", "BBB"]) - - for x in findAll("abcdef", re"^{.}", 3): - assert x == "d" - accum = @[] - for x in findAll("abcdef", re".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) - - assert("XYZ".find(re"^\d*") == 0) - assert("XYZ".match(re"^\d*") == true) - - block: - var matches: array[16, string] - if match("abcdefghijklmnop", re"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)(p)", matches): - for i in 0..matches.high: - assert matches[i] == $chr(i + 'a'.ord) - else: - assert false diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim deleted file mode 100644 index 721e5ce51..000000000 --- a/lib/impure/ssl.nim +++ /dev/null @@ -1,97 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module provides an easy to use sockets-style -## nim interface to the OpenSSL library. - -{.deprecated.} - -import openssl, strutils, os - -type - SecureSocket* = object - ssl: SslPtr - bio: BIO -{.deprecated: [TSecureSocket: SecureSocket].} - -proc connect*(sock: var SecureSocket, address: string, - port: int): int = - ## Connects to the specified `address` on the specified `port`. - ## Returns the result of the certificate validation. - SslLoadErrorStrings() - ERR_load_BIO_strings() - - if SSL_library_init() != 1: - raiseOSError(osLastError()) - - var ctx = SSL_CTX_new(SSLv23_client_method()) - if ctx == nil: - ERR_print_errors_fp(stderr) - raiseOSError(osLastError()) - - #if SSL_CTX_load_verify_locations(ctx, - # "/tmp/openssl-0.9.8e/certs/vsign1.pem", NIL) == 0: - # echo("Failed load verify locations") - # ERR_print_errors_fp(stderr) - - sock.bio = BIO_new_ssl_connect(ctx) - if BIO_get_ssl(sock.bio, addr(sock.ssl)) == 0: - raiseOSError(osLastError()) - - if BIO_set_conn_hostname(sock.bio, address & ":" & $port) != 1: - raiseOSError(osLastError()) - - if BIO_do_connect(sock.bio) <= 0: - ERR_print_errors_fp(stderr) - raiseOSError(osLastError()) - - result = SSL_get_verify_result(sock.ssl) - -proc recvLine*(sock: SecureSocket, line: var TaintedString): bool = - ## Acts in a similar fashion to the `recvLine` in the sockets module. - ## Returns false when no data is available to be read. - ## `Line` must be initialized and not nil! - setLen(line.string, 0) - while true: - var c: array[0..0, char] - var n = BIO_read(sock.bio, c, c.len.cint) - if n <= 0: return false - if c[0] == '\r': - n = BIO_read(sock.bio, c, c.len.cint) - if n > 0 and c[0] == '\L': - return true - elif n <= 0: - return false - elif c[0] == '\L': return true - add(line.string, c) - - -proc send*(sock: SecureSocket, data: string) = - ## Writes `data` to the socket. - if BIO_write(sock.bio, data, data.len.cint) <= 0: - raiseOSError(osLastError()) - -proc close*(sock: SecureSocket) = - ## Closes the socket - if BIO_free(sock.bio) <= 0: - ERR_print_errors_fp(stderr) - raiseOSError(osLastError()) - -when not defined(testing) and isMainModule: - var s: SecureSocket - echo connect(s, "smtp.gmail.com", 465) - - #var buffer: array[0..255, char] - #echo BIO_read(bio, buffer, buffer.len) - var buffer: string = "" - - echo s.recvLine(buffer) - echo buffer - echo buffer.len - diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim new file mode 100644 index 000000000..9b043f3e5 --- /dev/null +++ b/lib/js/asyncjs.nim @@ -0,0 +1,269 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim Authors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## This module implements types and macros for writing asynchronous code +## for the JS backend. It provides tools for interaction with JavaScript async API-s +## and libraries, writing async procedures in Nim and converting callback-based code +## to promises. +## +## A Nim procedure is asynchronous when it includes the `{.async.}` pragma. It +## should always have a `Future[T]` return type or not have a return type at all. +## A `Future[void]` return type is assumed by default. +## +## This is roughly equivalent to the `async` keyword in JavaScript code. +## +## ```nim +## proc loadGame(name: string): Future[Game] {.async.} = +## # code +## ``` +## +## should be equivalent to +## +## ```javascript +## async function loadGame(name) { +## // code +## } +## ``` +## +## A call to an asynchronous procedure usually needs `await` to wait for +## the completion of the `Future`. +## +## ```nim +## var game = await loadGame(name) +## ``` +## +## Often, you might work with callback-based API-s. You can wrap them with +## asynchronous procedures using promises and `newPromise`: +## +## ```nim +## proc loadGame(name: string): Future[Game] = +## var promise = newPromise() do (resolve: proc(response: Game)): +## cbBasedLoadGame(name) do (game: Game): +## resolve(game) +## return promise +## ``` +## +## Forward definitions work properly, you just need to always add the `{.async.}` pragma: +## +## ```nim +## proc loadGame(name: string): Future[Game] {.async.} +## ``` +## +## JavaScript compatibility +## ======================== +## +## Nim currently generates `async/await` JavaScript code which is supported in modern +## EcmaScript and most modern versions of browsers, Node.js and Electron. +## If you need to use this module with older versions of JavaScript, you can +## use a tool that backports the resulting JavaScript code, as babel. + +# xxx code: javascript above gives `LanguageXNotSupported` warning. + +when not defined(js) and not defined(nimsuggest): + {.fatal: "Module asyncjs is designed to be used with the JavaScript backend.".} + +import std/jsffi +import std/macros +import std/private/since + +type + Future*[T] = ref object + future*: T + ## Wraps the return type of an asynchronous procedure. + + PromiseJs* {.importjs: "Promise".} = ref object + ## A JavaScript Promise. + + +proc replaceReturn(node: var NimNode) = + var z = 0 + for s in node: + var son = node[z] + let jsResolve = ident("jsResolve") + if son.kind == nnkReturnStmt: + let value = if son[0].kind != nnkEmpty: nnkCall.newTree(jsResolve, son[0]) else: jsResolve + node[z] = nnkReturnStmt.newTree(value) + elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result": + node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(jsResolve, son[1])) + elif son.kind in RoutineNodes: + discard + else: + replaceReturn(son) + inc z + +proc isFutureVoid(node: NimNode): bool = + result = node.kind == nnkBracketExpr and + node[0].kind == nnkIdent and $node[0] == "Future" and + node[1].kind == nnkIdent and $node[1] == "void" + +proc generateJsasync(arg: NimNode): NimNode = + if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo, nnkProcTy}: + error("Cannot transform this node kind into an async proc." & + " proc/method definition or lambda node expected.") + + # Transform type X = proc (): something {.async.} + # into type X = proc (): Future[something] + if arg.kind == nnkProcTy: + result = arg + if arg[0][0].kind == nnkEmpty: + result[0][0] = quote do: Future[void] + return result + + result = arg + var isVoid = false + let jsResolve = ident("jsResolve") + if arg.params[0].kind == nnkEmpty: + result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void")) + isVoid = true + elif isFutureVoid(arg.params[0]): + isVoid = true + + var code = result.body + replaceReturn(code) + result.body = nnkStmtList.newTree() + + if len(code) > 0: + var awaitFunction = quote: + proc await[T](f: Future[T]): T {.importjs: "(await #)", used.} + result.body.add(awaitFunction) + + var resolve: NimNode + if isVoid: + resolve = quote: + var `jsResolve` {.importjs: "undefined".}: Future[void] + else: + resolve = quote: + proc jsResolve[T](a: T): Future[T] {.importjs: "#", used.} + proc jsResolve[T](a: Future[T]): Future[T] {.importjs: "#", used.} + result.body.add(resolve) + else: + result.body = newEmptyNode() + for child in code: + result.body.add(child) + + if len(code) > 0 and isVoid: + var voidFix = quote: + return `jsResolve` + result.body.add(voidFix) + + let asyncPragma = quote: + {.codegenDecl: "async function $2($3)".} + + result.addPragma(asyncPragma[0]) + +macro async*(arg: untyped): untyped = + ## Macro which converts normal procedures into + ## javascript-compatible async procedures. + if arg.kind == nnkStmtList: + result = newStmtList() + for oneProc in arg: + result.add generateJsasync(oneProc) + else: + result = generateJsasync(arg) + +proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importjs: "(new Promise(#))".} + ## A helper for wrapping callback-based functions + ## into promises and async procedures. + +proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importjs: "(new Promise(#))".} + ## A helper for wrapping callback-based functions + ## into promises and async procedures. + +template maybeFuture(T): untyped = + # avoids `Future[Future[T]]` + when T is Future: T + else: Future[T] + + +since (1, 5, 1): + #[ + TODO: + * map `Promise.all()` + * proc toString*(a: Error): cstring {.importjs: "#.toString()".} + + Note: + We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible + in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options + and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho + ]# + + type Error* {.importjs: "Error".} = ref object of JsRoot + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error + message*: cstring + name*: cstring + + type OnReject* = proc(reason: Error) + + proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then + ## Returns a `Future` from the return type of `onSuccess(T.default)`. + runnableExamples("-r:off"): + from std/sugar import `=>` + + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 + + proc asyncFact(n: int): Future[int] {.async.} = + if n > 0: result = n * await asyncFact(n-1) + else: result = 1 + + proc main() {.async.} = + block: # then + assert asyncFact(3).await == 3*2 + assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2 + let x1 = await fn(3) + assert x1 == 3 * 2 + let x2 = await fn(4) + .then((a: int) => a.float) + .then((a: float) => $a) + assert x2 == "8.0" + + block: # then with `onReject` callback + var witness = 1 + await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 2 + await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 3 + + template impl(call): untyped = + # see D20210421T014713 + when typeof(block: call) is void: + var ret: Future[void] + else: + var ret = default(maybeFuture(typeof(call))) + typeof(ret) + when T is void: + type A = impl(onSuccess()) + else: + type A = impl(onSuccess(default(T))) + var ret: A + {.emit: "`ret` = `future`.then(`onSuccess`, `onReject`);".} + return ret + + proc catch*[T](future: Future[T], onReject: OnReject): Future[void] = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch + runnableExamples("-r:off"): + from std/sugar import `=>` + from std/strutils import contains + + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 + + proc main() {.async.} = + var reason: Error + await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work + assert reason == nil + await fn(7).catch((r: Error) => (reason = r)) + assert reason != nil + assert "foobar: 7" in $reason.message + + discard main() + + {.emit: "`result` = `future`.catch(`onReject`);".} diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 11df959d7..be2a34db1 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -8,61 +8,189 @@ # ## Declaration of the Document Object Model for the `JavaScript backend -## <backends.html#the-javascript-target>`_. +## <backends.html#backends-the-javascript-target>`_. +## +## +## Document Ready +## -------------- +## +## * Basic example of a document ready: +runnableExamples"-b:js -r:off": + proc example(e: Event) = echo "Document is ready" + document.addEventListener("DOMContentLoaded", example) # You can also use "load" event. +## * This example runs 5 seconds after the document ready: +runnableExamples"-b:js -r:off": + proc example() = echo "5 seconds after document ready" + proc domReady(e: Event) = discard setTimeout(example, 5_000) # Document is ready. + document.addEventListener("DOMContentLoaded", domReady) +## Document onUnload +## ----------------- +## +## * Simple example of how to implement code that runs when the page unloads: +runnableExamples"-b:js -r:off": + proc example(e: Event) = echo "Document is unloaded" + document.addEventListener("unload", example) # You can also use "beforeunload". +## Document Autorefresh +## -------------------- +## +## * Minimal example of a document autorefresh: +runnableExamples"-b:js -r:off": + proc example() = window.location.reload() + discard setTimeout(example, 5_000) +## - For more examples, see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener -when not defined(js) and not defined(Nimdoc): + +import std/private/since +when not defined(js): {.error: "This module only works on the JavaScript platform".} +const + DomApiVersion* = 3 ## the version of DOM API we try to follow. No guarantees though. + type - EventTarget* = ref EventTargetObj - EventTargetObj {.importc.} = object of RootObj - onabort*: proc (event: Event) {.nimcall.} - onblur*: proc (event: Event) {.nimcall.} - onchange*: proc (event: Event) {.nimcall.} - onclick*: proc (event: Event) {.nimcall.} - ondblclick*: proc (event: Event) {.nimcall.} - onerror*: proc (event: Event) {.nimcall.} - onfocus*: proc (event: Event) {.nimcall.} - onkeydown*: proc (event: Event) {.nimcall.} - onkeypress*: proc (event: Event) {.nimcall.} - onkeyup*: proc (event: Event) {.nimcall.} - onload*: proc (event: Event) {.nimcall.} - onmousedown*: proc (event: Event) {.nimcall.} - onmousemove*: proc (event: Event) {.nimcall.} - onmouseout*: proc (event: Event) {.nimcall.} - onmouseover*: proc (event: Event) {.nimcall.} - onmouseup*: proc (event: Event) {.nimcall.} - onreset*: proc (event: Event) {.nimcall.} - onselect*: proc (event: Event) {.nimcall.} - onsubmit*: proc (event: Event) {.nimcall.} - onunload*: proc (event: Event) {.nimcall.} - - Window* = ref WindowObj - WindowObj {.importc.} = object of EventTargetObj + EventTarget* {.importc.} = ref object of RootObj + onabort*: proc (event: Event) {.closure.} + onblur*: proc (event: Event) {.closure.} + onchange*: proc (event: Event) {.closure.} + onclick*: proc (event: Event) {.closure.} + ondblclick*: proc (event: Event) {.closure.} + onerror*: proc (event: Event) {.closure.} + onfocus*: proc (event: Event) {.closure.} + onkeydown*: proc (event: Event) {.closure.} + onkeypress*: proc (event: Event) {.closure.} + onkeyup*: proc (event: Event) {.closure.} + onload*: proc (event: Event) {.closure.} + onmousedown*: proc (event: Event) {.closure.} + onmousemove*: proc (event: Event) {.closure.} + onmouseout*: proc (event: Event) {.closure.} + onmouseover*: proc (event: Event) {.closure.} + onmouseup*: proc (event: Event) {.closure.} + onreset*: proc (event: Event) {.closure.} + onselect*: proc (event: Event) {.closure.} + onstorage*: proc (event: Event) {.closure.} + onsubmit*: proc (event: Event) {.closure.} + onunload*: proc (event: Event) {.closure.} + onloadstart*: proc (event: Event) {.closure.} + onprogress*: proc (event: Event) {.closure.} + onloadend*: proc (event: Event) {.closure.} + + DomEvent* {.pure.} = enum + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/Events>`_ + Abort = "abort", + BeforeInput = "beforeinput", + Blur = "blur", + Click = "click", + CompositionEnd = "compositionend", + CompositionStart = "compositionstart", + CompositionUpdate = "compositionupdate", + DblClick = "dblclick", + Error = "error", + Focus = "focus", + FocusIn = "focusin", + FocusOut = "focusout", + Input = "input", + KeyDown = "keydown", + KeyPress = "keypress", + KeyUp = "keyup", + Load = "load", + MouseDown = "mousedown", + MouseEnter = "mouseenter", + MouseLeave = "mouseleave", + MouseMove = "mousemove", + MouseOut = "mouseout", + MouseOver = "mouseover", + MouseUp = "mouseup", + Resize = "resize", + Scroll = "scroll", + Select = "select", + Storage = "storage", + Unload = "unload", + Wheel = "wheel" + + PerformanceMemory* {.importc.} = ref object + jsHeapSizeLimit*: float + totalJSHeapSize*: float + usedJSHeapSize*: float + + PerformanceTiming* {.importc.} = ref object + connectStart*: float + domComplete*: float + domContentLoadedEventEnd*: float + domContentLoadedEventStart*: float + domInteractive*: float + domLoading*: float + domainLookupEnd*: float + domainLookupStart*: float + fetchStart*: float + loadEventEnd*: float + loadEventStart*: float + navigationStart*: float + redirectEnd*: float + redirectStart*: float + requestStart*: float + responseEnd*: float + responseStart*: float + secureConnectionStart*: float + unloadEventEnd*: float + unloadEventStart*: float + + Performance* {.importc.} = ref object + memory*: PerformanceMemory + timing*: PerformanceTiming + + Range* {.importc.} = ref object + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Range>`_ + collapsed*: bool + commonAncestorContainer*: Node + endContainer*: Node + endOffset*: int + startContainer*: Node + startOffset*: int + + Selection* {.importc.} = ref object + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Selection>`_ + anchorNode*: Node + anchorOffset*: int + focusNode*: Node + focusOffset*: int + isCollapsed*: bool + rangeCount*: int + `type`*: cstring + + Storage* {.importc.} = ref object + + Window* {.importc.} = ref object of EventTarget document*: Document event*: Event history*: History location*: Location closed*: bool defaultStatus*: cstring + devicePixelRatio*: float innerHeight*, innerWidth*: int - locationbar*: ref TLocationBar - menubar*: ref TMenuBar + locationbar*: ref LocationBar + menubar*: ref MenuBar name*: cstring outerHeight*, outerWidth*: int pageXOffset*, pageYOffset*: int - personalbar*: ref TPersonalBar - scrollbars*: ref TScrollBars - statusbar*: ref TStatusBar + scrollX*: float + scrollY*: float + personalbar*: ref PersonalBar + scrollbars*: ref ScrollBars + statusbar*: ref StatusBar status*: cstring - toolbar*: ref TToolBar - frames*: seq[TFrame] + toolbar*: ref ToolBar + frames*: seq[Frame] + screen*: Screen + performance*: Performance + onpopstate*: proc (event: Event) + localStorage*: Storage + sessionStorage*: Storage + parent*: Window - Frame* = ref FrameObj - FrameObj {.importc.} = object of WindowObj + Frame* {.importc.} = ref object of Window - ClassList* = ref ClassListObj - ClassListObj {.importc.} = object of RootObj + ClassList* {.importc.} = ref object of RootObj NodeType* = enum ElementNode = 1, @@ -78,8 +206,7 @@ type DocumentFragmentNode, NotationNode - Node* = ref NodeObj - NodeObj {.importc.} = object of EventTargetObj + Node* {.importc.} = ref object of EventTarget attributes*: seq[Node] childNodes*: seq[Node] children*: seq[Node] @@ -91,12 +218,21 @@ type nodeType*: NodeType nodeValue*: cstring parentNode*: Node + content*: Node previousSibling*: Node + ownerDocument*: Document innerHTML*: cstring + outerHTML*: cstring + innerText*: cstring + textContent*: cstring style*: Style + baseURI*: cstring + parentElement*: Element + isConnected*: bool - Document* = ref DocumentObj - DocumentObj {.importc.} = object of NodeObj + Document* {.importc.} = ref object of Node + activeElement*: Element + documentElement*: Element alinkColor*: cstring bgColor*: cstring body*: Element @@ -105,11 +241,13 @@ type defaultCharset*: cstring fgColor*: cstring head*: Element + hidden*: bool lastModified*: cstring linkColor*: cstring referrer*: cstring title*: cstring URL*: cstring + visibilityState*: cstring vlinkColor*: cstring anchors*: seq[AnchorElement] forms*: seq[FormElement] @@ -117,10 +255,11 @@ type applets*: seq[Element] embeds*: seq[EmbedElement] links*: seq[LinkElement] + fonts*: FontFaceSet - Element* = ref ElementObj - ElementObj {.importc.} = object of NodeObj - classList*: Classlist + Element* {.importc.} = ref object of Node + className*: cstring + classList*: ClassList checked*: bool defaultChecked*: bool defaultValue*: cstring @@ -129,16 +268,102 @@ type name*: cstring readOnly*: bool options*: seq[OptionElement] + selectedOptions*: seq[OptionElement] + clientWidth*, clientHeight*: int + contentEditable*: cstring + isContentEditable*: bool + dir*: cstring + offsetHeight*: int + offsetWidth*: int + offsetLeft*: int + offsetTop*: int + + ValidityState* {.importc.} = ref object ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/ValidityState>`_ + badInput*: bool + customError*: bool + patternMismatch*: bool + rangeOverflow*: bool + rangeUnderflow*: bool + stepMismatch*: bool + tooLong*: bool + tooShort*: bool + typeMismatch*: bool + valid*: bool + valueMissing*: bool + + Blob* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Blob>`_ + size*: int + `type`*: cstring + + File* {.importc.} = ref object of Blob ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/File>`_ + lastModified*: int + name*: cstring + + TextAreaElement* {.importc.} = ref object of Element ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/HTMLTextAreaElement>`_ + value*: cstring + selectionStart*, selectionEnd*: int + selectionDirection*: cstring + rows*, cols*: int - LinkElement* = ref LinkObj - LinkObj {.importc.} = object of ElementObj + InputElement* {.importc.} = ref object of Element ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement>`_ + # Properties related to the parent form + formAction*: cstring + formEncType*: cstring + formMethod*: cstring + formNoValidate*: bool + formTarget*: cstring + + # Properties that apply to any type of input element that is not hidden + `type`*: cstring + autofocus*: bool + required*: bool + value*: cstring + validity*: ValidityState + validationMessage*: cstring + willValidate*: bool + + # Properties that apply only to elements of type "checkbox" or "radio" + indeterminate*: bool + + # Properties that apply only to elements of type "image" + alt*: cstring + height*: cstring + src*: cstring + width*: cstring + + # Properties that apply only to elements of type "file" + accept*: cstring + files*: seq[Blob] + + # Properties that apply only to text/number-containing or elements + autocomplete*: cstring + maxLength*: int + size*: int + pattern*: cstring + placeholder*: cstring + min*: cstring + max*: cstring + selectionStart*: int + selectionEnd*: int + selectionDirection*: cstring + + # Properties not yet categorized + dirName*: cstring + accessKey*: cstring + list*: Element + multiple*: bool + labels*: seq[Element] + step*: cstring + valueAsDate*: cstring + valueAsNumber*: float + + LinkElement* {.importc.} = ref object of Element target*: cstring text*: cstring x*: int y*: int - EmbedElement* = ref EmbedObj - EmbedObj {.importc.} = object of ElementObj + EmbedElement* {.importc.} = ref object of Element height*: int hspace*: int src*: cstring @@ -146,29 +371,30 @@ type `type`*: cstring vspace*: int - AnchorElement* = ref AnchorObj - AnchorObj {.importc.} = object of ElementObj + AnchorElement* {.importc.} = ref object of Element text*: cstring x*, y*: int - OptionElement* = ref OptionObj - OptionObj {.importc.} = object of ElementObj + OptionElement* {.importc.} = ref object of Element defaultSelected*: bool selected*: bool selectedIndex*: int text*: cstring value*: cstring - FormElement* = ref FormObj - FormObj {.importc.} = object of ElementObj + FormElement* {.importc.} = ref object of Element ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/HTMLFormElement>`_ + acceptCharset*: cstring action*: cstring + autocomplete*: cstring + elements*: seq[Element] encoding*: cstring + enctype*: cstring + length*: int `method`*: cstring + noValidate*: bool target*: cstring - elements*: seq[Element] - ImageElement* = ref ImageObj - ImageObj {.importc.} = object of ElementObj + ImageElement* {.importc.} = ref object of Element border*: int complete*: bool height*: int @@ -178,140 +404,844 @@ type vspace*: int width*: int - Style = ref StyleObj - StyleObj {.importc.} = object of RootObj + Style* {.importc.} = ref object of RootObj + alignContent*: cstring + alignItems*: cstring + alignSelf*: cstring + all*: cstring + animation*: cstring + animationDelay*: cstring + animationDirection*: cstring + animationDuration*: cstring + animationFillMode*: cstring + animationIterationCount*: cstring + animationName*: cstring + animationPlayState*: cstring + animationTimingFunction*: cstring + backdropFilter*: cstring + backfaceVisibility*: cstring background*: cstring backgroundAttachment*: cstring + backgroundBlendMode*: cstring + backgroundClip*: cstring backgroundColor*: cstring backgroundImage*: cstring + backgroundOrigin*: cstring backgroundPosition*: cstring backgroundRepeat*: cstring + backgroundSize*: cstring + blockSize*: cstring border*: cstring + borderBlock*: cstring + borderBlockColor*: cstring + borderBlockEnd*: cstring + borderBlockEndColor*: cstring + borderBlockEndStyle*: cstring + borderBlockEndWidth*: cstring + borderBlockStart*: cstring + borderBlockStartColor*: cstring + borderBlockStartStyle*: cstring + borderBlockStartWidth*: cstring + borderBlockStyle*: cstring + borderBlockWidth*: cstring borderBottom*: cstring borderBottomColor*: cstring + borderBottomLeftRadius*: cstring + borderBottomRightRadius*: cstring borderBottomStyle*: cstring borderBottomWidth*: cstring + borderCollapse*: cstring borderColor*: cstring + borderEndEndRadius*: cstring + borderEndStartRadius*: cstring + borderImage*: cstring + borderImageOutset*: cstring + borderImageRepeat*: cstring + borderImageSlice*: cstring + borderImageSource*: cstring + borderImageWidth*: cstring + borderInline*: cstring + borderInlineColor*: cstring + borderInlineEnd*: cstring + borderInlineEndColor*: cstring + borderInlineEndStyle*: cstring + borderInlineEndWidth*: cstring + borderInlineStart*: cstring + borderInlineStartColor*: cstring + borderInlineStartStyle*: cstring + borderInlineStartWidth*: cstring + borderInlineStyle*: cstring + borderInlineWidth*: cstring borderLeft*: cstring borderLeftColor*: cstring borderLeftStyle*: cstring borderLeftWidth*: cstring + borderRadius*: cstring borderRight*: cstring borderRightColor*: cstring borderRightStyle*: cstring borderRightWidth*: cstring + borderSpacing*: cstring + borderStartEndRadius*: cstring + borderStartStartRadius*: cstring borderStyle*: cstring borderTop*: cstring borderTopColor*: cstring + borderTopLeftRadius*: cstring + borderTopRightRadius*: cstring borderTopStyle*: cstring borderTopWidth*: cstring borderWidth*: cstring bottom*: cstring + boxDecorationBreak*: cstring + boxShadow*: cstring + boxSizing*: cstring + breakAfter*: cstring + breakBefore*: cstring + breakInside*: cstring captionSide*: cstring + caretColor*: cstring clear*: cstring clip*: cstring + clipPath*: cstring color*: cstring + colorAdjust*: cstring + columnCount*: cstring + columnFill*: cstring + columnGap*: cstring + columnRule*: cstring + columnRuleColor*: cstring + columnRuleStyle*: cstring + columnRuleWidth*: cstring + columnSpan*: cstring + columnWidth*: cstring + columns*: cstring + contain*: cstring + content*: cstring + counterIncrement*: cstring + counterReset*: cstring + counterSet*: cstring cursor*: cstring direction*: cstring display*: cstring emptyCells*: cstring + filter*: cstring + flex*: cstring + flexBasis*: cstring + flexDirection*: cstring + flexFlow*: cstring + flexGrow*: cstring + flexShrink*: cstring + flexWrap*: cstring cssFloat*: cstring font*: cstring fontFamily*: cstring + fontFeatureSettings*: cstring + fontKerning*: cstring + fontLanguageOverride*: cstring + fontOpticalSizing*: cstring fontSize*: cstring + fontSizeAdjust*: cstring fontStretch*: cstring fontStyle*: cstring + fontSynthesis*: cstring fontVariant*: cstring + fontVariantAlternates*: cstring + fontVariantCaps*: cstring + fontVariantEastAsian*: cstring + fontVariantLigatures*: cstring + fontVariantNumeric*: cstring + fontVariantPosition*: cstring + fontVariationSettings*: cstring fontWeight*: cstring + gap*: cstring + grid*: cstring + gridArea*: cstring + gridAutoColumns*: cstring + gridAutoFlow*: cstring + gridAutoRows*: cstring + gridColumn*: cstring + gridColumnEnd*: cstring + gridColumnStart*: cstring + gridRow*: cstring + gridRowEnd*: cstring + gridRowStart*: cstring + gridTemplate*: cstring + gridTemplateAreas*: cstring + gridTemplateColumns*: cstring + gridTemplateRows*: cstring + hangingPunctuation*: cstring height*: cstring + hyphens*: cstring + imageOrientation*: cstring + imageRendering*: cstring + inlineSize*: cstring + inset*: cstring + insetBlock*: cstring + insetBlockEnd*: cstring + insetBlockStart*: cstring + insetInline*: cstring + insetInlineEnd*: cstring + insetInlineStart*: cstring + isolation*: cstring + justifyContent*: cstring + justifyItems*: cstring + justifySelf*: cstring left*: cstring letterSpacing*: cstring + lineBreak*: cstring lineHeight*: cstring listStyle*: cstring listStyleImage*: cstring listStylePosition*: cstring listStyleType*: cstring margin*: cstring + marginBlock*: cstring + marginBlockEnd*: cstring + marginBlockStart*: cstring marginBottom*: cstring + marginInline*: cstring + marginInlineEnd*: cstring + marginInlineStart*: cstring marginLeft*: cstring marginRight*: cstring marginTop*: cstring + mask*: cstring + maskBorder*: cstring + maskBorderMode*: cstring + maskBorderOutset*: cstring + maskBorderRepeat*: cstring + maskBorderSlice*: cstring + maskBorderSource*: cstring + maskBorderWidth*: cstring + maskClip*: cstring + maskComposite*: cstring + maskImage*: cstring + maskMode*: cstring + maskOrigin*: cstring + maskPosition*: cstring + maskRepeat*: cstring + maskSize*: cstring + maskType*: cstring + maxBlockSize*: cstring maxHeight*: cstring + maxInlineSize*: cstring maxWidth*: cstring + minBlockSize*: cstring minHeight*: cstring + minInlineSize*: cstring minWidth*: cstring + mixBlendMode*: cstring + objectFit*: cstring + objectPosition*: cstring + offset*: cstring + offsetAnchor*: cstring + offsetDistance*: cstring + offsetPath*: cstring + offsetRotate*: cstring + opacity*: cstring + order*: cstring + orphans*: cstring + outline*: cstring + outlineColor*: cstring + outlineOffset*: cstring + outlineStyle*: cstring + outlineWidth*: cstring overflow*: cstring + overflowAnchor*: cstring + overflowBlock*: cstring + overflowInline*: cstring + overflowWrap*: cstring + overflowX*: cstring + overflowY*: cstring + overscrollBehavior*: cstring + overscrollBehaviorBlock*: cstring + overscrollBehaviorInline*: cstring + overscrollBehaviorX*: cstring + overscrollBehaviorY*: cstring padding*: cstring + paddingBlock*: cstring + paddingBlockEnd*: cstring + paddingBlockStart*: cstring paddingBottom*: cstring + paddingInline*: cstring + paddingInlineEnd*: cstring + paddingInlineStart*: cstring paddingLeft*: cstring paddingRight*: cstring paddingTop*: cstring pageBreakAfter*: cstring pageBreakBefore*: cstring + pageBreakInside*: cstring + paintOrder*: cstring + perspective*: cstring + perspectiveOrigin*: cstring + placeContent*: cstring + placeItems*: cstring + placeSelf*: cstring + pointerEvents*: cstring position*: cstring + quotes*: cstring + resize*: cstring right*: cstring + rotate*: cstring + rowGap*: cstring + scale*: cstring + scrollBehavior*: cstring + scrollMargin*: cstring + scrollMarginBlock*: cstring + scrollMarginBlockEnd*: cstring + scrollMarginBlockStart*: cstring + scrollMarginBottom*: cstring + scrollMarginInline*: cstring + scrollMarginInlineEnd*: cstring + scrollMarginInlineStart*: cstring + scrollMarginLeft*: cstring + scrollMarginRight*: cstring + scrollMarginTop*: cstring + scrollPadding*: cstring + scrollPaddingBlock*: cstring + scrollPaddingBlockEnd*: cstring + scrollPaddingBlockStart*: cstring + scrollPaddingBottom*: cstring + scrollPaddingInline*: cstring + scrollPaddingInlineEnd*: cstring + scrollPaddingInlineStart*: cstring + scrollPaddingLeft*: cstring + scrollPaddingRight*: cstring + scrollPaddingTop*: cstring + scrollSnapAlign*: cstring + scrollSnapStop*: cstring + scrollSnapType*: cstring scrollbar3dLightColor*: cstring scrollbarArrowColor*: cstring scrollbarBaseColor*: cstring + scrollbarColor*: cstring scrollbarDarkshadowColor*: cstring scrollbarFaceColor*: cstring scrollbarHighlightColor*: cstring scrollbarShadowColor*: cstring scrollbarTrackColor*: cstring + scrollbarWidth*: cstring + shapeImageThreshold*: cstring + shapeMargin*: cstring + shapeOutside*: cstring + tabSize*: cstring tableLayout*: cstring textAlign*: cstring + textAlignLast*: cstring + textCombineUpright*: cstring textDecoration*: cstring + textDecorationColor*: cstring + textDecorationLine*: cstring + textDecorationSkipInk*: cstring + textDecorationStyle*: cstring + textDecorationThickness*: cstring + textEmphasis*: cstring + textEmphasisColor*: cstring + textEmphasisPosition*: cstring + textEmphasisStyle*: cstring textIndent*: cstring + textJustify*: cstring + textOrientation*: cstring + textOverflow*: cstring + textRendering*: cstring + textShadow*: cstring textTransform*: cstring + textUnderlineOffset*: cstring + textUnderlinePosition*: cstring top*: cstring + touchAction*: cstring + transform*: cstring + transformBox*: cstring + transformOrigin*: cstring + transformStyle*: cstring + transition*: cstring + transitionDelay*: cstring + transitionDuration*: cstring + transitionProperty*: cstring + transitionTimingFunction*: cstring + translate*: cstring + unicodeBidi*: cstring verticalAlign*: cstring visibility*: cstring + whiteSpace*: cstring + widows*: cstring width*: cstring + willChange*: cstring + wordBreak*: cstring wordSpacing*: cstring - zIndex*: int + writingMode*: cstring + zIndex*: cstring - Event* = ref EventObj - EventObj {.importc.} = object of RootObj + EventPhase* = enum + None = 0, + CapturingPhase, + AtTarget, + BubblingPhase + + Event* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Event>`_ + bubbles*: bool + cancelBubble*: bool + cancelable*: bool + composed*: bool + currentTarget*: Node + defaultPrevented*: bool + eventPhase*: int target*: Node - altKey*, ctrlKey*, shiftKey*: bool + `type`*: cstring + isTrusted*: bool + + UIEvent* {.importc.} = ref object of Event ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/UIEvent>`_ + detail*: int64 + view*: Window + + KeyboardEvent* {.importc.} = ref object of UIEvent ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent>`_ + altKey*, ctrlKey*, metaKey*, shiftKey*: bool + code*: cstring + isComposing*: bool + key*: cstring + keyCode*: int + location*: int + + KeyboardEventKey* {.pure.} = enum ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key/Key_Values>`_ + # Modifier keys + Alt, + AltGraph, + CapsLock, + Control, + Fn, + FnLock, + Hyper, + Meta, + NumLock, + ScrollLock, + Shift, + Super, + Symbol, + SymbolLock, + + # Whitespace keys + ArrowDown, + ArrowLeft, + ArrowRight, + ArrowUp, + End, + Home, + PageDown, + PageUp, + + # Editing keys + Backspace, + Clear, + Copy, + CrSel, + Cut, + Delete, + EraseEof, + ExSel, + Insert, + Paste, + Redo, + Undo, + + # UI keys + Accept, + Again, + Attn, + Cancel, + ContextMenu, + Escape, + Execute, + Find, + Finish, + Help, + Pause, + Play, + Props, + Select, + ZoomIn, + ZoomOut, + + # Device keys + BrigtnessDown, + BrigtnessUp, + Eject, + LogOff, + Power, + PowerOff, + PrintScreen, + Hibernate, + Standby, + WakeUp, + + # Common IME keys + AllCandidates, + Alphanumeric, + CodeInput, + Compose, + Convert, + Dead, + FinalMode, + GroupFirst, + GroupLast, + GroupNext, + GroupPrevious, + ModeChange, + NextCandidate, + NonConvert, + PreviousCandidate, + Process, + SingleCandidate, + + # Korean keyboards only + HangulMode, + HanjaMode, + JunjaMode, + + # Japanese keyboards only + Eisu, + Hankaku, + Hiragana, + HiraganaKatakana, + KanaMode, + KanjiMode, + Katakana, + Romaji, + Zenkaku, + ZenkakuHanaku, + + # Function keys + F1, + F2, + F3, + F4, + F5, + F6, + F7, + F8, + F9, + F10, + F11, + F12, + F13, + F14, + F15, + F16, + F17, + F18, + F19, + F20, + Soft1, + Soft2, + Soft3, + Soft4, + + # Phone keys + AppSwitch, + Call, + Camera, + CameraFocus, + EndCall, + GoBack, + GoHome, + HeadsetHook, + LastNumberRedial, + Notification, + MannerMode, + VoiceDial, + + # Multimedia keys + ChannelDown, + ChannelUp, + MediaFastForward, + MediaPause, + MediaPlay, + MediaPlayPause, + MediaRecord, + MediaRewind, + MediaStop, + MediaTrackNext, + MediaTrackPrevious, + + # Audio control keys + AudioBalanceLeft, + AudioBalanceRight, + AudioBassDown, + AudioBassBoostDown, + AudioBassBoostToggle, + AudioBassBoostUp, + AudioBassUp, + AudioFaderFront, + AudioFaderRear, + AudioSurroundModeNext, + AudioTrebleDown, + AudioTrebleUp, + AudioVolumeDown, + AUdioVolumeMute, + AudioVolumeUp, + MicrophoneToggle, + MicrophoneVolumeDown, + MicrophoneVolumeMute, + MicrophoneVolumeUp, + + # TV control keys + TV, + TV3DMode, + TVAntennaCable, + TVAudioDescription, + TVAudioDescriptionMixDown, + TVAudioDescriptionMixUp, + TVContentsMenu, + TVDataService, + TVInput, + TVInputComponent1, + TVInputComponent2, + TVInputComposite1, + TVInputComposite2, + TVInputHDMI1, + TVInputHDMI2, + TVInputHDMI3, + TVInputHDMI4, + TVInputVGA1, + TVMediaContext, + TVNetwork, + TVNumberEntry, + TVPower, + TVRadioService, + TVSatellite, + TVSatelliteBS, + TVSatelliteCS, + TVSatelliteToggle, + TVTerrestrialAnalog, + TVTerrestrialDigital, + TVTimer, + + # Media controller keys + AVRInput, + AVRPower, + ColorF0Red, + ColorF1Green, + ColorF2Yellow, + ColorF3Blue, + ColorF4Grey, + ColorF5Brown, + ClosedCaptionToggle, + Dimmer, + DisplaySwap, + DVR, + Exit, + FavoriteClear0, + FavoriteClear1, + FavoriteClear2, + FavoriteClear3, + FavoriteRecall0, + FavoriteRecall1, + FavoriteRecall2, + FavoriteRecall3, + FavoriteStore0, + FavoriteStore1, + FavoriteStore2, + FavoriteStore3, + Guide, + GuideNextDay, + GuidePreviousDay, + Info, + InstantReplay, + Link, + ListProgram, + LiveContent, + Lock, + MediaApps, + MediaAudioTrack, + MediaLast, + MediaSkipBackward, + MediaSkipForward, + MediaStepBackward, + MediaStepForward, + MediaTopMenu, + NavigateIn, + NavigateNext, + NavigateOut, + NavigatePrevious, + NextFavoriteChannel, + NextUserProfile, + OnDemand, + Pairing, + PinPDown, + PinPMove, + PinPUp, + PlaySpeedDown, + PlaySpeedReset, + PlaySpeedUp, + RandomToggle, + RcLowBattery, + RecordSpeedNext, + RfBypass, + ScanChannelsToggle, + ScreenModeNext, + Settings, + SplitScreenToggle, + STBInput, + STBPower, + Subtitle, + Teletext, + VideoModeNext, + Wink, + ZoomToggle, + + # Speech recognition keys + SpeechCorrectionList, + SpeechInputToggle, + + # Document keys + Close, + New, + Open, + Print, + Save, + SpellCheck, + MailForward, + MailReply, + MailSend, + + # Application selector keys + LaunchCalculator, + LaunchCalendar, + LaunchContacts, + LaunchMail, + LaunchMediaPlayer, + LaunchMusicPlayer, + LaunchMyComputer, + LaunchPhone, + LaunchScreenSaver, + LaunchSpreadsheet, + LaunchWebBrowser, + LaunchWebCam, + LaunchWordProcessor, + LaunchApplication1, + LaunchApplication2, + LaunchApplication3, + LaunchApplication4, + LaunchApplication5, + LaunchApplication6, + LaunchApplication7, + LaunchApplication8, + LaunchApplication9, + LaunchApplication10, + LaunchApplication11, + LaunchApplication12, + LaunchApplication13, + LaunchApplication14, + LaunchApplication15, + LaunchApplication16, + + # Browser control keys + BrowserBack, + BrowserFavorites, + BrowserForward, + BrowserHome, + BrowserRefresh, + BrowserSearch, + BrowserStop, + + # Numeric keypad keys + Key11, + Key12, + Separator + + MouseButtons* = enum + NoButton = 0, + PrimaryButton = 1, + SecondaryButton = 2, + AuxilaryButton = 4, + FourthButton = 8, + FifthButton = 16 + + MouseEvent* {.importc.} = ref object of UIEvent ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent>`_ + altKey*, ctrlKey*, metaKey*, shiftKey*: bool button*: int + buttons*: int clientX*, clientY*: int - keyCode*: int - layerX*, layerY*: int - modifiers*: int - ALT_MASK*, CONTROL_MASK*, SHIFT_MASK*, META_MASK*: int + movementX*, movementY*: int offsetX*, offsetY*: int pageX*, pageY*: int + relatedTarget*: EventTarget + #region*: cstring screenX*, screenY*: int - which*: int - `type`*: cstring x*, y*: int - ABORT*: int - BLUR*: int - CHANGE*: int - CLICK*: int - DBLCLICK*: int - DRAGDROP*: int - ERROR*: int - FOCUS*: int - KEYDOWN*: int - KEYPRESS*: int - KEYUP*: int - LOAD*: int - MOUSEDOWN*: int - MOUSEMOVE*: int - MOUSEOUT*: int - MOUSEOVER*: int - MOUSEUP*: int - MOVE*: int - RESET*: int - RESIZE*: int - SELECT*: int - SUBMIT*: int - UNLOAD*: int - - Location* = ref LocationObj - LocationObj {.importc.} = object of RootObj + + DataTransferItemKind* {.pure.} = enum + File = "file", + String = "string" + + DataTransferItem* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/DataTransferItem>`_ + kind*: cstring + `type`*: cstring + + DataTransfer* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/DataTransfer>`_ + dropEffect*: cstring + effectAllowed*: cstring + files*: seq[Element] + items*: seq[DataTransferItem] + types*: seq[cstring] + + DataTransferDropEffect* {.pure.} = enum + None = "none", + Copy = "copy", + Link = "link", + Move = "move" + + DataTransferEffectAllowed* {.pure.} = enum + None = "none", + Copy = "copy", + CopyLink = "copyLink", + CopyMove = "copyMove", + Link = "link", + LinkMove = "linkMove", + Move = "move", + All = "all", + Uninitialized = "uninitialized" + + DragEventTypes* = enum + Drag = "drag", + DragEnd = "dragend", + DragEnter = "dragenter", + DragExit = "dragexit", + DragLeave = "dragleave", + DragOver = "dragover", + DragStart = "dragstart", + Drop = "drop" + + DragEvent* {.importc.} = object of MouseEvent + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/DragEvent>`_ + dataTransfer*: DataTransfer + + ClipboardEvent* {.importc.} = object of Event + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/ClipboardEvent>`_ + clipboardData*: DataTransfer + + StorageEvent* {.importc.} = ref object of Event ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/StorageEvent>`_ + key*: cstring + newValue*, oldValue*: cstring + storageArea*: Storage + url*: cstring + + TouchList* {.importc.} = ref object of RootObj + length*: int + + Touch* {.importc.} = ref object of RootObj + identifier*: int + screenX*, screenY*, clientX*, clientY*, pageX*, pageY*: int + target*: Element + radiusX*, radiusY*: int + rotationAngle*: int + force*: float + + TouchEvent* {.importc.} = ref object of UIEvent + changedTouches*, targetTouches*, touches*: seq[Touch] + + Location* {.importc.} = ref object of RootObj hash*: cstring host*: cstring hostname*: cstring @@ -320,43 +1250,50 @@ type port*: cstring protocol*: cstring search*: cstring + origin*: cstring - History* = ref HistoryObj - HistoryObj {.importc.} = object of RootObj + History* {.importc.} = ref object of RootObj length*: int - Navigator* = ref NavigatorObj - NavigatorObj {.importc.} = object of RootObj + Navigator* {.importc.} = ref object of RootObj appCodeName*: cstring appName*: cstring appVersion*: cstring + buildID*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/buildID cookieEnabled*: bool + deviceMemory*: float ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory + doNotTrack*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/doNotTrack language*: cstring + languages*: seq[cstring] ## https://developer.mozilla.org/en-US/docs/Web/API/NavigatorLanguage/languages + maxTouchPoints*: cint ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/maxTouchPoints + onLine*: bool ## https://developer.mozilla.org/en-US/docs/Web/API/NavigatorOnLine/onLine + oscpu*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/oscpu platform*: cstring userAgent*: cstring - mimeTypes*: seq[ref TMimeType] + vendor*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendor + webdriver*: bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/webdriver + mimeTypes*: seq[ref MimeType] - TPlugin* {.importc.} = object of RootObj + Plugin* {.importc.} = object of RootObj description*: cstring filename*: cstring name*: cstring - TMimeType* {.importc.} = object of RootObj + MimeType* {.importc.} = object of RootObj description*: cstring - enabledPlugin*: ref TPlugin + enabledPlugin*: ref Plugin suffixes*: seq[cstring] `type`*: cstring - TLocationBar* {.importc.} = object of RootObj + LocationBar* {.importc.} = object of RootObj visible*: bool - TMenuBar* = TLocationBar - TPersonalBar* = TLocationBar - TScrollBars* = TLocationBar - TToolBar* = TLocationBar - TStatusBar* = TLocationBar - - Screen = ref ScreenObj - ScreenObj {.importc.} = object of RootObj + MenuBar* = LocationBar + PersonalBar* = LocationBar + ScrollBars* = LocationBar + ToolBar* = LocationBar + StatusBar* = LocationBar + + Screen* {.importc.} = ref object of RootObj availHeight*: int availWidth*: int colorDepth*: int @@ -364,29 +1301,214 @@ type pixelDepth*: int width*: int - TTimeOut* {.importc.} = object of RootObj - TInterval* {.importc.} = object of RootObj + TimeOut* {.importc.} = ref object of RootObj + Interval* {.importc.} = ref object of RootObj + + AddEventListenerOptions* = object + capture*: bool + once*: bool + passive*: bool + + FontFaceSetReady* {.importc.} = ref object + ## see: `docs<https://developer.mozilla.org/en-US/docs/Web/API/FontFaceSet/ready>`_ + then*: proc(cb: proc()) + + FontFaceSet* {.importc.} = ref object + ## see: `docs<https://developer.mozilla.org/en-US/docs/Web/API/FontFaceSet>`_ + ready*: FontFaceSetReady + onloadingdone*: proc(event: Event) + + ScrollIntoViewOptions* = object + behavior*: cstring + `block`*: cstring + inline*: cstring + + MediaQueryList* {.importc.} = ref object of EventTarget + matches*: bool + media*: cstring + +since (1, 3): + type + DomParser* = ref object + ## DOM Parser object (defined on browser only, may not be on NodeJS). + ## * https://developer.mozilla.org/en-US/docs/Web/API/DOMParser + ## + ## ```nim + ## let prsr = newDomParser() + ## discard prsr.parseFromString("<html><marquee>Hello World</marquee></html>".cstring, "text/html".cstring) + ## ``` + + DomException* {.importc.} = ref object + ## The DOMException interface represents an abnormal event (called an exception) + ## which occurs as a result of calling a method or accessing a property of a web API. + ## Each exception has a name, which is a short "CamelCase" style string identifying + ## the error or abnormal condition. + ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException + + FileReader* {.importc.} = ref object of EventTarget + ## The FileReader object lets web applications asynchronously read the contents of files + ## (or raw data buffers) stored on the user's computer, using File or Blob objects to specify + ## the file or data to read. + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader + + FileReaderState* = distinct range[0'u16..2'u16] + RootNodeOptions* = object of RootObj + composed*: bool + DocumentOrShadowRoot* {.importc.} = object of RootObj + activeElement*: Element + # styleSheets*: StyleSheetList + ShadowRoot* {.importc.} = ref object of DocumentOrShadowRoot + delegatesFocus*: bool + host*: Element + innerHTML*: cstring + mode*: cstring # "open" or "closed" + ShadowRootInit* = object of RootObj + mode*: cstring + delegatesFocus*: bool + + HTMLSlotElement* {.importc.} = ref object of RootObj + name*: cstring + SlotOptions* = object of RootObj + flatten*: bool + + const + fileReaderEmpty* = 0.FileReaderState + fileReaderLoading* = 1.FileReaderState + fileReaderDone* = 2.FileReaderState + +proc id*(n: Node): cstring {.importcpp: "#.id", nodecl.} +proc `id=`*(n: Node; x: cstring) {.importcpp: "#.id = #", nodecl.} +proc class*(n: Node): cstring {.importcpp: "#.className", nodecl.} +proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} + +proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.} +proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.} + +proc checked*(n: Node): bool {.importcpp: "#.checked", nodecl.} +proc `checked=`*(n: Node; v: bool) {.importcpp: "#.checked = #", nodecl.} + +proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.} + +when defined(nodejs): + # we provide a dummy DOM for nodejs for testing purposes + proc len*(x: Node): int = x.childNodes.len + proc `[]`*(x: Node; idx: int): Element = + assert idx >= 0 and idx < x.childNodes.len + result = cast[Element](x.childNodes[idx]) + + var document* = Document(nodeType: DocumentNode) + document.ownerDocument = document + + proc getElem(x: Element; id: cstring): Element = + if x.id == id: return x + for i in 0..<x.len: + result = getElem(x[i], id) + if result != nil: return result + + proc getElementById*(doc: Document; id: cstring): Element = + getElem(doc.body, id) + proc getElementById*(id: cstring): Element = document.getElementById(id) + + proc appendChild*(parent, n: Node) = + n.parentNode = parent + n.ownerDocument = parent.ownerDocument + parent.childNodes.add n + + proc replaceChild*(parent, newNode, oldNode: Node) = + newNode.parentNode = parent + oldNode.parentNode = nil + var i = 0 + while i < parent.len: + if Node(parent[i]) == oldNode: + parent.childNodes[i] = newNode + return + inc i + raiseAssert "old node not in node list" + + proc removeChild*(parent, child: Node) = + child.parentNode = nil + var i = 0 + while i < parent.len: + if Node(parent[i]) == child: + parent.childNodes.delete(i) + return + inc i + raiseAssert "old node not in node list" + + proc insertBefore*(parent, newNode, before: Node) = + appendChild(parent, newNode) + var i = 0 + while i < parent.len-1: + if Node(parent[i]) == before: + for j in countdown(parent.len-1, i-1): + parent.childNodes[j] = parent.childNodes[j-1] + parent.childNodes[i-1] = newNode + return + inc i + #raiseAssert "before not in node list" + + proc createElement*(d: Document, identifier: cstring): Element = + new(result) + result.nodeName = identifier + result.nodeType = NodeType.ElementNode + + proc createTextNode*(d: Document, identifier: cstring): Node = + new(result) + result.nodeName = "#text" + result.nodeValue = identifier + result.nodeType = NodeType.TextNode + + proc createComment*(d: Document, data: cstring): Node = + new(result) + result.nodeName = "#comment" + result.nodeValue = data + result.nodeType = NodeType.CommentNode + +else: + proc len*(x: Node): int {.importcpp: "#.childNodes.length".} + proc `[]`*(x: Node; idx: int): Element {.importcpp: "#.childNodes[#]".} + proc getElementById*(id: cstring): Element {.importc: "document.getElementById", nodecl.} + proc appendChild*(n, child: Node) {.importcpp.} + proc removeChild*(n, child: Node) {.importcpp.} + proc remove*(child: Node) {.importcpp.} + proc replaceChild*(n, newNode, oldNode: Node) {.importcpp.} + proc insertBefore*(n, newNode, before: Node) {.importcpp.} + proc getElementById*(d: Document, id: cstring): Element {.importcpp.} + proc createElement*(d: Document, identifier: cstring): Element {.importcpp.} + proc createElementNS*(d: Document, namespaceURI, qualifiedIdentifier: cstring): Element {.importcpp.} + proc createTextNode*(d: Document, identifier: cstring): Node {.importcpp.} + proc createComment*(d: Document, data: cstring): Node {.importcpp.} + +proc setTimeout*(action: proc(); ms: int): TimeOut {.importc, nodecl.} +proc clearTimeout*(t: TimeOut) {.importc, nodecl.} +proc setInterval*(action: proc(); ms: int): Interval {.importc, nodecl.} +proc clearInterval*(i: Interval) {.importc, nodecl.} {.push importcpp.} # EventTarget "methods" proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false) +proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), options: AddEventListenerOptions) +proc dispatchEvent*(et: EventTarget, ev: Event) +proc removeEventListener*(et: EventTarget; ev: cstring; cb: proc(ev: Event)) # Window "methods" proc alert*(w: Window, msg: cstring) proc back*(w: Window) proc blur*(w: Window) -proc captureEvents*(w: Window, eventMask: int) {.deprecated.} -proc clearInterval*(w: Window, interval: ref TInterval) -proc clearTimeout*(w: Window, timeout: ref TTimeOut) +proc clearInterval*(w: Window, interval: Interval) +proc clearTimeout*(w: Window, timeout: TimeOut) proc close*(w: Window) proc confirm*(w: Window, msg: cstring): bool proc disableExternalCapture*(w: Window) proc enableExternalCapture*(w: Window) proc find*(w: Window, text: cstring, caseSensitive = false, - backwards = false) + backwards = false): bool proc focus*(w: Window) proc forward*(w: Window) +proc getComputedStyle*(w: Window, e: Node, pe: Node = nil): Style + ## .. warning:: The returned Style may or may not be read-only at run-time in the browser. getComputedStyle is performance costly. + proc handleEvent*(w: Window, e: Event) proc home*(w: Window) proc moveBy*(w: Window, x, y: int) @@ -395,51 +1517,74 @@ proc open*(w: Window, uri, windowname: cstring, properties: cstring = nil): Window proc print*(w: Window) proc prompt*(w: Window, text, default: cstring): cstring -proc releaseEvents*(w: Window, eventMask: int) {.deprecated.} proc resizeBy*(w: Window, x, y: int) proc resizeTo*(w: Window, x, y: int) proc routeEvent*(w: Window, event: Event) proc scrollBy*(w: Window, x, y: int) proc scrollTo*(w: Window, x, y: int) -proc setInterval*(w: Window, code: cstring, pause: int): ref TInterval -proc setTimeout*(w: Window, code: cstring, pause: int): ref TTimeOut +proc setInterval*(w: Window, code: cstring, pause: int): Interval +proc setInterval*(w: Window, function: proc (), pause: int): Interval +proc setTimeout*(w: Window, code: cstring, pause: int): TimeOut +proc setTimeout*(w: Window, function: proc (), pause: int): Interval proc stop*(w: Window) +proc requestAnimationFrame*(w: Window, function: proc (time: float)): int +proc cancelAnimationFrame*(w: Window, id: int) +proc matchMedia*(w: Window, mediaQueryString: cstring): MediaQueryList # Node "methods" -proc appendChild*(n, child: Node) proc appendData*(n: Node, data: cstring) proc cloneNode*(n: Node, copyContent: bool): Node proc deleteData*(n: Node, start, len: int) +proc focus*(e: Node) proc getAttribute*(n: Node, attr: cstring): cstring proc getAttributeNode*(n: Node, attr: cstring): Node +proc hasAttribute*(n: Node, attr: cstring): bool proc hasChildNodes*(n: Node): bool -proc insertBefore*(n, newNode, before: Node) +proc normalize*(n: Node) proc insertData*(n: Node, position: int, data: cstring) proc removeAttribute*(n: Node, attr: cstring) proc removeAttributeNode*(n, attr: Node) -proc removeChild*(n, child: Node) -proc replaceChild*(n, newNode, oldNode: Node) proc replaceData*(n: Node, start, len: int, text: cstring) proc scrollIntoView*(n: Node) +proc scrollIntoView*(n: Node, options: ScrollIntoViewOptions) proc setAttribute*(n: Node, name, value: cstring) proc setAttributeNode*(n: Node, attr: Node) +proc querySelector*(n: Node, selectors: cstring): Element +proc querySelectorAll*(n: Node, selectors: cstring): seq[Element] +proc compareDocumentPosition*(n: Node, otherNode:Node): int +proc lookupPrefix*(n: Node): cstring +proc lookupNamespaceURI*(n: Node): cstring +proc isDefaultNamespace*(n: Node): bool +proc contains*(n: Node): bool +proc isEqualNode*(n: Node): bool +proc isSameNode*(n: Node): bool + +since (1, 3): + proc getRootNode*(n: Node,options: RootNodeOptions): Node + + # DocumentOrShadowRoot + proc getSelection*(n: DocumentOrShadowRoot): Selection + proc elementFromPoint*(n: DocumentOrShadowRoot; x, y: float): Element + + # shadow dom + proc attachShadow*(n: Element): ShadowRoot + proc assignedNodes*(n: HTMLSlotElement; options: SlotOptions): seq[Node] + proc assignedElements*(n: HTMLSlotElement; options: SlotOptions): seq[Element] # Document "methods" -proc captureEvents*(d: Document, eventMask: int) {.deprecated.} proc createAttribute*(d: Document, identifier: cstring): Node -proc createElement*(d: Document, identifier: cstring): Element -proc createTextNode*(d: Document, identifier: cstring): Node -proc getElementById*(d: Document, id: cstring): Element proc getElementsByName*(d: Document, name: cstring): seq[Element] proc getElementsByTagName*(d: Document, name: cstring): seq[Element] proc getElementsByClassName*(d: Document, name: cstring): seq[Element] -proc getSelection*(d: Document): cstring +proc insertNode*(range: Range, node: Node) +proc getSelection*(d: Document): Selection proc handleEvent*(d: Document, event: Event) proc open*(d: Document) -proc releaseEvents*(d: Document, eventMask: int) {.deprecated.} proc routeEvent*(d: Document, event: Event) proc write*(d: Document, text: cstring) proc writeln*(d: Document, text: cstring) +proc querySelector*(d: Document, selectors: cstring): Element +proc querySelectorAll*(d: Document, selectors: cstring): seq[Element] # Element "methods" proc blur*(e: Element) @@ -453,6 +1598,8 @@ proc getElementsByClassName*(e: Element, name: cstring): seq[Element] # FormElement "methods" proc reset*(f: FormElement) proc submit*(f: FormElement) +proc checkValidity*(e: FormElement): bool +proc reportValidity*(e: FormElement): bool # EmbedElement "methods" proc play*(e: EmbedElement) @@ -466,29 +1613,90 @@ proc replace*(loc: Location, s: cstring) proc back*(h: History) proc forward*(h: History) proc go*(h: History, pagesToJump: int) +proc pushState*[T](h: History, stateObject: T, title, url: cstring) # Navigator "methods" proc javaEnabled*(h: Navigator): bool +since (1, 3): + proc canShare*(self: Navigator; data: cstring): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/canShare + proc sendBeacon*(self: Navigator; url, data: cstring): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/sendBeacon + proc vibrate*(self: Navigator; pattern: cint): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vibrate + proc vibrate*(self: Navigator; pattern: openArray[cint]): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vibrate + proc registerProtocolHandler*(self: Navigator; scheme, url, title: cstring) ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler # ClassList "methods" proc add*(c: ClassList, class: cstring) proc remove*(c: ClassList, class: cstring) -proc contains*(c: ClassList, class: cstring):bool +proc contains*(c: ClassList, class: cstring): bool proc toggle*(c: ClassList, class: cstring) # Style "methods" -proc getAttribute*(s: Style, attr: cstring, caseSensitive=false): cstring -proc removeAttribute*(s: Style, attr: cstring, caseSensitive=false) -proc setAttribute*(s: Style, attr, value: cstring, caseSensitive=false) +proc getPropertyValue*(s: Style, property: cstring): cstring +proc removeProperty*(s: Style, property: cstring) +proc setProperty*(s: Style, property, value: cstring, priority = "") +proc getPropertyPriority*(s: Style, property: cstring): cstring + +# Event "methods" +proc preventDefault*(ev: Event) +proc stopImmediatePropagation*(ev: Event) +proc stopPropagation*(ev: Event) + +# KeyboardEvent "methods" +proc getModifierState*(ev: KeyboardEvent, keyArg: cstring): bool + +# MouseEvent "methods" +proc getModifierState*(ev: MouseEvent, keyArg: cstring): bool + +# TouchEvent "methods" +proc identifiedTouch*(list: TouchList): Touch +proc item*(list: TouchList, i: int): Touch + +# DataTransfer "methods" +proc clearData*(dt: DataTransfer, format: cstring) +proc getData*(dt: DataTransfer, format: cstring): cstring +proc setData*(dt: DataTransfer, format: cstring, data: cstring) +proc setDragImage*(dt: DataTransfer, img: Element, xOffset: int, yOffset: int) + +# DataTransferItem "methods" +proc getAsFile*(dti: DataTransferItem): File + +# InputElement "methods" +proc setSelectionRange*(e: InputElement, selectionStart: int, selectionEnd: int, selectionDirection: cstring = "none") +proc setRangeText*(e: InputElement, replacement: cstring, startindex: int = 0, endindex: int = 0, selectionMode: cstring = "preserve") +proc setCustomValidity*(e: InputElement, error: cstring) +proc checkValidity*(e: InputElement): bool + +# Blob "methods" +proc slice*(e: Blob, startindex: int = 0, endindex: int = e.size, contentType: cstring = "") + +# Performance "methods" +proc now*(p: Performance): float + +# Selection "methods" +proc removeAllRanges*(s: Selection) +proc deleteFromDocument*(s: Selection) +proc getRangeAt*(s: Selection, index: int): Range +converter toString*(s: Selection): cstring +proc `$`*(s: Selection): string = $(s.toString()) + +# Storage "methods" +proc getItem*(s: Storage, key: cstring): cstring +proc setItem*(s: Storage, key, value: cstring) +proc clear*(s: Storage) +proc removeItem*(s: Storage, key: cstring) {.pop.} +proc setAttr*(n: Node; key, val: cstring) {.importcpp: "#.setAttribute(@)".} + var window* {.importc, nodecl.}: Window - document* {.importc, nodecl.}: Document navigator* {.importc, nodecl.}: Navigator screen* {.importc, nodecl.}: Screen +when not defined(nodejs): + var document* {.importc, nodecl.}: Document + proc decodeURI*(uri: cstring): cstring {.importc, nodecl.} proc encodeURI*(uri: cstring): cstring {.importc, nodecl.} @@ -499,29 +1707,135 @@ proc decodeURIComponent*(uri: cstring): cstring {.importc, nodecl.} proc encodeURIComponent*(uri: cstring): cstring {.importc, nodecl.} proc isFinite*(x: BiggestFloat): bool {.importc, nodecl.} proc isNaN*(x: BiggestFloat): bool {.importc, nodecl.} -proc parseFloat*(s: cstring): BiggestFloat {.importc, nodecl.} -proc parseInt*(s: cstring): int {.importc, nodecl.} -proc parseInt*(s: cstring, radix: int):int {.importc, nodecl.} + ## see also `math.isNaN`. + +proc newEvent*(name: cstring): Event {.importcpp: "new Event(@)", constructor.} + +proc getElementsByClass*(n: Node; name: cstring): seq[Node] {. + importcpp: "#.getElementsByClassName(#)", nodecl.} type - TEventHandlers* {.deprecated.} = EventTargetObj - TWindow* {.deprecated.} = WindowObj - TFrame* {.deprecated.} = FrameObj - TNode* {.deprecated.} = NodeObj - TDocument* {.deprecated.} = DocumentObj - TElement* {.deprecated.} = ElementObj - TLink* {.deprecated.} = LinkObj - TEmbed* {.deprecated.} = EmbedObj - TAnchor* {.deprecated.} = AnchorObj - TOption* {.deprecated.} = OptionObj - TForm* {.deprecated.} = FormObj - TImage* {.deprecated.} = ImageObj - TNodeType* {.deprecated.} = NodeType - TEvent* {.deprecated.} = EventObj - TLocation* {.deprecated.} = LocationObj - THistory* {.deprecated.} = HistoryObj - TNavigator* {.deprecated.} = NavigatorObj - TStyle* {.deprecated.} = StyleObj - TScreen* {.deprecated.} = ScreenObj - TApplet* {.importc, deprecated.} = object of RootObj + BoundingRect* {.importc.} = object + top*, bottom*, left*, right*, x*, y*, width*, height*: float + +proc getBoundingClientRect*(e: Node): BoundingRect {. + importcpp: "getBoundingClientRect", nodecl.} +proc clientHeight*(): int {. + importcpp: "(window.innerHeight || document.documentElement.clientHeight)@", nodecl.} +proc clientWidth*(): int {. + importcpp: "(window.innerWidth || document.documentElement.clientWidth)@", nodecl.} + +proc inViewport*(el: Node): bool = + let rect = el.getBoundingClientRect() + result = rect.top >= 0 and rect.left >= 0 and + rect.bottom <= clientHeight().float and + rect.right <= clientWidth().float + +proc scrollTop*(e: Node): int {.importcpp: "#.scrollTop", nodecl.} +proc `scrollTop=`*(e: Node, value: int) {.importcpp: "#.scrollTop = #", nodecl.} +proc scrollLeft*(e: Node): int {.importcpp: "#.scrollLeft", nodecl.} +proc scrollHeight*(e: Node): int {.importcpp: "#.scrollHeight", nodecl.} +proc scrollWidth*(e: Node): int {.importcpp: "#.scrollWidth", nodecl.} +proc offsetHeight*(e: Node): int {.importcpp: "#.offsetHeight", nodecl.} +proc offsetWidth*(e: Node): int {.importcpp: "#.offsetWidth", nodecl.} +proc offsetTop*(e: Node): int {.importcpp: "#.offsetTop", nodecl.} +proc offsetLeft*(e: Node): int {.importcpp: "#.offsetLeft", nodecl.} + +since (1, 3): + func newDomParser*(): DomParser {.importcpp: "new DOMParser()".} + ## DOM Parser constructor. + func parseFromString*(this: DomParser; str: cstring; mimeType: cstring): Document {.importcpp.} + ## Parse from string to `Document`. + + proc newDomException*(): DomException {.importcpp: "new DomException()", constructor.} + ## DOM Exception constructor + proc message*(ex: DomException): cstring {.importcpp: "#.message", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException/message + proc name*(ex: DomException): cstring {.importcpp: "#.name", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException/name + + proc newFileReader*(): FileReader {.importcpp: "new FileReader()", constructor.} + ## File Reader constructor + proc error*(f: FileReader): DomException {.importcpp: "#.error", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/error + proc readyState*(f: FileReader): FileReaderState {.importcpp: "#.readyState", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readyState + proc resultAsString*(f: FileReader): cstring {.importcpp: "#.result", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/result + proc abort*(f: FileReader) {.importcpp: "#.abort()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/abort + proc readAsBinaryString*(f: FileReader, b: Blob) {.importcpp: "#.readAsBinaryString(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsBinaryString + proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL + proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText + +since (1, 5): + proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] {.importcpp.} + + +since (1, 7): + + proc insertAdjacentText*(self: Node; position, data: cstring) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentText + + proc insertAdjacentElement*(self: Node; position: cstring; element: Node) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentElement + + proc insertAdjacentHTML*(self: Node; position, html: cstring) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML + + proc after*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/after + + proc before*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/before + + proc append*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/append + + proc closest*(self: Node; cssSelector: cstring): Node {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/closest + + proc hasAttributeNS*(self: Node; namespace, localName: cstring): bool {.importjs: "(#.$1(#, #) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/hasAttributeNS + + proc removeAttributeNS*(self: Node; namespace, attributeName: cstring) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/removeAttributeNS + + proc hasPointerCapture*(self: Node; pointerId: SomeNumber): bool {.importjs: "(#.$1(#) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/hasPointerCapture + + proc releasePointerCapture*(self: Node; pointerId: SomeNumber) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/releasePointerCapture + + proc requestPointerLock*(self: Node) {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/requestPointerLock + + proc replaceChildren*(self: Node; replacements: Node) {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/replaceChildren + + proc replaceWith*(self: Node; replacements: Node) {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/replaceWith + + proc scrollIntoViewIfNeeded*(self: Node; centerIfNeeded: bool) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoViewIfNeeded + + proc setHTML*(self: Node; html: cstring) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/setHTML + + proc toggleAttribute*(self: Node; name: cstring; force = false): bool {.importjs: "(#.$1(#, #) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/toggleAttribute + + proc matches*(self: Node; cssSelector: cstring): bool {.importjs: "(#.$1(#) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/matches + + +since (2, 1): + type VisualViewport* {.importc.} = ref object of EventTarget + offsetLeft*, offsetTop*, pageLeft*, pageTop*, width*, height*, scale*: float + onResize*, onScroll*: proc (event: Event) {.closure.} + + func visualViewport*(self: Window): VisualViewport {.importjs: "#.$1", nodecl.} diff --git a/lib/js/jsconsole.nim b/lib/js/jsconsole.nim new file mode 100644 index 000000000..e74127334 --- /dev/null +++ b/lib/js/jsconsole.nim @@ -0,0 +1,125 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Wrapper for the `console` object for the `JavaScript backend +## <backends.html#backends-the-javascript-target>`_. +## +## Styled Messages +## =============== +## +## CSS-styled messages in the browser are useful for debugging purposes. +## To use them, prefix the message with one or more `%c`, +## and provide the CSS style as the last argument. +## The amount of `%c`'s must match the amount of CSS-styled strings. +## +runnableExamples("-r:off"): + console.log "%c My Debug Message", "color: red" # Notice the "%c" + console.log "%c My Debug %c Message", "color: red", "font-size: 2em" + +import std/private/since, std/private/miscdollars # toLocation + +when not defined(js): + {.error: "This module only works on the JavaScript platform".} + +type Console* = ref object of JsRoot + +proc log*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/log + +proc debug*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/debug + +proc info*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/info + +proc error*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/error + +template exception*(console: Console, args: varargs[untyped]) = + ## Alias for `console.error()`. + error(console, args) + +proc trace*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/trace + +proc warn*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/warn + +proc clear*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/clear + +proc count*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/count + +proc countReset*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/countReset + +proc group*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/group + +proc groupCollapsed*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/groupCollapsed + +proc groupEnd*(console: Console) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/groupEnd + +proc time*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/time + +proc timeEnd*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/timeEnd + +proc timeLog*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/timeLog + +proc table*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/table + +since (1, 5): + type InstantiationInfo = tuple[filename: string, line: int, column: int] + + func getMsg(info: InstantiationInfo; msg: string): string = + var temp = "" + temp.toLocation(info.filename, info.line, info.column + 1) + result.addQuoted("[jsAssert] " & temp) + result.add ',' + result.addQuoted(msg) + + template jsAssert*(console: Console; assertion) = + ## JavaScript `console.assert`, for NodeJS this prints to stderr, + ## assert failure just prints to console and do not quit the program, + ## this is not meant to be better or even equal than normal assertions, + ## is just for when you need faster performance *and* assertions, + ## otherwise use the normal assertions for better user experience. + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/assert + runnableExamples: + console.jsAssert(42 == 42) # OK + console.jsAssert(42 != 42) # Fail, prints "Assertion failed" and continues + console.jsAssert('`' == '\n' and '\t' == '\0') # Message correctly formatted + assert 42 == 42 # Normal assertions keep working + + const + loc = instantiationInfo(fullPaths = compileOption("excessiveStackTrace")) + msg = getMsg(loc, astToStr(assertion)).cstring + {.line: loc.}: + {.emit: ["console.assert(", assertion, ", ", msg, ");"].} + + func dir*(console: Console; obj: auto) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/dir + + func dirxml*(console: Console; obj: auto) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/dirxml + + func timeStamp*(console: Console; label: cstring) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeStamp + ## + ## ..warning:: non-standard + + +var console* {.importc, nodecl.}: Console diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim new file mode 100644 index 000000000..be353875c --- /dev/null +++ b/lib/js/jscore.nim @@ -0,0 +1,153 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module wraps core JavaScript functions. +## +## Unless your application has very +## specific requirements and solely targets JavaScript, you should be using +## the relevant functions in the `math`, `json`, and `times` stdlib +## modules instead. +import std/private/[since, jsutils] + +when not defined(js): + {.error: "This module only works on the JavaScript platform".} + +type + MathLib* = ref object + JsonLib* = ref object + DateLib* = ref object + DateTime* = ref object + +var + Math* {.importc, nodecl.}: MathLib + Date* {.importc, nodecl.}: DateLib + JSON* {.importc, nodecl.}: JsonLib + +# Math library +proc abs*(m: MathLib, a: SomeNumber): SomeNumber {.importcpp.} +proc acos*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc acosh*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc asin*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc asinh*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc atan*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc atan2*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc atanh*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc cbrt*(m: MathLib, f: SomeFloat): SomeFloat {.importcpp.} +proc ceil*(m: MathLib, f: SomeFloat): SomeFloat {.importcpp.} +proc clz32*(m: MathLib, f: SomeInteger): int {.importcpp.} +proc cos*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc cosh*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc exp*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc expm1*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc floor*(m: MathLib, f: SomeFloat): int {.importcpp.} +proc fround*(m: MathLib, f: SomeFloat): float32 {.importcpp.} +proc hypot*(m: MathLib, args: varargs[distinct SomeNumber]): float {.importcpp.} +proc imul*(m: MathLib, a, b: int32): int32 {.importcpp.} +proc log*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc log10*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc log1p*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc log2*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc max*(m: MathLib, a, b: SomeNumber): SomeNumber {.importcpp.} +proc min*[T: SomeNumber | JsRoot](m: MathLib, a, b: T): T {.importcpp.} +proc pow*(m: MathLib, a, b: distinct SomeNumber): float {.importcpp.} +proc random*(m: MathLib): float {.importcpp.} +proc round*(m: MathLib, f: SomeFloat): int {.importcpp.} +proc sign*(m: MathLib, f: SomeNumber): int {.importcpp.} +proc sin*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc sinh*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc sqrt*(m: MathLib, f: SomeFloat): SomeFloat {.importcpp.} +proc tan*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc tanh*(m: MathLib, a: SomeNumber): float {.importcpp.} +proc trunc*(m: MathLib, f: SomeFloat): int {.importcpp.} + +# Date library +proc now*(d: DateLib): int {.importcpp.} +proc UTC*(d: DateLib): int {.importcpp.} +proc parse*(d: DateLib, s: cstring): int {.importcpp.} + +proc newDate*(): DateTime {. + importcpp: "new Date()".} + +proc newDate*(date: int|string): DateTime {. + importcpp: "new Date(#)".} + +whenJsNoBigInt64: + proc newDate*(date: int64): DateTime {. + importcpp: "new Date(#)".} +do: + proc newDate*(date: int64): DateTime {. + importcpp: "new Date(Number(#))".} + +proc newDate*(year, month, day, hours, minutes, + seconds, milliseconds: int): DateTime {. + importcpp: "new Date(#,#,#,#,#,#,#)".} + +proc getDay*(d: DateTime): int {.importcpp.} +proc getFullYear*(d: DateTime): int {.importcpp.} +proc getHours*(d: DateTime): int {.importcpp.} +proc getMilliseconds*(d: DateTime): int {.importcpp.} +proc getMinutes*(d: DateTime): int {.importcpp.} +proc getMonth*(d: DateTime): int {.importcpp.} +proc getSeconds*(d: DateTime): int {.importcpp.} +proc getTime*(d: DateTime): int {.importcpp.} +proc getTimezoneOffset*(d: DateTime): int {.importcpp.} +proc getUTCDate*(d: DateTime): int {.importcpp.} +proc getUTCDay*(d: DateTime): int {.importcpp.} +proc getUTCFullYear*(d: DateTime): int {.importcpp.} +proc getUTCHours*(d: DateTime): int {.importcpp.} +proc getUTCMilliseconds*(d: DateTime): int {.importcpp.} +proc getUTCMinutes*(d: DateTime): int {.importcpp.} +proc getUTCMonth*(d: DateTime): int {.importcpp.} +proc getUTCSeconds*(d: DateTime): int {.importcpp.} +proc getYear*(d: DateTime): int {.importcpp.} + +proc setFullYear*(d: DateTime, year: int) {.importcpp.} + +func toDateString*(d: DateTime): cstring {.importcpp.} +func toISOString*(d: DateTime): cstring {.importcpp.} +func toJSON*(d: DateTime): cstring {.importcpp.} +proc toString*(d: DateTime): cstring {.importcpp.} +func toTimeString*(d: DateTime): cstring {.importcpp.} +func toUTCString*(d: DateTime): cstring {.importcpp.} + +#JSON library +proc stringify*(l: JsonLib, s: JsRoot): cstring {.importcpp.} +proc parse*(l: JsonLib, s: cstring): JsRoot {.importcpp.} + + +since (1, 5): + func debugger*() {.importjs: "debugger@".} + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger + + func copyWithin*[T](self: openArray[T]; target: int): seq[T] {.importjs: "#.copyWithin(#)".} + func copyWithin*[T](self: openArray[T]; target, start: int): seq[T] {.importjs: "#.copyWithin(#, #)".} + func copyWithin*[T](self: openArray[T]; target, start, ends: int): seq[T] {.importjs: "#.copyWithin(#, #, #)".} = + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin + ## `copyWithin` uses shallow copy. + runnableExamples: + assert ['a', 'b', 'c', 'd', 'e'].copyWithin(0, 3, 4) == @['d', 'b', 'c', 'd', 'e'] + assert ['a', 'b', 'c', 'd', 'e'].copyWithin(1, 3) == @['a', 'd', 'e', 'd', 'e'] + assert [1, 2, 3, 4, 5].copyWithin(-2) == @[1, 2, 3, 1, 2] + assert [1, 2, 3, 4, 5].copyWithin(0, 3) == @[4, 5, 3, 4, 5] + assert [1, 2, 3, 4, 5].copyWithin(0, 3, 4) == @[4, 2, 3, 4, 5] + assert [1, 2, 3, 4, 5].copyWithin(-2, -3, -1) == @[1, 2, 3, 3, 4] + + +since (1, 7): + func shift*[T](self: seq[T]): T {.importjs: "#.$1()".} = + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift + runnableExamples: + var arrai = @[1, 2, 3] + assert arrai.shift() == 1 + assert arrai == @[2, 3] + + func queueMicrotask*(function: proc) {.importjs: "$1(#)".} = + ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask + ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide + runnableExamples"-r:off": queueMicrotask(proc() = echo "Microtask") diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim new file mode 100644 index 000000000..d50d58ae5 --- /dev/null +++ b/lib/js/jsffi.nim @@ -0,0 +1,527 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim Authors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This Module implements types and macros to facilitate the wrapping of, and +## interaction with JavaScript libraries. Using the provided types `JsObject` +## and `JsAssoc` together with the provided macros allows for smoother +## interfacing with JavaScript, allowing for example quick and easy imports of +## JavaScript variables: + +runnableExamples: + # Here, we are using jQuery for just a few calls and do not want to wrap the + # whole library: + + # import the document object and the console + var document {.importc, nodecl.}: JsObject + var console {.importc, nodecl.}: JsObject + # import the "$" function + proc jq(selector: JsObject): JsObject {.importjs: "$$(#)".} + + # Use jQuery to make the following code run, after the document is ready. + # This uses an experimental `.()` operator for `JsObject`, to emit + # JavaScript calls, when no corresponding proc exists for `JsObject`. + proc main = + jq(document).ready(proc() = + console.log("Hello JavaScript!") + ) + + +when not defined(js) and not defined(nimsuggest): + {.fatal: "Module jsFFI is designed to be used with the JavaScript backend.".} + +import std/[macros, tables] + +const + setImpl = "#[#] = #" + getImpl = "#[#]" + +var + mangledNames {.compileTime.} = initTable[string, string]() + nameCounter {.compileTime.} = 0 + +proc validJsName(name: string): bool = + result = true + const reservedWords = ["break", "case", "catch", "class", "const", "continue", + "debugger", "default", "delete", "do", "else", "export", "extends", + "finally", "for", "function", "if", "import", "in", "instanceof", "new", + "return", "super", "switch", "this", "throw", "try", "typeof", "var", + "void", "while", "with", "yield", "enum", "implements", "interface", + "let", "package", "private", "protected", "public", "static", "await", + "abstract", "boolean", "byte", "char", "double", "final", "float", "goto", + "int", "long", "native", "short", "synchronized", "throws", "transient", + "volatile", "null", "true", "false"] + case name + of reservedWords: return false + else: discard + if name[0] notin {'A'..'Z','a'..'z','_','$'}: return false + for chr in name: + if chr notin {'A'..'Z','a'..'z','_','$','0'..'9'}: + return false + +template mangleJsName(name: string): string = + inc nameCounter + "mangledName" & $nameCounter + +# only values that can be mapped 1 to 1 with cstring should be keys: they have an injective function with cstring + +proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importjs: "parseInt(#)".} + +proc toJsKey*[T: enum](text: cstring, t: type T): T = + T(text.toJsKey(int)) + +proc toJsKey*(text: cstring, t: type cstring): cstring = + text + +proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importjs: "parseFloat(#)".} + +type + JsKey* = concept a, type T + cstring.toJsKey(T) is T + + JsObject* = ref object of JsRoot + ## Dynamically typed wrapper around a JavaScript object. + JsAssoc*[K: JsKey, V] = ref object of JsRoot + ## Statically typed wrapper around a JavaScript object. + + js* = JsObject + +var + jsArguments* {.importc: "arguments", nodecl}: JsObject + ## JavaScript's arguments pseudo-variable. + jsNull* {.importc: "null", nodecl.}: JsObject + ## JavaScript's null literal. + jsUndefined* {.importc: "undefined", nodecl.}: JsObject + ## JavaScript's undefined literal. + jsDirname* {.importc: "__dirname", nodecl.}: cstring + ## JavaScript's __dirname pseudo-variable. + jsFilename* {.importc: "__filename", nodecl.}: cstring + ## JavaScript's __filename pseudo-variable. + +proc isNull*[T](x: T): bool {.noSideEffect, importjs: "(# === null)".} + ## Checks if a value is exactly null. + +proc isUndefined*[T](x: T): bool {.noSideEffect, importjs: "(# === undefined)".} + ## Checks if a value is exactly undefined. + +# Exceptions +type + JsError* {.importc: "Error".} = object of JsRoot + message*: cstring + JsEvalError* {.importc: "EvalError".} = object of JsError + JsRangeError* {.importc: "RangeError".} = object of JsError + JsReferenceError* {.importc: "ReferenceError".} = object of JsError + JsSyntaxError* {.importc: "SyntaxError".} = object of JsError + JsTypeError* {.importc: "TypeError".} = object of JsError + JsURIError* {.importc: "URIError".} = object of JsError + +# New +proc newJsObject*: JsObject {.importjs: "{@}".} + ## Creates a new empty JsObject. + +proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importjs: "{@}".} + ## Creates a new empty JsAssoc with key type `K` and value type `V`. + +# Checks +proc hasOwnProperty*(x: JsObject, prop: cstring): bool + {.importjs: "#.hasOwnProperty(#)".} + ## Checks, whether `x` has a property of name `prop`. + +proc jsTypeOf*(x: JsObject): cstring {.importjs: "typeof(#)".} + ## Returns the name of the JsObject's JavaScript type as a cstring. + +proc jsNew*(x: auto): JsObject {.importjs: "(new #)".} + ## Turns a regular function call into an invocation of the + ## JavaScript's `new` operator. + +proc jsDelete*(x: auto): JsObject {.importjs: "(delete #)".} + ## JavaScript's `delete` operator. + +proc require*(module: cstring): JsObject {.importc.} + ## JavaScript's `require` function. + +# Conversion to and from JsObject +proc to*(x: JsObject, T: typedesc): T {.importjs: "(#)".} + ## Converts a JsObject `x` to type `T`. + +proc toJs*[T](val: T): JsObject {.importjs: "(#)".} + ## Converts a value of any type to type JsObject. + +template toJs*(s: string): JsObject = cstring(s).toJs + +macro jsFromAst*(n: untyped): untyped = + result = n + if n.kind == nnkStmtList: + result = newProc(procType = nnkDo, body = result) + return quote: toJs(`result`) + +proc `&`*(a, b: cstring): cstring {.importjs: "(# + #)".} + ## Concatenation operator for JavaScript strings. + +proc `+`*(x, y: JsObject): JsObject {.importjs: "(# + #)".} +proc `-`*(x, y: JsObject): JsObject {.importjs: "(# - #)".} +proc `*`*(x, y: JsObject): JsObject {.importjs: "(# * #)".} +proc `/`*(x, y: JsObject): JsObject {.importjs: "(# / #)".} +proc `%`*(x, y: JsObject): JsObject {.importjs: "(# % #)".} +proc `+=`*(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} +proc `-=`*(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} +proc `*=`*(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} +proc `/=`*(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} +proc `%=`*(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} +proc `++`*(x: JsObject): JsObject {.importjs: "(++#)".} +proc `--`*(x: JsObject): JsObject {.importjs: "(--#)".} +proc `>`*(x, y: JsObject): JsObject {.importjs: "(# > #)".} +proc `<`*(x, y: JsObject): JsObject {.importjs: "(# < #)".} +proc `>=`*(x, y: JsObject): JsObject {.importjs: "(# >= #)".} +proc `<=`*(x, y: JsObject): JsObject {.importjs: "(# <= #)".} +proc `**`*(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} + # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 +proc `and`*(x, y: JsObject): JsObject {.importjs: "(# && #)".} +proc `or`*(x, y: JsObject): JsObject {.importjs: "(# || #)".} +proc `not`*(x: JsObject): JsObject {.importjs: "(!#)".} +proc `in`*(x, y: JsObject): JsObject {.importjs: "(# in #)".} + +proc `[]`*(obj: JsObject, field: cstring): JsObject {.importjs: getImpl.} + ## Returns the value of a property of name `field` from a JsObject `obj`. + +proc `[]`*(obj: JsObject, field: int): JsObject {.importjs: getImpl.} + ## Returns the value of a property of name `field` from a JsObject `obj`. + +proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importjs: setImpl.} + ## Sets the value of a property of name `field` in a JsObject `obj` to `v`. + +proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importjs: setImpl.} + ## Sets the value of a property of name `field` in a JsObject `obj` to `v`. + +proc `[]`*[K: JsKey, V](obj: JsAssoc[K, V], field: K): V + {.importjs: getImpl.} + ## Returns the value of a property of name `field` from a JsAssoc `obj`. + +proc `[]=`*[K: JsKey, V](obj: JsAssoc[K, V], field: K, val: V) + {.importjs: setImpl.} + ## Sets the value of a property of name `field` in a JsAssoc `obj` to `v`. + +proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V = + obj[cstring(field)] + +proc `[]=`*[V](obj: JsAssoc[cstring, V], field: string, val: V) = + obj[cstring(field)] = val + +proc `==`*(x, y: JsRoot): bool {.importjs: "(# === #)".} + ## Compares two JsObjects or JsAssocs. Be careful though, as this is comparison + ## like in JavaScript, so if your JsObjects are in fact JavaScript Objects, + ## and not strings or numbers, this is a *comparison of references*. + +{.experimental.} +macro `.`*(obj: JsObject, field: untyped): JsObject = + ## Experimental dot accessor (get) for type JsObject. + ## Returns the value of a property of name `field` from a JsObject `x`. + runnableExamples: + let obj = newJsObject() + obj.a = 20 + assert obj.a.to(int) == 20 + if validJsName($field): + let importString = "#." & $field + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: JsObject): JsObject + {.importjs: `importString`.} + `helperName`(`obj`) + else: + if not mangledNames.hasKey($field): + mangledNames[$field] = mangleJsName($field) + let importString = "#." & mangledNames[$field] + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: JsObject): JsObject + {.importjs: `importString`.} + `helperName`(`obj`) + +macro `.=`*(obj: JsObject, field, value: untyped): untyped = + ## Experimental dot accessor (set) for type JsObject. + ## Sets the value of a property of name `field` in a JsObject `x` to `value`. + if validJsName($field): + let importString = "#." & $field & " = #" + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: JsObject, v: auto) + {.importjs: `importString`.} + `helperName`(`obj`, `value`) + else: + if not mangledNames.hasKey($field): + mangledNames[$field] = mangleJsName($field) + let importString = "#." & mangledNames[$field] & " = #" + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: JsObject, v: auto) + {.importjs: `importString`.} + `helperName`(`obj`, `value`) + +macro `.()`*(obj: JsObject, + field: untyped, + args: varargs[JsObject, jsFromAst]): JsObject = + ## Experimental "method call" operator for type JsObject. + ## Takes the name of a method of the JavaScript object (`field`) and calls + ## it with `args` as arguments, returning a JsObject (which may be discarded, + ## and may be `undefined`, if the method does not return anything, + ## so be careful when using this.) + ## + ## Example: + ## ```nim + ## # Let's get back to the console example: + ## var console {.importc, nodecl.}: JsObject + ## let res = console.log("I return undefined!") + ## console.log(res) # This prints undefined, as console.log always returns + ## # undefined. Thus one has to be careful, when using + ## # JsObject calls. + ## ``` + var importString: string + if validJsName($field): + importString = "#." & $field & "(@)" + else: + if not mangledNames.hasKey($field): + mangledNames[$field] = mangleJsName($field) + importString = "#." & mangledNames[$field] & "(@)" + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: JsObject): JsObject + {.importjs: `importString`, discardable.} + `helperName`(`obj`) + for idx in 0 ..< args.len: + let paramName = newIdentNode("param" & $idx) + result[0][3].add newIdentDefs(paramName, newIdentNode("JsObject")) + result[1].add args[idx].copyNimTree + +macro `.`*[K: cstring, V](obj: JsAssoc[K, V], + field: untyped): V = + ## Experimental dot accessor (get) for type JsAssoc. + ## Returns the value of a property of name `field` from a JsObject `x`. + var importString: string + if validJsName($field): + importString = "#." & $field + else: + if not mangledNames.hasKey($field): + mangledNames[$field] = mangleJsName($field) + importString = "#." & mangledNames[$field] + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: type(`obj`)): `obj`.V + {.importjs: `importString`.} + `helperName`(`obj`) + +macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], + field: untyped, + value: V): untyped = + ## Experimental dot accessor (set) for type JsAssoc. + ## Sets the value of a property of name `field` in a JsObject `x` to `value`. + var importString: string + if validJsName($field): + importString = "#." & $field & " = #" + else: + if not mangledNames.hasKey($field): + mangledNames[$field] = mangleJsName($field) + importString = "#." & mangledNames[$field] & " = #" + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: type(`obj`), v: `obj`.V) + {.importjs: `importString`.} + `helperName`(`obj`, `value`) + +macro `.()`*[K: cstring, V: proc](obj: JsAssoc[K, V], + field: untyped, + args: varargs[untyped]): auto = + ## Experimental "method call" operator for type JsAssoc. + ## Takes the name of a method of the JavaScript object (`field`) and calls + ## it with `args` as arguments. Here, everything is typechecked, so you do not + ## have to worry about `undefined` return values. + let dotOp = bindSym"." + result = quote do: + (`dotOp`(`obj`, `field`))() + for elem in args: + result.add elem + +# Iterators: + +iterator pairs*(obj: JsObject): (cstring, JsObject) = + ## Yields tuples of type `(cstring, JsObject)`, with the first entry + ## being the `name` of a fields in the JsObject and the second being its + ## value wrapped into a JsObject. + var k: cstring + var v: JsObject + {.emit: "for (var `k` in `obj`) {".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `obj`[`k`];".} + yield (k, v) + {.emit: "}".} + +iterator items*(obj: JsObject): JsObject = + ## Yields the `values` of each field in a JsObject, wrapped into a JsObject. + var v: JsObject + {.emit: "for (var k in `obj`) {".} + {.emit: " if (!`obj`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `obj`[k];".} + yield v + {.emit: "}".} + +iterator keys*(obj: JsObject): cstring = + ## Yields the `names` of each field in a JsObject. + var k: cstring + {.emit: "for (var `k` in `obj`) {".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} + yield k + {.emit: "}".} + +iterator pairs*[K: JsKey, V](assoc: JsAssoc[K, V]): (K,V) = + ## Yields tuples of type `(K, V)`, with the first entry + ## being a `key` in the JsAssoc and the second being its corresponding value. + var k: cstring + var v: V + {.emit: "for (var `k` in `assoc`) {".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `assoc`[`k`];".} + yield (k.toJsKey(K), v) + {.emit: "}".} + +iterator items*[K, V](assoc: JsAssoc[K, V]): V = + ## Yields the `values` in a JsAssoc. + var v: V + {.emit: "for (var k in `assoc`) {".} + {.emit: " if (!`assoc`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `assoc`[k];".} + yield v + {.emit: "}".} + +iterator keys*[K: JsKey, V](assoc: JsAssoc[K, V]): K = + ## Yields the `keys` in a JsAssoc. + var k: cstring + {.emit: "for (var `k` in `assoc`) {".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} + yield k.toJsKey(K) + {.emit: "}".} + +# Literal generation + +macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto = + ## Takes a `typedesc` as its first argument, and a series of expressions of + ## type `key: value`, and returns a value of the specified type with each + ## field `key` set to `value`, as specified in the arguments of `{}`. + ## + ## Example: + ## + ## ```nim + ## # Let's say we have a type with a ton of fields, where some fields do not + ## # need to be set, and we do not want those fields to be set to `nil`: + ## type + ## ExtremelyHugeType = ref object + ## a, b, c, d, e, f, g: int + ## h, i, j, k, l: cstring + ## # And even more fields ... + ## + ## let obj = ExtremelyHugeType{ a: 1, k: "foo".cstring, d: 42 } + ## + ## # This generates roughly the same JavaScript as: + ## {.emit: "var obj = {a: 1, k: "foo", d: 42};".} + ## ``` + let a = ident"a" + var body = quote do: + var `a` {.noinit.}: `typ` + {.emit: "`a` = {};".} + for x in xs.children: + if x.kind == nnkExprColonExpr: + let + k = x[0] + kString = quote do: + when compiles($`k`): $`k` else: "invalid" + v = x[1] + body.add quote do: + when compiles(`a`.`k`): + `a`.`k` = `v` + elif compiles(`a`[`k`]): + `a`[`k`] = `v` + else: + `a`[`kString`] = `v` + + else: + error("Expression `" & $x.toStrLit & "` not allowed in `{}` macro") + + body.add quote do: + return `a` + + result = quote do: + proc inner(): `typ` {.gensym.} = + `body` + inner() + +# Macro to build a lambda using JavaScript's `this` +# from a proc, `this` being the first argument. + +proc replaceSyms(n: NimNode): NimNode = + if n.kind == nnkSym: + result = newIdentNode($n) + else: + result = n + for i in 0..<n.len: + result[i] = replaceSyms(n[i]) + +macro bindMethod*(procedure: typed): auto {.deprecated: "Don't use it with closures".} = + ## Takes the name of a procedure and wraps it into a lambda missing the first + ## argument, which passes the JavaScript builtin `this` as the first + ## argument to the procedure. Returns the resulting lambda. + ## + ## Example: + ## + ## We want to generate roughly this JavaScript: + ## ```js + ## var obj = {a: 10}; + ## obj.someMethod = function() { + ## return this.a + 42; + ## }; + ## ``` + ## + ## We can achieve this using the `bindMethod` macro: + ## + ## ```nim + ## let obj = JsObject{ a: 10 } + ## proc someMethodImpl(that: JsObject): int = + ## that.a.to(int) + 42 + ## obj.someMethod = bindMethod someMethodImpl + ## + ## # Alternatively: + ## obj.someMethod = bindMethod + ## proc(that: JsObject): int = that.a.to(int) + 42 + ## ``` + if not (procedure.kind == nnkSym or procedure.kind == nnkLambda): + error("Argument has to be a proc or a symbol corresponding to a proc.") + var + rawProc = if procedure.kind == nnkSym: + getImpl(procedure) + else: + procedure + args = rawProc[3].copyNimTree.replaceSyms + thisType = args[1][1] + params = newNimNode(nnkFormalParams).add(args[0]) + body = newNimNode(nnkLambda) + this = newIdentNode("this") + # construct the `this` parameter: + thisQuote = quote do: + var `this` {.nodecl, importc: "this".}: `thisType` + call = newNimNode(nnkCall).add(rawProc[0], thisQuote[0][0][0]) + # construct the procedure call inside the method + if args.len > 2: + for idx in 2..args.len-1: + params.add(args[idx]) + call.add(args[idx][0]) + body.add(newNimNode(nnkEmpty), + rawProc[1], + rawProc[2], + params, + rawProc[4], + rawProc[5], + newTree(nnkStmtList, thisQuote, call) + ) + result = body diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim new file mode 100644 index 000000000..2d931eb20 --- /dev/null +++ b/lib/js/jsre.nim @@ -0,0 +1,97 @@ +## Regular Expressions for the JavaScript target. +## * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions +when not defined(js): + {.error: "This module only works on the JavaScript platform".} + +type RegExp* = ref object of JsRoot + ## Regular Expressions for JavaScript target. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp + flags*: cstring ## cstring that contains the flags of the RegExp object. + dotAll*: bool ## Whether `.` matches newlines or not. + global*: bool ## Whether to test against all possible matches in a string, or only against the first. + ignoreCase*: bool ## Whether to ignore case while attempting a match in a string. + multiline*: bool ## Whether to search in strings across multiple lines. + source*: cstring ## The text of the pattern. + sticky*: bool ## Whether the search is sticky. + unicode*: bool ## Whether Unicode features are enabled. + lastIndex*: cint ## Index at which to start the next match (read/write property). + input*: cstring ## Read-only and modified on successful match. + lastMatch*: cstring ## Ditto. + lastParen*: cstring ## Ditto. + leftContext*: cstring ## Ditto. + rightContext*: cstring ## Ditto. + hasIndices*: bool ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices + + +func newRegExp*(pattern: cstring; flags: cstring): RegExp {.importjs: "new RegExp(@)".} + ## Creates a new RegExp object. + +func newRegExp*(pattern: cstring): RegExp {.importjs: "new RegExp(@)".} + +func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.compile(@)".} + ## Recompiles a regular expression during execution of a script. + +func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} + ## Returns a new string with some or all matches of a pattern replaced by given replacement + +func replace*(pattern: cstring, self: RegExp, cb: proc (args: varargs[cstring]): cstring): cstring {.importcpp.} + ## Returns a new string with some or all matches of a pattern replaced by given callback function + +func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.split(#) || [])".} + ## Divides a string into an ordered list of substrings and returns the array + +func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.match(#) || [])".} + ## Returns an array of matches of a RegExp against given string + +func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "(#.exec(#) || [])".} + ## Executes a search for a match in its string parameter. + +func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} + ## Returns a string representing the RegExp object. + +func `$`*(self: RegExp): string = $toCstring(self) + +func contains*(pattern: cstring; self: RegExp): bool = + ## Tests for a substring match in its string parameter. + runnableExamples: + let jsregex: RegExp = newRegExp(r"bc$", r"i") + assert jsregex in r"abc" + assert jsregex notin r"abcd" + assert "xabc".contains jsregex + {.emit: "`result` = `self`.test(`pattern`);".} + +func startsWith*(pattern: cstring; self: RegExp): bool = + ## Tests if string starts with given RegExp + runnableExamples: + let jsregex: RegExp = newRegExp(r"abc", r"i") + assert "abcd".startsWith jsregex + pattern.contains(newRegExp(("^" & $(self.source)).cstring, self.flags)) + +func endsWith*(pattern: cstring; self: RegExp): bool = + ## Tests if string ends with given RegExp + runnableExamples: + let jsregex: RegExp = newRegExp(r"bcd", r"i") + assert "abcd".endsWith jsregex + pattern.contains(newRegExp(($(self.source) & "$").cstring, self.flags)) + + +runnableExamples: + let jsregex: RegExp = newRegExp(r"\s+", r"i") + jsregex.compile(r"\w+", r"i") + assert "nim javascript".contains jsregex + assert jsregex.exec(r"nim javascript") == @["nim".cstring] + assert jsregex.toCstring() == r"/\w+/i" + jsregex.compile(r"[0-9]", r"i") + assert "0123456789abcd".contains jsregex + assert $jsregex == "/[0-9]/i" + jsregex.compile(r"abc", r"i") + assert "abcd".startsWith jsregex + assert "dabc".endsWith jsregex + jsregex.compile(r"\d", r"i") + assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] + jsregex.compile(r"[lw]", r"i") + assert "hello world".replace(jsregex,"X") == "heXlo world" + jsregex.compile(r"([a-z])\1*", r"g") + assert "abbcccdddd".replace(jsregex, proc (m: varargs[cstring]): cstring = ($m[0] & $(m.len)).cstring) == "a1b2c3d4" + let digitsRegex: RegExp = newRegExp(r"\d") + assert "foo".match(digitsRegex) == @[] diff --git a/lib/nimbase.h b/lib/nimbase.h index bba5ac023..cf0c8002b 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -10,20 +10,71 @@ /* compiler symbols: __BORLANDC__ _MSC_VER -__WATCOMC__ -__LCC__ __GNUC__ -__DMC__ -__POCC__ __TINYC__ __clang__ +__AVR__ +__arm__ +__EMSCRIPTEN__ */ #ifndef NIMBASE_H #define NIMBASE_H -#if defined(__GNUC__) +/*------------ declaring a custom attribute to support using LLVM's Address Sanitizer ------------ */ + +/* + This definition exists to provide support for using the LLVM ASAN (Address SANitizer) tooling with Nim. This + should only be used to mark implementations of the GC system that raise false flags with the ASAN tooling, or + for functions that are hot and need to be disabled for performance reasons. Based on the official ASAN + documentation, both the clang and gcc compilers are supported. In addition to that, a check is performed to + verify that the necessary attribute is supported by the compiler. + + To flag a proc as ignored, append the following code pragma to the proc declaration: + {.codegenDecl: "CLANG_NO_SANITIZE_ADDRESS $# $#$#".} + + For further information, please refer to the official documentation: + https://github.com/google/sanitizers/wiki/AddressSanitizer + */ +#define CLANG_NO_SANITIZE_ADDRESS +#if defined(__clang__) +# if __has_attribute(no_sanitize_address) +# undef CLANG_NO_SANITIZE_ADDRESS +# define CLANG_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +# endif +#endif + + +/* ------------ ignore typical warnings in Nim-generated files ------------- */ +#if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic ignored "-Wpragmas" +# pragma GCC diagnostic ignored "-Wwritable-strings" +# pragma GCC diagnostic ignored "-Winvalid-noreturn" +# pragma GCC diagnostic ignored "-Wformat" +# pragma GCC diagnostic ignored "-Wlogical-not-parentheses" +# pragma GCC diagnostic ignored "-Wlogical-op-parentheses" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wunused-function" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Winvalid-offsetof" +# pragma GCC diagnostic ignored "-Wtautological-compare" +# pragma GCC diagnostic ignored "-Wswitch-bool" +# pragma GCC diagnostic ignored "-Wmacro-redefined" +# pragma GCC diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers" +# pragma GCC diagnostic ignored "-Wpointer-bool-conversion" +# pragma GCC diagnostic ignored "-Wconstant-conversion" +#endif + +#if defined(_MSC_VER) +# pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) +# pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) +# pragma warning(disable: 4710 4711 4774 4800 4809 4820 4996 4090 4297) +#endif +/* ------------------------------------------------------------------------- */ + +#if defined(__GNUC__) && !defined(__ZEPHYR__) +/* Zephyr does some magic in it's headers that override the GCC stdlib. This breaks that. */ # define _GNU_SOURCE 1 #endif @@ -36,11 +87,8 @@ __clang__ #endif /* calling convention mess ----------------------------------------------- */ -#if defined(__GNUC__) || defined(__LCC__) || defined(__POCC__) \ - || defined(__TINYC__) +#if defined(__GNUC__) || defined(__TINYC__) /* these should support C99's inline */ - /* the test for __POCC__ has to come before the test for _MSC_VER, - because PellesC defines _MSC_VER too. This is brain-dead. */ # define N_INLINE(rettype, name) inline rettype name #elif defined(__BORLANDC__) || defined(_MSC_VER) /* Borland's compiler is really STRANGE here; note that the __fastcall @@ -48,33 +96,52 @@ __clang__ the return type, so we do not handle this mess in the code generator but rather here. */ # define N_INLINE(rettype, name) __inline rettype name -#elif defined(__DMC__) -# define N_INLINE(rettype, name) inline rettype name -#elif defined(__WATCOMC__) -# define N_INLINE(rettype, name) __inline rettype name #else /* others are less picky: */ # define N_INLINE(rettype, name) rettype __inline name #endif -#if defined(__POCC__) -# define NIM_CONST /* PCC is really picky with const modifiers */ -# undef _MSC_VER /* Yeah, right PCC defines _MSC_VER even if it is - not that compatible. Well done. */ -#elif defined(__cplusplus) +#define N_INLINE_PTR(rettype, name) rettype (*name) + +#if defined(__cplusplus) # define NIM_CONST /* C++ is picky with const modifiers */ #else # define NIM_CONST const #endif -#if (defined(WIN32) || defined(_WIN32) || defined(__WIN32__)) +/* + NIM_THREADVAR declaration based on + http://stackoverflow.com/questions/18298280/how-to-declare-a-variable-as-thread-local-portably +*/ +#if defined _WIN32 +# if defined _MSC_VER || defined __BORLANDC__ +# define NIM_THREADVAR __declspec(thread) +# else +# define NIM_THREADVAR __thread +# endif +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__ +# define NIM_THREADVAR _Thread_local +#elif defined _WIN32 && ( \ + defined _MSC_VER || \ + defined __ICL || \ + defined __BORLANDC__ ) # define NIM_THREADVAR __declspec(thread) -#else +#elif defined(__TINYC__) || defined(__GENODE__) +# define NIM_THREADVAR +/* note that ICC (linux) and Clang are covered by __GNUC__ */ +#elif defined __GNUC__ || \ + defined __SUNPRO_C || \ + defined __xlC__ # define NIM_THREADVAR __thread +#else +# error "Cannot define NIM_THREADVAR" +#endif + +#if defined(__cplusplus) + #define NIM_THREAD_LOCAL thread_local #endif /* --------------- how int64 constants should be declared: ----------- */ -#if defined(__GNUC__) || defined(__LCC__) || \ - defined(__POCC__) || defined(__DMC__) || defined(_MSC_VER) +#if defined(__GNUC__) || defined(_MSC_VER) # define IL64(x) x##LL #else /* works only without LL */ # define IL64(x) ((NI64)x) @@ -88,28 +155,40 @@ __clang__ # define NIM_CAST(type, ptr) ((type)(ptr)) #endif + /* ------------------------------------------------------------------- */ +#ifdef __cplusplus +# define NIM_EXTERNC extern "C" +#else +# define NIM_EXTERNC +#endif #if defined(WIN32) || defined(_WIN32) /* only Windows has this mess... */ +# define N_LIB_PRIVATE # define N_CDECL(rettype, name) rettype __cdecl name # define N_STDCALL(rettype, name) rettype __stdcall name # define N_SYSCALL(rettype, name) rettype __syscall name # define N_FASTCALL(rettype, name) rettype __fastcall name -# define N_SAFECALL(rettype, name) rettype __safecall name +# define N_THISCALL(rettype, name) rettype __thiscall name +# define N_SAFECALL(rettype, name) rettype __stdcall name /* function pointers with calling convention: */ # define N_CDECL_PTR(rettype, name) rettype (__cdecl *name) # define N_STDCALL_PTR(rettype, name) rettype (__stdcall *name) # define N_SYSCALL_PTR(rettype, name) rettype (__syscall *name) # define N_FASTCALL_PTR(rettype, name) rettype (__fastcall *name) -# define N_SAFECALL_PTR(rettype, name) rettype (__safecall *name) +# define N_THISCALL_PTR(rettype, name) rettype (__thiscall *name) +# define N_SAFECALL_PTR(rettype, name) rettype (__stdcall *name) -# ifdef __cplusplus -# define N_LIB_EXPORT extern "C" __declspec(dllexport) +# ifdef __EMSCRIPTEN__ +# define N_LIB_EXPORT NIM_EXTERNC __declspec(dllexport) __attribute__((used)) +# define N_LIB_EXPORT_VAR __declspec(dllexport) __attribute__((used)) # else -# define N_LIB_EXPORT extern __declspec(dllexport) +# define N_LIB_EXPORT NIM_EXTERNC __declspec(dllexport) +# define N_LIB_EXPORT_VAR __declspec(dllexport) # endif # define N_LIB_IMPORT extern __declspec(dllimport) #else +# define N_LIB_PRIVATE __attribute__((visibility("hidden"))) # if defined(__GNUC__) # define N_CDECL(rettype, name) rettype name # define N_STDCALL(rettype, name) rettype name @@ -135,10 +214,12 @@ __clang__ # define N_FASTCALL_PTR(rettype, name) rettype (*name) # define N_SAFECALL_PTR(rettype, name) rettype (*name) # endif -# ifdef __cplusplus -# define N_LIB_EXPORT extern "C" +# ifdef __EMSCRIPTEN__ +# define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default"), used)) +# define N_LIB_EXPORT_VAR __attribute__((visibility("default"), used)) # else -# define N_LIB_EXPORT extern +# define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default"))) +# define N_LIB_EXPORT_VAR __attribute__((visibility("default"))) # endif # define N_LIB_IMPORT extern #endif @@ -148,7 +229,7 @@ __clang__ #define N_NOCONV_PTR(rettype, name) rettype (*name) #if defined(__GNUC__) || defined(__ICC__) -# define N_NOINLINE(rettype, name) rettype __attribute__((noinline)) name +# define N_NOINLINE(rettype, name) rettype __attribute__((__noinline__)) name #elif defined(_MSC_VER) # define N_NOINLINE(rettype, name) __declspec(noinline) rettype name #else @@ -157,8 +238,7 @@ __clang__ #define N_NOINLINE_PTR(rettype, name) rettype (*name) -#if defined(__BORLANDC__) || defined(__WATCOMC__) || \ - defined(__POCC__) || defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) +#if defined(__BORLANDC__) || defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) /* these compilers have a fastcall so use it: */ # ifdef __TINYC__ # define N_NIMCALL(rettype, name) rettype __attribute((__fastcall)) name @@ -180,83 +260,174 @@ __clang__ /* ----------------------------------------------------------------------- */ +#define COMMA , + #include <limits.h> #include <stddef.h> +// define NIM_STATIC_ASSERT +// example use case: CT sizeof for importc types verification +// where we have {.completeStruct.} (or lack of {.incompleteStruct.}) +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) +#define NIM_STATIC_ASSERT(x, msg) _Static_assert((x), msg) +#elif defined(__cplusplus) +#define NIM_STATIC_ASSERT(x, msg) static_assert((x), msg) +#else +#define _NIM_STATIC_ASSERT_FINAL(x, append_name) typedef int NIM_STATIC_ASSERT_AUX ## append_name[(x) ? 1 : -1]; +#define _NIM_STATIC_ASSERT_STAGE_3(x, line) _NIM_STATIC_ASSERT_FINAL(x, _AT_LINE_##line) +#define _NIM_STATIC_ASSERT_STAGE_2(x, line) _NIM_STATIC_ASSERT_STAGE_3(x, line) +#define NIM_STATIC_ASSERT(x, msg) _NIM_STATIC_ASSERT_STAGE_2(x,__LINE__) +// On failure, your C compiler will say something like: +// "error: 'NIM_STATIC_ASSERT_AUX_AT_LINE_XXX' declared as an array with a negative size" +// Adding the line number helps to avoid redefinitions which are not allowed in +// old GCC versions, however the order of evaluation for __LINE__ is a little tricky, +// hence all the helper macros. See https://stackoverflow.com/a/3385694 for more info. +#endif + /* C99 compiler? */ -#if (defined(__STD_VERSION__) && (__STD_VERSION__ >= 199901)) +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) # define HAVE_STDINT_H #endif -#if defined(__LCC__) || defined(__DMC__) || defined(__POCC__) +/* Known compiler with stdint.h that doesn't fit the general pattern? */ +#if defined(__AVR__) || (defined(__cplusplus) && (__cplusplus < 201103)) # define HAVE_STDINT_H #endif +#if (!defined(HAVE_STDINT_H) && defined(__cplusplus) && (__cplusplus >= 201103)) +# define HAVE_CSTDINT +#endif + + +/* wrap all Nim typedefs into namespace Nim */ +#ifdef USE_NIM_NAMESPACE +#ifdef HAVE_CSTDINT +#include <cstdint> +#else +#include <stdint.h> +#endif +namespace USE_NIM_NAMESPACE { +#endif + +// preexisting check, seems paranoid, maybe remove +#if defined(NIM_TRUE) || defined(NIM_FALSE) || defined(NIM_BOOL) +#error "nim reserved preprocessor macros clash" +#endif + /* bool types (C++ has it): */ #ifdef __cplusplus -# ifndef NIM_TRUE -# define NIM_TRUE true -# endif -# ifndef NIM_FALSE -# define NIM_FALSE false -# endif -# define NIM_BOOL bool -# define NIM_NIL 0 -struct NimException -{ - NimException(struct Exception* exp, const char* msg): exp(exp), msg(msg) {} - - struct Exception* exp; - const char* msg; -}; +#define NIM_BOOL bool +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901) +// see #13798: to avoid conflicts for code emitting `#include <stdbool.h>` +#define NIM_BOOL _Bool #else -# ifdef bool -# define NIM_BOOL bool +typedef unsigned char NIM_BOOL; // best effort +#endif + +NIM_STATIC_ASSERT(sizeof(NIM_BOOL) == 1, ""); // check whether really needed +NIM_STATIC_ASSERT(CHAR_BIT == 8, ""); + // fail fast for (rare) environments where this doesn't hold, as some implicit + // assumptions would need revisiting (e.g. `uint8` or https://github.com/nim-lang/Nim/pull/18505) + +#define NIM_TRUE true +#define NIM_FALSE false + +#ifdef __cplusplus +# if __cplusplus >= 201103L +# /* nullptr is more type safe (less implicit conversions than 0) */ +# define NIM_NIL nullptr # else - typedef unsigned char NIM_BOOL; -# endif -# ifndef NIM_TRUE -# define NIM_TRUE ((NIM_BOOL) 1) -# endif -# ifndef NIM_FALSE -# define NIM_FALSE ((NIM_BOOL) 0) +# // both `((void*)0)` and `NULL` would cause codegen to emit +# // error: assigning to 'Foo *' from incompatible type 'void *' +# // but codegen could be fixed if need. See also potential caveat regarding +# // NULL. +# // However, `0` causes other issues, see #13798 +# define NIM_NIL 0 # endif +#else +# include <stdbool.h> # define NIM_NIL ((void*)0) /* C's NULL is fucked up in some C compilers, so the generated code does not rely on it anymore */ #endif -#if defined(__BORLANDC__) || defined(__DMC__) \ - || defined(__WATCOMC__) || defined(_MSC_VER) +#if defined(__BORLANDC__) || defined(_MSC_VER) typedef signed char NI8; typedef signed short int NI16; typedef signed int NI32; +typedef __int64 NI64; /* XXX: Float128? */ typedef unsigned char NU8; typedef unsigned short int NU16; -typedef unsigned __int64 NU64; -typedef __int64 NI64; typedef unsigned int NU32; +typedef unsigned __int64 NU64; #elif defined(HAVE_STDINT_H) +#ifndef USE_NIM_NAMESPACE # include <stdint.h> +#endif typedef int8_t NI8; typedef int16_t NI16; typedef int32_t NI32; typedef int64_t NI64; -typedef uint64_t NU64; typedef uint8_t NU8; typedef uint16_t NU16; typedef uint32_t NU32; +typedef uint64_t NU64; +#elif defined(HAVE_CSTDINT) +#ifndef USE_NIM_NAMESPACE +# include <cstdint> +#endif +typedef std::int8_t NI8; +typedef std::int16_t NI16; +typedef std::int32_t NI32; +typedef std::int64_t NI64; +typedef std::uint8_t NU8; +typedef std::uint16_t NU16; +typedef std::uint32_t NU32; +typedef std::uint64_t NU64; +#else +/* Unknown compiler/version, do our best */ +#ifdef __INT8_TYPE__ +typedef __INT8_TYPE__ NI8; #else typedef signed char NI8; +#endif +#ifdef __INT16_TYPE__ +typedef __INT16_TYPE__ NI16; +#else typedef signed short int NI16; +#endif +#ifdef __INT32_TYPE__ +typedef __INT32_TYPE__ NI32; +#else typedef signed int NI32; +#endif +#ifdef __INT64_TYPE__ +typedef __INT64_TYPE__ NI64; +#else +typedef long long int NI64; +#endif /* XXX: Float128? */ +#ifdef __UINT8_TYPE__ +typedef __UINT8_TYPE__ NU8; +#else typedef unsigned char NU8; +#endif +#ifdef __UINT16_TYPE__ +typedef __UINT16_TYPE__ NU16; +#else typedef unsigned short int NU16; -typedef unsigned long long int NU64; -typedef long long int NI64; +#endif +#ifdef __UINT32_TYPE__ +typedef __UINT32_TYPE__ NU32; +#else typedef unsigned int NU32; #endif +#ifdef __UINT64_TYPE__ +typedef __UINT64_TYPE__ NU64; +#else +typedef unsigned long long int NU64; +#endif +#endif #ifdef NIM_INTBITS # if NIM_INTBITS == 64 @@ -276,7 +447,12 @@ typedef NU8 NU; # endif #endif +// for now there isn't an easy way for C code to reach the program result +// when hot code reloading is ON - users will have to: +// load the nimhcr.dll, get the hcrGetGlobal proc from there and use it +#ifndef NIM_HOT_CODE_RELOADING extern NI nim_program_result; +#endif typedef float NF32; typedef double NF64; @@ -291,31 +467,18 @@ typedef char* NCSTRING; # define NIM_IMAN 0 #endif -static N_INLINE(NI, float64ToInt32)(double x) { - /* nowadays no hack necessary anymore */ - return x >= 0 ? (NI)(x+0.5) : (NI)(x-0.5); -} - -static N_INLINE(NI32, float32ToInt32)(float x) { - /* nowadays no hack necessary anymore */ - return x >= 0 ? (NI32)(x+0.5) : (NI32)(x-0.5); -} - -#define float64ToInt64(x) ((NI64) (x)) - -#define zeroMem(a, size) memset(a, 0, size) -#define equalMem(a, b, size) (memcmp(a, b, size) == 0) +#define NIM_STRLIT_FLAG ((NU)(1) << ((NIM_INTBITS) - 2)) /* This has to be the same as system.strlitFlag! */ #define STRING_LITERAL(name, str, length) \ - static const struct { \ - TGenericSeq Sup; \ - NIM_CHAR data[(length) + 1]; \ - } name = {{length, length}, str} - -typedef struct TStringDesc* string; + static const struct { \ + TGenericSeq Sup; \ + NIM_CHAR data[(length) + 1]; \ + } name = {{length, (NI) ((NU)length | NIM_STRLIT_FLAG)}, str} /* declared size of a sequence/variable length array: */ -#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) +#if defined(__cplusplus) && defined(__clang__) +# define SEQ_DECL_SIZE 1 +#elif defined(__GNUC__) || defined(_MSC_VER) # define SEQ_DECL_SIZE /* empty is correct! */ #else # define SEQ_DECL_SIZE 1000000 @@ -324,18 +487,15 @@ typedef struct TStringDesc* string; #define ALLOC_0(size) calloc(1, size) #define DL_ALLOC_0(size) dlcalloc(1, size) -#define GenericSeqSize sizeof(TGenericSeq) #define paramCount() cmdCount -#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__i386__) -# ifndef NAN -static unsigned long nimNaN[2]={0xffffffff, 0x7fffffff}; -# define NAN (*(double*) nimNaN) -# endif -#endif - +// NAN definition copied from math.h included in the Windows SDK version 10.0.14393.0 #ifndef NAN -# define NAN (0.0 / 0.0) +# ifndef _HUGE_ENUF +# define _HUGE_ENUF 1e+300 // _HUGE_ENUF*_HUGE_ENUF must overflow +# endif +# define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF)) +# define NAN ((float)(NAN_INFINITY * 0.0F)) #endif #ifndef INF @@ -351,50 +511,30 @@ static unsigned long nimNaN[2]={0xffffffff, 0x7fffffff}; # endif #endif -typedef struct TFrame TFrame; -struct TFrame { +typedef struct TFrame_ TFrame; +struct TFrame_ { TFrame* prev; NCSTRING procname; NI line; NCSTRING filename; NI16 len; NI16 calldepth; + NI frameMsgLen; }; -#define nimfr(proc, file) \ - TFrame FR; \ - FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR); - -#define nimfrs(proc, file, slots, length) \ - struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR; \ - FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR); - -#define nimln(n, file) \ - FR.line = n; FR.filename = file; - #define NIM_POSIX_INIT __attribute__((constructor)) -#if defined(_MSCVER) && defined(__i386__) -__declspec(naked) int __fastcall NimXadd(volatile int* pNum, int val) { - __asm { - lock xadd dword ptr [ECX], EDX - mov EAX, EDX - ret - } -} -#endif - #ifdef __GNUC__ -# define likely(x) __builtin_expect(x, 1) -# define unlikely(x) __builtin_expect(x, 0) +# define NIM_LIKELY(x) __builtin_expect(x, 1) +# define NIM_UNLIKELY(x) __builtin_expect(x, 0) /* We need the following for the posix wrapper. In particular it will give us POSIX_SPAWN_USEVFORK: */ # ifndef _GNU_SOURCE # define _GNU_SOURCE # endif #else -# define likely(x) (x) -# define unlikely(x) (x) +# define NIM_LIKELY(x) (x) +# define NIM_UNLIKELY(x) (x) #endif #if 0 // defined(__GNUC__) || defined(__clang__) @@ -406,16 +546,19 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } # define GC_GUARD #endif -/* Test to see if Nim and the C compiler agree on the size of a pointer. - On disagreement, your C compiler will say something like: - "error: 'assert_numbits' declared as an array with a negative size" */ -typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; +// Test to see if Nim and the C compiler agree on the size of a pointer. +NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, "Pointer size mismatch between Nim and C/C++ backend. You probably need to setup the backend compiler for target CPU."); + +#ifdef USE_NIM_NAMESPACE +} #endif -#ifdef __cplusplus -# define NIM_EXTERNC extern "C" +#if defined(_MSC_VER) +# define NIM_ALIGN(x) __declspec(align(x)) +# define NIM_ALIGNOF(x) __alignof(x) #else -# define NIM_EXTERNC +# define NIM_ALIGN(x) __attribute__((aligned(x))) +# define NIM_ALIGNOF(x) __alignof__(x) #endif /* ---------------- platform specific includes ----------------------- */ @@ -428,3 +571,39 @@ typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof( #elif defined(__FreeBSD__) # include <sys/types.h> #endif + +/* these exist to make the codegen logic simpler */ +#define nimModInt(a, b, res) (((*res) = (a) % (b)), 0) +#define nimModInt64(a, b, res) (((*res) = (a) % (b)), 0) + +#if (!defined(_MSC_VER) || defined(__clang__)) && !defined(NIM_EmulateOverflowChecks) + /* these exist because we cannot have .compilerProcs that are importc'ed + by a different name */ + + #define nimAddInt64(a, b, res) __builtin_saddll_overflow(a, b, (long long int*)res) + #define nimSubInt64(a, b, res) __builtin_ssubll_overflow(a, b, (long long int*)res) + #define nimMulInt64(a, b, res) __builtin_smulll_overflow(a, b, (long long int*)res) + + #if NIM_INTBITS == 32 + #if defined(__arm__) && defined(__GNUC__) + /* arm-none-eabi-gcc targets defines int32_t as long int */ + #define nimAddInt(a, b, res) __builtin_saddl_overflow(a, b, res) + #define nimSubInt(a, b, res) __builtin_ssubl_overflow(a, b, res) + #define nimMulInt(a, b, res) __builtin_smull_overflow(a, b, res) + #else + #define nimAddInt(a, b, res) __builtin_sadd_overflow(a, b, res) + #define nimSubInt(a, b, res) __builtin_ssub_overflow(a, b, res) + #define nimMulInt(a, b, res) __builtin_smul_overflow(a, b, res) + #endif + #else + /* map it to the 'long long' variant */ + #define nimAddInt(a, b, res) __builtin_saddll_overflow(a, b, (long long int*)res) + #define nimSubInt(a, b, res) __builtin_ssubll_overflow(a, b, (long long int*)res) + #define nimMulInt(a, b, res) __builtin_smulll_overflow(a, b, (long long int*)res) + #endif +#endif + +#define NIM_NOALIAS __restrict +/* __restrict is said to work for all the C(++) compilers out there that we support */ + +#endif /* NIMBASE_H */ diff --git a/lib/nimhcr.nim b/lib/nimhcr.nim new file mode 100644 index 000000000..e87bb2413 --- /dev/null +++ b/lib/nimhcr.nim @@ -0,0 +1,671 @@ +discard """ +batchable: false +""" + +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This is the Nim hot code reloading run-time for the native targets. +# +# This minimal dynamic library is not subject to reloading when the +# `hotCodeReloading` build mode is enabled. It's responsible for providing +# a permanent memory location for all globals and procs within a program +# and orchestrating the reloading. For globals, this is easily achieved +# by storing them on the heap. For procs, we produce on the fly simple +# trampolines that can be dynamically overwritten to jump to a different +# target. In the host program, all globals and procs are first registered +# here with `hcrRegisterGlobal` and `hcrRegisterProc` and then the +# returned permanent locations are used in every reference to these symbols +# onwards. +# +# Detailed description: +# +# When code is compiled with the hotCodeReloading option for native targets +# a couple of things happen for all modules in a project: +# - the useNimRtl option is forced (including when building the HCR runtime too) +# - all modules of a target get built into separate shared libraries +# - the smallest granularity of reloads is modules +# - for each .c (or .cpp) in the corresponding nimcache folder of the project +# a shared object is built with the name of the source file + DLL extension +# - only the main module produces whatever the original project type intends +# (again in nimcache) and is then copied to its original destination +# - linking is done in parallel - just like compilation +# - function calls to functions from the same project go through function pointers: +# - with a few exceptions - see the nonReloadable pragma +# - the forward declarations of the original functions become function +# pointers as static globals with the same names +# - the original function definitions get suffixed with <name>_actual +# - the function pointers get initialized with the address of the corresponding +# function in the DatInit of their module through a call to either hcrRegisterProc +# or hcrGetProc. When being registered, the <name>_actual address is passed to +# hcrRegisterProc and a permanent location is returned and assigned to the pointer. +# This way the implementation (<name>_actual) can change but the address for it +# will be the same - this works by just updating a jump instruction (trampoline). +# For functions from other modules hcrGetProc is used (after they are registered). +# - globals are initialized only once and their state is preserved +# - including locals with the {.global.} pragma +# - their definitions are changed into pointer definitions which are initialized +# in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the +# size of the type that this HCR runtime should allocate) and a bool is returned +# which when true triggers the initialization code for the global (only once). +# Globals from other modules: a global pointer coupled with a hcrGetGlobal call. +# - globals which have already been initialized cannot have their values changed +# by changing their initialization - use a handler or some other mechanism +# - new globals can be introduced when reloading +# - top-level code (global scope) is executed only once - at the first module load +# - the runtime knows every symbol's module owner (globals and procs) +# - both the RTL and HCR shared libraries need to be near the program for execution +# - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS) +# - the main module is responsible for initializing the HCR runtime +# - the main module loads the RTL and HCR shared objects +# - after that a call to hcrInit() is done in the main module which triggers +# the loading of all modules the main one imports, and doing that for the +# dependencies of each module recursively. Basically a DFS traversal. +# - then initialization takes place with several passes over all modules: +# - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc +# - HcrCreateTypeInfos - creates globals which will be referenced in the next pass +# - DatInit - usual dat init + register/get procs and get globals +# - Init - it does the following multiplexed operations: +# - register globals (if already registered - then just retrieve pointer) +# - execute top level scope (only if loaded for the first time) +# - when modules are loaded the originally built shared libraries get copied in +# the same folder and the copies are loaded instead of the original files +# - a module import tree is built in the runtime (and maintained when reloading) +# - hcrPerformCodeReload +# - named `performCodeReload`, requires the hotcodereloading module +# - explicitly called by the user - the current active callstack shouldn't contain +# any functions which are defined in modules that will be reloaded (or crash!). +# The reason is that old dynamic libraries get unloaded. +# Example: +# if A is the main module and it imports B, then only B is reloadable and only +# if when calling hcrPerformCodeReload there is no function defined in B in the +# current active callstack at the point of the call (it has to be done from A) +# - for reloading to take place the user has to have rebuilt parts of the application +# without changes affecting the main module in any way - it shouldn't be rebuilt. +# - to determine what needs to be reloaded the runtime starts traversing the import +# tree from the root and checks the timestamps of the loaded shared objects +# - modules that are no longer referenced are unloaded and cleaned up properly +# - symbols (procs/globals) that have been removed in the code are also cleaned up +# - so changing the init of a global does nothing, but removing it, reloading, +# and then re-introducing it with a new initializer works +# - new modules can be imported, and imports can also be reodereded/removed +# - hcrReloadNeeded() can be used to determine if any module needs reloading +# - named `hasAnyModuleChanged`, requires the hotcodereloading module +# - code in the beforeCodeReload/afterCodeReload handlers is executed on each reload +# - require the hotcodereloading module +# - such handlers can be added and removed +# - before each reload all "beforeCodeReload" handlers are executed and after +# that all handlers (including "after") from the particular module are deleted +# - the order of execution is the same as the order of top-level code execution. +# Example: if A imports B which imports C, then all handlers in C will be executed +# first (from top to bottom) followed by all from B and lastly all from A +# - after the reload all "after" handlers are executed the same way as "before" +# - the handlers for a reloaded module are always removed when reloading and then +# registered when the top-level scope is executed (thanks to `executeOnReload`) +# +# TODO next: +# +# - implement the before/after handlers and hasModuleChanged for the javascript target +# - ARM support for the trampolines +# - investigate: +# - soon the system module might be importing other modules - the init order...? +# (revert https://github.com/nim-lang/Nim/pull/11971 when working on this) +# - rethink the closure iterators +# - ability to keep old versions of dynamic libraries alive +# - because of async server code +# - perhaps with refcounting of .dlls for unfinished closures +# - linking with static libs +# - all shared objects for each module will (probably) have to link to them +# - state in static libs gets duplicated +# - linking is slow and therefore iteration time suffers +# - have just a single .dll for all .nim files and bulk reload? +# - think about the compile/link/passc/passl/emit/injectStmt pragmas +# - if a passc pragma is introduced (either written or dragged in by a new +# import) the whole command line for compilation changes - for example: +# winlean.nim: {.passc: "-DWIN32_LEAN_AND_MEAN".} +# - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*' +# - everything thread-local related +# - tests +# - add a new travis build matrix entry which builds everything with HCR enabled +# - currently building with useNimRtl is problematic - lots of problems... +# - how to supply the nimrtl/nimhcr shared objects to all test binaries...? +# - think about building to C++ instead of only to C - added type safety +# - run tests through valgrind and the sanitizers! +# +# TODO - nice to have cool stuff: +# +# - separate handling of global state for much faster reloading and manipulation +# - imagine sliders in an IDE for tweaking variables +# - perhaps using shared memory +# - multi-dll projects - how everything can be reloaded..? +# - a single HCR instance shared across multiple .dlls +# - instead of having to call hcrPerformCodeReload from a function in each dll +# - which currently renders the main module of each dll not reloadable +# - ability to check with the current callstack if a reload is "legal" +# - if it is in any function which is in a module about to be reloaded ==> error +# - pragma annotations for files - to be excluded from dll shenanigans +# - for such file-global pragmas look at codeReordering or injectStmt +# - how would the initialization order be kept? messy... +# - C code calling stable exportc interface of nim code (for bindings) +# - generate proxy functions with the stable names +# - in a non-reloadable part (the main binary) that call the function pointers +# - parameter passing/forwarding - how? use the same trampoline jumping? +# - extracting the dependencies for these stubs/proxies will be hard... +# - changing memory layout of types - detecting this..? +# - implement with registerType() call to HCR runtime...? +# - and checking if a previously registered type matches +# - issue an error +# - or let the user handle this by transferring the state properly +# - perhaps in the before/afterCodeReload handlers +# - implement executeOnReload for global vars too - not just statements (and document!) +# - cleanup at shutdown - freeing all globals +# - fallback mechanism if the program crashes (the program should detect crashes +# by itself using SEH/signals on Windows/Unix) - should be able to revert to +# previous versions of the .dlls by calling some function from HCR +# - improve runtime performance - possibilities +# - implement a way for multiple .nim files to be bundled into the same dll +# and have all calls within that domain to use the "_actual" versions of +# procs so there are no indirections (or the ability to just bundle everything +# except for a few unreloadable modules into a single mega reloadable dll) +# - try to load the .dlls at specific addresses of memory (close to each other) +# allocated with execution flags - check this: https://github.com/fancycode/MemoryModule +# +# TODO - unimportant: +# +# - have a "bad call" trampoline that all no-longer-present functions are routed to call there +# - so the user gets some error msg if he calls a dangling pointer instead of a crash +# - before/afterCodeReload and hasModuleChanged should be accessible only where appropriate +# - nim_program_result is inaccessible in HCR mode from external C code (see nimbase.h) +# - proper .json build file - but the format is different... multiple link commands... +# - avoid registering globals on each loop when using an iterator in global scope +# +# TODO - REPL: +# - proper way (as proposed by Zahary): +# - parse the input code and put everything in global scope except for +# statements with side effects only - those go in afterCodeReload blocks +# - my very hacky idea: just append to a closure iterator the new statements +# followed by a yield statement. So far I can think of 2 problems: +# - import and some other code cannot be written inside of a proc - +# has to be parsed and extracted in the outer scope +# - when new variables are created they are actually locals to the closure +# so the struct for the closure state grows in memory, but it has already +# been allocated when the closure was created with the previous smaller size. +# That would lead to working with memory outside of the initially allocated +# block. Perhaps something can be done about this - some way of re-allocating +# the state and transferring the old... + +when defined(nimPreviewSlimSystem): + import std/assertions + +when not defined(js) and (defined(hotcodereloading) or + defined(createNimHcr) or + defined(testNimHcr)): + const + dllExt = when defined(windows): "dll" + elif defined(macosx): "dylib" + else: "so" + type + HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.} + HcrGcMarkerProc = proc () {.nimcall, raises: [].} + HcrModuleInitializer* = proc () {.nimcall.} + +when defined(createNimHcr): + when system.appType != "lib": + {.error: "This file has to be compiled as a library!".} + + import std/[os, tables, sets, times, strutils, reservedmem, dynlib] + + template trace(args: varargs[untyped]) = + when defined(testNimHcr) or defined(traceHcr): + echo args + + proc sanitize(arg: Time): string = + when defined(testNimHcr): return "<time>" + else: return $arg + + proc sanitize(arg: string|cstring): string = + when defined(testNimHcr): return ($arg).splitFile.name.splitFile.name + else: return $arg + + {.pragma: nimhcr, compilerproc, exportc, dynlib.} + + # XXX these types are CPU specific and need ARM etc support + type + ShortJumpInstruction {.packed.} = object + opcode: byte + offset: int32 + + LongJumpInstruction {.packed.} = object + opcode1: byte + opcode2: byte + offset: int32 + absoluteAddr: pointer + + proc writeJump(jumpTableEntry: ptr LongJumpInstruction, targetFn: pointer) = + let + jumpFrom = jumpTableEntry.shift(sizeof(ShortJumpInstruction)) + jumpDistance = distance(jumpFrom, targetFn) + + if abs(jumpDistance) < 0x7fff0000: + let shortJump = cast[ptr ShortJumpInstruction](jumpTableEntry) + shortJump.opcode = 0xE9 # relative jump + shortJump.offset = int32(jumpDistance) + else: + jumpTableEntry.opcode1 = 0xff # indirect absolute jump + jumpTableEntry.opcode2 = 0x25 + when hostCPU == "i386": + # on x86 we write the absolute address of the following pointer + jumpTableEntry.offset = cast[int32](addr jumpTableEntry.absoluteAddr) + else: + # on x64, we use a relative address for the same location + jumpTableEntry.offset = 0 + jumpTableEntry.absoluteAddr = targetFn + + if hostCPU == "arm": + const jumpSize = 8 + elif hostCPU == "arm64": + const jumpSize = 16 + + const defaultJumpTableSize = case hostCPU + of "i386": 50 + of "amd64": 500 + else: 50 + + let jumpTableSizeStr = getEnv("HOT_CODE_RELOADING_JUMP_TABLE_SIZE") + let jumpTableSize = if jumpTableSizeStr.len > 0: parseInt(jumpTableSizeStr) + else: defaultJumpTableSize + + # TODO: perhaps keep track of free slots due to removed procs using a free list + var jumpTable = ReservedMemSeq[LongJumpInstruction].init( + memStart = cast[pointer](0x10000000), + maxLen = jumpTableSize * 1024 * 1024 div sizeof(LongJumpInstruction), + accessFlags = memExecReadWrite) + + type + ProcSym = object + jump: ptr LongJumpInstruction + gen: int + + GlobalVarSym = object + p: pointer + markerProc: HcrGcMarkerProc + gen: int + + ModuleDesc = object + procs: Table[string, ProcSym] + globals: Table[string, GlobalVarSym] + imports: seq[string] + handle: LibHandle + hash: string + gen: int + lastModification: Time + handlers: seq[tuple[isBefore: bool, cb: proc () {.nimcall.}]] + + proc newModuleDesc(): ModuleDesc = + result.procs = initTable[string, ProcSym]() + result.globals = initTable[string, GlobalVarSym]() + result.handle = nil + result.gen = -1 + result.lastModification = low(Time) + + # the global state necessary for traversing and reloading the module import tree + var modules = initTable[string, ModuleDesc]() + var root: string + var system: string + var mainDatInit: HcrModuleInitializer + var generation = 0 + + # necessary for queries such as "has module X changed" - contains all but the main module + var hashToModuleMap = initTable[string, string]() + + # necessary for registering handlers and keeping them up-to-date + var currentModule: string + + # supplied from the main module - used by others to initialize pointers to this runtime + var hcrDynlibHandle: pointer + var getProcAddr: HcrProcGetter + + proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} = + trace " register proc: ", module.sanitize, " ", name + # Please note: We must allocate a local copy of the strings, because the supplied + # `cstring` will reside in the data segment of a DLL that will be later unloaded. + let name = $name + let module = $module + + var jumpTableEntryAddr: ptr LongJumpInstruction + + modules[module].procs.withValue(name, p): + trace " update proc: ", name + jumpTableEntryAddr = p.jump + p.gen = generation + do: + let len = jumpTable.len + jumpTable.setLen(len + 1) + jumpTableEntryAddr = addr jumpTable[len] + modules[module].procs[name] = ProcSym(jump: jumpTableEntryAddr, gen: generation) + + writeJump jumpTableEntryAddr, fn + return jumpTableEntryAddr + + proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} = + trace " get proc: ", module.sanitize, " ", name + return modules[$module].procs.getOrDefault($name, ProcSym()).jump + + proc hcrRegisterGlobal*(module: cstring, + name: cstring, + size: Natural, + gcMarker: HcrGcMarkerProc, + outPtr: ptr pointer): bool {.nimhcr.} = + trace " register global: ", module.sanitize, " ", name + # Please note: We must allocate local copies of the strings, because the supplied + # `cstring` will reside in the data segment of a DLL that will be later unloaded. + # Also using a ptr pointer instead of a var pointer (an output parameter) + # because for the C++ backend var parameters use references and in this use case + # it is not possible to cast an int* (for example) to a void* and then pass it + # to void*& since the casting yields an rvalue and references bind only to lvalues. + let name = $name + let module = $module + + modules[module].globals.withValue(name, global): + trace " update global: ", name + outPtr[] = global.p + global.gen = generation + global.markerProc = gcMarker + return false + do: + outPtr[] = alloc0(size) + modules[module].globals[name] = GlobalVarSym(p: outPtr[], + gen: generation, + markerProc: gcMarker) + return true + + proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} = + trace " get global: ", module.sanitize, " ", name + return modules[$module].globals[$name].p + + proc getListOfModules(cstringArray: ptr pointer): seq[string] = + var curr = cast[ptr cstring](cstringArray) + while len(curr[]) > 0: + result.add($curr[]) + curr = cast[ptr cstring](cast[int64](curr) + sizeof(ptr cstring)) + + template cleanup(collection, body) = + var toDelete: seq[string] + for name, data in collection.pairs: + if data.gen < generation: + toDelete.add(name) + trace "HCR Cleaning ", astToStr(collection), " :: ", name, " ", data.gen + for name {.inject.} in toDelete: + body + + proc cleanupGlobal(module: string, name: string) = + var g: GlobalVarSym + if modules[module].globals.take(name, g): + dealloc g.p + + proc cleanupSymbols(module: string) = + cleanup modules[module].globals: + cleanupGlobal(module, name) + + cleanup modules[module].procs: + modules[module].procs.del(name) + + proc unloadDll(name: string) = + if modules[name].handle != nil: + unloadLib(modules[name].handle) + + proc loadDll(name: cstring) {.nimhcr.} = + let name = $name + trace "HCR LOADING: ", name.sanitize + if modules.contains(name): + unloadDll(name) + else: + modules[name] = newModuleDesc() + + let copiedName = name & ".copy." & dllExt + copyFileWithPermissions(name, copiedName) + + let lib = loadLib(copiedName) + assert lib != nil + modules[name].handle = lib + modules[name].gen = generation + modules[name].lastModification = getLastModificationTime(name) + + # update the list of imports by the module + let getImportsProc = cast[proc (): ptr pointer {.nimcall.}]( + checkedSymAddr(lib, "HcrGetImportedModules")) + modules[name].imports = getListOfModules(getImportsProc()) + # get the hash of the module + let getHashProc = cast[proc (): cstring {.nimcall.}]( + checkedSymAddr(lib, "HcrGetSigHash")) + modules[name].hash = $getHashProc() + hashToModuleMap[modules[name].hash] = name + + # Remove handlers for this module if reloading - they will be re-registered. + # In order for them to be re-registered we need to de-register all globals + # that trigger the registering of handlers through calls to hcrAddEventHandler + modules[name].handlers.setLen(0) + + proc initHcrData(name: cstring) {.nimhcr.} = + trace "HCR Hcr init: ", name.sanitize + cast[proc (h: pointer, gpa: HcrProcGetter) {.nimcall.}]( + checkedSymAddr(modules[$name].handle, "HcrInit000"))(hcrDynlibHandle, getProcAddr) + + proc initTypeInfoGlobals(name: cstring) {.nimhcr.} = + trace "HCR TypeInfo globals init: ", name.sanitize + cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "HcrCreateTypeInfos"))() + + proc initPointerData(name: cstring) {.nimhcr.} = + trace "HCR Dat init: ", name.sanitize + cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "DatInit000"))() + + proc initGlobalScope(name: cstring) {.nimhcr.} = + trace "HCR Init000: ", name.sanitize + # set the currently inited module - necessary for registering the before/after HCR handlers + currentModule = $name + cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "Init000"))() + + var modulesToInit: seq[string] = @[] + var allModulesOrderedByDFS: seq[string] = @[] + + proc recursiveDiscovery(dlls: seq[string]) = + for curr in dlls: + if modules.contains(curr): + # skip updating modules that have already been updated to the latest generation + if modules[curr].gen >= generation: + trace "HCR SKIP: ", curr.sanitize, " gen is already: ", modules[curr].gen + continue + # skip updating an unmodified module but continue traversing its dependencies + if modules[curr].lastModification >= getLastModificationTime(curr): + trace "HCR SKIP (not modified): ", curr.sanitize, " ", modules[curr].lastModification.sanitize + # update generation so module doesn't get collected + modules[curr].gen = generation + # recurse to imported modules - they might be changed + recursiveDiscovery(modules[curr].imports) + allModulesOrderedByDFS.add(curr) + continue + loadDll(curr.cstring) + # first load all dependencies of the current module and init it after that + recursiveDiscovery(modules[curr].imports) + + allModulesOrderedByDFS.add(curr) + modulesToInit.add(curr) + + proc initModules() = + # first init the pointers to hcr functions and also do the registering of typeinfo globals + for curr in modulesToInit: + initHcrData(curr.cstring) + initTypeInfoGlobals(curr.cstring) + # for now system always gets fully inited before any other module (including when reloading) + initPointerData(system.cstring) + initGlobalScope(system.cstring) + # proceed with the DatInit calls - for all modules - including the main one! + for curr in allModulesOrderedByDFS: + if curr != system: + initPointerData(curr.cstring) + mainDatInit() + # execute top-level code (in global scope) + for curr in modulesToInit: + if curr != system: + initGlobalScope(curr.cstring) + # cleanup old symbols which are gone now + for curr in modulesToInit: + cleanupSymbols(curr) + + proc hcrInit*(moduleList: ptr pointer, main, sys: cstring, + datInit: HcrModuleInitializer, handle: pointer, gpa: HcrProcGetter) {.nimhcr.} = + trace "HCR INITING: ", main.sanitize, " gen: ", generation + # initialize globals + root = $main + system = $sys + mainDatInit = datInit + hcrDynlibHandle = handle + getProcAddr = gpa + # the root is already added and we need it because symbols from it will also be registered in the HCR system + modules[root].imports = getListOfModules(moduleList) + modules[root].gen = high(int) # something huge so it doesn't get collected + # recursively initialize all modules + recursiveDiscovery(modules[root].imports) + initModules() + # the next module to be inited will be the root + currentModule = root + + proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} = + let module = hashToModuleMap[moduleHash] + return modules[module].lastModification < getLastModificationTime(module) + + proc hcrReloadNeeded*(): bool {.nimhcr.} = + for hash, _ in hashToModuleMap: + if hcrHasModuleChanged(hash): + return true + return false + + proc hcrPerformCodeReload*() {.nimhcr.} = + if not hcrReloadNeeded(): + trace "HCR - no changes" + return + + # We disable the GC during the reload, because the reloading procedures + # will replace type info objects and GC marker procs. This seems to create + # problems when the GC is executed while the reload is underway. + # Future versions of NIMHCR won't use the GC, because all globals and the + # metadata needed to access them will be placed in shared memory, so they + # can be manipulated from external programs without reloading. + when declared(GC_disable): + GC_disable() + defer: GC_enable() + elif declared(GC_disableOrc): + GC_disableOrc() + defer: GC_enableOrc() + + inc(generation) + trace "HCR RELOADING: ", generation + + var traversedHandlerModules = initHashSet[string]() + + proc recursiveExecuteHandlers(isBefore: bool, module: string) = + # do not process an already traversed module + if traversedHandlerModules.containsOrIncl(module): return + traversedHandlerModules.incl module + # first recurse to do a DFS traversal + for curr in modules[module].imports: + recursiveExecuteHandlers(isBefore, curr) + # and then execute the handlers - from leaf modules all the way up to the root module + for curr in modules[module].handlers: + if curr.isBefore == isBefore: + curr.cb() + + # first execute the before reload handlers + traversedHandlerModules.clear() + recursiveExecuteHandlers(true, root) + + # do the reloading + modulesToInit = @[] + allModulesOrderedByDFS = @[] + recursiveDiscovery(modules[root].imports) + initModules() + + # execute the after reload handlers + traversedHandlerModules.clear() + recursiveExecuteHandlers(false, root) + + # collecting no longer referenced modules - based on their generation + cleanup modules: + cleanupSymbols(name) + unloadDll(name) + hashToModuleMap.del(modules[name].hash) + modules.del(name) + + proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.} = + modules[currentModule].handlers.add( + (isBefore: isBefore, cb: cb)) + + proc hcrAddModule*(module: cstring) {.nimhcr.} = + if not modules.contains($module): + modules[$module] = newModuleDesc() + + proc hcrGeneration*(): int {.nimhcr.} = + generation + + proc hcrMarkGlobals*() {.compilerproc, exportc, dynlib, nimcall, gcsafe.} = + # This is gcsafe, because it will be registered + # only in the GC of the main thread. + {.gcsafe.}: + for _, module in modules: + for _, global in module.globals: + if global.markerProc != nil: + global.markerProc() + +elif defined(hotcodereloading) or defined(testNimHcr): + when not defined(js): + const + nimhcrLibname = when defined(windows): "nimhcr." & dllExt + elif defined(macosx): "libnimhcr." & dllExt + else: "libnimhcr." & dllExt + + {.pragma: nimhcr, compilerproc, importc, dynlib: nimhcrLibname.} + + proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} + + proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} + + proc hcrRegisterGlobal*(module: cstring, name: cstring, size: Natural, + gcMarker: HcrGcMarkerProc, outPtr: ptr pointer): bool {.nimhcr.} + proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} + + proc hcrInit*(moduleList: ptr pointer, + main, sys: cstring, + datInit: HcrModuleInitializer, + handle: pointer, + gpa: HcrProcGetter) {.nimhcr.} + + proc hcrAddModule*(module: cstring) {.nimhcr.} + + proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} + + proc hcrReloadNeeded*(): bool {.nimhcr.} + + proc hcrPerformCodeReload*() {.nimhcr.} + + proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.} + + proc hcrMarkGlobals*() {.raises: [], nimhcr, nimcall, gcsafe.} + + when declared(nimRegisterGlobalMarker): + nimRegisterGlobalMarker(cast[GlobalMarkerProc](hcrMarkGlobals)) + + else: + proc hcrHasModuleChanged*(moduleHash: string): bool = + # TODO + false + + proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) = + # TODO + discard + diff --git a/lib/nimhcr.nim.cfg b/lib/nimhcr.nim.cfg new file mode 100644 index 000000000..282bec27c --- /dev/null +++ b/lib/nimhcr.nim.cfg @@ -0,0 +1,5 @@ +--app:lib +--threads:on +-d:useNimRtl +-d:createNimHcr + diff --git a/lib/nimrtl.nim b/lib/nimrtl.nim index 4e4cf7e0e..a2fb6ce60 100644 --- a/lib/nimrtl.nim +++ b/lib/nimrtl.nim @@ -1,3 +1,7 @@ +discard """ +batchable: false +""" + # # # Nim's Runtime Library @@ -8,7 +12,7 @@ # ## Main file to generate a DLL from the standard library. -## The default Nimrtl does not only contain the ``system`` module, but these +## The default Nimrtl does not only contain the `system` module, but these ## too: ## ## * parseutils @@ -22,6 +26,7 @@ ## * unicode ## * pegs ## * ropes +## * cstrutils ## when system.appType != "lib": @@ -31,6 +36,5 @@ when not defined(createNimRtl): {.error: "This file has to be compiled with '-d:createNimRtl'".} import - parseutils, strutils, parseopt, parsecfg, strtabs, unicode, pegs, ropes, - os, osproc, times - + std/[parseutils, strutils, parseopt, parsecfg, strtabs, unicode, pegs, ropes, + os, osproc, times, cstrutils] diff --git a/lib/nimrtl.nim.cfg b/lib/nimrtl.nim.cfg index b60de183a..a5cec596f 100644 --- a/lib/nimrtl.nim.cfg +++ b/lib/nimrtl.nim.cfg @@ -1,5 +1,6 @@ # The RTL.dll needs to be compiled with these options! --app:lib +--threads:on --define:createNimRtl diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim new file mode 100644 index 000000000..0a41d85b5 --- /dev/null +++ b/lib/packages/docutils/dochelpers.nim @@ -0,0 +1,298 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integration helpers between ``docgen.nim`` and ``rst.nim``. +## +## Function `toLangSymbol(linkText)`_ produces a signature `docLink` of +## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ +## matches it with `generated`, produced from `PNode` by ``docgen.rst``. + +import rstast +import std/strutils + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + + +type + LangSymbol* = object ## symbol signature in Nim + symKind*: string ## "proc", "const", "type", etc + symTypeKind*: string ## ""|enum|object|tuple - + ## valid only when `symKind == "type"` + name*: string ## plain symbol name without any parameters + generics*: string ## generic parameters (without brackets) + isGroup*: bool ## is LangSymbol a group with overloads? + # the following fields are valid iff `isGroup` == false + # (always false when parsed by `toLangSymbol` because link like foo_ + # can point to just a single symbol foo, e.g. proc). + parametersProvided*: bool ## to disambiguate `proc f`_ and `proc f()`_ + parameters*: seq[tuple[name: string, `type`: string]] + ## name-type seq, e.g. for proc + outType*: string ## result type, e.g. for proc + +proc `$`*(s: LangSymbol): string = # for debug + ("(symkind=$1, symTypeKind=$2, name=$3, generics=$4, isGroup=$5, " & + "parametersProvided=$6, parameters=$7, outType=$8)") % [ + s.symKind, s.symTypeKind , s.name, s.generics, $s.isGroup, + $s.parametersProvided, $s.parameters, s.outType] + +func nimIdentBackticksNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## Unlike `nimIdentNormalize` removes spaces and backticks. + ## + ## .. Warning:: No checking (e.g. that identifiers cannot start from + ## digits or '_', or that number of backticks is even) is performed. + runnableExamples: + doAssert nimIdentBackticksNormalize("Foo_bar") == "Foobar" + doAssert nimIdentBackticksNormalize("FoO BAr") == "Foobar" + doAssert nimIdentBackticksNormalize("`Foo BAR`") == "Foobar" + doAssert nimIdentBackticksNormalize("` Foo BAR `") == "Foobar" + # not a valid identifier: + doAssert nimIdentBackticksNormalize("`_x_y`") == "_xy" + result = newString(s.len) + var firstChar = true + var j = 0 + for i in 0..len(s) - 1: + if s[i] in {'A'..'Z'}: + if not firstChar: # to lowercase + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + else: + result[j] = s[i] + firstChar = false + inc j + elif s[i] notin {'_', ' ', '`'}: + result[j] = s[i] + inc j + firstChar = false + elif s[i] == '_' and firstChar: + result[j] = '_' + inc j + firstChar = false + else: discard # just omit '`' or ' ' + if j != s.len: setLen(result, j) + +proc langSymbolGroup*(kind: string, name: string): LangSymbol = + if kind notin ["proc", "func", "macro", "method", "iterator", + "template", "converter"]: + raise newException(ValueError, "unknown symbol kind $1" % [kind]) + result = LangSymbol(symKind: kind, name: name, isGroup: true) + +proc toLangSymbol*(linkText: PRstNode): LangSymbol = + ## Parses `linkText` into a more structured form using a state machine. + ## + ## This proc is designed to allow link syntax with operators even + ## without escaped backticks inside: + ## + ## `proc *`_ + ## `proc []`_ + ## + ## This proc should be kept in sync with the `renderTypes` proc from + ## ``compiler/typesrenderer.nim``. + template fail(msg: string) = + raise newException(ValueError, msg) + if linkText.kind notin {rnRstRef, rnInner}: + fail("toLangSymbol: wrong input kind " & $linkText.kind) + + const NimDefs = ["proc", "func", "macro", "method", "iterator", + "template", "converter", "const", "type", "var", + "enum", "object", "tuple", "module"] + template resolveSymKind(x: string) = + if x in ["enum", "object", "tuple"]: + result.symKind = "type" + result.symTypeKind = x + else: + result.symKind = x + type + State = enum + inBeginning + afterSymKind + beforeSymbolName # auxiliary state to catch situations like `proc []`_ after space + atSymbolName + afterSymbolName + genericsPar + parameterName + parameterType + outType + var state = inBeginning + var curIdent = "" + template flushIdent() = + if curIdent != "": + case state + of inBeginning: fail("incorrect state inBeginning") + of afterSymKind: resolveSymKind curIdent + of beforeSymbolName: fail("incorrect state beforeSymbolName") + of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize + of afterSymbolName: fail("incorrect state afterSymbolName") + of genericsPar: result.generics = curIdent + of parameterName: result.parameters.add (curIdent, "") + of parameterType: + for a in countdown(result.parameters.len - 1, 0): + if result.parameters[a].`type` == "": + result.parameters[a].`type` = curIdent + of outType: result.outType = curIdent + curIdent = "" + var parens = 0 + let L = linkText.sons.len + template s(i: int): string = linkText.sons[i].text + var i = 0 + template nextState = + case s(i) + of " ": + if state == afterSymKind: + flushIdent + state = beforeSymbolName + of "`": + curIdent.add "`" + inc i + while i < L: # add contents between ` ` as a whole + curIdent.add s(i) + if s(i) == "`": + break + inc i + curIdent = curIdent.nimIdentBackticksNormalize + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + flushIdent + state = afterSymbolName + of "[": + if state notin {inBeginning, afterSymKind, beforeSymbolName}: + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + curIdent.add s(i) + elif state in {atSymbolName, afterSymbolName} and parens == 1: + flushIdent + state = genericsPar + curIdent.add s(i) + else: curIdent.add s(i) + of "]": + if state notin {inBeginning, afterSymKind, beforeSymbolName, atSymbolName}: + dec parens + if state == genericsPar and parens == 0: + curIdent.add s(i) + flushIdent + else: curIdent.add s(i) + of "(": + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + result.parametersProvided = true + state = atSymbolName + flushIdent + state = parameterName + elif state in {atSymbolName, afterSymbolName, genericsPar} and parens == 1: + result.parametersProvided = true + flushIdent + state = parameterName + else: curIdent.add s(i) + of ")": + dec parens + if state in {parameterName, parameterType} and parens == 0: + flushIdent + state = outType + else: curIdent.add s(i) + of "{": # remove pragmas + while i < L: + if s(i) == "}": + break + inc i + of ",", ";": + if state in {parameterName, parameterType} and parens == 1: + flushIdent + state = parameterName + else: curIdent.add s(i) + of "*": # skip export symbol + if state == atSymbolName: + flushIdent + state = afterSymbolName + elif state == afterSymbolName: + discard + else: curIdent.add "*" + of ":": + if state == outType: discard + elif state == parameterName: + flushIdent + state = parameterType + else: curIdent.add ":" + else: + let isPostfixSymKind = i > 0 and i == L - 1 and + result.symKind == "" and s(i) in NimDefs + if isPostfixSymKind: # for links like `foo proc`_ + resolveSymKind s(i) + else: + case state + of inBeginning: + if s(i) in NimDefs: + state = afterSymKind + else: + state = atSymbolName + curIdent.add s(i) + of afterSymKind, beforeSymbolName: + state = atSymbolName + curIdent.add s(i) + of parameterType: + case s(i) + of "ref": curIdent.add "ref." + of "ptr": curIdent.add "ptr." + of "var": discard + else: curIdent.add s(i).nimIdentBackticksNormalize + of atSymbolName: + curIdent.add s(i) + else: + curIdent.add s(i).nimIdentBackticksNormalize + while i < L: + nextState + inc i + if state == afterSymKind: # treat `type`_ as link to symbol `type` + state = atSymbolName + flushIdent + result.isGroup = false + +proc match*(generated: LangSymbol, docLink: LangSymbol): bool = + ## Returns true if `generated` can be a target for `docLink`. + ## If `generated` is an overload group then only `symKind` and `name` + ## are compared for success. + result = true + if docLink.symKind != "": + if generated.symKind == "proc": + result = docLink.symKind in ["proc", "func"] + else: + result = generated.symKind == docLink.symKind + if result and docLink.symKind == "type" and docLink.symTypeKind != "": + result = generated.symTypeKind == docLink.symTypeKind + if not result: return + result = generated.name == docLink.name + if not result: return + if generated.isGroup: + # if `()` were added then it's not a reference to the whole group: + return not docLink.parametersProvided + if docLink.generics != "": + result = generated.generics == docLink.generics + if not result: return + if docLink.outType != "": + result = generated.outType == docLink.outType + if not result: return + if docLink.parametersProvided: + result = generated.parameters.len == docLink.parameters.len + if not result: return + var onlyType = false + for i in 0 ..< generated.parameters.len: + let g = generated.parameters[i] + let d = docLink.parameters[i] + if i == 0: + if g.`type` == d.name: + onlyType = true # only types, not names, are provided in `docLink` + if onlyType: + result = g.`type` == d.name + else: + if d.`type` != "": + result = g.`type` == d.`type` + if not result: return + result = g.name == d.name + if not result: return diff --git a/lib/packages/docutils/docutils.babel b/lib/packages/docutils/docutils.babel deleted file mode 100644 index 1ed86ca05..000000000 --- a/lib/packages/docutils/docutils.babel +++ /dev/null @@ -1,6 +0,0 @@ -[Package] -name = "docutils" -version = "0.9.0" -author = "Andreas Rumpf" -description = "Nimrod's reStructuredText processor." -license = "MIT" diff --git a/lib/packages/docutils/docutils.nimble.old b/lib/packages/docutils/docutils.nimble.old new file mode 100644 index 000000000..f97c3bdde --- /dev/null +++ b/lib/packages/docutils/docutils.nimble.old @@ -0,0 +1,7 @@ +# xxx disabled this as this isn't really a nimble package and it affects logic +# used to compute canonical imports, refs https://github.com/nim-lang/Nim/pull/16999#issuecomment-805442914 + +version = "0.10.0" +author = "Andreas Rumpf" +description = "Nim's reStructuredText processor." +license = "MIT" diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 640b8cd5a..f8376f46c 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -10,11 +10,64 @@ ## Source highlighter for programming or markup languages. ## Currently only few languages are supported, other languages may be added. ## The interface supports one language nested in another. +## +## You can use this to build your own syntax highlighting, check this example: +## +## ```Nim +## let code = """for x in $int.high: echo x.ord mod 2 == 0""" +## var toknizr: GeneralTokenizer +## initGeneralTokenizer(toknizr, code) +## while true: +## getNextToken(toknizr, langNim) +## case toknizr.kind +## of gtEof: break # End Of File (or string) +## of gtWhitespace: +## echo gtWhitespace # Maybe you want "visible" whitespaces?. +## echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1) +## of gtOperator: +## echo gtOperator # Maybe you want Operators to use a specific color?. +## echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1) +## # of gtSomeSymbol: syntaxHighlight("Comic Sans", "bold", "99px", "pink") +## else: +## echo toknizr.kind # All the kinds of tokens can be processed here. +## echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1) +## ``` +## +## The proc `getSourceLanguage` can get the language `enum` from a string: +## ```Nim +## for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l) +## ``` +## +## There is also a `Cmd` pseudo-language supported, which is a simple generic +## shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command): +## no escaping, no programming language constructs besides variable definition +## at the beginning of line. It supports these operators: +## ```Cmd +## & && | || ( ) '' "" ; # for comments +## ``` +## +## Instead of escaping always use quotes like here +## `nimgrep --ext:'nim|nims' file.name`:cmd: shows how to input ``|``. +## Any argument that contains ``.`` or ``/`` or ``\`` will be treated +## as a file or directory. +## +## In addition to `Cmd` there is also `Console` language for +## displaying interactive sessions. +## Lines with a command should start with ``$``, other lines are considered +## as program output. import - strutils + std/strutils +from std/algorithm import binarySearch + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + type + SourceLanguage* = enum + langNone, langNim, langCpp, langCsharp, langC, langJava, + langYaml, langPython, langCmd, langConsole TokenClass* = enum gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, @@ -22,48 +75,52 @@ type gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression, gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, - gtReference, gtOther + gtReference, gtPrompt, gtProgramOutput, gtProgram, gtOption, gtOther GeneralTokenizer* = object of RootObj kind*: TokenClass start*, length*: int buf: cstring pos: int state: TokenClass - - SourceLanguage* = enum - langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava -{.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass, - TGeneralTokenizer: GeneralTokenizer].} + lang: SourceLanguage const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "Nimrod", "C++", "C#", "C", "Java"] + "Nim", "C++", "C#", "C", "Java", "Yaml", "Python", "Cmd", "Console"] + sourceLanguageToAlpha*: array[SourceLanguage, string] = ["none", + "Nim", "cpp", "csharp", "C", "Java", "Yaml", "Python", "Cmd", "Console"] + ## list of languages spelled with alpabetic characters tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment", "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", - "Label", "Reference", "Other"] + "Label", "Reference", "Prompt", "ProgramOutput", + # start from lower-case if there is a corresponding RST role (see rst.nim) + "program", "option", + "Other"] # The following list comes from doc/keywords.txt, make sure it is # synchronized with this array by running the module itself as a test case. - nimKeywords = ["addr", "and", "as", "asm", "atomic", "bind", "block", + nimKeywords = ["addr", "and", "as", "asm", "bind", "block", "break", "case", "cast", "concept", "const", "continue", "converter", "defer", "discard", "distinct", "div", "do", "elif", "else", "end", "enum", "except", "export", "finally", "for", "from", "func", - "generic", "if", "import", "in", "include", + "if", "import", "in", "include", "interface", "is", "isnot", "iterator", "let", "macro", "method", "mixin", "mod", "nil", "not", "notin", "object", "of", "or", "out", "proc", "ptr", "raise", "ref", "return", "shl", "shr", "static", - "template", "try", "tuple", "type", "using", "var", "when", "while", "with", - "without", "xor", "yield"] + "template", "try", "tuple", "type", "using", "var", "when", "while", + "xor", "yield"] proc getSourceLanguage*(name: string): SourceLanguage = - for i in countup(succ(low(SourceLanguage)), high(SourceLanguage)): + for i in succ(low(SourceLanguage)) .. high(SourceLanguage): if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: return i + if cmpIgnoreStyle(name, sourceLanguageToAlpha[i]) == 0: + return i result = langNone proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) = @@ -72,9 +129,8 @@ proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) = g.start = 0 g.length = 0 g.state = low(TokenClass) - var pos = 0 # skip initial whitespace: - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) - g.pos = pos + g.lang = low(SourceLanguage) + g.pos = 0 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: string) = initGeneralTokenizer(g, cstring(buf)) @@ -130,9 +186,12 @@ proc nimNumber(g: var GeneralTokenizer, position: int): int = const OpChars = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.', - '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'} + '|', '=', '%', '&', '$', '@', '~', ':'} -proc nimNextToken(g: var GeneralTokenizer) = +proc isKeyword(x: openArray[string], y: string): int = + binarySearch(x, y) + +proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) = const hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'} octChars = {'0'..'7', '_'} @@ -141,39 +200,75 @@ proc nimNextToken(g: var GeneralTokenizer) = var pos = g.pos g.start = g.pos if g.state == gtStringLit: - g.kind = gtStringLit - while true: + if g.buf[pos] == '\\': + g.kind = gtEscapeSequence + inc(pos) case g.buf[pos] - of '\\': - g.kind = gtEscapeSequence + of 'x', 'X': inc(pos) + if g.buf[pos] in hexChars: inc(pos) + if g.buf[pos] in hexChars: inc(pos) + of '0'..'9': + while g.buf[pos] in {'0'..'9'}: inc(pos) + of '\0': + g.state = gtNone + else: inc(pos) + else: + g.kind = gtStringLit + while true: case g.buf[pos] - of 'x', 'X': + of '\\': + break + of '\0', '\r', '\n': + g.state = gtNone + break + of '\"': inc(pos) - if g.buf[pos] in hexChars: inc(pos) - if g.buf[pos] in hexChars: inc(pos) - of '0'..'9': - while g.buf[pos] in {'0'..'9'}: inc(pos) - of '\0': g.state = gtNone + break else: inc(pos) - break - of '\0', '\x0D', '\x0A': - g.state = gtNone - break - of '\"': - inc(pos) - g.state = gtNone - break - else: inc(pos) else: case g.buf[pos] - of ' ', '\x09'..'\x0D': + of ' ', '\t'..'\r': g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) of '#': g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) + inc(pos) + var isDoc = false + if g.buf[pos] == '#': + inc(pos) + isDoc = true + if g.buf[pos] == '[' and g.lang == langNim: + g.kind = gtLongComment + var nesting = 0 + while true: + case g.buf[pos] + of '\0': break + of '#': + if isDoc: + if g.buf[pos+1] == '#' and g.buf[pos+2] == '[': + inc nesting + elif g.buf[pos+1] == '[': + inc nesting + inc pos + of ']': + if isDoc: + if g.buf[pos+1] == '#' and g.buf[pos+2] == '#': + if nesting == 0: + inc(pos, 3) + break + dec nesting + elif g.buf[pos+1] == '#': + if nesting == 0: + inc(pos, 2) + break + dec nesting + inc pos + else: + inc pos + else: + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': var id = "" while g.buf[pos] in SymChars + {'_'}: @@ -197,24 +292,30 @@ proc nimNextToken(g: var GeneralTokenizer) = else: g.kind = gtRawData inc(pos) - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): + while not (g.buf[pos] in {'\0', '\n', '\r'}): if g.buf[pos] == '"' and g.buf[pos+1] != '"': break inc(pos) if g.buf[pos] == '\"': inc(pos) else: - g.kind = nimGetKeyword(id) + if g.lang == langNim: + g.kind = nimGetKeyword(id) + elif isKeyword(keywords, id) >= 0: + g.kind = gtKeyword of '0': inc(pos) case g.buf[pos] of 'b', 'B': + g.kind = gtBinNumber inc(pos) while g.buf[pos] in binChars: inc(pos) pos = nimNumberPostfix(g, pos) of 'x', 'X': + g.kind = gtHexNumber inc(pos) while g.buf[pos] in hexChars: inc(pos) pos = nimNumberPostfix(g, pos) of 'o', 'O': + g.kind = gtOctNumber inc(pos) while g.buf[pos] in octChars: inc(pos) pos = nimNumberPostfix(g, pos) @@ -223,17 +324,18 @@ proc nimNextToken(g: var GeneralTokenizer) = pos = nimNumber(g, pos) of '\'': inc(pos) - g.kind = gtCharLit - while true: - case g.buf[pos] - of '\0', '\x0D', '\x0A': - break - of '\'': - inc(pos) - break - of '\\': - inc(pos, 2) - else: inc(pos) + if g.kind != gtPunctuation: + g.kind = gtCharLit + while true: + case g.buf[pos] + of '\0', '\r', '\n': + break + of '\'': + inc(pos) + break + of '\\': + inc(pos, 2) + else: inc(pos) of '\"': inc(pos) if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'): @@ -254,7 +356,7 @@ proc nimNextToken(g: var GeneralTokenizer) = g.kind = gtStringLit while true: case g.buf[pos] - of '\0', '\x0D', '\x0A': + of '\0', '\r', '\n': break of '\"': inc(pos) @@ -276,7 +378,7 @@ proc nimNextToken(g: var GeneralTokenizer) = inc(pos) g.kind = gtNone g.length = pos - g.pos - if g.kind != gtEof and g.length <= 0: + if g.kind != gtEof and g.state != gtNone and g.length <= 0: assert false, "nimNextToken: produced an empty token" g.pos = pos @@ -328,39 +430,10 @@ proc generalStrLit(g: var GeneralTokenizer, position: int): int = inc(pos) result = pos -proc isKeyword(x: openArray[string], y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmp(x[mid], y) - if c < 0: - a = mid + 1 - elif c > 0: - b = mid - 1 - else: - return mid - result = - 1 - -proc isKeywordIgnoreCase(x: openArray[string], y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmpIgnoreCase(x[mid], y) - if c < 0: - a = mid + 1 - elif c > 0: - b = mid - 1 - else: - return mid - result = - 1 - type TokenizerFlag = enum hasPreprocessor, hasNestedComments TokenizerFlags = set[TokenizerFlag] -{.deprecated: [TTokenizerFlag: TokenizerFlag, TTokenizerFlags: TokenizerFlags].} proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], flags: TokenizerFlags) = @@ -389,7 +462,7 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], g.state = gtNone else: inc(pos) break - of '\0', '\x0D', '\x0A': + of '\0', '\r', '\n': g.state = gtNone break of '\"': @@ -399,14 +472,14 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], else: inc(pos) else: case g.buf[pos] - of ' ', '\x09'..'\x0D': + of ' ', '\t'..'\r': g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) of '/': inc(pos) if g.buf[pos] == '/': g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) + while not (g.buf[pos] in {'\0', '\n', '\r'}): inc(pos) elif g.buf[pos] == '*': g.kind = gtLongComment var nested = 0 @@ -426,6 +499,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], of '\0': break else: inc(pos) + else: + g.kind = gtOperator + while g.buf[pos] in OpChars: inc(pos) of '#': inc(pos) if hasPreprocessor in flags: @@ -544,27 +620,420 @@ proc javaNextToken(g: var GeneralTokenizer) = "try", "void", "volatile", "while"] clikeNextToken(g, keywords, {}) +proc yamlPlainStrLit(g: var GeneralTokenizer, pos: var int) = + g.kind = gtStringLit + while g.buf[pos] notin {'\0', '\t'..'\r', ',', ']', '}'}: + if g.buf[pos] == ':' and + g.buf[pos + 1] in {'\0', '\t'..'\r', ' '}: + break + inc(pos) + +proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) = + g.kind = gtNone + if g.buf[pos] == '-': inc(pos) + if g.buf[pos] == '0': inc(pos) + elif g.buf[pos] in '1'..'9': + inc(pos) + while g.buf[pos] in {'0'..'9'}: inc(pos) + else: yamlPlainStrLit(g, pos) + if g.kind == gtNone: + if g.buf[pos] in {'\0', '\t'..'\r', ' ', ',', ']', '}'}: + g.kind = gtDecNumber + elif g.buf[pos] == '.': + inc(pos) + if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) + else: + while g.buf[pos] in {'0'..'9'}: inc(pos) + if g.buf[pos] in {'\0', '\t'..'\r', ' ', ',', ']', '}'}: + g.kind = gtFloatNumber + if g.kind == gtNone: + if g.buf[pos] in {'e', 'E'}: + inc(pos) + if g.buf[pos] in {'-', '+'}: inc(pos) + if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) + else: + while g.buf[pos] in {'0'..'9'}: inc(pos) + if g.buf[pos] in {'\0', '\t'..'\r', ' ', ',', ']', '}'}: + g.kind = gtFloatNumber + else: yamlPlainStrLit(g, pos) + else: yamlPlainStrLit(g, pos) + while g.buf[pos] notin {'\0', ',', ']', '}', '\n', '\r'}: + inc(pos) + if g.buf[pos] notin {'\t'..'\r', ' ', ',', ']', '}'}: + yamlPlainStrLit(g, pos) + break + # theoretically, we would need to parse indentation (like with block scalars) + # because of possible multiline flow scalars that start with number-like + # content, but that is far too troublesome. I think it is fine that the + # highlighter is sloppy here. + +proc yamlNextToken(g: var GeneralTokenizer) = + const + hexChars = {'0'..'9', 'A'..'F', 'a'..'f'} + var pos = g.pos + g.start = g.pos + if g.state == gtStringLit: + g.kind = gtStringLit + while true: + case g.buf[pos] + of '\\': + if pos != g.pos: break + g.kind = gtEscapeSequence + inc(pos) + case g.buf[pos] + of 'x': + inc(pos) + for i in 1..2: + if g.buf[pos] in hexChars: inc(pos) + break + of 'u': + inc(pos) + for i in 1..4: + if g.buf[pos] in hexChars: inc(pos) + break + of 'U': + inc(pos) + for i in 1..8: + if g.buf[pos] in hexChars: inc(pos) + break + else: inc(pos) + break + of '\0': + g.state = gtOther + break + of '\"': + inc(pos) + g.state = gtOther + break + else: inc(pos) + elif g.state == gtCharLit: + # abusing gtCharLit as single-quoted string lit + g.kind = gtStringLit + inc(pos) # skip the starting ' + while true: + case g.buf[pos] + of '\'': + inc(pos) + if g.buf[pos] == '\'': + inc(pos) + g.kind = gtEscapeSequence + else: g.state = gtOther + break + else: inc(pos) + elif g.state == gtCommand: + # gtCommand means 'block scalar header' + case g.buf[pos] + of ' ', '\t': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'}: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + of '\n', '\r': discard + else: + # illegal here. just don't parse a block scalar + g.kind = gtNone + g.state = gtOther + if g.buf[pos] in {'\n', '\r'} and g.state == gtCommand: + g.state = gtLongStringLit + elif g.state == gtLongStringLit: + # beware, this is the only token where we actually have to parse + # indentation. + + g.kind = gtLongStringLit + # first, we have to find the parent indentation of the block scalar, so that + # we know when to stop + assert g.buf[pos] in {'\n', '\r'} + var lookbehind = pos - 1 + var headerStart = -1 + while lookbehind >= 0 and g.buf[lookbehind] notin {'\n', '\r'}: + if headerStart == -1 and g.buf[lookbehind] in {'|', '>'}: + headerStart = lookbehind + dec(lookbehind) + assert headerStart != -1 + var indentation = 1 + while g.buf[lookbehind + indentation] == ' ': inc(indentation) + if g.buf[lookbehind + indentation] in {'|', '>'}: + # when the header is alone in a line, this line does not show the parent's + # indentation, so we must go further. search the first previous line with + # non-whitespace content. + while lookbehind >= 0 and g.buf[lookbehind] in {'\n', '\r'}: + dec(lookbehind) + while lookbehind >= 0 and + g.buf[lookbehind] in {' ', '\t'}: dec(lookbehind) + # now, find the beginning of the line... + while lookbehind >= 0 and g.buf[lookbehind] notin {'\n', '\r'}: + dec(lookbehind) + # ... and its indentation + indentation = 1 + while g.buf[lookbehind + indentation] == ' ': inc(indentation) + if lookbehind == -1: indentation = 0 # top level + elif g.buf[lookbehind + 1] == '-' and g.buf[lookbehind + 2] == '-' and + g.buf[lookbehind + 3] == '-' and + g.buf[lookbehind + 4] in {'\t'..'\r', ' '}: + # this is a document start, therefore, we are at top level + indentation = 0 + # because lookbehind was at newline char when calculating indentation, we're + # off by one. fix that. top level's parent will have indentation of -1. + let parentIndentation = indentation - 1 + + # find first content + while g.buf[pos] in {' ', '\n', '\r'}: + if g.buf[pos] == ' ': inc(indentation) + else: indentation = 0 + inc(pos) + var minIndentation = indentation + + # for stupid edge cases, we must check whether an explicit indentation depth + # is given at the header. + while g.buf[headerStart] in {'>', '|', '+', '-'}: inc(headerStart) + if g.buf[headerStart] in {'0'..'9'}: + minIndentation = min(minIndentation, ord(g.buf[headerStart]) - ord('0')) + + # process content lines + while indentation > parentIndentation and g.buf[pos] != '\0': + if (indentation < minIndentation and g.buf[pos] == '#') or + (indentation == 0 and g.buf[pos] == '.' and g.buf[pos + 1] == '.' and + g.buf[pos + 2] == '.' and + g.buf[pos + 3] in {'\0', '\t'..'\r', ' '}): + # comment after end of block scalar, or end of document + break + minIndentation = min(indentation, minIndentation) + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + while g.buf[pos] in {' ', '\n', '\r'}: + if g.buf[pos] == ' ': inc(indentation) + else: indentation = 0 + inc(pos) + + g.state = gtOther + elif g.state == gtOther: + # gtOther means 'inside YAML document' + case g.buf[pos] + of ' ', '\t'..'\r': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) + of '#': + g.kind = gtComment + inc(pos) + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + of '-': + inc(pos) + if g.buf[pos] in {'\0', ' ', '\t'..'\r'}: + g.kind = gtPunctuation + elif g.buf[pos] == '-' and + (pos == 1 or g.buf[pos - 2] in {'\n', '\r'}): # start of line + inc(pos) + if g.buf[pos] == '-' and g.buf[pos + 1] in {'\0', '\t'..'\r', ' '}: + inc(pos) + g.kind = gtKeyword + else: yamlPossibleNumber(g, pos) + else: yamlPossibleNumber(g, pos) + of '.': + if pos == 0 or g.buf[pos - 1] in {'\n', '\r'}: + inc(pos) + for i in 1..2: + if g.buf[pos] != '.': break + inc(pos) + if pos == g.start + 3: + g.kind = gtKeyword + g.state = gtNone + else: yamlPlainStrLit(g, pos) + else: yamlPlainStrLit(g, pos) + of '?': + inc(pos) + if g.buf[pos] in {'\0', ' ', '\t'..'\r'}: + g.kind = gtPunctuation + else: yamlPlainStrLit(g, pos) + of ':': + inc(pos) + if g.buf[pos] in {'\0', '\t'..'\r', ' ', '\'', '\"'} or + (pos > 0 and g.buf[pos - 2] in {'}', ']', '\"', '\''}): + g.kind = gtPunctuation + else: yamlPlainStrLit(g, pos) + of '[', ']', '{', '}', ',': + inc(pos) + g.kind = gtPunctuation + of '\"': + inc(pos) + g.state = gtStringLit + g.kind = gtStringLit + of '\'': + g.state = gtCharLit + g.kind = gtNone + of '!': + g.kind = gtTagStart + inc(pos) + if g.buf[pos] == '<': + # literal tag (e.g. `!<tag:yaml.org,2002:str>`) + while g.buf[pos] notin {'\0', '>', '\t'..'\r', ' '}: inc(pos) + if g.buf[pos] == '>': inc(pos) + else: + while g.buf[pos] in {'A'..'Z', 'a'..'z', '0'..'9', '-'}: inc(pos) + case g.buf[pos] + of '!': + # prefixed tag (e.g. `!!str`) + inc(pos) + while g.buf[pos] notin + {'\0', '\t'..'\r', ' ', ',', '[', ']', '{', '}'}: inc(pos) + of '\0', '\t'..'\r', ' ': discard + else: + # local tag (e.g. `!nim:system:int`) + while g.buf[pos] notin {'\0', '\t'..'\r', ' '}: inc(pos) + of '&': + g.kind = gtLabel + while g.buf[pos] notin {'\0', '\t'..'\r', ' '}: inc(pos) + of '*': + g.kind = gtReference + while g.buf[pos] notin {'\0', '\t'..'\r', ' '}: inc(pos) + of '|', '>': + # this can lead to incorrect tokenization when | or > appear inside flow + # content. checking whether we're inside flow content is not + # chomsky type-3, so we won't do that here. + g.kind = gtCommand + g.state = gtCommand + inc(pos) + while g.buf[pos] in {'0'..'9', '+', '-'}: inc(pos) + of '0'..'9': yamlPossibleNumber(g, pos) + of '\0': g.kind = gtEof + else: yamlPlainStrLit(g, pos) + else: + # outside document + case g.buf[pos] + of '%': + if pos == 0 or g.buf[pos - 1] in {'\n', '\r'}: + g.kind = gtDirective + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + else: + g.state = gtOther + yamlPlainStrLit(g, pos) + of ' ', '\t'..'\r': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + of '\0': g.kind = gtEof + else: + g.kind = gtNone + g.state = gtOther + g.length = pos - g.pos + g.pos = pos + +proc pythonNextToken(g: var GeneralTokenizer) = + const + keywords: array[0..34, string] = [ + "False", "None", "True", "and", "as", "assert", "async", "await", + "break", "class", "continue", "def", "del", "elif", "else", "except", + "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", + "with", "yield"] + nimNextToken(g, keywords) + +proc cmdNextToken(g: var GeneralTokenizer, dollarPrompt = false) = + var pos = g.pos + g.start = g.pos + if g.state == low(TokenClass): + g.state = if dollarPrompt: gtPrompt else: gtProgram + case g.buf[pos] + of ' ', '\t'..'\r': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'..'\r'}: + if g.buf[pos] == '\n': + g.state = if dollarPrompt: gtPrompt else: gtProgram + inc(pos) + of '\'', '"': + g.kind = gtOption + let q = g.buf[pos] + inc(pos) + while g.buf[pos] notin {q, '\0'}: + inc(pos) + if g.buf[pos] == q: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\n', '\0'}: + inc(pos) + of '&', '|': + g.kind = gtOperator + inc(pos) + if g.buf[pos] == g.buf[pos-1]: inc(pos) + g.state = gtProgram + of '(': + g.kind = gtOperator + g.state = gtProgram + inc(pos) + of ')': + g.kind = gtOperator + inc(pos) + of ';': + g.state = gtProgram + g.kind = gtOperator + inc(pos) + of '\0': g.kind = gtEof + elif dollarPrompt and g.state == gtPrompt: + if g.buf[pos] == '$' and g.buf[pos+1] in {' ', '\t'}: + g.kind = gtPrompt + inc pos, 2 + g.state = gtProgram + else: + g.kind = gtProgramOutput + while g.buf[pos] notin {'\n', '\0'}: + inc(pos) + else: + if g.state == gtProgram: + g.kind = gtProgram + g.state = gtOption + else: + g.kind = gtOption + while g.buf[pos] notin {' ', '\t'..'\r', '&', '|', '(', ')', '\'', '"', '\0'}: + if g.buf[pos] == ';' and g.buf[pos+1] == ' ': + # (check space because ';' can be used inside arguments in Win bat) + break + if g.kind == gtOption and g.buf[pos] in {'/', '\\', '.'}: + g.kind = gtIdentifier # for file/dir name + elif g.kind == gtProgram and g.buf[pos] == '=': + g.kind = gtIdentifier # for env variable setting at beginning of line + g.state = gtProgram + inc(pos) + g.length = pos - g.pos + g.pos = pos + proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = + g.lang = lang case lang of langNone: assert false - of langNim, langNimrod: nimNextToken(g) + of langNim: nimNextToken(g) of langCpp: cppNextToken(g) of langCsharp: csharpNextToken(g) of langC: cNextToken(g) of langJava: javaNextToken(g) + of langYaml: yamlNextToken(g) + of langPython: pythonNextToken(g) + of langCmd: cmdNextToken(g) + of langConsole: cmdNextToken(g, dollarPrompt=true) + +proc tokenize*(text: string, lang: SourceLanguage): seq[(string, TokenClass)] = + var g: GeneralTokenizer + initGeneralTokenizer(g, text) + var prevPos = 0 + while true: + getNextToken(g, lang) + if g.kind == gtEof: + break + var s = text[prevPos ..< g.pos] + result.add (s, g.kind) + prevPos = g.pos when isMainModule: var keywords: seq[string] # Try to work running in both the subdir or at the root. for filename in ["doc/keywords.txt", "../../../doc/keywords.txt"]: try: - let input = string(readFile(filename)) - keywords = input.split() + let input = readFile(filename) + keywords = input.splitWhitespace() break except: echo filename, " not found" - doAssert(not keywords.isNil, "Couldn't read any keywords.txt file!") - doAssert keywords.len == nimKeywords.len, "No matching lengths" - for i in 0..keywords.len-1: - #echo keywords[i], " == ", nimKeywords[i] + doAssert(keywords.len > 0, "Couldn't read any keywords.txt file!") + for i in 0..min(keywords.len, nimKeywords.len)-1: doAssert keywords[i] == nimKeywords[i], "Unexpected keyword" + doAssert keywords.len == nimKeywords.len, "No matching lengths" diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 2fbde632e..706c50689 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -7,21 +7,46 @@ # distribution, for details about the copyright. # -## This module implements a `reStructuredText`:idx: parser. A large -## subset is implemented. Some features of the `markdown`:idx: wiki syntax are -## also supported. +## This module implements a `reStructuredText`:idx: (RST) and +## `Markdown`:idx: parser. +## User's manual on supported markup syntax and command line usage can be +## found in [Nim-flavored Markdown and reStructuredText](markdown_rst.html). +## +## * See also [Nim DocGen Tools Guide](docgen.html) for handling of +## ``.nim`` files. +## * See also [packages/docutils/rstgen module](rstgen.html) to know how to +## generate HTML or Latex strings (for embedding them into custom documents). +## +## Choice between Markdown and RST as well as optional additional features are +## turned on by passing ``options:`` [RstParseOptions] to [proc rstParse]. import - os, strutils, rstast + std/[os, strutils, enumutils, algorithm, lists, sequtils, + tables, strscans] +import dochelpers, rstidx, rstast +import std/private/miscdollars +from highlite import SourceLanguage, getSourceLanguage + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + type RstParseOption* = enum ## options for the RST parser - roSkipPounds, ## skip ``#`` at line beginning (documentation - ## embedded in Nim comments) roSupportSmilies, ## make the RST parser support smilies like ``:)`` roSupportRawDirective, ## support the ``raw`` directive (don't support ## it for sandboxing) - roSupportMarkdown ## support additional features of markdown + roSupportMarkdown, ## support additional features of Markdown + roPreferMarkdown, ## parse as Markdown (keeping RST as "extension" + ## to Markdown) -- implies `roSupportMarkdown` + roNimFile ## set for Nim files where default interpreted + ## text role should be :nim: + roSandboxDisabled ## this option enables certain options + ## (e.g. raw, include, importdoc) + ## which are disabled by default as they can + ## enable users to read arbitrary data and + ## perform XSS if the parser is used in a web + ## app. RstParseOptions* = set[RstParseOption] @@ -30,41 +55,41 @@ type mcWarning = "Warning", mcError = "Error" + # keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim: MsgKind* = enum ## the possible messages - meCannotOpenFile, - meExpected, - meGridTableNotImplemented, - meNewSectionExpected, - meGeneralParseError, - meInvalidDirective, - mwRedefinitionOfLabel, - mwUnknownSubstitution, - mwUnsupportedLanguage, - mwUnsupportedField + meCannotOpenFile = "cannot open '$1'", + meExpected = "'$1' expected", + meMissingClosing = "$1", + meGridTableNotImplemented = "grid table is not implemented", + meMarkdownIllformedTable = "illformed delimiter row of a Markdown table", + meIllformedTable = "Illformed table: $1", + meNewSectionExpected = "new section expected $1", + meGeneralParseError = "general parse error", + meInvalidDirective = "invalid directive: '$1'", + meInvalidField = "invalid field: $1", + meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", + mwRedefinitionOfLabel = "redefinition of label '$1'", + mwUnknownSubstitution = "unknown substitution '$1'", + mwAmbiguousLink = "ambiguous doc link $1", + mwBrokenLink = "broken link '$1'", + mwUnsupportedLanguage = "language '$1' not supported", + mwUnsupportedField = "field '$1' not supported", + mwRstStyle = "RST style: $1", + mwUnusedImportdoc = "importdoc for '$1' is not used", + meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, - arg: string) {.nimcall.} ## what to do in case of an error - FindFileHandler* = proc (filename: string): string {.nimcall.} -{.deprecated: [TRstParseOptions: RstParseOptions, TRstParseOption: RstParseOption, - TMsgKind: MsgKind].} - -const - messages: array [MsgKind, string] = [ - meCannotOpenFile: "cannot open '$1'", - meExpected: "'$1' expected", - meGridTableNotImplemented: "grid table is not implemented", - meNewSectionExpected: "new section expected", - meGeneralParseError: "general parse error", - meInvalidDirective: "invalid directive: '$1'", - mwRedefinitionOfLabel: "redefinition of label '$1'", - mwUnknownSubstitution: "unknown substitution '$1'", - mwUnsupportedLanguage: "language '$1' not supported", - mwUnsupportedField: "field '$1' not supported" - ] + arg: string) {.closure, gcsafe.} ## what to do in case of an error + FindFileHandler* = proc (filename: string): string {.closure, gcsafe.} + FindRefFileHandler* = + proc (targetRelPath: string): + tuple[targetPath: string, linkRelPath: string] {.closure, gcsafe.} + ## returns where .html or .idx file should be found by its relative path; + ## `linkRelPath` is a prefix to be added before a link anchor from such file proc rstnodeToRefname*(n: PRstNode): string proc addNodes*(n: PRstNode): string -proc getFieldValue*(n: PRstNode, fieldname: string): string +proc getFieldValue*(n: PRstNode, fieldname: string): string {.gcsafe.} proc getArgument*(n: PRstNode): string # ----------------------------- scanner part -------------------------------- @@ -111,12 +136,21 @@ const ":geek:": "icon_e_geek", ":ugeek:": "icon_e_ugeek" } + SandboxDirAllowlist = [ + "image", "code", "code-block", "admonition", "attention", "caution", + "container", "contents", "danger", "default-role", "error", "figure", + "hint", "important", "index", "note", "role", "tip", "title", "warning"] type TokType = enum - tkEof, tkIndent, tkWhite, tkWord, tkAdornment, tkPunct, tkOther - Token = object # a RST token - kind*: TokType # the type of the token + tkEof, tkIndent, + tkWhite, tkWord, + tkAdornment, # used for chapter adornment, transitions and + # horizontal table borders + tkPunct, # one or many punctuation characters + tkOther + Token = object # a RST token + kind*: TokType # the type of the token ival*: int # the indentation or parsed integer value symbol*: string # the parsed symbol as string line*, col*: int # line and column of the token @@ -126,9 +160,8 @@ type buf*: cstring bufpos*: int line*, col*, baseIndent*: int - skipPounds*: bool -{.deprecated: [TTokType: TokType, TToken: Token, TTokenSeq: TokenSeq, - TLexer: Lexer].} + adornmentLine*: bool + escapeNext*: bool proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.kind = tkWord @@ -136,50 +169,82 @@ proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.col = L.col var pos = L.bufpos while true: - add(tok.symbol, L.buf[pos]) - inc(pos) + tok.symbol.add(L.buf[pos]) + inc pos if L.buf[pos] notin s: break - inc(L.col, pos - L.bufpos) + inc L.col, pos - L.bufpos L.bufpos = pos -proc getAdornment(L: var Lexer, tok: var Token) = - tok.kind = tkAdornment +proc isCurrentLineAdornment(L: var Lexer): bool = + var pos = L.bufpos + let c = L.buf[pos] + while true: + inc pos + if L.buf[pos] in {'\c', '\l', '\0'}: + break + if c == '+': # grid table + if L.buf[pos] notin {'-', '=', '+'}: + return false + else: # section adornment or table horizontal border + if L.buf[pos] notin {c, ' ', '\t', '\v', '\f'}: + return false + result = true + +proc getPunctAdornment(L: var Lexer, tok: var Token) = + if L.adornmentLine: + tok.kind = tkAdornment + else: + tok.kind = tkPunct tok.line = L.line tok.col = L.col var pos = L.bufpos - var c = L.buf[pos] - while true: - add(tok.symbol, L.buf[pos]) - inc(pos) - if L.buf[pos] != c: break - inc(L.col, pos - L.bufpos) + let c = L.buf[pos] + if not L.escapeNext and (c != '\\' or L.adornmentLine): + while true: + tok.symbol.add(L.buf[pos]) + inc pos + if L.buf[pos] != c: break + elif L.escapeNext: + tok.symbol.add(L.buf[pos]) + inc pos + else: # not L.escapeNext and c == '\\' and not L.adornmentLine + tok.symbol.add '\\' + inc pos + L.escapeNext = true + inc L.col, pos - L.bufpos L.bufpos = pos + if tok.symbol == "\\": tok.kind = tkPunct + # nim extension: standalone \ can not be adornment + +proc getBracket(L: var Lexer, tok: var Token) = + tok.kind = tkPunct + tok.line = L.line + tok.col = L.col + tok.symbol.add(L.buf[L.bufpos]) + inc L.col + inc L.bufpos proc getIndentAux(L: var Lexer, start: int): int = var pos = start - var buf = L.buf # skip the newline (but include it in the token!) - if buf[pos] == '\x0D': - if buf[pos + 1] == '\x0A': inc(pos, 2) - else: inc(pos) - elif buf[pos] == '\x0A': - inc(pos) - if L.skipPounds: - if buf[pos] == '#': inc(pos) - if buf[pos] == '#': inc(pos) + if L.buf[pos] == '\r': + if L.buf[pos + 1] == '\n': inc pos, 2 + else: inc pos + elif L.buf[pos] == '\n': + inc pos while true: - case buf[pos] - of ' ', '\x0B', '\x0C': - inc(pos) - inc(result) - of '\x09': - inc(pos) + case L.buf[pos] + of ' ', '\v', '\f': + inc pos + inc result + of '\t': + inc pos result = result - (result mod 8) + 8 else: break # EndOfFile also leaves the loop - if buf[pos] == '\0': + if L.buf[pos] == '\0': result = 0 - elif (buf[pos] == '\x0A') or (buf[pos] == '\x0D'): + elif L.buf[pos] == '\n' or L.buf[pos] == '\r': # look at the next line for proper indentation: result = getIndentAux(L, pos) L.bufpos = pos # no need to set back buf @@ -197,22 +262,28 @@ proc getIndent(L: var Lexer, tok: var Token) = proc rawGetTok(L: var Lexer, tok: var Token) = tok.symbol = "" tok.ival = 0 + if L.col == 0: + L.adornmentLine = false var c = L.buf[L.bufpos] case c of 'a'..'z', 'A'..'Z', '\x80'..'\xFF', '0'..'9': getThing(L, tok, SymChars) - of ' ', '\x09', '\x0B', '\x0C': - getThing(L, tok, {' ', '\x09'}) + of ' ', '\t', '\v', '\f': + getThing(L, tok, {' ', '\t'}) tok.kind = tkWhite - if L.buf[L.bufpos] in {'\x0D', '\x0A'}: + if L.buf[L.bufpos] in {'\r', '\n'}: rawGetTok(L, tok) # ignore spaces before \n - of '\x0D', '\x0A': + of '\r', '\n': getIndent(L, tok) - of '!', '\"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', - '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', - '|', '}', '~': - getAdornment(L, tok) - if len(tok.symbol) <= 3: tok.kind = tkPunct + L.adornmentLine = false + of '!', '\"', '#', '$', '%', '&', '\'', '*', '+', ',', '-', '.', + '/', ':', ';', '<', '=', '>', '?', '@', '\\', '^', '_', '`', + '|', '~': + if L.col == 0: + L.adornmentLine = L.isCurrentLineAdornment() + getPunctAdornment(L, tok) + of '(', ')', '[', ']', '{', '}': + getBracket(L, tok) else: tok.line = L.line tok.col = L.col @@ -220,299 +291,909 @@ proc rawGetTok(L: var Lexer, tok: var Token) = tok.kind = tkEof else: tok.kind = tkOther - add(tok.symbol, c) - inc(L.bufpos) - inc(L.col) + tok.symbol.add(c) + inc L.bufpos + inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq) = var L: Lexer - var length = len(tokens) + var length = tokens.len L.buf = cstring(buffer) L.line = 0 # skip UTF-8 BOM - if (L.buf[0] == '\xEF') and (L.buf[1] == '\xBB') and (L.buf[2] == '\xBF'): - inc(L.bufpos, 3) - L.skipPounds = skipPounds - if skipPounds: - if L.buf[L.bufpos] == '#': - inc(L.bufpos) - inc(result) - if L.buf[L.bufpos] == '#': - inc(L.bufpos) - inc(result) - L.baseIndent = 0 - while L.buf[L.bufpos] == ' ': - inc(L.bufpos) - inc(L.baseIndent) - inc(result) + if L.buf[0] == '\xEF' and L.buf[1] == '\xBB' and L.buf[2] == '\xBF': + inc L.bufpos, 3 while true: - inc(length) + inc length setLen(tokens, length) + let toEscape = L.escapeNext rawGetTok(L, tokens[length - 1]) + if toEscape: L.escapeNext = false if tokens[length - 1].kind == tkEof: break if tokens[0].kind == tkWhite: # BUGFIX - tokens[0].ival = len(tokens[0].symbol) + tokens[0].ival = tokens[0].symbol.len tokens[0].kind = tkIndent type - LevelMap = array[char, int] + LevelInfo = object + symbol: char # adornment character + hasOverline: bool # has also overline (besides underline)? + line: int # the last line of this style occurrence + # (for error message) + hasPeers: bool # has headings on the same level of hierarchy? + LiteralBlockKind = enum # RST-style literal blocks after `::` + lbNone, + lbIndentedLiteralBlock, + lbQuotedLiteralBlock + LevelMap = seq[LevelInfo] # Saves for each possible title adornment + # style its level in the current document. + SubstitutionKind = enum + rstSubstitution = "substitution", + hyperlinkAlias = "hyperlink alias", + implicitHyperlinkAlias = "implicitly-generated hyperlink alias" Substitution = object + kind*: SubstitutionKind key*: string value*: PRstNode - - SharedState = object - options: RstParseOptions # parsing options - uLevel, oLevel: int # counters for the section levels + info*: TLineInfo # place where the substitution was defined + AnchorRule = enum + arInternalRst, ## For automatically generated RST anchors (from + ## headings, footnotes, inline internal targets): + ## case-insensitive, 1-space-significant (by RST spec) + arExternalRst, ## For external .nim doc comments or .rst/.md + arNim ## For anchors generated by ``docgen.nim``: Nim-style case + ## sensitivity, etc. (see `proc normalizeNimName`_ for details) + arHyperlink, ## For links with manually set anchors in + ## form `text <pagename.html#anchor>`_ + RstAnchorKind = enum + manualDirectiveAnchor = "manual directive anchor", + manualInlineAnchor = "manual inline anchor", + footnoteAnchor = "footnote anchor", + headlineAnchor = "implicitly-generated headline anchor" + AnchorSubst = object + info: TLineInfo # the file where the anchor was defined + priority: int + case kind: range[arInternalRst .. arNim] + of arInternalRst: + anchorType: RstAnchorKind + target: PRstNode + of arExternalRst: + anchorTypeExt: RstAnchorKind + refnameExt: string + of arNim: + module: FileIndex # anchor's module (generally not the same as file) + tooltip: string # displayed tooltip for Nim-generated anchors + langSym: LangSymbol + refname: string # A reference name that will be inserted directly + # into HTML/Latex. + external: bool + AnchorSubstTable = Table[string, seq[AnchorSubst]] + # use `seq` to account for duplicate anchors + FootnoteType = enum + fnManualNumber, # manually numbered footnote like [3] + fnAutoNumber, # auto-numbered footnote [#] + fnAutoNumberLabel, # auto-numbered with label [#label] + fnAutoSymbol, # auto-symbol footnote [*] + fnCitation # simple text label like [citation2021] + FootnoteSubst = tuple + kind: FootnoteType # discriminator + number: int # valid for fnManualNumber (always) and fnAutoNumber, + # fnAutoNumberLabel after resolveSubs is called + autoNumIdx: int # order of occurrence: fnAutoNumber, fnAutoNumberLabel + autoSymIdx: int # order of occurrence: fnAutoSymbol + label: string # valid for fnAutoNumberLabel + RstFileTable* = object + filenameToIdx*: Table[string, FileIndex] + idxToFilename*: seq[string] + ImportdocInfo = object + used: bool # was this import used? + fromInfo: TLineInfo # place of `.. importdoc::` directive + idxPath: string # full path to ``.idx`` file + linkRelPath: string # prefix before target anchor + title: string # document title obtained from ``.idx`` + RstSharedState = object + options*: RstParseOptions # parsing options + hLevels: LevelMap # hierarchy of heading styles + hTitleCnt: int # =0 if no title, =1 if only main title, + # =2 if both title and subtitle are present + hCurLevel: int # current section level + currRole: string # current interpreted text role + currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions - refs: seq[Substitution] # references - underlineToLevel: LevelMap # Saves for each possible title adornment - # character its level in the - # current document. - # This is for single underline adornments. - overlineToLevel: LevelMap # Saves for each possible title adornment - # character its level in the current - # document. - # This is for over-underline adornments. + refs*: seq[Substitution] # references + anchors*: AnchorSubstTable + # internal target substitutions + lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#] + lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_ + currFootnoteNumRef: int # ... their counter for `resolveSubs` + lineFootnoteSym: seq[TLineInfo] # footnote line, auto symbols .. [*] + lineFootnoteSymRef: seq[TLineInfo] # footnote line, their reference [*]_ + currFootnoteSymRef: int # ... their counter for `resolveSubs` + footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, + # number, order of occurrence msgHandler: MsgHandler # How to handle errors. - findFile: FindFileHandler # How to find files. - - PSharedState = ref SharedState + findFile: FindFileHandler # How to find files for include. + findRefFile: FindRefFileHandler + # How to find files imported by importdoc. + filenames*: RstFileTable # map file name <-> FileIndex (for storing + # file names for warnings after 1st stage) + currFileIdx*: FileIndex # current index in `filenames` + tocPart*: seq[PRstNode] # all the headings of a document + hasToc*: bool + idxImports*: Table[string, ImportdocInfo] + # map `importdoc`ed filename -> it's info + nimFileImported*: bool # Was any ``.nim`` module `importdoc`ed ? + + PRstSharedState* = ref RstSharedState + ManualAnchor = object + alias: string # a (short) name that can substitute the `anchor` + anchor: string # anchor = id = refname + info: TLineInfo RstParser = object of RootObj idx*: int tok*: TokenSeq - s*: PSharedState + s*: PRstSharedState indentStack*: seq[int] - filename*: string - line*, col*: int - hasToc*: bool + line*, col*: int ## initial line/column of whole text or + ## documenation fragment that will be added + ## in case of error/warning reporting to + ## (relative) line/column of the token. + curAnchors*: seq[ManualAnchor] + ## seq to accumulate aliases for anchors: + ## because RST can have >1 alias per 1 anchor EParseError* = object of ValueError -{.deprecated: [TLevelMap: LevelMap, TSubstitution: Substitution, - TSharedState: SharedState, TRstParser: RstParser, - TMsgHandler: MsgHandler, TFindFileHandler: FindFileHandler, - TMsgClass: MsgClass].} + SectionParser = proc (p: var RstParser): PRstNode {.nimcall, gcsafe.} + +const + LineRstInit* = 1 ## Initial line number for standalone RST text + ColRstInit* = 0 ## Initial column number for standalone RST text + ## (Nim global reporting adds ColOffset=1) + ColRstOffset* = 1 ## 1: a replica of ColOffset for internal use + +template currentTok(p: RstParser): Token = p.tok[p.idx] +template prevTok(p: RstParser): Token = p.tok[p.idx - 1] +template nextTok(p: RstParser): Token = p.tok[p.idx + 1] proc whichMsgClass*(k: MsgKind): MsgClass = ## returns which message class `k` belongs to. - case ($k)[1] + case k.symbolName[1] of 'e', 'E': result = mcError of 'w', 'W': result = mcWarning of 'h', 'H': result = mcHint else: assert false, "msgkind does not fit naming scheme" proc defaultMsgHandler*(filename: string, line, col: int, msgkind: MsgKind, - arg: string) {.procvar.} = + arg: string) = let mc = msgkind.whichMsgClass - let a = messages[msgkind] % arg - let message = "$1($2, $3) $4: $5" % [filename, $line, $col, $mc, a] + let a = $msgkind % arg + var message: string + toLocation(message, filename, line, col + ColRstOffset) + message.add " $1: $2" % [$mc, a] if mc == mcError: raise newException(EParseError, message) else: writeLine(stdout, message) -proc defaultFindFile*(filename: string): string {.procvar.} = - if existsFile(filename): result = filename +proc defaultFindFile*(filename: string): string = + if fileExists(filename): result = filename else: result = "" -proc newSharedState(options: RstParseOptions, - findFile: FindFileHandler, - msgHandler: MsgHandler): PSharedState = - new(result) - result.subs = @[] - result.refs = @[] - result.options = options - result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler - result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc defaultFindRefFile*(filename: string): (string, string) = + (filename, "") + +proc defaultRole(options: RstParseOptions): string = + if roNimFile in options: "nim" else: "literal" + +proc whichRoleAux(sym: string): RstNodeKind = + let r = sym.toLowerAscii + case r + of "idx": result = rnIdx + of "literal": result = rnInlineLiteral + of "strong": result = rnStrongEmphasis + of "emphasis": result = rnEmphasis + of "sub", "subscript": result = rnSub + of "sup", "superscript": result = rnSup + # literal and code are the same in our implementation + of "code": result = rnInlineLiteral + of "program", "option", "tok": result = rnCodeFragment + # c++ currently can be spelled only as cpp, c# only as csharp + elif getSourceLanguage(r) != langNone: + result = rnInlineCode + else: # unknown role + result = rnUnknownRole + +proc len(filenames: RstFileTable): int = filenames.idxToFilename.len + +proc addFilename*(s: PRstSharedState, file1: string): FileIndex = + ## Returns index of filename, adding it if it has not been used before + let nextIdx = s.filenames.len.FileIndex + result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) + if result == nextIdx: + s.filenames.filenameToIdx[file1] = result + s.filenames.idxToFilename.add file1 + +proc setCurrFilename*(s: PRstSharedState, file1: string) = + s.currFileIdx = addFilename(s, file1) + +proc getFilename(filenames: RstFileTable, fid: FileIndex): string = + doAssert(0 <= fid.int and fid.int < filenames.len, + "incorrect FileIndex $1 (range 0..$2)" % [ + $fid.int, $(filenames.len - 1)]) + result = filenames.idxToFilename[fid.int] + +proc getFilename(s: PRstSharedState, subst: AnchorSubst): string = + getFilename(s.filenames, subst.info.fileIndex) + +proc getModule(s: PRstSharedState, subst: AnchorSubst): string = + result = getFilename(s.filenames, subst.module) + +proc currFilename(s: PRstSharedState): string = + getFilename(s.filenames, s.currFileIdx) + +proc newRstSharedState*(options: RstParseOptions, + filename: string, + findFile: FindFileHandler, + findRefFile: FindRefFileHandler, + msgHandler: MsgHandler, + hasToc: bool): PRstSharedState = + let r = defaultRole(options) + result = PRstSharedState( + currRole: r, + currRoleKind: whichRoleAux(r), + options: options, + msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, + findFile: if not isNil(findFile): findFile else: defaultFindFile, + findRefFile: + if not isNil(findRefFile): findRefFile + else: defaultFindRefFile, + hasToc: hasToc + ) + setCurrFilename(result, filename) + +proc curLine(p: RstParser): int = p.line + currentTok(p).line + +proc findRelativeFile(p: RstParser; filename: string): string = + result = p.s.currFilename.splitFile.dir / filename + if not fileExists(result): + result = p.s.findFile(filename) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = - p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, - p.col + p.tok[p.idx].col, msgKind, arg) + p.s.msgHandler(p.s.currFilename, curLine(p), + p.col + currentTok(p).col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = + s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) + +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string; + line, col: int) = + s.msgHandler(s.currFilename, line, col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, filename: string, msgKind: MsgKind, + arg: string) = + s.msgHandler(filename, LineRstInit, ColRstInit, msgKind, arg) + +proc rstMessage*(filenames: RstFileTable, f: MsgHandler, + info: TLineInfo, msgKind: MsgKind, arg: string) = + ## Print warnings using `info`, i.e. in 2nd-pass warnings for + ## footnotes/substitutions/references or from ``rstgen.nim``. + let file = getFilename(filenames, info.fileIndex) + f(file, info.line.int, info.col.int, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = - p.s.msgHandler(p.filename, p.line + line, + p.s.msgHandler(p.s.currFilename, p.line + line, p.col + col, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind) = - p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, - p.col + p.tok[p.idx].col, msgKind, - p.tok[p.idx].symbol) + p.s.msgHandler(p.s.currFilename, curLine(p), + p.col + currentTok(p).col, msgKind, + currentTok(p).symbol) + +# Functions `isPureRst` & `stopOrWarn` address differences between +# Markdown and RST: +# * Markdown always tries to continue working. If it is really impossible +# to parse a markup element, its proc just returns `nil` and parsing +# continues for it as for normal text paragraph. +# The downside is that real mistakes/typos are often silently ignored. +# The same applies to legacy `RstMarkdown` mode for nimforum. +# * RST really signals errors. The downside is that it's more intrusive - +# the user must escape special syntax with \ explicitly. +# +# TODO: we need to apply this strategy to all markup elements eventually. -when false: - proc corrupt(p: RstParser) = - assert p.indentStack[0] == 0 - for i in 1 .. high(p.indentStack): assert p.indentStack[i] < 1_000 +func isPureRst(p: RstParser): bool = roSupportMarkdown notin p.s.options +func isRst(p: RstParser): bool = roPreferMarkdown notin p.s.options +func isMd(p: RstParser): bool = roPreferMarkdown in p.s.options +func isMd(s: PRstSharedState): bool = roPreferMarkdown in s.options + +proc stopOrWarn(p: RstParser, errorType: MsgKind, arg: string) = + let realMsgKind = if isPureRst(p): errorType else: mwRstStyle + rstMessage(p, realMsgKind, arg) + +proc stopOrWarn(p: RstParser, errorType: MsgKind, arg: string, line, col: int) = + let realMsgKind = if isPureRst(p): errorType else: mwRstStyle + rstMessage(p, realMsgKind, arg, line, col) proc currInd(p: RstParser): int = result = p.indentStack[high(p.indentStack)] proc pushInd(p: var RstParser, ind: int) = - add(p.indentStack, ind) + p.indentStack.add(ind) proc popInd(p: var RstParser) = - if len(p.indentStack) > 1: setLen(p.indentStack, len(p.indentStack) - 1) + if p.indentStack.len > 1: setLen(p.indentStack, p.indentStack.len - 1) + +# Working with indentation in rst.nim +# ----------------------------------- +# +# Every line break has an associated tkIndent. +# The tokenizer writes back the first column of next non-blank line +# in all preceeding tkIndent tokens to the `ival` field of tkIndent. +# +# RST document is separated into body elements (B.E.), every of which +# has a dedicated handler proc (or block of logic when B.E. is a block quote) +# that should follow the next rule: +# Every B.E. handler proc should finish at tkIndent (newline) +# after its B.E. finishes. +# Then its callers (which is `parseSection` or another B.E. handler) +# check for tkIndent ival (without necessity to advance `p.idx`) +# and decide themselves whether they continue processing or also stop. +# +# An example:: +# +# L RST text fragment indentation +# +--------------------+ +# 1 | | <- (empty line at the start of file) no tokens +# 2 |First paragraph. | <- tkIndent has ival=0, and next tkWord has col=0 +# 3 | | <- tkIndent has ival=0 +# 4 |* bullet item and | <- tkIndent has ival=0, and next tkPunct has col=0 +# 5 | its continuation | <- tkIndent has ival=2, and next tkWord has col=2 +# 6 | | <- tkIndent has ival=4 +# 7 | Block quote | <- tkIndent has ival=4, and next tkWord has col=4 +# 8 | | <- tkIndent has ival=0 +# 9 | | <- tkIndent has ival=0 +# 10|Final paragraph | <- tkIndent has ival=0, and tkWord has col=0 +# +--------------------+ +# C:01234 +# +# Here parser starts with initial `indentStack=[0]` and then calls the +# 1st `parseSection`: +# +# - `parseSection` calls `parseParagraph` and "First paragraph" is parsed +# - bullet list handler is started at reaching ``*`` (L4 C0), it +# starts bullet item logic (L4 C2), which calls `pushInd(p, ind=2)`, +# then calls `parseSection` (2nd call, nested) which parses +# paragraph "bullet list and its continuation" and then starts +# a block quote logic (L7 C4). +# The block quote logic calls calls `pushInd(p, ind=4)` and +# calls `parseSection` again, so a (simplified) sequence of calls now is:: +# +# parseSection -> parseBulletList -> +# parseSection (+block quote logic) -> parseSection +# +# 3rd `parseSection` finishes, block quote logic calls `popInd(p)`, +# it returns to bullet item logic, which sees that next tkIndent has +# ival=0 and stops there since the required indentation for a bullet item +# is 2 and 0<2; the bullet item logic calls `popInd(p)`. +# Then bullet list handler checks that next tkWord (L10 C0) has the +# right indentation but does not have ``*`` so stops at tkIndent (L10). +# - 1st `parseSection` invocation calls `parseParagraph` and the +# "Final paragraph" is parsed. +# +# If a B.E. handler has advanced `p.idx` past tkIndent to check +# whether it should continue its processing or not, and decided not to, +# then this B.E. handler should step back (e.g. do `dec p.idx`). -proc initParser(p: var RstParser, sharedState: PSharedState) = +proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.indentStack = @[0] p.tok = @[] p.idx = 0 - p.filename = "" - p.hasToc = false - p.col = 0 - p.line = 1 + p.col = ColRstInit + p.line = LineRstInit p.s = sharedState proc addNodesAux(n: PRstNode, result: var string) = + if n == nil: + return if n.kind == rnLeaf: - add(result, n.text) + result.add(n.text) else: - for i in countup(0, len(n) - 1): addNodesAux(n.sons[i], result) + for i in 0 ..< n.len: addNodesAux(n.sons[i], result) proc addNodes(n: PRstNode): string = - result = "" - addNodesAux(n, result) + n.addNodesAux(result) + +proc linkName(n: PRstNode): string = + ## Returns a normalized reference name, see: + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + n.addNodes.toLowerAscii proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = + template special(s) = + if b: + r.add('-') + b = false + r.add(s) + + if n == nil: return if n.kind == rnLeaf: - for i in countup(0, len(n.text) - 1): + for i in 0 ..< n.text.len: case n.text[i] of '0'..'9': if b: - add(r, '-') + r.add('-') b = false - if len(r) == 0: add(r, 'Z') - add(r, n.text[i]) - of 'a'..'z': + if r.len == 0: r.add('Z') + r.add(n.text[i]) + of 'a'..'z', '\128'..'\255': if b: - add(r, '-') + r.add('-') b = false - add(r, n.text[i]) + r.add(n.text[i]) of 'A'..'Z': if b: - add(r, '-') + r.add('-') b = false - add(r, chr(ord(n.text[i]) - ord('A') + ord('a'))) + r.add(chr(ord(n.text[i]) - ord('A') + ord('a'))) + of '$': special "dollar" + of '%': special "percent" + of '&': special "amp" + of '^': special "roof" + of '!': special "emark" + of '?': special "qmark" + of '*': special "star" + of '+': special "plus" + of '-': special "minus" + of '/': special "slash" + of '\\': special "backslash" + of '=': special "eq" + of '<': special "lt" + of '>': special "gt" + of '~': special "tilde" + of ':': special "colon" + of '.': special "dot" + of '@': special "at" + of '|': special "bar" else: - if (len(r) > 0): b = true + if r.len > 0: b = true else: - for i in countup(0, len(n) - 1): rstnodeToRefnameAux(n.sons[i], r, b) + for i in 0 ..< n.len: rstnodeToRefnameAux(n.sons[i], r, b) proc rstnodeToRefname(n: PRstNode): string = - result = "" var b = false rstnodeToRefnameAux(n, result, b) -proc findSub(p: var RstParser, n: PRstNode): int = +proc findSub(s: PRstSharedState, n: PRstNode): int = var key = addNodes(n) # the spec says: if no exact match, try one without case distinction: - for i in countup(0, high(p.s.subs)): - if key == p.s.subs[i].key: + for i in countup(0, high(s.subs)): + if key == s.subs[i].key: return i - for i in countup(0, high(p.s.subs)): - if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: + for i in countup(0, high(s.subs)): + if cmpIgnoreStyle(key, s.subs[i].key) == 0: return i result = -1 +proc lineInfo(p: RstParser, iTok: int): TLineInfo = + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx + +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) +# TODO: we need this simplification because we don't preserve exact starting +# token of currently parsed element: +proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1) + proc setSub(p: var RstParser, key: string, value: PRstNode) = - var length = len(p.s.subs) - for i in countup(0, length - 1): + var length = p.s.subs.len + for i in 0 ..< length: if key == p.s.subs[i].key: p.s.subs[i].value = value return - setLen(p.s.subs, length + 1) - p.s.subs[length].key = key - p.s.subs[length].value = value + p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p))) -proc setRef(p: var RstParser, key: string, value: PRstNode) = - var length = len(p.s.refs) - for i in countup(0, length - 1): +proc setRef(p: var RstParser, key: string, value: PRstNode, + refType: SubstitutionKind) = + var length = p.s.refs.len + for i in 0 ..< length: if key == p.s.refs[i].key: if p.s.refs[i].value.addNodes != value.addNodes: rstMessage(p, mwRedefinitionOfLabel, key) - p.s.refs[i].value = value return - setLen(p.s.refs, length + 1) - p.s.refs[length].key = key - p.s.refs[length].value = value + p.s.refs.add(Substitution(kind: refType, key: key, value: value, + info: prevLineInfo(p))) + +proc findRef(s: PRstSharedState, key: string): seq[Substitution] = + for i in countup(0, high(s.refs)): + if key == s.refs[i].key: + result.add s.refs[i] + +# Ambiguity in links: we don't follow procedure of removing implicit targets +# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets +# Instead we just give explicit links a higher priority than to implicit ones +# and report ambiguities as warnings. Hopefully it is easy to remove +# ambiguities manually. Nim auto-generated links from ``docgen.nim`` +# have lowest priority: 1 (for procs) and below for other symbol types. + +proc refPriority(k: SubstitutionKind): int = + case k + of rstSubstitution: result = 8 + of hyperlinkAlias: result = 7 + of implicitHyperlinkAlias: result = 2 + +proc internalRefPriority(k: RstAnchorKind): int = + case k + of manualDirectiveAnchor: result = 6 + of manualInlineAnchor: result = 5 + of footnoteAnchor: result = 4 + of headlineAnchor: result = 3 + +proc `$`(subst: AnchorSubst): string = # for debug + let s = + case subst.kind + of arInternalRst: "type=" & $subst.anchorType + of arExternalRst: "type=" & $subst.anchorTypeExt + of arNim: "langsym=" & $subst.langSym + result = "(kind=$1, priority=$2, $3)" % [$subst.kind, $subst.priority, s] + +proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, + anchorType: RstAnchorKind) = + ## Associates node `target` (which has field `anchor`) with an + ## alias `name` and updates the corresponding aliases in `p.curAnchors`. + let prio = internalRefPriority(anchorType) + for a in p.curAnchors: + p.s.anchors.mgetOrPut(a.alias, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, target: target, priority: prio, + info: a.info, anchorType: manualDirectiveAnchor)) + if name != "": + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, target: target, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) + p.curAnchors.setLen 0 + +proc addAnchorExtRst(s: var PRstSharedState, key: string, refn: string, + anchorType: RstAnchorKind, info: TLineInfo) = + let name = key.toLowerAscii + let prio = internalRefPriority(anchorType) + s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arExternalRst, refnameExt: refn, priority: prio, + info: info, + anchorTypeExt: anchorType)) + +proc addAnchorNim*(s: var PRstSharedState, external: bool, refn: string, tooltip: string, + langSym: LangSymbol, priority: int, + info: TLineInfo, module: FileIndex) = + ## Adds an anchor `refn`, which follows + ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) + s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arNim, external: external, refname: refn, langSym: langSym, + tooltip: tooltip, priority: priority, + info: info)) + +proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, + info: TLineInfo): + seq[AnchorSubst] = + var langSym: LangSymbol + try: + langSym = toLangSymbol(signature) + except ValueError: # parsing failed, not a Nim symbol + return + let substitutions = s.anchors.getOrDefault(langSym.name, + newSeq[AnchorSubst]()) + if substitutions.len == 0: + return + # logic to select only groups instead of concrete symbols + # with overloads, note that the same symbol can be defined + # in multiple modules and `importdoc`ed: + type GroupKey = tuple[symKind: string, origModule: string] + # map (symKind, file) (like "proc", "os.nim") -> found symbols/groups: + var found: Table[GroupKey, seq[AnchorSubst]] + for subst in substitutions: + if subst.kind == arNim: + if match(subst.langSym, langSym): + let key: GroupKey = (subst.langSym.symKind, getModule(s, subst)) + found.mgetOrPut(key, newSeq[AnchorSubst]()).add subst + for key, sList in found: + if sList.len == 1: + result.add sList[0] + else: # > 1, there are overloads, potential ambiguity in this `symKind` + if langSym.parametersProvided: + # there are non-group signatures, select only them + for s in sList: + if not s.langSym.isGroup: + result.add s + else: # when there are many overloads a link like foo_ points to all + # of them, so selecting the group + var foundGroup = false + for s in sList: + if s.langSym.isGroup: + result.add s + foundGroup = true + break + doAssert(foundGroup, + "docgen has not generated the group for $1 (file $2)" % [ + langSym.name, getModule(s, sList[0]) ]) + +proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): + seq[AnchorSubst] = + let name = linkText.toLowerAscii + let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) + for s in substitutions: + if s.kind in {arInternalRst, arExternalRst}: + result.add s + +proc addFootnoteNumManual(p: var RstParser, num: int) = + ## add manually-numbered footnote + for fnote in p.s.footnotes: + if fnote.number == num: + rstMessage(p, mwRedefinitionOfLabel, $num) + return + p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) + +proc addFootnoteNumAuto(p: var RstParser, label: string) = + ## add auto-numbered footnote. + ## Empty label [#] means it'll be resolved by the occurrence. + if label == "": # simple auto-numbered [#] + p.s.lineFootnoteNum.add lineInfo(p) + p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label)) + else: # auto-numbered with label [#label] + for fnote in p.s.footnotes: + if fnote.label == label: + rstMessage(p, mwRedefinitionOfLabel, label) + return + p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label)) + +proc addFootnoteSymAuto(p: var RstParser) = + p.s.lineFootnoteSym.add lineInfo(p) + p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) + +proc orderFootnotes(s: PRstSharedState) = + ## numerate auto-numbered footnotes taking into account that all + ## manually numbered ones always have preference. + ## Save the result back to `s.footnotes`. + + # Report an error if found any mismatch in number of automatic footnotes + proc listFootnotes(locations: seq[TLineInfo]): string = + var lines: seq[string] + for info in locations: + if s.filenames.len > 1: + let file = getFilename(s.filenames, info.fileIndex) + lines.add file & ":" + else: # no need to add file name here if there is only 1 + lines.add "" + lines[^1].add $info.line + result.add $lines.len & " (lines " & join(lines, ", ") & ")" + if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteNum), + listFootnotes(s.lineFootnoteNumRef)] & + " for auto-numbered footnotes") + if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteSym), + listFootnotes(s.lineFootnoteSymRef)] & + " for auto-symbol footnotes") + + var result: seq[FootnoteSubst] + var manuallyN, autoN, autoSymbol: seq[FootnoteSubst] + for fs in s.footnotes: + if fs.kind == fnManualNumber: manuallyN.add fs + elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs + else: autoSymbol.add fs + + if autoN.len == 0: + result = manuallyN + else: + # fill gaps between manually numbered footnotes in ascending order + manuallyN.sort() # sort by number - its first field + var lst = initSinglyLinkedList[FootnoteSubst]() + for elem in manuallyN: lst.append(elem) + var firstAuto = 0 + if lst.head == nil or lst.head.value.number != 1: + # no manual footnote [1], start numeration from 1 for auto-numbered + lst.prepend (autoN[0].kind, 1, autoN[0].autoNumIdx, -1, autoN[0].label) + firstAuto = 1 + var curNode = lst.head + var nextNode: SinglyLinkedNode[FootnoteSubst] + # go simultaneously through `autoN` and `lst` looking for gaps + for (kind, x, autoNumIdx, y, label) in autoN[firstAuto .. ^1]: + while (nextNode = curNode.next; nextNode != nil): + if nextNode.value.number - curNode.value.number > 1: + # gap found, insert new node `n` between curNode and nextNode: + var n = newSinglyLinkedNode((kind, curNode.value.number + 1, + autoNumIdx, -1, label)) + curNode.next = n + n.next = nextNode + curNode = n + break + else: + curNode = nextNode + if nextNode == nil: # no gap found, just append + lst.append (kind, curNode.value.number + 1, autoNumIdx, -1, label) + curNode = lst.tail + result = lst.toSeq + + # we use ASCII symbols instead of those recommended in RST specification: + const footnoteAutoSymbols = ["*", "^", "+", "=", "~", "$", "@", "%", "&"] + for fs in autoSymbol: + # assignment order: *, **, ***, ^, ^^, ^^^, ... &&&, ****, *****, ... + let i = fs.autoSymIdx - 1 + let symbolNum = (i div 3) mod footnoteAutoSymbols.len + let nSymbols = (1 + i mod 3) + 3 * (i div (3 * footnoteAutoSymbols.len)) + let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols) + result.add((fs.kind, -1, -1, fs.autoSymIdx, label)) + + s.footnotes = result + +proc getFootnoteNum(s: PRstSharedState, label: string): int = + ## get number from label. Must be called after `orderFootnotes`. + result = -1 + for fnote in s.footnotes: + if fnote.label == label: + return fnote.number -proc findRef(p: var RstParser, key: string): PRstNode = - for i in countup(0, high(p.s.refs)): - if key == p.s.refs[i].key: - return p.s.refs[i].value +proc getFootnoteNum(s: PRstSharedState, order: int): int = + ## get number from occurrence. Must be called after `orderFootnotes`. + result = -1 + for fnote in s.footnotes: + if fnote.autoNumIdx == order: + return fnote.number + +proc getAutoSymbol(s: PRstSharedState, order: int): string = + ## get symbol from occurrence of auto-symbol footnote. + result = "???" + for fnote in s.footnotes: + if fnote.autoSymIdx == order: + return fnote.label + +proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = + ## create node and consume the current anchor + result = newRstNode(kind) + if p.curAnchors.len > 0: + result.anchor = p.curAnchors[0].anchor + addAnchorRst(p, "", result, manualDirectiveAnchor) + +template newLeaf(s: string): PRstNode = newRstLeaf(s) proc newLeaf(p: var RstParser): PRstNode = - result = newRstNode(rnLeaf, p.tok[p.idx].symbol) + result = newLeaf(currentTok(p).symbol) + +proc validRefnamePunct(x: string): bool = + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + x.len == 1 and x[0] in {'-', '_', '.', ':', '+'} + +func getRefnameIdx(p: RstParser, startIdx: int): int = + ## Gets last token index of a refname ("word" in RST terminology): + ## + ## reference names are single words consisting of alphanumerics plus + ## isolated (no two adjacent) internal hyphens, underscores, periods, + ## colons and plus signs; no whitespace or other characters are allowed. + ## + ## Refnames are used for: + ## - reference names + ## - role names + ## - directive names + ## - footnote labels + ## + # TODO: use this func in all other relevant places + var j = startIdx + if p.tok[j].kind == tkWord: + inc j + while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and + p.tok[j+1].kind == tkWord: + inc j, 2 + result = j - 1 + +func getRefname(p: RstParser, startIdx: int): (string, int) = + let lastIdx = getRefnameIdx(p, startIdx) + result[1] = lastIdx + for j in startIdx..lastIdx: + result[0].add p.tok[j].symbol proc getReferenceName(p: var RstParser, endStr: string): PRstNode = var res = newRstNode(rnInner) while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWord, tkOther, tkWhite: - add(res, newLeaf(p)) + res.add(newLeaf(p)) of tkPunct: - if p.tok[p.idx].symbol == endStr: - inc(p.idx) + if currentTok(p).symbol == endStr: + inc p.idx break else: - add(res, newLeaf(p)) + res.add(newLeaf(p)) else: rstMessage(p, meExpected, endStr) break - inc(p.idx) + inc p.idx result = res proc untilEol(p: var RstParser): PRstNode = result = newRstNode(rnInner) - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - add(result, newLeaf(p)) - inc(p.idx) + while currentTok(p).kind notin {tkIndent, tkEof}: + result.add(newLeaf(p)) + inc p.idx proc expect(p: var RstParser, tok: string) = - if p.tok[p.idx].symbol == tok: inc(p.idx) + if currentTok(p).symbol == tok: inc p.idx else: rstMessage(p, meExpected, tok) -proc isInlineMarkupEnd(p: RstParser, markup: string): bool = - result = p.tok[p.idx].symbol == markup - if not result: - return # Rule 3: - result = not (p.tok[p.idx - 1].kind in {tkIndent, tkWhite}) - if not result: - return # Rule 4: - result = (p.tok[p.idx + 1].kind in {tkIndent, tkWhite, tkEof}) or - (p.tok[p.idx + 1].symbol[0] in - {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', - '?', '_'}) - if not result: - return # Rule 7: +proc inlineMarkdownEnd(p: RstParser): bool = + result = prevTok(p).kind notin {tkIndent, tkWhite} + ## (For a special case of ` we don't allow spaces surrounding it + ## unlike original Markdown because this behavior confusing/useless) + +proc inlineRstEnd(p: RstParser): bool = + # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules + # Rule 2: + result = prevTok(p).kind notin {tkIndent, tkWhite} + if not result: return + # Rule 7: + result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or + nextTok(p).symbol[0] in + {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'} + +proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool = + if exact: + result = currentTok(p).symbol == markup + else: + result = currentTok(p).symbol.endsWith markup + if (not result) and markup == "``": + # check that escaping may have splitted `` to 2 tokens ` and ` + result = currentTok(p).symbol == "`" and prevTok(p).symbol == "`" + if not result: return + # surroundings check + if markup in ["_", "__"]: + result = inlineRstEnd(p) + else: + if roPreferMarkdown in p.s.options: result = inlineMarkdownEnd(p) + else: result = inlineRstEnd(p) + +proc rstRuleSurround(p: RstParser): bool = + result = true + # Rules 4 & 5: if p.idx > 0: - if (markup != "``") and (p.tok[p.idx - 1].symbol == "\\"): - result = false + var d: char + var c = prevTok(p).symbol[0] + case c + of '\'', '\"': d = c + of '(': d = ')' + of '[': d = ']' + of '{': d = '}' + of '<': d = '>' + else: d = '\0' + if d != '\0': result = nextTok(p).symbol[0] != d + +proc inlineMarkdownStart(p: RstParser): bool = + result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof} + if not result: return + # this rst rule is really nice, let us use it in Markdown mode too. + result = rstRuleSurround(p) + +proc inlineRstStart(p: RstParser): bool = + ## rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules + # Rule 6 + result = p.idx == 0 or prevTok(p).kind in {tkIndent, tkWhite} or + prevTok(p).symbol[0] in {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'} + if not result: return + # Rule 1: + result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof} + if not result: return + result = rstRuleSurround(p) proc isInlineMarkupStart(p: RstParser, markup: string): bool = - var d: char - result = p.tok[p.idx].symbol == markup - if not result: - return # Rule 1: - result = (p.idx == 0) or (p.tok[p.idx - 1].kind in {tkIndent, tkWhite}) or - (p.tok[p.idx - 1].symbol[0] in - {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'}) - if not result: - return # Rule 2: - result = not (p.tok[p.idx + 1].kind in {tkIndent, tkWhite, tkEof}) - if not result: - return # Rule 5 & 7: - if p.idx > 0: - if p.tok[p.idx - 1].symbol == "\\": - result = false - else: - var c = p.tok[p.idx - 1].symbol[0] - case c - of '\'', '\"': d = c - of '(': d = ')' - of '[': d = ']' - of '{': d = '}' - of '<': d = '>' - else: d = '\0' - if d != '\0': result = p.tok[p.idx + 1].symbol[0] != d + if markup != "_`": + result = currentTok(p).symbol == markup + else: # _` is a 2 token case + result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`" + if not result: return + # surroundings check + if markup in ["_", "__", "[", "|"]: + # Note: we require space/punctuation even before [markdown link](...) + result = inlineRstStart(p) + else: + if roPreferMarkdown in p.s.options: result = inlineMarkdownStart(p) + else: result = inlineRstStart(p) proc match(p: RstParser, start: int, expr: string): bool = # regular expressions are: @@ -521,97 +1202,159 @@ proc match(p: RstParser, start: int, expr: string): bool = # ' ' tkWhite # 'a' tkAdornment # 'i' tkIndent + # 'I' tkIndent or tkEof # 'p' tkPunct # 'T' always true # 'E' whitespace, indent or eof - # 'e' tkWord or '#' (for enumeration lists) + # 'e' any enumeration sequence or '#' (for enumeration lists) + # 'x' a..z or '#' (for enumeration lists) + # 'n' 0..9 or '#' (for enumeration lists) var i = 0 var j = start - var last = len(expr) - 1 + var last = expr.len - 1 while i <= last: case expr[i] - of 'w': result = p.tok[j].kind == tkWord + of 'w': + let lastIdx = getRefnameIdx(p, j) + result = lastIdx >= j + if result: j = lastIdx of ' ': result = p.tok[j].kind == tkWhite of 'i': result = p.tok[j].kind == tkIndent + of 'I': result = p.tok[j].kind in {tkIndent, tkEof} of 'p': result = p.tok[j].kind == tkPunct of 'a': result = p.tok[j].kind == tkAdornment of 'o': result = p.tok[j].kind == tkOther of 'T': result = true of 'E': result = p.tok[j].kind in {tkEof, tkWhite, tkIndent} - of 'e': - result = (p.tok[j].kind == tkWord) or (p.tok[j].symbol == "#") + of 'e', 'x', 'n': + result = p.tok[j].kind == tkWord or p.tok[j].symbol == "#" if result: case p.tok[j].symbol[0] - of 'a'..'z', 'A'..'Z': result = len(p.tok[j].symbol) == 1 - of '0'..'9': result = allCharsInSet(p.tok[j].symbol, {'0'..'9'}) - else: discard + of '#': result = true + of 'a'..'z', 'A'..'Z': + result = expr[i] in {'e', 'x'} and p.tok[j].symbol.len == 1 + of '0'..'9': + result = expr[i] in {'e', 'n'} and + allCharsInSet(p.tok[j].symbol, {'0'..'9'}) + else: result = false else: var c = expr[i] var length = 0 - while (i <= last) and (expr[i] == c): - inc(i) - inc(length) - dec(i) - result = (p.tok[j].kind in {tkPunct, tkAdornment}) and - (len(p.tok[j].symbol) == length) and (p.tok[j].symbol[0] == c) + while i <= last and expr[i] == c: + inc i + inc length + dec i + result = p.tok[j].kind in {tkPunct, tkAdornment} and + p.tok[j].symbol.len == length and p.tok[j].symbol[0] == c if not result: return - inc(j) - inc(i) + inc j + inc i result = true -proc fixupEmbeddedRef(n, a, b: PRstNode) = +proc safeProtocol*(linkStr: var string): string = + # Returns link's protocol and, if it's not safe, clears `linkStr` + result = "" + if scanf(linkStr, "$w:", result): + # if it has a protocol at all, ensure that it's not 'javascript:' or worse: + if cmpIgnoreCase(result, "http") == 0 or + cmpIgnoreCase(result, "https") == 0 or + cmpIgnoreCase(result, "ftp") == 0: + discard "it's fine" + else: + linkStr = "" + +proc fixupEmbeddedRef(p: var RstParser, n, a, b: PRstNode): bool = + # Returns `true` if the link belongs to an allowed protocol var sep = - 1 - for i in countdown(len(n) - 2, 0): + for i in countdown(n.len - 2, 0): if n.sons[i].text == "<": sep = i break - var incr = if (sep > 0) and (n.sons[sep - 1].text[0] == ' '): 2 else: 1 - for i in countup(0, sep - incr): add(a, n.sons[i]) - for i in countup(sep + 1, len(n) - 2): add(b, n.sons[i]) + var incr = if sep > 0 and n.sons[sep - 1].text[0] == ' ': 2 else: 1 + for i in countup(0, sep - incr): a.add(n.sons[i]) + var linkStr = "" + for i in countup(sep + 1, n.len - 2): linkStr.add(n.sons[i].addNodes) + if linkStr != "": + let protocol = safeProtocol(linkStr) + result = linkStr != "" + if not result: + rstMessage(p, mwBrokenLink, protocol, + p.tok[p.idx-3].line, p.tok[p.idx-3].col) + b.add newLeaf(linkStr) + +proc whichRole(p: RstParser, sym: string): RstNodeKind = + result = whichRoleAux(sym) + if result == rnUnknownRole: + rstMessage(p, mwUnsupportedLanguage, sym) + +proc toInlineCode(n: PRstNode, language: string): PRstNode = + ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). + result = newRstNode(rnInlineCode, info=n.info) + let args = newRstNode(rnDirArg) + var lang = language + if language == "cpp": lang = "c++" + elif language == "csharp": lang = "c#" + args.add newLeaf(lang) + result.add args + result.add PRstNode(nil) + var lb = newRstNode(rnLiteralBlock) + var s: string + for i in n.sons: + assert i.kind == rnLeaf + s.add i.text + lb.add newLeaf(s) + result.add lb + +proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode = + let newN = newRstNode(rnInner, n.sons) + let newSons = @[newN, newLeaf(roleName)] + result = newRstNode(kind, newSons) proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = - result = n - if isInlineMarkupEnd(p, "_") or isInlineMarkupEnd(p, "__"): - inc(p.idx) + ## Finalizes node `n` that was tentatively determined as interpreted text. + var newKind = n.kind + var newSons = n.sons + + proc finalizeInterpreted(node: PRstNode, newKind: RstNodeKind, + newSons: seq[PRstNode], roleName: string): + PRstNode {.nimcall.} = + # fixes interpreted text (`x` or `y`:role:) to proper internal AST format + if newKind in {rnUnknownRole, rnCodeFragment}: + result = node.toOtherRole(newKind, roleName) + elif newKind == rnInlineCode: + result = node.toInlineCode(language=roleName) + else: + result = newRstNode(newKind, newSons) + + if isInlineMarkupEnd(p, "_", exact=true) or + isInlineMarkupEnd(p, "__", exact=true): + inc p.idx if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">": var a = newRstNode(rnInner) var b = newRstNode(rnInner) - fixupEmbeddedRef(n, a, b) - if len(a) == 0: - result = newRstNode(rnStandaloneHyperlink) - add(result, b) - else: - result = newRstNode(rnHyperlink) - add(result, a) - add(result, b) - setRef(p, rstnodeToRefname(a), b) - elif n.kind == rnInterpretedText: - n.kind = rnRef - else: - result = newRstNode(rnRef) - add(result, n) + if fixupEmbeddedRef(p, n, a, b): + if a.len == 0: # e.g. `<a_named_relative_link>`_ + newKind = rnStandaloneHyperlink + newSons = @[b] + else: # e.g. `link title <http://site>`_ + newKind = rnHyperlink + newSons = @[a, b] + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) + else: # include as plain text, not a link + newKind = rnInner + newSons = n.sons + result = newRstNode(newKind, newSons) + else: # some link that will be resolved in `resolveSubs` + newKind = rnRstRef + result = newRstNode(newKind, sons=newSons, info=n.info) elif match(p, p.idx, ":w:"): # a role: - if p.tok[p.idx + 1].symbol == "idx": - n.kind = rnIdx - elif p.tok[p.idx + 1].symbol == "literal": - n.kind = rnInlineLiteral - elif p.tok[p.idx + 1].symbol == "strong": - n.kind = rnStrongEmphasis - elif p.tok[p.idx + 1].symbol == "emphasis": - n.kind = rnEmphasis - elif (p.tok[p.idx + 1].symbol == "sub") or - (p.tok[p.idx + 1].symbol == "subscript"): - n.kind = rnSub - elif (p.tok[p.idx + 1].symbol == "sup") or - (p.tok[p.idx + 1].symbol == "supscript"): - n.kind = rnSup - else: - result = newRstNode(rnGeneralRole) - n.kind = rnInner - add(result, n) - add(result, newRstNode(rnLeaf, p.tok[p.idx + 1].symbol)) - inc(p.idx, 3) + let (roleName, lastIdx) = getRefname(p, p.idx+1) + newKind = whichRole(p, roleName) + result = n.finalizeInterpreted(newKind, newSons, roleName) + p.idx = lastIdx + 2 + else: + result = n.finalizeInterpreted(p.s.currRoleKind, newSons, p.s.currRole) proc matchVerbatim(p: RstParser, start: int, expr: string): int = result = start @@ -623,7 +1366,7 @@ proc matchVerbatim(p: RstParser, start: int, expr: string): int = if j < expr.len: result = 0 proc parseSmiley(p: var RstParser): PRstNode = - if p.tok[p.idx].symbol[0] notin SmileyStartChars: return + if currentTok(p).symbol[0] notin SmileyStartChars: return for key, val in items(Smilies): let m = matchVerbatim(p, p.idx, key) if m > 0: @@ -632,278 +1375,683 @@ proc parseSmiley(p: var RstParser): PRstNode = result.text = val return -when false: - const - urlChars = {'A'..'Z', 'a'..'z', '0'..'9', ':', '#', '@', '%', '/', ';', - '$', '(', ')', '~', '_', '?', '+', '-', '=', '\\', '.', '&', - '\128'..'\255'} - proc isUrl(p: RstParser, i: int): bool = - result = (p.tok[i+1].symbol == ":") and (p.tok[i+2].symbol == "//") and - (p.tok[i+3].kind == tkWord) and - (p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"]) - -proc parseUrl(p: var RstParser, father: PRstNode) = - #if p.tok[p.idx].symbol[strStart] == '<': - if isUrl(p, p.idx): - var n = newRstNode(rnStandaloneHyperlink) - while true: - case p.tok[p.idx].kind - of tkWord, tkAdornment, tkOther: discard - of tkPunct: - if p.tok[p.idx+1].kind notin {tkWord, tkAdornment, tkOther, tkPunct}: + result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and + p.tok[i+3].kind == tkWord and + p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"] + +proc checkParen(token: Token, parensStack: var seq[char]): bool {.inline.} = + ## Returns `true` iff `token` is a closing parenthesis for some + ## previous opening parenthesis saved in `parensStack`. + ## This is according Markdown balanced parentheses rule + ## (https://spec.commonmark.org/0.29/#link-destination) + ## to allow links like + ## https://en.wikipedia.org/wiki/APL_(programming_language), + ## we use it for RST also. + result = false + if token.kind == tkPunct: + let c = token.symbol[0] + if c in {'(', '[', '{'}: # push + parensStack.add c + elif c in {')', ']', '}'}: # try pop + # a case like ([) inside a link is allowed and [ is also `pop`ed: + for i in countdown(parensStack.len - 1, 0): + if (parensStack[i] == '(' and c == ')' or + parensStack[i] == '[' and c == ']' or + parensStack[i] == '{' and c == '}'): + parensStack.setLen i + result = true break - else: break - add(n, newLeaf(p)) - inc(p.idx) - add(father, n) + +proc parseUrl(p: var RstParser): PRstNode = + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#standalone-hyperlinks + result = newRstNode(rnStandaloneHyperlink) + var lastIdx = p.idx + var closedParenIdx = p.idx - 1 # for balanced parens rule + var parensStack: seq[char] + while p.tok[lastIdx].kind in {tkWord, tkPunct, tkOther}: + let isClosing = checkParen(p.tok[lastIdx], parensStack) + if isClosing: + closedParenIdx = lastIdx + inc lastIdx + dec lastIdx + # standalone URL can not end with punctuation in RST + while lastIdx > closedParenIdx and p.tok[lastIdx].kind == tkPunct and + p.tok[lastIdx].symbol != "/": + dec lastIdx + var s = "" + for i in p.idx .. lastIdx: s.add p.tok[i].symbol + result.add s + p.idx = lastIdx + 1 + +proc parseWordOrRef(p: var RstParser, father: PRstNode) = + ## Parses a normal word or may be a reference or URL. + if nextTok(p).kind != tkPunct: # <- main path, a normal word + father.add newLeaf(p) + inc p.idx + elif isUrl(p, p.idx): # URL http://something + father.add parseUrl(p) else: - var n = newLeaf(p) - inc(p.idx) - if p.tok[p.idx].symbol == "_": n = parsePostfix(p, n) - add(father, n) + # check for reference (probably, long one like some.ref.with.dots_ ) + var saveIdx = p.idx + var reference: PRstNode = nil + inc p.idx + while currentTok(p).kind in {tkWord, tkPunct}: + if currentTok(p).kind == tkPunct: + if isInlineMarkupEnd(p, "_", exact=true): + reference = newRstNode(rnRstRef, info=lineInfo(p, saveIdx)) + break + if not validRefnamePunct(currentTok(p).symbol): + break + inc p.idx + if reference != nil: + for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol) + father.add reference + inc p.idx # skip final _ + else: # 1 normal word + father.add newLeaf(p.tok[saveIdx].symbol) + p.idx = saveIdx + 1 proc parseBackslash(p: var RstParser, father: PRstNode) = - assert(p.tok[p.idx].kind == tkPunct) - if p.tok[p.idx].symbol == "\\\\": - add(father, newRstNode(rnLeaf, "\\")) - inc(p.idx) - elif p.tok[p.idx].symbol == "\\": + assert(currentTok(p).kind == tkPunct) + if currentTok(p).symbol == "\\": # XXX: Unicode? - inc(p.idx) - if p.tok[p.idx].kind != tkWhite: add(father, newLeaf(p)) - if p.tok[p.idx].kind != tkEof: inc(p.idx) + inc p.idx + if currentTok(p).kind != tkWhite: father.add(newLeaf(p)) + if currentTok(p).kind != tkEof: inc p.idx else: - add(father, newLeaf(p)) - inc(p.idx) - -when false: - proc parseAdhoc(p: var RstParser, father: PRstNode, verbatim: bool) = - if not verbatim and isURL(p, p.idx): - var n = newRstNode(rnStandaloneHyperlink) - while true: - case p.tok[p.idx].kind - of tkWord, tkAdornment, tkOther: nil - of tkPunct: - if p.tok[p.idx+1].kind notin {tkWord, tkAdornment, tkOther, tkPunct}: - break - else: break - add(n, newLeaf(p)) - inc(p.idx) - add(father, n) - elif not verbatim and roSupportSmilies in p.sharedState.options: - let n = parseSmiley(p) - if s != nil: - add(father, n) - else: - var n = newLeaf(p) - inc(p.idx) - if p.tok[p.idx].symbol == "_": n = parsePostfix(p, n) - add(father, n) + father.add(newLeaf(p)) + inc p.idx proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, interpretBackslash: bool) = let - line = p.tok[p.idx].line - col = p.tok[p.idx].col + line = currentTok(p).line + col = currentTok(p).col inc p.idx while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkPunct: - if isInlineMarkupEnd(p, postfix): - inc(p.idx) + if isInlineMarkupEnd(p, postfix, exact=false): + let l = currentTok(p).symbol.len + if l > postfix.len: + # handle cases like *emphasis with stars****. (It's valid RST!) + father.add newLeaf(currentTok(p).symbol[0 ..< l - postfix.len]) + elif postfix == "``" and currentTok(p).symbol == "`" and + prevTok(p).symbol == "`": + # handle cases like ``literal\`` - delete ` already added after \ + father.sons.setLen(father.sons.len - 1) + inc p.idx break - elif interpretBackslash: - parseBackslash(p, father) else: - add(father, newLeaf(p)) - inc(p.idx) + if postfix == "`": + if currentTok(p).symbol == "\\": + if nextTok(p).symbol == "\\": + father.add newLeaf("\\") + father.add newLeaf("\\") + inc p.idx, 2 + elif nextTok(p).symbol == "`": # escape ` + father.add newLeaf("`") + inc p.idx, 2 + else: + father.add newLeaf("\\") + inc p.idx + else: + father.add(newLeaf(p)) + inc p.idx + else: + if interpretBackslash: + parseBackslash(p, father) + else: + father.add(newLeaf(p)) + inc p.idx of tkAdornment, tkWord, tkOther: - add(father, newLeaf(p)) - inc(p.idx) + father.add(newLeaf(p)) + inc p.idx of tkIndent: - add(father, newRstNode(rnLeaf, " ")) - inc(p.idx) - if p.tok[p.idx].kind == tkIndent: + father.add newLeaf(" ") + inc p.idx + if currentTok(p).kind == tkIndent: rstMessage(p, meExpected, postfix, line, col) break of tkWhite: - add(father, newRstNode(rnLeaf, " ")) - inc(p.idx) + father.add newLeaf(" ") + inc p.idx else: rstMessage(p, meExpected, postfix, line, col) +proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode = + ## Parses additional (after language string) code block parameters + ## in a format *suggested* in the `CommonMark Spec`_ with handling of `"`. + if currentTok(p).kind == tkIndent: + result = nil + else: + result = newRstNode(rnFieldList) + while currentTok(p).kind notin {tkIndent, tkEof}: + if currentTok(p).kind == tkWhite: + inc p.idx + else: + let field = newRstNode(rnField) + var fieldName = "" + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof} and + currentTok(p).symbol != "=": + fieldName.add currentTok(p).symbol + inc p.idx + field.add(newRstNode(rnFieldName, @[newLeaf(fieldName)])) + if currentTok(p).kind == tkWhite: inc p.idx + let fieldBody = newRstNode(rnFieldBody) + if currentTok(p).symbol == "=": + inc p.idx + if currentTok(p).kind == tkWhite: inc p.idx + var fieldValue = "" + if currentTok(p).symbol == "\"": + while true: + fieldValue.add currentTok(p).symbol + inc p.idx + if currentTok(p).kind == tkEof: + rstMessage(p, meExpected, "\"") + elif currentTok(p).symbol == "\"": + fieldValue.add "\"" + inc p.idx + break + else: + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof}: + fieldValue.add currentTok(p).symbol + inc p.idx + fieldBody.add newLeaf(fieldValue) + field.add(fieldBody) + result.add(field) + +proc mayLoadFile(p: RstParser, result: var PRstNode) = + var filename = strip(getFieldValue(result, "file"), + chars = Whitespace + {'"'}) + if filename != "": + if roSandboxDisabled notin p.s.options: + let tok = p.tok[p.idx-2] + rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col) + var path = p.findRelativeFile(filename) + if path == "": rstMessage(p, meCannotOpenFile, filename) + var n = newRstNode(rnLiteralBlock) + n.add newLeaf(readFile(path)) + result.sons[2] = n + +proc defaultCodeLangNim(p: RstParser, result: var PRstNode) = + # Create a field block if the input block didn't have any. + if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) + assert result.sons[1].kind == rnFieldList + # Hook the extra field and specify the Nim language as value. + var extraNode = newRstNode(rnField, info=lineInfo(p)) + extraNode.add(newRstNode(rnFieldName)) + extraNode.add(newRstNode(rnFieldBody)) + extraNode.sons[0].add newLeaf("default-language") + extraNode.sons[1].add newLeaf("Nim") + result.sons[1].add(extraNode) + proc parseMarkdownCodeblock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnCodeBlock) + result.sons.setLen(3) + let line = curLine(p) + let baseCol = currentTok(p).col + let baseSym = currentTok(p).symbol # usually just ``` + inc p.idx + result.info = lineInfo(p) var args = newRstNode(rnDirArg) - if p.tok[p.idx].kind == tkWord: - add(args, newLeaf(p)) - inc(p.idx) + if currentTok(p).kind == tkWord: + args.add(newLeaf(p)) + inc p.idx + result.sons[1] = parseMarkdownCodeblockFields(p) + mayLoadFile(p, result) else: args = nil - var n = newRstNode(rnLeaf, "") + var n = newLeaf("") + var isFirstLine = true while true: - case p.tok[p.idx].kind - of tkEof: - rstMessage(p, meExpected, "```") + if currentTok(p).kind == tkEof: + rstMessage(p, meMissingClosing, + "$1 (started at line $2)" % [baseSym, $line]) break - of tkPunct: - if p.tok[p.idx].symbol == "```": - inc(p.idx) + elif nextTok(p).kind in {tkPunct, tkAdornment} and + nextTok(p).symbol[0] == baseSym[0] and + nextTok(p).symbol.len >= baseSym.len: + inc p.idx, 2 + break + elif currentTok(p).kind == tkIndent: + if not isFirstLine: + n.text.add "\n" + if currentTok(p).ival > baseCol: + n.text.add " ".repeat(currentTok(p).ival - baseCol) + elif currentTok(p).ival < baseCol: + rstMessage(p, mwRstStyle, + "unexpected de-indentation in Markdown code block") + inc p.idx + else: + n.text.add(currentTok(p).symbol) + inc p.idx + isFirstLine = false + result.sons[0] = args + if result.sons[2] == nil: + var lb = newRstNode(rnLiteralBlock) + lb.add(n) + result.sons[2] = lb + if result.sons[0].isNil and roNimFile in p.s.options: + defaultCodeLangNim(p, result) + +proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = + # Parses Markdown link. If it's Pandoc auto-link then its second + # son (target) will be in tokenized format (rnInner with leafs). + var desc = newRstNode(rnInner) + var i = p.idx + + var parensStack: seq[char] + template parse(endToken, dest) = + parensStack.setLen 0 + inc i # skip begin token + while true: + if p.tok[i].kind == tkEof: return false + if p.tok[i].kind == tkIndent and p.tok[i+1].kind == tkIndent: + return false + let isClosing = checkParen(p.tok[i], parensStack) + if p.tok[i].symbol == endToken and not isClosing: break + let symbol = if p.tok[i].kind == tkIndent: " " else: p.tok[i].symbol + when dest is string: dest.add symbol + else: dest.add newLeaf(symbol) + inc i + inc i # skip end token + + parse("]", desc) + if p.tok[i].symbol == "(": + var link = "" + let linkIdx = i + 1 + parse(")", link) + # only commit if we detected no syntax error: + let protocol = safeProtocol(link) + if link == "": + result = false + rstMessage(p, mwBrokenLink, protocol, + p.tok[linkIdx].line, p.tok[linkIdx].col) + else: + let child = newRstNode(rnHyperlink) + child.add newLeaf(desc.addNodes) + child.add link + father.add child + p.idx = i + result = true + elif roPreferMarkdown in p.s.options: + # Use Pandoc's implicit_header_references extension + var n = newRstNode(rnPandocRef) + if p.tok[i].symbol == "[": + var link = newRstNode(rnInner) + let targetIdx = i + 1 + parse("]", link) + n.add desc + if link.len != 0: # [description][target] + n.add link + n.info = lineInfo(p, targetIdx) + else: # [description=target][] + n.add desc + n.info = lineInfo(p, p.idx + 1) + else: # [description=target] + n.add desc + n.add desc # target is the same as description + n.info = lineInfo(p, p.idx + 1) + father.add n + p.idx = i + result = true + else: + result = false + +proc getRstFootnoteType(label: PRstNode): (FootnoteType, int) = + if label.sons.len >= 1 and label.sons[0].kind == rnLeaf and + label.sons[0].text == "#": + if label.sons.len == 1: + result = (fnAutoNumber, -1) + else: + result = (fnAutoNumberLabel, -1) + elif label.len == 1 and label.sons[0].kind == rnLeaf and + label.sons[0].text == "*": + result = (fnAutoSymbol, -1) + elif label.len == 1 and label.sons[0].kind == rnLeaf: + try: + result = (fnManualNumber, parseInt(label.sons[0].text)) + except ValueError: + result = (fnCitation, -1) + else: + result = (fnCitation, -1) + +proc getMdFootnoteType(label: PRstNode): (FootnoteType, int) = + try: + result = (fnManualNumber, parseInt(label.sons[0].text)) + except ValueError: + result = (fnAutoNumberLabel, -1) + +proc getFootnoteType(s: PRstSharedState, label: PRstNode): (FootnoteType, int) = + ## Returns footnote/citation type and manual number (if present). + if isMd(s): getMdFootnoteType(label) + else: getRstFootnoteType(label) + +proc parseRstFootnoteName(p: var RstParser, reference: bool): PRstNode = + ## parse footnote/citation label. Precondition: start at `[`. + ## Label text should be valid ref. name symbol, otherwise nil is returned. + var i = p.idx + 1 + result = newRstNode(rnInner) + while true: + if p.tok[i].kind in {tkEof, tkIndent, tkWhite}: + return nil + if p.tok[i].kind == tkPunct: + case p.tok[i].symbol: + of "]": + if i > p.idx + 1 and (not reference or (p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol == "_")): + inc i # skip ] + if reference: inc i # skip _ + break # to succeed, it's a footnote/citation indeed + else: + return nil + of "#": + if i != p.idx + 1: + return nil + of "*": + if i != p.idx + 1 and p.tok[i].kind != tkPunct and p.tok[i+1].symbol != "]": + return nil else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) + if not validRefnamePunct(p.tok[i].symbol): + return nil + result.add newLeaf(p.tok[i].symbol) + inc i + p.idx = i + +proc isMdFootnoteName(p: RstParser, reference: bool): bool = + ## Pandoc Markdown footnote extension. + let j = p.idx + result = p.tok[j].symbol == "[" and p.tok[j+1].symbol == "^" and + p.tok[j+2].kind == tkWord + +proc parseMdFootnoteName(p: var RstParser, reference: bool): PRstNode = + if isMdFootnoteName(p, reference): + result = newRstNode(rnInner) + var j = p.idx + 2 + while p.tok[j].kind in {tkWord, tkOther} or + validRefnamePunct(p.tok[j].symbol): + result.add newLeaf(p.tok[j].symbol) + inc j + if j == p.idx + 2: + return nil + if p.tok[j].symbol == "]": + if reference: + p.idx = j + 1 # skip ] + else: + if p.tok[j+1].symbol == ":": + p.idx = j + 2 # skip ]: + else: + result = nil else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) - var lb = newRstNode(rnLiteralBlock) - add(lb, n) - result = newRstNode(rnCodeBlock) - add(result, args) - add(result, nil) - add(result, lb) + result = nil + else: + result = nil + +proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode = + if isMd(p): parseMdFootnoteName(p, reference) + else: + if isInlineMarkupStart(p, "["): parseRstFootnoteName(p, reference) + else: nil + +proc isMarkdownCodeBlock(p: RstParser, idx: int): bool = + let tok = p.tok[idx] + template allowedSymbol: bool = + (tok.symbol[0] == '`' or + roPreferMarkdown in p.s.options and tok.symbol[0] == '~') + result = (roSupportMarkdown in p.s.options and + tok.kind in {tkPunct, tkAdornment} and + allowedSymbol and + tok.symbol.len >= 3) + +proc isMarkdownCodeBlock(p: RstParser): bool = + isMarkdownCodeBlock(p, p.idx) proc parseInline(p: var RstParser, father: PRstNode) = - case p.tok[p.idx].kind + var n: PRstNode # to be used in `if` condition + let saveIdx = p.idx + case currentTok(p).kind of tkPunct: if isInlineMarkupStart(p, "***"): var n = newRstNode(rnTripleEmphasis) parseUntil(p, n, "***", true) - add(father, n) + father.add(n) elif isInlineMarkupStart(p, "**"): var n = newRstNode(rnStrongEmphasis) parseUntil(p, n, "**", true) - add(father, n) + father.add(n) elif isInlineMarkupStart(p, "*"): var n = newRstNode(rnEmphasis) parseUntil(p, n, "*", true) - add(father, n) - elif roSupportMarkdown in p.s.options and p.tok[p.idx].symbol == "```": - inc(p.idx) - add(father, parseMarkdownCodeblock(p)) + father.add(n) + elif isInlineMarkupStart(p, "_`"): + var n = newRstNode(rnInlineTarget) + inc p.idx + parseUntil(p, n, "`", false) + n.anchor = rstnodeToRefname(n) + addAnchorRst(p, name = linkName(n), target = n, + anchorType=manualInlineAnchor) + father.add(n) + elif isMarkdownCodeBlock(p): + father.add(parseMarkdownCodeblock(p)) elif isInlineMarkupStart(p, "``"): var n = newRstNode(rnInlineLiteral) parseUntil(p, n, "``", false) - add(father, n) + father.add(n) + elif match(p, p.idx, ":w:") and + (var lastIdx = getRefnameIdx(p, p.idx + 1); + p.tok[lastIdx+2].symbol == "`"): + let (roleName, _) = getRefname(p, p.idx+1) + let k = whichRole(p, roleName) + var n = newRstNode(k) + p.idx = lastIdx + 2 + if k == rnInlineCode: + n = n.toInlineCode(language=roleName) + parseUntil(p, n, "`", false) # bug #17260 + if k in {rnUnknownRole, rnCodeFragment}: + n = n.toOtherRole(k, roleName) + father.add(n) elif isInlineMarkupStart(p, "`"): - var n = newRstNode(rnInterpretedText) - parseUntil(p, n, "`", true) + var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1)) + parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) - add(father, n) + father.add(n) elif isInlineMarkupStart(p, "|"): - var n = newRstNode(rnSubstitutionReferences) + var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1)) parseUntil(p, n, "|", false) - add(father, n) + father.add(n) + elif currentTok(p).symbol == "[" and nextTok(p).symbol != "[" and + (n = parseFootnoteName(p, reference=true); n != nil): + var nn = newRstNode(rnFootnoteRef) + nn.info = lineInfo(p, saveIdx+1) + nn.add n + let (fnType, _) = getFootnoteType(p.s, n) + case fnType + of fnAutoSymbol: + p.s.lineFootnoteSymRef.add lineInfo(p) + of fnAutoNumber: + p.s.lineFootnoteNumRef.add lineInfo(p) + else: discard + father.add(nn) + elif roSupportMarkdown in p.s.options and + currentTok(p).symbol == "[" and nextTok(p).symbol != "[" and + parseMarkdownLink(p, father): + discard "parseMarkdownLink already processed it" else: if roSupportSmilies in p.s.options: let n = parseSmiley(p) if n != nil: - add(father, n) + father.add(n) return parseBackslash(p, father) of tkWord: if roSupportSmilies in p.s.options: let n = parseSmiley(p) if n != nil: - add(father, n) + father.add(n) return - parseUrl(p, father) + parseWordOrRef(p, father) of tkAdornment, tkOther, tkWhite: + if isMarkdownCodeBlock(p): + father.add(parseMarkdownCodeblock(p)) + return if roSupportSmilies in p.s.options: let n = parseSmiley(p) if n != nil: - add(father, n) + father.add(n) return - add(father, newLeaf(p)) - inc(p.idx) + father.add(newLeaf(p)) + inc p.idx else: discard proc getDirective(p: var RstParser): string = - if p.tok[p.idx].kind == tkWhite and p.tok[p.idx+1].kind == tkWord: - var j = p.idx - inc(p.idx) - result = p.tok[p.idx].symbol - inc(p.idx) - while p.tok[p.idx].kind in {tkWord, tkPunct, tkAdornment, tkOther}: - if p.tok[p.idx].symbol == "::": break - add(result, p.tok[p.idx].symbol) - inc(p.idx) - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - if p.tok[p.idx].symbol == "::": - inc(p.idx) - if (p.tok[p.idx].kind == tkWhite): inc(p.idx) - else: - p.idx = j # set back - result = "" # error - else: - result = "" - -proc parseComment(p: var RstParser): PRstNode = - case p.tok[p.idx].kind - of tkIndent, tkEof: - if p.tok[p.idx].kind != tkEof and p.tok[p.idx + 1].kind == tkIndent: - inc(p.idx) # empty comment - else: - var indent = p.tok[p.idx].ival - while true: - case p.tok[p.idx].kind - of tkEof: - break - of tkIndent: - if (p.tok[p.idx].ival < indent): break - else: - discard - inc(p.idx) + result = "" + if currentTok(p).kind == tkWhite: + let (name, lastIdx) = getRefname(p, p.idx + 1) + let afterIdx = lastIdx + 1 + if name.len > 0: + if p.tok[afterIdx].symbol == "::": + result = name + p.idx = afterIdx + 1 + if currentTok(p).kind == tkWhite: + inc p.idx + elif currentTok(p).kind != tkIndent: + rstMessage(p, mwRstStyle, + "whitespace or newline expected after directive " & name) + result = result.toLowerAscii() + elif p.tok[afterIdx].symbol == ":": + rstMessage(p, mwRstStyle, + "double colon :: may be missing at end of '" & name & "'", + p.tok[afterIdx].line, p.tok[afterIdx].col) + elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':': + rstMessage(p, mwRstStyle, + "too many colons for a directive (should be ::)", + p.tok[afterIdx].line, p.tok[afterIdx].col) + +proc parseComment(p: var RstParser, col: int): PRstNode = + if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: + inc p.idx # empty comment else: - while p.tok[p.idx].kind notin {tkIndent, tkEof}: inc(p.idx) + while currentTok(p).kind != tkEof: + if currentTok(p).kind == tkIndent and currentTok(p).ival > col or + currentTok(p).kind != tkIndent and currentTok(p).col > col: + inc p.idx + else: + break result = nil -type - DirKind = enum # must be ordered alphabetically! - dkNone, dkAuthor, dkAuthors, dkCode, dkCodeBlock, dkContainer, dkContents, - dkFigure, dkImage, dkInclude, dkIndex, dkRaw, dkTitle -{.deprecated: [TDirKind: DirKind].} - -const - DirIds: array[0..12, string] = ["", "author", "authors", "code", - "code-block", "container", "contents", "figure", "image", "include", - "index", "raw", "title"] - -proc getDirKind(s: string): DirKind = - let i = find(DirIds, s) - if i >= 0: result = DirKind(i) - else: result = dkNone - proc parseLine(p: var RstParser, father: PRstNode) = while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWhite, tkWord, tkOther, tkPunct: parseInline(p, father) else: break proc parseUntilNewline(p: var RstParser, father: PRstNode) = while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWhite, tkWord, tkAdornment, tkOther, tkPunct: parseInline(p, father) of tkEof, tkIndent: break proc parseSection(p: var RstParser, result: PRstNode) {.gcsafe.} + +proc tokenAfterNewline(p: RstParser, start: int): int = + result = start + while true: + case p.tok[result].kind + of tkEof: + break + of tkIndent: + inc result + break + else: inc result + +proc tokenAfterNewline(p: RstParser): int {.inline.} = + result = tokenAfterNewline(p, p.idx) + +proc getWrappableIndent(p: RstParser): int = + ## Gets baseline indentation for bodies of field lists and directives. + ## Handles situations like this (with possible de-indent in [case.3]):: + ## + ## :field: definition [case.1] + ## + ## currInd currentTok(p).col + ## | | + ## v v + ## + ## .. Note:: defItem: [case.2] + ## definition + ## + ## ^ + ## | + ## nextIndent + ## + ## .. Note:: - point1 [case.3] + ## - point 2 + ## + ## ^ + ## | + ## nextIndent + if currentTok(p).kind == tkIndent: + result = currentTok(p).ival + else: + var nextIndent = p.tok[tokenAfterNewline(p)-1].ival + if nextIndent <= currInd(p): # parse only this line [case.1] + result = currentTok(p).col + elif nextIndent >= currentTok(p).col: # may be a definition list [case.2] + result = currentTok(p).col + else: + result = nextIndent # allow parsing next lines [case.3] + +proc getMdBlockIndent(p: RstParser): int = + ## Markdown version of `getWrappableIndent`. + if currentTok(p).kind == tkIndent: + result = currentTok(p).ival + else: + var nextIndent = p.tok[tokenAfterNewline(p)-1].ival + # TODO: Markdown-compliant definition should allow nextIndent == currInd(p): + if nextIndent <= currInd(p): # parse only this line + result = currentTok(p).col + else: + result = nextIndent # allow parsing next lines [case.3] + +proc indFollows(p: RstParser): bool = + result = currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) + +proc parseBlockContent(p: var RstParser, father: var PRstNode, + contentParser: SectionParser): bool {.gcsafe.} = + ## parse the final content part of explicit markup blocks (directives, + ## footnotes, etc). Returns true if succeeded. + if currentTok(p).kind != tkIndent or indFollows(p): + let blockIndent = getWrappableIndent(p) + pushInd(p, blockIndent) + let content = contentParser(p) + popInd(p) + father.add content + result = true + +proc parseSectionWrapper(p: var RstParser): PRstNode = + result = newRstNode(rnInner) + parseSection(p, result) + while result.kind == rnInner and result.len == 1: + result = result.sons[0] + proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. - result = newRstNode(rnField) - var col = p.tok[p.idx].col + result = newRstNode(rnField, info=lineInfo(p)) + var col = currentTok(p).col var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) var fieldbody = newRstNode(rnFieldBody) - if p.tok[p.idx].kind != tkIndent: parseLine(p, fieldbody) - if p.tok[p.idx].kind == tkIndent: - var indent = p.tok[p.idx].ival - if indent > col: - pushInd(p, indent) - parseSection(p, fieldbody) - popInd(p) - add(result, fieldname) - add(result, fieldbody) + if currentTok(p).kind == tkWhite: inc p.idx + let indent = getWrappableIndent(p) + if indent > col: + pushInd(p, indent) + parseSection(p, fieldbody) + popInd(p) + result.add(fieldname) + result.add(fieldbody) proc parseFields(p: var RstParser): PRstNode = ## Parses fields for a section or directive block. @@ -912,16 +2060,16 @@ proc parseFields(p: var RstParser): PRstNode = ## otherwise it will return a node of rnFieldList type with children. result = nil var atStart = p.idx == 0 and p.tok[0].symbol == ":" - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx + 1].symbol == ":") or + if currentTok(p).kind == tkIndent and nextTok(p).symbol == ":" or atStart: - var col = if atStart: p.tok[p.idx].col else: p.tok[p.idx].ival - result = newRstNode(rnFieldList) - if not atStart: inc(p.idx) + var col = if atStart: currentTok(p).col else: currentTok(p).ival + result = newRstNodeA(p, rnFieldList) + if not atStart: inc p.idx while true: - add(result, parseField(p)) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == ":"): - inc(p.idx) + result.add(parseField(p)) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).symbol == ":": + inc p.idx else: break @@ -938,13 +2086,12 @@ proc getFieldValue*(n: PRstNode): string = result = addNodes(n.sons[1]).strip proc getFieldValue(n: PRstNode, fieldname: string): string = - result = "" if n.sons[1] == nil: return - if (n.sons[1].kind != rnFieldList): + if n.sons[1].kind != rnFieldList: #InternalError("getFieldValue (2): " & $n.sons[1].kind) # We don't like internal errors here anymore as that would break the forum! return - for i in countup(0, len(n.sons[1]) - 1): + for i in 0 ..< n.sons[1].len: var f = n.sons[1].sons[i] if cmpIgnoreStyle(addNodes(f.sons[0]), fieldname) == 0: result = addNodes(f.sons[1]) @@ -957,153 +2104,462 @@ proc getArgument(n: PRstNode): string = proc parseDotDot(p: var RstParser): PRstNode {.gcsafe.} proc parseLiteralBlock(p: var RstParser): PRstNode = - result = newRstNode(rnLiteralBlock) - var n = newRstNode(rnLeaf, "") - if p.tok[p.idx].kind == tkIndent: - var indent = p.tok[p.idx].ival - inc(p.idx) + result = newRstNodeA(p, rnLiteralBlock) + var n = newLeaf("") + if currentTok(p).kind == tkIndent: + var indent = currentTok(p).ival + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkEof: break of tkIndent: - if (p.tok[p.idx].ival < indent): + if currentTok(p).ival < indent: break else: - add(n.text, "\n") - add(n.text, spaces(p.tok[p.idx].ival - indent)) - inc(p.idx) + n.text.add("\n") + n.text.add(spaces(currentTok(p).ival - indent)) + inc p.idx else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) + n.text.add(currentTok(p).symbol) + inc p.idx else: - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) - add(result, n) - -proc getLevel(map: var LevelMap, lvl: var int, c: char): int = - if map[c] == 0: - inc(lvl) - map[c] = lvl - result = map[c] - -proc tokenAfterNewline(p: RstParser): int = - result = p.idx - while true: - case p.tok[result].kind - of tkEof: - break - of tkIndent: - inc(result) - break - else: inc(result) + while currentTok(p).kind notin {tkIndent, tkEof}: + n.text.add(currentTok(p).symbol) + inc p.idx + result.add(n) + +proc parseQuotedLiteralBlock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnLiteralBlock) + var n = newLeaf("") + if currentTok(p).kind == tkIndent: + var indent = currInd(p) + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines + var quoteSym = currentTok(p).symbol[0] + while true: + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if currentTok(p).ival < indent: + break + elif currentTok(p).ival == indent: + if nextTok(p).kind == tkPunct and nextTok(p).symbol[0] == quoteSym: + n.text.add("\n") + inc p.idx + elif nextTok(p).kind == tkIndent: + break + else: + rstMessage(p, mwRstStyle, "no newline after quoted literal block") + break + else: + rstMessage(p, mwRstStyle, + "unexpected indentation in quoted literal block") + break + else: + n.text.add(currentTok(p).symbol) + inc p.idx + result.add(n) + +proc parseRstLiteralBlock(p: var RstParser, kind: LiteralBlockKind): PRstNode = + if kind == lbIndentedLiteralBlock: + result = parseLiteralBlock(p) + else: + result = parseQuotedLiteralBlock(p) + +proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = + ## Returns (preliminary) heading level corresponding to `c` and + ## `hasOverline`. If level does not exist, add it first. + for i, hType in p.s.hLevels: + if hType.symbol == c and hType.hasOverline == hasOverline: + p.s.hLevels[i].line = curLine(p) + p.s.hLevels[i].hasPeers = true + return i + p.s.hLevels.add LevelInfo(symbol: c, hasOverline: hasOverline, + line: curLine(p), hasPeers: false) + result = p.s.hLevels.len - 1 + +proc countTitles(s: PRstSharedState, n: PRstNode) = + ## Fill `s.hTitleCnt` + if n == nil: return + for node in n.sons: + if node != nil: + if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}: + break + if node.kind == rnOverline: + if s.hLevels[s.hTitleCnt].hasPeers: + break + inc s.hTitleCnt + if s.hTitleCnt >= 2: + break + +proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = + ## check that underline/overline length is enough for the heading. + ## No support for Unicode. + if p.tok[adornmentIdx].symbol in ["::", "..", "|"]: + return false + if isMarkdownCodeBlock(p, adornmentIdx): + return false + var headlineLen = 0 + var failure = "" + if p.idx < adornmentIdx: # check for underline + if p.idx > 0: + headlineLen = currentTok(p).col - p.tok[adornmentIdx].col + if headlineLen > 0: + rstMessage(p, mwRstStyle, "indentation of heading text allowed" & + " only for overline titles") + for i in p.idx ..< adornmentIdx-1: # adornmentIdx-1 is a linebreak + headlineLen += p.tok[i].symbol.len + result = p.tok[adornmentIdx].symbol.len >= headlineLen and headlineLen != 0 + if not result: + failure = "(underline '" & p.tok[adornmentIdx].symbol & "' is too short)" + else: # p.idx == adornmentIdx, at overline. Check overline and underline + var i = p.idx + 2 + headlineLen = p.tok[i].col - p.tok[adornmentIdx].col + while p.tok[i].kind notin {tkEof, tkIndent}: + headlineLen += p.tok[i].symbol.len + inc i + if p.tok[i].kind == tkIndent and + p.tok[i+1].kind == tkAdornment and + p.tok[i+1].symbol[0] == p.tok[adornmentIdx].symbol[0]: + result = p.tok[adornmentIdx].symbol.len >= headlineLen and + headlineLen != 0 + if result: + result = p.tok[i+1].symbol == p.tok[adornmentIdx].symbol + if not result: + failure = "(underline '" & p.tok[i+1].symbol & "' does not match " & + "overline '" & p.tok[adornmentIdx].symbol & "')" + else: + failure = "(overline '" & p.tok[adornmentIdx].symbol & "' is too short)" + else: # it's not overline/underline section, not reporting error + return false + if not result: + rstMessage(p, meNewSectionExpected, failure) proc isLineBlock(p: RstParser): bool = var j = tokenAfterNewline(p) - result = (p.tok[p.idx].col == p.tok[j].col) and (p.tok[j].symbol == "|") or - (p.tok[j].col > p.tok[p.idx].col) + result = currentTok(p).col == p.tok[j].col and p.tok[j].symbol == "|" or + p.tok[j].col > currentTok(p).col or + p.tok[j].symbol == "\n" + +proc isMarkdownBlockQuote(p: RstParser): bool = + result = currentTok(p).symbol[0] == '>' + +proc whichRstLiteralBlock(p: RstParser): LiteralBlockKind = + ## Checks that the following tokens are either Indented Literal Block or + ## Quoted Literal Block (which is not quite the same as Markdown quote block). + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks + if currentTok(p).symbol == "::" and nextTok(p).kind == tkIndent: + if currInd(p) > nextTok(p).ival: + result = lbNone + if currInd(p) < nextTok(p).ival: + result = lbIndentedLiteralBlock + elif currInd(p) == nextTok(p).ival: + var i = p.idx + 1 + while p.tok[i].kind == tkIndent: inc i + const validQuotingCharacters = { + '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', + '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', + '_', '`', '{', '|', '}', '~'} + if p.tok[i].kind in {tkPunct, tkAdornment} and + p.tok[i].symbol[0] in validQuotingCharacters: + result = lbQuotedLiteralBlock + else: + result = lbNone + else: + result = lbNone proc predNL(p: RstParser): bool = result = true if p.idx > 0: - result = p.tok[p.idx-1].kind == tkIndent and - p.tok[p.idx-1].ival == currInd(p) + result = prevTok(p).kind == tkIndent and + prevTok(p).ival == currInd(p) proc isDefList(p: RstParser): bool = var j = tokenAfterNewline(p) - result = (p.tok[p.idx].col < p.tok[j].col) and - (p.tok[j].kind in {tkWord, tkOther, tkPunct}) and - (p.tok[j - 2].symbol != "::") + result = currentTok(p).col < p.tok[j].col and + p.tok[j].kind in {tkWord, tkOther, tkPunct} and + p.tok[j - 2].symbol != "::" + +proc `$`(t: Token): string = # for debugging only + result = "(" & $t.kind & " line=" & $t.line & " col=" & $t.col + if t.kind == tkIndent: result = result & " ival=" & $t.ival & ")" + else: result = result & " symbol=" & t.symbol & ")" + +proc skipNewlines(p: RstParser, j: int): int = + result = j + while p.tok[result].kind != tkEof and p.tok[result].kind == tkIndent: + inc result # skip blank lines + +proc skipNewlines(p: var RstParser) = + p.idx = skipNewlines(p, p.idx) + +const maxMdRelInd = 3 ## In Markdown: maximum indentation that does not yet + ## make the indented block a code + +proc isMdRelInd(outerInd, nestedInd: int): bool = + result = outerInd <= nestedInd and nestedInd <= outerInd + maxMdRelInd + +proc isMdDefBody(p: RstParser, j: int, termCol: int): bool = + let defCol = p.tok[j].col + result = p.tok[j].symbol == ":" and + isMdRelInd(termCol, defCol) and + p.tok[j+1].kind == tkWhite and + p.tok[j+2].kind in {tkWord, tkOther, tkPunct} + +proc isMdDefListItem(p: RstParser, idx: int): bool = + var j = tokenAfterNewline(p, idx) + j = skipNewlines(p, j) + let termCol = p.tok[j].col + result = isMdRelInd(currInd(p), termCol) and + isMdDefBody(p, j, termCol) proc isOptionList(p: RstParser): bool = result = match(p, p.idx, "-w") or match(p, p.idx, "--w") or match(p, p.idx, "/w") or match(p, p.idx, "//w") +proc isMarkdownHeadlinePattern(s: string): bool = + if s.len >= 1 and s.len <= 6: + for c in s: + if c != '#': return false + result = true + +proc isMarkdownHeadline(p: RstParser): bool = + if roSupportMarkdown in p.s.options: + if isMarkdownHeadlinePattern(currentTok(p).symbol) and nextTok(p).kind == tkWhite: + if p.tok[p.idx+2].kind in {tkWord, tkOther, tkPunct}: + result = true + +proc findPipe(p: RstParser, start: int): bool = + var i = start + while true: + if p.tok[i].symbol == "|": return true + if p.tok[i].kind in {tkIndent, tkEof}: return false + inc i + proc whichSection(p: RstParser): RstNodeKind = - case p.tok[p.idx].kind + if currentTok(p).kind in {tkAdornment, tkPunct}: + # for punctuation sequences that can be both tkAdornment and tkPunct + if isMarkdownCodeBlock(p): + return rnCodeBlock + elif isRst(p) and currentTok(p).symbol == "::": + return rnLiteralBlock + elif currentTok(p).symbol == ".." and + nextTok(p).kind in {tkWhite, tkIndent}: + return rnDirective + case currentTok(p).kind of tkAdornment: - if match(p, p.idx + 1, "ii"): result = rnTransition + if match(p, p.idx + 1, "iI") and currentTok(p).symbol.len >= 4: + result = rnTransition + elif match(p, p.idx, "+a+"): + result = rnGridTable + rstMessage(p, meGridTableNotImplemented) elif match(p, p.idx + 1, " a"): result = rnTable - elif match(p, p.idx + 1, "i"): result = rnOverline - else: result = rnLeaf + elif currentTok(p).symbol == "|" and isLineBlock(p): + result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote + elif (match(p, p.idx + 1, "i") and not match(p, p.idx + 2, "I")) and + isAdornmentHeadline(p, p.idx): + result = rnOverline + else: + result = rnParagraph of tkPunct: - if match(p, tokenAfterNewline(p), "ai"): + if isMarkdownHeadline(p): + result = rnMarkdownHeadline + elif roSupportMarkdown in p.s.options and predNL(p) and + match(p, p.idx, "| w") and findPipe(p, p.idx+3): + result = rnMarkdownTable + elif isMd(p) and isMdFootnoteName(p, reference=false): + result = rnFootnote + elif currentTok(p).symbol == "|" and isLineBlock(p): + result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote + elif match(p, tokenAfterNewline(p), "aI") and + isAdornmentHeadline(p, tokenAfterNewline(p)): result = rnHeadline - elif p.tok[p.idx].symbol == "::": - result = rnLiteralBlock - elif predNL(p) and - ((p.tok[p.idx].symbol == "+") or (p.tok[p.idx].symbol == "*") or - (p.tok[p.idx].symbol == "-")) and (p.tok[p.idx + 1].kind == tkWhite): + elif currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite: result = rnBulletList - elif (p.tok[p.idx].symbol == "|") and isLineBlock(p): - result = rnLineBlock - elif (p.tok[p.idx].symbol == "..") and predNL(p): - result = rnDirective - elif match(p, p.idx, ":w:") and predNL(p): - # (p.tok[p.idx].symbol == ":") + elif match(p, p.idx, ":w:E"): + # (currentTok(p).symbol == ":") result = rnFieldList - elif match(p, p.idx, "(e) "): + elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or + match(p, p.idx, "e. "): result = rnEnumList - elif match(p, p.idx, "+a+"): - result = rnGridTable - rstMessage(p, meGridTableNotImplemented) - elif isDefList(p): - result = rnDefList elif isOptionList(p): result = rnOptionList + elif isRst(p) and isDefList(p): + result = rnDefList + elif isMd(p) and isMdDefListItem(p, p.idx): + result = rnMdDefList else: result = rnParagraph of tkWord, tkOther, tkWhite: - if match(p, tokenAfterNewline(p), "ai"): result = rnHeadline + let tokIdx = tokenAfterNewline(p) + if match(p, tokIdx, "aI"): + if isAdornmentHeadline(p, tokIdx): result = rnHeadline + else: result = rnParagraph elif match(p, p.idx, "e) ") or match(p, p.idx, "e. "): result = rnEnumList - elif isDefList(p): result = rnDefList + elif isRst(p) and isDefList(p): result = rnDefList + elif isMd(p) and isMdDefListItem(p, p.idx): + result = rnMdDefList else: result = rnParagraph else: result = rnLeaf proc parseLineBlock(p: var RstParser): PRstNode = + ## Returns rnLineBlock with all sons of type rnLineBlockItem result = nil - if p.tok[p.idx + 1].kind == tkWhite: - var col = p.tok[p.idx].col - result = newRstNode(rnLineBlock) - pushInd(p, p.tok[p.idx + 2].col) - inc(p.idx, 2) + if nextTok(p).kind in {tkWhite, tkIndent}: + var col = currentTok(p).col + result = newRstNodeA(p, rnLineBlock) while true: var item = newRstNode(rnLineBlockItem) - parseSection(p, item) - add(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == "|") and - (p.tok[p.idx + 2].kind == tkWhite): - inc(p.idx, 3) + if nextTok(p).kind == tkWhite: + if nextTok(p).symbol.len > 1: # pass additional indentation after '| ' + item.lineIndent = nextTok(p).symbol + inc p.idx, 2 + pushInd(p, p.tok[p.idx].col) + parseSection(p, item) + popInd(p) + else: # tkIndent => add an empty line + item.lineIndent = "\n" + inc p.idx, 1 + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).symbol == "|" and + p.tok[p.idx + 2].kind in {tkWhite, tkIndent}: + inc p.idx, 1 else: break - popInd(p) + +proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} + +proc getQuoteSymbol(p: RstParser, idx: int): tuple[sym: string, depth: int, tokens: int] = + result = ("", 0, 0) + var i = idx + result.sym &= p.tok[i].symbol + result.depth += p.tok[i].symbol.len + inc result.tokens + inc i + while p.tok[i].kind == tkWhite and i+1 < p.tok.len and + p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol[0] == '>': + result.sym &= p.tok[i].symbol + result.sym &= p.tok[i+1].symbol + result.depth += p.tok[i+1].symbol.len + inc result.tokens, 2 + inc i, 2 + +proc parseMarkdownQuoteSegment(p: var RstParser, curSym: string, col: int): + PRstNode = + ## We define *segment* as a group of lines that starts with exactly the + ## same quote symbol. If the following lines don't contain any `>` (*lazy* + ## continuation) they considered as continuation of the current segment. + var q: RstParser # to delete `>` at a start of line and then parse normally + initParser(q, p.s) + q.col = p.col + q.line = p.line + var minCol = int.high # minimum colum num in the segment + while true: # move tokens of segment from `p` to `q` skipping `curSym` + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if nextTok(p).kind in {tkIndent, tkEof}: + break + else: + if nextTok(p).symbol[0] == '>': + var (quoteSym, _, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + if quoteSym == curSym: # the segment continues + var iTok = tokenAfterNewline(p, p.idx+1) + if p.tok[iTok].kind notin {tkEof, tkIndent} and + p.tok[iTok].symbol[0] != '>': + rstMessage(p, mwRstStyle, + "two or more quoted lines are followed by unquoted line " & + $(curLine(p) + 1)) + break + q.tok.add currentTok(p) + var ival = currentTok(p).ival + quoteSym.len + inc p.idx, (1 + quoteTokens) # skip newline and > > > + if currentTok(p).kind == tkWhite: + ival += currentTok(p).symbol.len + inc p.idx + # fix up previous `tkIndent`s to ival (as if >>> were not there) + var j = q.tok.len - 1 + while j >= 0 and q.tok[j].kind == tkIndent: + q.tok[j].ival = ival + dec j + else: # next segment started + break + elif currentTok(p).ival < col: + break + else: # the segment continues, a case like: + # > beginning + # continuation + q.tok.add currentTok(p) + inc p.idx + else: + if currentTok(p).col < minCol: minCol = currentTok(p).col + q.tok.add currentTok(p) + inc p.idx + q.indentStack = @[minCol] + # if initial indentation `minCol` is > 0 then final newlines + # should be omitted so that parseDoc could advance to the end of tokens: + var j = q.tok.len - 1 + while q.tok[j].kind == tkIndent: dec j + q.tok.setLen (j+1) + q.tok.add Token(kind: tkEof, line: currentTok(p).line) + result = parseDoc(q) + +proc parseMarkdownBlockQuote(p: var RstParser): PRstNode = + var (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx) + let col = currentTok(p).col + result = newRstNodeA(p, rnMarkdownBlockQuote) + inc p.idx, quoteTokens # skip first > + while true: + var item = newRstNode(rnMarkdownBlockQuoteItem) + item.quotationDepth = quotationDepth + if currentTok(p).kind == tkWhite: inc p.idx + item.add parseMarkdownQuoteSegment(p, curSym, col) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).kind != tkEof and nextTok(p).symbol[0] == '>': + (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + inc p.idx, (1 + quoteTokens) # skip newline and > > > + else: + break proc parseParagraph(p: var RstParser, result: PRstNode) = while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkIndent: - if p.tok[p.idx + 1].kind == tkIndent: - inc(p.idx) - break - elif (p.tok[p.idx].ival == currInd(p)): - inc(p.idx) + if nextTok(p).kind == tkIndent: + inc p.idx + break # blank line breaks paragraph for both Md & Rst + elif currentTok(p).ival == currInd(p) or ( + isMd(p) and currentTok(p).ival > currInd(p)): + # (Md allows adding additional indentation inside paragraphs) + inc p.idx case whichSection(p) - of rnParagraph, rnLeaf, rnHeadline, rnOverline, rnDirective: - add(result, newRstNode(rnLeaf, " ")) + of rnParagraph, rnLeaf, rnHeadline, rnMarkdownHeadline, + rnOverline, rnDirective: + result.add newLeaf(" ") of rnLineBlock: - addIfNotNil(result, parseLineBlock(p)) - else: break + result.addIfNotNil(parseLineBlock(p)) + of rnMarkdownBlockQuote: + result.addIfNotNil(parseMarkdownBlockQuote(p)) + else: + dec p.idx # allow subsequent block to be parsed as another section + break else: break of tkPunct: - if (p.tok[p.idx].symbol == "::") and - (p.tok[p.idx + 1].kind == tkIndent) and - (currInd(p) < p.tok[p.idx + 1].ival): - add(result, newRstNode(rnLeaf, ":")) - inc(p.idx) # skip '::' - add(result, parseLiteralBlock(p)) + if isRst(p) and ( + let literalBlockKind = whichRstLiteralBlock(p); + literalBlockKind != lbNone): + result.add newLeaf(":") + inc p.idx # skip '::' + result.add(parseRstLiteralBlock(p, literalBlockKind)) break else: parseInline(p, result) @@ -1111,385 +2567,743 @@ proc parseParagraph(p: var RstParser, result: PRstNode) = parseInline(p, result) else: break +proc checkHeadingHierarchy(p: RstParser, lvl: int) = + if lvl - p.s.hCurLevel > 1: # broken hierarchy! + proc descr(l: int): string = + (if p.s.hLevels[l].hasOverline: "overline " else: "underline ") & + repeat(p.s.hLevels[l].symbol, 5) + var msg = "(section level inconsistent: " + msg.add descr(lvl) & " unexpectedly found, " & + "while the following intermediate section level(s) are missing on lines " + msg.add $p.s.hLevels[p.s.hCurLevel].line & ".." & $curLine(p) & ":" + for l in p.s.hCurLevel+1 .. lvl-1: + msg.add " " & descr(l) + if l != lvl-1: msg.add "," + rstMessage(p, meNewSectionExpected, msg & ")") + proc parseHeadline(p: var RstParser): PRstNode = - result = newRstNode(rnHeadline) - parseUntilNewline(p, result) - assert(p.tok[p.idx].kind == tkIndent) - assert(p.tok[p.idx + 1].kind == tkAdornment) - var c = p.tok[p.idx + 1].symbol[0] - inc(p.idx, 2) - result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c) + if isMarkdownHeadline(p): + result = newRstNode(rnMarkdownHeadline) + # Note that level hierarchy is not checked for markdown headings + result.level = currentTok(p).symbol.len + assert(nextTok(p).kind == tkWhite) + inc p.idx, 2 + parseUntilNewline(p, result) + else: + result = newRstNode(rnHeadline) + parseUntilNewline(p, result) + assert(currentTok(p).kind == tkIndent) + assert(nextTok(p).kind == tkAdornment) + var c = nextTok(p).symbol[0] + inc p.idx, 2 + result.level = getLevel(p, c, hasOverline=false) + checkHeadingHierarchy(p, result.level) + p.s.hCurLevel = result.level + addAnchorRst(p, linkName(result), result, anchorType=headlineAnchor) + p.s.tocPart.add result + +proc parseOverline(p: var RstParser): PRstNode = + var c = currentTok(p).symbol[0] + inc p.idx, 2 + result = newRstNode(rnOverline) + while true: + parseUntilNewline(p, result) + if currentTok(p).kind == tkIndent: + inc p.idx + if prevTok(p).ival > currInd(p): + result.add newLeaf(" ") + else: + break + else: + break + result.level = getLevel(p, c, hasOverline=true) + checkHeadingHierarchy(p, result.level) + p.s.hCurLevel = result.level + if currentTok(p).kind == tkAdornment: + inc p.idx + if currentTok(p).kind == tkIndent: inc p.idx + addAnchorRst(p, linkName(result), result, anchorType=headlineAnchor) + p.s.tocPart.add result + +proc fixHeadlines(s: PRstSharedState) = + # Fix up section levels depending on presence of a title and subtitle: + for n in s.tocPart: + if n.kind in {rnHeadline, rnOverline}: + if s.hTitleCnt == 2: + if n.level == 1: # it's the subtitle + n.level = 0 + elif n.level >= 2: # normal sections, start numbering from 1 + n.level -= 1 + elif s.hTitleCnt == 0: + n.level += 1 + # Set headline anchors: + for iHeading in 0 .. s.tocPart.high: + let n: PRstNode = s.tocPart[iHeading] + if n.level >= 1: + n.anchor = rstnodeToRefname(n) + # Fix anchors for uniqueness if `.. contents::` is present + if s.hasToc: + # Find the last higher level section for unique reference name + var sectionPrefix = "" + for i in countdown(iHeading - 1, 0): + if s.tocPart[i].level >= 1 and s.tocPart[i].level < n.level: + sectionPrefix = rstnodeToRefname(s.tocPart[i]) & "-" + break + if sectionPrefix != "": + n.anchor = sectionPrefix & n.anchor + s.tocPart.setLen 0 type - IntSeq = seq[int] -{.deprecated: [TIntSeq: IntSeq].} + ColSpec = object + start, stop: int + RstCols = seq[ColSpec] + ColumnLimits = tuple # for Markdown + first, last: int + ColSeq = seq[ColumnLimits] + +proc tokStart(p: RstParser, idx: int): int = + result = p.tok[idx].col + +proc tokStart(p: RstParser): int = + result = tokStart(p, p.idx) + +proc tokEnd(p: RstParser, idx: int): int = + result = p.tok[idx].col + p.tok[idx].symbol.len - 1 proc tokEnd(p: RstParser): int = - result = p.tok[p.idx].col + len(p.tok[p.idx].symbol) - 1 + result = tokEnd(p, p.idx) -proc getColumns(p: var RstParser, cols: var IntSeq) = +proc getColumns(p: RstParser, cols: var RstCols, startIdx: int): int = + # Fills table column specification (or separator) `cols` and returns + # the next parser index after it. var L = 0 + result = startIdx while true: - inc(L) + inc L setLen(cols, L) - cols[L - 1] = tokEnd(p) - assert(p.tok[p.idx].kind == tkAdornment) - inc(p.idx) - if p.tok[p.idx].kind != tkWhite: break - inc(p.idx) - if p.tok[p.idx].kind != tkAdornment: break - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - # last column has no limit: - cols[L - 1] = 32000 - -proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} + cols[L - 1].start = tokStart(p, result) + cols[L - 1].stop = tokEnd(p, result) + assert(p.tok[result].kind == tkAdornment) + inc result + if p.tok[result].kind != tkWhite: break + inc result + if p.tok[result].kind != tkAdornment: break + if p.tok[result].kind == tkIndent: inc result + +proc checkColumns(p: RstParser, cols: RstCols) = + var i = p.idx + if p.tok[i].symbol[0] != '=': + stopOrWarn(p, meIllformedTable, + "only tables with `=` columns specification are allowed") + for col in 0 ..< cols.len: + if tokEnd(p, i) != cols[col].stop: + stopOrWarn(p, meIllformedTable, + "end of table column #$1 should end at position $2" % [ + $(col+1), $(cols[col].stop+ColRstOffset)], + p.tok[i].line, tokEnd(p, i)) + inc i + if col == cols.len - 1: + if p.tok[i].kind == tkWhite: + inc i + if p.tok[i].kind notin {tkIndent, tkEof}: + stopOrWarn(p, meIllformedTable, "extraneous column specification") + elif p.tok[i].kind == tkWhite: + inc i + else: + stopOrWarn(p, meIllformedTable, + "no enough table columns", p.tok[i].line, p.tok[i].col) + +proc getSpans(p: RstParser, nextLine: int, + cols: RstCols, unitedCols: RstCols): seq[int] = + ## Calculates how many columns a joined cell occupies. + if unitedCols.len > 0: + result = newSeq[int](unitedCols.len) + var + iCell = 0 + jCell = 0 + uCell = 0 + while jCell < cols.len: + if cols[jCell].stop < unitedCols[uCell].stop: + inc jCell + elif cols[jCell].stop == unitedCols[uCell].stop: + result[uCell] = jCell - iCell + 1 + iCell = jCell + 1 + jCell = jCell + 1 + inc uCell + else: + rstMessage(p, meIllformedTable, + "spanning underline does not match main table columns", + p.tok[nextLine].line, p.tok[nextLine].col) + +proc parseSimpleTableRow(p: var RstParser, cols: RstCols, colChar: char): PRstNode = + ## Parses 1 row in RST simple table. + # Consider that columns may be spanning (united by using underline like ----): + let nextLine = tokenAfterNewline(p) + var unitedCols: RstCols + var afterSpan: int + if p.tok[nextLine].kind == tkAdornment and p.tok[nextLine].symbol[0] == '-': + afterSpan = getColumns(p, unitedCols, nextLine) + if unitedCols == cols and p.tok[nextLine].symbol[0] == colChar: + # legacy rst.nim compat.: allow punctuation like `----` in main boundaries + afterSpan = nextLine + unitedCols.setLen 0 + else: + afterSpan = nextLine + template colEnd(i): int = + if i == cols.len - 1: high(int) # last column has no limit + elif unitedCols.len > 0: unitedCols[i].stop else: cols[i].stop + template colStart(i): int = + if unitedCols.len > 0: unitedCols[i].start else: cols[i].start + var row = newSeq[string](if unitedCols.len > 0: unitedCols.len else: cols.len) + var spans: seq[int] = getSpans(p, nextLine, cols, unitedCols) + + let line = currentTok(p).line + # Iterate over the lines a single cell may span: + while true: + var nCell = 0 + # distribute tokens between cells in the current line: + while currentTok(p).kind notin {tkIndent, tkEof}: + if tokEnd(p) <= colEnd(nCell): + if tokStart(p) < colStart(nCell): + if currentTok(p).kind != tkWhite: + stopOrWarn(p, meIllformedTable, + "this word crosses table column from the left") + row[nCell].add(currentTok(p).symbol) + else: + row[nCell].add(currentTok(p).symbol) + inc p.idx + else: + if tokStart(p) < colEnd(nCell) and currentTok(p).kind != tkWhite: + stopOrWarn(p, meIllformedTable, + "this word crosses table column from the right") + row[nCell].add(currentTok(p).symbol) + inc p.idx + inc nCell + if currentTok(p).kind == tkIndent: inc p.idx + if tokEnd(p) <= colEnd(0): break + # Continued current cells because the 1st column is empty. + if currentTok(p).kind in {tkEof, tkAdornment}: + break + for nCell in countup(1, high(row)): row[nCell].add('\n') + result = newRstNode(rnTableRow) + var q: RstParser + for uCell in 0 ..< row.len: + initParser(q, p.s) + q.col = colStart(uCell) + q.line = line - 1 + getTokens(row[uCell], q.tok) + let cell = newRstNode(rnTableDataCell) + cell.span = if spans.len == 0: 0 else: spans[uCell] + cell.add(parseDoc(q)) + result.add(cell) + if afterSpan > p.idx: + p.idx = afterSpan proc parseSimpleTable(p: var RstParser): PRstNode = - var - cols: IntSeq - row: seq[string] - i, last, line: int - c: char - q: RstParser - a, b: PRstNode - result = newRstNode(rnTable) - cols = @[] - row = @[] - a = nil - c = p.tok[p.idx].symbol[0] + var cols: RstCols + result = newRstNodeA(p, rnTable) + let startIdx = getColumns(p, cols, p.idx) + let colChar = currentTok(p).symbol[0] + checkColumns(p, cols) + p.idx = startIdx + result.colCount = cols.len while true: - if p.tok[p.idx].kind == tkAdornment: - last = tokenAfterNewline(p) - if p.tok[last].kind in {tkEof, tkIndent}: + if currentTok(p).kind == tkAdornment: + checkColumns(p, cols) + p.idx = tokenAfterNewline(p) + if currentTok(p).kind in {tkEof, tkIndent}: # skip last adornment line: - p.idx = last break - getColumns(p, cols) - setLen(row, len(cols)) - if a != nil: - for j in 0..len(a)-1: a.sons[j].kind = rnTableHeaderCell - if p.tok[p.idx].kind == tkEof: break - for j in countup(0, high(row)): row[j] = "" - # the following while loop iterates over the lines a single cell may span: - line = p.tok[p.idx].line - while true: - i = 0 - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - if (tokEnd(p) <= cols[i]): - add(row[i], p.tok[p.idx].symbol) - inc(p.idx) - else: - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - inc(i) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - if tokEnd(p) <= cols[0]: break - if p.tok[p.idx].kind in {tkEof, tkAdornment}: break - for j in countup(1, high(row)): add(row[j], '\x0A') + if result.sons.len > 0: result.sons[^1].endsHeader = true + # fix rnTableDataCell -> rnTableHeaderCell for previous table rows: + for nRow in 0 ..< result.sons.len: + for nCell in 0 ..< result.sons[nRow].len: + template cell: PRstNode = result.sons[nRow].sons[nCell] + cell = PRstNode(kind: rnTableHeaderCell, sons: cell.sons, + span: cell.span, anchor: cell.anchor) + if currentTok(p).kind == tkEof: break + let tabRow = parseSimpleTableRow(p, cols, colChar) + result.add tabRow + +proc readTableRow(p: var RstParser): ColSeq = + if currentTok(p).symbol == "|": inc p.idx + while currentTok(p).kind notin {tkIndent, tkEof}: + var limits: ColumnLimits + limits.first = p.idx + while currentTok(p).kind notin {tkIndent, tkEof}: + if currentTok(p).symbol == "|" and prevTok(p).symbol != "\\": break + inc p.idx + limits.last = p.idx + result.add(limits) + if currentTok(p).kind in {tkIndent, tkEof}: break + inc p.idx + p.idx = tokenAfterNewline(p) + +proc getColContents(p: var RstParser, colLim: ColumnLimits): string = + for i in colLim.first ..< colLim.last: + result.add(p.tok[i].symbol) + result.strip + +proc isValidDelimiterRow(p: var RstParser, colNum: int): bool = + let row = readTableRow(p) + if row.len != colNum: return false + for limits in row: + let content = getColContents(p, limits) + if content.len < 3 or not (content.startsWith("--") or content.startsWith(":-")): + return false + return true + +proc parseMarkdownTable(p: var RstParser): PRstNode = + var + row: ColSeq + a, b: PRstNode + q: RstParser + result = newRstNodeA(p, rnMarkdownTable) + + proc parseRow(p: var RstParser, cellKind: RstNodeKind, result: PRstNode) = + row = readTableRow(p) + if result.colCount == 0: result.colCount = row.len # table header + elif row.len < result.colCount: row.setLen(result.colCount) a = newRstNode(rnTableRow) - for j in countup(0, high(row)): + for j in 0 ..< result.colCount: + b = newRstNode(cellKind) initParser(q, p.s) - q.col = cols[j] - q.line = line - 1 - q.filename = p.filename - q.col += getTokens(row[j], false, q.tok) - b = newRstNode(rnTableDataCell) - add(b, parseDoc(q)) - add(a, b) - add(result, a) + q.col = p.col + q.line = currentTok(p).line - 1 + getTokens(getColContents(p, row[j]), q.tok) + b.add(parseDoc(q)) + a.add(b) + result.add(a) + + parseRow(p, rnTableHeaderCell, result) + if not isValidDelimiterRow(p, result.colCount): + rstMessage(p, meMarkdownIllformedTable) + while predNL(p) and currentTok(p).symbol == "|": + parseRow(p, rnTableDataCell, result) proc parseTransition(p: var RstParser): PRstNode = - result = newRstNode(rnTransition) - inc(p.idx) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - -proc parseOverline(p: var RstParser): PRstNode = - var c = p.tok[p.idx].symbol[0] - inc(p.idx, 2) - result = newRstNode(rnOverline) - while true: - parseUntilNewline(p, result) - if p.tok[p.idx].kind == tkIndent: - inc(p.idx) - if p.tok[p.idx - 1].ival > currInd(p): - add(result, newRstNode(rnLeaf, " ")) - else: - break - else: - break - result.level = getLevel(p.s.overlineToLevel, p.s.oLevel, c) - if p.tok[p.idx].kind == tkAdornment: - inc(p.idx) # XXX: check? - if p.tok[p.idx].kind == tkIndent: inc(p.idx) + result = newRstNodeA(p, rnTransition) + inc p.idx + if currentTok(p).kind == tkIndent: inc p.idx + if currentTok(p).kind == tkIndent: inc p.idx proc parseBulletList(p: var RstParser): PRstNode = result = nil - if p.tok[p.idx + 1].kind == tkWhite: - var bullet = p.tok[p.idx].symbol - var col = p.tok[p.idx].col - result = newRstNode(rnBulletList) + if nextTok(p).kind == tkWhite: + var bullet = currentTok(p).symbol + var col = currentTok(p).col + result = newRstNodeA(p, rnBulletList) pushInd(p, p.tok[p.idx + 2].col) - inc(p.idx, 2) + inc p.idx, 2 while true: var item = newRstNode(rnBulletItem) parseSection(p, item) - add(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == bullet) and - (p.tok[p.idx + 2].kind == tkWhite): - inc(p.idx, 3) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).symbol == bullet and + p.tok[p.idx + 2].kind == tkWhite: + inc p.idx, 3 else: break popInd(p) proc parseOptionList(p: var RstParser): PRstNode = - result = newRstNode(rnOptionList) + result = newRstNodeA(p, rnOptionList) + let col = currentTok(p).col + var order = 1 while true: - if isOptionList(p): + if currentTok(p).col == col and isOptionList(p): var a = newRstNode(rnOptionGroup) var b = newRstNode(rnDescription) var c = newRstNode(rnOptionListItem) - if match(p, p.idx, "//w"): inc(p.idx) - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - if (p.tok[p.idx].kind == tkWhite) and (len(p.tok[p.idx].symbol) > 1): - inc(p.idx) + if match(p, p.idx, "//w"): inc p.idx + while currentTok(p).kind notin {tkIndent, tkEof}: + if currentTok(p).kind == tkWhite and currentTok(p).symbol.len > 1: + inc p.idx break - add(a, newLeaf(p)) - inc(p.idx) + a.add(newLeaf(p)) + inc p.idx var j = tokenAfterNewline(p) - if (j > 0) and (p.tok[j - 1].kind == tkIndent) and - (p.tok[j - 1].ival > currInd(p)): + if j > 0 and p.tok[j - 1].kind == tkIndent and p.tok[j - 1].ival > currInd(p): pushInd(p, p.tok[j - 1].ival) parseSection(p, b) popInd(p) else: parseLine(p, b) - if (p.tok[p.idx].kind == tkIndent): inc(p.idx) - add(c, a) - add(c, b) - add(result, c) + while currentTok(p).kind == tkIndent: inc p.idx + c.add(a) + c.add(b) + c.order = order; inc order + result.add(c) + else: + if currentTok(p).kind != tkEof: dec p.idx # back to tkIndent + break + +proc parseMdDefinitionList(p: var RstParser): PRstNode = + ## Parses (Pandoc/kramdown/PHPextra) Markdown definition lists. + result = newRstNodeA(p, rnMdDefList) + let termCol = currentTok(p).col + while true: + var item = newRstNode(rnDefItem) + var term = newRstNode(rnDefName) + parseLine(p, term) + skipNewlines(p) + inc p.idx, 2 # skip ":" and space + item.add(term) + while true: + var def = newRstNode(rnDefBody) + let indent = getMdBlockIndent(p) + pushInd(p, indent) + parseSection(p, def) + popInd(p) + item.add(def) + let j = skipNewlines(p, p.idx) + if isMdDefBody(p, j, termCol): # parse next definition body + p.idx = j + 2 # skip ":" and space + else: + break + result.add(item) + let j = skipNewlines(p, p.idx) + if p.tok[j].col == termCol and isMdDefListItem(p, j): + p.idx = j # parse next item else: break proc parseDefinitionList(p: var RstParser): PRstNode = result = nil var j = tokenAfterNewline(p) - 1 - if (j >= 1) and (p.tok[j].kind == tkIndent) and - (p.tok[j].ival > currInd(p)) and (p.tok[j - 1].symbol != "::"): - var col = p.tok[p.idx].col - result = newRstNode(rnDefList) + if j >= 1 and p.tok[j].kind == tkIndent and + p.tok[j].ival > currInd(p) and p.tok[j - 1].symbol != "::": + var col = currentTok(p).col + result = newRstNodeA(p, rnDefList) while true: + if isOptionList(p): + break # option list has priority over def.list j = p.idx var a = newRstNode(rnDefName) parseLine(p, a) - if (p.tok[p.idx].kind == tkIndent) and - (p.tok[p.idx].ival > currInd(p)) and - (p.tok[p.idx + 1].symbol != "::") and - not (p.tok[p.idx + 1].kind in {tkIndent, tkEof}): - pushInd(p, p.tok[p.idx].ival) + if currentTok(p).kind == tkIndent and + currentTok(p).ival > currInd(p) and + nextTok(p).symbol != "::" and + nextTok(p).kind notin {tkIndent, tkEof}: + pushInd(p, currentTok(p).ival) var b = newRstNode(rnDefBody) parseSection(p, b) var c = newRstNode(rnDefItem) - add(c, a) - add(c, b) - add(result, c) + c.add(a) + c.add(b) + result.add(c) popInd(p) else: p.idx = j break - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col): - inc(p.idx) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col: + inc p.idx j = tokenAfterNewline(p) - 1 if j >= 1 and p.tok[j].kind == tkIndent and p.tok[j].ival > col and p.tok[j-1].symbol != "::" and p.tok[j+1].kind != tkIndent: discard else: break - if len(result) == 0: result = nil + if result.len == 0: result = nil proc parseEnumList(p: var RstParser): PRstNode = const - wildcards: array[0..2, string] = ["(e) ", "e) ", "e. "] - wildpos: array[0..2, int] = [1, 0, 0] - result = nil + wildcards: array[0..5, string] = ["(n) ", "n) ", "n. ", + "(x) ", "x) ", "x. "] + # enumerator patterns, where 'x' means letter and 'n' means number + wildToken: array[0..5, int] = [4, 3, 3, 4, 3, 3] # number of tokens + wildIndex: array[0..5, int] = [1, 0, 0, 1, 0, 0] + # position of enumeration sequence (number/letter) in enumerator + let col = currentTok(p).col var w = 0 - while w <= 2: + while w < wildcards.len: if match(p, p.idx, wildcards[w]): break - inc(w) - if w <= 2: - var col = p.tok[p.idx].col - result = newRstNode(rnEnumList) - inc(p.idx, wildpos[w] + 3) - var j = tokenAfterNewline(p) - if (p.tok[j].col == p.tok[p.idx].col) or match(p, j, wildcards[w]): - pushInd(p, p.tok[p.idx].col) - while true: - var item = newRstNode(rnEnumItem) - parseSection(p, item) - add(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - match(p, p.idx + 1, wildcards[w]): - inc(p.idx, wildpos[w] + 4) - else: + inc w + assert w < wildcards.len + + proc checkAfterNewline(p: RstParser, report: bool): bool = + ## If no indentation on the next line then parse as a normal paragraph + ## according to the RST spec. And report a warning with suggestions + let j = tokenAfterNewline(p, start=p.idx+1) + let requiredIndent = p.tok[p.idx+wildToken[w]].col + if p.tok[j].kind notin {tkIndent, tkEof} and + p.tok[j].col < requiredIndent and + (p.tok[j].col > col or + (p.tok[j].col == col and not match(p, j, wildcards[w]))): + if report: + let n = p.line + p.tok[j].line + let msg = "\n" & """ + not enough indentation on line $2 + (should be at column $3 if it's a continuation of enum. list), + or no blank line after line $1 (if it should be the next paragraph), + or no escaping \ at the beginning of line $1 + (if lines $1..$2 are a normal paragraph, not enum. list)""".dedent + let c = p.col + requiredIndent + ColRstOffset + rstMessage(p, mwRstStyle, msg % [$(n-1), $n, $c], + p.tok[j].line, p.tok[j].col) + result = false + else: + result = true + + if not checkAfterNewline(p, report = true): + return nil + result = newRstNodeA(p, rnEnumList) + let autoEnums = if roSupportMarkdown in p.s.options: @["#", "1"] else: @["#"] + var prevAE = "" # so as not allow mixing auto-enumerators `1` and `#` + var curEnum = 1 + for i in 0 ..< wildToken[w]-1: # add first enumerator with (, ), and . + if p.tok[p.idx + i].symbol == "#": + prevAE = "#" + result.labelFmt.add "1" + else: + result.labelFmt.add p.tok[p.idx + i].symbol + var prevEnum = p.tok[p.idx + wildIndex[w]].symbol + inc p.idx, wildToken[w] + while true: + var item = newRstNode(rnEnumItem) + pushInd(p, currentTok(p).col) + parseSection(p, item) + popInd(p) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + match(p, p.idx+1, wildcards[w]): + # don't report to avoid duplication of warning since for + # subsequent enum. items parseEnumList will be called second time: + if not checkAfterNewline(p, report = false): + break + let enumerator = p.tok[p.idx + 1 + wildIndex[w]].symbol + # check that it's in sequence: enumerator == next(prevEnum) + if "n" in wildcards[w]: # arabic numeral + let prevEnumI = try: parseInt(prevEnum) except ValueError: 1 + if enumerator in autoEnums: + if prevAE != "" and enumerator != prevAE: + break + prevAE = enumerator + curEnum = prevEnumI + 1 + else: curEnum = (try: parseInt(enumerator) except ValueError: 1) + if curEnum - prevEnumI != 1: break - popInd(p) + prevEnum = enumerator + else: # a..z + let prevEnumI = ord(prevEnum[0]) + if enumerator == "#": curEnum = prevEnumI + 1 + else: curEnum = ord(enumerator[0]) + if curEnum - prevEnumI != 1: + break + prevEnum = $chr(curEnum) + inc p.idx, 1 + wildToken[w] else: - dec(p.idx, wildpos[w] + 3) - result = nil + break + +proc prefix(ftnType: FootnoteType): string = + case ftnType + of fnManualNumber: result = "footnote-" + of fnAutoNumber: result = "footnoteauto-" + of fnAutoNumberLabel: result = "footnote-" + of fnAutoSymbol: result = "footnotesym-" + of fnCitation: result = "citation-" + +proc parseFootnote(p: var RstParser): PRstNode {.gcsafe.} = + ## Parses footnotes and citations, always returns 2 sons: + ## + ## 1) footnote label, always containing rnInner with 1 or more sons + ## 2) footnote body, which may be nil + var label: PRstNode + if isRst(p): + inc p.idx # skip space after `..` + label = parseFootnoteName(p, reference=false) + if label == nil: + if isRst(p): + dec p.idx + return nil + result = newRstNode(rnFootnote) + result.add label + let (fnType, i) = getFootnoteType(p.s, label) + var name = "" + var anchor = fnType.prefix + case fnType + of fnManualNumber: + addFootnoteNumManual(p, i) + anchor.add $i + of fnAutoNumber, fnAutoNumberLabel: + name = rstnodeToRefname(label) + addFootnoteNumAuto(p, name) + if fnType == fnAutoNumberLabel: + anchor.add name + else: # fnAutoNumber + result.order = p.s.lineFootnoteNum.len + anchor.add $result.order + of fnAutoSymbol: + addFootnoteSymAuto(p) + result.order = p.s.lineFootnoteSym.len + anchor.add $p.s.lineFootnoteSym.len + of fnCitation: + anchor.add rstnodeToRefname(label) + addAnchorRst(p, anchor, target = result, anchorType = footnoteAnchor) + result.anchor = anchor + if currentTok(p).kind == tkWhite: inc p.idx + discard parseBlockContent(p, result, parseSectionWrapper) + if result.len < 2: + result.add nil proc sonKind(father: PRstNode, i: int): RstNodeKind = result = rnLeaf - if i < len(father): result = father.sons[i].kind + if i < father.len: result = father.sons[i].kind proc parseSection(p: var RstParser, result: PRstNode) = + ## parse top-level RST elements: sections, transitions and body elements. while true: var leave = false assert(p.idx >= 0) - while p.tok[p.idx].kind == tkIndent: - if currInd(p) == p.tok[p.idx].ival: - inc(p.idx) - elif p.tok[p.idx].ival > currInd(p): - pushInd(p, p.tok[p.idx].ival) - var a = newRstNode(rnBlockQuote) - parseSection(p, a) - add(result, a) - popInd(p) + while currentTok(p).kind == tkIndent: + if currInd(p) == currentTok(p).ival: + inc p.idx + elif currentTok(p).ival > currInd(p): + if roPreferMarkdown in p.s.options: # Markdown => normal paragraphs + if currentTok(p).ival - currInd(p) >= 4: + result.add parseLiteralBlock(p) + else: + pushInd(p, currentTok(p).ival) + parseSection(p, result) + popInd(p) + else: # RST mode => block quotes + pushInd(p, currentTok(p).ival) + var a = newRstNodeA(p, rnBlockQuote) + parseSection(p, a) + result.add(a) + popInd(p) else: + while currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: + inc p.idx # skip blank lines leave = true break - if leave or p.tok[p.idx].kind == tkEof: break + if leave or currentTok(p).kind == tkEof: break var a: PRstNode = nil var k = whichSection(p) case k of rnLiteralBlock: - inc(p.idx) # skip '::' + inc p.idx # skip '::' a = parseLiteralBlock(p) of rnBulletList: a = parseBulletList(p) of rnLineBlock: a = parseLineBlock(p) + of rnMarkdownBlockQuote: a = parseMarkdownBlockQuote(p) of rnDirective: a = parseDotDot(p) + of rnFootnote: a = parseFootnote(p) of rnEnumList: a = parseEnumList(p) - of rnLeaf: rstMessage(p, meNewSectionExpected) + of rnLeaf: rstMessage(p, meNewSectionExpected, "(syntax error)") of rnParagraph: discard of rnDefList: a = parseDefinitionList(p) + of rnMdDefList: a = parseMdDefinitionList(p) of rnFieldList: - if p.idx > 0: dec(p.idx) + if p.idx > 0: dec p.idx a = parseFields(p) of rnTransition: a = parseTransition(p) - of rnHeadline: a = parseHeadline(p) + of rnHeadline, rnMarkdownHeadline: a = parseHeadline(p) of rnOverline: a = parseOverline(p) of rnTable: a = parseSimpleTable(p) + of rnMarkdownTable: a = parseMarkdownTable(p) of rnOptionList: a = parseOptionList(p) else: #InternalError("rst.parseSection()") discard if a == nil and k != rnDirective: - a = newRstNode(rnParagraph) + a = newRstNodeA(p, rnParagraph) parseParagraph(p, a) - addIfNotNil(result, a) + result.addIfNotNil(a) if sonKind(result, 0) == rnParagraph and sonKind(result, 1) != rnParagraph: - result.sons[0].kind = rnInner - -proc parseSectionWrapper(p: var RstParser): PRstNode = - result = newRstNode(rnInner) - parseSection(p, result) - while (result.kind == rnInner) and (len(result) == 1): - result = result.sons[0] - -proc `$`(t: Token): string = - result = $t.kind & ' ' & (if isNil(t.symbol): "NIL" else: t.symbol) + result.sons[0] = newRstNode(rnInner, result.sons[0].sons, + anchor=result.sons[0].anchor) proc parseDoc(p: var RstParser): PRstNode = result = parseSectionWrapper(p) - if p.tok[p.idx].kind != tkEof: - when false: - assert isAllocatedPtr(cast[pointer](p.tok)) - for i in 0 .. high(p.tok): - assert isNil(p.tok[i].symbol) or - isAllocatedPtr(cast[pointer](p.tok[i].symbol)) - echo "index: ", p.idx, " length: ", high(p.tok), "##", - p.tok[p.idx-1], p.tok[p.idx], p.tok[p.idx+1] - #assert isAllocatedPtr(cast[pointer](p.indentStack)) + if currentTok(p).kind != tkEof: rstMessage(p, meGeneralParseError) type DirFlag = enum hasArg, hasOptions, argIsFile, argIsWord DirFlags = set[DirFlag] - SectionParser = proc (p: var RstParser): PRstNode {.nimcall.} -{.deprecated: [TDirFlag: DirFlag, TDirFlags: DirFlags, - TSectionParser: SectionParser].} -proc parseDirective(p: var RstParser, flags: DirFlags): PRstNode = +proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode = ## Parses arguments and options for a directive block. ## ## A directive block will always have three sons: the arguments for the - ## directive (rnDirArg), the options (rnFieldList) and the block - ## (rnLineBlock). This proc parses the two first nodes, the block is left to + ## directive (rnDirArg), the options (rnFieldList) and the directive + ## content block. This proc parses the two first nodes, the 3rd is left to ## the outer `parseDirective` call. ## ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## check them before accessing. - result = newRstNode(rnDirective) + result = newRstNodeA(p, k) + if k == rnCodeBlock: result.info = lineInfo(p) var args: PRstNode = nil var options: PRstNode = nil if hasArg in flags: args = newRstNode(rnDirArg) if argIsFile in flags: while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWord, tkOther, tkPunct, tkAdornment: - add(args, newLeaf(p)) - inc(p.idx) + args.add(newLeaf(p)) + inc p.idx else: break elif argIsWord in flags: - while p.tok[p.idx].kind == tkWhite: inc(p.idx) - if p.tok[p.idx].kind == tkWord: - add(args, newLeaf(p)) - inc(p.idx) + while currentTok(p).kind == tkWhite: inc p.idx + if currentTok(p).kind == tkWord: + args.add(newLeaf(p)) + inc p.idx else: args = nil else: parseLine(p, args) - add(result, args) + result.add(args) if hasOptions in flags: - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival >= 3) and - (p.tok[p.idx + 1].symbol == ":"): + if currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) and + nextTok(p).symbol == ":": + pushInd(p, currentTok(p).ival) options = parseFields(p) - add(result, options) - -proc indFollows(p: RstParser): bool = - result = p.tok[p.idx].kind == tkIndent and p.tok[p.idx].ival > currInd(p) + popInd(p) + result.add(options) -proc parseDirective(p: var RstParser, flags: DirFlags, +proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags, contentParser: SectionParser): PRstNode = - ## Returns a generic rnDirective tree. + ## A helper proc that does main work for specific directive procs. + ## Always returns a generic rnDirective tree with these 3 children: ## - ## The children are rnDirArg, rnFieldList and rnLineBlock. Any might be nil. - result = parseDirective(p, flags) - if not isNil(contentParser) and indFollows(p): - pushInd(p, p.tok[p.idx].ival) - var content = contentParser(p) - popInd(p) - add(result, content) + ## 1) rnDirArg + ## 2) rnFieldList + ## 3) a node returned by `contentParser`. + ## + ## .. warning:: Any of the 3 children may be nil. + result = parseDirective(p, k, flags) + if not isNil(contentParser) and + parseBlockContent(p, result, contentParser): + discard "result is updated by parseBlockContent" else: - add(result, nil) + result.add(PRstNode(nil)) proc parseDirBody(p: var RstParser, contentParser: SectionParser): PRstNode = if indFollows(p): - pushInd(p, p.tok[p.idx].ival) + pushInd(p, currentTok(p).ival) result = contentParser(p) popInd(p) proc dirInclude(p: var RstParser): PRstNode = - # - #The following options are recognized: - # - #start-after : text to find in the external data file - # Only the content after the first occurrence of the specified text will - # be included. - #end-before : text to find in the external data file - # Only the content before the first occurrence of the specified text - # (but after any after text) will be included. + ## + ## The following options are recognized: + ## + ## :start-after: text to find in the external data file + ## + ## Only the content after the first occurrence of the specified + ## text will be included. If text is not found inclusion will + ## start from beginning of the file + ## + ## :end-before: text to find in the external data file + ## + ## Only the content before the first occurrence of the specified + ## text (but after any after text) will be included. If text is + ## not found inclusion will happen until the end of the file. #literal : flag (empty) # The entire included text is inserted into the document as a single # literal block (useful for program listings). @@ -1498,27 +3312,52 @@ proc dirInclude(p: var RstParser): PRstNode = # encoding (if specified). # result = nil - var n = parseDirective(p, {hasArg, argIsFile, hasOptions}, nil) + var n = parseDirective(p, rnDirective, {hasArg, argIsFile, hasOptions}, nil) var filename = strip(addNodes(n.sons[0])) - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) else: # XXX: error handling; recursive file inclusion! if getFieldValue(n, "literal") != "": result = newRstNode(rnLiteralBlock) - add(result, newRstNode(rnLeaf, readFile(path))) + result.add newLeaf(readFile(path)) else: + let inputString = readFile(path) + let startPosition = + block: + let searchFor = n.getFieldValue("start-after").strip() + if searchFor != "": + let pos = inputString.find(searchFor) + if pos != -1: pos + searchFor.len + else: 0 + else: + 0 + + let endPosition = + block: + let searchFor = n.getFieldValue("end-before").strip() + if searchFor != "": + let pos = inputString.find(searchFor, start = startPosition) + if pos != -1: pos - 1 + else: 0 + else: + inputString.len - 1 + var q: RstParser initParser(q, p.s) - q.filename = filename - q.col += getTokens(readFile(path), false, q.tok) + let saveFileIdx = p.s.currFileIdx + setCurrFilename(p.s, path) + getTokens( + inputString[startPosition..endPosition], + q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: # InternalError("Too many binary zeros in include file") result = parseDoc(q) + p.s.currFileIdx = saveFileIdx -proc dirCodeBlock(p: var RstParser, nimrodExtension = false): PRstNode = +proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = ## Parses a code block. ## ## Code blocks are rnDirective trees with a `kind` of rnCodeBlock. See the @@ -1526,80 +3365,77 @@ proc dirCodeBlock(p: var RstParser, nimrodExtension = false): PRstNode = ## ## Code blocks can come in two forms, the standard `code directive ## <http://docutils.sourceforge.net/docs/ref/rst/directives.html#code>`_ and - ## the nimrod extension ``.. code-block::``. If the block is an extension, we - ## want the default language syntax highlighting to be Nimrod, so we create a - ## fake internal field to comminicate with the generator. The field is named + ## the nim extension ``.. code-block::``. If the block is an extension, we + ## want the default language syntax highlighting to be Nim, so we create a + ## fake internal field to communicate with the generator. The field is named ## ``default-language``, which is unlikely to collide with a field specified ## by any random rst input file. ## ## As an extension this proc will process the ``file`` extension field and if ## present will replace the code block with the contents of the referenced - ## file. - result = parseDirective(p, {hasArg, hasOptions}, parseLiteralBlock) - var filename = strip(getFieldValue(result, "file")) - if filename != "": - var path = p.s.findFile(filename) - if path == "": rstMessage(p, meCannotOpenFile, filename) - var n = newRstNode(rnLiteralBlock) - add(n, newRstNode(rnLeaf, readFile(path))) - result.sons[2] = n + ## file. This behaviour is disabled in sandboxed mode and can be re-enabled + ## with the `roSandboxDisabled` flag. + result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock) + mayLoadFile(p, result) - # Extend the field block if we are using our custom extension. - if nimrodExtension: - # Create a field block if the input block didn't have any. - if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) - assert result.sons[1].kind == rnFieldList - # Hook the extra field and specify the Nimrod language as value. - var extraNode = newRstNode(rnField) - extraNode.add(newRstNode(rnFieldName)) - extraNode.add(newRstNode(rnFieldBody)) - extraNode.sons[0].add(newRstNode(rnLeaf, "default-language")) - extraNode.sons[1].add(newRstNode(rnLeaf, "Nimrod")) - result.sons[1].add(extraNode) - - result.kind = rnCodeBlock + # Extend the field block if we are using our custom Nim extension. + if nimExtension: + defaultCodeLangNim(p, result) proc dirContainer(p: var RstParser): PRstNode = - result = parseDirective(p, {hasArg}, parseSectionWrapper) - assert(result.kind == rnDirective) - assert(len(result) == 3) - result.kind = rnContainer + result = parseDirective(p, rnContainer, {hasArg}, parseSectionWrapper) + assert(result.len == 3) proc dirImage(p: var RstParser): PRstNode = - result = parseDirective(p, {hasOptions, hasArg, argIsFile}, nil) - result.kind = rnImage + result = parseDirective(p, rnImage, {hasOptions, hasArg, argIsFile}, nil) proc dirFigure(p: var RstParser): PRstNode = - result = parseDirective(p, {hasOptions, hasArg, argIsFile}, + result = parseDirective(p, rnFigure, {hasOptions, hasArg, argIsFile}, parseSectionWrapper) - result.kind = rnFigure proc dirTitle(p: var RstParser): PRstNode = - result = parseDirective(p, {hasArg}, nil) - result.kind = rnTitle + result = parseDirective(p, rnTitle, {hasArg}, nil) proc dirContents(p: var RstParser): PRstNode = - result = parseDirective(p, {hasArg}, nil) - result.kind = rnContents + result = parseDirective(p, rnContents, {hasArg}, nil) + p.s.hasToc = true proc dirIndex(p: var RstParser): PRstNode = - result = parseDirective(p, {}, parseSectionWrapper) - result.kind = rnIndex + result = parseDirective(p, rnIndex, {}, parseSectionWrapper) + +proc dirAdmonition(p: var RstParser, d: string): PRstNode = + result = parseDirective(p, rnAdmonition, {}, parseSectionWrapper) + result.adType = d + +proc dirDefaultRole(p: var RstParser): PRstNode = + result = parseDirective(p, rnDefaultRole, {hasArg}, nil) + if result.sons[0].len == 0: p.s.currRole = defaultRole(p.s.options) + else: + assert result.sons[0].sons[0].kind == rnLeaf + p.s.currRole = result.sons[0].sons[0].text + p.s.currRoleKind = whichRole(p, p.s.currRole) + +proc dirRole(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {hasArg, hasOptions}, nil) + # just check that language is supported, TODO: real role association + let lang = getFieldValue(result, "language").strip + if lang != "" and getSourceLanguage(lang) == langNone: + rstMessage(p, mwUnsupportedLanguage, lang) proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind, contentParser: SectionParser) = var filename = getFieldValue(result, "file") if filename.len > 0: - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path.len == 0: rstMessage(p, meCannotOpenFile, filename) else: var f = readFile(path) result = newRstNode(kind) - add(result, newRstNode(rnLeaf, f)) + result.add newLeaf(f) else: - result.kind = kind - add(result, parseDirBody(p, contentParser)) + result = newRstNode(kind, result.sons) + result.add(parseDirBody(p, contentParser)) proc dirRaw(p: var RstParser): PRstNode = # @@ -1610,7 +3446,7 @@ proc dirRaw(p: var RstParser): PRstNode = # # html # latex - result = parseDirective(p, {hasOptions, hasArg, argIsWord}) + result = parseDirective(p, rnDirective, {hasOptions, hasArg, argIsWord}) if result.sons[0] != nil: if cmpIgnoreCase(result.sons[0].sons[0].text, "html") == 0: dirRawAux(p, result, rnRawHtml, parseLiteralBlock) @@ -1621,99 +3457,430 @@ proc dirRaw(p: var RstParser): PRstNode = else: dirRawAux(p, result, rnRaw, parseSectionWrapper) +proc dirImportdoc(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {}, parseLiteralBlock) + assert result.sons[2].kind == rnLiteralBlock + assert result.sons[2].sons[0].kind == rnLeaf + let filenames: seq[string] = split(result.sons[2].sons[0].text, seps = {','}) + proc rmSpaces(s: string): string = s.split.join("") + for origFilename in filenames: + p.s.idxImports[origFilename.rmSpaces] = ImportdocInfo(fromInfo: lineInfo(p)) + +proc selectDir(p: var RstParser, d: string): PRstNode = + result = nil + let tok = p.tok[p.idx-2] # report on directive in ".. directive::" + if roSandboxDisabled notin p.s.options: + if d notin SandboxDirAllowlist: + rstMessage(p, meSandboxedDirective, d, tok.line, tok.col) + + case d + of "admonition", "attention", "caution": result = dirAdmonition(p, d) + of "code": result = dirCodeBlock(p) + of "code-block": result = dirCodeBlock(p, nimExtension = true) + of "container": result = dirContainer(p) + of "contents": result = dirContents(p) + of "danger": result = dirAdmonition(p, d) + of "default-role": result = dirDefaultRole(p) + of "error": result = dirAdmonition(p, d) + of "figure": result = dirFigure(p) + of "hint": result = dirAdmonition(p, d) + of "image": result = dirImage(p) + of "important": result = dirAdmonition(p, d) + of "importdoc": result = dirImportdoc(p) + of "include": result = dirInclude(p) + of "index": result = dirIndex(p) + of "note": result = dirAdmonition(p, d) + of "raw": + if roSupportRawDirective in p.s.options: + result = dirRaw(p) + else: + rstMessage(p, meInvalidDirective, d) + of "role": result = dirRole(p) + of "tip": result = dirAdmonition(p, d) + of "title": result = dirTitle(p) + of "warning": result = dirAdmonition(p, d) + else: + rstMessage(p, meInvalidDirective, d, tok.line, tok.col) + proc parseDotDot(p: var RstParser): PRstNode = + # parse "explicit markup blocks" result = nil - var col = p.tok[p.idx].col - inc(p.idx) + var n: PRstNode # to store result, workaround for bug 16855 + var col = currentTok(p).col + inc p.idx var d = getDirective(p) if d != "": pushInd(p, col) - case getDirKind(d) - of dkInclude: result = dirInclude(p) - of dkImage: result = dirImage(p) - of dkFigure: result = dirFigure(p) - of dkTitle: result = dirTitle(p) - of dkContainer: result = dirContainer(p) - of dkContents: result = dirContents(p) - of dkRaw: - if roSupportRawDirective in p.s.options: - result = dirRaw(p) - else: - rstMessage(p, meInvalidDirective, d) - of dkCode: result = dirCodeBlock(p) - of dkCodeBlock: result = dirCodeBlock(p, nimrodExtension = true) - of dkIndex: result = dirIndex(p) - else: rstMessage(p, meInvalidDirective, d) + result = selectDir(p, d) popInd(p) elif match(p, p.idx, " _"): # hyperlink target: - inc(p.idx, 2) - var a = getReferenceName(p, ":") - if p.tok[p.idx].kind == tkWhite: inc(p.idx) + inc p.idx, 2 + var ending = ":" + if currentTok(p).symbol == "`": + inc p.idx + ending = "`" + var a = getReferenceName(p, ending) + if ending == "`": + if currentTok(p).symbol == ":": + inc p.idx + else: + rstMessage(p, meExpected, ":") + if currentTok(p).kind == tkWhite: inc p.idx var b = untilEol(p) - setRef(p, rstnodeToRefname(a), b) + if len(b) == 0: # set internal anchor + p.curAnchors.add ManualAnchor( + alias: linkName(a), anchor: rstnodeToRefname(a), info: prevLineInfo(p) + ) + else: # external hyperlink + setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias) elif match(p, p.idx, " |"): # substitution definitions: - inc(p.idx, 2) + inc p.idx, 2 var a = getReferenceName(p, "|") var b: PRstNode - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - if cmpIgnoreStyle(p.tok[p.idx].symbol, "replace") == 0: - inc(p.idx) + if currentTok(p).kind == tkWhite: inc p.idx + if cmpIgnoreStyle(currentTok(p).symbol, "replace") == 0: + inc p.idx expect(p, "::") b = untilEol(p) - elif cmpIgnoreStyle(p.tok[p.idx].symbol, "image") == 0: - inc(p.idx) + elif cmpIgnoreStyle(currentTok(p).symbol, "image") == 0: + inc p.idx b = dirImage(p) else: - rstMessage(p, meInvalidDirective, p.tok[p.idx].symbol) + rstMessage(p, meInvalidDirective, currentTok(p).symbol) setSub(p, addNodes(a), b) - elif match(p, p.idx, " ["): - # footnotes, citations - inc(p.idx, 2) - var a = getReferenceName(p, "]") - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - var b = untilEol(p) - setRef(p, rstnodeToRefname(a), b) + elif match(p, p.idx, " [") and + (n = parseFootnote(p); n != nil): + result = n else: - result = parseComment(p) - -proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = + result = parseComment(p, col) + +proc rstParsePass1*(fragment: string, + line, column: int, + sharedState: PRstSharedState): PRstNode = + ## Parses an RST `fragment`. + ## The result should be further processed by + ## preparePass2_ and resolveSubs_ (which is pass 2). + var p: RstParser + initParser(p, sharedState) + p.line = line + p.col = column + getTokens(fragment, p.tok) + result = parseDoc(p) + +proc extractLinkEnd(x: string): string = + ## From links like `path/to/file.html#/%` extract `file.html#/%`. + let i = find(x, '#') + let last = + if i >= 0: i + else: x.len - 1 + let j = rfind(x, '/', start=0, last=last) + if j >= 0: + result = x[j+1 .. ^1] + else: + result = x + +proc loadIdxFile(s: var PRstSharedState, origFilename: string) = + doAssert roSandboxDisabled in s.options + var info: TLineInfo + info.fileIndex = addFilename(s, origFilename) + var (dir, basename, ext) = origFilename.splitFile + if ext notin [".md", ".rst", ".nim", ""]: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, origFilename & ": unknown extension") + let idxFilename = dir / basename & ".idx" + let (idxPath, linkRelPath) = s.findRefFile(idxFilename) + s.idxImports[origFilename].linkRelPath = linkRelPath + var + fileEntries: seq[IndexEntry] + title: IndexEntry + try: + (fileEntries, title) = parseIdxFile(idxPath) + except IOError: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, idxPath) + except ValueError as e: + s.msgHandler(idxPath, LineRstInit, ColRstInit, meInvalidField, e.msg) + + var isMarkup = false # for sanity check to avoid mixing .md <-> .nim + for entry in fileEntries: + # Though target .idx already has inside it the path to HTML relative + # project's root, we won't rely on it and use `linkRelPath` instead. + let refn = extractLinkEnd(entry.link) + # select either markup (rst/md) or Nim cases: + if entry.kind in {ieMarkupTitle, ieNimTitle}: + s.idxImports[origFilename].title = entry.keyword + case entry.kind + of ieIdxRole, ieHeading, ieMarkupTitle: + if ext == ".nim" and entry.kind == ieMarkupTitle: + rstMessage(s, idxPath, meInvalidField, + $ieMarkupTitle & " in supposedly .nim-derived file") + if entry.kind == ieMarkupTitle: + isMarkup = true + info.line = entry.line.uint16 + addAnchorExtRst(s, key = entry.keyword, refn = refn, + anchorType = headlineAnchor, info=info) + of ieNim, ieNimGroup, ieNimTitle: + if ext in [".md", ".rst"] or isMarkup: + rstMessage(s, idxPath, meInvalidField, + $entry.kind & " in supposedly markup-derived file") + s.nimFileImported = true + var langSym: LangSymbol + if entry.kind in {ieNim, ieNimTitle}: + var q: RstParser + initParser(q, s) + info.line = entry.line.uint16 + setLen(q.tok, 0) + q.idx = 0 + getTokens(entry.linkTitle, q.tok) + var sons = newSeq[PRstNode](q.tok.len) + for i in 0 ..< q.tok.len: sons[i] = newLeaf(q.tok[i].symbol) + let linkTitle = newRstNode(rnInner, sons) + langSym = linkTitle.toLangSymbol + else: # entry.kind == ieNimGroup + langSym = langSymbolGroup(kind=entry.linkTitle, name=entry.keyword) + addAnchorNim(s, external = true, refn = refn, tooltip = entry.linkDesc, + langSym = langSym, priority = -4, # lowest + info = info, module = info.fileIndex) + doAssert s.idxImports[origFilename].title != "" + +proc preparePass2*(s: var PRstSharedState, mainNode: PRstNode, importdoc = true) = + ## Records titles in node `mainNode` and orders footnotes. + countTitles(s, mainNode) + fixHeadlines(s) + orderFootnotes(s) + if importdoc: + for origFilename in s.idxImports.keys: + loadIdxFile(s, origFilename) + +proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = + # Associate this link alias with its target and change node kind to + # rnHyperlink or rnInternalRef appropriately. + var desc, alias: PRstNode + if n.kind == rnPandocRef: # link like [desc][alias] + desc = n.sons[0] + alias = n.sons[1] + else: # n.kind == rnRstRef, link like `desc=alias`_ + desc = n + alias = n + type LinkDef = object + ar: AnchorRule + priority: int + tooltip: string + target: PRstNode + info: TLineInfo + externFilename: string + # when external anchor: origin filename where anchor was defined + isTitle: bool + proc cmp(x, y: LinkDef): int = + result = cmp(x.priority, y.priority) + if result == 0: + result = cmp(x.target, y.target) + var foundLinks: seq[LinkDef] + let refn = rstnodeToRefname(alias) + var hyperlinks = findRef(s, refn) + for y in hyperlinks: + foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind), + target: y.value, info: y.info, + tooltip: "(" & $y.kind & ")") + let substRst = findMainAnchorRst(s, alias.addNodes, n.info) + template getExternFilename(subst: AnchorSubst): string = + if subst.kind == arExternalRst or + (subst.kind == arNim and subst.external): + getFilename(s, subst) + else: "" + for subst in substRst: + var refname, fullRefname: string + if subst.kind == arInternalRst: + refname = subst.target.anchor + fullRefname = refname + else: # arExternalRst + refname = subst.refnameExt + fullRefname = s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & refname + let anchorType = + if subst.kind == arInternalRst: subst.anchorType + else: subst.anchorTypeExt # arExternalRst + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + info: subst.info, + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(refname), + tooltip: "(" & $anchorType & ")") + # find anchors automatically generated from Nim symbols + if roNimFile in s.options or s.nimFileImported: + let substNim = findMainAnchorNim(s, signature=alias, n.info) + for subst in substNim: + let fullRefname = + if subst.external: + s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & subst.refname + else: subst.refname + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(subst.refname), + info: subst.info, tooltip: subst.tooltip) + foundLinks.sort(cmp = cmp, order = Descending) + let aliasStr = addNodes(alias) + if foundLinks.len >= 1: + if foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].used = true + let kind = if foundLinks[0].ar in {arHyperlink, arExternalRst}: rnHyperlink + elif foundLinks[0].ar == arNim: + if foundLinks[0].externFilename == "": rnNimdocRef + else: rnHyperlink + else: rnInternalRef + result = newRstNode(kind) + let documentName = # filename without ext for `.nim`, title for `.md` + if foundLinks[0].ar == arNim: + changeFileExt(foundLinks[0].externFilename.extractFilename, "") + elif foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].title + else: foundLinks[0].externFilename.extractFilename + let linkText = + if foundLinks[0].externFilename != "": + if foundLinks[0].isTitle: newLeaf(addNodes(desc)) + else: newLeaf(documentName & ": " & addNodes(desc)) + else: + newRstNode(rnInner, desc.sons) + result.sons = @[linkText, foundLinks[0].target] + if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip + if foundLinks.len > 1: # report ambiguous link + var targets = newSeq[string]() + for l in foundLinks: + var t = " " + if s.filenames.len > 1: + t.add getFilename(s.filenames, l.info.fileIndex) + let n = l.info.line + let c = l.info.col + ColRstOffset + t.add "($1, $2): $3" % [$n, $c, l.tooltip] + targets.add t + rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink, + "`$1`\n clash:\n$2" % [ + aliasStr, targets.join("\n")]) + else: # nothing found + result = n + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, aliasStr) + +proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = + ## Makes pass 2 of RST parsing. + ## Resolves substitutions and anchor aliases, groups footnotes. + ## Takes input node `n` and returns the same node with recursive + ## substitutions in `n.sons` to `result`. result = n if n == nil: return case n.kind of rnSubstitutionReferences: - var x = findSub(p, n) + var x = findSub(s, n) if x >= 0: - result = p.s.subs[x].value + result = s.subs[x].value else: var key = addNodes(n) var e = getEnv(key) - if e != "": result = newRstNode(rnLeaf, e) - else: rstMessage(p, mwUnknownSubstitution, key) - of rnRef: - var y = findRef(p, rstnodeToRefname(n)) - if y != nil: - result = newRstNode(rnHyperlink) - n.kind = rnInner - add(result, n) - add(result, y) + if e != "": result = newLeaf(e) + else: rstMessage(s.filenames, s.msgHandler, n.info, + mwUnknownSubstitution, key) + of rnRstRef, rnPandocRef: + result = resolveLink(s, n) + of rnFootnote: + var (fnType, num) = getFootnoteType(s, n.sons[0]) + case fnType + of fnManualNumber, fnCitation: + discard "no need to alter fixed text" + of fnAutoNumberLabel, fnAutoNumber: + if fnType == fnAutoNumberLabel: + let labelR = rstnodeToRefname(n.sons[0]) + num = getFootnoteNum(s, labelR) + else: + num = getFootnoteNum(s, n.order) + var nn = newRstNode(rnInner) + nn.add newLeaf($num) + result.sons[0] = nn + of fnAutoSymbol: + let sym = getAutoSymbol(s, n.order) + n.sons[0].sons[0].text = sym + n.sons[1] = resolveSubs(s, n.sons[1]) + of rnFootnoteRef: + var (fnType, num) = getFootnoteType(s, n.sons[0]) + template addLabel(number: int | string) = + var nn = newRstNode(rnInner) + nn.add newLeaf($number) + result.add(nn) + var refn = fnType.prefix + # create new rnFootnoteRef, add final label, and finalize target refn: + result = newRstNode(rnFootnoteRef, info = n.info) + case fnType + of fnManualNumber: + addLabel num + refn.add $num + of fnAutoNumber: + inc s.currFootnoteNumRef + addLabel getFootnoteNum(s, s.currFootnoteNumRef) + refn.add $s.currFootnoteNumRef + of fnAutoNumberLabel: + addLabel getFootnoteNum(s, rstnodeToRefname(n)) + refn.add rstnodeToRefname(n) + of fnAutoSymbol: + inc s.currFootnoteSymRef + addLabel getAutoSymbol(s, s.currFootnoteSymRef) + refn.add $s.currFootnoteSymRef + of fnCitation: + result.add n.sons[0] + refn.add rstnodeToRefname(n) + # TODO: correctly report ambiguities + let anchorInfo = findMainAnchorRst(s, refn, n.info) + if anchorInfo.len != 0: + result.add newLeaf(anchorInfo[0].target.anchor) # add link + else: + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) + result.add newLeaf(refn) # add link of rnLeaf: discard - of rnContents: - p.hasToc = true else: - for i in countup(0, len(n) - 1): n.sons[i] = resolveSubs(p, n.sons[i]) + var regroup = false + for i in 0 ..< n.len: + n.sons[i] = resolveSubs(s, n.sons[i]) + if n.sons[i] != nil and n.sons[i].kind == rnFootnote: + regroup = true + if regroup: # group footnotes together into rnFootnoteGroup + var newSons: seq[PRstNode] + var i = 0 + while i < n.len: + if n.sons[i] != nil and n.sons[i].kind == rnFootnote: + var grp = newRstNode(rnFootnoteGroup) + while i < n.len and n.sons[i].kind == rnFootnote: + grp.sons.add n.sons[i] + inc i + newSons.add grp + else: + newSons.add n.sons[i] + inc i + result.sons = newSons + +proc completePass2*(s: PRstSharedState) = + for (filename, importdocInfo) in s.idxImports.pairs: + if not importdocInfo.used: + rstMessage(s.filenames, s.msgHandler, importdocInfo.fromInfo, + mwUnusedImportdoc, filename) proc rstParse*(text, filename: string, - line, column: int, hasToc: var bool, + line, column: int, options: RstParseOptions, findFile: FindFileHandler = nil, - msgHandler: MsgHandler = nil): PRstNode = - var p: RstParser - initParser(p, newSharedState(options, findFile, msgHandler)) - p.filename = filename - p.line = line - p.col = column + getTokens(text, roSkipPounds in options, p.tok) - result = resolveSubs(p, parseDoc(p)) - hasToc = p.hasToc + findRefFile: FindRefFileHandler = nil, + msgHandler: MsgHandler = nil): + tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] = + ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, + ## note that 2nd tuple element should be fed to `initRstGenerator` + ## argument `filenames` (it is being filled here at least with `filename` + ## and possibly with other files from RST ``.. include::`` statement). + var sharedState = newRstSharedState(options, filename, findFile, findRefFile, + msgHandler, hasToc=false) + let unresolved = rstParsePass1(text, line, column, sharedState) + preparePass2(sharedState, unresolved) + result.node = resolveSubs(sharedState, unresolved) + completePass2(sharedState) + result.filenames = sharedState.filenames + result.hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 7be4470c1..2bbb0d0b8 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -9,20 +9,25 @@ ## This module implements an AST for the `reStructuredText`:idx: parser. -import strutils, json +import std/[strutils, json] + +when defined(nimPreviewSlimSystem): + import std/assertions + type RstNodeKind* = enum ## the possible node kinds of an PRstNode rnInner, # an inner node or a root rnHeadline, # a headline rnOverline, # an over- and underlined headline + rnMarkdownHeadline, # a Markdown headline rnTransition, # a transition (the ------------- <hr> thingie) rnParagraph, # a paragraph rnBulletList, # a bullet list rnBulletItem, # a bullet item rnEnumList, # an enumerated list rnEnumItem, # an enumerated item - rnDefList, # a definition list + rnDefList, rnMdDefList, # a definition list (RST/Markdown) rnDefItem, # an item of a definition list consisting of ... rnDefName, # ... a name part ... rnDefBody, # ... and a body part ... @@ -31,16 +36,28 @@ type rnFieldName, # consisting of a field name ... rnFieldBody, # ... and a field body rnOptionList, rnOptionListItem, rnOptionGroup, rnOption, rnOptionString, - rnOptionArgument, rnDescription, rnLiteralBlock, rnQuotedLiteralBlock, + rnOptionArgument, rnDescription, rnLiteralBlock, + rnMarkdownBlockQuote, # a quote starting from punctuation like >>> + rnMarkdownBlockQuoteItem, # a quotation block, quote lines starting with + # the same number of chars rnLineBlock, # the | thingie - rnLineBlockItem, # sons of the | thing + rnLineBlockItem, # a son of rnLineBlock - one line inside it. + # When `RstNode` lineIndent="\n" the line's empty rnBlockQuote, # text just indented - rnTable, rnGridTable, rnTableRow, rnTableHeaderCell, rnTableDataCell, - rnLabel, # used for footnotes and other things + rnTable, rnGridTable, rnMarkdownTable, rnTableRow, rnTableHeaderCell, rnTableDataCell, rnFootnote, # a footnote - rnCitation, # similar to footnote - rnStandaloneHyperlink, rnHyperlink, rnRef, rnDirective, # a directive - rnDirArg, rnRaw, rnTitle, rnContents, rnImage, rnFigure, rnCodeBlock, + rnCitation, # similar to footnote, so use rnFootnote instead + rnFootnoteGroup, # footnote group - exists for a purely stylistic + # reason: to display a few footnotes as 1 block + rnStandaloneHyperlink, rnHyperlink, + rnRstRef, # RST reference like `section name`_ + rnPandocRef, # Pandoc Markdown reference like [section name] + rnInternalRef, rnFootnoteRef, + rnNimdocRef, # reference to automatically generated Nim symbol + rnDirective, # a general directive + rnDirArg, # a directive argument (for some directives). + # here are directives that are not rnDirective: + rnRaw, rnTitle, rnContents, rnImage, rnFigure, rnCodeBlock, rnAdmonition, rnRawHtml, rnRawLatex, rnContainer, # ``container`` directive rnIndex, # index directve: @@ -49,48 +66,103 @@ type # * `file#id <file#id>`_ # * `file#id <file#id>'_ rnSubstitutionDef, # a definition of a substitution - rnGeneralRole, # Inline markup: + # Inline markup: + rnInlineCode, # interpreted text with code in a known language + rnCodeFragment, # inline code for highlighting with the specified + # class (which cannot be inferred from context) + rnUnknownRole, # interpreted text with an unknown role rnSub, rnSup, rnIdx, rnEmphasis, # "*" rnStrongEmphasis, # "**" rnTripleEmphasis, # "***" - rnInterpretedText, # "`" + rnInterpretedText, # "`" an auxiliary role for parsing that will + # be converted into other kinds like rnInlineCode rnInlineLiteral, # "``" + rnInlineTarget, # "_`target`" rnSubstitutionReferences, # "|" rnSmiley, # some smiley + rnDefaultRole, # .. default-role:: code rnLeaf # a leaf; the node's text field contains the # leaf val + FileIndex* = distinct int32 + TLineInfo* = object + line*: uint16 + col*: int16 + fileIndex*: FileIndex PRstNode* = ref RstNode ## an RST node RstNodeSeq* = seq[PRstNode] - RstNode* {.acyclic, final.} = object ## an RST node's description - kind*: RstNodeKind ## the node's kind - text*: string ## valid for leafs in the AST; and the title of - ## the document or the section - level*: int ## valid for some node kinds + RstNode* {.acyclic, final.} = object ## AST node (result of RST parsing) + case kind*: RstNodeKind ## the node's kind + of rnLeaf, rnSmiley: + text*: string ## string that is expected to be displayed + of rnEnumList: + labelFmt*: string ## label format like "(1)" + of rnLineBlockItem: + lineIndent*: string ## a few spaces or newline at the line beginning + of rnAdmonition: + adType*: string ## admonition type: "note", "caution", etc. This + ## text will set the style and also be displayed + of rnOverline, rnHeadline, rnMarkdownHeadline: + level*: int ## level of headings starting from 1 (main + ## chapter) to larger ones (minor sub-sections) + ## level=0 means it's document title or subtitle + of rnFootnote, rnCitation, rnOptionListItem: + order*: int ## footnote order (for auto-symbol footnotes and + ## auto-numbered ones without a label) + of rnMarkdownBlockQuoteItem: + quotationDepth*: int ## number of characters in line prefix + of rnRstRef, rnPandocRef, rnSubstitutionReferences, + rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: + info*: TLineInfo ## To have line/column info for warnings at + ## nodes that are post-processed after parsing + of rnNimdocRef: + tooltip*: string + of rnTable, rnGridTable, rnMarkdownTable: + colCount*: int ## Number of (not-united) cells in the table + of rnTableRow: + endsHeader*: bool ## Is last row in the header of table? + of rnTableHeaderCell, rnTableDataCell: + span*: int ## Number of table columns that the cell occupies + else: + discard + anchor*: string ## anchor, internal link target + ## (aka HTML id tag, aka Latex label/hypertarget) sons*: RstNodeSeq ## the node's sons -{.deprecated: [TRstNodeKind: RstNodeKind, TRstNodeSeq: RstNodeSeq, - TRstNode: RstNode].} + +proc `==`*(a, b: FileIndex): bool {.borrow.} proc len*(n: PRstNode): int = result = len(n.sons) -proc newRstNode*(kind: RstNodeKind): PRstNode = - new(result) - result.sons = @[] - result.kind = kind +proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], + anchor = ""): PRstNode = + result = PRstNode(kind: kind, sons: sons, anchor: anchor) -proc newRstNode*(kind: RstNodeKind, s: string): PRstNode = +proc newRstNode*(kind: RstNodeKind, info: TLineInfo, + sons: seq[PRstNode] = @[]): PRstNode = + result = PRstNode(kind: kind, sons: sons) + result.info = info + +proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = + assert kind in {rnLeaf, rnSmiley} result = newRstNode(kind) result.text = s +proc newRstLeaf*(s: string): PRstNode = + result = newRstNode(rnLeaf) + result.text = s + proc lastSon*(n: PRstNode): PRstNode = result = n.sons[len(n.sons)-1] proc add*(father, son: PRstNode) = add(father.sons, son) +proc add*(father: PRstNode; s: string) = + add(father.sons, newRstLeaf(s)) + proc addIfNotNil*(father, son: PRstNode) = if son != nil: add(father, son) @@ -99,7 +171,6 @@ type RenderContext {.pure.} = object indent: int verbatim: int -{.deprecated: [TRenderContext: RenderContext].} proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) {.gcsafe.} @@ -213,7 +284,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) = inc(d.indent, 2) renderRstSons(d, n, result) dec(d.indent, 2) - of rnRef: + of rnRstRef: result.add("`") renderRstSons(d, n, result) result.add("`_") @@ -223,7 +294,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) = result.add(" <") renderRstToRst(d, n.sons[1], result) result.add(">`_") - of rnGeneralRole: + of rnUnknownRole: result.add('`') renderRstToRst(d, n.sons[0],result) result.add("`:") @@ -296,9 +367,9 @@ proc renderRstToJsonNode(node: PRstNode): JsonNode = (key: "kind", val: %($node.kind)), (key: "level", val: %BiggestInt(node.level)) ] - if node.text != nil: + if node.kind in {rnLeaf, rnSmiley} and node.text.len > 0: result.add("text", %node.text) - if node.sons != nil and len(node.sons) > 0: + if len(node.sons) > 0: var accm = newSeq[JsonNode](len(node.sons)) for i, son in node.sons: accm[i] = renderRstToJsonNode(son) @@ -306,11 +377,68 @@ proc renderRstToJsonNode(node: PRstNode): JsonNode = proc renderRstToJson*(node: PRstNode): string = ## Writes the given RST node as JSON that is in the form - ## :: - ## { - ## "kind":string node.kind, - ## "text":optional string node.text, - ## "level":optional int node.level, - ## "sons":optional node array - ## } + ## + ## { + ## "kind":string node.kind, + ## "text":optional string node.text, + ## "level":optional int node.level, + ## "sons":optional node array + ## } renderRstToJsonNode(node).pretty + +proc renderRstToText*(node: PRstNode): string = + ## minimal text representation of markup node + const code = {rnCodeFragment, rnInterpretedText, rnInlineLiteral, rnInlineCode} + if node == nil: + return "" + case node.kind + of rnLeaf, rnSmiley: + result.add node.text + else: + if node.kind in code: result.add "`" + for i in 0 ..< node.sons.len: + if node.kind in {rnInlineCode, rnCodeBlock} and i == 0: + continue # omit language specifier + result.add renderRstToText(node.sons[i]) + if node.kind in code: result.add "`" + +proc treeRepr*(node: PRstNode, indent=0): string = + ## Writes the parsed RST `node` into an AST tree with compact string + ## representation in the format (one line per every sub-node): + ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)`` + ## (suitable for debugging of RST parsing). + if node == nil: + result.add " ".repeat(indent) & "[nil]\n" + return + result.add " ".repeat(indent) & $node.kind + case node.kind + of rnLeaf, rnSmiley: + result.add (if node.text == "": "" else: " '" & node.text & "'") + of rnEnumList: + result.add " labelFmt=" & node.labelFmt + of rnLineBlockItem: + var txt: string + if node.lineIndent == "\n": txt = " (blank line)" + else: txt = " lineIndent=" & $node.lineIndent.len + result.add txt + of rnAdmonition: + result.add " adType=" & node.adType + of rnHeadline, rnOverline, rnMarkdownHeadline: + result.add " level=" & $node.level + of rnFootnote, rnCitation, rnOptionListItem: + result.add (if node.order == 0: "" else: " order=" & $node.order) + of rnMarkdownBlockQuoteItem: + result.add " quotationDepth=" & $node.quotationDepth + of rnTable, rnGridTable, rnMarkdownTable: + result.add " colCount=" & $node.colCount + of rnTableHeaderCell, rnTableDataCell: + if node.span > 0: + result.add " span=" & $node.span + of rnTableRow: + if node.endsHeader: result.add " endsHeader" + else: + discard + result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") + result.add "\n" + for son in node.sons: + result.add treeRepr(son, indent=indent+2) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 4a0304a7c..7fc0ac03a 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -22,9 +22,33 @@ ## other lower level methods to finally build complete documents. This requires ## many options and tweaking, but you are not limited to snippets and can ## generate `LaTeX documents <https://en.wikipedia.org/wiki/LaTeX>`_ too. +## +## `Docutils configuration files`_ are not supported. Instead HTML generation +## can be tweaked by editing file ``config/nimdoc.cfg``. +## +## .. _Docutils configuration files: https://docutils.sourceforge.io/docs/user/config.htm +## +## There are stylistic difference between how this module renders some elements +## and how original Python Docutils does: +## +## * Backreferences to TOC in section headings are not generated. +## In HTML each section is also a link that points to the section itself: +## this is done for user to be able to copy the link into clipboard. +## +## * The same goes for footnotes/citations links: they point to themselves. +## No backreferences are generated since finding all references of a footnote +## can be done by simply searching for ``[footnoteName]``. + +import std/[strutils, os, hashes, strtabs, tables, sequtils, + algorithm, parseutils, strbasics] + +import rstast, rst, rstidx, highlite -import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio, formatfloat] + + +import ../../std/private/since const HtmlExt = "html" @@ -35,25 +59,27 @@ type outHtml, # output is HTML outLatex # output is Latex - TocEntry = object - n*: PRstNode - refname*, header*: string - MetaEnum* = enum - metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion + metaNone, metaTitleRaw, metaTitle, metaSubtitle, metaAuthor, metaVersion + + EscapeMode* = enum # in Latex text inside options [] and URLs is + # escaped slightly differently than in normal text + emText, emOption, emUrl # emText is currently used for code also RstGenerator* = object of RootObj target*: OutputTarget config*: StringTableRef splitAfter*: int # split too long entries in the TOC - listingCounter: int - tocPart*: seq[TocEntry] + listingCounter*: int + tocPart*: seq[PRstNode] # headings for Table of Contents hasToc*: bool theIndex: string # Contents of the index file to be dumped at the end. - options*: RstParseOptions findFile*: FindFileHandler msgHandler*: MsgHandler - filename*: string + outDir*: string ## output directory, initialized by docgen.nim + destFile*: string ## output (HTML) file, initialized by docgen.nim + filenames*: RstFileTable + filename*: string ## source Nim or Rst file meta*: array[MetaEnum, string] currentSection: string ## \ ## Stores the empty string or the last headline/overline found in the rst @@ -61,6 +87,11 @@ type seenIndexTerms: Table[string, int] ## \ ## Keeps count of same text index terms to generate different identifiers ## for hyperlinks. See renderIndexTerm proc for details. + id*: int ## A counter useful for generating IDs. + onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int; + content: string) {.gcsafe.} + escMode*: EscapeMode + curQuotationDepth: int PDoc = var RstGenerator ## Alias to type less. @@ -69,8 +100,12 @@ type startLine: int ## The starting line of the code block, by default 1. langStr: string ## Input string used to specify the language. lang: SourceLanguage ## Type of highlighting, by default none. -{.deprecated: [TRstGenerator: RstGenerator, TTocEntry: TocEntry, - TOutputTarget: OutputTarget, TMetaEnum: MetaEnum].} + filename: string + testCmd: string + status: int + +proc prettyLink*(file: string): string = + changeFileExt(file, "").replace("_._", "..") proc init(p: var CodeBlockParams) = ## Default initialisation of CodeBlockParams to sane values. @@ -80,9 +115,10 @@ proc init(p: var CodeBlockParams) = proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, config: StringTableRef, filename: string, - options: RstParseOptions, - findFile: FindFileHandler=nil, - msgHandler: MsgHandler=nil) = + findFile: FindFileHandler = nil, + msgHandler: MsgHandler = nil, + filenames = default(RstFileTable), + hasToc = false) = ## Initializes a ``RstGenerator``. ## ## You need to call this before using a ``RstGenerator`` with any other @@ -95,11 +131,11 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, ## index hyperlinks to the file, but you can pass an empty string here if you ## are parsing a stream in memory. If `filename` ends with the ``.nim`` ## extension, the title for the document will be set by default to ``Module - ## filename``. This default title can be overriden by the embedded rst, but + ## filename``. This default title can be overridden by the embedded rst, but ## it helps to prettify the generated index if no title is found. ## ## The ``RstParseOptions``, ``FindFileHandler`` and ``MsgHandler`` types - ## are defined in the the `packages/docutils/rst module <rst.html>`_. + ## are defined in the `packages/docutils/rst module <rst.html>`_. ## ``options`` selects the behaviour of the rst parser. ## ## ``findFile`` is a proc used by the rst ``include`` directive among others. @@ -118,21 +154,25 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, ## ## Example: ## - ## .. code-block:: nim - ## + ## ```nim ## import packages/docutils/rstgen ## ## var gen: RstGenerator ## gen.initRstGenerator(outHtml, defaultConfig(), "filename", {}) + ## ``` g.config = config g.target = target g.tocPart = @[] + g.hasToc = hasToc g.filename = filename + g.filenames = filenames g.splitAfter = 20 g.theIndex = "" - g.options = options g.findFile = findFile g.currentSection = "" + g.id = 0 + g.escMode = emText + g.curQuotationDepth = 0 let fileParts = filename.splitFile if fileParts.ext == ".nim": g.currentSection = "Module " & fileParts.name @@ -147,10 +187,13 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) = ## Writes the current index buffer to the specified output file. ## ## You previously need to add entries to the index with the `setIndexTerm() - ## <#setIndexTerm>`_ proc. If the index is empty the file won't be created. + ## <#setIndexTerm,RstGenerator,string,string,string,string,string>`_ proc. + ## If the index is empty the file won't be created. if g.theIndex.len > 0: writeFile(outfile, g.theIndex) -proc addXmlChar(dest: var string, c: char) = +proc addHtmlChar(dest: var string, c: char) = + # Escapes HTML characters. Note that single quote ' is not escaped as + # ' -- unlike XML (for standards pre HTML5 it was even forbidden). case c of '&': add(dest, "&") of '<': add(dest, "<") @@ -158,37 +201,41 @@ proc addXmlChar(dest: var string, c: char) = of '\"': add(dest, """) else: add(dest, c) -proc addRtfChar(dest: var string, c: char) = +proc addTexChar(dest: var string, c: char, escMode: EscapeMode) = + ## Escapes 10 special Latex characters and sometimes ` and [, ]. + ## TODO: @ is always a normal symbol (besides the header), am I wrong? + ## All escapes that need to work in text and code blocks (`emText` mode) + ## should start from \ (to be compatible with fancyvrb/fvextra). case c - of '{': add(dest, "\\{") - of '}': add(dest, "\\}") - of '\\': add(dest, "\\\\") + of '_', '&', '#', '%': add(dest, "\\" & c) + # commands \label and \pageref don't accept \$ by some reason but OK with $: + of '$': (if escMode == emUrl: add(dest, c) else: add(dest, "\\" & c)) + # \~ and \^ have a special meaning unless they are followed by {} + of '~', '^': add(dest, "\\" & c & "{}") + # Latex loves to substitute ` to opening quote, even in texttt mode! + of '`': add(dest, "\\textasciigrave{}") + # add {} to avoid gobbling up space by \textbackslash + of '\\': add(dest, "\\textbackslash{}") + # Using { and } in URL in Latex: https://tex.stackexchange.com/a/469175 + of '{': + add(dest, if escMode == emUrl: "\\%7B" else: "\\{") + of '}': + add(dest, if escMode == emUrl: "\\%7D" else: "\\}") + of ']': + # escape ] inside an optional argument in e.g. \section[static[T]]{.. + add(dest, if escMode == emOption: "\\text{]}" else: "]") else: add(dest, c) -proc addTexChar(dest: var string, c: char) = - case c - of '_': add(dest, "\\_") - of '{': add(dest, "\\symbol{123}") - of '}': add(dest, "\\symbol{125}") - of '[': add(dest, "\\symbol{91}") - of ']': add(dest, "\\symbol{93}") - of '\\': add(dest, "\\symbol{92}") - of '$': add(dest, "\\$") - of '&': add(dest, "\\&") - of '#': add(dest, "\\#") - of '%': add(dest, "\\%") - of '~': add(dest, "\\symbol{126}") - of '@': add(dest, "\\symbol{64}") - of '^': add(dest, "\\symbol{94}") - of '`': add(dest, "\\symbol{96}") - else: add(dest, c) - -var splitter*: string = "<wbr />" +proc escChar*(target: OutputTarget, dest: var string, + c: char, escMode: EscapeMode) {.inline.} = + case target + of outHtml: addHtmlChar(dest, c) + of outLatex: addTexChar(dest, c, escMode) -proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} = +proc addSplitter(target: OutputTarget; dest: var string) {.inline.} = case target - of outHtml: addXmlChar(dest, c) - of outLatex: addTexChar(dest, c) + of outHtml: add(dest, "<wbr />") + of outLatex: add(dest, "\\-") proc nextSplitPoint*(s: string, start: int): int = result = start @@ -202,21 +249,22 @@ proc nextSplitPoint*(s: string, start: int): int = inc(result) dec(result) # last valid index -proc esc*(target: OutputTarget, s: string, splitAfter = -1): string = +proc esc*(target: OutputTarget, s: string, splitAfter = -1, escMode = emText): string = + ## Escapes the HTML. result = "" if splitAfter >= 0: var partLen = 0 var j = 0 while j < len(s): var k = nextSplitPoint(s, j) - if (splitter != " ") or (partLen + k - j + 1 > splitAfter): - partLen = 0 - add(result, splitter) - for i in countup(j, k): escChar(target, result, s[i]) + #if (splitter != " ") or (partLen + k - j + 1 > splitAfter): + partLen = 0 + addSplitter(target, result) + for i in countup(j, k): escChar(target, result, s[i], escMode) inc(partLen, k - j + 1) j = k + 1 else: - for i in countup(0, len(s) - 1): escChar(target, result, s[i]) + for i in countup(0, len(s) - 1): escChar(target, result, s[i], escMode) proc disp(target: OutputTarget, xml, tex: string): string = @@ -234,64 +282,58 @@ proc dispA(target: OutputTarget, dest: var string, else: addf(dest, tex, args) proc `or`(x, y: string): string {.inline.} = - result = if x.isNil: y else: x + result = if x.len == 0: y else: x -proc renderRstToOut*(d: var RstGenerator, n: PRstNode, result: var string) +proc renderRstToOut*(d: var RstGenerator, n: PRstNode, result: var string) {.gcsafe.} ## Writes into ``result`` the rst ast ``n`` using the ``d`` configuration. ## ## Before using this proc you need to initialise a ``RstGenerator`` with ## ``initRstGenerator`` and parse a rst file with ``rstParse`` from the ## `packages/docutils/rst module <rst.html>`_. Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## # ...configure gen and rst vars... - ## var generatedHTML = "" - ## renderRstToOut(gen, rst, generatedHTML) - ## echo generatedHTML + ## var generatedHtml = "" + ## renderRstToOut(gen, rst, generatedHtml) + ## echo generatedHtml + ## ``` proc renderAux(d: PDoc, n: PRstNode, result: var string) = for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], result) -proc renderAux(d: PDoc, n: PRstNode, frmtA, frmtB: string, result: var string) = +template idS(txt: string): string = + if txt == "": "" + else: + case d.target + of outHtml: + " id=\"" & txt & "\"" + of outLatex: + "\\label{" & txt & "}\\hypertarget{" & txt & "}{}" + # we add \label for page number references via \pageref, while + # \hypertarget is for clickable links via \hyperlink. + +proc renderAux(d: PDoc, n: PRstNode, html, tex: string, result: var string) = + # formats sons of `n` as substitution variable $1 inside strings `html` and + # `tex`, internal target (anchor) is provided as substitute $2. var tmp = "" for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], tmp) - if d.target != outLatex: - result.addf(frmtA, [tmp]) - else: - result.addf(frmtB, [tmp]) + case d.target + of outHtml: result.addf(html, [tmp, n.anchor.idS]) + of outLatex: result.addf(tex, [tmp, n.anchor.idS]) # ---------------- index handling -------------------------------------------- -proc quoteIndexColumn(text: string): string = - ## Returns a safe version of `text` for serialization to the ``.idx`` file. - ## - ## The returned version can be put without worries in a line based tab - ## separated column text file. The following character sequence replacements - ## will be performed for that goal: - ## - ## * ``"\\"`` => ``"\\\\"`` - ## * ``"\n"`` => ``"\\n"`` - ## * ``"\t"`` => ``"\\t"`` - result = text.replace("\\", "\\\\").replace("\n", "\\n").replace("\t", "\\t") - -proc unquoteIndexColumn(text: string): string = - ## Returns the unquoted version generated by ``quoteIndexColumn``. - result = text.replace("\\t", "\t").replace("\\n", "\n").replace("\\\\", "\\") - -proc setIndexTerm*(d: var RstGenerator, id, term: string, - linkTitle, linkDesc = "") = +proc setIndexTerm*(d: var RstGenerator; k: IndexEntryKind, htmlFile, id, term: string, + linkTitle, linkDesc = "", line = 0) = ## Adds a `term` to the index using the specified hyperlink identifier. ## ## A new entry will be added to the index using the format - ## ``term<tab>file#id``. The file part will come from the `filename` - ## parameter used in a previous call to the `initRstGenerator() - ## <#initRstGenerator>`_ proc. + ## ``term<tab>file#id``. The file part will come from the `htmlFile` + ## parameter. ## ## The `id` will be appended with a hash character only if its length is not ## zero, otherwise no specific anchor will be generated. In general you ## should only pass an empty `id` value for the title of standalone rst - ## documents (they are special for the `mergeIndexes() <#mergeIndexes>`_ + ## documents (they are special for the `mergeIndexes() <#mergeIndexes,string>`_ ## proc, see `Index (idx) file format <docgen.html#index-idx-file-format>`_ ## for more information). Unlike other index terms, title entries are ## inserted at the beginning of the accumulated buffer to maintain a logical @@ -301,25 +343,11 @@ proc setIndexTerm*(d: var RstGenerator, id, term: string, ## columns with their contents will be added. ## ## The index won't be written to disk unless you call `writeIndexFile() - ## <#writeIndexFile>`_. The purpose of the index is documented in the `docgen - ## tools guide <docgen.html#index-switch>`_. - assert(not d.theIndex.isNil) - var - entry = term - isTitle = false - entry.add('\t') - let htmlFile = changeFileExt(extractFilename(d.filename), HtmlExt) - entry.add(htmlFile) - if id.len > 0: - entry.add('#') - entry.add(id) - else: - isTitle = true - if linkTitle.len > 0 or linkDesc.len > 0: - entry.add('\t' & linkTitle.quoteIndexColumn) - entry.add('\t' & linkDesc.quoteIndexColumn) - entry.add("\n") - + ## <#writeIndexFile,RstGenerator,string>`_. The purpose of the index is + ## documented in the `docgen tools guide + ## <docgen.html#related-options-index-switch>`_. + let (entry, isTitle) = formatIndexEntry(k, htmlFile, id, term, + linkTitle, linkDesc, line) if isTitle: d.theIndex.insert(entry) else: d.theIndex.add(entry) @@ -328,14 +356,23 @@ proc hash(n: PRstNode): int = result = hash(n.text) elif n.len > 0: result = hash(n.sons[0]) - for i in 1 .. <len(n): + for i in 1 ..< len(n): result = result !& hash(n.sons[i]) result = !$result +proc htmlFileRelPath(d: PDoc): string = + if d.outDir.len == 0: + # /foo/bar/zoo.nim -> zoo.html + changeFileExt(extractFilename(d.filename), HtmlExt) + else: # d is initialized in docgen.nim + # outDir = /foo -\ + # destFile = /foo/bar/zoo.html -|-> bar/zoo.html + d.destFile.relativePath(d.outDir, '/') + proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = ## Renders the string decorated within \`foobar\`\:idx\: markers. ## - ## Additionally adds the encosed text to the index as a term. Since we are + ## Additionally adds the enclosed text to the index as a term. Since we are ## interested in different instances of the same term to have different ## entries, a table is used to keep track of the amount of times a term has ## previously appeared to give a different identifier value for each. @@ -348,52 +385,31 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) - setIndexTerm(d, id, term, d.currentSection) - dispA(d.target, result, "<span id=\"$1\">$2</span>", "$2\\label{$1}", + setIndexTerm(d, ieIdxRole, + htmlFileRelPath(d), id, term, d.currentSection) + dispA(d.target, result, "<span id=\"$1\">$2</span>", "\\nimindexterm{$1}{$2}", [id, term]) type - IndexEntry = object - keyword: string - link: string - linkTitle: string ## If not nil, contains a prettier text for the href - linkDesc: string ## If not nil, the title attribute of the final href - - IndexedDocs = Table[IndexEntry, seq[IndexEntry]] ## \ + IndexedDocs* = Table[IndexEntry, seq[IndexEntry]] ## \ ## Contains the index sequences for doc types. ## ## The key is a *fake* IndexEntry which will contain the title of the ## document in the `keyword` field and `link` will contain the html - ## filename for the document. `linkTitle` and `linkDesc` will be nil. + ## filename for the document. `linkTitle` and `linkDesc` will be empty. ## ## The value indexed by this IndexEntry is a sequence with the real index ## entries found in the ``.idx`` file. -{.deprecated: [TIndexEntry: IndexEntry, TIndexedDocs: IndexedDocs].} -proc cmp(a, b: IndexEntry): int = - ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. - result = cmpIgnoreStyle(a.keyword, b.keyword) - if result == 0: - result = cmpIgnoreStyle(a.link, b.link) - -proc hash(x: IndexEntry): Hash = - ## Returns the hash for the combined fields of the type. - ## - ## The hash is computed as the chained hash of the individual string hashes. - assert(not x.keyword.isNil) - assert(not x.link.isNil) - result = x.keyword.hash !& x.link.hash - result = result !& (x.linkTitle or "").hash - result = result !& (x.linkDesc or "").hash - result = !$result - -proc `<-`(a: var IndexEntry, b: IndexEntry) = - shallowCopy a.keyword, b.keyword - shallowCopy a.link, b.link - if b.linkTitle.isNil: a.linkTitle = nil - else: shallowCopy a.linkTitle, b.linkTitle - if b.linkDesc.isNil: a.linkDesc = nil - else: shallowCopy a.linkDesc, b.linkDesc +when defined(gcDestructors): + template `<-`(a, b: var IndexEntry) = a = move(b) +else: + proc `<-`(a: var IndexEntry, b: IndexEntry) = + shallowCopy a.keyword, b.keyword + shallowCopy a.link, b.link + shallowCopy a.linkTitle, b.linkTitle + shallowCopy a.linkDesc, b.linkDesc + shallowCopy a.module, b.module proc sortIndex(a: var openArray[IndexEntry]) = # we use shellsort here; fast and simple @@ -415,44 +431,57 @@ proc sortIndex(a: var openArray[IndexEntry]) = a[j] <- v if h == 1: break +proc escapeLink(s: string): string = + ## This proc is mostly copied from uri/encodeUrl except that + ## these chars are also left unencoded: '#', '/'. + result = newStringOfCap(s.len + s.len shr 2) + for c in items(s): + case c + of 'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~': # same as that in uri/encodeUrl + add(result, c) + of '#', '/': # example.com/foo/#bar (don't escape the '/' and '#' in such links) + add(result, c) + else: + add(result, "%") + add(result, toHex(ord(c), 2)) + proc generateSymbolIndex(symbols: seq[IndexEntry]): string = - result = "" + result = "<dl>" var i = 0 while i < symbols.len: - let keyword= symbols[i].keyword - let cleaned_keyword = keyword[1..keyword.high - 1] - result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><ul class=\"simple\"><dd>\n", - [keyword, cleaned_keyword]) + let keyword = esc(outHtml, symbols[i].keyword) + let cleanedKeyword = keyword.escapeLink + result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><dd><ul class=\"simple\">\n", + [keyword, cleanedKeyword]) var j = i - while j < symbols.len and keyword == symbols[j].keyword: + while j < symbols.len and symbols[i].keyword == symbols[j].keyword: let - url = symbols[j].link - text = if not symbols[j].linkTitle.isNil: symbols[j].linkTitle else: url - desc = if not symbols[j].linkDesc.isNil: symbols[j].linkDesc else: "" + url = symbols[j].link.escapeLink + module = symbols[j].module + text = + if symbols[j].linkTitle.len > 0: + esc(outHtml, module & ": " & symbols[j].linkTitle) + else: url + desc = symbols[j].linkDesc if desc.len > 0: result.addf("""<li><a class="reference external" - title="$3" href="$1">$2</a></li> + title="$3" data-doc-search-tag="$2" href="$1">$2</a></li> """, [url, text, desc]) else: - result.addf("""<li><a class="reference external" href="$1">$2</a></li> + result.addf("""<li><a class="reference external" + data-doc-search-tag="$2" href="$1">$2</a></li> """, [url, text]) inc j result.add("</ul></dd>\n") i = j + result.add("</dl>") -proc isDocumentationTitle(hyperlink: string): bool = - ## Returns true if the hyperlink is actually a documentation title. - ## - ## Documentation titles lack the hash. See `mergeIndexes() <#mergeIndexes>`_ - ## for a more detailed explanation. - result = hyperlink.find('#') < 0 - -proc stripTOCLevel(s: string): tuple[level: int, text: string] = +proc stripTocLevel(s: string): tuple[level: int, text: string] = ## Returns the *level* of the toc along with the text without it. - for c in 0 .. <s.len: + for c in 0 ..< s.len: result.level = c if s[c] != ' ': break - result.text = s[result.level .. <s.len] + result.text = s[result.level ..< s.len] proc indentToLevel(level: var int, newLevel: int): string = ## Returns the sequence of <ul>|</ul> characters to switch to `newLevel`. @@ -463,33 +492,32 @@ proc indentToLevel(level: var int, newLevel: int): string = if level == newLevel: return if newLevel > level: - result = repeat("<ul>", newLevel - level) + result = repeat("<li><ul>", newLevel - level) else: - result = repeat("</ul>", level - newLevel) + result = repeat("</ul></li>", level - newLevel) level = newLevel -proc generateDocumentationTOC(entries: seq[IndexEntry]): string = +proc generateDocumentationToc(entries: seq[IndexEntry]): string = ## Returns the sequence of index entries in an HTML hierarchical list. result = "" # Build a list of levels and extracted titles to make processing easier. var titleRef: string + titleTag: string levels: seq[tuple[level: int, text: string]] L = 0 level = 1 levels.newSeq(entries.len) for entry in entries: - let (rawLevel, rawText) = stripTOCLevel(entry.linkTitle or entry.keyword) + let (rawLevel, rawText) = stripTocLevel(entry.linkTitle) if rawLevel < 1: # This is a normal symbol, push it *inside* one level from the last one. levels[L].level = level + 1 - # Also, ignore the linkTitle and use directly the keyword. - levels[L].text = entry.keyword else: # The level did change, update the level indicator. level = rawLevel levels[L].level = rawLevel - levels[L].text = rawText + levels[L].text = rawText inc L # Now generate hierarchical lists based on the precalculated levels. @@ -500,14 +528,14 @@ proc generateDocumentationTOC(entries: seq[IndexEntry]): string = let link = entries[L].link if link.isDocumentationTitle: titleRef = link + titleTag = levels[L].text else: result.add(level.indentToLevel(levels[L].level)) - result.add("<li><a href=\"" & link & "\">" & - levels[L].text & "</a>\n") + result.addf("""<li><a class="reference" data-doc-search-tag="$1: $2" href="$3"> + $3</a></li> + """, [titleTag, levels[L].text, link, levels[L].text]) inc L result.add(level.indentToLevel(1) & "</ul>\n") - assert(not titleRef.isNil, - "Can't use this proc on an API index, docs always have a title entry") proc generateDocumentationIndex(docs: IndexedDocs): string = ## Returns all the documentation TOCs in an HTML hierarchical list. @@ -518,9 +546,9 @@ proc generateDocumentationIndex(docs: IndexedDocs): string = sort(titles, cmp) for title in titles: - let tocList = generateDocumentationTOC(docs.getOrDefault(title)) + let tocList = generateDocumentationToc(docs.getOrDefault(title)) result.add("<ul><li><a href=\"" & - title.link & "\">" & title.keyword & "</a>\n" & tocList & "</ul>\n") + title.link & "\">" & title.linkTitle & "</a>\n" & tocList & "</li></ul>\n") proc generateDocumentationJumps(docs: IndexedDocs): string = ## Returns a plain list of hyperlinks to documentation TOCs in HTML. @@ -532,9 +560,9 @@ proc generateDocumentationJumps(docs: IndexedDocs): string = var chunks: seq[string] = @[] for title in titles: - chunks.add("<a href=\"" & title.link & "\">" & title.keyword & "</a>") + chunks.add("<a href=\"" & title.link & "\">" & title.linkTitle & "</a>") - result.add(chunks.join(", ") & ".<br>") + result.add(chunks.join(", ") & ".<br/>") proc generateModuleJumps(modules: seq[string]): string = ## Returns a plain list of hyperlinks to the list of modules. @@ -542,11 +570,11 @@ proc generateModuleJumps(modules: seq[string]): string = var chunks: seq[string] = @[] for name in modules: - chunks.add("<a href=\"" & name & ".html\">" & name & "</a>") + chunks.add("<a href=\"$1.html\">$2</a>" % [name, name.prettyLink]) - result.add(chunks.join(", ") & ".<br>") + result.add(chunks.join(", ") & ".<br/>") -proc readIndexDir(dir: string): +proc readIndexDir*(dir: string): tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] = ## Walks `dir` reading ``.idx`` files converting them in IndexEntry items. ## @@ -559,53 +587,39 @@ proc readIndexDir(dir: string): setLen(result.symbols, 0) var L = 0 # Scan index files and build the list of symbols. - for kind, path in walkDir(dir): - if kind == pcFile and path.endsWith(IndexExt): - var - fileEntries: seq[IndexEntry] - title: IndexEntry - F = 0 - newSeq(fileEntries, 500) - setLen(fileEntries, 0) - for line in lines(path): - let s = line.find('\t') - if s < 0: continue - setLen(fileEntries, F+1) - fileEntries[F].keyword = line.substr(0, s-1) - fileEntries[F].link = line.substr(s+1) - # See if we detect a title, a link without a `#foobar` trailing part. - if title.keyword.isNil and fileEntries[F].link.isDocumentationTitle: - title.keyword = fileEntries[F].keyword - title.link = fileEntries[F].link - - if fileEntries[F].link.find('\t') > 0: - let extraCols = fileEntries[F].link.split('\t') - fileEntries[F].link = extraCols[0] - assert extraCols.len == 3 - fileEntries[F].linkTitle = extraCols[1].unquoteIndexColumn - fileEntries[F].linkDesc = extraCols[2].unquoteIndexColumn - else: - fileEntries[F].linkTitle = nil - fileEntries[F].linkDesc = nil - inc F + for path in walkDirRec(dir): + if path.endsWith(IndexExt): + var (fileEntries, title) = parseIdxFile(path) # Depending on type add this to the list of symbols or table of APIs. - if title.keyword.isNil: - for i in 0 .. <F: - # Don't add to symbols TOC entries (they start with a whitespace). - let toc = fileEntries[i].linkTitle - if not toc.isNil and toc.len > 0 and toc[0] == ' ': + + if title.kind == ieNimTitle: + for i in 0 ..< fileEntries.len: + if fileEntries[i].kind != ieNim: continue # Ok, non TOC entry, add it. setLen(result.symbols, L + 1) result.symbols[L] = fileEntries[i] inc L - result.modules.add(path.splitFile.name) + if fileEntries.len > 0: + var x = fileEntries[0].link + let i = find(x, '#') + if i > 0: + x.setLen(i) + if i != 0: + # don't add entries starting with '#' + result.modules.add(x.changeFileExt("")) else: # Generate the symbolic anchor for index quickjumps. - title.linkTitle = "doc_toc_" & $result.docs.len + title.aux = "doc_toc_" & $result.docs.len result.docs[title] = fileEntries - sort(result.modules, system.cmp) + for i in 0 ..< fileEntries.len: + if fileEntries[i].kind != ieIdxRole: + continue + + setLen(result.symbols, L + 1) + result.symbols[L] = fileEntries[i] + inc L proc mergeIndexes*(dir: string): string = ## Merges all index files in `dir` and returns the generated index as HTML. @@ -613,13 +627,15 @@ proc mergeIndexes*(dir: string): string = ## This proc will first scan `dir` for index files with the ``.idx`` ## extension previously created by commands like ``nim doc|rst2html`` ## which use the ``--index:on`` switch. These index files are the result of - ## calls to `setIndexTerm() <#setIndexTerm>`_ and `writeIndexFile() - ## <#writeIndexFile>`_, so they are simple tab separated files. + ## calls to `setIndexTerm() + ## <#setIndexTerm,RstGenerator,string,string,string,string,string>`_ + ## and `writeIndexFile() <#writeIndexFile,RstGenerator,string>`_, so they are + ## simple tab separated files. ## ## As convention this proc will split index files into two categories: ## documentation and API. API indices will be all joined together into a ## single big sorted index, making the bulk of the final index. This is good - ## for API documentation because many symbols are repated in different + ## for API documentation because many symbols are repeated in different ## modules. On the other hand, documentation indices are essentially table of ## contents plus a few special markers. These documents will be rendered in a ## separate section which tries to maintain the order and hierarchy of the @@ -634,7 +650,7 @@ proc mergeIndexes*(dir: string): string = ## Returns the merged and sorted indices into a single HTML block which can ## be further embedded into nimdoc templates. var (modules, symbols, docs) = readIndexDir(dir) - assert(not symbols.isNil) + sort(modules, system.cmp) result = "" # Generate a quick jump list of documents. @@ -647,10 +663,11 @@ proc mergeIndexes*(dir: string): string = result.add(generateModuleJumps(modules)) result.add("<p />") - # Generate the HTML block with API documents. - if docs.len > 0: - result.add("<h2>Documentation files</h2>\n") - result.add(generateDocumentationIndex(docs)) + when false: + # Generate the HTML block with API documents. + if docs.len > 0: + result.add("<h2>Documentation files</h2>\n") + result.add(generateDocumentationIndex(docs)) # Generate the HTML block with symbols. if symbols.len > 0: @@ -661,63 +678,36 @@ proc mergeIndexes*(dir: string): string = # ---------------------------------------------------------------------------- -proc stripTOCHTML(s: string): string = - ## Ugly quick hack to remove HTML tags from TOC titles. - ## - ## A TocEntry.header field already contains rendered HTML tags. Instead of - ## implementing a proper version of renderRstToOut() which recursively - ## renders an rst tree to plain text, we simply remove text found between - ## angled brackets. Given the limited possibilities of rst inside TOC titles - ## this should be enough. - result = s - var first = result.find('<') - while first >= 0: - let last = result.find('>', first) - if last < 0: - # Abort, since we didn't found a closing angled bracket. - return - result.delete(first, last) - first = result.find('<', first) - proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) d.currentSection = tmp - # Find the last higher level section for unique reference name - var sectionPrefix = "" - for i in countdown(d.tocPart.high, 0): - let n2 = d.tocPart[i].n - if n2.level < n.level: - sectionPrefix = rstnodeToRefname(n2) & "-" - break - var refname = sectionPrefix & rstnodeToRefname(n) + var tocName = esc(d.target, renderRstToText(n), escMode = emOption) + # for Latex: simple text without commands that may break TOC/hyperref if d.hasToc: - var length = len(d.tocPart) - setLen(d.tocPart, length + 1) - d.tocPart[length].refname = refname - d.tocPart[length].n = n - d.tocPart[length].header = tmp - - dispA(d.target, result, "\n<h$1><a class=\"toc-backref\" " & - "id=\"$2\" href=\"#$2\">$3</a></h$1>", "\\rsth$4{$3}\\label{$2}\n", - [$n.level, d.tocPart[length].refname, tmp, $chr(n.level - 1 + ord('A'))]) + d.tocPart.add n + dispA(d.target, result, "\n<h$1><a class=\"toc-backref\"" & + "$2 href=\"#$5\">$3</a></h$1>", "\\rsth$4[$6]{$3}$2\n", + [$n.level, n.anchor.idS, tmp, + $chr(n.level - 1 + ord('A')), n.anchor, tocName]) else: - dispA(d.target, result, "\n<h$1 id=\"$2\">$3</h$1>", - "\\rsth$4{$3}\\label{$2}\n", [ - $n.level, refname, tmp, - $chr(n.level - 1 + ord('A'))]) + dispA(d.target, result, "\n<h$1$2>$3</h$1>", + "\\rsth$4[$5]{$3}$2\n", [ + $n.level, n.anchor.idS, tmp, + $chr(n.level - 1 + ord('A')), tocName]) # Generate index entry using spaces to indicate TOC level for the output HTML. assert n.level >= 0 - setIndexTerm(d, refname, tmp.stripTOCHTML, - spaces(max(0, n.level)) & tmp) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) proc renderOverline(d: PDoc, n: PRstNode, result: var string) = - if d.meta[metaTitle].len == 0: + if n.level == 0 and d.meta[metaTitle].len == 0: + d.meta[metaTitleRaw] = n.addNodes for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], d.meta[metaTitle]) d.currentSection = d.meta[metaTitle] - elif d.meta[metaSubtitle].len == 0: + elif n.level == 0 and d.meta[metaSubtitle].len == 0: for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], d.meta[metaSubtitle]) d.currentSection = d.meta[metaSubtitle] @@ -725,21 +715,25 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) d.currentSection = tmp - dispA(d.target, result, "<h$1 id=\"$2\"><center>$3</center></h$1>", - "\\rstov$4{$3}\\label{$2}\n", [$n.level, - rstnodeToRefname(n), tmp, $chr(n.level - 1 + ord('A'))]) - - -proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) = + var tocName = esc(d.target, renderRstToText(n), escMode=emOption) + dispA(d.target, result, "<h$1$2><center>$3</center></h$1>", + "\\rstov$4[$5]{$3}$2\n", [$n.level, + n.anchor.idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) + +proc renderTocEntry(d: PDoc, n: PRstNode, result: var string) = + var header = "" + for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], header) dispA(d.target, result, "<li><a class=\"reference\" id=\"$1_toc\" href=\"#$1\">$2</a></li>\n", - "\\item\\label{$1_toc} $2\\ref{$1}\n", [e.refname, e.header]) + "\\item\\label{$1_toc} $2\\ref{$1}\n", [n.anchor, header]) proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int, result: var string) = var tmp = "" while j <= high(d.tocPart): - var a = abs(d.tocPart[j].n.level) + var a = abs(d.tocPart[j].level) if a == lvl: renderTocEntry(d, d.tocPart[j], tmp) inc(j) @@ -754,32 +748,59 @@ proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int, result.add(tmp) proc renderImage(d: PDoc, n: PRstNode, result: var string) = - template valid(s): expr = - s.len > 0 and allCharsInSet(s, {'.','/',':','%','_','\\','\128'..'\xFF'} + - Digits + Letters + WhiteSpace) + let + arg = getArgument(n) + var + options = "" - var options = "" - var s = getFieldValue(n, "scale") - if s.valid: dispA(d.target, options, " scale=\"$1\"", " scale=$1", [strip(s)]) + var s = esc(d.target, getFieldValue(n, "scale").strip()) + if s.len > 0: + dispA(d.target, options, " scale=\"$1\"", " scale=$1", [s]) - s = getFieldValue(n, "height") - if s.valid: dispA(d.target, options, " height=\"$1\"", " height=$1", [strip(s)]) + s = esc(d.target, getFieldValue(n, "height").strip()) + if s.len > 0: + dispA(d.target, options, " height=\"$1\"", " height=$1", [s]) - s = getFieldValue(n, "width") - if s.valid: dispA(d.target, options, " width=\"$1\"", " width=$1", [strip(s)]) + s = esc(d.target, getFieldValue(n, "width").strip()) + if s.len > 0: + dispA(d.target, options, " width=\"$1\"", " width=$1", [s]) - s = getFieldValue(n, "alt") - if s.valid: dispA(d.target, options, " alt=\"$1\"", "", [strip(s)]) + s = esc(d.target, getFieldValue(n, "alt").strip()) + if s.len > 0: + dispA(d.target, options, " alt=\"$1\"", "", [s]) - s = getFieldValue(n, "align") - if s.valid: dispA(d.target, options, " align=\"$1\"", "", [strip(s)]) + s = esc(d.target, getFieldValue(n, "align").strip()) + if s.len > 0: + dispA(d.target, options, " align=\"$1\"", "", [s]) if options.len > 0: options = dispF(d.target, "$1", "[$1]", [options]) - let arg = getArgument(n) - if arg.valid: - dispA(d.target, result, "<img src=\"$1\"$2 />", "\\includegraphics$2{$1}", - [arg, options]) + var htmlOut = "" + if arg.endsWith(".mp4") or arg.endsWith(".ogg") or + arg.endsWith(".webm"): + htmlOut = """ + <video$3 src="$1"$2 autoPlay='true' loop='true' muted='true'> + Sorry, your browser doesn't support embedded videos + </video> + """ + else: + htmlOut = "<img$3 src=\"$1\"$2/>" + + # support for `:target:` links for images: + var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl) + discard safeProtocol(target) + + if target.len > 0: + # `htmlOut` needs to be of the following format for link to work for images: + # <a class="reference external" href="target"><img src=\"$1\"$2/></a> + var htmlOutWithLink = "" + dispA(d.target, htmlOutWithLink, + "<a class=\"reference external\" href=\"$2\">$1</a>", + "\\href{$2}{$1}", [htmlOut, target]) + htmlOut = htmlOutWithLink + + dispA(d.target, result, htmlOut, "$3\\includegraphics$2{$1}", + [esc(d.target, arg), options, n.anchor.idS]) if len(n) >= 3: renderRstToOut(d, n.sons[2], result) proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = @@ -789,30 +810,59 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = "\\includegraphics{$1}", [d.config.getOrDefault"doc.smiley_format" % n.text]) +proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = + template err(msg: string) = + rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg) + let value = n.getFieldValue + var number: int + let nChars = parseInt(value, number) + if nChars == 0: + if value.len == 0: + # use a good default value: + result = 1 + else: + err("field $1 requires an integer, but '$2' was given" % + [fieldName, value]) + elif nChars < value.len: + err("extra arguments were given to $1: '$2'" % + [fieldName, value[nChars..^1]]) + else: + result = number + proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = ## Parses useful fields which can appear before a code block. ## ## This supports the special ``default-language`` internal string generated ## by the ``rst`` module to communicate a specific default language. - case n.getArgument.toLower + case n.getArgument.toLowerAscii of "number-lines": params.numberLines = true # See if the field has a parameter specifying a different line than 1. - var number: int - if parseInt(n.getFieldValue, number) > 0: - params.startLine = number - of "file": - # The ``file`` option is a Nimrod extension to the official spec, it acts + params.startLine = getField1Int(d, n, "number-lines") + of "file", "filename": + # The ``file`` option is a Nim extension to the official spec, it acts # like it would for other directives like ``raw`` or ``cvs-table``. This # field is dealt with in ``rst.nim`` which replaces the existing block with # the referenced file, so we only need to ignore it here to avoid incorrect # warning messages. - discard + params.filename = n.getFieldValue.strip + of "test": + params.testCmd = n.getFieldValue.strip + if params.testCmd.len == 0: + # factor with D20210224T221756. Note that `$docCmd` should appear before `$file` + # but after all other options, but currently `$options` merges both options and `$file` so it's tricky. + params.testCmd = "$nim r --backend:$backend --lib:$libpath $docCmd $options" + else: + # consider whether `$docCmd` should be appended here too + params.testCmd = unescape(params.testCmd) + of "status", "exitcode": + params.status = getField1Int(d, n, n.getArgument) of "default-language": params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage else: - d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument) + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField, + n.getArgument) proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = ## Iterates over all code block fields and returns processed params. @@ -822,8 +872,7 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = result.init if n.isNil: return - assert n.kind == rnCodeBlock - assert(not n.sons[2].isNil) + assert n.kind in {rnCodeBlock, rnInlineCode} # Parse the field list for rendering parameters if there are any. if not n.sons[1].isNil: @@ -834,7 +883,8 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = if result.langStr != "": result.lang = getSourceLanguage(result.langStr) -proc buildLinesHTMLTable(d: PDoc; params: CodeBlockParams, code: string): +proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string, + idStr: string): tuple[beginTable, endTable: string] = ## Returns the necessary tags to start/end a code block in HTML. ## @@ -845,76 +895,100 @@ proc buildLinesHTMLTable(d: PDoc; params: CodeBlockParams, code: string): inc d.listingCounter let id = $d.listingCounter if not params.numberLines: - result = (d.config.getOrDefault"doc.listing_start" % id, + result = (d.config.getOrDefault"doc.listing_start" % + [id, sourceLanguageToStr[params.lang], idStr], d.config.getOrDefault"doc.listing_end" % id) return - var codeLines = 1 + code.strip.countLines + var codeLines = code.strip.countLines assert codeLines > 0 - result.beginTable = """<table class="line-nums-table"><tbody><tr><td class="blob-line-nums"><pre class="line-nums">""" + result.beginTable = """<table$1 class="line-nums-table">""" % [idStr] & + """<tbody><tr><td class="blob-line-nums"><pre class="line-nums">""" var line = params.startLine while codeLines > 0: result.beginTable.add($line & "\n") line.inc codeLines.dec result.beginTable.add("</pre></td><td>" & ( - d.config.getOrDefault"doc.listing_start" % id)) + d.config.getOrDefault"doc.listing_start" % + [id, sourceLanguageToStr[params.lang], idStr])) result.endTable = (d.config.getOrDefault"doc.listing_end" % id) & "</td></tr></tbody></table>" & ( d.config.getOrDefault"doc.listing_button" % id) -proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = - ## Renders a code block, appending it to `result`. +proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string, + target: OutputTarget) = + var g: GeneralTokenizer + initGeneralTokenizer(g, code) + while true: + getNextToken(g, lang) + case g.kind + of gtEof: break + of gtNone, gtWhitespace: + add(result, substr(code, g.start, g.length + g.start - 1)) + else: + dispA(target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [ + esc(target, substr(code, g.start, g.length+g.start-1)), + tokenClassToStr[g.kind]]) + deinitGeneralTokenizer(g) + +proc renderNimCode*(result: var string, code: string, target: OutputTarget) = + renderCodeLang(result, langNim, code, target) + +proc renderCode(d: PDoc, n: PRstNode, result: var string) {.gcsafe.} = + ## Renders a code (code block or inline code), appending it to `result`. ## ## If the code block uses the ``number-lines`` option, a table will be ## generated with two columns, the first being a list of numbers and the ## second the code block itself. The code block can use syntax highlighting, ## which depends on the directive argument specified by the rst input, and ## may also come from the parser through the internal ``default-language`` - ## option to differentiate between a plain code block and nimrod's code block + ## option to differentiate between a plain code block and Nim's code block ## extension. - assert n.kind == rnCodeBlock - if n.sons[2] == nil: return + assert n.kind in {rnCodeBlock, rnInlineCode} var params = d.parseCodeBlockParams(n) + if n.sons[2] == nil: return var m = n.sons[2].sons[0] assert m.kind == rnLeaf - let (blockStart, blockEnd) = buildLinesHTMLTable(d, params, m.text) - - dispA(d.target, result, blockStart, "\\begin{rstpre}\n", []) + if params.testCmd.len > 0 and d.onTestSnippet != nil: + d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text) + + var blockStart, blockEnd: string + case d.target + of outHtml: + if n.kind == rnCodeBlock: + (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text, + n.anchor.idS) + else: # rnInlineCode + blockStart = "<tt class=\"docutils literal\"><span class=\"pre\">" + blockEnd = "</span></tt>" + of outLatex: + if n.kind == rnCodeBlock: + blockStart = "\n\n" & n.anchor.idS & "\\begin{rstpre}\n" + blockEnd = "\n\\end{rstpre}\n\n" + else: # rnInlineCode + blockStart = "\\rstcode{" + blockEnd = "}" + dispA(d.target, result, blockStart, blockStart, []) if params.lang == langNone: - if len(params.langStr) > 0: - d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr) - for letter in m.text: escChar(d.target, result, letter) + if len(params.langStr) > 0 and params.langStr.toLowerAscii != "none": + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage, + params.langStr) + for letter in m.text: escChar(d.target, result, letter, emText) else: - var g: GeneralTokenizer - initGeneralTokenizer(g, m.text) - while true: - getNextToken(g, params.lang) - case g.kind - of gtEof: break - of gtNone, gtWhitespace: - add(result, substr(m.text, g.start, g.length + g.start - 1)) - else: - dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [ - esc(d.target, substr(m.text, g.start, g.length+g.start-1)), - tokenClassToStr[g.kind]]) - deinitGeneralTokenizer(g) - dispA(d.target, result, blockEnd, "\n\\end{rstpre}\n") + renderCodeLang(result, params.lang, m.text, d.target) + dispA(d.target, result, blockEnd, blockEnd) proc renderContainer(d: PDoc, n: PRstNode, result: var string) = var tmp = "" renderRstToOut(d, n.sons[2], tmp) - var arg = strip(getArgument(n)) + var arg = esc(d.target, strip(getArgument(n))) if arg == "": dispA(d.target, result, "<div>$1</div>", "$1", [tmp]) else: dispA(d.target, result, "<div class=\"$1\">$2</div>", "$2", [arg, tmp]) -proc texColumns(n: PRstNode): string = - result = "" - for i in countup(1, len(n)): add(result, "|X") - proc renderField(d: PDoc, n: PRstNode, result: var string) = var b = false if d.target == outLatex: @@ -932,41 +1006,150 @@ proc renderField(d: PDoc, n: PRstNode, result: var string) = if not b: renderAux(d, n, "<tr>$1</tr>\n", "$1", result) +proc renderEnumList(d: PDoc, n: PRstNode, result: var string) = + var + specifier = "" + specStart = "" + i1 = 0 + pre = "" + i2 = n.labelFmt.len - 1 + post = "" + if n.labelFmt[0] == '(': + i1 = 1 + pre = "(" + if n.labelFmt[^1] == ')' or n.labelFmt[^1] == '.': + i2 = n.labelFmt.len - 2 + post = $n.labelFmt[^1] + let enumR = i1 .. i2 # enumerator range without surrounding (, ), . + if d.target == outLatex: + result.add ("\n%" & n.labelFmt & "\n") + # use enumerate parameters from package enumitem + if n.labelFmt[i1].isDigit: + var labelDef = "" + if pre != "" or post != "": + labelDef = "label=" & pre & "\\arabic*" & post & "," + if n.labelFmt[enumR] != "1": + specStart = "start=$1" % [n.labelFmt[enumR]] + if labelDef != "" or specStart != "": + specifier = "[$1$2]" % [labelDef, specStart] + else: + let (first, labelDef) = + if n.labelFmt[i1].isUpperAscii: ('A', "label=" & pre & "\\Alph*" & post) + else: ('a', "label=" & pre & "\\alph*" & post) + if n.labelFmt[i1] != first: + specStart = ",start=" & $(ord(n.labelFmt[i1]) - ord(first) + 1) + specifier = "[$1$2]" % [labelDef, specStart] + else: # HTML + # TODO: implement enumerator formatting using pre and post ( and ) for HTML + if n.labelFmt[i1].isDigit: + if n.labelFmt[enumR] != "1": + specStart = " start=\"$1\"" % [n.labelFmt[enumR]] + specifier = "class=\"simple\"" & specStart + else: + let (first, labelDef) = + if n.labelFmt[i1].isUpperAscii: ('A', "class=\"upperalpha simple\"") + else: ('a', "class=\"loweralpha simple\"") + if n.labelFmt[i1] != first: + specStart = " start=\"$1\"" % [ $(ord(n.labelFmt[i1]) - ord(first) + 1) ] + specifier = labelDef & specStart + renderAux(d, n, "<ol$2 " & specifier & ">$1</ol>\n", + "\\begin{enumerate}" & specifier & "$2$1\\end{enumerate}\n", + result) + +proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = + var + htmlCls = "admonition_warning" + texSz = "\\large" + texColor = "orange" + case n.adType + of "hint", "note", "tip": + htmlCls = "admonition-info"; texSz = "\\normalsize"; texColor = "green" + of "attention", "admonition", "important", "warning", "caution": + htmlCls = "admonition-warning"; texSz = "\\large"; texColor = "orange" + of "danger", "error": + htmlCls = "admonition-error"; texSz = "\\Large"; texColor = "red" + else: discard + let txt = n.adType.capitalizeAscii() + let htmlHead = "<div class=\"admonition " & htmlCls & "\">" + renderAux(d, n, + htmlHead & "<span$2 class=\"" & htmlCls & "-text\"><b>" & txt & + ":</b></span>\n" & "$1</div>\n", + "\n\n\\begin{rstadmonition}[borderline west={0.2em}{0pt}{" & + texColor & "}]$2\n" & + "{" & texSz & "\\color{" & texColor & "}{\\textbf{" & txt & ":}}} " & + "$1\n\\end{rstadmonition}\n", + result) + +proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, + external: bool, nimdoc = false, tooltip="") = + var linkStr = "" + block: + let mode = d.escMode + d.escMode = emUrl + renderRstToOut(d, link, linkStr) + d.escMode = mode + discard safeProtocol(linkStr) + var textStr = "" + renderRstToOut(d, text, textStr) + let nimDocStr = if nimdoc: " nimdoc" else: "" + var tooltipStr = "" + if tooltip != "": + tooltipStr = """ title="$1"""" % [ esc(d.target, tooltip) ] + if external: + dispA(d.target, result, + "<a class=\"reference external$3\"$4 href=\"$2\">$1</a>", + "\\href{$2}{$1}", [textStr, linkStr, nimDocStr, tooltipStr]) + else: + dispA(d.target, result, + "<a class=\"reference internal$3\"$4 href=\"#$2\">$1</a>", + "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", + [textStr, linkStr, nimDocStr, tooltipStr]) + +proc traverseForIndex*(d: PDoc, n: PRstNode) = + ## A version of [renderRstToOut] that only fills entries for ``.idx`` files. + var discarded: string + if n == nil: return + case n.kind + of rnIdx: renderIndexTerm(d, n, discarded) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, discarded) + of rnOverline: renderOverline(d, n, discarded) + else: + for i in 0 ..< len(n): + traverseForIndex(d, n.sons[i]) + proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return case n.kind of rnInner: renderAux(d, n, result) - of rnHeadline: renderHeadline(d, n, result) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result) of rnOverline: renderOverline(d, n, result) - of rnTransition: renderAux(d, n, "<hr />\n", "\\hrule\n", result) - of rnParagraph: renderAux(d, n, "<p>$1</p>\n", "$1\n\n", result) + of rnTransition: renderAux(d, n, "<hr$2 />\n", "\n\n\\vspace{0.6em}\\hrule$2\n", result) + of rnParagraph: renderAux(d, n, "<p$2>$1</p>\n", "\n\n$2\n$1\n\n", result) of rnBulletList: - renderAux(d, n, "<ul class=\"simple\">$1</ul>\n", - "\\begin{itemize}$1\\end{itemize}\n", result) + renderAux(d, n, "<ul$2 class=\"simple\">$1</ul>\n", + "\\begin{itemize}\n$2\n$1\\end{itemize}\n", result) of rnBulletItem, rnEnumItem: - renderAux(d, n, "<li>$1</li>\n", "\\item $1\n", result) - of rnEnumList: - renderAux(d, n, "<ol class=\"simple\">$1</ol>\n", - "\\begin{enumerate}$1\\end{enumerate}\n", result) - of rnDefList: - renderAux(d, n, "<dl class=\"docutils\">$1</dl>\n", - "\\begin{description}$1\\end{description}\n", result) + renderAux(d, n, "<li$2>$1</li>\n", "\\item $2$1\n", result) + of rnEnumList: renderEnumList(d, n, result) + of rnDefList, rnMdDefList: + renderAux(d, n, "<dl$2 class=\"docutils\">$1</dl>\n", + "\\begin{description}\n$2\n$1\\end{description}\n", result) of rnDefItem: renderAux(d, n, result) - of rnDefName: renderAux(d, n, "<dt>$1</dt>\n", "\\item[$1] ", result) - of rnDefBody: renderAux(d, n, "<dd>$1</dd>\n", "$1\n", result) + of rnDefName: renderAux(d, n, "<dt$2>$1</dt>\n", "$2\\item[$1]\\ ", result) + of rnDefBody: renderAux(d, n, "<dd$2>$1</dd>\n", "$2\n$1\n", result) of rnFieldList: var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) if tmp.len != 0: dispA(d.target, result, - "<table class=\"docinfo\" frame=\"void\" rules=\"none\">" & + "<table$2 class=\"docinfo\" frame=\"void\" rules=\"none\">" & "<col class=\"docinfo-name\" />" & "<col class=\"docinfo-content\" />" & "<tbody valign=\"top\">$1" & "</tbody></table>", - "\\begin{description}$1\\end{description}\n", - [tmp]) + "\\begin{description}\n$2\n$1\\end{description}\n", + [tmp, n.anchor.idS]) of rnField: renderField(d, n, result) of rnFieldName: renderAux(d, n, "<th class=\"docinfo-name\">$1:</th>", @@ -976,96 +1159,190 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnIndex: renderRstToOut(d, n.sons[2], result) of rnOptionList: - renderAux(d, n, "<table frame=\"void\">$1</table>", - "\\begin{description}\n$1\\end{description}\n", result) + renderAux(d, n, "<div$2 class=\"option-list\">$1</div>", + "\\begin{rstoptlist}$2\n$1\\end{rstoptlist}", result) of rnOptionListItem: - renderAux(d, n, "<tr>$1</tr>\n", "$1", result) + var addclass = if n.order mod 2 == 1: " odd" else: "" + renderAux(d, n, + "<div class=\"option-list-item" & addclass & "\">$1</div>\n", + "$1", result) of rnOptionGroup: - renderAux(d, n, "<th align=\"left\">$1</th>", "\\item[$1]", result) + renderAux(d, n, + "<div class=\"option-list-label\"><tt><span class=\"option\">" & + "$1</span></tt></div>", + "\\item[\\rstcodeitem{\\spanoption{$1}}]", result) of rnDescription: - renderAux(d, n, "<td align=\"left\">$1</td>\n", " $1\n", result) + renderAux(d, n, "<div class=\"option-list-description\">$1</div>", + " $1\n", result) of rnOption, rnOptionString, rnOptionArgument: - doAssert false, "renderRstToOut" + raiseAssert "renderRstToOut" of rnLiteralBlock: - renderAux(d, n, "<pre>$1</pre>\n", - "\\begin{rstpre}\n$1\n\\end{rstpre}\n", result) - of rnQuotedLiteralBlock: - doAssert false, "renderRstToOut" + renderAux(d, n, "<pre$2>$1</pre>\n", + "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) + of rnMarkdownBlockQuote: + d.curQuotationDepth = 1 + var tmp = "" + renderAux(d, n, "$1", "$1", tmp) + let itemEnding = + if d.target == outHtml: "</blockquote>" else: "\\end{rstquote}" + tmp.add itemEnding.repeat(d.curQuotationDepth - 1) + dispA(d.target, result, + "<blockquote$2 class=\"markdown-quote\">$1</blockquote>\n", + "\n\\begin{rstquote}\n$2\n$1\\end{rstquote}\n", [tmp, n.anchor.idS]) + of rnMarkdownBlockQuoteItem: + let addQuotationDepth = n.quotationDepth - d.curQuotationDepth + var itemPrefix: string # start or ending (quotation grey bar on the left) + if addQuotationDepth >= 0: + let s = + if d.target == outHtml: "<blockquote class=\"markdown-quote\">" + else: "\\begin{rstquote}" + itemPrefix = s.repeat(addQuotationDepth) + else: + let s = + if d.target == outHtml: "</blockquote>" + else: "\\end{rstquote}" + itemPrefix = s.repeat(-addQuotationDepth) + renderAux(d, n, itemPrefix & "<p>$1</p>", itemPrefix & "\n$1", result) + d.curQuotationDepth = n.quotationDepth of rnLineBlock: - renderAux(d, n, "<p>$1</p>", "$1\n\n", result) + if n.sons.len == 1 and n.sons[0].lineIndent == "\n": + # whole line block is one empty line, no need to add extra spacing + renderAux(d, n, "<p$2>$1</p> ", "\n\n$2\n$1", result) + else: # add extra spacing around the line block for Latex + renderAux(d, n, "<p$2>$1</p>", + "\n\\vspace{0.5em}$2\n$1\\vspace{0.5em}\n", result) of rnLineBlockItem: - renderAux(d, n, "$1<br />", "$1\\\\\n", result) + if n.lineIndent.len == 0: # normal case - no additional indentation + renderAux(d, n, "$1<br/>", "\\noindent $1\n\n", result) + elif n.lineIndent == "\n": # add one empty line + renderAux(d, n, "<br/>", "\\vspace{1em}\n", result) + else: # additional indentation w.r.t. '| ' + let indent = $(0.5 * (n.lineIndent.len - 1).toFloat) & "em" + renderAux(d, n, + "<span style=\"margin-left: " & indent & "\">$1</span><br/>", + "\\noindent\\hspace{" & indent & "}$1\n\n", result) of rnBlockQuote: - renderAux(d, n, "<blockquote><p>$1</p></blockquote>\n", - "\\begin{quote}$1\\end{quote}\n", result) - of rnTable, rnGridTable: + renderAux(d, n, "<blockquote$2><p>$1</p></blockquote>\n", + "\\begin{quote}\n$2\n$1\\end{quote}\n", result) + of rnAdmonition: renderAdmonition(d, n, result) + of rnTable, rnGridTable, rnMarkdownTable: renderAux(d, n, - "<table border=\"1\" class=\"docutils\">$1</table>", - "\\begin{table}\\begin{rsttab}{" & - texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result) + "<table$2 border=\"1\" class=\"docutils\">$1</table>", + "\n$2\n\\begin{rsttab}{" & + "L".repeat(n.colCount) & "}\n\\toprule\n$1" & + "\\addlinespace[0.1em]\\bottomrule\n\\end{rsttab}", result) of rnTableRow: if len(n) >= 1: - if d.target == outLatex: - #var tmp = "" - renderRstToOut(d, n.sons[0], result) - for i in countup(1, len(n) - 1): - result.add(" & ") - renderRstToOut(d, n.sons[i], result) - result.add("\\\\\n\\hline\n") - else: + case d.target + of outHtml: result.add("<tr>") renderAux(d, n, result) result.add("</tr>\n") - of rnTableDataCell: - renderAux(d, n, "<td>$1</td>", "$1", result) - of rnTableHeaderCell: - renderAux(d, n, "<th>$1</th>", "\\textbf{$1}", result) - of rnLabel: - doAssert false, "renderRstToOut" # used for footnotes and other - of rnFootnote: - doAssert false, "renderRstToOut" # a footnote - of rnCitation: - doAssert false, "renderRstToOut" # similar to footnote - of rnRef: - var tmp = "" - renderAux(d, n, tmp) + of outLatex: + if n.sons[0].kind == rnTableHeaderCell: + result.add "\\rowcolor{gray!15} " + var spanLines: seq[(int, int)] + var nCell = 0 + for uCell in 0 .. n.len - 1: + renderRstToOut(d, n.sons[uCell], result) + if n.sons[uCell].span > 0: + spanLines.add (nCell + 1, nCell + n.sons[uCell].span) + nCell += n.sons[uCell].span + else: + nCell += 1 + if uCell != n.len - 1: + result.add(" & ") + result.add("\\\\") + if n.endsHeader: result.add("\\midrule\n") + for (start, stop) in spanLines: + result.add("\\cmidrule(lr){$1-$2}" % [$start, $stop]) + result.add("\n") + of rnTableHeaderCell, rnTableDataCell: + case d.target + of outHtml: + let tag = if n.kind == rnTableHeaderCell: "th" else: "td" + var spanSpec: string + if n.span <= 1: spanSpec = "" + else: + spanSpec = " colspan=\"" & $n.span & "\" style=\"text-align: center\"" + renderAux(d, n, "<$1$2>$$1</$1>" % [tag, spanSpec], "", result) + of outLatex: + let text = if n.kind == rnTableHeaderCell: "\\textbf{$1}" else: "$1" + var latexStr: string + if n.span <= 1: latexStr = text + else: latexStr = "\\multicolumn{" & $n.span & "}{c}{" & text & "}" + renderAux(d, n, "", latexStr, result) + of rnFootnoteGroup: + renderAux(d, n, + "<hr class=\"footnote\">" & + "<div class=\"footnote-group\">\n$1</div>\n", + "\n\n\\noindent\\rule{0.25\\linewidth}{.4pt}\n" & + "\\begin{rstfootnote}\n$1\\end{rstfootnote}\n\n", + result) + of rnFootnote, rnCitation: + var mark = "" + renderAux(d, n.sons[0], mark) + var body = "" + renderRstToOut(d, n.sons[1], body) dispA(d.target, result, - "<a class=\"reference external\" href=\"#$2\">$1</a>", - "$1\\ref{$2}", [tmp, rstnodeToRefname(n)]) + "<div$2><div class=\"footnote-label\">" & + "<sup><strong><a href=\"#$4\">[$3]</a></strong></sup>" & + "</div>   $1\n</div>\n", + "\\item[\\textsuperscript{[$3]}]$2 $1\n", + [body, n.anchor.idS, mark, n.anchor]) + of rnPandocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnRstRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=false) of rnStandaloneHyperlink: - renderAux(d, n, - "<a class=\"reference external\" href=\"$1\">$1</a>", - "\\href{$1}{$1}", result) + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) + of rnInternalRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnNimdocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false, + nimdoc=true, tooltip=n.tooltip) of rnHyperlink: - var tmp0 = "" - var tmp1 = "" - renderRstToOut(d, n.sons[0], tmp0) - renderRstToOut(d, n.sons[1], tmp1) + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) + of rnFootnoteRef: + var tmp = "[" + renderAux(d, n.sons[0], tmp) + tmp.add "]" dispA(d.target, result, - "<a class=\"reference external\" href=\"$2\">$1</a>", - "\\href{$2}{$1}", [tmp0, tmp1]) + "<sup><strong><a class=\"reference internal\" href=\"#$2\">" & + "$1</a></strong></sup>", + "\\textsuperscript{\\hyperlink{$2}{\\textbf{$1}}}", + [tmp, n.sons[1].text]) of rnDirArg, rnRaw: renderAux(d, n, result) of rnRawHtml: - if d.target != outLatex: + if d.target != outLatex and not lastSon(n).isNil: result.add addNodes(lastSon(n)) of rnRawLatex: - if d.target == outLatex: + if d.target == outLatex and not lastSon(n).isNil: result.add addNodes(lastSon(n)) of rnImage, rnFigure: renderImage(d, n, result) - of rnCodeBlock: renderCodeBlock(d, n, result) + of rnCodeBlock, rnInlineCode: renderCode(d, n, result) of rnContainer: renderContainer(d, n, result) of rnSubstitutionReferences, rnSubstitutionDef: renderAux(d, n, "|$1|", "|$1|", result) of rnDirective: renderAux(d, n, "", "", result) - of rnGeneralRole: + of rnUnknownRole, rnCodeFragment: var tmp0 = "" var tmp1 = "" renderRstToOut(d, n.sons[0], tmp0) renderRstToOut(d, n.sons[1], tmp1) - dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", - [tmp0, tmp1]) + var class = tmp1 + # don't allow missing role break latex compilation: + if d.target == outLatex and n.kind == rnUnknownRole: class = "Other" + if n.kind == rnCodeFragment: + dispA(d.target, result, + "<tt class=\"docutils literal\"><span class=\"pre $2\">" & + "$1</span></tt>", + "\\rstcode{\\span$2{$1}}", [tmp0, class]) + else: # rnUnknownRole, not necessarily code/monospace font + dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", + [tmp0, class]) of rnSub: renderAux(d, n, "<sub>$1</sub>", "\\rstsub{$1}", result) of rnSup: renderAux(d, n, "<sup>$1</sup>", "\\rstsup{$1}", result) of rnEmphasis: renderAux(d, n, "<em>$1</em>", "\\emph{$1}", result) @@ -1074,20 +1351,27 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTripleEmphasis: renderAux(d, n, "<strong><em>$1</em></strong>", "\\textbf{emph{$1}}", result) - of rnInterpretedText: - renderAux(d, n, "<cite>$1</cite>", "\\emph{$1}", result) of rnIdx: renderIndexTerm(d, n, result) - of rnInlineLiteral: + of rnInlineLiteral, rnInterpretedText: renderAux(d, n, "<tt class=\"docutils literal\"><span class=\"pre\">$1</span></tt>", - "\\texttt{$1}", result) + "\\rstcode{$1}", result) + of rnInlineTarget: + var tmp = "" + renderAux(d, n, tmp) + dispA(d.target, result, + "<span class=\"target\" id=\"$2\">$1</span>", + "\\label{$2}\\hypertarget{$2}{$1}", + [tmp, rstnodeToRefname(n)]) of rnSmiley: renderSmiley(d, n, result) - of rnLeaf: result.add(esc(d.target, n.text)) + of rnLeaf: result.add(esc(d.target, n.text, escMode=d.escMode)) of rnContents: d.hasToc = true + of rnDefaultRole: discard of rnTitle: d.meta[metaTitle] = "" renderRstToOut(d, n.sons[0], d.meta[metaTitle]) + d.meta[metaTitleRaw] = n.sons[0].addNodes # ----------------------------------------------------------------------------- @@ -1173,7 +1457,7 @@ proc defaultConfig*(): StringTableRef = ## ``rstToHtml`` to generate the bare minimum HTML. result = newStringTable(modeStyleInsensitive) - template setConfigVar(key, val: expr) = + template setConfigVar(key, val) = result[key] = val # If you need to modify these values, it might be worth updating the template @@ -1217,7 +1501,7 @@ $moduledesc $content </div> """) - setConfigVar("doc.listing_start", "<pre class = \"listing\">") + setConfigVar("doc.listing_start", "<pre$3 class = \"listing\">") setConfigVar("doc.listing_end", "</pre>") setConfigVar("doc.listing_button", "</pre>") setConfigVar("doc.body_no_toc", "$moduledesc $content") @@ -1227,7 +1511,8 @@ $content # ---------- forum --------------------------------------------------------- proc rstToHtml*(s: string, options: RstParseOptions, - config: StringTableRef): string = + config: StringTableRef, + msgHandler: MsgHandler = rst.defaultMsgHandler): string {.gcsafe.} = ## Converts an input rst string into embeddable HTML. ## ## This convenience proc parses any input string using rst markup (it doesn't @@ -1237,12 +1522,13 @@ proc rstToHtml*(s: string, options: RstParseOptions, ## work. For an explanation of the ``config`` parameter see the ## ``initRstGenerator`` proc. Example: ## - ## .. code-block:: nim + ## ```nim ## import packages/docutils/rstgen, strtabs ## ## echo rstToHtml("*Hello* **world**!", {}, ## newStringTable(modeStyleInsensitive)) ## # --> <em>Hello</em> <strong>world</strong>! + ## ``` ## ## If you need to allow the rst ``include`` directive or tweak the generated ## output you have to create your own ``RstGenerator`` with @@ -1251,18 +1537,30 @@ proc rstToHtml*(s: string, options: RstParseOptions, proc myFindFile(filename: string): string = # we don't find any files in online mode: result = "" + proc myFindRefFile(filename: string): (string, string) = + result = ("", "") const filen = "input" + let (rst, filenames, t) = rstParse(s, filen, + line=LineRstInit, column=ColRstInit, + options, myFindFile, myFindRefFile, msgHandler) var d: RstGenerator - initRstGenerator(d, outHtml, config, filen, options, myFindFile, - rst.defaultMsgHandler) - var dummyHasToc = false - var rst = rstParse(s, filen, 0, 1, dummyHasToc, options) + initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, + filenames, hasToc = t) result = "" renderRstToOut(d, rst, result) - - -when isMainModule: - assert rstToHtml("*Hello* **world**!", {}, - newStringTable(modeStyleInsensitive)) == - "<em>Hello</em> <strong>world</strong>!" + strbasics.strip(result) + + +proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} = + ## Convenience proc for `renderRstToOut` and `initRstGenerator`. + runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" + if rstSource.len == 0: return + let (rst, filenames, t) = rstParse(rstSource, "", + line=LineRstInit, column=ColRstInit, + options) + var rstGenera: RstGenerator + rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", + filenames=filenames, hasToc = t) + rstGenera.renderRstToOut(rst, result) + strbasics.strip(result) diff --git a/lib/packages/docutils/rstidx.nim b/lib/packages/docutils/rstidx.nim new file mode 100644 index 000000000..1472d28fd --- /dev/null +++ b/lib/packages/docutils/rstidx.nim @@ -0,0 +1,141 @@ +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## Nim `idx`:idx: file format related definitions. + +import std/[strutils, syncio, hashes] +from std/os import splitFile + +type + IndexEntryKind* = enum ## discriminator tag + ieMarkupTitle = "markupTitle" + ## RST/Markdown title, text in `keyword` + + ## HTML text in `linkTitle` + ieNimTitle = "nimTitle" + ## Nim title + ieHeading = "heading" ## RST/Markdown markup heading, escaped + ieIdxRole = "idx" ## RST :idx: definition, escaped + ieNim = "nim" ## Nim symbol, unescaped + ieNimGroup = "nimgrp" ## Nim overload group, unescaped + IndexEntry* = object + kind*: IndexEntryKind ## 0. + keyword*: string ## 1. + link*: string ## 2. + linkTitle*: string ## 3. contains a prettier text for the href + linkDesc*: string ## 4. the title attribute of the final href + line*: int ## 5. + module*: string ## origin file, NOT a field in ``.idx`` file + aux*: string ## auxuliary field, NOT a field in ``.idx`` file + +proc isDocumentationTitle*(hyperlink: string): bool = + ## Returns true if the hyperlink is actually a documentation title. + ## + ## Documentation titles lack the hash. See `mergeIndexes() + ## <#mergeIndexes,string>`_ for a more detailed explanation. + result = hyperlink.find('#') < 0 + +proc `$`*(e: IndexEntry): string = + """("$1", "$2", "$3", "$4", $5)""" % [ + e.keyword, e.link, e.linkTitle, e.linkDesc, $e.line] + +proc quoteIndexColumn(text: string): string = + ## Returns a safe version of `text` for serialization to the ``.idx`` file. + ## + ## The returned version can be put without worries in a line based tab + ## separated column text file. The following character sequence replacements + ## will be performed for that goal: + ## + ## * ``"\\"`` => ``"\\\\"`` + ## * ``"\n"`` => ``"\\n"`` + ## * ``"\t"`` => ``"\\t"`` + result = newStringOfCap(text.len + 3) + for c in text: + case c + of '\\': result.add "\\" + of '\L': result.add "\\n" + of '\C': discard + of '\t': result.add "\\t" + else: result.add c + +proc unquoteIndexColumn*(text: string): string = + ## Returns the unquoted version generated by ``quoteIndexColumn``. + result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) + +proc formatIndexEntry*(kind: IndexEntryKind; htmlFile, id, term, linkTitle, + linkDesc: string, line: int): + tuple[entry: string, isTitle: bool] = + result.entry = $kind + result.entry.add('\t') + result.entry.add term + result.entry.add('\t') + result.entry.add(htmlFile) + if id.len > 0: + result.entry.add('#') + result.entry.add(id) + result.isTitle = false + else: + result.isTitle = true + result.entry.add('\t' & linkTitle.quoteIndexColumn) + result.entry.add('\t' & linkDesc.quoteIndexColumn) + result.entry.add('\t' & $line) + result.entry.add("\n") + +proc parseIndexEntryKind(s: string): IndexEntryKind = + result = case s: + of "nim": ieNim + of "nimgrp": ieNimGroup + of "heading": ieHeading + of "idx": ieIdxRole + of "nimTitle": ieNimTitle + of "markupTitle": ieMarkupTitle + else: raise newException(ValueError, "unknown index entry value $1" % [s]) + +proc parseIdxFile*(path: string): + tuple[fileEntries: seq[IndexEntry], title: IndexEntry] = + var + f = 0 + newSeq(result.fileEntries, 500) + setLen(result.fileEntries, 0) + let (_, base, _) = path.splitFile + for line in lines(path): + let s = line.find('\t') + if s < 0: continue + setLen(result.fileEntries, f+1) + let cols = line.split('\t') + result.fileEntries[f].kind = parseIndexEntryKind(cols[0]) + result.fileEntries[f].keyword = cols[1] + result.fileEntries[f].link = cols[2] + if result.fileEntries[f].kind == ieIdxRole: + result.fileEntries[f].module = base + else: + if result.title.keyword.len == 0: + result.fileEntries[f].module = base + else: + result.fileEntries[f].module = result.title.keyword + + result.fileEntries[f].linkTitle = cols[3].unquoteIndexColumn + result.fileEntries[f].linkDesc = cols[4].unquoteIndexColumn + result.fileEntries[f].line = parseInt(cols[5]) + + if result.fileEntries[f].kind in {ieNimTitle, ieMarkupTitle}: + result.title = result.fileEntries[f] + inc f + +proc cmp*(a, b: IndexEntry): int = + ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. + result = cmpIgnoreStyle(a.keyword, b.keyword) + if result == 0: + result = cmpIgnoreStyle(a.link, b.link) + +proc hash*(x: IndexEntry): Hash = + ## Returns the hash for the combined fields of the type. + ## + ## The hash is computed as the chained hash of the individual string hashes. + result = x.keyword.hash !& x.link.hash + result = result !& x.linkTitle.hash + result = result !& x.linkDesc.hash + result = !$result diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index 276dd812d..007488354 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -7,9 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim:on.} - -from posix import SocketHandle +from std/posix import SocketHandle const EPOLLIN* = 0x00000001 @@ -23,6 +21,7 @@ const EPOLLWRBAND* = 0x00000200 EPOLLMSG* = 0x00000400 EPOLLRDHUP* = 0x00002000 + EPOLLEXCLUSIVE* = 1 shl 28 EPOLLWAKEUP* = 1 shl 29 EPOLLONESHOT* = 1 shl 30 EPOLLET* = 1 shl 31 @@ -34,59 +33,67 @@ const EPOLL_CTL_DEL* = 2 # Remove a file descriptor from the interface. EPOLL_CTL_MOD* = 3 # Change file descriptor epoll_event structure. +# https://github.com/torvalds/linux/blob/ff6992735ade75aae3e35d16b17da1008d753d28/include/uapi/linux/eventpoll.h#L77 +when defined(linux) and defined(amd64): + {.pragma: epollPacked, packed.} +else: + {.pragma: epollPacked.} + type - epoll_data* {.importc: "union epoll_data", - header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union. - #thePtr* {.importc: "ptr".}: pointer - fd* {.importc: "fd".}: cint # \ - #u32*: uint32 - #u64*: uint64 + EpollData* {.importc: "epoll_data_t", + header: "<sys/epoll.h>", pure, final, union.} = object + `ptr`* {.importc: "ptr".}: pointer + fd* {.importc: "fd".}: cint + u32* {.importc: "u32".}: uint32 + u64* {.importc: "u64".}: uint64 - epoll_event* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final.} = object + EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final, epollPacked.} = object events*: uint32 # Epoll events - data*: epoll_data # User data variable + data*: EpollData # User data variable proc epoll_create*(size: cint): cint {.importc: "epoll_create", header: "<sys/epoll.h>".} ## Creates an epoll instance. Returns an fd for the new instance. - ## The "size" parameter is a hint specifying the number of file - ## descriptors to be associated with the new instance. The fd - ## returned by epoll_create() should be closed with close(). + ## + ## The "size" parameter is a hint specifying the number of file + ## descriptors to be associated with the new instance. The fd + ## returned by epoll_create() should be closed with close(). proc epoll_create1*(flags: cint): cint {.importc: "epoll_create1", header: "<sys/epoll.h>".} ## Same as epoll_create but with an FLAGS parameter. The unused SIZE - ## parameter has been dropped. + ## parameter has been dropped. -proc epoll_ctl*(epfd: cint; op: cint; fd: cint | SocketHandle; event: ptr epoll_event): cint {. +proc epoll_ctl*(epfd: cint; op: cint; fd: cint | SocketHandle; event: ptr EpollEvent): cint {. importc: "epoll_ctl", header: "<sys/epoll.h>".} - ## Manipulate an epoll instance "epfd". Returns 0 in case of success, - ## -1 in case of error ( the "errno" variable will contain the - ## specific error code ) The "op" parameter is one of the EPOLL_CTL_* - ## constants defined above. The "fd" parameter is the target of the - ## operation. The "event" parameter describes which events the caller - ## is interested in and any associated user data. + ## Manipulate an epoll instance "epfd". Returns `0` in case of success, + ## `-1` in case of error (the "errno" variable will contain the specific error code). + ## + ## The "op" parameter is one of the `EPOLL_CTL_*` + ## constants defined above. The "fd" parameter is the target of the + ## operation. The "event" parameter describes which events the caller + ## is interested in and any associated user data. -proc epoll_wait*(epfd: cint; events: ptr epoll_event; maxevents: cint; +proc epoll_wait*(epfd: cint; events: ptr EpollEvent; maxevents: cint; timeout: cint): cint {.importc: "epoll_wait", header: "<sys/epoll.h>".} ## Wait for events on an epoll instance "epfd". Returns the number of - ## triggered events returned in "events" buffer. Or -1 in case of - ## error with the "errno" variable set to the specific error code. The - ## "events" parameter is a buffer that will contain triggered - ## events. The "maxevents" is the maximum number of events to be - ## returned ( usually size of "events" ). The "timeout" parameter - ## specifies the maximum wait time in milliseconds (-1 == infinite). + ## triggered events returned in "events" buffer. Or -1 in case of + ## error with the "errno" variable set to the specific error code. The + ## "events" parameter is a buffer that will contain triggered + ## events. The "maxevents" is the maximum number of events to be + ## returned ( usually size of "events" ). The "timeout" parameter + ## specifies the maximum wait time in milliseconds (-1 == infinite). ## - ## This function is a cancellation point and therefore not marked with - ## __THROW. + ## This function is a cancellation point and therefore not marked with + ## __THROW. -#proc epoll_pwait*(epfd: cint; events: ptr epoll_event; maxevents: cint; +#proc epoll_pwait*(epfd: cint; events: ptr EpollEvent; maxevents: cint; # timeout: cint; ss: ptr sigset_t): cint {. # importc: "epoll_pwait", header: "<sys/epoll.h>".} # Same as epoll_wait, but the thread's signal mask is temporarily -# and atomically replaced with the one provided as parameter. +# and atomically replaced with the one provided as parameter. # -# This function is a cancellation point and therefore not marked with -# __THROW. +# This function is a cancellation point and therefore not marked with +# __THROW. diff --git a/lib/posix/inotify.nim b/lib/posix/inotify.nim index a206f8067..575accc18 100644 --- a/lib/posix/inotify.nim +++ b/lib/posix/inotify.nim @@ -7,67 +7,105 @@ # distribution, for details about the copyright. # -{.deadCodeElim:on.} +when defined(nimPreviewSlimSystem): + import std/syncio # Get the platform-dependent flags. # Structure describing an inotify event. type - InotifyEvent*{.pure, final, importc: "struct inotify_event", - header: "<sys/inotify.h>".} = object - wd*{.importc: "wd".}: cint # Watch descriptor. - mask*{.importc: "mask".}: uint32 # Watch mask. - cookie*{.importc: "cookie".}: uint32 # Cookie to synchronize two events. - len*{.importc: "len".}: uint32 # Length (including NULs) of name. - name*{.importc: "name".}: char # Name. -{.deprecated: [Tinotify_event: InotifyEvent].} + InotifyEvent* {.pure, final, importc: "struct inotify_event", + header: "<sys/inotify.h>", + completeStruct.} = object ## An Inotify event. + wd* {.importc: "wd".}: FileHandle ## Watch descriptor. + mask* {.importc: "mask".}: uint32 ## Watch mask. + cookie* {.importc: "cookie".}: uint32 ## Cookie to synchronize two events. + len* {.importc: "len".}: uint32 ## Length (including NULs) of name. + name* {.importc: "name".}: UncheckedArray[char] ## Name. # Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH. const - IN_ACCESS* = 0x00000001 # File was accessed. - IN_MODIFY* = 0x00000002 # File was modified. - IN_ATTRIB* = 0x00000004 # Metadata changed. - IN_CLOSE_WRITE* = 0x00000008 # Writtable file was closed. - IN_CLOSE_NOWRITE* = 0x00000010 # Unwrittable file closed. - IN_CLOSE* = (IN_CLOSE_WRITE or IN_CLOSE_NOWRITE) # Close. - IN_OPEN* = 0x00000020 # File was opened. - IN_MOVED_FROM* = 0x00000040 # File was moved from X. - IN_MOVED_TO* = 0x00000080 # File was moved to Y. - IN_MOVE* = (IN_MOVED_FROM or IN_MOVED_TO) # Moves. - IN_CREATE* = 0x00000100 # Subfile was created. - IN_DELETE* = 0x00000200 # Subfile was deleted. - IN_DELETE_SELF* = 0x00000400 # Self was deleted. - IN_MOVE_SELF* = 0x00000800 # Self was moved. + IN_ACCESS* = 0x00000001 ## File was accessed. + IN_MODIFY* = 0x00000002 ## File was modified. + IN_ATTRIB* = 0x00000004 ## Metadata changed. + IN_CLOSE_WRITE* = 0x00000008 ## Writtable file was closed. + IN_CLOSE_NOWRITE* = 0x00000010 ## Unwrittable file closed. + IN_CLOSE* = (IN_CLOSE_WRITE or IN_CLOSE_NOWRITE) ## Close. + IN_OPEN* = 0x00000020 ## File was opened. + IN_MOVED_FROM* = 0x00000040 ## File was moved from X. + IN_MOVED_TO* = 0x00000080 ## File was moved to Y. + IN_MOVE* = (IN_MOVED_FROM or IN_MOVED_TO) ## Moves. + IN_CREATE* = 0x00000100 ## Subfile was created. + IN_DELETE* = 0x00000200 ## Subfile was deleted. + IN_DELETE_SELF* = 0x00000400 ## Self was deleted. + IN_MOVE_SELF* = 0x00000800 ## Self was moved. + # Events sent by the kernel. const - IN_UNMOUNT* = 0x00002000 # Backing fs was unmounted. - IN_Q_OVERFLOW* = 0x00004000 # Event queued overflowed. - IN_IGNORED* = 0x00008000 # File was ignored. + IN_UNMOUNT* = 0x00002000 ## Backing fs was unmounted. + IN_Q_OVERFLOW* = 0x00004000 ## Event queued overflowed. + IN_IGNORED* = 0x00008000 ## File was ignored. + # Special flags. const - IN_ONLYDIR* = 0x01000000 # Only watch the path if it is a - # directory. - IN_DONT_FOLLOW* = 0x02000000 # Do not follow a sym link. - IN_EXCL_UNLINK* = 0x04000000 # Exclude events on unlinked - # objects. - IN_MASK_ADD* = 0x20000000 # Add to the mask of an already - # existing watch. - IN_ISDIR* = 0x40000000 # Event occurred against dir. - IN_ONESHOT* = 0x80000000 # Only send event once. + IN_ONLYDIR* = 0x01000000 ## Only watch the path if it is a directory. + IN_DONT_FOLLOW* = 0x02000000 ## Do not follow a sym link. + IN_EXCL_UNLINK* = 0x04000000 ## Exclude events on unlinked objects. + IN_MASK_ADD* = 0x20000000 ## Add to the mask of an already existing watch. + IN_ISDIR* = 0x40000000 ## Event occurred against dir. + IN_ONESHOT* = 0x80000000 ## Only send event once. + # All events which a program can wait on. const IN_ALL_EVENTS* = (IN_ACCESS or IN_MODIFY or IN_ATTRIB or IN_CLOSE_WRITE or IN_CLOSE_NOWRITE or IN_OPEN or IN_MOVED_FROM or IN_MOVED_TO or IN_CREATE or IN_DELETE or IN_DELETE_SELF or IN_MOVE_SELF) -# Create and initialize inotify instance. -proc inotify_init*(): cint{.cdecl, importc: "inotify_init", - header: "<sys/inotify.h>".} -# Create and initialize inotify instance. -proc inotify_init1*(flags: cint): cint{.cdecl, importc: "inotify_init1", + + +proc inotify_init*(): FileHandle {.cdecl, importc: "inotify_init", + header: "<sys/inotify.h>".} + ## Create and initialize inotify instance. + +proc inotify_init1*(flags: cint): FileHandle {.cdecl, importc: "inotify_init1", header: "<sys/inotify.h>".} -# Add watch of object NAME to inotify instance FD. Notify about -# events specified by MASK. -proc inotify_add_watch*(fd: cint; name: cstring; mask: uint32): cint{. - cdecl, importc: "inotify_add_watch", header: "<sys/inotify.h>".} -# Remove the watch specified by WD from the inotify instance FD. -proc inotify_rm_watch*(fd: cint; wd: cint): cint{.cdecl, + ## Like `inotify_init<#inotify_init>`_ , + ## but has a flags argument that provides access to some extra functionality. + +proc inotify_add_watch*(fd: cint; name: cstring; mask: uint32): cint {.cdecl, + importc: "inotify_add_watch", header: "<sys/inotify.h>".} + ## Add watch of object NAME to inotify instance FD. Notify about events specified by MASK. + +proc inotify_rm_watch*(fd: cint; wd: cint): cint {.cdecl, importc: "inotify_rm_watch", header: "<sys/inotify.h>".} + ## Remove the watch specified by WD from the inotify instance FD. + +iterator inotify_events*(evs: pointer, n: int): ptr InotifyEvent = + ## Abstract the packed buffer interface to yield event object pointers. + runnableExamples("-r:off"): + when defined(linux): + import std/posix # needed for FileHandle read procedure + const MaxWatches = 8192 + + let inotifyFd = inotify_init() # create new inotify instance and get it's FileHandle + let wd = inotifyFd.inotify_add_watch("/tmp", IN_CREATE or IN_DELETE) # Add new watch + + var events: array[MaxWatches, byte] # event buffer + while (let n = read(inotifyFd, addr events, MaxWatches); n) > 0: # blocks until any events have been read + for e in inotify_events(addr events, n): + echo (e[].wd, e[].mask, cast[cstring](addr e[].name)) # echo watch id, mask, and name value of each event + var ev: ptr InotifyEvent = cast[ptr InotifyEvent](evs) + var n = n + while n > 0: + yield ev + let sz = InotifyEvent.sizeof + int(ev[].len) + n -= sz + ev = cast[ptr InotifyEvent](cast[uint](ev) + uint(sz)) + +runnableExamples: + when defined(linux): + let inotifyFd = inotify_init() # create and get new inotify FileHandle + doAssert inotifyFd >= 0 # check for errors + + let wd = inotifyFd.inotify_add_watch("/tmp", IN_CREATE or IN_DELETE) # Add new watch + doAssert wd >= 0 # check for errors + + discard inotifyFd.inotify_rm_watch(wd) # remove watch diff --git a/lib/posix/kqueue.nim b/lib/posix/kqueue.nim index 511ada9ac..2450cdb42 100644 --- a/lib/posix/kqueue.nim +++ b/lib/posix/kqueue.nim @@ -1,30 +1,49 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Adam Strzelecki +# (c) Copyright 2016 Eugene Kabanov # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -{.deadCodeElim:on.} +from std/posix import Timespec -from posix import Timespec - -# Filters: -const - EVFILT_READ* = -1 - EVFILT_WRITE* = -2 - EVFILT_AIO* = -3 - EVFILT_VNODE* = -4 - EVFILT_PROC* = -5 - EVFILT_SIGNAL* = -6 - EVFILT_TIMER* = -7 - EVFILT_MACHPORT* = -8 - EVFILT_FS* = -9 - EVFILT_USER* = -10 - # -11 is unused - EVFILT_VM* = -12 +when defined(macosx) or defined(freebsd) or defined(openbsd) or + defined(dragonfly): + const + EVFILT_READ* = -1 + EVFILT_WRITE* = -2 + EVFILT_AIO* = -3 ## attached to aio requests + EVFILT_VNODE* = -4 ## attached to vnodes + EVFILT_PROC* = -5 ## attached to struct proc + EVFILT_SIGNAL* = -6 ## attached to struct proc + EVFILT_TIMER* = -7 ## timers +elif defined(netbsd): + const + EVFILT_READ* = 0 + EVFILT_WRITE* = 1 + EVFILT_AIO* = 2 ## attached to aio requests + EVFILT_VNODE* = 3 ## attached to vnodes + EVFILT_PROC* = 4 ## attached to struct proc + EVFILT_SIGNAL* = 5 ## attached to struct proc + EVFILT_TIMER* = 6 ## timers (in ms) +when defined(macosx): + const + EVFILT_MACHPORT* = -8 ## Mach portsets + EVFILT_FS* = -9 ## filesystem events + EVFILT_USER* = -10 ## user events + EVFILT_VM = -12 ## virtual memory events +elif defined(freebsd): + const + EVFILT_FS* = -9 ## filesystem events + EVFILT_LIO* = -10 ## attached to lio requests + EVFILT_USER* = -11 ## user events +elif defined(dragonfly): + const + EVFILT_EXCEPT* = -8 ## exceptional conditions + EVFILT_USER* = -9 ## user events + EVFILT_FS* = -10 ## filesystem events # Actions: const @@ -41,20 +60,91 @@ const EV_RECEIPT* = 0x0040 ## Force EV_ERROR on success, data == 0 EV_DISPATCH* = 0x0080 ## Disable event after reporting. + EV_SYSFLAGS* = 0xF000 ## Reserved by system + EV_DROP* = 0x1000 ## Not should be dropped + EV_FLAG1* = 0x2000 ## Filter-specific flag + # Return values: const EV_EOF* = 0x8000 ## EOF detected EV_ERROR* = 0x4000 ## Error, data contains errno + EV_NODATA* = 0x1000 ## EOF and no more data + +when defined(macosx) or defined(freebsd) or defined(dragonfly): + # EVFILT_USER is not supported by OpenBSD and NetBSD + # + # data/hint flags/masks for EVFILT_USER, shared with userspace + # + # On input, the top two bits of fflags specifies how the lower twenty four + # bits should be applied to the stored value of fflags. + # + # On output, the top two bits will always be set to NOTE_FFNOP and the + # remaining twenty four bits will contain the stored fflags value. + const + NOTE_FFNOP* = 0x00000000'u32 ## ignore input fflags + NOTE_FFAND* = 0x40000000'u32 ## AND fflags + NOTE_FFOR* = 0x80000000'u32 ## OR fflags + NOTE_FFCOPY* = 0xc0000000'u32 ## copy fflags + NOTE_FFCTRLMASK* = 0xc0000000'u32 ## masks for operations + NOTE_FFLAGSMASK* = 0x00ffffff'u32 + + NOTE_TRIGGER* = 0x01000000'u32 ## Cause the event to be triggered + ## for output. + +# data/hint flags for EVFILT_{READ|WRITE}, shared with userspace +const + NOTE_LOWAT* = 0x0001 ## low water mark + +# data/hint flags for EVFILT_VNODE, shared with userspace +const + NOTE_DELETE* = 0x0001 ## vnode was removed + NOTE_WRITE* = 0x0002 ## data contents changed + NOTE_EXTEND* = 0x0004 ## size increased + NOTE_ATTRIB* = 0x0008 ## attributes changed + NOTE_LINK* = 0x0010 ## link count changed + NOTE_RENAME* = 0x0020 ## vnode was renamed + NOTE_REVOKE* = 0x0040 ## vnode access was revoked + +# data/hint flags for EVFILT_PROC, shared with userspace +const + NOTE_EXIT* = 0x80000000'u32 ## process exited + NOTE_FORK* = 0x40000000'u32 ## process forked + NOTE_EXEC* = 0x20000000'u32 ## process exec'd + NOTE_PCTRLMASK* = 0xf0000000'u32 ## mask for hint bits + NOTE_PDATAMASK* = 0x000fffff'u32 ## mask for pid + +# additional flags for EVFILT_PROC +const + NOTE_TRACK* = 0x00000001'u32 ## follow across forks + NOTE_TRACKERR* = 0x00000002'u32 ## could not track child + NOTE_CHILD* = 0x00000004'u32 ## am a child process + +when defined(macosx) or defined(freebsd): + # additional flags for EVFILE_TIMER + const + NOTE_SECONDS* = 0x00000001'u32 ## data is seconds + NOTE_MSECONDS* = 0x00000002'u32 ## data is milliseconds + NOTE_USECONDS* = 0x00000004'u32 ## data is microseconds + NOTE_NSECONDS* = 0x00000008'u32 ## data is nanoseconds +else: + # NetBSD and OpenBSD doesn't support NOTE_{TIME} constants, but + # support EVFILT_TIMER with granularity of milliseconds. + const + NOTE_MSECONDS* = 0x00000000'u32 type + ## This define not fully satisfy NetBSD "struct kevent" + ## but it works and tested. KEvent* {.importc: "struct kevent", - header: "<sys/event.h>", pure, final.} = object - ident*: cuint ## identifier for this event (uintptr_t) - filter*: cshort ## filter for event - flags*: cushort ## general flags - fflags*: cuint ## filter-specific flags - data*: cuint ## filter-specific data (intptr_t) - #udata*: ptr void ## opaque user data identifier + header: """#include <sys/types.h> + #include <sys/event.h> + #include <sys/time.h>""", pure, final.} = object + ident* : uint ## identifier for this event (uintptr_t) + filter* : cshort ## filter for event + flags* : cushort ## general flags + fflags* : cuint ## filter-specific flags + data* : int ## filter-specific data (intptr_t) + udata* : pointer ## opaque user data identifier proc kqueue*(): cint {.importc: "kqueue", header: "<sys/event.h>".} ## Creates new queue and returns its descriptor. @@ -63,9 +153,9 @@ proc kevent*(kqFD: cint, changelist: ptr KEvent, nchanges: cint, eventlist: ptr KEvent, nevents: cint, timeout: ptr Timespec): cint {.importc: "kevent", header: "<sys/event.h>".} - ## Manipulates queue for given ``kqFD`` descriptor. + ## Manipulates queue for given `kqFD` descriptor. -proc EV_SET*(event: ptr KEvent, ident: cuint, filter: cshort, flags: cushort, - fflags: cuint, data: cuint, udata: ptr void) +proc EV_SET*(event: ptr KEvent, ident: uint, filter: cshort, flags: cushort, + fflags: cuint, data: int, udata: pointer) {.importc: "EV_SET", header: "<sys/event.h>".} ## Fills event with given data. diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim index 01d5e57de..29fd4288d 100644 --- a/lib/posix/linux.nim +++ b/lib/posix/linux.nim @@ -1,28 +1,39 @@ -{.deadCodeElim:on.} - -import posix +import std/posix +## Flags of `clone` syscall. +## See `clone syscall manual +## <https://man7.org/linux/man-pages/man2/clone.2.html>`_ for more information. const - CSIGNAL* = 0x000000FF - CLONE_VM* = 0x00000100 - CLONE_FS* = 0x00000200 - CLONE_FILES* = 0x00000400 - CLONE_SIGHAND* = 0x00000800 - CLONE_PTRACE* = 0x00002000 - CLONE_VFORK* = 0x00004000 - CLONE_PARENT* = 0x00008000 - CLONE_THREAD* = 0x00010000 - CLONE_NEWNS* = 0x00020000 - CLONE_SYSVSEM* = 0x00040000 - CLONE_SETTLS* = 0x00080000 - CLONE_PARENT_SETTID* = 0x00100000 - CLONE_CHILD_CLEARTID* = 0x00200000 - CLONE_DETACHED* = 0x00400000 - CLONE_UNTRACED* = 0x00800000 - CLONE_CHILD_SETTID* = 0x01000000 - CLONE_STOPPED* = 0x02000000 + CSIGNAL* = 0x000000FF'i32 + CLONE_VM* = 0x00000100'i32 + CLONE_FS* = 0x00000200'i32 + CLONE_FILES* = 0x00000400'i32 + CLONE_SIGHAND* = 0x00000800'i32 + CLONE_PIDFD* = 0x00001000'i32 + CLONE_PTRACE* = 0x00002000'i32 + CLONE_VFORK* = 0x00004000'i32 + CLONE_PARENT* = 0x00008000'i32 + CLONE_THREAD* = 0x00010000'i32 + CLONE_NEWNS* = 0x00020000'i32 + CLONE_SYSVSEM* = 0x00040000'i32 + CLONE_SETTLS* = 0x00080000'i32 + CLONE_PARENT_SETTID* = 0x00100000'i32 + CLONE_CHILD_CLEARTID* = 0x00200000'i32 + CLONE_DETACHED* = 0x00400000'i32 + CLONE_UNTRACED* = 0x00800000'i32 + CLONE_CHILD_SETTID* = 0x01000000'i32 + CLONE_NEWCGROUP* = 0x02000000'i32 + CLONE_NEWUTS* = 0x04000000'i32 + CLONE_NEWIPC* = 0x08000000'i32 + CLONE_NEWUSER* = 0x10000000'i32 + CLONE_NEWPID* = 0x20000000'i32 + CLONE_NEWNET* = 0x40000000'i32 + CLONE_IO* = 0x80000000'i32 + -# fn should be of type proc (a2: pointer): void {.cdecl.} +# fn should be of type proc (a2: pointer) {.cdecl.} proc clone*(fn: pointer; child_stack: pointer; flags: cint; arg: pointer; ptid: ptr Pid; tls: pointer; ctid: ptr Pid): cint {.importc, header: "<sched.h>".} + +proc pipe2*(a: array[0..1, cint], flags: cint): cint {.importc, header: "<unistd.h>".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 5f1dfcfcd..fbe945df3 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -16,7 +16,10 @@ ## convenience: cstrings are used instead of proper Nim strings and ## return codes indicate errors. If you want exceptions ## and a proper Nim-like interface, use the OS module or write a wrapper. - +## +## For high-level wrappers specialized for Linux and BSDs see: +## `posix_utils <posix_utils.html>`_ +## ## Coding conventions: ## ALL types are named the same as in the POSIX standard except that they start ## with 'T' or 'P' (if they are pointers) and without the '_t' suffix to be @@ -24,39 +27,43 @@ ## the \`identifier\` notation is used. ## ## This library relies on the header files of your C compiler. The -## resulting C code will just ``#include <XYZ.h>`` and *not* define the +## resulting C code will just `#include <XYZ.h>` and *not* define the ## symbols declared here. -{.deadCodeElim:on.} +# Dead code elimination ensures that we don't accidentally generate #includes +# for files that might not exist on a specific platform! The user will get an +# error only if they actually try to use the missing declaration -from times import Time +when defined(nimHasStyleChecks): + {.push styleChecks: off.} -const - hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays - hasAioH = defined(linux) +when defined(nimPreviewSlimSystem): + import std/syncio +# TODO these constants don't seem to be fetched from a header file for unknown +# platforms - where do they come from and why are they here? when false: const - C_IRUSR = 0c000400 ## Read by owner. - C_IWUSR = 0c000200 ## Write by owner. - C_IXUSR = 0c000100 ## Execute by owner. - C_IRGRP = 0c000040 ## Read by group. - C_IWGRP = 0c000020 ## Write by group. - C_IXGRP = 0c000010 ## Execute by group. - C_IROTH = 0c000004 ## Read by others. - C_IWOTH = 0c000002 ## Write by others. - C_IXOTH = 0c000001 ## Execute by others. - C_ISUID = 0c004000 ## Set user ID. - C_ISGID = 0c002000 ## Set group ID. - C_ISVTX = 0c001000 ## On directories, restricted deletion flag. - C_ISDIR = 0c040000 ## Directory. - C_ISFIFO = 0c010000 ##FIFO. - C_ISREG = 0c100000 ## Regular file. - C_ISBLK = 0c060000 ## Block special. - C_ISCHR = 0c020000 ## Character special. - C_ISCTG = 0c110000 ## Reserved. - C_ISLNK = 0c120000 ## Symbolic link.</p> - C_ISSOCK = 0c140000 ## Socket. + C_IRUSR = 0o000400 ## Read by owner. + C_IWUSR = 0o000200 ## Write by owner. + C_IXUSR = 0o000100 ## Execute by owner. + C_IRGRP = 0o000040 ## Read by group. + C_IWGRP = 0o000020 ## Write by group. + C_IXGRP = 0o000010 ## Execute by group. + C_IROTH = 0o000004 ## Read by others. + C_IWOTH = 0o000002 ## Write by others. + C_IXOTH = 0o000001 ## Execute by others. + C_ISUID = 0o004000 ## Set user ID. + C_ISGID = 0o002000 ## Set group ID. + C_ISVTX = 0o001000 ## On directories, restricted deletion flag. + C_ISDIR = 0o040000 ## Directory. + C_ISFIFO = 0o010000 ##FIFO. + C_ISREG = 0o100000 ## Regular file. + C_ISBLK = 0o060000 ## Block special. + C_ISCHR = 0o020000 ## Character special. + C_ISCTG = 0o110000 ## Reserved. + C_ISLNK = 0o120000 ## Symbolic link.</p> + C_ISSOCK = 0o140000 ## Socket. const MM_NULLLBL* = nil @@ -80,1714 +87,54 @@ const DT_SOCK* = 12 ## UNIX domain socket. DT_WHT* = 14 -type - DIR* {.importc: "DIR", header: "<dirent.h>", - incompleteStruct.} = object - ## A type representing a directory stream. -{.deprecated: [TDIR: DIR].} - -type - SocketHandle* = distinct cint # The type used to represent socket descriptors - -{.deprecated: [TSocketHandle: SocketHandle].} - -type - Dirent* {.importc: "struct dirent", - header: "<dirent.h>", final, pure.} = object ## dirent_t struct - d_ino*: Ino ## File serial number. - when defined(linux) or defined(macosx) or defined(bsd): - d_reclen*: cshort ## Length of this record. (not POSIX) - d_type*: int8 ## Type of file; not supported by all filesystem types. - ## (not POSIX) - when defined(linux) or defined(bsd): - d_off*: Off ## Not an offset. Value that ``telldir()`` would return. - d_name*: array [0..255, char] ## Name of entry. - - Tflock* {.importc: "struct flock", final, pure, - header: "<fcntl.h>".} = object ## flock type - l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. - l_whence*: cshort ## Flag for starting offset. - l_start*: Off ## Relative offset in bytes. - l_len*: Off ## Size; if 0 then until EOF. - l_pid*: Pid ## Process ID of the process holding the lock; - ## returned with F_GETLK. - - FTW* {.importc: "struct FTW", header: "<ftw.h>", final, pure.} = object - base*: cint - level*: cint - - Glob* {.importc: "glob_t", header: "<glob.h>", - final, pure.} = object ## glob_t - gl_pathc*: int ## Count of paths matched by pattern. - gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. - gl_offs*: int ## Slots to reserve at the beginning of gl_pathv. - - Group* {.importc: "struct group", header: "<grp.h>", - final, pure.} = object ## struct group - gr_name*: cstring ## The name of the group. - gr_gid*: Gid ## Numerical group ID. - gr_mem*: cstringArray ## Pointer to a null-terminated array of character - ## pointers to member names. - - Iconv* {.importc: "iconv_t", header: "<iconv.h>", final, pure.} = - object ## Identifies the conversion from one codeset to another. - - Lconv* {.importc: "struct lconv", header: "<locale.h>", final, - pure.} = object - currency_symbol*: cstring - decimal_point*: cstring - frac_digits*: char - grouping*: cstring - int_curr_symbol*: cstring - int_frac_digits*: char - int_n_cs_precedes*: char - int_n_sep_by_space*: char - int_n_sign_posn*: char - int_p_cs_precedes*: char - int_p_sep_by_space*: char - int_p_sign_posn*: char - mon_decimal_point*: cstring - mon_grouping*: cstring - mon_thousands_sep*: cstring - negative_sign*: cstring - n_cs_precedes*: char - n_sep_by_space*: char - n_sign_posn*: char - positive_sign*: cstring - p_cs_precedes*: char - p_sep_by_space*: char - p_sign_posn*: char - thousands_sep*: cstring - - Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object - MqAttr* {.importc: "struct mq_attr", - header: "<mqueue.h>", - final, pure.} = object ## message queue attribute - mq_flags*: int ## Message queue flags. - mq_maxmsg*: int ## Maximum number of messages. - mq_msgsize*: int ## Maximum message size. - mq_curmsgs*: int ## Number of messages currently queued. - - Passwd* {.importc: "struct passwd", header: "<pwd.h>", - final, pure.} = object ## struct passwd - pw_name*: cstring ## User's login name. - pw_uid*: Uid ## Numerical user ID. - pw_gid*: Gid ## Numerical group ID. - pw_dir*: cstring ## Initial working directory. - pw_shell*: cstring ## Program to use as shell. - - Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = int - ## used for file block counts - Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = int - ## used for block sizes - Clock* {.importc: "clock_t", header: "<sys/types.h>".} = int - ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int - Dev* {.importc: "dev_t", header: "<sys/types.h>".} = int - Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int - Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int - Gid* {.importc: "gid_t", header: "<sys/types.h>".} = int - Id* {.importc: "id_t", header: "<sys/types.h>".} = int - Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int - Key* {.importc: "key_t", header: "<sys/types.h>".} = int - Mode* {.importc: "mode_t", header: "<sys/types.h>".} = cint - Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int - Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 - Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int - Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = int - Pthread_barrier* {.importc: "pthread_barrier_t", - header: "<sys/types.h>".} = int - Pthread_barrierattr* {.importc: "pthread_barrierattr_t", - header: "<sys/types.h>".} = int - Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>".} = int - Pthread_condattr* {.importc: "pthread_condattr_t", - header: "<sys/types.h>".} = int - Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = int - Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>".} = int - Pthread_mutexattr* {.importc: "pthread_mutexattr_t", - header: "<sys/types.h>".} = int - Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = int - Pthread_rwlock* {.importc: "pthread_rwlock_t", - header: "<sys/types.h>".} = int - Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", - header: "<sys/types.h>".} = int - Pthread_spinlock* {.importc: "pthread_spinlock_t", - header: "<sys/types.h>".} = int - Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = int - Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = int - #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int - Timer* {.importc: "timer_t", header: "<sys/types.h>".} = int - Trace_attr* {.importc: "trace_attr_t", header: "<sys/types.h>".} = int - Trace_event_id* {.importc: "trace_event_id_t", - header: "<sys/types.h>".} = int - Trace_event_set* {.importc: "trace_event_set_t", - header: "<sys/types.h>".} = int - Trace_id* {.importc: "trace_id_t", header: "<sys/types.h>".} = int - Uid* {.importc: "uid_t", header: "<sys/types.h>".} = int - Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = int - - Utsname* {.importc: "struct utsname", - header: "<sys/utsname.h>", - final, pure.} = object ## struct utsname - sysname*, ## Name of this implementation of the operating system. - nodename*, ## Name of this node within the communications - ## network to which this node is attached, if any. - release*, ## Current release level of this implementation. - version*, ## Current version level of this release. - machine*: array [0..255, char] ## Name of the hardware type on which the - ## system is running. - - Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object - Ipc_perm* {.importc: "struct ipc_perm", - header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm - uid*: Uid ## Owner's user ID. - gid*: Gid ## Owner's group ID. - cuid*: Uid ## Creator's user ID. - cgid*: Gid ## Creator's group ID. - mode*: Mode ## Read/write permission. - - Stat* {.importc: "struct stat", - header: "<sys/stat.h>", final, pure.} = object ## struct stat - st_dev*: Dev ## Device ID of device containing file. - st_ino*: Ino ## File serial number. - st_mode*: Mode ## Mode of file (see below). - st_nlink*: Nlink ## Number of hard links to the file. - st_uid*: Uid ## User ID of file. - st_gid*: Gid ## Group ID of file. - st_rdev*: Dev ## Device ID (if file is character or block special). - st_size*: Off ## For regular files, the file size in bytes. - ## For symbolic links, the length in bytes of the - ## pathname contained in the symbolic link. - ## For a shared memory object, the length in bytes. - ## For a typed memory object, the length in bytes. - ## For other file types, the use of this field is - ## unspecified. - st_atime*: Time ## Time of last access. - st_mtime*: Time ## Time of last data modification. - st_ctime*: Time ## Time of last status change. - st_blksize*: Blksize ## A file system-specific preferred I/O block size - ## for this object. In some file system types, this - ## may vary from file to file. - st_blocks*: Blkcnt ## Number of blocks allocated for this object. - - - Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", - final, pure.} = object ## struct statvfs - f_bsize*: int ## File system block size. - f_frsize*: int ## Fundamental file system block size. - f_blocks*: Fsblkcnt ## Total number of blocks on file system - ## in units of f_frsize. - f_bfree*: Fsblkcnt ## Total number of free blocks. - f_bavail*: Fsblkcnt ## Number of free blocks available to - ## non-privileged process. - f_files*: Fsfilcnt ## Total number of file serial numbers. - f_ffree*: Fsfilcnt ## Total number of free file serial numbers. - f_favail*: Fsfilcnt ## Number of file serial numbers available to - ## non-privileged process. - f_fsid*: int ## File system ID. - f_flag*: int ## Bit mask of f_flag values. - f_namemax*: int ## Maximum filename length. - - Posix_typed_mem_info* {.importc: "struct posix_typed_mem_info", - header: "<sys/mman.h>", final, pure.} = object - posix_tmi_length*: int - - Tm* {.importc: "struct tm", header: "<time.h>", - final, pure.} = object ## struct tm - tm_sec*: cint ## Seconds [0,60]. - tm_min*: cint ## Minutes [0,59]. - tm_hour*: cint ## Hour [0,23]. - tm_mday*: cint ## Day of month [1,31]. - tm_mon*: cint ## Month of year [0,11]. - tm_year*: cint ## Years since 1900. - tm_wday*: cint ## Day of week [0,6] (Sunday =0). - tm_yday*: cint ## Day of year [0,365]. - tm_isdst*: cint ## Daylight Savings flag. - Timespec* {.importc: "struct timespec", - header: "<time.h>", final, pure.} = object ## struct timespec - tv_sec*: Time ## Seconds. - tv_nsec*: int ## Nanoseconds. - Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", - final, pure.} = object ## struct itimerspec - it_interval*: Timespec ## Timer period. - it_value*: Timespec ## Timer expiration. - - Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint - ## Possibly volatile-qualified integer type of an object that can be - ## accessed as an atomic entity, even in the presence of asynchronous - ## interrupts. - Sigset* {.importc: "sigset_t", header: "<signal.h>", final, pure.} = object - - SigEvent* {.importc: "struct sigevent", - header: "<signal.h>", final, pure.} = object ## struct sigevent - sigev_notify*: cint ## Notification type. - sigev_signo*: cint ## Signal number. - sigev_value*: SigVal ## Signal value. - sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. - sigev_notify_attributes*: ptr PthreadAttr ## Notification attributes. - - SigVal* {.importc: "union sigval", - header: "<signal.h>", final, pure.} = object ## struct sigval - sival_ptr*: pointer ## pointer signal value; - ## integer signal value not defined! - Sigaction* {.importc: "struct sigaction", - header: "<signal.h>", final, pure.} = object ## struct sigaction - sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching - ## function or one of the macros - ## SIG_IGN or SIG_DFL. - sa_mask*: Sigset ## Set of signals to be blocked during execution of - ## the signal handling function. - sa_flags*: cint ## Special flags. - sa_sigaction*: proc (x: cint, y: var SigInfo, z: pointer) {.noconv.} - - Stack* {.importc: "stack_t", - header: "<signal.h>", final, pure.} = object ## stack_t - ss_sp*: pointer ## Stack base or pointer. - ss_size*: int ## Stack size. - ss_flags*: cint ## Flags. - - SigStack* {.importc: "struct sigstack", - header: "<signal.h>", final, pure.} = object ## struct sigstack - ss_onstack*: cint ## Non-zero when signal stack is in use. - ss_sp*: pointer ## Signal stack pointer. - - SigInfo* {.importc: "siginfo_t", - header: "<signal.h>", final, pure.} = object ## siginfo_t - si_signo*: cint ## Signal number. - si_code*: cint ## Signal code. - si_errno*: cint ## If non-zero, an errno value associated with - ## this signal, as defined in <errno.h>. - si_pid*: Pid ## Sending process ID. - si_uid*: Uid ## Real user ID of sending process. - si_addr*: pointer ## Address of faulting instruction. - si_status*: cint ## Exit value or signal. - si_band*: int ## Band event for SIGPOLL. - si_value*: SigVal ## Signal value. - - Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint - Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = cint - - Sched_param* {.importc: "struct sched_param", - header: "<sched.h>", - final, pure.} = object ## struct sched_param - sched_priority*: cint - sched_ss_low_priority*: cint ## Low scheduling priority for - ## sporadic server. - sched_ss_repl_period*: Timespec ## Replenishment period for - ## sporadic server. - sched_ss_init_budget*: Timespec ## Initial budget for sporadic server. - sched_ss_max_repl*: cint ## Maximum pending replenishments for - ## sporadic server. - - Timeval* {.importc: "struct timeval", header: "<sys/select.h>", - final, pure.} = object ## struct timeval - tv_sec*: int ## Seconds. - tv_usec*: int ## Microseconds. - TFdSet* {.importc: "fd_set", header: "<sys/select.h>", - final, pure.} = object - Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", - final, pure.} = object - Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", - final, pure.} = object ## ucontext_t - uc_link*: ptr Ucontext ## Pointer to the context that is resumed - ## when this context returns. - uc_sigmask*: Sigset ## The set of signals that are blocked when this - ## context is active. - uc_stack*: Stack ## The stack used by this context. - uc_mcontext*: Mcontext ## A machine-specific representation of the saved - ## context. -{.deprecated: [TOff: Off, TPid: Pid, TGid: Gid, TMode: Mode, TDev: Dev, - TNlink: Nlink, TStack: Stack, TGroup: Group, TMqd: Mqd, - TPasswd: Passwd, TClock: Clock, TClockId: ClockId, TKey: Key, - TSem: Sem, Tpthread_attr: PthreadAttr, Ttimespec: Timespec, - Tdirent: Dirent, TFTW: FTW, TGlob: Glob, - # Tflock: Flock, # Naming conflict if we drop the `T` - Ticonv: Iconv, Tlconv: Lconv, TMqAttr: MqAttr, Tblkcnt: Blkcnt, - Tblksize: Blksize, Tfsblkcnt: Fsblkcnt, Tfsfilcnt: Fsfilcnt, - Tid: Id, Tino: Ino, Tpthread_barrier: Pthread_barrier, - Tpthread_barrierattr: Pthread_barrierattr, Tpthread_cond: Pthread_cond, - TPthread_condattr: Pthread_condattr, Tpthread_key: Pthread_key, - Tpthread_mutex: Pthread_mutex, Tpthread_mutexattr: Pthread_mutexattr, - Tpthread_once: Pthread_once, Tpthread_rwlock: Pthread_rwlock, - Tpthread_rwlockattr: Pthread_rwlockattr, Tpthread_spinlock: Pthread_spinlock, - Tpthread: Pthread, Tsuseconds: Suseconds, Ttimer: Timer, - Ttrace_attr: Trace_attr, Ttrace_event_id: Trace_event_id, - Ttrace_event_set: Trace_event_set, Ttrace_id: Trace_id, - Tuid: Uid, Tuseconds: Useconds, Tutsname: Utsname, Tipc_perm: Ipc_perm, - TStat: Stat, TStatvfs: Statvfs, Tposix_typed_mem_info: Posix_typed_mem_info, - Ttm: Tm, titimerspec: Itimerspec, Tsig_atomic: Sig_atomic, Tsigset: Sigset, - TsigEvent: SigEvent, TsigVal: SigVal, TSigaction: Sigaction, - TSigStack: SigStack, TsigInfo: SigInfo, Tnl_item: Nl_item, - Tnl_catd: Nl_catd, Tsched_param: Sched_param, - # TFdSet: FdSet, # Naming conflict if we drop the `T` - Tmcontext: Mcontext, Tucontext: Ucontext].} -when hasAioH: - type - Taiocb* {.importc: "struct aiocb", header: "<aio.h>", - final, pure.} = object ## struct aiocb - aio_fildes*: cint ## File descriptor. - aio_offset*: Off ## File offset. - aio_buf*: pointer ## Location of buffer. - aio_nbytes*: int ## Length of transfer. - aio_reqprio*: cint ## Request priority offset. - aio_sigevent*: SigEvent ## Signal number and value. - aio_lio_opcode: cint ## Operation to be performed. - -when hasSpawnH: - type - Tposix_spawnattr* {.importc: "posix_spawnattr_t", - header: "<spawn.h>", final, pure.} = object - Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", - header: "<spawn.h>", final, pure.} = object - -type - Socklen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint - TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cint - - SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", - pure, final.} = object ## struct sockaddr - sa_family*: TSa_Family ## Address family. - sa_data*: array [0..255, char] ## Socket address (variable-length data). - - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "<sys/socket.h>", - pure, final.} = object ## struct sockaddr_storage - ss_family*: TSa_Family ## Address family. - - Tif_nameindex* {.importc: "struct if_nameindex", final, - pure, header: "<net/if.h>".} = object ## struct if_nameindex - if_index*: cint ## Numeric index of the interface. - if_name*: cstring ## Null-terminated name of the interface. - - - IOVec* {.importc: "struct iovec", pure, final, - header: "<sys/uio.h>".} = object ## struct iovec - iov_base*: pointer ## Base address of a memory region for input or output. - iov_len*: int ## The size of the memory pointed to by iov_base. - - Tmsghdr* {.importc: "struct msghdr", pure, final, - header: "<sys/socket.h>".} = object ## struct msghdr - msg_name*: pointer ## Optional address. - msg_namelen*: Socklen ## Size of address. - msg_iov*: ptr IOVec ## Scatter/gather array. - msg_iovlen*: cint ## Members in msg_iov. - msg_control*: pointer ## Ancillary data; see below. - msg_controllen*: Socklen ## Ancillary data buffer len. - msg_flags*: cint ## Flags on received message. - - - Tcmsghdr* {.importc: "struct cmsghdr", pure, final, - header: "<sys/socket.h>".} = object ## struct cmsghdr - cmsg_len*: Socklen ## Data byte count, including the cmsghdr. - cmsg_level*: cint ## Originating protocol. - cmsg_type*: cint ## Protocol-specific type. - - TLinger* {.importc: "struct linger", pure, final, - header: "<sys/socket.h>".} = object ## struct linger - l_onoff*: cint ## Indicates whether linger option is enabled. - l_linger*: cint ## Linger time, in seconds. - - InPort* = int16 ## unsigned! - InAddrScalar* = int32 ## unsigned! - - InAddrT* {.importc: "in_addr_t", pure, final, - header: "<netinet/in.h>".} = int32 ## unsigned! - - InAddr* {.importc: "struct in_addr", pure, final, - header: "<netinet/in.h>".} = object ## struct in_addr - s_addr*: InAddrScalar - - Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, - header: "<netinet/in.h>".} = object ## struct sockaddr_in - sin_family*: TSa_Family ## AF_INET. - sin_port*: InPort ## Port number. - sin_addr*: InAddr ## IP address. - - In6Addr* {.importc: "struct in6_addr", pure, final, - header: "<netinet/in.h>".} = object ## struct in6_addr - s6_addr*: array [0..15, char] - - Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, - header: "<netinet/in.h>".} = object ## struct sockaddr_in6 - sin6_family*: TSa_Family ## AF_INET6. - sin6_port*: InPort ## Port number. - sin6_flowinfo*: int32 ## IPv6 traffic class and flow information. - sin6_addr*: In6Addr ## IPv6 address. - sin6_scope_id*: int32 ## Set of interfaces for a scope. - - Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, - header: "<netinet/in.h>".} = object ## struct ipv6_mreq - ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. - ipv6mr_interface*: cint ## Interface index. - - Hostent* {.importc: "struct hostent", pure, final, - header: "<netdb.h>".} = object ## struct hostent - h_name*: cstring ## Official name of the host. - h_aliases*: cstringArray ## A pointer to an array of pointers to - ## alternative host names, terminated by a - ## null pointer. - h_addrtype*: cint ## Address type. - h_length*: cint ## The length, in bytes, of the address. - h_addr_list*: cstringArray ## A pointer to an array of pointers to network - ## addresses (in network byte order) for the - ## host, terminated by a null pointer. - - Tnetent* {.importc: "struct netent", pure, final, - header: "<netdb.h>".} = object ## struct netent - n_name*: cstring ## Official, fully-qualified (including the - ## domain) name of the host. - n_aliases*: cstringArray ## A pointer to an array of pointers to - ## alternative network names, terminated by a - ## null pointer. - n_addrtype*: cint ## The address type of the network. - n_net*: int32 ## The network number, in host byte order. - - Protoent* {.importc: "struct protoent", pure, final, - header: "<netdb.h>".} = object ## struct protoent - p_name*: cstring ## Official name of the protocol. - p_aliases*: cstringArray ## A pointer to an array of pointers to - ## alternative protocol names, terminated by - ## a null pointer. - p_proto*: cint ## The protocol number. - - Servent* {.importc: "struct servent", pure, final, - header: "<netdb.h>".} = object ## struct servent - s_name*: cstring ## Official name of the service. - s_aliases*: cstringArray ## A pointer to an array of pointers to - ## alternative service names, terminated by - ## a null pointer. - s_port*: cint ## The port number at which the service - ## resides, in network byte order. - s_proto*: cstring ## The name of the protocol to use when - ## contacting the service. - - AddrInfo* {.importc: "struct addrinfo", pure, final, - header: "<netdb.h>".} = object ## struct addrinfo - ai_flags*: cint ## Input flags. - ai_family*: cint ## Address family of socket. - ai_socktype*: cint ## Socket type. - ai_protocol*: cint ## Protocol of socket. - ai_addrlen*: Socklen ## Length of socket address. - ai_addr*: ptr SockAddr ## Socket address of socket. - ai_canonname*: cstring ## Canonical name of service location. - ai_next*: ptr AddrInfo ## Pointer to next in list. - - TPollfd* {.importc: "struct pollfd", pure, final, - header: "<poll.h>".} = object ## struct pollfd - fd*: cint ## The following descriptor being polled. - events*: cshort ## The input event flags (see below). - revents*: cshort ## The output event flags (see below). - - Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint - -{.deprecated: [TSockaddr_in: Sockaddr_in, TAddrinfo: AddrInfo, - TSockAddr: SockAddr, TSockLen: SockLen, TTimeval: Timeval, - Tsockaddr_storage: Sockaddr_storage, Tsockaddr_in6: Sockaddr_in6, - Thostent: Hostent, TServent: Servent, - TInAddr: InAddr, TIOVec: IOVec, TInPort: InPort, TInAddrT: InAddrT, - TIn6Addr: In6Addr, TInAddrScalar: InAddrScalar, TProtoent: Protoent].} - -var - errno* {.importc, header: "<errno.h>".}: cint ## error variable - h_errno* {.importc, header: "<netdb.h>".}: cint - daylight* {.importc, header: "<time.h>".}: cint - timezone* {.importc, header: "<time.h>".}: int - -# Constants as variables: -when hasAioH: - var - AIO_ALLDONE* {.importc, header: "<aio.h>".}: cint - ## A return value indicating that none of the requested operations - ## could be canceled since they are already complete. - AIO_CANCELED* {.importc, header: "<aio.h>".}: cint - ## A return value indicating that all requested operations have - ## been canceled. - AIO_NOTCANCELED* {.importc, header: "<aio.h>".}: cint - ## A return value indicating that some of the requested operations could - ## not be canceled since they are in progress. - LIO_NOP* {.importc, header: "<aio.h>".}: cint - ## A lio_listio() element operation option indicating that no transfer is - ## requested. - LIO_NOWAIT* {.importc, header: "<aio.h>".}: cint - ## A lio_listio() synchronization operation indicating that the calling - ## thread is to continue execution while the lio_listio() operation is - ## being performed, and no notification is given when the operation is - ## complete. - LIO_READ* {.importc, header: "<aio.h>".}: cint - ## A lio_listio() element operation option requesting a read. - LIO_WAIT* {.importc, header: "<aio.h>".}: cint - ## A lio_listio() synchronization operation indicating that the calling - ## thread is to suspend until the lio_listio() operation is complete. - LIO_WRITE* {.importc, header: "<aio.h>".}: cint - ## A lio_listio() element operation option requesting a write. - -var - RTLD_LAZY* {.importc, header: "<dlfcn.h>".}: cint - ## Relocations are performed at an implementation-defined time. - RTLD_NOW* {.importc, header: "<dlfcn.h>".}: cint - ## Relocations are performed when the object is loaded. - RTLD_GLOBAL* {.importc, header: "<dlfcn.h>".}: cint - ## All symbols are available for relocation processing of other modules. - RTLD_LOCAL* {.importc, header: "<dlfcn.h>".}: cint - ## All symbols are not made available for relocation processing by - ## other modules. - - E2BIG* {.importc, header: "<errno.h>".}: cint - ## Argument list too long. - EACCES* {.importc, header: "<errno.h>".}: cint - ## Permission denied. - EADDRINUSE* {.importc, header: "<errno.h>".}: cint - ## Address in use. - EADDRNOTAVAIL* {.importc, header: "<errno.h>".}: cint - ## Address not available. - EAFNOSUPPORT* {.importc, header: "<errno.h>".}: cint - ## Address family not supported. - EAGAIN* {.importc, header: "<errno.h>".}: cint - ## Resource unavailable, try again (may be the same value as EWOULDBLOCK). - EALREADY* {.importc, header: "<errno.h>".}: cint - ## Connection already in progress. - EBADF* {.importc, header: "<errno.h>".}: cint - ## Bad file descriptor. - EBADMSG* {.importc, header: "<errno.h>".}: cint - ## Bad message. - EBUSY* {.importc, header: "<errno.h>".}: cint - ## Device or resource busy. - ECANCELED* {.importc, header: "<errno.h>".}: cint - ## Operation canceled. - ECHILD* {.importc, header: "<errno.h>".}: cint - ## No child processes. - ECONNABORTED* {.importc, header: "<errno.h>".}: cint - ## Connection aborted. - ECONNREFUSED* {.importc, header: "<errno.h>".}: cint - ## Connection refused. - ECONNRESET* {.importc, header: "<errno.h>".}: cint - ## Connection reset. - EDEADLK* {.importc, header: "<errno.h>".}: cint - ## Resource deadlock would occur. - EDESTADDRREQ* {.importc, header: "<errno.h>".}: cint - ## Destination address required. - EDOM* {.importc, header: "<errno.h>".}: cint - ## Mathematics argument out of domain of function. - EDQUOT* {.importc, header: "<errno.h>".}: cint - ## Reserved. - EEXIST* {.importc, header: "<errno.h>".}: cint - ## File exists. - EFAULT* {.importc, header: "<errno.h>".}: cint - ## Bad address. - EFBIG* {.importc, header: "<errno.h>".}: cint - ## File too large. - EHOSTUNREACH* {.importc, header: "<errno.h>".}: cint - ## Host is unreachable. - EIDRM* {.importc, header: "<errno.h>".}: cint - ## Identifier removed. - EILSEQ* {.importc, header: "<errno.h>".}: cint - ## Illegal byte sequence. - EINPROGRESS* {.importc, header: "<errno.h>".}: cint - ## Operation in progress. - EINTR* {.importc, header: "<errno.h>".}: cint - ## Interrupted function. - EINVAL* {.importc, header: "<errno.h>".}: cint - ## Invalid argument. - EIO* {.importc, header: "<errno.h>".}: cint - ## I/O error. - EISCONN* {.importc, header: "<errno.h>".}: cint - ## Socket is connected. - EISDIR* {.importc, header: "<errno.h>".}: cint - ## Is a directory. - ELOOP* {.importc, header: "<errno.h>".}: cint - ## Too many levels of symbolic links. - EMFILE* {.importc, header: "<errno.h>".}: cint - ## Too many open files. - EMLINK* {.importc, header: "<errno.h>".}: cint - ## Too many links. - EMSGSIZE* {.importc, header: "<errno.h>".}: cint - ## Message too large. - EMULTIHOP* {.importc, header: "<errno.h>".}: cint - ## Reserved. - ENAMETOOLONG* {.importc, header: "<errno.h>".}: cint - ## Filename too long. - ENETDOWN* {.importc, header: "<errno.h>".}: cint - ## Network is down. - ENETRESET* {.importc, header: "<errno.h>".}: cint - ## Connection aborted by network. - ENETUNREACH* {.importc, header: "<errno.h>".}: cint - ## Network unreachable. - ENFILE* {.importc, header: "<errno.h>".}: cint - ## Too many files open in system. - ENOBUFS* {.importc, header: "<errno.h>".}: cint - ## No buffer space available. - ENODATA* {.importc, header: "<errno.h>".}: cint - ## No message is available on the STREAM head read queue. - ENODEV* {.importc, header: "<errno.h>".}: cint - ## No such device. - ENOENT* {.importc, header: "<errno.h>".}: cint - ## No such file or directory. - ENOEXEC* {.importc, header: "<errno.h>".}: cint - ## Executable file format error. - ENOLCK* {.importc, header: "<errno.h>".}: cint - ## No locks available. - ENOLINK* {.importc, header: "<errno.h>".}: cint - ## Reserved. - ENOMEM* {.importc, header: "<errno.h>".}: cint - ## Not enough space. - ENOMSG* {.importc, header: "<errno.h>".}: cint - ## No message of the desired type. - ENOPROTOOPT* {.importc, header: "<errno.h>".}: cint - ## Protocol not available. - ENOSPC* {.importc, header: "<errno.h>".}: cint - ## No space left on device. - ENOSR* {.importc, header: "<errno.h>".}: cint - ## No STREAM resources. - ENOSTR* {.importc, header: "<errno.h>".}: cint - ## Not a STREAM. - ENOSYS* {.importc, header: "<errno.h>".}: cint - ## Function not supported. - ENOTCONN* {.importc, header: "<errno.h>".}: cint - ## The socket is not connected. - ENOTDIR* {.importc, header: "<errno.h>".}: cint - ## Not a directory. - ENOTEMPTY* {.importc, header: "<errno.h>".}: cint - ## Directory not empty. - ENOTSOCK* {.importc, header: "<errno.h>".}: cint - ## Not a socket. - ENOTSUP* {.importc, header: "<errno.h>".}: cint - ## Not supported. - ENOTTY* {.importc, header: "<errno.h>".}: cint - ## Inappropriate I/O control operation. - ENXIO* {.importc, header: "<errno.h>".}: cint - ## No such device or address. - EOPNOTSUPP* {.importc, header: "<errno.h>".}: cint - ## Operation not supported on socket. - EOVERFLOW* {.importc, header: "<errno.h>".}: cint - ## Value too large to be stored in data type. - EPERM* {.importc, header: "<errno.h>".}: cint - ## Operation not permitted. - EPIPE* {.importc, header: "<errno.h>".}: cint - ## Broken pipe. - EPROTO* {.importc, header: "<errno.h>".}: cint - ## Protocol error. - EPROTONOSUPPORT* {.importc, header: "<errno.h>".}: cint - ## Protocol not supported. - EPROTOTYPE* {.importc, header: "<errno.h>".}: cint - ## Protocol wrong type for socket. - ERANGE* {.importc, header: "<errno.h>".}: cint - ## Result too large. - EROFS* {.importc, header: "<errno.h>".}: cint - ## Read-only file system. - ESPIPE* {.importc, header: "<errno.h>".}: cint - ## Invalid seek. - ESRCH* {.importc, header: "<errno.h>".}: cint - ## No such process. - ESTALE* {.importc, header: "<errno.h>".}: cint - ## Reserved. - ETIME* {.importc, header: "<errno.h>".}: cint - ## Stream ioctl() timeout. - ETIMEDOUT* {.importc, header: "<errno.h>".}: cint - ## Connection timed out. - ETXTBSY* {.importc, header: "<errno.h>".}: cint - ## Text file busy. - EWOULDBLOCK* {.importc, header: "<errno.h>".}: cint - ## Operation would block (may be the same value as [EAGAIN]). - EXDEV* {.importc, header: "<errno.h>".}: cint - ## Cross-device link. - - F_DUPFD* {.importc, header: "<fcntl.h>".}: cint - ## Duplicate file descriptor. - F_GETFD* {.importc, header: "<fcntl.h>".}: cint - ## Get file descriptor flags. - F_SETFD* {.importc, header: "<fcntl.h>".}: cint - ## Set file descriptor flags. - F_GETFL* {.importc, header: "<fcntl.h>".}: cint - ## Get file status flags and file access modes. - F_SETFL* {.importc, header: "<fcntl.h>".}: cint - ## Set file status flags. - F_GETLK* {.importc, header: "<fcntl.h>".}: cint - ## Get record locking information. - F_SETLK* {.importc, header: "<fcntl.h>".}: cint - ## Set record locking information. - F_SETLKW* {.importc, header: "<fcntl.h>".}: cint - ## Set record locking information; wait if blocked. - F_GETOWN* {.importc, header: "<fcntl.h>".}: cint - ## Get process or process group ID to receive SIGURG signals. - F_SETOWN* {.importc, header: "<fcntl.h>".}: cint - ## Set process or process group ID to receive SIGURG signals. - FD_CLOEXEC* {.importc, header: "<fcntl.h>".}: cint - ## Close the file descriptor upon execution of an exec family function. - F_RDLCK* {.importc, header: "<fcntl.h>".}: cint - ## Shared or read lock. - F_UNLCK* {.importc, header: "<fcntl.h>".}: cint - ## Unlock. - F_WRLCK* {.importc, header: "<fcntl.h>".}: cint - ## Exclusive or write lock. - O_CREAT* {.importc, header: "<fcntl.h>".}: cint - ## Create file if it does not exist. - O_EXCL* {.importc, header: "<fcntl.h>".}: cint - ## Exclusive use flag. - O_NOCTTY* {.importc, header: "<fcntl.h>".}: cint - ## Do not assign controlling terminal. - O_TRUNC* {.importc, header: "<fcntl.h>".}: cint - ## Truncate flag. - O_APPEND* {.importc, header: "<fcntl.h>".}: cint - ## Set append mode. - O_DSYNC* {.importc, header: "<fcntl.h>".}: cint - ## Write according to synchronized I/O data integrity completion. - O_NONBLOCK* {.importc, header: "<fcntl.h>".}: cint - ## Non-blocking mode. - O_RSYNC* {.importc, header: "<fcntl.h>".}: cint - ## Synchronized read I/O operations. - O_SYNC* {.importc, header: "<fcntl.h>".}: cint - ## Write according to synchronized I/O file integrity completion. - O_ACCMODE* {.importc, header: "<fcntl.h>".}: cint - ## Mask for file access modes. - O_RDONLY* {.importc, header: "<fcntl.h>".}: cint - ## Open for reading only. - O_RDWR* {.importc, header: "<fcntl.h>".}: cint - ## Open for reading and writing. - O_WRONLY* {.importc, header: "<fcntl.h>".}: cint - ## Open for writing only. - POSIX_FADV_NORMAL* {.importc, header: "<fcntl.h>".}: cint - ## The application has no advice to give on its behavior with - ## respect to the specified data. It is the default characteristic - ## if no advice is given for an open file. - POSIX_FADV_SEQUENTIAL* {.importc, header: "<fcntl.h>".}: cint - ## The application expects to access the specified data - # sequentially from lower offsets to higher offsets. - POSIX_FADV_RANDOM* {.importc, header: "<fcntl.h>".}: cint - ## The application expects to access the specified data in a random order. - POSIX_FADV_WILLNEED* {.importc, header: "<fcntl.h>".}: cint - ## The application expects to access the specified data in the near future. - POSIX_FADV_DONTNEED* {.importc, header: "<fcntl.h>".}: cint - ## The application expects that it will not access the specified data - ## in the near future. - POSIX_FADV_NOREUSE* {.importc, header: "<fcntl.h>".}: cint - ## The application expects to access the specified data once and - ## then not reuse it thereafter. - -when not defined(haiku) and not defined(OpenBSD): - var - MM_HARD* {.importc, header: "<fmtmsg.h>".}: cint - ## Source of the condition is hardware. - MM_SOFT* {.importc, header: "<fmtmsg.h>".}: cint - ## Source of the condition is software. - MM_FIRM* {.importc, header: "<fmtmsg.h>".}: cint - ## Source of the condition is firmware. - MM_APPL* {.importc, header: "<fmtmsg.h>".}: cint - ## Condition detected by application. - MM_UTIL* {.importc, header: "<fmtmsg.h>".}: cint - ## Condition detected by utility. - MM_OPSYS* {.importc, header: "<fmtmsg.h>".}: cint - ## Condition detected by operating system. - MM_RECOVER* {.importc, header: "<fmtmsg.h>".}: cint - ## Recoverable error. - MM_NRECOV* {.importc, header: "<fmtmsg.h>".}: cint - ## Non-recoverable error. - MM_HALT* {.importc, header: "<fmtmsg.h>".}: cint - ## Error causing application to halt. - MM_ERROR* {.importc, header: "<fmtmsg.h>".}: cint - ## Application has encountered a non-fatal fault. - MM_WARNING* {.importc, header: "<fmtmsg.h>".}: cint - ## Application has detected unusual non-error condition. - MM_INFO* {.importc, header: "<fmtmsg.h>".}: cint - ## Informative message. - MM_NOSEV* {.importc, header: "<fmtmsg.h>".}: cint - ## No severity level provided for the message. - MM_PRINT* {.importc, header: "<fmtmsg.h>".}: cint - ## Display message on standard error. - MM_CONSOLE* {.importc, header: "<fmtmsg.h>".}: cint - ## Display message on system console. - - MM_OK* {.importc, header: "<fmtmsg.h>".}: cint - ## The function succeeded. - MM_NOTOK* {.importc, header: "<fmtmsg.h>".}: cint - ## The function failed completely. - MM_NOMSG* {.importc, header: "<fmtmsg.h>".}: cint - ## The function was unable to generate a message on standard error, - ## but otherwise succeeded. - MM_NOCON* {.importc, header: "<fmtmsg.h>".}: cint - ## The function was unable to generate a console message, but - ## otherwise succeeded. - -var - FNM_NOMATCH* {.importc, header: "<fnmatch.h>".}: cint - ## The string does not match the specified pattern. - FNM_PATHNAME* {.importc, header: "<fnmatch.h>".}: cint - ## Slash in string only matches slash in pattern. - FNM_PERIOD* {.importc, header: "<fnmatch.h>".}: cint - ## Leading period in string must be exactly matched by period in pattern. - FNM_NOESCAPE* {.importc, header: "<fnmatch.h>".}: cint - ## Disable backslash escaping. - FNM_NOSYS* {.importc, header: "<fnmatch.h>".}: cint - ## Reserved. - - FTW_F* {.importc, header: "<ftw.h>".}: cint - ## File. - FTW_D* {.importc, header: "<ftw.h>".}: cint - ## Directory. - FTW_DNR* {.importc, header: "<ftw.h>".}: cint - ## Directory without read permission. - FTW_DP* {.importc, header: "<ftw.h>".}: cint - ## Directory with subdirectories visited. - FTW_NS* {.importc, header: "<ftw.h>".}: cint - ## Unknown type; stat() failed. - FTW_SL* {.importc, header: "<ftw.h>".}: cint - ## Symbolic link. - FTW_SLN* {.importc, header: "<ftw.h>".}: cint - ## Symbolic link that names a nonexistent file. - - FTW_PHYS* {.importc, header: "<ftw.h>".}: cint - ## Physical walk, does not follow symbolic links. Otherwise, nftw() - ## follows links but does not walk down any path that crosses itself. - FTW_MOUNT* {.importc, header: "<ftw.h>".}: cint - ## The walk does not cross a mount point. - FTW_DEPTH* {.importc, header: "<ftw.h>".}: cint - ## All subdirectories are visited before the directory itself. - FTW_CHDIR* {.importc, header: "<ftw.h>".}: cint - ## The walk changes to each directory before reading it. - - GLOB_APPEND* {.importc, header: "<glob.h>".}: cint - ## Append generated pathnames to those previously obtained. - GLOB_DOOFFS* {.importc, header: "<glob.h>".}: cint - ## Specify how many null pointers to add to the beginning of gl_pathv. - GLOB_ERR* {.importc, header: "<glob.h>".}: cint - ## Cause glob() to return on error. - GLOB_MARK* {.importc, header: "<glob.h>".}: cint - ## Each pathname that is a directory that matches pattern has a - ## slash appended. - GLOB_NOCHECK* {.importc, header: "<glob.h>".}: cint - ## If pattern does not match any pathname, then return a list - ## consisting of only pattern. - GLOB_NOESCAPE* {.importc, header: "<glob.h>".}: cint - ## Disable backslash escaping. - GLOB_NOSORT* {.importc, header: "<glob.h>".}: cint - ## Do not sort the pathnames returned. - GLOB_ABORTED* {.importc, header: "<glob.h>".}: cint - ## The scan was stopped because GLOB_ERR was set or errfunc() - ## returned non-zero. - GLOB_NOMATCH* {.importc, header: "<glob.h>".}: cint - ## The pattern does not match any existing pathname, and GLOB_NOCHECK - ## was not set in flags. - GLOB_NOSPACE* {.importc, header: "<glob.h>".}: cint - ## An attempt to allocate memory failed. - GLOB_NOSYS* {.importc, header: "<glob.h>".}: cint - ## Reserved - - CODESET* {.importc, header: "<langinfo.h>".}: cint - ## Codeset name. - D_T_FMT* {.importc, header: "<langinfo.h>".}: cint - ## String for formatting date and time. - D_FMT * {.importc, header: "<langinfo.h>".}: cint - ## Date format string. - T_FMT* {.importc, header: "<langinfo.h>".}: cint - ## Time format string. - T_FMT_AMPM* {.importc, header: "<langinfo.h>".}: cint - ## a.m. or p.m. time format string. - AM_STR* {.importc, header: "<langinfo.h>".}: cint - ## Ante-meridiem affix. - PM_STR* {.importc, header: "<langinfo.h>".}: cint - ## Post-meridiem affix. - DAY_1* {.importc, header: "<langinfo.h>".}: cint - ## Name of the first day of the week (for example, Sunday). - DAY_2* {.importc, header: "<langinfo.h>".}: cint - ## Name of the second day of the week (for example, Monday). - DAY_3* {.importc, header: "<langinfo.h>".}: cint - ## Name of the third day of the week (for example, Tuesday). - DAY_4* {.importc, header: "<langinfo.h>".}: cint - ## Name of the fourth day of the week (for example, Wednesday). - DAY_5* {.importc, header: "<langinfo.h>".}: cint - ## Name of the fifth day of the week (for example, Thursday). - DAY_6* {.importc, header: "<langinfo.h>".}: cint - ## Name of the sixth day of the week (for example, Friday). - DAY_7* {.importc, header: "<langinfo.h>".}: cint - ## Name of the seventh day of the week (for example, Saturday). - ABDAY_1* {.importc, header: "<langinfo.h>".}: cint - ## Abbreviated name of the first day of the week. - ABDAY_2* {.importc, header: "<langinfo.h>".}: cint - ABDAY_3* {.importc, header: "<langinfo.h>".}: cint - ABDAY_4* {.importc, header: "<langinfo.h>".}: cint - ABDAY_5* {.importc, header: "<langinfo.h>".}: cint - ABDAY_6* {.importc, header: "<langinfo.h>".}: cint - ABDAY_7* {.importc, header: "<langinfo.h>".}: cint - MON_1* {.importc, header: "<langinfo.h>".}: cint - ## Name of the first month of the year. - MON_2* {.importc, header: "<langinfo.h>".}: cint - MON_3* {.importc, header: "<langinfo.h>".}: cint - MON_4* {.importc, header: "<langinfo.h>".}: cint - MON_5* {.importc, header: "<langinfo.h>".}: cint - MON_6* {.importc, header: "<langinfo.h>".}: cint - MON_7* {.importc, header: "<langinfo.h>".}: cint - MON_8* {.importc, header: "<langinfo.h>".}: cint - MON_9* {.importc, header: "<langinfo.h>".}: cint - MON_10* {.importc, header: "<langinfo.h>".}: cint - MON_11* {.importc, header: "<langinfo.h>".}: cint - MON_12* {.importc, header: "<langinfo.h>".}: cint - ABMON_1* {.importc, header: "<langinfo.h>".}: cint - ## Abbreviated name of the first month. - ABMON_2* {.importc, header: "<langinfo.h>".}: cint - ABMON_3* {.importc, header: "<langinfo.h>".}: cint - ABMON_4* {.importc, header: "<langinfo.h>".}: cint - ABMON_5* {.importc, header: "<langinfo.h>".}: cint - ABMON_6* {.importc, header: "<langinfo.h>".}: cint - ABMON_7* {.importc, header: "<langinfo.h>".}: cint - ABMON_8* {.importc, header: "<langinfo.h>".}: cint - ABMON_9* {.importc, header: "<langinfo.h>".}: cint - ABMON_10* {.importc, header: "<langinfo.h>".}: cint - ABMON_11* {.importc, header: "<langinfo.h>".}: cint - ABMON_12* {.importc, header: "<langinfo.h>".}: cint - ERA* {.importc, header: "<langinfo.h>".}: cint - ## Era description segments. - ERA_D_FMT* {.importc, header: "<langinfo.h>".}: cint - ## Era date format string. - ERA_D_T_FMT* {.importc, header: "<langinfo.h>".}: cint - ## Era date and time format string. - ERA_T_FMT* {.importc, header: "<langinfo.h>".}: cint - ## Era time format string. - ALT_DIGITS* {.importc, header: "<langinfo.h>".}: cint - ## Alternative symbols for digits. - RADIXCHAR* {.importc, header: "<langinfo.h>".}: cint - ## Radix character. - THOUSEP* {.importc, header: "<langinfo.h>".}: cint - ## Separator for thousands. - YESEXPR* {.importc, header: "<langinfo.h>".}: cint - ## Affirmative response expression. - NOEXPR* {.importc, header: "<langinfo.h>".}: cint - ## Negative response expression. - CRNCYSTR* {.importc, header: "<langinfo.h>".}: cint - ## Local currency symbol, preceded by '-' if the symbol - ## should appear before the value, '+' if the symbol should appear - ## after the value, or '.' if the symbol should replace the radix - ## character. If the local currency symbol is the empty string, - ## implementations may return the empty string ( "" ). - - LC_ALL* {.importc, header: "<locale.h>".}: cint - LC_COLLATE* {.importc, header: "<locale.h>".}: cint - LC_CTYPE* {.importc, header: "<locale.h>".}: cint - LC_MESSAGES* {.importc, header: "<locale.h>".}: cint - LC_MONETARY* {.importc, header: "<locale.h>".}: cint - LC_NUMERIC* {.importc, header: "<locale.h>".}: cint - LC_TIME* {.importc, header: "<locale.h>".}: cint - - PTHREAD_BARRIER_SERIAL_THREAD* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CANCEL_ASYNCHRONOUS* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CANCEL_ENABLE* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CANCEL_DEFERRED* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CANCEL_DISABLE* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CANCELED* {.importc, header: "<pthread.h>".}: cint - PTHREAD_COND_INITIALIZER* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CREATE_DETACHED* {.importc, header: "<pthread.h>".}: cint - PTHREAD_CREATE_JOINABLE* {.importc, header: "<pthread.h>".}: cint - PTHREAD_EXPLICIT_SCHED* {.importc, header: "<pthread.h>".}: cint - PTHREAD_INHERIT_SCHED* {.importc, header: "<pthread.h>".}: cint - PTHREAD_MUTEX_DEFAULT* {.importc, header: "<pthread.h>".}: cint - PTHREAD_MUTEX_ERRORCHECK* {.importc, header: "<pthread.h>".}: cint - PTHREAD_MUTEX_INITIALIZER* {.importc, header: "<pthread.h>".}: cint - PTHREAD_MUTEX_NORMAL* {.importc, header: "<pthread.h>".}: cint - PTHREAD_MUTEX_RECURSIVE* {.importc, header: "<pthread.h>".}: cint - PTHREAD_ONCE_INIT* {.importc, header: "<pthread.h>".}: cint - PTHREAD_PRIO_INHERIT* {.importc, header: "<pthread.h>".}: cint - PTHREAD_PRIO_NONE* {.importc, header: "<pthread.h>".}: cint - PTHREAD_PRIO_PROTECT* {.importc, header: "<pthread.h>".}: cint - PTHREAD_PROCESS_SHARED* {.importc, header: "<pthread.h>".}: cint - PTHREAD_PROCESS_PRIVATE* {.importc, header: "<pthread.h>".}: cint - PTHREAD_SCOPE_PROCESS* {.importc, header: "<pthread.h>".}: cint - PTHREAD_SCOPE_SYSTEM* {.importc, header: "<pthread.h>".}: cint - - POSIX_ASYNC_IO* {.importc: "_POSIX_ASYNC_IO", header: "<unistd.h>".}: cint - POSIX_PRIO_IO* {.importc: "_POSIX_PRIO_IO", header: "<unistd.h>".}: cint - POSIX_SYNC_IO* {.importc: "_POSIX_SYNC_IO", header: "<unistd.h>".}: cint - F_OK* {.importc: "F_OK", header: "<unistd.h>".}: cint - R_OK* {.importc: "R_OK", header: "<unistd.h>".}: cint - W_OK* {.importc: "W_OK", header: "<unistd.h>".}: cint - X_OK* {.importc: "X_OK", header: "<unistd.h>".}: cint - - CS_PATH* {.importc: "_CS_PATH", header: "<unistd.h>".}: cint - CS_POSIX_V6_ILP32_OFF32_CFLAGS* {.importc: "_CS_POSIX_V6_ILP32_OFF32_CFLAGS", - header: "<unistd.h>".}: cint - CS_POSIX_V6_ILP32_OFF32_LDFLAGS* {. - importc: "_CS_POSIX_V6_ILP32_OFF32_LDFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_ILP32_OFF32_LIBS* {.importc: "_CS_POSIX_V6_ILP32_OFF32_LIBS", - header: "<unistd.h>".}: cint - CS_POSIX_V6_ILP32_OFFBIG_CFLAGS* {. - importc: "_CS_POSIX_V6_ILP32_OFFBIG_CFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS* {. - importc: "_CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_ILP32_OFFBIG_LIBS* {. - importc: "_CS_POSIX_V6_ILP32_OFFBIG_LIBS", header: "<unistd.h>".}: cint - CS_POSIX_V6_LP64_OFF64_CFLAGS* {. - importc: "_CS_POSIX_V6_LP64_OFF64_CFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_LP64_OFF64_LDFLAGS* {. - importc: "_CS_POSIX_V6_LP64_OFF64_LDFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_LP64_OFF64_LIBS* {. - importc: "_CS_POSIX_V6_LP64_OFF64_LIBS", header: "<unistd.h>".}: cint - CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS* {. - importc: "_CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS* {. - importc: "_CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS", header: "<unistd.h>".}: cint - CS_POSIX_V6_LPBIG_OFFBIG_LIBS* {. - importc: "_CS_POSIX_V6_LPBIG_OFFBIG_LIBS", header: "<unistd.h>".}: cint - CS_POSIX_V6_WIDTH_RESTRICTED_ENVS* {. - importc: "_CS_POSIX_V6_WIDTH_RESTRICTED_ENVS", header: "<unistd.h>".}: cint - F_LOCK* {.importc: "F_LOCK", header: "<unistd.h>".}: cint - F_TEST* {.importc: "F_TEST", header: "<unistd.h>".}: cint - F_TLOCK* {.importc: "F_TLOCK", header: "<unistd.h>".}: cint - F_ULOCK* {.importc: "F_ULOCK", header: "<unistd.h>".}: cint - PC_2_SYMLINKS* {.importc: "_PC_2_SYMLINKS", header: "<unistd.h>".}: cint - PC_ALLOC_SIZE_MIN* {.importc: "_PC_ALLOC_SIZE_MIN", - header: "<unistd.h>".}: cint - PC_ASYNC_IO* {.importc: "_PC_ASYNC_IO", header: "<unistd.h>".}: cint - PC_CHOWN_RESTRICTED* {.importc: "_PC_CHOWN_RESTRICTED", - header: "<unistd.h>".}: cint - PC_FILESIZEBITS* {.importc: "_PC_FILESIZEBITS", header: "<unistd.h>".}: cint - PC_LINK_MAX* {.importc: "_PC_LINK_MAX", header: "<unistd.h>".}: cint - PC_MAX_CANON* {.importc: "_PC_MAX_CANON", header: "<unistd.h>".}: cint - - PC_MAX_INPUT*{.importc: "_PC_MAX_INPUT", header: "<unistd.h>".}: cint - PC_NAME_MAX*{.importc: "_PC_NAME_MAX", header: "<unistd.h>".}: cint - PC_NO_TRUNC*{.importc: "_PC_NO_TRUNC", header: "<unistd.h>".}: cint - PC_PATH_MAX*{.importc: "_PC_PATH_MAX", header: "<unistd.h>".}: cint - PC_PIPE_BUF*{.importc: "_PC_PIPE_BUF", header: "<unistd.h>".}: cint - PC_PRIO_IO*{.importc: "_PC_PRIO_IO", header: "<unistd.h>".}: cint - PC_REC_INCR_XFER_SIZE*{.importc: "_PC_REC_INCR_XFER_SIZE", - header: "<unistd.h>".}: cint - PC_REC_MIN_XFER_SIZE*{.importc: "_PC_REC_MIN_XFER_SIZE", - header: "<unistd.h>".}: cint - PC_REC_XFER_ALIGN*{.importc: "_PC_REC_XFER_ALIGN", header: "<unistd.h>".}: cint - PC_SYMLINK_MAX*{.importc: "_PC_SYMLINK_MAX", header: "<unistd.h>".}: cint - PC_SYNC_IO*{.importc: "_PC_SYNC_IO", header: "<unistd.h>".}: cint - PC_VDISABLE*{.importc: "_PC_VDISABLE", header: "<unistd.h>".}: cint - SC_2_C_BIND*{.importc: "_SC_2_C_BIND", header: "<unistd.h>".}: cint - SC_2_C_DEV*{.importc: "_SC_2_C_DEV", header: "<unistd.h>".}: cint - SC_2_CHAR_TERM*{.importc: "_SC_2_CHAR_TERM", header: "<unistd.h>".}: cint - SC_2_FORT_DEV*{.importc: "_SC_2_FORT_DEV", header: "<unistd.h>".}: cint - SC_2_FORT_RUN*{.importc: "_SC_2_FORT_RUN", header: "<unistd.h>".}: cint - SC_2_LOCALEDEF*{.importc: "_SC_2_LOCALEDEF", header: "<unistd.h>".}: cint - SC_2_PBS*{.importc: "_SC_2_PBS", header: "<unistd.h>".}: cint - SC_2_PBS_ACCOUNTING*{.importc: "_SC_2_PBS_ACCOUNTING", - header: "<unistd.h>".}: cint - SC_2_PBS_CHECKPOINT*{.importc: "_SC_2_PBS_CHECKPOINT", - header: "<unistd.h>".}: cint - SC_2_PBS_LOCATE*{.importc: "_SC_2_PBS_LOCATE", header: "<unistd.h>".}: cint - SC_2_PBS_MESSAGE*{.importc: "_SC_2_PBS_MESSAGE", header: "<unistd.h>".}: cint - SC_2_PBS_TRACK*{.importc: "_SC_2_PBS_TRACK", header: "<unistd.h>".}: cint - SC_2_SW_DEV*{.importc: "_SC_2_SW_DEV", header: "<unistd.h>".}: cint - SC_2_UPE*{.importc: "_SC_2_UPE", header: "<unistd.h>".}: cint - SC_2_VERSION*{.importc: "_SC_2_VERSION", header: "<unistd.h>".}: cint - SC_ADVISORY_INFO*{.importc: "_SC_ADVISORY_INFO", header: "<unistd.h>".}: cint - SC_AIO_LISTIO_MAX*{.importc: "_SC_AIO_LISTIO_MAX", header: "<unistd.h>".}: cint - SC_AIO_MAX*{.importc: "_SC_AIO_MAX", header: "<unistd.h>".}: cint - SC_AIO_PRIO_DELTA_MAX*{.importc: "_SC_AIO_PRIO_DELTA_MAX", - header: "<unistd.h>".}: cint - SC_ARG_MAX*{.importc: "_SC_ARG_MAX", header: "<unistd.h>".}: cint - SC_ASYNCHRONOUS_IO*{.importc: "_SC_ASYNCHRONOUS_IO", - header: "<unistd.h>".}: cint - SC_ATEXIT_MAX*{.importc: "_SC_ATEXIT_MAX", header: "<unistd.h>".}: cint - SC_BARRIERS*{.importc: "_SC_BARRIERS", header: "<unistd.h>".}: cint - SC_BC_BASE_MAX*{.importc: "_SC_BC_BASE_MAX", header: "<unistd.h>".}: cint - SC_BC_DIM_MAX*{.importc: "_SC_BC_DIM_MAX", header: "<unistd.h>".}: cint - SC_BC_SCALE_MAX*{.importc: "_SC_BC_SCALE_MAX", header: "<unistd.h>".}: cint - SC_BC_STRING_MAX*{.importc: "_SC_BC_STRING_MAX", header: "<unistd.h>".}: cint - SC_CHILD_MAX*{.importc: "_SC_CHILD_MAX", header: "<unistd.h>".}: cint - SC_CLK_TCK*{.importc: "_SC_CLK_TCK", header: "<unistd.h>".}: cint - SC_CLOCK_SELECTION*{.importc: "_SC_CLOCK_SELECTION", - header: "<unistd.h>".}: cint - SC_COLL_WEIGHTS_MAX*{.importc: "_SC_COLL_WEIGHTS_MAX", - header: "<unistd.h>".}: cint - SC_CPUTIME*{.importc: "_SC_CPUTIME", header: "<unistd.h>".}: cint - SC_DELAYTIMER_MAX*{.importc: "_SC_DELAYTIMER_MAX", header: "<unistd.h>".}: cint - SC_EXPR_NEST_MAX*{.importc: "_SC_EXPR_NEST_MAX", header: "<unistd.h>".}: cint - SC_FSYNC*{.importc: "_SC_FSYNC", header: "<unistd.h>".}: cint - SC_GETGR_R_SIZE_MAX*{.importc: "_SC_GETGR_R_SIZE_MAX", - header: "<unistd.h>".}: cint - SC_GETPW_R_SIZE_MAX*{.importc: "_SC_GETPW_R_SIZE_MAX", - header: "<unistd.h>".}: cint - SC_HOST_NAME_MAX*{.importc: "_SC_HOST_NAME_MAX", header: "<unistd.h>".}: cint - SC_IOV_MAX*{.importc: "_SC_IOV_MAX", header: "<unistd.h>".}: cint - SC_IPV6*{.importc: "_SC_IPV6", header: "<unistd.h>".}: cint - SC_JOB_CONTROL*{.importc: "_SC_JOB_CONTROL", header: "<unistd.h>".}: cint - SC_LINE_MAX*{.importc: "_SC_LINE_MAX", header: "<unistd.h>".}: cint - SC_LOGIN_NAME_MAX*{.importc: "_SC_LOGIN_NAME_MAX", header: "<unistd.h>".}: cint - SC_MAPPED_FILES*{.importc: "_SC_MAPPED_FILES", header: "<unistd.h>".}: cint - SC_MEMLOCK*{.importc: "_SC_MEMLOCK", header: "<unistd.h>".}: cint - SC_MEMLOCK_RANGE*{.importc: "_SC_MEMLOCK_RANGE", header: "<unistd.h>".}: cint - SC_MEMORY_PROTECTION*{.importc: "_SC_MEMORY_PROTECTION", - header: "<unistd.h>".}: cint - SC_MESSAGE_PASSING*{.importc: "_SC_MESSAGE_PASSING", - header: "<unistd.h>".}: cint - SC_MONOTONIC_CLOCK*{.importc: "_SC_MONOTONIC_CLOCK", - header: "<unistd.h>".}: cint - SC_MQ_OPEN_MAX*{.importc: "_SC_MQ_OPEN_MAX", header: "<unistd.h>".}: cint - SC_MQ_PRIO_MAX*{.importc: "_SC_MQ_PRIO_MAX", header: "<unistd.h>".}: cint - SC_NGROUPS_MAX*{.importc: "_SC_NGROUPS_MAX", header: "<unistd.h>".}: cint - SC_OPEN_MAX*{.importc: "_SC_OPEN_MAX", header: "<unistd.h>".}: cint - SC_PAGE_SIZE*{.importc: "_SC_PAGE_SIZE", header: "<unistd.h>".}: cint - SC_PRIORITIZED_IO*{.importc: "_SC_PRIORITIZED_IO", header: "<unistd.h>".}: cint - SC_PRIORITY_SCHEDULING*{.importc: "_SC_PRIORITY_SCHEDULING", - header: "<unistd.h>".}: cint - SC_RAW_SOCKETS*{.importc: "_SC_RAW_SOCKETS", header: "<unistd.h>".}: cint - SC_RE_DUP_MAX*{.importc: "_SC_RE_DUP_MAX", header: "<unistd.h>".}: cint - SC_READER_WRITER_LOCKS*{.importc: "_SC_READER_WRITER_LOCKS", - header: "<unistd.h>".}: cint - SC_REALTIME_SIGNALS*{.importc: "_SC_REALTIME_SIGNALS", - header: "<unistd.h>".}: cint - SC_REGEXP*{.importc: "_SC_REGEXP", header: "<unistd.h>".}: cint - SC_RTSIG_MAX*{.importc: "_SC_RTSIG_MAX", header: "<unistd.h>".}: cint - SC_SAVED_IDS*{.importc: "_SC_SAVED_IDS", header: "<unistd.h>".}: cint - SC_SEM_NSEMS_MAX*{.importc: "_SC_SEM_NSEMS_MAX", header: "<unistd.h>".}: cint - SC_SEM_VALUE_MAX*{.importc: "_SC_SEM_VALUE_MAX", header: "<unistd.h>".}: cint - SC_SEMAPHORES*{.importc: "_SC_SEMAPHORES", header: "<unistd.h>".}: cint - SC_SHARED_MEMORY_OBJECTS*{.importc: "_SC_SHARED_MEMORY_OBJECTS", - header: "<unistd.h>".}: cint - SC_SHELL*{.importc: "_SC_SHELL", header: "<unistd.h>".}: cint - SC_SIGQUEUE_MAX*{.importc: "_SC_SIGQUEUE_MAX", header: "<unistd.h>".}: cint - SC_SPAWN*{.importc: "_SC_SPAWN", header: "<unistd.h>".}: cint - SC_SPIN_LOCKS*{.importc: "_SC_SPIN_LOCKS", header: "<unistd.h>".}: cint - SC_SPORADIC_SERVER*{.importc: "_SC_SPORADIC_SERVER", - header: "<unistd.h>".}: cint - SC_SS_REPL_MAX*{.importc: "_SC_SS_REPL_MAX", header: "<unistd.h>".}: cint - SC_STREAM_MAX*{.importc: "_SC_STREAM_MAX", header: "<unistd.h>".}: cint - SC_SYMLOOP_MAX*{.importc: "_SC_SYMLOOP_MAX", header: "<unistd.h>".}: cint - SC_SYNCHRONIZED_IO*{.importc: "_SC_SYNCHRONIZED_IO", - header: "<unistd.h>".}: cint - SC_THREAD_ATTR_STACKADDR*{.importc: "_SC_THREAD_ATTR_STACKADDR", - header: "<unistd.h>".}: cint - SC_THREAD_ATTR_STACKSIZE*{.importc: "_SC_THREAD_ATTR_STACKSIZE", - header: "<unistd.h>".}: cint - SC_THREAD_CPUTIME*{.importc: "_SC_THREAD_CPUTIME", header: "<unistd.h>".}: cint - SC_THREAD_DESTRUCTOR_ITERATIONS*{.importc: "_SC_THREAD_DESTRUCTOR_ITERATIONS", - header: "<unistd.h>".}: cint - SC_THREAD_KEYS_MAX*{.importc: "_SC_THREAD_KEYS_MAX", - header: "<unistd.h>".}: cint - SC_THREAD_PRIO_INHERIT*{.importc: "_SC_THREAD_PRIO_INHERIT", - header: "<unistd.h>".}: cint - SC_THREAD_PRIO_PROTECT*{.importc: "_SC_THREAD_PRIO_PROTECT", - header: "<unistd.h>".}: cint - SC_THREAD_PRIORITY_SCHEDULING*{.importc: "_SC_THREAD_PRIORITY_SCHEDULING", - header: "<unistd.h>".}: cint - SC_THREAD_PROCESS_SHARED*{.importc: "_SC_THREAD_PROCESS_SHARED", - header: "<unistd.h>".}: cint - SC_THREAD_SAFE_FUNCTIONS*{.importc: "_SC_THREAD_SAFE_FUNCTIONS", - header: "<unistd.h>".}: cint - SC_THREAD_SPORADIC_SERVER*{.importc: "_SC_THREAD_SPORADIC_SERVER", - header: "<unistd.h>".}: cint - SC_THREAD_STACK_MIN*{.importc: "_SC_THREAD_STACK_MIN", - header: "<unistd.h>".}: cint - SC_THREAD_THREADS_MAX*{.importc: "_SC_THREAD_THREADS_MAX", - header: "<unistd.h>".}: cint - SC_THREADS*{.importc: "_SC_THREADS", header: "<unistd.h>".}: cint - SC_TIMEOUTS*{.importc: "_SC_TIMEOUTS", header: "<unistd.h>".}: cint - SC_TIMER_MAX*{.importc: "_SC_TIMER_MAX", header: "<unistd.h>".}: cint - SC_TIMERS*{.importc: "_SC_TIMERS", header: "<unistd.h>".}: cint - SC_TRACE*{.importc: "_SC_TRACE", header: "<unistd.h>".}: cint - SC_TRACE_EVENT_FILTER*{.importc: "_SC_TRACE_EVENT_FILTER", header: "<unistd.h>".}: cint - SC_TRACE_EVENT_NAME_MAX*{.importc: "_SC_TRACE_EVENT_NAME_MAX", header: "<unistd.h>".}: cint - SC_TRACE_INHERIT*{.importc: "_SC_TRACE_INHERIT", header: "<unistd.h>".}: cint - SC_TRACE_LOG*{.importc: "_SC_TRACE_LOG", header: "<unistd.h>".}: cint - SC_TRACE_NAME_MAX*{.importc: "_SC_TRACE_NAME_MAX", header: "<unistd.h>".}: cint - SC_TRACE_SYS_MAX*{.importc: "_SC_TRACE_SYS_MAX", header: "<unistd.h>".}: cint - SC_TRACE_USER_EVENT_MAX*{.importc: "_SC_TRACE_USER_EVENT_MAX", header: "<unistd.h>".}: cint - SC_TTY_NAME_MAX*{.importc: "_SC_TTY_NAME_MAX", header: "<unistd.h>".}: cint - SC_TYPED_MEMORY_OBJECTS*{.importc: "_SC_TYPED_MEMORY_OBJECTS", header: "<unistd.h>".}: cint - SC_TZNAME_MAX*{.importc: "_SC_TZNAME_MAX", header: "<unistd.h>".}: cint - SC_V6_ILP32_OFF32*{.importc: "_SC_V6_ILP32_OFF32", header: "<unistd.h>".}: cint - SC_V6_ILP32_OFFBIG*{.importc: "_SC_V6_ILP32_OFFBIG", header: "<unistd.h>".}: cint - SC_V6_LP64_OFF64*{.importc: "_SC_V6_LP64_OFF64", header: "<unistd.h>".}: cint - SC_V6_LPBIG_OFFBIG*{.importc: "_SC_V6_LPBIG_OFFBIG", header: "<unistd.h>".}: cint - SC_VERSION*{.importc: "_SC_VERSION", header: "<unistd.h>".}: cint - SC_XBS5_ILP32_OFF32*{.importc: "_SC_XBS5_ILP32_OFF32", header: "<unistd.h>".}: cint - SC_XBS5_ILP32_OFFBIG*{.importc: "_SC_XBS5_ILP32_OFFBIG", header: "<unistd.h>".}: cint - SC_XBS5_LP64_OFF64*{.importc: "_SC_XBS5_LP64_OFF64", header: "<unistd.h>".}: cint - SC_XBS5_LPBIG_OFFBIG*{.importc: "_SC_XBS5_LPBIG_OFFBIG", - header: "<unistd.h>".}: cint - SC_XOPEN_CRYPT*{.importc: "_SC_XOPEN_CRYPT", header: "<unistd.h>".}: cint - SC_XOPEN_ENH_I18N*{.importc: "_SC_XOPEN_ENH_I18N", header: "<unistd.h>".}: cint - SC_XOPEN_LEGACY*{.importc: "_SC_XOPEN_LEGACY", header: "<unistd.h>".}: cint - SC_XOPEN_REALTIME*{.importc: "_SC_XOPEN_REALTIME", header: "<unistd.h>".}: cint - SC_XOPEN_REALTIME_THREADS*{.importc: "_SC_XOPEN_REALTIME_THREADS", - header: "<unistd.h>".}: cint - SC_XOPEN_SHM*{.importc: "_SC_XOPEN_SHM", header: "<unistd.h>".}: cint - SC_XOPEN_STREAMS*{.importc: "_SC_XOPEN_STREAMS", header: "<unistd.h>".}: cint - SC_XOPEN_UNIX*{.importc: "_SC_XOPEN_UNIX", header: "<unistd.h>".}: cint - SC_XOPEN_VERSION*{.importc: "_SC_XOPEN_VERSION", header: "<unistd.h>".}: cint - SC_NPROCESSORS_ONLN*{.importc: "_SC_NPROCESSORS_ONLN", - header: "<unistd.h>".}: cint - - SEM_FAILED* {.importc, header: "<semaphore.h>".}: pointer - IPC_CREAT* {.importc, header: "<sys/ipc.h>".}: cint - ## Create entry if key does not exist. - IPC_EXCL* {.importc, header: "<sys/ipc.h>".}: cint - ## Fail if key exists. - IPC_NOWAIT* {.importc, header: "<sys/ipc.h>".}: cint - ## Error if request must wait. - - IPC_PRIVATE* {.importc, header: "<sys/ipc.h>".}: cint - ## Private key. - - IPC_RMID* {.importc, header: "<sys/ipc.h>".}: cint - ## Remove identifier. - IPC_SET* {.importc, header: "<sys/ipc.h>".}: cint - ## Set options. - IPC_STAT* {.importc, header: "<sys/ipc.h>".}: cint - ## Get options. - - S_IFMT* {.importc, header: "<sys/stat.h>".}: cint - ## Type of file. - S_IFBLK* {.importc, header: "<sys/stat.h>".}: cint - ## Block special. - S_IFCHR* {.importc, header: "<sys/stat.h>".}: cint - ## Character special. - S_IFIFO* {.importc, header: "<sys/stat.h>".}: cint - ## FIFO special. - S_IFREG* {.importc, header: "<sys/stat.h>".}: cint - ## Regular. - S_IFDIR* {.importc, header: "<sys/stat.h>".}: cint - ## Directory. - S_IFLNK* {.importc, header: "<sys/stat.h>".}: cint - ## Symbolic link. - S_IFSOCK* {.importc, header: "<sys/stat.h>".}: cint - ## Socket. - S_IRWXU* {.importc, header: "<sys/stat.h>".}: cint - ## Read, write, execute/search by owner. - S_IRUSR* {.importc, header: "<sys/stat.h>".}: cint - ## Read permission, owner. - S_IWUSR* {.importc, header: "<sys/stat.h>".}: cint - ## Write permission, owner. - S_IXUSR* {.importc, header: "<sys/stat.h>".}: cint - ## Execute/search permission, owner. - S_IRWXG* {.importc, header: "<sys/stat.h>".}: cint - ## Read, write, execute/search by group. - S_IRGRP* {.importc, header: "<sys/stat.h>".}: cint - ## Read permission, group. - S_IWGRP* {.importc, header: "<sys/stat.h>".}: cint - ## Write permission, group. - S_IXGRP* {.importc, header: "<sys/stat.h>".}: cint - ## Execute/search permission, group. - S_IRWXO* {.importc, header: "<sys/stat.h>".}: cint - ## Read, write, execute/search by others. - S_IROTH* {.importc, header: "<sys/stat.h>".}: cint - ## Read permission, others. - S_IWOTH* {.importc, header: "<sys/stat.h>".}: cint - ## Write permission, others. - S_IXOTH* {.importc, header: "<sys/stat.h>".}: cint - ## Execute/search permission, others. - S_ISUID* {.importc, header: "<sys/stat.h>".}: cint - ## Set-user-ID on execution. - S_ISGID* {.importc, header: "<sys/stat.h>".}: cint - ## Set-group-ID on execution. - S_ISVTX* {.importc, header: "<sys/stat.h>".}: cint - ## On directories, restricted deletion flag. - - ST_RDONLY* {.importc, header: "<sys/statvfs.h>".}: cint - ## Read-only file system. - ST_NOSUID* {.importc, header: "<sys/statvfs.h>".}: cint - ## Does not support the semantics of the ST_ISUID and ST_ISGID file mode bits. - - PROT_READ* {.importc, header: "<sys/mman.h>".}: cint - ## Page can be read. - PROT_WRITE* {.importc, header: "<sys/mman.h>".}: cint - ## Page can be written. - PROT_EXEC* {.importc, header: "<sys/mman.h>".}: cint - ## Page can be executed. - PROT_NONE* {.importc, header: "<sys/mman.h>".}: cint - ## Page cannot be accessed. - MAP_SHARED* {.importc, header: "<sys/mman.h>".}: cint - ## Share changes. - MAP_PRIVATE* {.importc, header: "<sys/mman.h>".}: cint - ## Changes are private. - MAP_FIXED* {.importc, header: "<sys/mman.h>".}: cint - ## Interpret addr exactly. - MS_ASYNC* {.importc, header: "<sys/mman.h>".}: cint - ## Perform asynchronous writes. - MS_SYNC* {.importc, header: "<sys/mman.h>".}: cint - ## Perform synchronous writes. - MS_INVALIDATE* {.importc, header: "<sys/mman.h>".}: cint - ## Invalidate mappings. - MCL_CURRENT* {.importc, header: "<sys/mman.h>".}: cint - ## Lock currently mapped pages. - MCL_FUTURE* {.importc, header: "<sys/mman.h>".}: cint - ## Lock pages that become mapped. - MAP_FAILED* {.importc, header: "<sys/mman.h>".}: cint - POSIX_MADV_NORMAL* {.importc, header: "<sys/mman.h>".}: cint - ## The application has no advice to give on its behavior with - ## respect to the specified range. It is the default characteristic - ## if no advice is given for a range of memory. - POSIX_MADV_SEQUENTIAL* {.importc, header: "<sys/mman.h>".}: cint - ## The application expects to access the specified range sequentially - ## from lower addresses to higher addresses. - POSIX_MADV_RANDOM* {.importc, header: "<sys/mman.h>".}: cint - ## The application expects to access the specified range in a random order. - POSIX_MADV_WILLNEED* {.importc, header: "<sys/mman.h>".}: cint - ## The application expects to access the specified range in the near future. - POSIX_MADV_DONTNEED* {.importc, header: "<sys/mman.h>".}: cint - POSIX_TYPED_MEM_ALLOCATE* {.importc, header: "<sys/mman.h>".}: cint - POSIX_TYPED_MEM_ALLOCATE_CONTIG* {.importc, header: "<sys/mman.h>".}: cint - POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc, header: "<sys/mman.h>".}: cint - - - CLOCKS_PER_SEC* {.importc, header: "<time.h>".}: int - ## A number used to convert the value returned by the clock() function - ## into seconds. - CLOCK_PROCESS_CPUTIME_ID* {.importc, header: "<time.h>".}: cint - ## The identifier of the CPU-time clock associated with the process - ## making a clock() or timer*() function call. - CLOCK_THREAD_CPUTIME_ID* {.importc, header: "<time.h>".}: cint - CLOCK_REALTIME* {.importc, header: "<time.h>".}: cint - ## The identifier of the system-wide realtime clock. - TIMER_ABSTIME* {.importc, header: "<time.h>".}: cint - ## Flag indicating time is absolute. For functions taking timer - ## objects, this refers to the clock associated with the timer. - CLOCK_MONOTONIC* {.importc, header: "<time.h>".}: cint - - WNOHANG* {.importc, header: "<sys/wait.h>".}: cint - ## Do not hang if no status is available; return immediately. - WUNTRACED* {.importc, header: "<sys/wait.h>".}: cint - ## Report status of stopped child process. - WEXITSTATUS* {.importc, header: "<sys/wait.h>".}: cint - ## Return exit status. - WSTOPSIG* {.importc, header: "<sys/wait.h>".}: cint - ## Return signal number that caused process to stop. - WTERMSIG* {.importc, header: "<sys/wait.h>".}: cint - ## Return signal number that caused process to terminate. - WEXITED* {.importc, header: "<sys/wait.h>".}: cint - ## Wait for processes that have exited. - WSTOPPED* {.importc, header: "<sys/wait.h>".}: cint - ## Status is returned for any child that has stopped upon receipt - ## of a signal. - WCONTINUED* {.importc, header: "<sys/wait.h>".}: cint - ## Status is returned for any child that was stopped and has been continued. - WNOWAIT* {.importc, header: "<sys/wait.h>".}: cint - ## Keep the process whose status is returned in infop in a waitable state. - P_ALL* {.importc, header: "<sys/wait.h>".}: cint - P_PID* {.importc, header: "<sys/wait.h>".}: cint - P_PGID* {.importc, header: "<sys/wait.h>".}: cint - - SIG_DFL* {.importc, header: "<signal.h>".}: proc (x: cint) {.noconv.} - ## Request for default signal handling. - SIG_ERR* {.importc, header: "<signal.h>".}: proc (x: cint) {.noconv.} - ## Return value from signal() in case of error. - cSIG_HOLD* {.importc: "SIG_HOLD", - header: "<signal.h>".}: proc (x: cint) {.noconv.} - ## Request that signal be held. - SIG_IGN* {.importc, header: "<signal.h>".}: proc (x: cint) {.noconv.} - ## Request that signal be ignored. - - SIGEV_NONE* {.importc, header: "<signal.h>".}: cint - SIGEV_SIGNAL* {.importc, header: "<signal.h>".}: cint - SIGEV_THREAD* {.importc, header: "<signal.h>".}: cint - SIGABRT* {.importc, header: "<signal.h>".}: cint - SIGALRM* {.importc, header: "<signal.h>".}: cint - SIGBUS* {.importc, header: "<signal.h>".}: cint - SIGCHLD* {.importc, header: "<signal.h>".}: cint - SIGCONT* {.importc, header: "<signal.h>".}: cint - SIGFPE* {.importc, header: "<signal.h>".}: cint - SIGHUP* {.importc, header: "<signal.h>".}: cint - SIGILL* {.importc, header: "<signal.h>".}: cint - SIGINT* {.importc, header: "<signal.h>".}: cint - SIGKILL* {.importc, header: "<signal.h>".}: cint - SIGPIPE* {.importc, header: "<signal.h>".}: cint - SIGQUIT* {.importc, header: "<signal.h>".}: cint - SIGSEGV* {.importc, header: "<signal.h>".}: cint - SIGSTOP* {.importc, header: "<signal.h>".}: cint - SIGTERM* {.importc, header: "<signal.h>".}: cint - SIGTSTP* {.importc, header: "<signal.h>".}: cint - SIGTTIN* {.importc, header: "<signal.h>".}: cint - SIGTTOU* {.importc, header: "<signal.h>".}: cint - SIGUSR1* {.importc, header: "<signal.h>".}: cint - SIGUSR2* {.importc, header: "<signal.h>".}: cint - SIGPOLL* {.importc, header: "<signal.h>".}: cint - SIGPROF* {.importc, header: "<signal.h>".}: cint - SIGSYS* {.importc, header: "<signal.h>".}: cint - SIGTRAP* {.importc, header: "<signal.h>".}: cint - SIGURG* {.importc, header: "<signal.h>".}: cint - SIGVTALRM* {.importc, header: "<signal.h>".}: cint - SIGXCPU* {.importc, header: "<signal.h>".}: cint - SIGXFSZ* {.importc, header: "<signal.h>".}: cint - SA_NOCLDSTOP* {.importc, header: "<signal.h>".}: cint - SIG_BLOCK* {.importc, header: "<signal.h>".}: cint - SIG_UNBLOCK* {.importc, header: "<signal.h>".}: cint - SIG_SETMASK* {.importc, header: "<signal.h>".}: cint - SA_ONSTACK* {.importc, header: "<signal.h>".}: cint - SA_RESETHAND* {.importc, header: "<signal.h>".}: cint - SA_RESTART* {.importc, header: "<signal.h>".}: cint - SA_SIGINFO* {.importc, header: "<signal.h>".}: cint - SA_NOCLDWAIT* {.importc, header: "<signal.h>".}: cint - SA_NODEFER* {.importc, header: "<signal.h>".}: cint - SS_ONSTACK* {.importc, header: "<signal.h>".}: cint - SS_DISABLE* {.importc, header: "<signal.h>".}: cint - MINSIGSTKSZ* {.importc, header: "<signal.h>".}: cint - SIGSTKSZ* {.importc, header: "<signal.h>".}: cint - - NL_SETD* {.importc, header: "<nl_types.h>".}: cint - NL_CAT_LOCALE* {.importc, header: "<nl_types.h>".}: cint - - SCHED_FIFO* {.importc, header: "<sched.h>".}: cint - SCHED_RR* {.importc, header: "<sched.h>".}: cint - SCHED_SPORADIC* {.importc, header: "<sched.h>".}: cint - SCHED_OTHER* {.importc, header: "<sched.h>".}: cint - FD_SETSIZE* {.importc, header: "<sys/select.h>".}: cint - - SEEK_SET* {.importc, header: "<unistd.h>".}: cint - SEEK_CUR* {.importc, header: "<unistd.h>".}: cint - SEEK_END* {.importc, header: "<unistd.h>".}: cint - - SCM_RIGHTS* {.importc, header: "<sys/socket.h>".}: cint - ## Indicates that the data array contains the access rights - ## to be sent or received. - - SOCK_DGRAM* {.importc, header: "<sys/socket.h>".}: cint ## Datagram socket. - SOCK_RAW* {.importc, header: "<sys/socket.h>".}: cint - ## Raw Protocol Interface. - SOCK_SEQPACKET* {.importc, header: "<sys/socket.h>".}: cint - ## Sequenced-packet socket. - SOCK_STREAM* {.importc, header: "<sys/socket.h>".}: cint - ## Byte-stream socket. - - SOL_SOCKET* {.importc, header: "<sys/socket.h>".}: cint - ## Options to be accessed at socket level, not protocol level. - - SO_ACCEPTCONN* {.importc, header: "<sys/socket.h>".}: cint - ## Socket is accepting connections. - SO_BROADCAST* {.importc, header: "<sys/socket.h>".}: cint - ## Transmission of broadcast messages is supported. - SO_DEBUG* {.importc, header: "<sys/socket.h>".}: cint - ## Debugging information is being recorded. - SO_DONTROUTE* {.importc, header: "<sys/socket.h>".}: cint - ## Bypass normal routing. - SO_ERROR* {.importc, header: "<sys/socket.h>".}: cint - ## Socket error status. - SO_KEEPALIVE* {.importc, header: "<sys/socket.h>".}: cint - ## Connections are kept alive with periodic messages. - SO_LINGER* {.importc, header: "<sys/socket.h>".}: cint - ## Socket lingers on close. - SO_OOBINLINE* {.importc, header: "<sys/socket.h>".}: cint - ## Out-of-band data is transmitted in line. - SO_RCVBUF* {.importc, header: "<sys/socket.h>".}: cint - ## Receive buffer size. - SO_RCVLOWAT* {.importc, header: "<sys/socket.h>".}: cint - ## Receive *low water mark*. - SO_RCVTIMEO* {.importc, header: "<sys/socket.h>".}: cint - ## Receive timeout. - SO_REUSEADDR* {.importc, header: "<sys/socket.h>".}: cint - ## Reuse of local addresses is supported. - SO_SNDBUF* {.importc, header: "<sys/socket.h>".}: cint - ## Send buffer size. - SO_SNDLOWAT* {.importc, header: "<sys/socket.h>".}: cint - ## Send *low water mark*. - SO_SNDTIMEO* {.importc, header: "<sys/socket.h>".}: cint - ## Send timeout. - SO_TYPE* {.importc, header: "<sys/socket.h>".}: cint - ## Socket type. - - SOMAXCONN* {.importc, header: "<sys/socket.h>".}: cint - ## The maximum backlog queue length. - - MSG_CTRUNC* {.importc, header: "<sys/socket.h>".}: cint - ## Control data truncated. - MSG_DONTROUTE* {.importc, header: "<sys/socket.h>".}: cint - ## Send without using routing tables. - MSG_EOR* {.importc, header: "<sys/socket.h>".}: cint - ## Terminates a record (if supported by the protocol). - MSG_OOB* {.importc, header: "<sys/socket.h>".}: cint - ## Out-of-band data. - -proc WIFCONTINUED*(s:cint) : bool {.importc, header: "<sys/wait.h>".} - ## True if child has been continued. -proc WIFEXITED*(s:cint) : bool {.importc, header: "<sys/wait.h>".} - ## True if child exited normally. -proc WIFSIGNALED*(s:cint) : bool {.importc, header: "<sys/wait.h>".} - ## True if child exited due to uncaught signal. -proc WIFSTOPPED*(s:cint) : bool {.importc, header: "<sys/wait.h>".} - ## True if child is currently stopped. - -when defined(linux): - var - MAP_POPULATE* {.importc, header: "<sys/mman.h>".}: cint - ## Populate (prefault) page tables for a mapping. +# Special types +type Sighandler = proc (a: cint) {.noconv.} + +const StatHasNanoseconds* = defined(linux) or defined(freebsd) or + defined(osx) or defined(openbsd) or defined(dragonfly) or defined(haiku) ## \ + ## Boolean flag that indicates if the system supports nanosecond time + ## resolution in the fields of `Stat`. Note that the nanosecond based fields + ## (`Stat.st_atim`, `Stat.st_mtim` and `Stat.st_ctim`) can be accessed + ## without checking this flag, because this module defines fallback procs + ## when they are not available. + +# Platform specific stuff + +when (defined(linux) and not defined(android)) and defined(amd64): + include posix_linux_amd64 +elif defined(openbsd) and defined(amd64): + include posix_openbsd_amd64 +elif (defined(macos) or defined(macosx) or defined(bsd)) and defined(cpu64): + include posix_macos_amd64 +elif defined(nintendoswitch): + include posix_nintendoswitch +elif defined(haiku): + include posix_haiku else: - var - MAP_POPULATE*: cint = 0 - - -when defined(macosx): - # We can't use the NOSIGNAL flag in the ``send`` function, it has no effect - # Instead we should use SO_NOSIGPIPE in setsockopt - const - MSG_NOSIGNAL* = 0'i32 - var - SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint + include posix_other + +# There used to be this name in posix.nim a long time ago, not sure why! + +when StatHasNanoseconds: + proc st_atime*(s: Stat): Time {.inline.} = + ## Second-granularity time of last access. + result = s.st_atim.tv_sec + proc st_mtime*(s: Stat): Time {.inline.} = + ## Second-granularity time of last data modification. + result = s.st_mtim.tv_sec + proc st_ctime*(s: Stat): Time {.inline.} = + ## Second-granularity time of last status change. + result = s.st_ctim.tv_sec else: - var - MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint - ## No SIGPIPE generated when an attempt to send is made on a stream-oriented socket that is no longer connected. - -var - MSG_PEEK* {.importc, header: "<sys/socket.h>".}: cint - ## Leave received data in queue. - MSG_TRUNC* {.importc, header: "<sys/socket.h>".}: cint - ## Normal data truncated. - MSG_WAITALL* {.importc, header: "<sys/socket.h>".}: cint - ## Attempt to fill the read buffer. - - AF_INET* {.importc, header: "<sys/socket.h>".}: cint - ## Internet domain sockets for use with IPv4 addresses. - AF_INET6* {.importc, header: "<sys/socket.h>".}: cint - ## Internet domain sockets for use with IPv6 addresses. - AF_UNIX* {.importc, header: "<sys/socket.h>".}: cint - ## UNIX domain sockets. - AF_UNSPEC* {.importc, header: "<sys/socket.h>".}: cint - ## Unspecified. - - SHUT_RD* {.importc, header: "<sys/socket.h>".}: cint - ## Disables further receive operations. - SHUT_RDWR* {.importc, header: "<sys/socket.h>".}: cint - ## Disables further send and receive operations. - SHUT_WR* {.importc, header: "<sys/socket.h>".}: cint - ## Disables further send operations. - - IF_NAMESIZE* {.importc, header: "<net/if.h>".}: cint - - IPPROTO_IP* {.importc, header: "<netinet/in.h>".}: cint - ## Internet protocol. - IPPROTO_IPV6* {.importc, header: "<netinet/in.h>".}: cint - ## Internet Protocol Version 6. - IPPROTO_ICMP* {.importc, header: "<netinet/in.h>".}: cint - ## Control message protocol. - IPPROTO_RAW* {.importc, header: "<netinet/in.h>".}: cint - ## Raw IP Packets Protocol. - IPPROTO_TCP* {.importc, header: "<netinet/in.h>".}: cint - ## Transmission control protocol. - IPPROTO_UDP* {.importc, header: "<netinet/in.h>".}: cint - ## User datagram protocol. - - INADDR_ANY* {.importc, header: "<netinet/in.h>".}: InAddrScalar - ## IPv4 local host address. - INADDR_BROADCAST* {.importc, header: "<netinet/in.h>".}: InAddrScalar - ## IPv4 broadcast address. - - INET_ADDRSTRLEN* {.importc, header: "<netinet/in.h>".}: cint - ## 16. Length of the string form for IP. - INET6_ADDRSTRLEN* {.importc, header: "<netinet/in.h>".}: cint - ## Length of the string form for IPv6. - - IPV6_JOIN_GROUP* {.importc, header: "<netinet/in.h>".}: cint - ## Join a multicast group. - IPV6_LEAVE_GROUP* {.importc, header: "<netinet/in.h>".}: cint - ## Quit a multicast group. - IPV6_MULTICAST_HOPS* {.importc, header: "<netinet/in.h>".}: cint - ## Multicast hop limit. - IPV6_MULTICAST_IF* {.importc, header: "<netinet/in.h>".}: cint - ## Interface to use for outgoing multicast packets. - IPV6_MULTICAST_LOOP* {.importc, header: "<netinet/in.h>".}: cint - ## Multicast packets are delivered back to the local application. - IPV6_UNICAST_HOPS* {.importc, header: "<netinet/in.h>".}: cint - ## Unicast hop limit. - IPV6_V6ONLY* {.importc, header: "<netinet/in.h>".}: cint - ## Restrict AF_INET6 socket to IPv6 communications only. - - TCP_NODELAY* {.importc, header: "<netinet/tcp.h>".}: cint - ## Avoid coalescing of small segments. - - IPPORT_RESERVED* {.importc, header: "<netdb.h>".}: cint - - HOST_NOT_FOUND* {.importc, header: "<netdb.h>".}: cint - NO_DATA* {.importc, header: "<netdb.h>".}: cint - NO_RECOVERY* {.importc, header: "<netdb.h>".}: cint - TRY_AGAIN* {.importc, header: "<netdb.h>".}: cint - - AI_PASSIVE* {.importc, header: "<netdb.h>".}: cint - ## Socket address is intended for bind(). - AI_CANONNAME* {.importc, header: "<netdb.h>".}: cint - ## Request for canonical name. - AI_NUMERICHOST* {.importc, header: "<netdb.h>".}: cint - ## Return numeric host address as name. - AI_NUMERICSERV* {.importc, header: "<netdb.h>".}: cint - ## Inhibit service name resolution. - AI_V4MAPPED* {.importc, header: "<netdb.h>".}: cint - ## If no IPv6 addresses are found, query for IPv4 addresses and - ## return them to the caller as IPv4-mapped IPv6 addresses. - AI_ALL* {.importc, header: "<netdb.h>".}: cint - ## Query for both IPv4 and IPv6 addresses. - AI_ADDRCONFIG* {.importc, header: "<netdb.h>".}: cint - ## Query for IPv4 addresses only when an IPv4 address is configured; - ## query for IPv6 addresses only when an IPv6 address is configured. - - NI_NOFQDN* {.importc, header: "<netdb.h>".}: cint - ## Only the nodename portion of the FQDN is returned for local hosts. - NI_NUMERICHOST* {.importc, header: "<netdb.h>".}: cint - ## The numeric form of the node's address is returned instead of its name. - NI_NAMEREQD* {.importc, header: "<netdb.h>".}: cint - ## Return an error if the node's name cannot be located in the database. - NI_NUMERICSERV* {.importc, header: "<netdb.h>".}: cint - ## The numeric form of the service address is returned instead of its name. - NI_NUMERICSCOPE* {.importc, header: "<netdb.h>".}: cint - ## For IPv6 addresses, the numeric form of the scope identifier is - ## returned instead of its name. - NI_DGRAM* {.importc, header: "<netdb.h>".}: cint - ## Indicates that the service is a datagram service (SOCK_DGRAM). - - EAI_AGAIN* {.importc, header: "<netdb.h>".}: cint - ## The name could not be resolved at this time. Future attempts may succeed. - EAI_BADFLAGS* {.importc, header: "<netdb.h>".}: cint - ## The flags had an invalid value. - EAI_FAIL* {.importc, header: "<netdb.h>".}: cint - ## A non-recoverable error occurred. - EAI_FAMILY* {.importc, header: "<netdb.h>".}: cint - ## The address family was not recognized or the address length - ## was invalid for the specified family. - EAI_MEMORY* {.importc, header: "<netdb.h>".}: cint - ## There was a memory allocation failure. - EAI_NONAME* {.importc, header: "<netdb.h>".}: cint - ## The name does not resolve for the supplied parameters. - ## NI_NAMEREQD is set and the host's name cannot be located, - ## or both nodename and servname were null. - EAI_SERVICE* {.importc, header: "<netdb.h>".}: cint - ## The service passed was not recognized for the specified socket type. - EAI_SOCKTYPE* {.importc, header: "<netdb.h>".}: cint - ## The intended socket type was not recognized. - EAI_SYSTEM* {.importc, header: "<netdb.h>".}: cint - ## A system error occurred. The error code can be found in errno. - EAI_OVERFLOW* {.importc, header: "<netdb.h>".}: cint - ## An argument buffer overflowed. - - POLLIN* {.importc, header: "<poll.h>".}: cshort - ## Data other than high-priority data may be read without blocking. - POLLRDNORM* {.importc, header: "<poll.h>".}: cshort - ## Normal data may be read without blocking. - POLLRDBAND* {.importc, header: "<poll.h>".}: cshort - ## Priority data may be read without blocking. - POLLPRI* {.importc, header: "<poll.h>".}: cshort - ## High priority data may be read without blocking. - POLLOUT* {.importc, header: "<poll.h>".}: cshort - ## Normal data may be written without blocking. - POLLWRNORM* {.importc, header: "<poll.h>".}: cshort - ## Equivalent to POLLOUT. - POLLWRBAND* {.importc, header: "<poll.h>".}: cshort - ## Priority data may be written. - POLLERR* {.importc, header: "<poll.h>".}: cshort - ## An error has occurred (revents only). - POLLHUP* {.importc, header: "<poll.h>".}: cshort - ## Device has been disconnected (revents only). - POLLNVAL* {.importc, header: "<poll.h>".}: cshort - ## Invalid fd member (revents only). - - -when hasSpawnH: - var - POSIX_SPAWN_RESETIDS* {.importc, header: "<spawn.h>".}: cint - POSIX_SPAWN_SETPGROUP* {.importc, header: "<spawn.h>".}: cint - POSIX_SPAWN_SETSCHEDPARAM* {.importc, header: "<spawn.h>".}: cint - POSIX_SPAWN_SETSCHEDULER* {.importc, header: "<spawn.h>".}: cint - POSIX_SPAWN_SETSIGDEF* {.importc, header: "<spawn.h>".}: cint - POSIX_SPAWN_SETSIGMASK* {.importc, header: "<spawn.h>".}: cint - - when defined(linux): - # better be safe than sorry; Linux has this flag, macosx doesn't, don't - # know about the other OSes - - # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we - # can't get the magic number from spawn.h - const POSIX_SPAWN_USEVFORK* = cint(0x40) - else: - # macosx lacks this, so we define the constant to be 0 to not affect - # OR'ing of flags: - const POSIX_SPAWN_USEVFORK* = cint(0) + proc st_atim*(s: Stat): Timespec {.inline.} = + ## Nanosecond-granularity time of last access. + result.tv_sec = s.st_atime + proc st_mtim*(s: Stat): Timespec {.inline.} = + ## Nanosecond-granularity time of last data modification. + result.tv_sec = s.st_mtime + proc st_ctim*(s: Stat): Timespec {.inline.} = + ## Nanosecond-granularity time of last data modification. + result.tv_sec = s.st_ctime when hasAioH: proc aio_cancel*(a1: cint, a2: ptr Taiocb): cint {.importc, header: "<aio.h>".} @@ -1802,16 +149,18 @@ when hasAioH: a4: ptr SigEvent): cint {.importc, header: "<aio.h>".} # arpa/inet.h -proc htonl*(a1: int32): int32 {.importc, header: "<arpa/inet.h>".} -proc htons*(a1: int16): int16 {.importc, header: "<arpa/inet.h>".} -proc ntohl*(a1: int32): int32 {.importc, header: "<arpa/inet.h>".} -proc ntohs*(a1: int16): int16 {.importc, header: "<arpa/inet.h>".} - -proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".} -proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".} -proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. +proc htonl*(a1: uint32): uint32 {.importc, header: "<arpa/inet.h>".} +proc htons*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".} +proc ntohl*(a1: uint32): uint32 {.importc, header: "<arpa/inet.h>".} +proc ntohs*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".} + +when not defined(zephyr): + proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".} + proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".} + +proc inet_ntop*(a1: cint, a2: pointer | ptr InAddr | ptr In6Addr, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "<arpa/inet.h>".} -proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. +proc inet_pton*(a1: cint, a2: cstring, a3: pointer | ptr InAddr | ptr In6Addr): cint {. importc, header: "<arpa/inet.h>".} var @@ -1823,29 +172,54 @@ proc IN6ADDR_LOOPBACK_INIT* (): In6Addr {.importc, header: "<netinet/in.h>".} # dirent.h proc closedir*(a1: ptr DIR): cint {.importc, header: "<dirent.h>".} -proc opendir*(a1: cstring): ptr DIR {.importc, header: "<dirent.h>".} -proc readdir*(a1: ptr DIR): ptr Dirent {.importc, header: "<dirent.h>".} +proc opendir*(a1: cstring): ptr DIR {.importc, header: "<dirent.h>", sideEffect.} +proc readdir*(a1: ptr DIR): ptr Dirent {.importc, header: "<dirent.h>", sideEffect.} proc readdir_r*(a1: ptr DIR, a2: ptr Dirent, a3: ptr ptr Dirent): cint {. - importc, header: "<dirent.h>".} + importc, header: "<dirent.h>", sideEffect.} proc rewinddir*(a1: ptr DIR) {.importc, header: "<dirent.h>".} proc seekdir*(a1: ptr DIR, a2: int) {.importc, header: "<dirent.h>".} proc telldir*(a1: ptr DIR): int {.importc, header: "<dirent.h>".} # dlfcn.h -proc dlclose*(a1: pointer): cint {.importc, header: "<dlfcn.h>".} -proc dlerror*(): cstring {.importc, header: "<dlfcn.h>".} -proc dlopen*(a1: cstring, a2: cint): pointer {.importc, header: "<dlfcn.h>".} -proc dlsym*(a1: pointer, a2: cstring): pointer {.importc, header: "<dlfcn.h>".} - -proc creat*(a1: cstring, a2: Mode): cint {.importc, header: "<fcntl.h>".} -proc fcntl*(a1: cint | SocketHandle, a2: cint): cint {.varargs, importc, header: "<fcntl.h>".} -proc open*(a1: cstring, a2: cint): cint {.varargs, importc, header: "<fcntl.h>".} +proc dlclose*(a1: pointer): cint {.importc, header: "<dlfcn.h>", sideEffect.} +proc dlerror*(): cstring {.importc, header: "<dlfcn.h>", sideEffect.} +proc dlopen*(a1: cstring, a2: cint): pointer {.importc, header: "<dlfcn.h>", sideEffect.} +proc dlsym*(a1: pointer, a2: cstring): pointer {.importc, header: "<dlfcn.h>", sideEffect.} + +proc creat*(a1: cstring, a2: Mode): cint {.importc, header: "<fcntl.h>", sideEffect.} +proc fcntl*(a1: cint | SocketHandle, a2: cint): cint {.varargs, importc, header: "<fcntl.h>", sideEffect.} +proc openImpl(a1: cstring, a2: cint): cint {.varargs, importc: "open", header: "<fcntl.h>", sideEffect.} +proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = + # prevents bug #17888 + openImpl(a1, a2, mode) + proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "<fcntl.h>".} -proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. - importc, header: "<fcntl.h>".} -when not defined(haiku) and not defined(OpenBSD): +proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "<unistd.h>".} +when defined(osx): # 2001 POSIX evidently does not concern Apple + type FStore {.importc: "fstore_t", header: "<fcntl.h>", bycopy.} = object + fst_flags: uint32 ## IN: flags word + fst_posmode: cint ## IN: indicates offset field + fst_offset, ## IN: start of the region + fst_length, ## IN: size of the region + fst_bytesalloc: Off ## OUT: number of bytes allocated + var F_PEOFPOSMODE {.importc, header: "<fcntl.h>".}: cint + var F_ALLOCATEALL {.importc, header: "<fcntl.h>".}: uint32 + var F_PREALLOCATE {.importc, header: "<fcntl.h>".}: cint + proc posix_fallocate*(a1: cint, a2, a3: Off): cint = + var fst = FStore(fst_flags: F_ALLOCATEALL, fst_posmode: F_PEOFPOSMODE, + fst_offset: a2, fst_length: a3) + # Must also call ftruncate to match what POSIX does. Unlike posix_fallocate, + # this can shrink files. Could guard w/getFileSize, but caller likely knows + # present size & has no good reason to call this unless it is growing. + if fcntl(a1, F_PREALLOCATE, fst.addr) != cint(-1): ftruncate(a1, a2 + a3) + else: cint(-1) +else: + proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. + importc, header: "<fcntl.h>".} + +when not defined(haiku) and not defined(openbsd): proc fmtmsg*(a1: int, a2: cstring, a3: cint, a4, a5, a6: cstring): cint {.importc, header: "<fmtmsg.h>".} @@ -1853,15 +227,18 @@ proc fnmatch*(a1, a2: cstring, a3: cint): cint {.importc, header: "<fnmatch.h>". proc ftw*(a1: cstring, a2: proc (x1: cstring, x2: ptr Stat, x3: cint): cint {.noconv.}, a3: cint): cint {.importc, header: "<ftw.h>".} -proc nftw*(a1: cstring, - a2: proc (x1: cstring, x2: ptr Stat, - x3: cint, x4: ptr FTW): cint {.noconv.}, - a3: cint, - a4: cint): cint {.importc, header: "<ftw.h>".} +when not (defined(linux) and defined(amd64)) and not defined(nintendoswitch): + proc nftw*(a1: cstring, + a2: proc (x1: cstring, x2: ptr Stat, + x3: cint, x4: ptr FTW): cint {.noconv.}, + a3: cint, + a4: cint): cint {.importc, header: "<ftw.h>".} proc glob*(a1: cstring, a2: cint, a3: proc (x1: cstring, x2: cint): cint {.noconv.}, - a4: ptr Glob): cint {.importc, header: "<glob.h>".} + a4: ptr Glob): cint {.importc, header: "<glob.h>", sideEffect.} + ## Filename globbing. Use `os.walkPattern() <os.html#glob_1>`_ and similar. + proc globfree*(a1: ptr Glob) {.importc, header: "<glob.h>".} proc getgrgid*(a1: Gid): ptr Group {.importc, header: "<grp.h>".} @@ -1888,30 +265,57 @@ proc dirname*(a1: cstring): cstring {.importc, header: "<libgen.h>".} proc localeconv*(): ptr Lconv {.importc, header: "<locale.h>".} proc setlocale*(a1: cint, a2: cstring): cstring {. - importc, header: "<locale.h>".} + importc, header: "<locale.h>", sideEffect.} proc strfmon*(a1: cstring, a2: int, a3: cstring): int {.varargs, importc, header: "<monetary.h>".} -proc mq_close*(a1: Mqd): cint {.importc, header: "<mqueue.h>".} -proc mq_getattr*(a1: Mqd, a2: ptr MqAttr): cint {. - importc, header: "<mqueue.h>".} -proc mq_notify*(a1: Mqd, a2: ptr SigEvent): cint {. - importc, header: "<mqueue.h>".} -proc mq_open*(a1: cstring, a2: cint): Mqd {. - varargs, importc, header: "<mqueue.h>".} -proc mq_receive*(a1: Mqd, a2: cstring, a3: int, a4: var int): int {. - importc, header: "<mqueue.h>".} -proc mq_send*(a1: Mqd, a2: cstring, a3: int, a4: int): cint {. - importc, header: "<mqueue.h>".} -proc mq_setattr*(a1: Mqd, a2, a3: ptr MqAttr): cint {. - importc, header: "<mqueue.h>".} - -proc mq_timedreceive*(a1: Mqd, a2: cstring, a3: int, a4: int, - a5: ptr Timespec): int {.importc, header: "<mqueue.h>".} -proc mq_timedsend*(a1: Mqd, a2: cstring, a3: int, a4: int, - a5: ptr Timespec): cint {.importc, header: "<mqueue.h>".} -proc mq_unlink*(a1: cstring): cint {.importc, header: "<mqueue.h>".} +when not (defined(nintendoswitch) or defined(macos) or defined(macosx)): + proc mq_notify*(mqdes: Mqd, event: ptr SigEvent): cint {. + importc, header: "<mqueue.h>".} + + proc mq_open*(name: cstring, flags: cint): Mqd {. + varargs, importc, header: "<mqueue.h>".} + + proc mq_close*(mqdes: Mqd): cint {.importc, header: "<mqueue.h>".} + + proc mq_receive*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: var cuint + ): int {.importc, header: "<mqueue.h>".} + + proc mq_timedreceive*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint, + timeout: ptr Timespec + ): int {.importc, header: "<mqueue.h>".} + + proc mq_send*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint + ): cint {.importc, header: "<mqueue.h>".} + + proc mq_timedsend*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint, + timeout: ptr Timespec + ): cint {.importc, header: "<mqueue.h>".} + + proc mq_getattr*(mqdes: Mqd, attribute: ptr MqAttr): cint {. + importc, header: "<mqueue.h>".} + + proc mq_setattr*(mqdes: Mqd, newAttribute, oldAttribute: ptr MqAttr): cint {. + importc, header: "<mqueue.h>".} + + proc mq_unlink*(mqdes: cstring): cint {.importc, header: "<mqueue.h>".} proc getpwnam*(a1: cstring): ptr Passwd {.importc, header: "<pwd.h>".} @@ -1926,47 +330,49 @@ proc setpwent*() {.importc, header: "<pwd.h>".} proc uname*(a1: var Utsname): cint {.importc, header: "<sys/utsname.h>".} +proc strerror*(errnum: cint): cstring {.importc, header: "<string.h>".} + proc pthread_atfork*(a1, a2, a3: proc () {.noconv.}): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_destroy*(a1: ptr PthreadAttr): cint {. +proc pthread_attr_destroy*(a1: ptr Pthread_attr): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_getdetachstate*(a1: ptr PthreadAttr, a2: cint): cint {. +proc pthread_attr_getdetachstate*(a1: ptr Pthread_attr, a2: cint): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_getguardsize*(a1: ptr PthreadAttr, a2: var cint): cint {. +proc pthread_attr_getguardsize*(a1: ptr Pthread_attr, a2: var cint): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_getinheritsched*(a1: ptr PthreadAttr, +proc pthread_attr_getinheritsched*(a1: ptr Pthread_attr, a2: var cint): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_getschedparam*(a1: ptr PthreadAttr, +proc pthread_attr_getschedparam*(a1: ptr Pthread_attr, a2: ptr Sched_param): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_getschedpolicy*(a1: ptr PthreadAttr, +proc pthread_attr_getschedpolicy*(a1: ptr Pthread_attr, a2: var cint): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_getscope*(a1: ptr PthreadAttr, +proc pthread_attr_getscope*(a1: ptr Pthread_attr, a2: var cint): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_getstack*(a1: ptr PthreadAttr, +proc pthread_attr_getstack*(a1: ptr Pthread_attr, a2: var pointer, a3: var int): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_getstackaddr*(a1: ptr PthreadAttr, +proc pthread_attr_getstackaddr*(a1: ptr Pthread_attr, a2: var pointer): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_getstacksize*(a1: ptr PthreadAttr, +proc pthread_attr_getstacksize*(a1: ptr Pthread_attr, a2: var int): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_init*(a1: ptr PthreadAttr): cint {. +proc pthread_attr_init*(a1: ptr Pthread_attr): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setdetachstate*(a1: ptr PthreadAttr, a2: cint): cint {. +proc pthread_attr_setdetachstate*(a1: ptr Pthread_attr, a2: cint): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setguardsize*(a1: ptr PthreadAttr, a2: int): cint {. +proc pthread_attr_setguardsize*(a1: ptr Pthread_attr, a2: int): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setinheritsched*(a1: ptr PthreadAttr, a2: cint): cint {. +proc pthread_attr_setinheritsched*(a1: ptr Pthread_attr, a2: cint): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setschedparam*(a1: ptr PthreadAttr, +proc pthread_attr_setschedparam*(a1: ptr Pthread_attr, a2: ptr Sched_param): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_setschedpolicy*(a1: ptr PthreadAttr, a2: cint): cint {. +proc pthread_attr_setschedpolicy*(a1: ptr Pthread_attr, a2: cint): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setscope*(a1: ptr PthreadAttr, a2: cint): cint {.importc, +proc pthread_attr_setscope*(a1: ptr Pthread_attr, a2: cint): cint {.importc, header: "<pthread.h>".} -proc pthread_attr_setstack*(a1: ptr PthreadAttr, a2: pointer, a3: int): cint {. +proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setstackaddr*(a1: ptr PthreadAttr, a2: pointer): cint {. +proc pthread_attr_setstackaddr*(a1: ptr Pthread_attr, a2: pointer): cint {. importc, header: "<pthread.h>".} -proc pthread_attr_setstacksize*(a1: ptr PthreadAttr, a2: int): cint {. +proc pthread_attr_setstacksize*(a1: ptr Pthread_attr, a2: int): cint {. importc, header: "<pthread.h>".} proc pthread_barrier_destroy*(a1: ptr Pthread_barrier): cint {. importc, header: "<pthread.h>".} @@ -2009,7 +415,7 @@ proc pthread_condattr_init*(a1: ptr Pthread_condattr): cint {.importc, header: " proc pthread_condattr_setclock*(a1: ptr Pthread_condattr,a2: ClockId): cint {.importc, header: "<pthread.h>".} proc pthread_condattr_setpshared*(a1: ptr Pthread_condattr, a2: cint): cint {.importc, header: "<pthread.h>".} -proc pthread_create*(a1: ptr Pthread, a2: ptr PthreadAttr, +proc pthread_create*(a1: ptr Pthread, a2: ptr Pthread_attr, a3: proc (x: pointer): pointer {.noconv.}, a4: pointer): cint {.importc, header: "<pthread.h>".} proc pthread_detach*(a1: Pthread): cint {.importc, header: "<pthread.h>".} proc pthread_equal*(a1, a2: Pthread): cint {.importc, header: "<pthread.h>".} @@ -2097,7 +503,7 @@ proc pthread_spin_unlock*(a1: ptr Pthread_spinlock): cint {. proc pthread_testcancel*() {.importc, header: "<pthread.h>".} -proc exitnow*(code: int): void {.importc: "_exit", header: "<unistd.h>".} +proc exitnow*(code: int) {.importc: "_exit", header: "<unistd.h>".} proc access*(a1: cstring, a2: cint): cint {.importc, header: "<unistd.h>".} proc alarm*(a1: cint): cint {.importc, header: "<unistd.h>".} proc chdir*(a1: cstring): cint {.importc, header: "<unistd.h>".} @@ -2110,41 +516,55 @@ proc dup*(a1: cint): cint {.importc, header: "<unistd.h>".} proc dup2*(a1, a2: cint): cint {.importc, header: "<unistd.h>".} proc encrypt*(a1: array[0..63, char], a2: cint) {.importc, header: "<unistd.h>".} -proc execl*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>".} -proc execle*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>".} -proc execlp*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>".} -proc execv*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>".} +proc execl*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>", sideEffect.} +proc execle*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>", sideEffect.} +proc execlp*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>", sideEffect.} +proc execv*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>", sideEffect.} proc execve*(a1: cstring, a2, a3: cstringArray): cint {. - importc, header: "<unistd.h>".} -proc execvp*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>".} -proc execvpe*(a1: cstring, a2: cstringArray, a3: cstringArray): cint {.importc, header: "<unistd.h>".} -proc fchown*(a1: cint, a2: Uid, a3: Gid): cint {.importc, header: "<unistd.h>".} -proc fchdir*(a1: cint): cint {.importc, header: "<unistd.h>".} + importc, header: "<unistd.h>", sideEffect.} +proc execvp*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>", sideEffect.} +proc execvpe*(a1: cstring, a2: cstringArray, a3: cstringArray): cint {.importc, header: "<unistd.h>", sideEffect.} +proc fchown*(a1: cint, a2: Uid, a3: Gid): cint {.importc, header: "<unistd.h>", sideEffect.} +proc fchdir*(a1: cint): cint {.importc, header: "<unistd.h>", sideEffect.} proc fdatasync*(a1: cint): cint {.importc, header: "<unistd.h>".} -proc fork*(): Pid {.importc, header: "<unistd.h>".} +proc fork*(): Pid {.importc, header: "<unistd.h>", sideEffect.} proc fpathconf*(a1, a2: cint): int {.importc, header: "<unistd.h>".} proc fsync*(a1: cint): cint {.importc, header: "<unistd.h>".} -proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "<unistd.h>".} -proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "<unistd.h>".} -proc getegid*(): Gid {.importc, header: "<unistd.h>".} -proc geteuid*(): Uid {.importc, header: "<unistd.h>".} -proc getgid*(): Gid {.importc, header: "<unistd.h>".} + ## synchronize a file's buffer cache to the storage device + +proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "<unistd.h>", sideEffect.} +proc getuid*(): Uid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the real user ID of the calling process + +proc geteuid*(): Uid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the effective user ID of the calling process + +proc getgid*(): Gid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the real group ID of the calling process + +proc getegid*(): Gid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the effective group ID of the calling process proc getgroups*(a1: cint, a2: ptr array[0..255, Gid]): cint {. importc, header: "<unistd.h>".} -proc gethostid*(): int {.importc, header: "<unistd.h>".} -proc gethostname*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>".} -proc getlogin*(): cstring {.importc, header: "<unistd.h>".} -proc getlogin_r*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>".} +proc gethostid*(): int {.importc, header: "<unistd.h>", sideEffect.} +proc gethostname*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>", sideEffect.} +proc getlogin*(): cstring {.importc, header: "<unistd.h>", sideEffect.} +proc getlogin_r*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>", sideEffect.} proc getopt*(a1: cint, a2: cstringArray, a3: cstring): cint {. importc, header: "<unistd.h>".} proc getpgid*(a1: Pid): Pid {.importc, header: "<unistd.h>".} proc getpgrp*(): Pid {.importc, header: "<unistd.h>".} -proc getpid*(): Pid {.importc, header: "<unistd.h>".} -proc getppid*(): Pid {.importc, header: "<unistd.h>".} -proc getsid*(a1: Pid): Pid {.importc, header: "<unistd.h>".} -proc getuid*(): Uid {.importc, header: "<unistd.h>".} +proc getpid*(): Pid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the process ID (PID) of the calling process + +proc getppid*(): Pid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the process ID of the parent of the calling process + +proc getsid*(a1: Pid): Pid {.importc, header: "<unistd.h>", sideEffect.} + ## returns the session ID of the calling process + proc getwd*(a1: cstring): cstring {.importc, header: "<unistd.h>".} proc isatty*(a1: cint): cint {.importc, header: "<unistd.h>".} proc lchown*(a1: cstring, a2: Uid, a3: Gid): cint {.importc, header: "<unistd.h>".} @@ -2164,7 +584,12 @@ proc pread*(a1: cint, a2: pointer, a3: int, a4: Off): int {. proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "<unistd.h>".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "<unistd.h>".} -proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".} +when not defined(nintendoswitch): + proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".} +proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", + header: "<sys/ioctl.h>", varargs, tags: [WriteIOEffect].} + ## A system call for device-specific input/output operations and other + ## operations which cannot be expressed by regular system calls proc rmdir*(a1: cstring): cint {.importc, header: "<unistd.h>".} proc setegid*(a1: Gid): cint {.importc, header: "<unistd.h>".} @@ -2179,7 +604,10 @@ proc setsid*(): Pid {.importc, header: "<unistd.h>".} proc setuid*(a1: Uid): cint {.importc, header: "<unistd.h>".} proc sleep*(a1: cint): cint {.importc, header: "<unistd.h>".} proc swab*(a1, a2: pointer, a3: int) {.importc, header: "<unistd.h>".} -proc symlink*(a1, a2: cstring): cint {.importc, header: "<unistd.h>".} +when not defined(nintendoswitch): + proc symlink*(a1, a2: cstring): cint {.importc, header: "<unistd.h>".} +else: + proc symlink*(a1, a2: cstring): cint = -1 proc sync*() {.importc, header: "<unistd.h>".} proc sysconf*(a1: cint): int {.importc, header: "<unistd.h>".} proc tcgetpgrp*(a1: cint): Pid {.importc, header: "<unistd.h>".} @@ -2216,11 +644,15 @@ proc statvfs*(a1: cstring, a2: var Statvfs): cint {. proc fstatvfs*(a1: cint, a2: var Statvfs): cint {. importc, header: "<sys/statvfs.h>".} -proc chmod*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>".} -proc fchmod*(a1: cint, a2: Mode): cint {.importc, header: "<sys/stat.h>".} -proc fstat*(a1: cint, a2: var Stat): cint {.importc, header: "<sys/stat.h>".} -proc lstat*(a1: cstring, a2: var Stat): cint {.importc, header: "<sys/stat.h>".} -proc mkdir*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>".} +proc chmod*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} +when defined(osx) or defined(freebsd): + proc lchmod*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc fchmod*(a1: cint, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc fstat*(a1: cint, a2: var Stat): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc lstat*(a1: cstring, a2: var Stat): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc mkdir*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} + ## Use `os.createDir() <os.html#createDir,string>`_ and similar. + proc mkfifo*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>".} proc mknod*(a1: cstring, a2: Mode, a3: Dev): cint {. importc, header: "<sys/stat.h>".} @@ -2259,6 +691,7 @@ proc mmap*(a1: pointer, a2: int, a3, a4, a5: cint, a6: Off): pointer {. proc mprotect*(a1: pointer, a2: int, a3: cint): cint {. importc, header: "<sys/mman.h>".} proc msync*(a1: pointer, a2: int, a3: cint): cint {.importc, header: "<sys/mman.h>".} + proc munlock*(a1: pointer, a2: int): cint {.importc, header: "<sys/mman.h>".} proc munlockall*(): cint {.importc, header: "<sys/mman.h>".} proc munmap*(a1: pointer, a2: int): cint {.importc, header: "<sys/mman.h>".} @@ -2266,8 +699,10 @@ proc posix_madvise*(a1: pointer, a2: int, a3: cint): cint {. importc, header: "<sys/mman.h>".} proc posix_mem_offset*(a1: pointer, a2: int, a3: var Off, a4: var int, a5: var cint): cint {.importc, header: "<sys/mman.h>".} -proc posix_typed_mem_get_info*(a1: cint, - a2: var Posix_typed_mem_info): cint {.importc, header: "<sys/mman.h>".} +when not (defined(linux) and defined(amd64)) and not defined(nintendoswitch) and + not defined(haiku): + proc posix_typed_mem_get_info*(a1: cint, + a2: var Posix_typed_mem_info): cint {.importc, header: "<sys/mman.h>".} proc posix_typed_mem_open*(a1: cstring, a2, a3: cint): cint {. importc, header: "<sys/mman.h>".} proc shm_open*(a1: cstring, a2: cint, a3: Mode): cint {. @@ -2277,37 +712,38 @@ proc shm_unlink*(a1: cstring): cint {.importc, header: "<sys/mman.h>".} proc asctime*(a1: var Tm): cstring{.importc, header: "<time.h>".} proc asctime_r*(a1: var Tm, a2: cstring): cstring {.importc, header: "<time.h>".} -proc clock*(): Clock {.importc, header: "<time.h>".} +proc clock*(): Clock {.importc, header: "<time.h>", sideEffect.} proc clock_getcpuclockid*(a1: Pid, a2: var ClockId): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc clock_getres*(a1: ClockId, a2: var Timespec): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc clock_gettime*(a1: ClockId, a2: var Timespec): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc clock_nanosleep*(a1: ClockId, a2: cint, a3: var Timespec, - a4: var Timespec): cint {.importc, header: "<time.h>".} + a4: var Timespec): cint {.importc, header: "<time.h>", sideEffect.} proc clock_settime*(a1: ClockId, a2: var Timespec): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} +proc `==`*(a, b: Time): bool {.borrow.} +proc `-`*(a, b: Time): Time {.borrow.} proc ctime*(a1: var Time): cstring {.importc, header: "<time.h>".} proc ctime_r*(a1: var Time, a2: cstring): cstring {.importc, header: "<time.h>".} proc difftime*(a1, a2: Time): cdouble {.importc, header: "<time.h>".} proc getdate*(a1: cstring): ptr Tm {.importc, header: "<time.h>".} - proc gmtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".} proc gmtime_r*(a1: var Time, a2: var Tm): ptr Tm {.importc, header: "<time.h>".} proc localtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".} proc localtime_r*(a1: var Time, a2: var Tm): ptr Tm {.importc, header: "<time.h>".} proc mktime*(a1: var Tm): Time {.importc, header: "<time.h>".} proc timegm*(a1: var Tm): Time {.importc, header: "<time.h>".} -proc nanosleep*(a1, a2: var Timespec): cint {.importc, header: "<time.h>".} +proc nanosleep*(a1, a2: var Timespec): cint {.importc, header: "<time.h>", sideEffect.} proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "<time.h>".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "<time.h>".} -proc time*(a1: var Time): Time {.importc, header: "<time.h>".} -proc timer_create*(a1: var ClockId, a2: var SigEvent, +proc time*(a1: var Time): Time {.importc, header: "<time.h>", sideEffect.} +proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "<time.h>".} -proc timer_delete*(a1: var Timer): cint {.importc, header: "<time.h>".} +proc timer_delete*(a1: Timer): cint {.importc, header: "<time.h>".} proc timer_gettime*(a1: Timer, a2: var Itimerspec): cint {. importc, header: "<time.h>".} proc timer_getoverrun*(a1: Timer): cint {.importc, header: "<time.h>".} @@ -2316,22 +752,46 @@ proc timer_settime*(a1: Timer, a2: cint, a3: var Itimerspec, proc tzset*() {.importc, header: "<time.h>".} -proc wait*(a1: ptr cint): Pid {.importc, discardable, header: "<sys/wait.h>".} +proc wait*(a1: ptr cint): Pid {.importc, discardable, header: "<sys/wait.h>", sideEffect.} proc waitid*(a1: cint, a2: Id, a3: var SigInfo, a4: cint): cint {. - importc, header: "<sys/wait.h>".} + importc, header: "<sys/wait.h>", sideEffect.} proc waitpid*(a1: Pid, a2: var cint, a3: cint): Pid {. - importc, header: "<sys/wait.h>".} + importc, header: "<sys/wait.h>", sideEffect.} + +type Rusage* {.importc: "struct rusage", header: "<sys/resource.h>", + bycopy.} = object + ru_utime*, ru_stime*: Timeval # User and system time + ru_maxrss*, ru_ixrss*, ru_idrss*, ru_isrss*, # memory sizes + ru_minflt*, ru_majflt*, ru_nswap*, # paging activity + ru_inblock*, ru_oublock*, ru_msgsnd*, ru_msgrcv*, # IO activity + ru_nsignals*, ru_nvcsw*, ru_nivcsw*: clong # switching activity + +proc wait4*(pid: Pid, status: ptr cint, options: cint, rusage: ptr Rusage): Pid + {.importc, header: "<sys/wait.h>", sideEffect.} + +const + RUSAGE_SELF* = cint(0) + RUSAGE_CHILDREN* = cint(-1) + RUSAGE_THREAD* = cint(1) # This one is less std; Linux, BSD agree though. + +# This can only fail if `who` is invalid or `rusage` ptr is invalid. +proc getrusage*(who: cint, rusage: ptr Rusage): cint + {.importc, header: "<sys/resource.h>", discardable.} proc bsd_signal*(a1: cint, a2: proc (x: pointer) {.noconv.}) {. importc, header: "<signal.h>".} -proc kill*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>".} -proc killpg*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>".} +proc kill*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>", sideEffect.} +proc killpg*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>", sideEffect.} proc pthread_kill*(a1: Pthread, a2: cint): cint {.importc, header: "<signal.h>".} proc pthread_sigmask*(a1: cint, a2, a3: var Sigset): cint {. importc, header: "<signal.h>".} proc `raise`*(a1: cint): cint {.importc, header: "<signal.h>".} proc sigaction*(a1: cint, a2, a3: var Sigaction): cint {. importc, header: "<signal.h>".} + +proc sigaction*(a1: cint, a2: var Sigaction; a3: ptr Sigaction = nil): cint {. + importc, header: "<signal.h>".} + proc sigaddset*(a1: var Sigset, a2: cint): cint {.importc, header: "<signal.h>".} proc sigaltstack*(a1, a2: var Stack): cint {.importc, header: "<signal.h>".} proc sigdelset*(a1: var Sigset, a2: cint): cint {.importc, header: "<signal.h>".} @@ -2341,7 +801,7 @@ proc sighold*(a1: cint): cint {.importc, header: "<signal.h>".} proc sigignore*(a1: cint): cint {.importc, header: "<signal.h>".} proc siginterrupt*(a1, a2: cint): cint {.importc, header: "<signal.h>".} proc sigismember*(a1: var Sigset, a2: cint): cint {.importc, header: "<signal.h>".} -proc signal*(a1: cint, a2: proc (x: cint) {.noconv.}) {. +proc signal*(a1: cint, a2: Sighandler) {. importc, header: "<signal.h>".} proc sigpause*(a1: cint): cint {.importc, header: "<signal.h>".} proc sigpending*(a1: var Sigset): cint {.importc, header: "<signal.h>".} @@ -2353,19 +813,36 @@ proc sigrelse*(a1: cint): cint {.importc, header: "<signal.h>".} proc sigset*(a1: int, a2: proc (x: cint) {.noconv.}) {. importc, header: "<signal.h>".} proc sigsuspend*(a1: var Sigset): cint {.importc, header: "<signal.h>".} -proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, - a3: var Timespec): cint {.importc, header: "<signal.h>".} + +when defined(android): + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} + var NR_rt_sigtimedwait {.importc: "__NR_rt_sigtimedwait", header: "<sys/syscall.h>".}: clong + var NSIGMAX {.importc: "NSIG", header: "<signal.h>".}: clong + + proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint = + result = cint(syscall(NR_rt_sigtimedwait, addr(a1), addr(a2), addr(a3), NSIGMAX div 8)) +else: + proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, + a3: var Timespec): cint {.importc, header: "<signal.h>".} + +when defined(sunos) or defined(solaris): + # The following compile time flag is needed on Illumos/Solaris to use the POSIX + # `sigwait` implementation. See the documentation here: + # https://docs.oracle.com/cd/E19455-01/806-5257/6je9h033k/index.html + # https://www.illumos.org/man/2/sigwait + {.passc: "-D_POSIX_PTHREAD_SEMANTICS".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "<signal.h>".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. importc, header: "<signal.h>".} - -proc catclose*(a1: Nl_catd): cint {.importc, header: "<nl_types.h>".} -proc catgets*(a1: Nl_catd, a2, a3: cint, a4: cstring): cstring {. - importc, header: "<nl_types.h>".} -proc catopen*(a1: cstring, a2: cint): Nl_catd {. - importc, header: "<nl_types.h>".} +when not defined(nintendoswitch): + proc catclose*(a1: Nl_catd): cint {.importc, header: "<nl_types.h>".} + proc catgets*(a1: Nl_catd, a2, a3: cint, a4: cstring): cstring {. + importc, header: "<nl_types.h>".} + proc catopen*(a1: cstring, a2: cint): Nl_catd {. + importc, header: "<nl_types.h>".} proc sched_get_priority_max*(a1: cint): cint {.importc, header: "<sched.h>".} proc sched_get_priority_min*(a1: cint): cint {.importc, header: "<sched.h>".} @@ -2380,7 +857,6 @@ proc sched_setscheduler*(a1: Pid, a2: cint, a3: var Sched_param): cint {. importc, header: "<sched.h>".} proc sched_yield*(): cint {.importc, header: "<sched.h>".} -proc strerror*(errnum: cint): cstring {.importc, header: "<string.h>".} proc hstrerror*(herrnum: cint): cstring {.importc:"(char *)$1", header: "<netdb.h>".} proc FD_CLR*(a1: cint, a2: var TFdSet) {.importc, header: "<sys/select.h>".} @@ -2447,11 +923,12 @@ when hasSpawnH: a4: var Tposix_spawnattr, a5, a6: cstringArray): cint {.importc, header: "<spawn.h>".} -proc getcontext*(a1: var Ucontext): cint {.importc, header: "<ucontext.h>".} -proc makecontext*(a1: var Ucontext, a4: proc (){.noconv.}, a3: cint) {. - varargs, importc, header: "<ucontext.h>".} -proc setcontext*(a1: var Ucontext): cint {.importc, header: "<ucontext.h>".} -proc swapcontext*(a1, a2: var Ucontext): cint {.importc, header: "<ucontext.h>".} +when not defined(nintendoswitch): + proc getcontext*(a1: var Ucontext): cint {.importc, header: "<ucontext.h>".} + proc makecontext*(a1: var Ucontext, a4: proc (){.noconv.}, a3: cint) {. + varargs, importc, header: "<ucontext.h>".} + proc setcontext*(a1: var Ucontext): cint {.importc, header: "<ucontext.h>".} + proc swapcontext*(a1, a2: var Ucontext): cint {.importc, header: "<ucontext.h>".} proc readv*(a1: cint, a2: ptr IOVec, a3: cint): int {. importc, header: "<sys/uio.h>".} @@ -2467,45 +944,55 @@ proc CMSG_NXTHDR*(mhdr: ptr Tmsghdr, cmsg: ptr Tcmsghdr): ptr Tcmsghdr {. proc CMSG_FIRSTHDR*(mhdr: ptr Tmsghdr): ptr Tcmsghdr {. importc, header: "<sys/socket.h>".} +proc CMSG_SPACE*(len: csize_t): csize_t {. + importc, header: "<sys/socket.h>".} + +proc CMSG_LEN*(len: csize_t): csize_t {. + importc, header: "<sys/socket.h>".} + const INVALID_SOCKET* = SocketHandle(-1) proc `==`*(x, y: SocketHandle): bool {.borrow.} -proc accept*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr Socklen): SocketHandle {. - importc, header: "<sys/socket.h>".} +proc accept*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen): SocketHandle {. + importc, header: "<sys/socket.h>", sideEffect.} + +when defined(linux) or defined(bsd) or defined(nuttx): + proc accept4*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen, + flags: cint): SocketHandle {.importc, header: "<sys/socket.h>".} -proc bindSocket*(a1: SocketHandle, a2: ptr SockAddr, a3: Socklen): cint {. +proc bindSocket*(a1: SocketHandle, a2: ptr SockAddr, a3: SockLen): cint {. importc: "bind", header: "<sys/socket.h>".} - ## is Posix's ``bind``, because ``bind`` is a reserved word + ## is Posix's `bind`, because `bind` is a reserved word -proc connect*(a1: SocketHandle, a2: ptr SockAddr, a3: Socklen): cint {. +proc connect*(a1: SocketHandle, a2: ptr SockAddr, a3: SockLen): cint {. importc, header: "<sys/socket.h>".} -proc getpeername*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr Socklen): cint {. +proc getpeername*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen): cint {. importc, header: "<sys/socket.h>".} -proc getsockname*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr Socklen): cint {. +proc getsockname*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen): cint {. importc, header: "<sys/socket.h>".} -proc getsockopt*(a1: SocketHandle, a2, a3: cint, a4: pointer, a5: ptr Socklen): cint {. +proc getsockopt*(a1: SocketHandle, a2, a3: cint, a4: pointer, a5: ptr SockLen): cint {. importc, header: "<sys/socket.h>".} proc listen*(a1: SocketHandle, a2: cint): cint {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc recv*(a1: SocketHandle, a2: pointer, a3: int, a4: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc recvfrom*(a1: SocketHandle, a2: pointer, a3: int, a4: cint, - a5: ptr SockAddr, a6: ptr Socklen): int {. - importc, header: "<sys/socket.h>".} + a5: ptr SockAddr, a6: ptr SockLen): int {. + importc, header: "<sys/socket.h>", sideEffect.} proc recvmsg*(a1: SocketHandle, a2: ptr Tmsghdr, a3: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc send*(a1: SocketHandle, a2: pointer, a3: int, a4: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc sendmsg*(a1: SocketHandle, a2: ptr Tmsghdr, a3: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc sendto*(a1: SocketHandle, a2: pointer, a3: int, a4: cint, a5: ptr SockAddr, - a6: Socklen): int {. - importc, header: "<sys/socket.h>".} -proc setsockopt*(a1: SocketHandle, a2, a3: cint, a4: pointer, a5: Socklen): cint {. + a6: SockLen): int {. + importc, header: "<sys/socket.h>", sideEffect.} +proc setsockopt*(a1: SocketHandle, a2, a3: cint, a4: pointer, a5: SockLen): cint {. importc, header: "<sys/socket.h>".} proc shutdown*(a1: SocketHandle, a2: cint): cint {. importc, header: "<sys/socket.h>".} @@ -2537,9 +1024,15 @@ proc IN6_IS_ADDR_LINKLOCAL* (a1: ptr In6Addr): cint {. proc IN6_IS_ADDR_SITELOCAL* (a1: ptr In6Addr): cint {. importc, header: "<netinet/in.h>".} ## Unicast site-local address. -proc IN6_IS_ADDR_V4MAPPED* (a1: ptr In6Addr): cint {. - importc, header: "<netinet/in.h>".} - ## IPv4 mapped address. +when defined(lwip): + proc IN6_IS_ADDR_V4MAPPED*(ipv6_address: ptr In6Addr): cint = + var bits32: ptr array[4, uint32] = cast[ptr array[4, uint32]](ipv6_address) + return (bits32[1] == 0'u32 and bits32[2] == htonl(0x0000FFFF)).cint +else: + proc IN6_IS_ADDR_V4MAPPED* (a1: ptr In6Addr): cint {. + importc, header: "<netinet/in.h>".} + ## IPv4 mapped address. + proc IN6_IS_ADDR_V4COMPAT* (a1: ptr In6Addr): cint {. importc, header: "<netinet/in.h>".} ## IPv4-compatible address. @@ -2563,21 +1056,25 @@ proc endhostent*() {.importc, header: "<netdb.h>".} proc endnetent*() {.importc, header: "<netdb.h>".} proc endprotoent*() {.importc, header: "<netdb.h>".} proc endservent*() {.importc, header: "<netdb.h>".} -proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "<netdb.h>".} +proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "<netdb.h>".} proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h>".} proc getaddrinfo*(a1, a2: cstring, a3: ptr AddrInfo, a4: var ptr AddrInfo): cint {.importc, header: "<netdb.h>".} -proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. - importc, header: "<netdb.h>".} +when not defined(android4): + proc gethostbyaddr*(a1: pointer, a2: SockLen, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} +else: + proc gethostbyaddr*(a1: cstring, a2: cint, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} proc gethostbyname*(a1: cstring): ptr Hostent {.importc, header: "<netdb.h>".} proc gethostent*(): ptr Hostent {.importc, header: "<netdb.h>".} -proc getnameinfo*(a1: ptr SockAddr, a2: Socklen, - a3: cstring, a4: Socklen, a5: cstring, - a6: Socklen, a7: cint): cint {.importc, header: "<netdb.h>".} +proc getnameinfo*(a1: ptr SockAddr, a2: SockLen, + a3: cstring, a4: SockLen, a5: cstring, + a6: SockLen, a7: cint): cint {.importc, header: "<netdb.h>".} proc getnetbyaddr*(a1: int32, a2: cint): ptr Tnetent {.importc, header: "<netdb.h>".} proc getnetbyname*(a1: cstring): ptr Tnetent {.importc, header: "<netdb.h>".} @@ -2597,14 +1094,37 @@ proc setnetent*(a1: cint) {.importc, header: "<netdb.h>".} proc setprotoent*(a1: cint) {.importc, header: "<netdb.h>".} proc setservent*(a1: cint) {.importc, header: "<netdb.h>".} -proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. - importc, header: "<poll.h>".} +when not defined(lwip): + proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. + importc, header: "<poll.h>", sideEffect.} proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "<stdlib.h>".} -proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. - importc: "utimes", header: "<sys/time.h>".} +proc mkstemp*(tmpl: cstring): cint {.importc, header: "<stdlib.h>", sideEffect.} + ## Creates a unique temporary file. + ## + ## .. warning:: The `tmpl` argument is written to by `mkstemp` and thus + ## can't be a string literal. If in doubt make a copy of the cstring before + ## passing it in. + +proc mkstemps*(tmpl: cstring, suffixlen: int): cint {.importc, header: "<stdlib.h>", sideEffect.} + ## Creates a unique temporary file. + ## + ## .. warning:: The `tmpl` argument is written to by `mkstemps` and thus + ## can't be a string literal. If in doubt make a copy of the cstring before + ## passing it in. + +proc mkdtemp*(tmpl: cstring): pointer {.importc, header: "<stdlib.h>", sideEffect.} + +when defined(linux) or defined(bsd) or defined(osx): + proc mkostemp*(tmpl: cstring, oflags: cint): cint {.importc, header: "<stdlib.h>", sideEffect.} + proc mkostemps*(tmpl: cstring, suffixlen: cint, oflags: cint): cint {.importc, header: "<stdlib.h>", sideEffect.} + + proc posix_memalign*(memptr: pointer, alignment: csize_t, size: csize_t): cint {.importc, header: "<stdlib.h>".} + +proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. + importc: "utimes", header: "<sys/time.h>", sideEffect.} ## Sets file access and modification times. ## ## Pass the filename and an array of times to set the access and modification @@ -2614,3 +1134,43 @@ proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. ## Returns zero on success. ## ## For more information read http://www.unix.com/man-page/posix/3/utimes/. + +proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "<signal.h>".} + +template onSignal*(signals: varargs[cint], body: untyped) = + ## Setup code to be executed when Unix signals are received. The + ## currently handled signal is injected as `sig` into the calling + ## scope. + ## + ## Example: + ## ```Nim + ## from std/posix import SIGINT, SIGTERM, onSignal + ## onSignal(SIGINT, SIGTERM): + ## echo "bye from signal ", sig + ## ``` + + for s in signals: + handle_signal(s, + proc (signal: cint) {.noconv.} = + let sig {.inject.} = signal + body + ) + +type + RLimit* {.importc: "struct rlimit", + header: "<sys/resource.h>", pure, final.} = object + rlim_cur*: int + rlim_max*: int + ## The getrlimit() and setrlimit() system calls get and set resource limits respectively. + ## Each resource has an associated soft and hard limit, as defined by the RLimit structure + +proc setrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "setrlimit", header: "<sys/resource.h>".} + ## The setrlimit() system calls sets resource limits. + +proc getrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "getrlimit", header: "<sys/resource.h>".} + ## The getrlimit() system call gets resource limits. + +when defined(nimHasStyleChecks): + {.pop.} # {.push styleChecks: off.} diff --git a/lib/posix/posix_freertos_consts.nim b/lib/posix/posix_freertos_consts.nim new file mode 100644 index 000000000..0f0fc0aae --- /dev/null +++ b/lib/posix/posix_freertos_consts.nim @@ -0,0 +1,506 @@ +# Generated by detect.nim + + + +# <errno.h> +var E2BIG* {.importc: "E2BIG", header: "<errno.h>".}: cint +var EACCES* {.importc: "EACCES", header: "<errno.h>".}: cint +var EADDRINUSE* {.importc: "EADDRINUSE", header: "<errno.h>".}: cint +var EADDRNOTAVAIL* {.importc: "EADDRNOTAVAIL", header: "<errno.h>".}: cint +var EAFNOSUPPORT* {.importc: "EAFNOSUPPORT", header: "<errno.h>".}: cint +var EAGAIN* {.importc: "EAGAIN", header: "<errno.h>".}: cint +var EALREADY* {.importc: "EALREADY", header: "<errno.h>".}: cint +var EBADF* {.importc: "EBADF", header: "<errno.h>".}: cint +var EBADMSG* {.importc: "EBADMSG", header: "<errno.h>".}: cint +var EBUSY* {.importc: "EBUSY", header: "<errno.h>".}: cint +var ECANCELED* {.importc: "ECANCELED", header: "<errno.h>".}: cint +var ECHILD* {.importc: "ECHILD", header: "<errno.h>".}: cint +var ECONNABORTED* {.importc: "ECONNABORTED", header: "<errno.h>".}: cint +var ECONNREFUSED* {.importc: "ECONNREFUSED", header: "<errno.h>".}: cint +var ECONNRESET* {.importc: "ECONNRESET", header: "<errno.h>".}: cint +var EDEADLK* {.importc: "EDEADLK", header: "<errno.h>".}: cint +var EDESTADDRREQ* {.importc: "EDESTADDRREQ", header: "<errno.h>".}: cint +var EDOM* {.importc: "EDOM", header: "<errno.h>".}: cint +var EDQUOT* {.importc: "EDQUOT", header: "<errno.h>".}: cint +var EEXIST* {.importc: "EEXIST", header: "<errno.h>".}: cint +var EFAULT* {.importc: "EFAULT", header: "<errno.h>".}: cint +var EFBIG* {.importc: "EFBIG", header: "<errno.h>".}: cint +var EHOSTUNREACH* {.importc: "EHOSTUNREACH", header: "<errno.h>".}: cint +var EIDRM* {.importc: "EIDRM", header: "<errno.h>".}: cint +var EILSEQ* {.importc: "EILSEQ", header: "<errno.h>".}: cint +var EINPROGRESS* {.importc: "EINPROGRESS", header: "<errno.h>".}: cint +var EINTR* {.importc: "EINTR", header: "<errno.h>".}: cint +var EINVAL* {.importc: "EINVAL", header: "<errno.h>".}: cint +var EIO* {.importc: "EIO", header: "<errno.h>".}: cint +var EISCONN* {.importc: "EISCONN", header: "<errno.h>".}: cint +var EISDIR* {.importc: "EISDIR", header: "<errno.h>".}: cint +var ELOOP* {.importc: "ELOOP", header: "<errno.h>".}: cint +var EMFILE* {.importc: "EMFILE", header: "<errno.h>".}: cint +var EMLINK* {.importc: "EMLINK", header: "<errno.h>".}: cint +var EMSGSIZE* {.importc: "EMSGSIZE", header: "<errno.h>".}: cint +var EMULTIHOP* {.importc: "EMULTIHOP", header: "<errno.h>".}: cint +var ENAMETOOLONG* {.importc: "ENAMETOOLONG", header: "<errno.h>".}: cint +var ENETDOWN* {.importc: "ENETDOWN", header: "<errno.h>".}: cint +var ENETRESET* {.importc: "ENETRESET", header: "<errno.h>".}: cint +var ENETUNREACH* {.importc: "ENETUNREACH", header: "<errno.h>".}: cint +var ENFILE* {.importc: "ENFILE", header: "<errno.h>".}: cint +var ENOBUFS* {.importc: "ENOBUFS", header: "<errno.h>".}: cint +var ENODATA* {.importc: "ENODATA", header: "<errno.h>".}: cint +var ENODEV* {.importc: "ENODEV", header: "<errno.h>".}: cint +var ENOENT* {.importc: "ENOENT", header: "<errno.h>".}: cint +var ENOEXEC* {.importc: "ENOEXEC", header: "<errno.h>".}: cint +var ENOLCK* {.importc: "ENOLCK", header: "<errno.h>".}: cint +var ENOLINK* {.importc: "ENOLINK", header: "<errno.h>".}: cint +var ENOMEM* {.importc: "ENOMEM", header: "<errno.h>".}: cint +var ENOMSG* {.importc: "ENOMSG", header: "<errno.h>".}: cint +var ENOPROTOOPT* {.importc: "ENOPROTOOPT", header: "<errno.h>".}: cint +var ENOSPC* {.importc: "ENOSPC", header: "<errno.h>".}: cint +var ENOSR* {.importc: "ENOSR", header: "<errno.h>".}: cint +var ENOSTR* {.importc: "ENOSTR", header: "<errno.h>".}: cint +var ENOSYS* {.importc: "ENOSYS", header: "<errno.h>".}: cint +var ENOTCONN* {.importc: "ENOTCONN", header: "<errno.h>".}: cint +var ENOTDIR* {.importc: "ENOTDIR", header: "<errno.h>".}: cint +var ENOTEMPTY* {.importc: "ENOTEMPTY", header: "<errno.h>".}: cint +var ENOTSOCK* {.importc: "ENOTSOCK", header: "<errno.h>".}: cint +var ENOTSUP* {.importc: "ENOTSUP", header: "<errno.h>".}: cint +var ENOTTY* {.importc: "ENOTTY", header: "<errno.h>".}: cint +var ENXIO* {.importc: "ENXIO", header: "<errno.h>".}: cint +var EOPNOTSUPP* {.importc: "EOPNOTSUPP", header: "<errno.h>".}: cint +var EOVERFLOW* {.importc: "EOVERFLOW", header: "<errno.h>".}: cint +var EPERM* {.importc: "EPERM", header: "<errno.h>".}: cint +var EPIPE* {.importc: "EPIPE", header: "<errno.h>".}: cint +var EPROTO* {.importc: "EPROTO", header: "<errno.h>".}: cint +var EPROTONOSUPPORT* {.importc: "EPROTONOSUPPORT", header: "<errno.h>".}: cint +var EPROTOTYPE* {.importc: "EPROTOTYPE", header: "<errno.h>".}: cint +var ERANGE* {.importc: "ERANGE", header: "<errno.h>".}: cint +var EROFS* {.importc: "EROFS", header: "<errno.h>".}: cint +var ESPIPE* {.importc: "ESPIPE", header: "<errno.h>".}: cint +var ESRCH* {.importc: "ESRCH", header: "<errno.h>".}: cint +var ESTALE* {.importc: "ESTALE", header: "<errno.h>".}: cint +var ETIME* {.importc: "ETIME", header: "<errno.h>".}: cint +var ETIMEDOUT* {.importc: "ETIMEDOUT", header: "<errno.h>".}: cint +var ETXTBSY* {.importc: "ETXTBSY", header: "<errno.h>".}: cint +var EWOULDBLOCK* {.importc: "EWOULDBLOCK", header: "<errno.h>".}: cint +var EXDEV* {.importc: "EXDEV", header: "<errno.h>".}: cint + +# <sys/fcntl.h> +var F_GETFD* {.importc: "F_GETFD", header: "<sys/fcntl.h>".}: cint +# var F_SETFD* {.importc: "F_SETFD", header: "<sys/fcntl.h>".}: cint +var F_GETFL* {.importc: "F_GETFL", header: "<sys/fcntl.h>".}: cint +var F_SETFL* {.importc: "F_SETFL", header: "<sys/fcntl.h>".}: cint +# var F_GETLK* {.importc: "F_GETLK", header: "<sys/fcntl.h>".}: cint +# var F_SETLK* {.importc: "F_SETLK", header: "<sys/fcntl.h>".}: cint +# var F_SETLKW* {.importc: "F_SETLKW", header: "<sys/fcntl.h>".}: cint +# var F_GETOWN* {.importc: "F_GETOWN", header: "<sys/fcntl.h>".}: cint +# var F_SETOWN* {.importc: "F_SETOWN", header: "<sys/fcntl.h>".}: cint +# var FD_CLOEXEC* {.importc: "FD_CLOEXEC", header: "<sys/fcntl.h>".}: cint +# var F_RDLCK* {.importc: "F_RDLCK", header: "<sys/fcntl.h>".}: cint +# var F_UNLCK* {.importc: "F_UNLCK", header: "<sys/fcntl.h>".}: cint +# var F_WRLCK* {.importc: "F_WRLCK", header: "<sys/fcntl.h>".}: cint +var O_CREAT* {.importc: "O_CREAT", header: "<sys/fcntl.h>".}: cint +var O_EXCL* {.importc: "O_EXCL", header: "<sys/fcntl.h>".}: cint +# var O_NOCTTY* {.importc: "O_NOCTTY", header: "<sys/fcntl.h>".}: cint +var O_TRUNC* {.importc: "O_TRUNC", header: "<sys/fcntl.h>".}: cint +var O_APPEND* {.importc: "O_APPEND", header: "<sys/fcntl.h>".}: cint +# var O_DSYNC* {.importc: "O_DSYNC", header: "<sys/fcntl.h>".}: cint +var O_NONBLOCK* {.importc: "O_NONBLOCK", header: "<sys/fcntl.h>".}: cint +# var O_RSYNC* {.importc: "O_RSYNC", header: "<sys/fcntl.h>".}: cint +# var O_SYNC* {.importc: "O_SYNC", header: "<sys/fcntl.h>".}: cint +# var O_ACCMODE* {.importc: "O_ACCMODE", header: "<sys/fcntl.h>".}: cint +var O_RDONLY* {.importc: "O_RDONLY", header: "<sys/fcntl.h>".}: cint +var O_RDWR* {.importc: "O_RDWR", header: "<sys/fcntl.h>".}: cint +var O_WRONLY* {.importc: "O_WRONLY", header: "<sys/fcntl.h>".}: cint +# var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "<sys/fcntl.h>".}: cint + +# # <locale.h> +# var LC_ALL* {.importc: "LC_ALL", header: "<locale.h>".}: cint +# var LC_COLLATE* {.importc: "LC_COLLATE", header: "<locale.h>".}: cint +# var LC_CTYPE* {.importc: "LC_CTYPE", header: "<locale.h>".}: cint +# var LC_MESSAGES* {.importc: "LC_MESSAGES", header: "<locale.h>".}: cint +# var LC_MONETARY* {.importc: "LC_MONETARY", header: "<locale.h>".}: cint +# var LC_NUMERIC* {.importc: "LC_NUMERIC", header: "<locale.h>".}: cint +# var LC_TIME* {.importc: "LC_TIME", header: "<locale.h>".}: cint + +# <netdb.h> +var HOST_NOT_FOUND* {.importc: "HOST_NOT_FOUND", header: "<netdb.h>".}: cint +var NO_DATA* {.importc: "NO_DATA", header: "<netdb.h>".}: cint +var NO_RECOVERY* {.importc: "NO_RECOVERY", header: "<netdb.h>".}: cint +var TRY_AGAIN* {.importc: "TRY_AGAIN", header: "<netdb.h>".}: cint +var AI_PASSIVE* {.importc: "AI_PASSIVE", header: "<netdb.h>".}: cint +var AI_CANONNAME* {.importc: "AI_CANONNAME", header: "<netdb.h>".}: cint +var AI_NUMERICHOST* {.importc: "AI_NUMERICHOST", header: "<netdb.h>".}: cint +var AI_NUMERICSERV* {.importc: "AI_NUMERICSERV", header: "<netdb.h>".}: cint +var AI_V4MAPPED* {.importc: "AI_V4MAPPED", header: "<netdb.h>".}: cint +var AI_ALL* {.importc: "AI_ALL", header: "<netdb.h>".}: cint +var AI_ADDRCONFIG* {.importc: "AI_ADDRCONFIG", header: "<netdb.h>".}: cint +var NI_NOFQDN* {.importc: "NI_NOFQDN", header: "<netdb.h>".}: cint +var NI_NUMERICHOST* {.importc: "NI_NUMERICHOST", header: "<netdb.h>".}: cint +var NI_NAMEREQD* {.importc: "NI_NAMEREQD", header: "<netdb.h>".}: cint +var NI_NUMERICSERV* {.importc: "NI_NUMERICSERV", header: "<netdb.h>".}: cint +var NI_DGRAM* {.importc: "NI_DGRAM", header: "<netdb.h>".}: cint +var EAI_AGAIN* {.importc: "EAI_AGAIN", header: "<netdb.h>".}: cint +var EAI_BADFLAGS* {.importc: "EAI_BADFLAGS", header: "<netdb.h>".}: cint +var EAI_FAIL* {.importc: "EAI_FAIL", header: "<netdb.h>".}: cint +var EAI_FAMILY* {.importc: "EAI_FAMILY", header: "<netdb.h>".}: cint +var EAI_MEMORY* {.importc: "EAI_MEMORY", header: "<netdb.h>".}: cint +var EAI_NONAME* {.importc: "EAI_NONAME", header: "<netdb.h>".}: cint +var EAI_SERVICE* {.importc: "EAI_SERVICE", header: "<netdb.h>".}: cint +var EAI_SOCKTYPE* {.importc: "EAI_SOCKTYPE", header: "<netdb.h>".}: cint + +var LWIP_DNS_API_DECLARE_H_ERRNO* {.importc: "LWIP_DNS_API_DECLARE_H_ERRNO", header: "<netdb.h>".}: cint +var LWIP_DNS_API_DEFINE_ERRORS* {.importc: "LWIP_DNS_API_DEFINE_ERRORS", header: "<netdb.h>".}: cint +var LWIP_DNS_API_DEFINE_FLAGS* {.importc: "LWIP_DNS_API_DEFINE_FLAGS", header: "<netdb.h>".}: cint +var LWIP_DNS_API_DECLARE_STRUCTS* {.importc: "LWIP_DNS_API_DECLARE_STRUCTS", header: "<netdb.h>".}: cint +var NETDB_ELEM_SIZE* {.importc: "NETDB_ELEM_SIZE", header: "<netdb.h>".}: cint + + +# <netif.h> +var ENABLE_LOOPBACK* {.importc: "ENABLE_LOOPBACK", header: "<netif.h>".}: cint +var NETIF_MAX_HWADDR_LEN* {.importc: "NETIF_MAX_HWADDR_LEN", header: "<netif.h>".}: cint +var NETIF_NAMESIZE* {.importc: "NETIF_NAMESIZE", header: "<netif.h>".}: cint +var NETIF_FLAG_UP* {.importc: "NETIF_FLAG_UP", header: "<netif.h>".}: cint +var NETIF_FLAG_BROADCAST* {.importc: "NETIF_FLAG_BROADCAST", header: "<netif.h>".}: cint +var NETIF_FLAG_LINK_UP* {.importc: "NETIF_FLAG_LINK_UP", header: "<netif.h>".}: cint +var NETIF_FLAG_ETHARP* {.importc: "NETIF_FLAG_ETHARP", header: "<netif.h>".}: cint +var NETIF_FLAG_ETHERNET* {.importc: "NETIF_FLAG_ETHERNET", header: "<netif.h>".}: cint +var NETIF_FLAG_IGMP* {.importc: "NETIF_FLAG_IGMP", header: "<netif.h>".}: cint +var NETIF_FLAG_MLD6* {.importc: "NETIF_FLAG_MLD6", header: "<netif.h>".}: cint +var NETIF_FLAG_GARP* {.importc: "NETIF_FLAG_GARP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_IP* {.importc: "NETIF_CHECKSUM_GEN_IP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_UDP* {.importc: "NETIF_CHECKSUM_GEN_UDP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_TCP* {.importc: "NETIF_CHECKSUM_GEN_TCP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_ICMP* {.importc: "NETIF_CHECKSUM_GEN_ICMP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_ICMP6* {.importc: "NETIF_CHECKSUM_GEN_ICMP6", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_IP* {.importc: "NETIF_CHECKSUM_CHECK_IP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_UDP* {.importc: "NETIF_CHECKSUM_CHECK_UDP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_TCP* {.importc: "NETIF_CHECKSUM_CHECK_TCP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_ICMP* {.importc: "NETIF_CHECKSUM_CHECK_ICMP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_ICMP6* {.importc: "NETIF_CHECKSUM_CHECK_ICMP6", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_ENABLE_ALL* {.importc: "NETIF_CHECKSUM_ENABLE_ALL", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_DISABLE_ALL* {.importc: "NETIF_CHECKSUM_DISABLE_ALL", header: "<netif.h>".}: cint +var NETIF_ADDR_IDX_MAX* {.importc: "NETIF_ADDR_IDX_MAX", header: "<netif.h>".}: cint +var LWIP_NETIF_USE_HINTS* {.importc: "LWIP_NETIF_USE_HINTS", header: "<netif.h>".}: cint +var NETIF_NO_INDEX* {.importc: "NETIF_NO_INDEX", header: "<netif.h>".}: cint +var LWIP_NSC_NONE* {.importc: "LWIP_NSC_NONE", header: "<netif.h>".}: cint +var LWIP_NSC_NETIF_ADDED* {.importc: "LWIP_NSC_NETIF_ADDED", header: "<netif.h>".}: cint +var LWIP_NSC_NETIF_REMOVED* {.importc: "LWIP_NSC_NETIF_REMOVED", header: "<netif.h>".}: cint +var LWIP_NSC_LINK_CHANGED* {.importc: "LWIP_NSC_LINK_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_STATUS_CHANGED* {.importc: "LWIP_NSC_STATUS_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_ADDRESS_CHANGED* {.importc: "LWIP_NSC_IPV4_ADDRESS_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_GATEWAY_CHANGED* {.importc: "LWIP_NSC_IPV4_GATEWAY_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_NETMASK_CHANGED* {.importc: "LWIP_NSC_IPV4_NETMASK_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_SETTINGS_CHANGED* {.importc: "LWIP_NSC_IPV4_SETTINGS_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV6_SET* {.importc: "LWIP_NSC_IPV6_SET", header: "<netif.h>".}: cint +var LWIP_NSC_IPV6_ADDR_STATE_CHANGED* {.importc: "LWIP_NSC_IPV6_ADDR_STATE_CHANGED", header: "<netif.h>".}: cint +#define NETIF_DECLARE_EXT_CALLBACK(name) + + +# <net/if.h> +var IF_NAMESIZE* {.importc: "IF_NAMESIZE", header: "<net/if.h>".}: cint + +# <sys/socket.h> +var IPPROTO_IP* {.importc: "IPPROTO_IP", header: "<sys/socket.h>".}: cint +var IPPROTO_IPV6* {.importc: "IPPROTO_IPV6", header: "<sys/socket.h>".}: cint +var IPPROTO_ICMP* {.importc: "IPPROTO_ICMP", header: "<sys/socket.h>".}: cint +var IPPROTO_ICMPV6* {.importc: "IPPROTO_ICMPV6", header: "<sys/socket.h>".}: cint +var IPPROTO_RAW* {.importc: "IPPROTO_RAW", header: "<sys/socket.h>".}: cint +var IPPROTO_TCP* {.importc: "IPPROTO_TCP", header: "<sys/socket.h>".}: cint +var IPPROTO_UDP* {.importc: "IPPROTO_UDP", header: "<sys/socket.h>".}: cint +var INADDR_ANY* {.importc: "INADDR_ANY", header: "<sys/socket.h>".}: InAddrScalar +var INADDR_LOOPBACK* {.importc: "INADDR_LOOPBACK", header: "<sys/socket.h>".}: InAddrScalar +var INADDR_BROADCAST* {.importc: "INADDR_BROADCAST", header: "<sys/socket.h>".}: InAddrScalar +var INET_ADDRSTRLEN* {.importc: "INET_ADDRSTRLEN", header: "<sys/socket.h>".}: cint +var INET6_ADDRSTRLEN* {.importc: "INET6_ADDRSTRLEN", header: "<sys/socket.h>".}: cint +var IPV6_JOIN_GROUP* {.importc: "IPV6_JOIN_GROUP", header: "<sys/socket.h>".}: cint +var IPV6_LEAVE_GROUP* {.importc: "IPV6_LEAVE_GROUP", header: "<sys/socket.h>".}: cint +var IPV6_MULTICAST_HOPS* {.importc: "IPV6_MULTICAST_HOPS", header: "<sys/socket.h>".}: cint +var IPV6_MULTICAST_IF* {.importc: "IPV6_MULTICAST_IF", header: "<sys/socket.h>".}: cint +var IPV6_MULTICAST_LOOP* {.importc: "IPV6_MULTICAST_LOOP", header: "<sys/socket.h>".}: cint +var IPV6_UNICAST_HOPS* {.importc: "IPV6_UNICAST_HOPS", header: "<sys/socket.h>".}: cint +var IPV6_V6ONLY* {.importc: "IPV6_V6ONLY", header: "<sys/socket.h>".}: cint + +# <netinet/tcp.h> +const TCP_NODELAY* = 0x01 # don't delay send to coalesce packets +const TCP_KEEPALIVE* = 0x02 # send KEEPALIVE probes when idle for pcb->keep_idle milliseconds +const TCP_KEEPIDLE* = 0x03 # set pcb->keep_idle - Same as TCP_KEEPALIVE, but use seconds for get/setsockopt +const TCP_KEEPINTVL* = 0x04 # set pcb->keep_intvl - Use seconds for get/setsockopt +const TCP_KEEPCNT* = 0x05 # set pcb->keep_cnt - Use number of probes sent for get/setsockopt + +# <nl_types.h> +# var NL_SETD* {.importc: "NL_SETD", header: "<nl_types.h>".}: cint +# var NL_CAT_LOCALE* {.importc: "NL_CAT_LOCALE", header: "<nl_types.h>".}: cint + +# <sys/poll.h> +# var POLLIN* {.importc: "POLLIN", header: "<sys/poll.h>".}: cshort +# var POLLRDNORM* {.importc: "POLLRDNORM", header: "<sys/poll.h>".}: cshort +# var POLLRDBAND* {.importc: "POLLRDBAND", header: "<sys/poll.h>".}: cshort +# var POLLPRI* {.importc: "POLLPRI", header: "<sys/poll.h>".}: cshort +# var POLLOUT* {.importc: "POLLOUT", header: "<sys/poll.h>".}: cshort +# var POLLWRNORM* {.importc: "POLLWRNORM", header: "<sys/poll.h>".}: cshort +# var POLLWRBAND* {.importc: "POLLWRBAND", header: "<sys/poll.h>".}: cshort +# var POLLERR* {.importc: "POLLERR", header: "<sys/poll.h>".}: cshort +# var POLLHUP* {.importc: "POLLHUP", header: "<sys/poll.h>".}: cshort +# var POLLNVAL* {.importc: "POLLNVAL", header: "<sys/poll.h>".}: cshort + +# <pthread.h> +var PTHREAD_STACK_MIN* {.importc: "PTHREAD_STACK_MIN", header: "<pthread.h>".}: cint +# var PTHREAD_BARRIER_SERIAL_THREAD* {.importc: "PTHREAD_BARRIER_SERIAL_THREAD", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_ASYNCHRONOUS* {.importc: "PTHREAD_CANCEL_ASYNCHRONOUS", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_ENABLE* {.importc: "PTHREAD_CANCEL_ENABLE", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_DEFERRED* {.importc: "PTHREAD_CANCEL_DEFERRED", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_DISABLE* {.importc: "PTHREAD_CANCEL_DISABLE", header: "<pthread.h>".}: cint +# var PTHREAD_CREATE_DETACHED* {.importc: "PTHREAD_CREATE_DETACHED", header: "<pthread.h>".}: cint +# var PTHREAD_CREATE_JOINABLE* {.importc: "PTHREAD_CREATE_JOINABLE", header: "<pthread.h>".}: cint +# var PTHREAD_EXPLICIT_SCHED* {.importc: "PTHREAD_EXPLICIT_SCHED", header: "<pthread.h>".}: cint +# var PTHREAD_INHERIT_SCHED* {.importc: "PTHREAD_INHERIT_SCHED", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_DEFAULT* {.importc: "PTHREAD_MUTEX_DEFAULT", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_ERRORCHECK* {.importc: "PTHREAD_MUTEX_ERRORCHECK", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_NORMAL* {.importc: "PTHREAD_MUTEX_NORMAL", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_RECURSIVE* {.importc: "PTHREAD_MUTEX_RECURSIVE", header: "<pthread.h>".}: cint +# var PTHREAD_PRIO_INHERIT* {.importc: "PTHREAD_PRIO_INHERIT", header: "<pthread.h>".}: cint +# var PTHREAD_PRIO_NONE* {.importc: "PTHREAD_PRIO_NONE", header: "<pthread.h>".}: cint +# var PTHREAD_PRIO_PROTECT* {.importc: "PTHREAD_PRIO_PROTECT", header: "<pthread.h>".}: cint +# var PTHREAD_PROCESS_SHARED* {.importc: "PTHREAD_PROCESS_SHARED", header: "<pthread.h>".}: cint +# var PTHREAD_PROCESS_PRIVATE* {.importc: "PTHREAD_PROCESS_PRIVATE", header: "<pthread.h>".}: cint +# var PTHREAD_SCOPE_PROCESS* {.importc: "PTHREAD_SCOPE_PROCESS", header: "<pthread.h>".}: cint +# var PTHREAD_SCOPE_SYSTEM* {.importc: "PTHREAD_SCOPE_SYSTEM", header: "<pthread.h>".}: cint + +# # <sched.h> +# var SCHED_FIFO* {.importc: "SCHED_FIFO", header: "<sched.h>".}: cint +# var SCHED_RR* {.importc: "SCHED_RR", header: "<sched.h>".}: cint +# var SCHED_SPORADIC* {.importc: "SCHED_SPORADIC", header: "<sched.h>".}: cint +# var SCHED_OTHER* {.importc: "SCHED_OTHER", header: "<sched.h>".}: cint + +# <semaphore.h> +var SEM_FAILED* {.importc: "SEM_FAILED", header: "<semaphore.h>".}: pointer + +# # <signal.h> +# var SIGEV_NONE* {.importc: "SIGEV_NONE", header: "<signal.h>".}: cint +# var SIGEV_SIGNAL* {.importc: "SIGEV_SIGNAL", header: "<signal.h>".}: cint +# var SIGEV_THREAD* {.importc: "SIGEV_THREAD", header: "<signal.h>".}: cint +# var SIGABRT* {.importc: "SIGABRT", header: "<signal.h>".}: cint +# var SIGALRM* {.importc: "SIGALRM", header: "<signal.h>".}: cint +# var SIGBUS* {.importc: "SIGBUS", header: "<signal.h>".}: cint +# var SIGCHLD* {.importc: "SIGCHLD", header: "<signal.h>".}: cint +# var SIGCONT* {.importc: "SIGCONT", header: "<signal.h>".}: cint +# var SIGFPE* {.importc: "SIGFPE", header: "<signal.h>".}: cint +# var SIGHUP* {.importc: "SIGHUP", header: "<signal.h>".}: cint +# var SIGILL* {.importc: "SIGILL", header: "<signal.h>".}: cint +# var SIGINT* {.importc: "SIGINT", header: "<signal.h>".}: cint +# var SIGKILL* {.importc: "SIGKILL", header: "<signal.h>".}: cint +# var SIGPIPE* {.importc: "SIGPIPE", header: "<signal.h>".}: cint +# var SIGQUIT* {.importc: "SIGQUIT", header: "<signal.h>".}: cint +# var SIGSEGV* {.importc: "SIGSEGV", header: "<signal.h>".}: cint +# var SIGSTOP* {.importc: "SIGSTOP", header: "<signal.h>".}: cint +# var SIGTERM* {.importc: "SIGTERM", header: "<signal.h>".}: cint +# var SIGTSTP* {.importc: "SIGTSTP", header: "<signal.h>".}: cint +# var SIGTTIN* {.importc: "SIGTTIN", header: "<signal.h>".}: cint +# var SIGTTOU* {.importc: "SIGTTOU", header: "<signal.h>".}: cint +# var SIGUSR1* {.importc: "SIGUSR1", header: "<signal.h>".}: cint +# var SIGUSR2* {.importc: "SIGUSR2", header: "<signal.h>".}: cint +# var SIGPOLL* {.importc: "SIGPOLL", header: "<signal.h>".}: cint +# var SIGPROF* {.importc: "SIGPROF", header: "<signal.h>".}: cint +# var SIGSYS* {.importc: "SIGSYS", header: "<signal.h>".}: cint +# var SIGTRAP* {.importc: "SIGTRAP", header: "<signal.h>".}: cint +# var SIGURG* {.importc: "SIGURG", header: "<signal.h>".}: cint +# var SIGVTALRM* {.importc: "SIGVTALRM", header: "<signal.h>".}: cint +# var SIGXCPU* {.importc: "SIGXCPU", header: "<signal.h>".}: cint +# var SIGXFSZ* {.importc: "SIGXFSZ", header: "<signal.h>".}: cint +# var SA_NOCLDSTOP* {.importc: "SA_NOCLDSTOP", header: "<signal.h>".}: cint +# var SIG_BLOCK* {.importc: "SIG_BLOCK", header: "<signal.h>".}: cint +# var SIG_UNBLOCK* {.importc: "SIG_UNBLOCK", header: "<signal.h>".}: cint +# var SIG_SETMASK* {.importc: "SIG_SETMASK", header: "<signal.h>".}: cint +# var SA_ONSTACK* {.importc: "SA_ONSTACK", header: "<signal.h>".}: cint +# var SA_RESETHAND* {.importc: "SA_RESETHAND", header: "<signal.h>".}: cint +# var SA_RESTART* {.importc: "SA_RESTART", header: "<signal.h>".}: cint +# var SA_SIGINFO* {.importc: "SA_SIGINFO", header: "<signal.h>".}: cint +# var SA_NOCLDWAIT* {.importc: "SA_NOCLDWAIT", header: "<signal.h>".}: cint +# var SA_NODEFER* {.importc: "SA_NODEFER", header: "<signal.h>".}: cint +# var SS_ONSTACK* {.importc: "SS_ONSTACK", header: "<signal.h>".}: cint +# var SS_DISABLE* {.importc: "SS_DISABLE", header: "<signal.h>".}: cint +# var MINSIGSTKSZ* {.importc: "MINSIGSTKSZ", header: "<signal.h>".}: cint +# var SIGSTKSZ* {.importc: "SIGSTKSZ", header: "<signal.h>".}: cint +# var SIG_HOLD* {.importc: "SIG_HOLD", header: "<signal.h>".}: Sighandler +# var SIG_DFL* {.importc: "SIG_DFL", header: "<signal.h>".}: Sighandler +# var SIG_ERR* {.importc: "SIG_ERR", header: "<signal.h>".}: Sighandler +# var SIG_IGN* {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler + +# # <sys/ipc.h> +# var IPC_CREAT* {.importc: "IPC_CREAT", header: "<sys/ipc.h>".}: cint +# var IPC_EXCL* {.importc: "IPC_EXCL", header: "<sys/ipc.h>".}: cint +# var IPC_NOWAIT* {.importc: "IPC_NOWAIT", header: "<sys/ipc.h>".}: cint +# var IPC_PRIVATE* {.importc: "IPC_PRIVATE", header: "<sys/ipc.h>".}: cint +# var IPC_RMID* {.importc: "IPC_RMID", header: "<sys/ipc.h>".}: cint +# var IPC_SET* {.importc: "IPC_SET", header: "<sys/ipc.h>".}: cint +# var IPC_STAT* {.importc: "IPC_STAT", header: "<sys/ipc.h>".}: cint + +# # <sys/mman.h> +# var PROT_READ* {.importc: "PROT_READ", header: "<sys/mman.h>".}: cint +# var PROT_WRITE* {.importc: "PROT_WRITE", header: "<sys/mman.h>".}: cint +# var PROT_EXEC* {.importc: "PROT_EXEC", header: "<sys/mman.h>".}: cint +# var PROT_NONE* {.importc: "PROT_NONE", header: "<sys/mman.h>".}: cint +# var MAP_ANONYMOUS* {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint +# var MAP_FIXED_NOREPLACE* {.importc: "MAP_FIXED_NOREPLACE", header: "<sys/mman.h>".}: cint +# var MAP_NORESERVE* {.importc: "MAP_NORESERVE", header: "<sys/mman.h>".}: cint +# var MAP_SHARED* {.importc: "MAP_SHARED", header: "<sys/mman.h>".}: cint +# var MAP_PRIVATE* {.importc: "MAP_PRIVATE", header: "<sys/mman.h>".}: cint +# var MAP_FIXED* {.importc: "MAP_FIXED", header: "<sys/mman.h>".}: cint +# var MS_ASYNC* {.importc: "MS_ASYNC", header: "<sys/mman.h>".}: cint +# var MS_SYNC* {.importc: "MS_SYNC", header: "<sys/mman.h>".}: cint +# var MS_INVALIDATE* {.importc: "MS_INVALIDATE", header: "<sys/mman.h>".}: cint +# var MCL_CURRENT* {.importc: "MCL_CURRENT", header: "<sys/mman.h>".}: cint +# var MCL_FUTURE* {.importc: "MCL_FUTURE", header: "<sys/mman.h>".}: cint +# var MAP_FAILED* {.importc: "MAP_FAILED", header: "<sys/mman.h>".}: pointer +# var POSIX_MADV_NORMAL* {.importc: "POSIX_MADV_NORMAL", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_SEQUENTIAL* {.importc: "POSIX_MADV_SEQUENTIAL", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_RANDOM* {.importc: "POSIX_MADV_RANDOM", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_WILLNEED* {.importc: "POSIX_MADV_WILLNEED", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_DONTNEED* {.importc: "POSIX_MADV_DONTNEED", header: "<sys/mman.h>".}: cint +# var POSIX_TYPED_MEM_ALLOCATE* {.importc: "POSIX_TYPED_MEM_ALLOCATE", header: "<sys/mman.h>".}: cint +# var POSIX_TYPED_MEM_ALLOCATE_CONTIG* {.importc: "POSIX_TYPED_MEM_ALLOCATE_CONTIG", header: "<sys/mman.h>".}: cint +# var POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc: "POSIX_TYPED_MEM_MAP_ALLOCATABLE", header: "<sys/mman.h>".}: cint + +# <sys/resource.h> +# var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint + +var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "<lwipopts.h>".}: cint + +# <sys/select.h> +var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint + +# <sys/socket.h> +# struct msghdr->msg_flags bit field values +const MSG_TRUNC* = 0x04 +const MSG_CTRUNC* = 0x08 + +# Flags we can use with send and recv. +const MSG_PEEK* = 0x01 # Peeks at an incoming message +const MSG_WAITALL* = 0x02 # Unimplemented: Requests that the function block until the full amount of data requested can be returned +const MSG_OOB* = 0x04 # Unimplemented: Requests out-of-band data. The significance and semantics of out-of-band data are protocol-specific +const MSG_DONTWAIT* = 0x08 # Nonblocking i/o for this operation only +const MSG_MORE* = 0x10 # Sender will send more +# const MSG_NOSIGNAL* = 0x20 # Uninmplemented: Requests not to send the SIGPIPE signal if an attempt to send is made on a stream-oriented socket that is no longer connected. + +# Alternately, they can defined like this, but the above seems better when they're known/stable values: +# var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "<sys/socket.h>".}: cint +# var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "<sys/socket.h>".}: cint +# var MSG_DONTROUTE* {.importc: "MSG_DONTROUTE", header: "<sys/socket.h>".}: cint # not defined in lwip +# var MSG_EOR* {.importc: "MSG_EOR", header: "<sys/socket.h>".}: cint # not defined in lwip +# var MSG_OOB* {.importc: "MSG_OOB", header: "<sys/socket.h>".}: cint + + +# var SCM_RIGHTS* {.importc: "SCM_RIGHTS", header: "<sys/socket.h>".}: cint +var SO_ACCEPTCONN* {.importc: "SO_ACCEPTCONN", header: "<sys/socket.h>".}: cint +var SO_BROADCAST* {.importc: "SO_BROADCAST", header: "<sys/socket.h>".}: cint +var SO_DEBUG* {.importc: "SO_DEBUG", header: "<sys/socket.h>".}: cint +var SO_DONTROUTE* {.importc: "SO_DONTROUTE", header: "<sys/socket.h>".}: cint +var SO_ERROR* {.importc: "SO_ERROR", header: "<sys/socket.h>".}: cint +var SO_KEEPALIVE* {.importc: "SO_KEEPALIVE", header: "<sys/socket.h>".}: cint +var SO_LINGER* {.importc: "SO_LINGER", header: "<sys/socket.h>".}: cint +var SO_OOBINLINE* {.importc: "SO_OOBINLINE", header: "<sys/socket.h>".}: cint +var SO_RCVBUF* {.importc: "SO_RCVBUF", header: "<sys/socket.h>".}: cint +var SO_RCVLOWAT* {.importc: "SO_RCVLOWAT", header: "<sys/socket.h>".}: cint +var SO_RCVTIMEO* {.importc: "SO_RCVTIMEO", header: "<sys/socket.h>".}: cint +var SO_REUSEADDR* {.importc: "SO_REUSEADDR", header: "<sys/socket.h>".}: cint +var SO_SNDBUF* {.importc: "SO_SNDBUF", header: "<sys/socket.h>".}: cint +var SO_SNDLOWAT* {.importc: "SO_SNDLOWAT", header: "<sys/socket.h>".}: cint +var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "<sys/socket.h>".}: cint +var SO_TYPE* {.importc: "SO_TYPE", header: "<sys/socket.h>".}: cint +var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "<sys/socket.h>".}: cint +var SOCK_RAW* {.importc: "SOCK_RAW", header: "<sys/socket.h>".}: cint + +# var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint +const SOCK_SEQPACKET* = cint(5) + +var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "<sys/socket.h>".}: cint +var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "<sys/socket.h>".}: cint + +const SocketMaxConnections {.intdefine.}: int = 32 +var SOMAXCONN*: cint = SocketMaxConnections.cint + +var AF_INET* {.importc: "AF_INET", header: "<sys/socket.h>".}: cint +var AF_INET6* {.importc: "AF_INET6", header: "<sys/socket.h>".}: cint +# var AF_UNIX* {.importc: "AF_UNIX", header: "<sys/socket.h>".}: cint +const AF_UNIX*: cint = 1 # for compat with Nim libraries, doesn't exist on freertos +var AF_UNSPEC* {.importc: "AF_UNSPEC", header: "<sys/socket.h>".}: cint +var SHUT_RD* {.importc: "SHUT_RD", header: "<sys/socket.h>".}: cint +var SHUT_RDWR* {.importc: "SHUT_RDWR", header: "<sys/socket.h>".}: cint +var SHUT_WR* {.importc: "SHUT_WR", header: "<sys/socket.h>".}: cint + +# # <sys/stat.h> +# <sys/stat.h> + +# var S_IFBLK* {.importc: "S_IFBLK", header: "<sys/stat.h>".}: cint +# var S_IFCHR* {.importc: "S_IFCHR", header: "<sys/stat.h>".}: cint +# var S_IFDIR* {.importc: "S_IFDIR", header: "<sys/stat.h>".}: cint +# var S_IFIFO* {.importc: "S_IFIFO", header: "<sys/stat.h>".}: cint +# var S_IFLNK* {.importc: "S_IFLNK", header: "<sys/stat.h>".}: cint +# var S_IFMT* {.importc: "S_IFMT", header: "<sys/stat.h>".}: cint +# var S_IFREG* {.importc: "S_IFREG", header: "<sys/stat.h>".}: cint +# var S_IFSOCK* {.importc: "S_IFSOCK", header: "<sys/stat.h>".}: cint +var S_IRGRP* {.importc: "S_IRGRP", header: "<sys/stat.h>".}: cint +var S_IROTH* {.importc: "S_IROTH", header: "<sys/stat.h>".}: cint +var S_IRUSR* {.importc: "S_IRUSR", header: "<sys/stat.h>".}: cint +# var S_IRWXG* {.importc: "S_IRWXG", header: "<sys/stat.h>".}: cint +# var S_IRWXO* {.importc: "S_IRWXO", header: "<sys/stat.h>".}: cint +# var S_IRWXU* {.importc: "S_IRWXU", header: "<sys/stat.h>".}: cint +# var S_ISGID* {.importc: "S_ISGID", header: "<sys/stat.h>".}: cint +# var S_ISUID* {.importc: "S_ISUID", header: "<sys/stat.h>".}: cint +# var S_ISVTX* {.importc: "S_ISVTX", header: "<sys/stat.h>".}: cint +var S_IWGRP* {.importc: "S_IWGRP", header: "<sys/stat.h>".}: cint +var S_IWOTH* {.importc: "S_IWOTH", header: "<sys/stat.h>".}: cint +var S_IWUSR* {.importc: "S_IWUSR", header: "<sys/stat.h>".}: cint +var S_IXGRP* {.importc: "S_IXGRP", header: "<sys/stat.h>".}: cint +var S_IXOTH* {.importc: "S_IXOTH", header: "<sys/stat.h>".}: cint +var S_IXUSR* {.importc: "S_IXUSR", header: "<sys/stat.h>".}: cint + +# # <sys/statvfs.h> +# var ST_RDONLY* {.importc: "ST_RDONLY", header: "<sys/statvfs.h>".}: cint +# var ST_NOSUID* {.importc: "ST_NOSUID", header: "<sys/statvfs.h>".}: cint + +# # <sys/wait.h> +# var WNOHANG* {.importc: "WNOHANG", header: "<sys/wait.h>".}: cint +# var WUNTRACED* {.importc: "WUNTRACED", header: "<sys/wait.h>".}: cint +# var WEXITED* {.importc: "WEXITED", header: "<sys/wait.h>".}: cint +# var WSTOPPED* {.importc: "WSTOPPED", header: "<sys/wait.h>".}: cint +# var WCONTINUED* {.importc: "WCONTINUED", header: "<sys/wait.h>".}: cint +# var WNOWAIT* {.importc: "WNOWAIT", header: "<sys/wait.h>".}: cint +# var P_ALL* {.importc: "P_ALL", header: "<sys/wait.h>".}: cint +# var P_PID* {.importc: "P_PID", header: "<sys/wait.h>".}: cint +# var P_PGID* {.importc: "P_PGID", header: "<sys/wait.h>".}: cint + +# # <spawn.h> +# var POSIX_SPAWN_RESETIDS* {.importc: "POSIX_SPAWN_RESETIDS", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETPGROUP* {.importc: "POSIX_SPAWN_SETPGROUP", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSCHEDPARAM* {.importc: "POSIX_SPAWN_SETSCHEDPARAM", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSCHEDULER* {.importc: "POSIX_SPAWN_SETSCHEDULER", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSIGDEF* {.importc: "POSIX_SPAWN_SETSIGDEF", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSIGMASK* {.importc: "POSIX_SPAWN_SETSIGMASK", header: "<spawn.h>".}: cint + +## <stdio.h> +# var IOFBF* {.importc: "_IOFBF", header: "<stdio.h>".}: cint +# var IONBF* {.importc: "_IONBF", header: "<stdio.h>".}: cint + +# <time.h> +var CLOCKS_PER_SEC* {.importc: "CLOCKS_PER_SEC", header: "<time.h>".}: clong +var CLOCK_PROCESS_CPUTIME_ID* {.importc: "CLOCK_PROCESS_CPUTIME_ID", header: "<time.h>".}: cint +var CLOCK_THREAD_CPUTIME_ID* {.importc: "CLOCK_THREAD_CPUTIME_ID", header: "<time.h>".}: cint +var CLOCK_REALTIME* {.importc: "CLOCK_REALTIME", header: "<time.h>".}: cint +var TIMER_ABSTIME* {.importc: "TIMER_ABSTIME", header: "<time.h>".}: cint +var CLOCK_MONOTONIC* {.importc: "CLOCK_MONOTONIC", header: "<time.h>".}: cint + +# <unistd.h> + +const F_OK* = cint(0) +const R_OK* = cint(4) +const W_OK* = cint(2) +const X_OK* = cint(1) +const F_LOCK* = cint(1) +const F_TEST* = cint(3) +const F_TLOCK* = cint(2) +const F_ULOCK* = cint(0) + +# <stdio.h> +const SEEK_SET* = cint(0) +const SEEK_CUR* = cint(1) +const SEEK_END* = cint(2) diff --git a/lib/posix/posix_haiku.nim b/lib/posix/posix_haiku.nim new file mode 100644 index 000000000..32a6d24e2 --- /dev/null +++ b/lib/posix/posix_haiku.nim @@ -0,0 +1,603 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +const + hasSpawnH = true # should exist for every Posix system nowadays + hasAioH = defined(linux) + +when defined(linux) and not defined(android): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passl: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passl: "-lresolv".} + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + ## A type representing a directory stream. + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct ( + when defined(i386): + int32 + else: + int64 + ) + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: int ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + when defined(haiku): + d_dev*: Dev ## Device (not POSIX) + d_pdev*: Dev ## Parent device (only for queries) (not POSIX) + d_ino*: Ino ## File serial number. + when defined(dragonfly): + # DragonflyBSD doesn't have `d_reclen` field. + d_type*: uint8 + elif defined(linux) or defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or defined(genode): + d_reclen*: cshort ## Length of this record. (not POSIX) + d_type*: int8 ## Type of file; not supported by all filesystem types. + ## (not POSIX) + when defined(linux) or defined(openbsd): + d_off*: Off ## Not an offset. Value that `telldir()` would return. + elif defined(haiku): + d_pino*: Ino ## Parent inode (only for queries) (not POSIX) + d_reclen*: cushort ## Length of this record. (not POSIX) + + when not defined(haiku): + d_name*: array[0..255, char] ## Name of entry. + else: + d_name*: UncheckedArray[char] ## Name of entry (NUL terminated). + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + FTW* {.importc: "struct FTW", header: "<ftw.h>", final, pure.} = object + base*: cint + level*: cint + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: int ## Count of paths matched by pattern. + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_offs*: int ## Slots to reserve at the beginning of gl_pathv. + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>", final, pure.} = + object ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + currency_symbol*: cstring + decimal_point*: cstring + frac_digits*: char + grouping*: cstring + int_curr_symbol*: cstring + int_frac_digits*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_p_sign_posn*: char + mon_decimal_point*: cstring + mon_grouping*: cstring + mon_thousands_sep*: cstring + negative_sign*: cstring + n_cs_precedes*: char + n_sep_by_space*: char + n_sign_posn*: char + positive_sign*: cstring + p_cs_precedes*: char + p_sep_by_space*: char + p_sign_posn*: char + thousands_sep*: cstring + + Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = int64 + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = int32 + ## used for block sizes + Clock* {.importc: "clock_t", header: "<time.h>".} = int32 + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int32 + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = int32 + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int64 + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int64 + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = uint32 + Id* {.importc: "id_t", header: "<sys/types.h>".} = int32 + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int64 + Key* {.importc: "key_t", header: "<sys/types.h>".} = int32 + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = ( + when defined(android) or defined(macos) or defined(macosx) or + (defined(bsd) and not defined(openbsd) and not defined(netbsd)): + uint16 + else: + uint32 + ) + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int32 + Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 + Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = object + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<sys/types.h>".} = object + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<sys/types.h>".} = object + Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>".} = object + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<sys/types.h>".} = object + Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = object + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>".} = object + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<sys/types.h>".} = object + Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = object + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<sys/types.h>".} = object + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<sys/types.h>".} = object + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<sys/types.h>".} = object + Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = object + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = int32 + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = object + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = uint32 + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = uint32 + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*: array[32, char] ## Name of the hardware type on which the + ## system is running. + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + Ipc_perm* {.importc: "struct ipc_perm", + header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm + uid*: Uid ## Owner's user ID. + gid*: Gid ## Owner's group ID. + cuid*: Uid ## Creator's user ID. + cgid*: Gid ## Creator's group ID. + mode*: Mode ## Read/write permission. + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_mode*: Mode ## Mode of file (see below). + st_nlink*: Nlink ## Number of hard links to the file. + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + when StatHasNanoseconds: + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + else: + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: culong ## File system block size. + f_frsize*: culong ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: culong ## File system ID. + f_flag*: culong ## Bit mask of f_flag values. + f_namemax*: culong ## Maximum filename length. + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>".} = uint64 + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_notify*: cint ## Notification type. + sigev_signo*: cint ## Signal number. + sigev_value*: SigVal ## Signal value. + sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. + sigev_notify_attributes*: ptr Pthread_attr ## Notification attributes. + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_ptr*: pointer ## pointer signal value; + ## integer signal value not defined! + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_size*: csize_t ## Stack size. + ss_flags*: cint ## Flags. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_code*: cint ## Signal code. + si_errno*: cint ## If non-zero, an errno value associated with + ## this signal, as defined in <errno.h>. + si_pid*: Pid ## Sending process ID. + si_uid*: Uid ## Real user ID of sending process. + si_addr*: pointer ## Address of faulting instruction. + si_status*: cint ## Exit value or signal. + si_band*: int ## Band event for SIGPOLL. + si_value*: SigVal ## Signal value. + + Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint + Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = cint + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", + final, pure.} = object + Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", + final, pure.} = object ## ucontext_t + uc_link*: ptr Ucontext ## Pointer to the context that is resumed + ## when this context returns. + uc_sigmask*: Sigset ## The set of signals that are blocked when this + ## context is active. + uc_stack*: Stack ## The stack used by this context. + uc_mcontext*: Mcontext ## A machine-specific representation of the saved + ## context. + +when hasAioH: + type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_offset*: Off ## File offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: int ## Length of transfer. + aio_reqprio*: cint ## Request priority offset. + aio_sigevent*: SigEvent ## Signal number and value. + aio_lio_opcode: cint ## Operation to be performed. + +when hasSpawnH: + type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + +when defined(linux): + # from sys/un.h + const Sockaddr_un_path_length* = 108 +elif defined(haiku): + # from sys/un.h + const Sockaddr_un_path_length* = 126 +else: + # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html + # this is >=92 + const Sockaddr_un_path_length* = 92 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = uint32 + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = uint8 + + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..255, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cuint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/uio.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: cint ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: SockLen ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: SockLen ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: int32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: int32 ## Set of interfaces for a scope. + + Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, + header: "<netinet/in.h>".} = object ## struct ipv6_mreq + ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. + ipv6mr_interface*: cuint ## Interface index. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: InAddrT ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_canonname*: cstring ## Canonical name of service location. + ai_next*: ptr AddrInfo ## Pointer to next in list. + + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = culong + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc, header: "<time.h>".}: cint + timezone* {.importc, header: "<time.h>".}: int + +# Regenerate using detect.nim! +include posix_other_consts + +when defined(linux): + var + MAP_POPULATE* {.importc, header: "<sys/mman.h>".}: cint + ## Populate (prefault) page tables for a mapping. +else: + var + MAP_POPULATE*: cint = 0 + +when defined(linux) or defined(nimdoc): + when defined(alpha) or defined(mips) or defined(mipsel) or + defined(mips64) or defined(mips64el) or defined(parisc) or + defined(sparc) or defined(sparc64) or defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) + ## Multiple binding: load balancing on incoming TCP connections + ## or UDP packets. (Requires Linux kernel > 3.9) + else: + const SO_REUSEPORT* = cint(15) +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint + +when defined(macosx): + # We can't use the NOSIGNAL flag in the `send` function, it has no effect + # Instead we should use SO_NOSIGPIPE in setsockopt + const + MSG_NOSIGNAL* = 0'i32 + var + SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint +elif defined(solaris): + # Solaris doesn't have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0'i32 +else: + var + MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint + ## No SIGPIPE generated when an attempt to send is made on a stream-oriented socket that is no longer connected. + +when defined(haiku): + const + SIGKILLTHR* = 21 ## BeOS specific: Kill just the thread, not team + +when hasSpawnH: + when defined(linux): + # better be safe than sorry; Linux has this flag, macosx doesn't, don't + # know about the other OSes + + # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we + # can't get the magic number from spawn.h + const POSIX_SPAWN_USEVFORK* = cint(0x40) + else: + # macosx lacks this, so we define the constant to be 0 to not affect + # OR'ing of flags: + const POSIX_SPAWN_USEVFORK* = cint(0) + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Exit code, if WIFEXITED(s) +proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Termination signal, if WIFSIGNALED(s) +proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Stop signal, if WIFSTOPPED(s) +proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited normally. +proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited due to uncaught signal. +proc WIFSTOPPED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child is currently stopped. +proc WIFCONTINUED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child has been continued. + +when defined(nimHasStyleChecks): + {.pop.} diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim new file mode 100644 index 000000000..8d11c507d --- /dev/null +++ b/lib/posix/posix_linux_amd64.nim @@ -0,0 +1,587 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Types here should conform to the glibc ABI on linux / x86_64 +# When adding a type, the order and size of fields must match + +# To be included from posix.nim! + +const + hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays + hasAioH = defined(linux) + +# On Linux: +# timer_{create,delete,settime,gettime}, +# clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt +{.passl: "-lrt".} + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +# Types + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + ## A type representing a directory stream. + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct clong + + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: clong ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + d_ino*: Ino + d_off*: Off + d_reclen*: cushort + d_type*: int8 # uint8 really! + d_name*: array[256, cchar] + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + # no struct FTW on linux + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: csize_t ## Count of paths matched by pattern. + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_offs*: csize_t ## Slots to reserve at the beginning of gl_pathv. + gl_flags*: cint + gl_closedir*: pointer + gl_readdir*: pointer + gl_opendir*: pointer + gl_lstat*: pointer + gl_stat*: pointer + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_passwd*: cstring + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>".} = pointer + ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + decimal_point*: cstring + thousands_sep*: cstring + grouping*: cstring + int_curr_symbol*: cstring + currency_symbol*: cstring + mon_decimal_point*: cstring + mon_thousands_sep*: cstring + mon_grouping*: cstring + positive_sign*: cstring + negative_sign*: cstring + int_frac_digits*: char + frac_digits*: char + p_cs_precedes*: char + p_sep_by_space*: char + n_cs_precedes*: char + n_sep_by_space*: char + p_sign_posn*: char + n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_p_sign_posn*: char + int_n_sign_posn*: char + + Mqd* {.importc: "mqd_t", header: "<mqueue.h>".} = cint + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: clong ## Message queue flags. + mq_maxmsg*: clong ## Maximum number of messages. + mq_msgsize*: clong ## Maximum message size. + mq_curmsgs*: clong ## Number of messages currently queued. + pad: array[4, clong] + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_passwd*: cstring + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_gecos*: cstring + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = clong + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = clong + ## used for block sizes + Clock* {.importc: "clock_t", header: "<sys/types.h>".} = clong + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = cint + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = culong + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = culong + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = culong + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = cuint + Id* {.importc: "id_t", header: "<sys/types.h>".} = cuint + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = culong + Key* {.importc: "key_t", header: "<sys/types.h>".} = cint + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = uint32 + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = culong + Off* {.importc: "off_t", header: "<sys/types.h>".} = clong + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = cint + Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>", + pure, final.} = object + abi: array[56 div sizeof(clong), clong] + + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[32 div sizeof(clong), clong] + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[4 div sizeof(cint), cint] + + Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>", + pure, final.} = object + abi: array[48 div sizeof(clonglong), clonglong] + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[4 div sizeof(cint), cint] + Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = cuint + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>", + pure, final.} = object + abi: array[40 div sizeof(clong), clong] + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[4 div sizeof(cint), cint] + Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = cint + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[56 div sizeof(clong), clong] + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<sys/types.h>".} = object + abi: array[8 div sizeof(clong), clong] + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<sys/types.h>".} = cint + Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = culong + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = clong + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = pointer + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = cuint + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = cuint + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*, ## Name of the hardware type on which the + ## system is running. + domainname*: array[65, char] + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + abi: array[32 div sizeof(clong), clong] + + Ipc_perm* {.importc: "struct ipc_perm", + header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm + key: Key + uid*: Uid ## Owner's user ID. + gid*: Gid ## Owner's group ID. + cuid*: Uid ## Creator's user ID. + cgid*: Gid ## Creator's group ID. + mode*: cshort ## Read/write permission. + pad1: cshort + seq1: cshort + pad2: cshort + reserved1: culong + reserved2: culong + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_nlink*: Nlink ## Number of hard links to the file. + st_mode*: Mode ## Mode of file (see below). + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + pad0 {.importc: "__pad0".}: cint + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: culong ## File system block size. + f_frsize*: culong ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: culong ## File system ID. + f_flag*: culong ## Bit mask of f_flag values. + f_namemax*: culong ## Maximum filename length. + f_spare: array[6, cint] + + # No Posix_typed_mem_info + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + tm_gmtoff*: clong + tm_zone*: cstring + + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>", final, pure.} = object + abi: array[1024 div (8 * sizeof(culong)), culong] + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_value*: SigVal ## Signal value. + sigev_signo*: cint ## Signal number. + sigev_notify*: cint ## Notification type. + sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. + sigev_notify_attributes*: ptr Pthread_attr ## Notification attributes. + abi: array[12, int] + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_ptr*: pointer ## pointer signal value; + ## integer signal value not defined! + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + sa_restorer: proc() {.noconv.} ## not intended for application use. + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_size*: int ## Stack size. + ss_flags*: cint ## Flags. + + SigStack* {.importc: "struct sigstack", + header: "<signal.h>", final, pure.} = object ## struct sigstack + ss_onstack*: cint ## Non-zero when signal stack is in use. + ss_sp*: pointer ## Signal stack pointer. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_errno*: cint ## If non-zero, an errno value associated with + ## this signal, as defined in <errno.h>. + si_code*: cint ## Signal code. + si_pid*: Pid ## Sending process ID. + si_uid*: Uid ## Real user ID of sending process. + si_addr*: pointer ## Address of faulting instruction. + si_status*: cint ## Exit value or signal. + si_band*: int ## Band event for SIGPOLL. + si_value*: SigVal ## Signal value. + pad {.importc: "_pad".}: array[128 - 56, uint8] + +template sa_sigaction*(v: Sigaction): proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} = + cast[proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.}](v.sa_handler) +proc `sa_sigaction=`*(v: var Sigaction, x: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.}) = + v.sa_handler = cast[proc (x: cint) {.noconv.}](x) + +type + Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint + Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = pointer + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + abi: array[1024 div (8 * sizeof(clong)), clong] + + Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", + final, pure.} = object + gregs: array[23, clonglong] + fpregs: pointer + reserved1: array[8, clonglong] + + Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", + final, pure.} = object ## ucontext_t + uc_flags: clong + uc_link*: ptr Ucontext ## Pointer to the context that is resumed + ## when this context returns. + uc_stack*: Stack ## The stack used by this context. + uc_mcontext*: Mcontext ## A machine-specific representation of the saved + ## context. + uc_sigmask*: Sigset ## The set of signals that are blocked when this + ## context is active. + # todo fpregds_mem + +type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_lio_opcode*: cint ## Operation to be performed. + aio_reqprio*: cint ## Request priority offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: csize_t ## Length of transfer. + aio_sigevent*: SigEvent ## Signal number and value. + next_prio: pointer + abs_prio: cint + policy: cint + error_Code: cint + return_value: clong + aio_offset*: Off ## File offset. + reserved: array[32, uint8] + + +when hasSpawnH: + type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + flags: cshort + pgrp: Pid + sd: Sigset + ss: Sigset + sp: Sched_param + policy: cint + pad: array[16, cint] + + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + allocated: cint + used: cint + actions: pointer + pad: array[16, cint] + +# from sys/un.h +const Sockaddr_un_path_length* = 108 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cushort + + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[14, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[108, char] ## Socket path + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + ss_padding {.importc: "__ss_padding".}: array[128 - sizeof(cshort) - sizeof(culong), char] + ss_align {.importc: "__ss_align".}: clong + + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cuint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/uio.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: csize_t ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: csize_t ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: csize_t ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + # data follows... + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + sin_zero: array[16 - 2 - 2 - 4, uint8] + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: uint32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: uint32 ## Set of interfaces for a scope. + + Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, + header: "<netinet/in.h>".} = object ## struct ipv6_mreq + ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. + ipv6mr_interface*: cuint ## Interface index. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: uint32 ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_canonname*: cstring ## Canonical name of service location. + ai_next*: ptr AddrInfo ## Pointer to next in list. + + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = culong + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc, header: "<time.h>".}: cint + timezone* {.importc, header: "<time.h>".}: clong + +# Regenerate using detect.nim! +include posix_linux_amd64_consts + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint = (s and 0xff00) shr 8 +proc WTERMSIG*(s:cint): cint = s and 0x7f +proc WSTOPSIG*(s:cint): cint = WEXITSTATUS(s) +proc WIFEXITED*(s:cint) : bool = WTERMSIG(s) == 0 +proc WIFSIGNALED*(s:cint) : bool = (cast[int8]((s and 0x7f) + 1) shr 1) > 0 +proc WIFSTOPPED*(s:cint) : bool = (s and 0xff) == 0x7f +proc WIFCONTINUED*(s:cint) : bool = s == WCONTINUED + +when defined(nimHasStyleChecks): + {.pop.} # {.push styleChecks: off.} diff --git a/lib/posix/posix_linux_amd64_consts.nim b/lib/posix/posix_linux_amd64_consts.nim new file mode 100644 index 000000000..fbe8d0666 --- /dev/null +++ b/lib/posix/posix_linux_amd64_consts.nim @@ -0,0 +1,730 @@ +# Generated by detect.nim + + +# <aio.h> +const AIO_ALLDONE* = cint(2) +const AIO_CANCELED* = cint(0) +const AIO_NOTCANCELED* = cint(1) +const LIO_NOP* = cint(2) +const LIO_NOWAIT* = cint(1) +const LIO_READ* = cint(0) +const LIO_WAIT* = cint(0) +const LIO_WRITE* = cint(1) + +# <dlfcn.h> +const RTLD_LAZY* = cint(1) +const RTLD_NOW* = cint(2) +const RTLD_GLOBAL* = cint(256) +const RTLD_LOCAL* = cint(0) + +# <errno.h> +const E2BIG* = cint(7) +const EACCES* = cint(13) +const EADDRINUSE* = cint(98) +const EADDRNOTAVAIL* = cint(99) +const EAFNOSUPPORT* = cint(97) +const EAGAIN* = cint(11) +const EALREADY* = cint(114) +const EBADF* = cint(9) +const EBADMSG* = cint(74) +const EBUSY* = cint(16) +const ECANCELED* = cint(125) +const ECHILD* = cint(10) +const ECONNABORTED* = cint(103) +const ECONNREFUSED* = cint(111) +const ECONNRESET* = cint(104) +const EDEADLK* = cint(35) +const EDESTADDRREQ* = cint(89) +const EDOM* = cint(33) +const EDQUOT* = cint(122) +const EEXIST* = cint(17) +const EFAULT* = cint(14) +const EFBIG* = cint(27) +const EHOSTUNREACH* = cint(113) +const EIDRM* = cint(43) +const EILSEQ* = cint(84) +const EINPROGRESS* = cint(115) +const EINTR* = cint(4) +const EINVAL* = cint(22) +const EIO* = cint(5) +const EISCONN* = cint(106) +const EISDIR* = cint(21) +const ELOOP* = cint(40) +const EMFILE* = cint(24) +const EMLINK* = cint(31) +const EMSGSIZE* = cint(90) +const EMULTIHOP* = cint(72) +const ENAMETOOLONG* = cint(36) +const ENETDOWN* = cint(100) +const ENETRESET* = cint(102) +const ENETUNREACH* = cint(101) +const ENFILE* = cint(23) +const ENOBUFS* = cint(105) +const ENODATA* = cint(61) +const ENODEV* = cint(19) +const ENOENT* = cint(2) +const ENOEXEC* = cint(8) +const ENOLCK* = cint(37) +const ENOLINK* = cint(67) +const ENOMEM* = cint(12) +const ENOMSG* = cint(42) +const ENOPROTOOPT* = cint(92) +const ENOSPC* = cint(28) +const ENOSR* = cint(63) +const ENOSTR* = cint(60) +const ENOSYS* = cint(38) +const ENOTCONN* = cint(107) +const ENOTDIR* = cint(20) +const ENOTEMPTY* = cint(39) +const ENOTSOCK* = cint(88) +const ENOTSUP* = cint(95) +const ENOTTY* = cint(25) +const ENXIO* = cint(6) +const EOPNOTSUPP* = cint(95) +const EOVERFLOW* = cint(75) +const EPERM* = cint(1) +const EPIPE* = cint(32) +const EPROTO* = cint(71) +const EPROTONOSUPPORT* = cint(93) +const EPROTOTYPE* = cint(91) +const ERANGE* = cint(34) +const EROFS* = cint(30) +const ESPIPE* = cint(29) +const ESRCH* = cint(3) +const ESTALE* = cint(116) +const ETIME* = cint(62) +const ETIMEDOUT* = cint(110) +const ETXTBSY* = cint(26) +const EWOULDBLOCK* = cint(11) +const EXDEV* = cint(18) + +# <fcntl.h> +const F_DUPFD* = cint(0) +const F_DUPFD_CLOEXEC* = cint(1030) +const F_GETFD* = cint(1) +const F_SETFD* = cint(2) +const F_GETFL* = cint(3) +const F_SETFL* = cint(4) +const F_GETLK* = cint(5) +const F_SETLK* = cint(6) +const F_SETLKW* = cint(7) +const F_GETOWN* = cint(9) +const F_SETOWN* = cint(8) +const FD_CLOEXEC* = cint(1) +const F_RDLCK* = cint(0) +const F_UNLCK* = cint(2) +const F_WRLCK* = cint(1) +const O_CREAT* = cint(64) +const O_EXCL* = cint(128) +const O_NOCTTY* = cint(256) +const O_TRUNC* = cint(512) +const O_APPEND* = cint(1024) +const O_DSYNC* = cint(4096) +const O_NONBLOCK* = cint(2048) +const O_RSYNC* = cint(1052672) +const O_SYNC* = cint(1052672) +const O_ACCMODE* = cint(3) +const O_RDONLY* = cint(0) +const O_RDWR* = cint(2) +const O_WRONLY* = cint(1) +const O_CLOEXEC* = cint(524288) +const O_DIRECT* = cint(16384) +const O_PATH* = cint(2097152) +const O_NOATIME* = cint(262144) +const O_TMPFILE* = cint(4259840) +const POSIX_FADV_NORMAL* = cint(0) +const POSIX_FADV_SEQUENTIAL* = cint(2) +const POSIX_FADV_RANDOM* = cint(1) +const POSIX_FADV_WILLNEED* = cint(3) +const POSIX_FADV_DONTNEED* = cint(4) +const POSIX_FADV_NOREUSE* = cint(5) + +# <fenv.h> +const FE_DIVBYZERO* = cint(4) +const FE_INEXACT* = cint(32) +const FE_INVALID* = cint(1) +const FE_OVERFLOW* = cint(8) +const FE_UNDERFLOW* = cint(16) +const FE_ALL_EXCEPT* = cint(61) +const FE_DOWNWARD* = cint(1024) +const FE_TONEAREST* = cint(0) +const FE_TOWARDZERO* = cint(3072) +const FE_UPWARD* = cint(2048) +const FE_DFL_ENV* = cint(-1) + +# <fmtmsg.h> +const MM_HARD* = cint(1) +const MM_SOFT* = cint(2) +const MM_FIRM* = cint(4) +const MM_APPL* = cint(8) +const MM_UTIL* = cint(16) +const MM_OPSYS* = cint(32) +const MM_RECOVER* = cint(64) +const MM_NRECOV* = cint(128) +const MM_HALT* = cint(1) +const MM_ERROR* = cint(2) +const MM_WARNING* = cint(3) +const MM_INFO* = cint(4) +const MM_NOSEV* = cint(0) +const MM_PRINT* = cint(256) +const MM_CONSOLE* = cint(512) +const MM_OK* = cint(0) +const MM_NOTOK* = cint(-1) +const MM_NOMSG* = cint(1) +const MM_NOCON* = cint(4) + +# <fnmatch.h> +const FNM_NOMATCH* = cint(1) +const FNM_PATHNAME* = cint(1) +const FNM_PERIOD* = cint(4) +const FNM_NOESCAPE* = cint(2) +const FNM_NOSYS* = cint(-1) + +# <ftw.h> +const FTW_F* = cint(0) +const FTW_D* = cint(1) +const FTW_DNR* = cint(2) +const FTW_DP* = cint(5) +const FTW_NS* = cint(3) +const FTW_SL* = cint(4) +const FTW_SLN* = cint(6) +const FTW_PHYS* = cint(1) +const FTW_MOUNT* = cint(2) +const FTW_DEPTH* = cint(8) +const FTW_CHDIR* = cint(4) + +# <glob.h> +const GLOB_APPEND* = cint(32) +const GLOB_DOOFFS* = cint(8) +const GLOB_ERR* = cint(1) +const GLOB_MARK* = cint(2) +const GLOB_NOCHECK* = cint(16) +const GLOB_NOESCAPE* = cint(64) +const GLOB_NOSORT* = cint(4) +const GLOB_ABORTED* = cint(2) +const GLOB_NOMATCH* = cint(3) +const GLOB_NOSPACE* = cint(1) +const GLOB_NOSYS* = cint(4) + +# <langinfo.h> +const CODESET* = cint(14) +const D_T_FMT* = cint(131112) +const D_FMT* = cint(131113) +const T_FMT* = cint(131114) +const T_FMT_AMPM* = cint(131115) +const AM_STR* = cint(131110) +const PM_STR* = cint(131111) +const DAY_1* = cint(131079) +const DAY_2* = cint(131080) +const DAY_3* = cint(131081) +const DAY_4* = cint(131082) +const DAY_5* = cint(131083) +const DAY_6* = cint(131084) +const DAY_7* = cint(131085) +const ABDAY_1* = cint(131072) +const ABDAY_2* = cint(131073) +const ABDAY_3* = cint(131074) +const ABDAY_4* = cint(131075) +const ABDAY_5* = cint(131076) +const ABDAY_6* = cint(131077) +const ABDAY_7* = cint(131078) +const MON_1* = cint(131098) +const MON_2* = cint(131099) +const MON_3* = cint(131100) +const MON_4* = cint(131101) +const MON_5* = cint(131102) +const MON_6* = cint(131103) +const MON_7* = cint(131104) +const MON_8* = cint(131105) +const MON_9* = cint(131106) +const MON_10* = cint(131107) +const MON_11* = cint(131108) +const MON_12* = cint(131109) +const ABMON_1* = cint(131086) +const ABMON_2* = cint(131087) +const ABMON_3* = cint(131088) +const ABMON_4* = cint(131089) +const ABMON_5* = cint(131090) +const ABMON_6* = cint(131091) +const ABMON_7* = cint(131092) +const ABMON_8* = cint(131093) +const ABMON_9* = cint(131094) +const ABMON_10* = cint(131095) +const ABMON_11* = cint(131096) +const ABMON_12* = cint(131097) +const ERA* = cint(131116) +const ERA_D_FMT* = cint(131118) +const ERA_D_T_FMT* = cint(131120) +const ERA_T_FMT* = cint(131121) +const ALT_DIGITS* = cint(131119) +const RADIXCHAR* = cint(65536) +const THOUSEP* = cint(65537) +const YESEXPR* = cint(327680) +const NOEXPR* = cint(327681) +const CRNCYSTR* = cint(262159) + +# <locale.h> +const LC_ALL* = cint(6) +const LC_COLLATE* = cint(3) +const LC_CTYPE* = cint(0) +const LC_MESSAGES* = cint(5) +const LC_MONETARY* = cint(4) +const LC_NUMERIC* = cint(1) +const LC_TIME* = cint(2) + +# <netdb.h> +const IPPORT_RESERVED* = cint(1024) +const HOST_NOT_FOUND* = cint(1) +const NO_DATA* = cint(4) +const NO_RECOVERY* = cint(3) +const TRY_AGAIN* = cint(2) +const AI_PASSIVE* = cint(1) +const AI_CANONNAME* = cint(2) +const AI_NUMERICHOST* = cint(4) +const AI_NUMERICSERV* = cint(1024) +const AI_V4MAPPED* = cint(8) +const AI_ALL* = cint(16) +const AI_ADDRCONFIG* = cint(32) +const NI_NOFQDN* = cint(4) +const NI_NUMERICHOST* = cint(1) +const NI_NAMEREQD* = cint(8) +const NI_NUMERICSERV* = cint(2) +const NI_DGRAM* = cint(16) +const EAI_AGAIN* = cint(-3) +const EAI_BADFLAGS* = cint(-1) +const EAI_FAIL* = cint(-4) +const EAI_FAMILY* = cint(-6) +const EAI_MEMORY* = cint(-10) +const EAI_NONAME* = cint(-2) +const EAI_SERVICE* = cint(-8) +const EAI_SOCKTYPE* = cint(-7) +const EAI_SYSTEM* = cint(-11) +const EAI_OVERFLOW* = cint(-12) + +# <net/if.h> +const IF_NAMESIZE* = cint(16) + +# <netinet/in.h> +const IPPROTO_IP* = cint(0) +const IPPROTO_IPV6* = cint(41) +const IPPROTO_ICMP* = cint(1) +const IPPROTO_ICMPV6* = cint(58) +const IPPROTO_RAW* = cint(255) +const IPPROTO_TCP* = cint(6) +const IPPROTO_UDP* = cint(17) +const INADDR_ANY* = InAddrScalar(0) +const INADDR_LOOPBACK* = InAddrScalar(2130706433) +const INADDR_BROADCAST* = InAddrScalar(4294967295) +const INET_ADDRSTRLEN* = cint(16) +const INET6_ADDRSTRLEN* = cint(46) +const IPV6_JOIN_GROUP* = cint(20) +const IPV6_LEAVE_GROUP* = cint(21) +const IPV6_MULTICAST_HOPS* = cint(18) +const IPV6_MULTICAST_IF* = cint(17) +const IPV6_MULTICAST_LOOP* = cint(19) +const IPV6_UNICAST_HOPS* = cint(16) +const IPV6_V6ONLY* = cint(26) + +# <netinet/tcp.h> +const TCP_NODELAY* = cint(1) + +# <nl_types.h> +const NL_SETD* = cint(1) +const NL_CAT_LOCALE* = cint(1) + +# <poll.h> +const POLLIN* = cshort(1) +const POLLRDNORM* = cshort(64) +const POLLRDBAND* = cshort(128) +const POLLPRI* = cshort(2) +const POLLOUT* = cshort(4) +const POLLWRNORM* = cshort(256) +const POLLWRBAND* = cshort(512) +const POLLERR* = cshort(8) +const POLLHUP* = cshort(16) +const POLLNVAL* = cshort(32) + +# <pthread.h> +const PTHREAD_BARRIER_SERIAL_THREAD* = cint(-1) +const PTHREAD_CANCEL_ASYNCHRONOUS* = cint(1) +const PTHREAD_CANCEL_ENABLE* = cint(0) +const PTHREAD_CANCEL_DEFERRED* = cint(0) +const PTHREAD_CANCEL_DISABLE* = cint(1) +const PTHREAD_CREATE_DETACHED* = cint(1) +const PTHREAD_CREATE_JOINABLE* = cint(0) +const PTHREAD_EXPLICIT_SCHED* = cint(1) +const PTHREAD_INHERIT_SCHED* = cint(0) +const PTHREAD_PROCESS_SHARED* = cint(1) +const PTHREAD_PROCESS_PRIVATE* = cint(0) +const PTHREAD_SCOPE_PROCESS* = cint(1) +const PTHREAD_SCOPE_SYSTEM* = cint(0) + +# <sched.h> +const SCHED_FIFO* = cint(1) +const SCHED_RR* = cint(2) +const SCHED_OTHER* = cint(0) + +# <semaphore.h> +const SEM_FAILED* = cast[pointer]((nil)) + +# <signal.h> +const SIGEV_NONE* = cint(1) +const SIGEV_SIGNAL* = cint(0) +const SIGEV_THREAD* = cint(2) +const SIGABRT* = cint(6) +const SIGALRM* = cint(14) +const SIGBUS* = cint(7) +const SIGCHLD* = cint(17) +const SIGCONT* = cint(18) +const SIGFPE* = cint(8) +const SIGHUP* = cint(1) +const SIGILL* = cint(4) +const SIGINT* = cint(2) +const SIGKILL* = cint(9) +const SIGPIPE* = cint(13) +const SIGQUIT* = cint(3) +const SIGSEGV* = cint(11) +const SIGSTOP* = cint(19) +const SIGTERM* = cint(15) +const SIGTSTP* = cint(20) +const SIGTTIN* = cint(21) +const SIGTTOU* = cint(22) +const SIGUSR1* = cint(10) +const SIGUSR2* = cint(12) +const SIGPOLL* = cint(29) +const SIGPROF* = cint(27) +const SIGSYS* = cint(31) +const SIGTRAP* = cint(5) +const SIGURG* = cint(23) +const SIGVTALRM* = cint(26) +const SIGXCPU* = cint(24) +const SIGXFSZ* = cint(25) +const SA_NOCLDSTOP* = cint(1) +const SIG_BLOCK* = cint(0) +const SIG_UNBLOCK* = cint(1) +const SIG_SETMASK* = cint(2) +const SA_ONSTACK* = cint(134217728) +const SA_RESETHAND* = cint(-2147483648) +const SA_RESTART* = cint(268435456) +const SA_SIGINFO* = cint(4) +const SA_NOCLDWAIT* = cint(2) +const SA_NODEFER* = cint(1073741824) +const SS_ONSTACK* = cint(1) +const SS_DISABLE* = cint(2) +const MINSIGSTKSZ* = cint(2048) +const SIGSTKSZ* = cint(8192) +const SIG_HOLD* = cast[Sighandler](2) +const SIG_DFL* = cast[Sighandler](0) +const SIG_ERR* = cast[Sighandler](-1) +const SIG_IGN* = cast[Sighandler](1) + +# <sys/ipc.h> +const IPC_CREAT* = cint(512) +const IPC_EXCL* = cint(1024) +const IPC_NOWAIT* = cint(2048) +const IPC_PRIVATE* = cint(0) +const IPC_RMID* = cint(0) +const IPC_SET* = cint(1) +const IPC_STAT* = cint(2) + +# <sys/mman.h> +const PROT_READ* = cint(1) +const PROT_WRITE* = cint(2) +const PROT_EXEC* = cint(4) +const PROT_NONE* = cint(0) +const MAP_ANONYMOUS* = cint(32) +const MAP_FIXED_NOREPLACE* = cint(1048576) +const MAP_NORESERVE* = cint(16384) +const MAP_SHARED* = cint(1) +const MAP_PRIVATE* = cint(2) +const MAP_FIXED* = cint(16) +const MS_ASYNC* = cint(1) +const MS_SYNC* = cint(4) +const MS_INVALIDATE* = cint(2) +const MCL_CURRENT* = cint(1) +const MCL_FUTURE* = cint(2) +const MAP_FAILED* = cast[pointer](0xffffffffffffffff) +const POSIX_MADV_NORMAL* = cint(0) +const POSIX_MADV_SEQUENTIAL* = cint(2) +const POSIX_MADV_RANDOM* = cint(1) +const POSIX_MADV_WILLNEED* = cint(3) +const POSIX_MADV_DONTNEED* = cint(4) +const MAP_POPULATE* = cint(32768) + +# <sys/resource.h> +const RLIMIT_NOFILE* = cint(7) +const RLIMIT_STACK* = cint(3) + +# <sys/select.h> +const FD_SETSIZE* = cint(1024) + +# <sys/socket.h> +const MSG_CTRUNC* = cint(8) +const MSG_DONTROUTE* = cint(4) +const MSG_EOR* = cint(128) +const MSG_OOB* = cint(1) +const SCM_RIGHTS* = cint(1) +const SO_ACCEPTCONN* = cint(30) +const SO_BINDTODEVICE* = cint(25) +const SO_BROADCAST* = cint(6) +const SO_DEBUG* = cint(1) +const SO_DONTROUTE* = cint(5) +const SO_ERROR* = cint(4) +const SO_KEEPALIVE* = cint(9) +const SO_LINGER* = cint(13) +const SO_OOBINLINE* = cint(10) +const SO_RCVBUF* = cint(8) +const SO_RCVLOWAT* = cint(18) +const SO_RCVTIMEO* = cint(20) +const SO_REUSEADDR* = cint(2) +const SO_SNDBUF* = cint(7) +const SO_SNDLOWAT* = cint(19) +const SO_SNDTIMEO* = cint(21) +const SO_TYPE* = cint(3) +const SOCK_DGRAM* = cint(2) +const SOCK_RAW* = cint(3) +const SOCK_SEQPACKET* = cint(5) +const SOCK_STREAM* = cint(1) +const SOCK_CLOEXEC* = cint(524288) +const SOL_SOCKET* = cint(1) +const SOMAXCONN* = cint(4096) +const SO_REUSEPORT* = cint(15) +const MSG_NOSIGNAL* = cint(16384) +const MSG_PEEK* = cint(2) +const MSG_TRUNC* = cint(32) +const MSG_WAITALL* = cint(256) +const AF_INET* = cint(2) +const AF_INET6* = cint(10) +const AF_UNIX* = cint(1) +const AF_UNSPEC* = cint(0) +const SHUT_RD* = cint(0) +const SHUT_RDWR* = cint(2) +const SHUT_WR* = cint(1) + +# <sys/stat.h> +const S_IFBLK* = cint(24576) +const S_IFCHR* = cint(8192) +const S_IFDIR* = cint(16384) +const S_IFIFO* = cint(4096) +const S_IFLNK* = cint(40960) +const S_IFMT* = cint(61440) +const S_IFREG* = cint(32768) +const S_IFSOCK* = cint(49152) +const S_IRGRP* = cint(32) +const S_IROTH* = cint(4) +const S_IRUSR* = cint(256) +const S_IRWXG* = cint(56) +const S_IRWXO* = cint(7) +const S_IRWXU* = cint(448) +const S_ISGID* = cint(1024) +const S_ISUID* = cint(2048) +const S_ISVTX* = cint(512) +const S_IWGRP* = cint(16) +const S_IWOTH* = cint(2) +const S_IWUSR* = cint(128) +const S_IXGRP* = cint(8) +const S_IXOTH* = cint(1) +const S_IXUSR* = cint(64) + +# <sys/statvfs.h> +const ST_RDONLY* = cint(1) +const ST_NOSUID* = cint(2) + +# <sys/wait.h> +const WNOHANG* = cint(1) +const WUNTRACED* = cint(2) +const WEXITED* = cint(4) +const WSTOPPED* = cint(2) +const WCONTINUED* = cint(8) +const WNOWAIT* = cint(16777216) + +# <spawn.h> +const POSIX_SPAWN_RESETIDS* = cint(1) +const POSIX_SPAWN_SETPGROUP* = cint(2) +const POSIX_SPAWN_SETSCHEDPARAM* = cint(16) +const POSIX_SPAWN_SETSCHEDULER* = cint(32) +const POSIX_SPAWN_SETSIGDEF* = cint(4) +const POSIX_SPAWN_SETSIGMASK* = cint(8) +const POSIX_SPAWN_USEVFORK* = cint(64) + +# <stdio.h> +const IOFBF* = cint(0) +const IONBF* = cint(2) + +# <time.h> +const CLOCKS_PER_SEC* = clong(1000000) +const CLOCK_PROCESS_CPUTIME_ID* = cint(2) +const CLOCK_THREAD_CPUTIME_ID* = cint(3) +const CLOCK_REALTIME* = cint(0) +const TIMER_ABSTIME* = cint(1) +const CLOCK_MONOTONIC* = cint(1) + +# <unistd.h> +const POSIX_ASYNC_IO* = cint(1) +const F_OK* = cint(0) +const R_OK* = cint(4) +const W_OK* = cint(2) +const X_OK* = cint(1) +const CS_PATH* = cint(0) +const CS_POSIX_V6_ILP32_OFF32_CFLAGS* = cint(1116) +const CS_POSIX_V6_ILP32_OFF32_LDFLAGS* = cint(1117) +const CS_POSIX_V6_ILP32_OFF32_LIBS* = cint(1118) +const CS_POSIX_V6_ILP32_OFFBIG_CFLAGS* = cint(1120) +const CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS* = cint(1121) +const CS_POSIX_V6_ILP32_OFFBIG_LIBS* = cint(1122) +const CS_POSIX_V6_LP64_OFF64_CFLAGS* = cint(1124) +const CS_POSIX_V6_LP64_OFF64_LDFLAGS* = cint(1125) +const CS_POSIX_V6_LP64_OFF64_LIBS* = cint(1126) +const CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS* = cint(1128) +const CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS* = cint(1129) +const CS_POSIX_V6_LPBIG_OFFBIG_LIBS* = cint(1130) +const CS_POSIX_V6_WIDTH_RESTRICTED_ENVS* = cint(1) +const F_LOCK* = cint(1) +const F_TEST* = cint(3) +const F_TLOCK* = cint(2) +const F_ULOCK* = cint(0) +const PC_2_SYMLINKS* = cint(20) +const PC_ALLOC_SIZE_MIN* = cint(18) +const PC_ASYNC_IO* = cint(10) +const PC_CHOWN_RESTRICTED* = cint(6) +const PC_FILESIZEBITS* = cint(13) +const PC_LINK_MAX* = cint(0) +const PC_MAX_CANON* = cint(1) +const PC_MAX_INPUT* = cint(2) +const PC_NAME_MAX* = cint(3) +const PC_NO_TRUNC* = cint(7) +const PC_PATH_MAX* = cint(4) +const PC_PIPE_BUF* = cint(5) +const PC_PRIO_IO* = cint(11) +const PC_REC_INCR_XFER_SIZE* = cint(14) +const PC_REC_MIN_XFER_SIZE* = cint(16) +const PC_REC_XFER_ALIGN* = cint(17) +const PC_SYMLINK_MAX* = cint(19) +const PC_SYNC_IO* = cint(9) +const PC_VDISABLE* = cint(8) +const SC_2_C_BIND* = cint(47) +const SC_2_C_DEV* = cint(48) +const SC_2_CHAR_TERM* = cint(95) +const SC_2_FORT_DEV* = cint(49) +const SC_2_FORT_RUN* = cint(50) +const SC_2_LOCALEDEF* = cint(52) +const SC_2_PBS* = cint(168) +const SC_2_PBS_ACCOUNTING* = cint(169) +const SC_2_PBS_CHECKPOINT* = cint(175) +const SC_2_PBS_LOCATE* = cint(170) +const SC_2_PBS_MESSAGE* = cint(171) +const SC_2_PBS_TRACK* = cint(172) +const SC_2_SW_DEV* = cint(51) +const SC_2_UPE* = cint(97) +const SC_2_VERSION* = cint(46) +const SC_ADVISORY_INFO* = cint(132) +const SC_AIO_LISTIO_MAX* = cint(23) +const SC_AIO_MAX* = cint(24) +const SC_AIO_PRIO_DELTA_MAX* = cint(25) +const SC_ARG_MAX* = cint(0) +const SC_ASYNCHRONOUS_IO* = cint(12) +const SC_ATEXIT_MAX* = cint(87) +const SC_BARRIERS* = cint(133) +const SC_BC_BASE_MAX* = cint(36) +const SC_BC_DIM_MAX* = cint(37) +const SC_BC_SCALE_MAX* = cint(38) +const SC_BC_STRING_MAX* = cint(39) +const SC_CHILD_MAX* = cint(1) +const SC_CLK_TCK* = cint(2) +const SC_CLOCK_SELECTION* = cint(137) +const SC_COLL_WEIGHTS_MAX* = cint(40) +const SC_CPUTIME* = cint(138) +const SC_DELAYTIMER_MAX* = cint(26) +const SC_EXPR_NEST_MAX* = cint(42) +const SC_FSYNC* = cint(15) +const SC_GETGR_R_SIZE_MAX* = cint(69) +const SC_GETPW_R_SIZE_MAX* = cint(70) +const SC_HOST_NAME_MAX* = cint(180) +const SC_IOV_MAX* = cint(60) +const SC_IPV6* = cint(235) +const SC_JOB_CONTROL* = cint(7) +const SC_LINE_MAX* = cint(43) +const SC_LOGIN_NAME_MAX* = cint(71) +const SC_MAPPED_FILES* = cint(16) +const SC_MEMLOCK* = cint(17) +const SC_MEMLOCK_RANGE* = cint(18) +const SC_MEMORY_PROTECTION* = cint(19) +const SC_MESSAGE_PASSING* = cint(20) +const SC_MONOTONIC_CLOCK* = cint(149) +const SC_MQ_OPEN_MAX* = cint(27) +const SC_MQ_PRIO_MAX* = cint(28) +const SC_NGROUPS_MAX* = cint(3) +const SC_OPEN_MAX* = cint(4) +const SC_PAGESIZE* = cint(30) +const SC_PRIORITIZED_IO* = cint(13) +const SC_PRIORITY_SCHEDULING* = cint(10) +const SC_RAW_SOCKETS* = cint(236) +const SC_RE_DUP_MAX* = cint(44) +const SC_READER_WRITER_LOCKS* = cint(153) +const SC_REALTIME_SIGNALS* = cint(9) +const SC_REGEXP* = cint(155) +const SC_RTSIG_MAX* = cint(31) +const SC_SAVED_IDS* = cint(8) +const SC_SEM_NSEMS_MAX* = cint(32) +const SC_SEM_VALUE_MAX* = cint(33) +const SC_SEMAPHORES* = cint(21) +const SC_SHARED_MEMORY_OBJECTS* = cint(22) +const SC_SHELL* = cint(157) +const SC_SIGQUEUE_MAX* = cint(34) +const SC_SPAWN* = cint(159) +const SC_SPIN_LOCKS* = cint(154) +const SC_SPORADIC_SERVER* = cint(160) +const SC_SS_REPL_MAX* = cint(241) +const SC_STREAM_MAX* = cint(5) +const SC_SYMLOOP_MAX* = cint(173) +const SC_SYNCHRONIZED_IO* = cint(14) +const SC_THREAD_ATTR_STACKADDR* = cint(77) +const SC_THREAD_ATTR_STACKSIZE* = cint(78) +const SC_THREAD_CPUTIME* = cint(139) +const SC_THREAD_DESTRUCTOR_ITERATIONS* = cint(73) +const SC_THREAD_KEYS_MAX* = cint(74) +const SC_THREAD_PRIO_INHERIT* = cint(80) +const SC_THREAD_PRIO_PROTECT* = cint(81) +const SC_THREAD_PRIORITY_SCHEDULING* = cint(79) +const SC_THREAD_PROCESS_SHARED* = cint(82) +const SC_THREAD_SAFE_FUNCTIONS* = cint(68) +const SC_THREAD_SPORADIC_SERVER* = cint(161) +const SC_THREAD_STACK_MIN* = cint(75) +const SC_THREAD_THREADS_MAX* = cint(76) +const SC_THREADS* = cint(67) +const SC_TIMEOUTS* = cint(164) +const SC_TIMER_MAX* = cint(35) +const SC_TIMERS* = cint(11) +const SC_TRACE* = cint(181) +const SC_TRACE_EVENT_FILTER* = cint(182) +const SC_TRACE_EVENT_NAME_MAX* = cint(242) +const SC_TRACE_INHERIT* = cint(183) +const SC_TRACE_LOG* = cint(184) +const SC_TRACE_NAME_MAX* = cint(243) +const SC_TRACE_SYS_MAX* = cint(244) +const SC_TRACE_USER_EVENT_MAX* = cint(245) +const SC_TTY_NAME_MAX* = cint(72) +const SC_TYPED_MEMORY_OBJECTS* = cint(165) +const SC_TZNAME_MAX* = cint(6) +const SC_V6_ILP32_OFF32* = cint(176) +const SC_V6_ILP32_OFFBIG* = cint(177) +const SC_V6_LP64_OFF64* = cint(178) +const SC_V6_LPBIG_OFFBIG* = cint(179) +const SC_VERSION* = cint(29) +const SC_XBS5_ILP32_OFF32* = cint(125) +const SC_XBS5_ILP32_OFFBIG* = cint(126) +const SC_XBS5_LP64_OFF64* = cint(127) +const SC_XBS5_LPBIG_OFFBIG* = cint(128) +const SC_XOPEN_CRYPT* = cint(92) +const SC_XOPEN_ENH_I18N* = cint(93) +const SC_XOPEN_LEGACY* = cint(129) +const SC_XOPEN_REALTIME* = cint(130) +const SC_XOPEN_REALTIME_THREADS* = cint(131) +const SC_XOPEN_SHM* = cint(94) +const SC_XOPEN_STREAMS* = cint(246) +const SC_XOPEN_UNIX* = cint(91) +const SC_XOPEN_VERSION* = cint(89) +const SC_NPROCESSORS_ONLN* = cint(84) +const SEEK_SET* = cint(0) +const SEEK_CUR* = cint(1) +const SEEK_END* = cint(2) diff --git a/lib/posix/posix_macos_amd64.nim b/lib/posix/posix_macos_amd64.nim new file mode 100644 index 000000000..a4b64ed62 --- /dev/null +++ b/lib/posix/posix_macos_amd64.nim @@ -0,0 +1,620 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +const + hasSpawnH = true # should exist for every Posix system nowadays + hasAioH = false + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + ## A type representing a directory stream. + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct clong + + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: int ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + when defined(haiku): + d_dev*: Dev ## Device (not POSIX) + d_pdev*: Dev ## Parent device (only for queries) (not POSIX) + d_ino*: Ino ## File serial number. + when defined(dragonfly): + # DragonflyBSD doesn't have `d_reclen` field. + d_type*: uint8 + elif defined(linux) or defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or defined(genode): + d_reclen*: cshort ## Length of this record. (not POSIX) + d_type*: int8 ## Type of file; not supported by all filesystem types. + ## (not POSIX) + when defined(linux) or defined(openbsd): + d_off*: Off ## Not an offset. Value that `telldir()` would return. + elif defined(haiku): + d_pino*: Ino ## Parent inode (only for queries) (not POSIX) + d_reclen*: cushort ## Length of this record. (not POSIX) + + d_name*: array[0..255, char] ## Name of entry. + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + FTW* {.importc: "struct FTW", header: "<ftw.h>", final, pure.} = object + base*: cint + level*: cint + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: int ## Count of paths matched by pattern. + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_offs*: int ## Slots to reserve at the beginning of gl_pathv. + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>", final, pure.} = + object ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + currency_symbol*: cstring + decimal_point*: cstring + frac_digits*: char + grouping*: cstring + int_curr_symbol*: cstring + int_frac_digits*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_p_sign_posn*: char + mon_decimal_point*: cstring + mon_grouping*: cstring + mon_thousands_sep*: cstring + negative_sign*: cstring + n_cs_precedes*: char + n_sep_by_space*: char + n_sign_posn*: char + positive_sign*: cstring + p_cs_precedes*: char + p_sep_by_space*: char + p_sign_posn*: char + thousands_sep*: cstring + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = int + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = int32 + ## used for block sizes + Clock* {.importc: "clock_t", header: "<sys/types.h>".} = int + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = ( + when defined(freebsd): + uint32 + else: + int32) + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = uint32 + Id* {.importc: "id_t", header: "<sys/types.h>".} = int + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int + Key* {.importc: "key_t", header: "<sys/types.h>".} = int + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = ( + when defined(openbsd) or defined(netbsd): + uint32 + else: + uint16 + ) + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = uint16 + Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 + Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = int + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<sys/types.h>".} = int + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<sys/types.h>".} = int + Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>".} = int + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<sys/types.h>".} = int + Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = int + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>".} = int + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<sys/types.h>".} = int + Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = int + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<sys/types.h>".} = int + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<sys/types.h>".} = int + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<sys/types.h>".} = int + Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = int32 + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = int + Trace_attr* {.importc: "trace_attr_t", header: "<sys/types.h>".} = int + Trace_event_id* {.importc: "trace_event_id_t", + header: "<sys/types.h>".} = int + Trace_event_set* {.importc: "trace_event_set_t", + header: "<sys/types.h>".} = int + Trace_id* {.importc: "trace_id_t", header: "<sys/types.h>".} = int + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = uint32 + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = int + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*: array[0..255, char] ## Name of the hardware type on which the + ## system is running. + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + Ipc_perm* {.importc: "struct ipc_perm", + header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm + uid*: Uid ## Owner's user ID. + gid*: Gid ## Owner's group ID. + cuid*: Uid ## Creator's user ID. + cgid*: Gid ## Creator's group ID. + mode*: Mode ## Read/write permission. + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_mode*: Mode ## Mode of file (see below). + st_nlink*: Nlink ## Number of hard links to the file. + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + when defined(osx): + st_atim* {.importc:"st_atimespec".}: Timespec ## Time of last access. + st_mtim* {.importc:"st_mtimespec".}: Timespec ## Time of last data modification. + st_ctim* {.importc:"st_ctimespec".}: Timespec ## Time of last status change. + elif StatHasNanoseconds: + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + else: + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. + + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: int ## File system block size. + f_frsize*: int ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: int ## File system ID. + f_flag*: int ## Bit mask of f_flag values. + f_namemax*: int ## Maximum filename length. + + Posix_typed_mem_info* {.importc: "struct posix_typed_mem_info", + header: "<sys/mman.h>", final, pure.} = object + posix_tmi_length*: int + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>", final, pure.} = object + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_notify*: cint ## Notification type. + sigev_signo*: cint ## Signal number. + sigev_value*: SigVal ## Signal value. + sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. + sigev_notify_attributes*: ptr Pthread_attr ## Notification attributes. + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_ptr*: pointer ## pointer signal value; + ## integer signal value not defined! + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_size*: int ## Stack size. + ss_flags*: cint ## Flags. + + SigStack* {.importc: "struct sigstack", + header: "<signal.h>", final, pure.} = object ## struct sigstack + ss_onstack*: cint ## Non-zero when signal stack is in use. + ss_sp*: pointer ## Signal stack pointer. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_code*: cint ## Signal code. + si_errno*: cint ## If non-zero, an errno value associated with + ## this signal, as defined in <errno.h>. + si_pid*: Pid ## Sending process ID. + si_uid*: Uid ## Real user ID of sending process. + si_addr*: pointer ## Address of faulting instruction. + si_status*: cint ## Exit value or signal. + si_band*: int ## Band event for SIGPOLL. + si_value*: SigVal ## Signal value. + + Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint + Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = cint + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + sched_ss_low_priority*: cint ## Low scheduling priority for + ## sporadic server. + sched_ss_repl_period*: Timespec ## Replenishment period for + ## sporadic server. + sched_ss_init_budget*: Timespec ## Initial budget for sporadic server. + sched_ss_max_repl*: cint ## Maximum pending replenishments for + ## sporadic server. + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", + final, pure.} = object + Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", + final, pure.} = object ## ucontext_t + uc_link*: ptr Ucontext ## Pointer to the context that is resumed + ## when this context returns. + uc_sigmask*: Sigset ## The set of signals that are blocked when this + ## context is active. + uc_stack*: Stack ## The stack used by this context. + uc_mcontext*: Mcontext ## A machine-specific representation of the saved + ## context. + +when hasAioH: + type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_offset*: Off ## File offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: int ## Length of transfer. + aio_reqprio*: cint ## Request priority offset. + aio_sigevent*: SigEvent ## Signal number and value. + aio_lio_opcode: cint ## Operation to be performed. + +when hasSpawnH: + type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + + +when not defined(macos) and not defined(macosx): # freebsd + type + Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + +when defined(linux): + # from sys/un.h + const Sockaddr_un_path_length* = 108 +else: + # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html + # this is >=92 + const Sockaddr_un_path_length* = 92 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = uint8 + + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..255, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/uio.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: cint ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: SockLen ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: SockLen ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: uint32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: uint32 ## Set of interfaces for a scope. + + Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, + header: "<netinet/in.h>".} = object ## struct ipv6_mreq + ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. + ipv6mr_interface*: cint ## Interface index. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: uint32 ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_canonname*: cstring ## Canonical name of service location. + ai_next*: ptr AddrInfo ## Pointer to next in list. + + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc, header: "<time.h>".}: cint + timezone* {.importc, header: "<time.h>".}: int + +# Regenerate using detect.nim! +include posix_other_consts + +when defined(linux): + var + MAP_POPULATE* {.importc, header: "<sys/mman.h>".}: cint + ## Populate (prefault) page tables for a mapping. +else: + var + MAP_POPULATE*: cint = 0 + +when defined(linux) or defined(nimdoc): + when defined(alpha) or defined(mips) or defined(mipsel) or + defined(mips64) or defined(mips64el) or defined(parisc) or + defined(sparc) or defined(sparc64) or defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) + ## Multiple binding: load balancing on incoming TCP connections + ## or UDP packets. (Requires Linux kernel > 3.9) + else: + const SO_REUSEPORT* = cint(15) +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint + +when defined(linux) or defined(bsd): + var SOCK_CLOEXEC* {.importc, header: "<sys/socket.h>".}: cint + +when defined(macosx): + # We can't use the NOSIGNAL flag in the `send` function, it has no effect + # Instead we should use SO_NOSIGPIPE in setsockopt + const + MSG_NOSIGNAL* = 0'i32 + var + SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint +elif defined(solaris): + # Solaris doesn't have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0'i32 +else: + var + MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint + ## No SIGPIPE generated when an attempt to send is made on a stream-oriented socket that is no longer connected. + +when defined(haiku): + const + SIGKILLTHR* = 21 ## BeOS specific: Kill just the thread, not team + +when hasSpawnH: + when defined(linux): + # better be safe than sorry; Linux has this flag, macosx doesn't, don't + # know about the other OSes + + # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we + # can't get the magic number from spawn.h + const POSIX_SPAWN_USEVFORK* = cint(0x40) + else: + # macosx lacks this, so we define the constant to be 0 to not affect + # OR'ing of flags: + const POSIX_SPAWN_USEVFORK* = cint(0) + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Exit code, if WIFEXITED(s) +proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Termination signal, if WIFSIGNALED(s) +proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Stop signal, if WIFSTOPPED(s) +proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited normally. +proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited due to uncaught signal. +proc WIFSTOPPED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child is currently stopped. +proc WIFCONTINUED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child has been continued. + +when defined(nimHasStyleChecks): + {.pop.} diff --git a/lib/posix/posix_nintendoswitch.nim b/lib/posix/posix_nintendoswitch.nim new file mode 100644 index 000000000..b66563695 --- /dev/null +++ b/lib/posix/posix_nintendoswitch.nim @@ -0,0 +1,506 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Joey Yakimowich-Payne +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# To be included from posix.nim! + +const + hasSpawnH = true + hasAioH = false + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + +const SIG_HOLD* = cast[Sighandler](2) + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct clong + + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: clong ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + d_ino*: Ino + d_type*: int8 # uint8 really! + d_name*: array[256, cchar] + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + # no struct FTW on linux + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: cint ## Count of paths matched by pattern. + gl_matchc*: cint ## Count of paths matching pattern + gl_offs*: cint ## Slots to reserve at the beginning of gl_pathv. + gl_flags*: cint + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_errfunc*: pointer + gl_closedir*: pointer + gl_readdir*: pointer + gl_opendir*: pointer + gl_lstat*: pointer + gl_stat*: pointer + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_passwd*: cstring + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>".} = pointer + ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + decimal_point*: cstring + thousands_sep*: cstring + grouping*: cstring + int_curr_symbol*: cstring + currency_symbol*: cstring + mon_decimal_point*: cstring + mon_thousands_sep*: cstring + mon_grouping*: cstring + positive_sign*: cstring + negative_sign*: cstring + int_frac_digits*: char + frac_digits*: char + p_cs_precedes*: char + p_sep_by_space*: char + n_cs_precedes*: char + n_sep_by_space*: char + p_sign_posn*: char + n_sign_posn*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_p_sign_posn*: char + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_passwd*: cstring + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_comment*: cstring + pw_gecos*: cstring + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = clong + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = clong + ## used for block sizes + Clock* {.importc: "clock_t", header: "<sys/types.h>".} = clong + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = cint + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = culong + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = culong + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = culong + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = cuint + Id* {.importc: "id_t", header: "<sys/types.h>".} = cuint + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = culong + Key* {.importc: "key_t", header: "<sys/types.h>".} = cint + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = uint16 + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = culong + Off* {.importc: "off_t", header: "<sys/types.h>".} = clong + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = cint + Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>", + pure, final.} = object + abi: array[56 div sizeof(clong), clong] + + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[32 div sizeof(clong), clong] + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[4 div sizeof(cint), cint] + + Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>", + pure, final.} = object + abi: array[48 div sizeof(clonglong), clonglong] + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[4 div sizeof(cint), cint] + Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = cuint + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>", + pure, final.} = object + abi: array[48 div sizeof(clong), clong] + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[4 div sizeof(cint), cint] + Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = cint + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<sys/types.h>", pure, final.} = object + abi: array[56 div sizeof(clong), clong] + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<sys/types.h>".} = object + abi: array[8 div sizeof(clong), clong] + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<sys/types.h>".} = cint + Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = culong + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = clong + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = pointer + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = cuint + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = cuint + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*, ## Name of the hardware type on which the + ## system is running. + domainname*: array[65, char] + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + abi: array[32 div sizeof(clong), clong] + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_mode*: Mode ## Mode of file (see below). + st_nlink*: Nlink ## Number of hard links to the file. + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + when StatHasNanoseconds: + st_atim*: Timespec ## Time of last access. + pad1: clong + st_mtim*: Timespec ## Time of last data modification. + pad2: clong + st_ctim*: Timespec ## Time of last status change. + pad3: clong + else: + st_atime*: Time ## Time of last access. + pad1: clong + st_mtime*: Time ## Time of last data modification. + pad2: clong + st_ctime*: Time ## Time of last status change. + pad3: clong + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + reserved: array[2, clong] + + + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: culong ## File system block size. + f_frsize*: culong ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: culong ## File system ID. + f_flag*: culong ## Bit mask of f_flag values. + f_namemax*: culong ## Maximum filename length. + + # No Posix_typed_mem_info + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>", final.} = culong + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_notify*: cint ## Notification type. + sigev_signo*: cint ## Signal number. + sigev_value*: SigVal ## Signal value. + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_int*: cint ## integer signal value + sival_ptr*: pointer ## pointer signal value; + + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_flags*: cint ## Flags. + ss_size*: csize_t ## Stack size. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_code*: cint ## Signal code. + si_value*: SigVal ## Signal value. + + Nl_item* {.importc: "nl_item", header: "<langinfo.h>".} = cint + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + abi: array[((64+(sizeof(clong) * 8)-1) div (sizeof(clong) * 8)), clong] + +proc si_pid*(info: SigInfo): Pid = + ## This might not be correct behavior. si_pid doesn't exist in Switch's + ## devkitpro headers + raise newException(OSError, "Nintendo switch cannot get si_pid!") + +type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_lio_opcode*: cint ## Operation to be performed. + aio_reqprio*: cint ## Request priority offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: csize_t ## Length of transfer. + aio_sigevent*: SigEvent ## Signal number and value. + next_prio: pointer + abs_prio: cint + policy: cint + error_Code: cint + return_value: clong + aio_offset*: Off ## File offset. + reserved: array[32, uint8] + +type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + +# from sys/un.h +const Sockaddr_un_path_length* = 108 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint + # cushort really + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cshort + + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_len: uint8 + sa_family*: TSa_Family ## Address family. + sa_data*: array[14, char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_len: uint8 + ss_family*: TSa_Family ## Address family. + ss_padding1: array[64 - sizeof(uint8) - sizeof(cshort), char] + ss_align: clonglong + ss_padding2: array[ + 128 - sizeof(uint8) - sizeof(cshort) - + (64 - sizeof(uint8) - sizeof(cshort)) - 64, char] + + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cuint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/socket.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: csize_t ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: csize_t ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: csize_t ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + # data follows... + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_len*: cushort + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + sin_zero: array[8, uint8] + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: uint32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: uint32 ## Set of interfaces for a scope. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: uint32 ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_canonname*: cstring ## Canonical name of service location. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_next*: ptr AddrInfo ## Pointer to next in list. + + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = culong + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc: "_daylight", header: "<time.h>".}: cint + timezone* {.importc: "_timezone", header: "<time.h>".}: clong + +# Regenerate using detect.nim! +include posix_nintendoswitch_consts + +const POSIX_SPAWN_USEVFORK* = cint(0x40) # needs _GNU_SOURCE! + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint = (s shr 8) and 0xff +proc WIFEXITED*(s:cint) : bool = (s and 0xff) == 0 +proc WTERMSIG*(s:cint): cint = s and 0x7f +proc WSTOPSIG*(s:cint): cint = WEXITSTATUS(s) +proc WIFSIGNALED*(s:cint) : bool = ((s and 0x7f) > 0) and ((s and 0x7f) < 0x7f) +proc WIFSTOPPED*(s:cint) : bool = (s and 0xff) == 0x7f diff --git a/lib/posix/posix_nintendoswitch_consts.nim b/lib/posix/posix_nintendoswitch_consts.nim new file mode 100644 index 000000000..8cd8824b5 --- /dev/null +++ b/lib/posix/posix_nintendoswitch_consts.nim @@ -0,0 +1,587 @@ +# Generated by detect.nim + + +# <aio.h> + +# <dlfcn.h> + +# <errno.h> +const E2BIG* = cint(7) +const EACCES* = cint(13) +const EADDRINUSE* = cint(112) +const EADDRNOTAVAIL* = cint(125) +const EAFNOSUPPORT* = cint(106) +const EAGAIN* = cint(11) +const EALREADY* = cint(120) +const EBADF* = cint(9) +const EBADMSG* = cint(77) +const EBUSY* = cint(16) +const ECANCELED* = cint(140) +const ECHILD* = cint(10) +const ECONNABORTED* = cint(113) +const ECONNREFUSED* = cint(111) +const ECONNRESET* = cint(104) +const EDEADLK* = cint(45) +const EDESTADDRREQ* = cint(121) +const EDOM* = cint(33) +const EDQUOT* = cint(132) +const EEXIST* = cint(17) +const EFAULT* = cint(14) +const EFBIG* = cint(27) +const EHOSTUNREACH* = cint(118) +const EIDRM* = cint(36) +const EILSEQ* = cint(138) +const EINPROGRESS* = cint(119) +const EINTR* = cint(4) +const EINVAL* = cint(22) +const EIO* = cint(5) +const EISCONN* = cint(127) +const EISDIR* = cint(21) +const ELOOP* = cint(92) +const EMFILE* = cint(24) +const EMLINK* = cint(31) +const EMSGSIZE* = cint(122) +const EMULTIHOP* = cint(74) +const ENAMETOOLONG* = cint(91) +const ENETDOWN* = cint(115) +const ENETRESET* = cint(126) +const ENETUNREACH* = cint(114) +const ENFILE* = cint(23) +const ENOBUFS* = cint(105) +const ENODATA* = cint(61) +const ENODEV* = cint(19) +const ENOENT* = cint(2) +const ENOEXEC* = cint(8) +const ENOLCK* = cint(46) +const ENOLINK* = cint(67) +const ENOMEM* = cint(12) +const ENOMSG* = cint(35) +const ENOPROTOOPT* = cint(109) +const ENOSPC* = cint(28) +const ENOSR* = cint(63) +const ENOSTR* = cint(60) +const ENOSYS* = cint(88) +const ENOTCONN* = cint(128) +const ENOTDIR* = cint(20) +const ENOTEMPTY* = cint(90) +const ENOTSOCK* = cint(108) +const ENOTSUP* = cint(134) +const ENOTTY* = cint(25) +const ENXIO* = cint(6) +const EOPNOTSUPP* = cint(95) +const EOVERFLOW* = cint(139) +const EPERM* = cint(1) +const EPIPE* = cint(32) +const EPROTO* = cint(71) +const EPROTONOSUPPORT* = cint(123) +const EPROTOTYPE* = cint(107) +const ERANGE* = cint(34) +const EROFS* = cint(30) +const ESPIPE* = cint(29) +const ESRCH* = cint(3) +const ESTALE* = cint(133) +const ETIME* = cint(62) +const ETIMEDOUT* = cint(116) +const ETXTBSY* = cint(26) +const EWOULDBLOCK* = cint(11) +const EXDEV* = cint(18) + +# <fcntl.h> +const F_DUPFD* = cint(0) +const F_GETFD* = cint(1) +const F_SETFD* = cint(2) +const F_GETFL* = cint(3) +const F_SETFL* = cint(4) +const F_GETLK* = cint(7) +const F_SETLK* = cint(8) +const F_SETLKW* = cint(9) +const F_GETOWN* = cint(5) +const F_SETOWN* = cint(6) +const FD_CLOEXEC* = cint(1) +const F_RDLCK* = cint(1) +const F_UNLCK* = cint(3) +const F_WRLCK* = cint(2) +const O_CREAT* = cint(512) +const O_EXCL* = cint(2048) +const O_NOCTTY* = cint(32768) +const O_TRUNC* = cint(1024) +const O_APPEND* = cint(8) +const O_NONBLOCK* = cint(16384) +const O_SYNC* = cint(8192) +const O_ACCMODE* = cint(3) +const O_RDONLY* = cint(0) +const O_RDWR* = cint(2) +const O_WRONLY* = cint(1) + +# <fenv.h> + +# <fmtmsg.h> + +# <fnmatch.h> +const FNM_NOMATCH* = cint(1) +const FNM_PATHNAME* = cint(2) +const FNM_PERIOD* = cint(4) +const FNM_NOESCAPE* = cint(1) + +# <ftw.h> + +# <glob.h> +const GLOB_APPEND* = cint(1) +const GLOB_DOOFFS* = cint(2) +const GLOB_ERR* = cint(4) +const GLOB_MARK* = cint(8) +const GLOB_NOCHECK* = cint(16) +const GLOB_NOSORT* = cint(32) +const GLOB_NOSPACE* = cint(-1) + +# <langinfo.h> +const CODESET* = cint(0) +const D_T_FMT* = cint(1) +const D_FMT* = cint(2) +const T_FMT* = cint(3) +const T_FMT_AMPM* = cint(4) +const AM_STR* = cint(5) +const PM_STR* = cint(6) +const DAY_1* = cint(7) +const DAY_2* = cint(8) +const DAY_3* = cint(9) +const DAY_4* = cint(10) +const DAY_5* = cint(11) +const DAY_6* = cint(12) +const DAY_7* = cint(13) +const ABDAY_1* = cint(14) +const ABDAY_2* = cint(15) +const ABDAY_3* = cint(16) +const ABDAY_4* = cint(17) +const ABDAY_5* = cint(18) +const ABDAY_6* = cint(19) +const ABDAY_7* = cint(20) +const MON_1* = cint(21) +const MON_2* = cint(22) +const MON_3* = cint(23) +const MON_4* = cint(24) +const MON_5* = cint(25) +const MON_6* = cint(26) +const MON_7* = cint(27) +const MON_8* = cint(28) +const MON_9* = cint(29) +const MON_10* = cint(30) +const MON_11* = cint(31) +const MON_12* = cint(32) +const ABMON_1* = cint(33) +const ABMON_2* = cint(34) +const ABMON_3* = cint(35) +const ABMON_4* = cint(36) +const ABMON_5* = cint(37) +const ABMON_6* = cint(38) +const ABMON_7* = cint(39) +const ABMON_8* = cint(40) +const ABMON_9* = cint(41) +const ABMON_10* = cint(42) +const ABMON_11* = cint(43) +const ABMON_12* = cint(44) +const ERA* = cint(45) +const ERA_D_FMT* = cint(46) +const ERA_D_T_FMT* = cint(47) +const ERA_T_FMT* = cint(48) +const ALT_DIGITS* = cint(49) +const RADIXCHAR* = cint(50) +const THOUSEP* = cint(51) +const YESEXPR* = cint(52) +const NOEXPR* = cint(53) +const CRNCYSTR* = cint(56) + +# <locale.h> +const LC_ALL* = cint(0) +const LC_COLLATE* = cint(1) +const LC_CTYPE* = cint(2) +const LC_MESSAGES* = cint(6) +const LC_MONETARY* = cint(3) +const LC_NUMERIC* = cint(4) +const LC_TIME* = cint(5) + +# <netdb.h> +const IPPORT_RESERVED* = cint(1024) +const HOST_NOT_FOUND* = cint(1) +const NO_DATA* = cint(4) +const NO_RECOVERY* = cint(3) +const TRY_AGAIN* = cint(2) +const AI_PASSIVE* = cint(1) +const AI_CANONNAME* = cint(2) +const AI_NUMERICHOST* = cint(4) +const AI_NUMERICSERV* = cint(8) +const AI_V4MAPPED* = cint(2048) +const AI_ALL* = cint(256) +const AI_ADDRCONFIG* = cint(1024) +const NI_NOFQDN* = cint(1) +const NI_NUMERICHOST* = cint(2) +const NI_NAMEREQD* = cint(4) +const NI_NUMERICSERV* = cint(8) +const NI_NUMERICSCOPE* = cint(32) +const NI_DGRAM* = cint(16) +const EAI_AGAIN* = cint(2) +const EAI_BADFLAGS* = cint(3) +const EAI_FAIL* = cint(4) +const EAI_FAMILY* = cint(5) +const EAI_MEMORY* = cint(6) +const EAI_NONAME* = cint(8) +const EAI_SERVICE* = cint(9) +const EAI_SOCKTYPE* = cint(10) +const EAI_SYSTEM* = cint(11) +const EAI_OVERFLOW* = cint(14) + +# <net/if.h> +const IF_NAMESIZE* = cint(16) + +# <netinet/in.h> +const IPPROTO_IP* = cint(0) +const IPPROTO_IPV6* = cint(41) +const IPPROTO_ICMP* = cint(1) +const IPPROTO_ICMPV6* = cint(58) +const IPPROTO_RAW* = cint(255) +const IPPROTO_TCP* = cint(6) +const IPPROTO_UDP* = cint(17) +const INADDR_ANY* = InAddrScalar(0) +const INADDR_LOOPBACK* = InAddrScalar(2130706433) +const INADDR_BROADCAST* = InAddrScalar(4294967295) +const INET_ADDRSTRLEN* = cint(16) +const INET6_ADDRSTRLEN* = cint(46) +const IPV6_JOIN_GROUP* = cint(12) +const IPV6_LEAVE_GROUP* = cint(13) +const IPV6_MULTICAST_HOPS* = cint(10) +const IPV6_MULTICAST_IF* = cint(9) +const IPV6_MULTICAST_LOOP* = cint(11) +const IPV6_UNICAST_HOPS* = cint(4) +const IPV6_V6ONLY* = cint(27) + +# <netinet/tcp.h> +const TCP_NODELAY* = cint(1) + +# <nl_types.h> + +# <poll.h> +const POLLIN* = cshort(1) +const POLLRDNORM* = cshort(64) +const POLLRDBAND* = cshort(128) +const POLLPRI* = cshort(2) +const POLLOUT* = cshort(4) +const POLLWRNORM* = cshort(4) +const POLLWRBAND* = cshort(256) +const POLLERR* = cshort(8) +const POLLHUP* = cshort(16) +const POLLNVAL* = cshort(32) + +# <pthread.h> +const PTHREAD_CREATE_DETACHED* = cint(0) +const PTHREAD_CREATE_JOINABLE* = cint(1) +const PTHREAD_EXPLICIT_SCHED* = cint(2) +const PTHREAD_INHERIT_SCHED* = cint(1) +const PTHREAD_SCOPE_PROCESS* = cint(0) +const PTHREAD_SCOPE_SYSTEM* = cint(1) + +# <sched.h> +const SCHED_FIFO* = cint(1) +const SCHED_RR* = cint(2) +const SCHED_OTHER* = cint(0) + +# <semaphore.h> + +# <signal.h> +const SIGEV_NONE* = cint(1) +const SIGEV_SIGNAL* = cint(2) +const SIGEV_THREAD* = cint(3) +const SIGABRT* = cint(6) +const SIGALRM* = cint(14) +const SIGBUS* = cint(10) +const SIGCHLD* = cint(20) +const SIGCONT* = cint(19) +const SIGFPE* = cint(8) +const SIGHUP* = cint(1) +const SIGILL* = cint(4) +const SIGINT* = cint(2) +const SIGKILL* = cint(9) +const SIGPIPE* = cint(13) +const SIGQUIT* = cint(3) +const SIGSEGV* = cint(11) +const SIGSTOP* = cint(17) +const SIGTERM* = cint(15) +const SIGTSTP* = cint(18) +const SIGTTIN* = cint(21) +const SIGTTOU* = cint(22) +const SIGUSR1* = cint(30) +const SIGUSR2* = cint(31) +const SIGPOLL* = cint(23) +const SIGPROF* = cint(27) +const SIGSYS* = cint(12) +const SIGTRAP* = cint(5) +const SIGURG* = cint(16) +const SIGVTALRM* = cint(26) +const SIGXCPU* = cint(24) +const SIGXFSZ* = cint(25) +const SA_NOCLDSTOP* = cint(1) +const SIG_BLOCK* = cint(1) +const SIG_UNBLOCK* = cint(2) +const SIG_SETMASK* = cint(0) +const SS_ONSTACK* = cint(1) +const SS_DISABLE* = cint(2) +const MINSIGSTKSZ* = cint(2048) +const SIGSTKSZ* = cint(8192) +const SIG_DFL* = cast[Sighandler](0) +const SIG_ERR* = cast[Sighandler](-1) +const SIG_IGN* = cast[Sighandler](1) + +# <sys/ipc.h> + +# <sys/mman.h> + +# <sys/resource.h> + +# <sys/select.h> +const FD_SETSIZE* = cint(64) + +# <sys/socket.h> +const MSG_CTRUNC* = cint(32) +const MSG_DONTROUTE* = cint(4) +const MSG_EOR* = cint(8) +const MSG_OOB* = cint(1) +const SCM_RIGHTS* = cint(1) +const SO_ACCEPTCONN* = cint(2) +const SO_BROADCAST* = cint(32) +const SO_DEBUG* = cint(1) +const SO_DONTROUTE* = cint(16) +const SO_ERROR* = cint(4103) +const SO_KEEPALIVE* = cint(8) +const SO_LINGER* = cint(128) +const SO_OOBINLINE* = cint(256) +const SO_RCVBUF* = cint(4098) +const SO_RCVLOWAT* = cint(4100) +const SO_RCVTIMEO* = cint(4102) +const SO_REUSEADDR* = cint(4) +const SO_SNDBUF* = cint(4097) +const SO_SNDLOWAT* = cint(4099) +const SO_SNDTIMEO* = cint(4101) +const SO_TYPE* = cint(4104) +const SOCK_DGRAM* = cint(2) +const SOCK_RAW* = cint(3) +const SOCK_SEQPACKET* = cint(5) +const SOCK_STREAM* = cint(1) +const SOL_SOCKET* = cint(65535) +const SOMAXCONN* = cint(128) +const SO_REUSEPORT* = cint(512) +const MSG_NOSIGNAL* = cint(131072) +const MSG_PEEK* = cint(2) +const MSG_TRUNC* = cint(16) +const MSG_WAITALL* = cint(64) +const AF_INET* = cint(2) +const AF_INET6* = cint(28) +const AF_UNIX* = cint(1) +const AF_UNSPEC* = cint(0) +const SHUT_RD* = cint(0) +const SHUT_RDWR* = cint(2) +const SHUT_WR* = cint(1) + +# <sys/stat.h> +const S_IFBLK* = cint(24576) +const S_IFCHR* = cint(8192) +const S_IFDIR* = cint(16384) +const S_IFIFO* = cint(4096) +const S_IFLNK* = cint(40960) +const S_IFMT* = cint(61440) +const S_IFREG* = cint(32768) +const S_IFSOCK* = cint(49152) +const S_IRGRP* = cint(32) +const S_IROTH* = cint(4) +const S_IRUSR* = cint(256) +const S_IRWXG* = cint(56) +const S_IRWXO* = cint(7) +const S_IRWXU* = cint(448) +const S_ISGID* = cint(1024) +const S_ISUID* = cint(2048) +const S_ISVTX* = cint(512) +const S_IWGRP* = cint(16) +const S_IWOTH* = cint(2) +const S_IWUSR* = cint(128) +const S_IXGRP* = cint(8) +const S_IXOTH* = cint(1) +const S_IXUSR* = cint(64) + +# <sys/statvfs.h> +const ST_RDONLY* = cint(1) +const ST_NOSUID* = cint(2) + +# <sys/wait.h> +const WNOHANG* = cint(1) +const WUNTRACED* = cint(2) + +# <spawn.h> +const POSIX_SPAWN_RESETIDS* = cint(1) +const POSIX_SPAWN_SETPGROUP* = cint(2) +const POSIX_SPAWN_SETSCHEDPARAM* = cint(4) +const POSIX_SPAWN_SETSCHEDULER* = cint(8) +const POSIX_SPAWN_SETSIGDEF* = cint(16) +const POSIX_SPAWN_SETSIGMASK* = cint(32) + +# <stdio.h> +const IOFBF* = cint(0) +const IONBF* = cint(2) + +# <time.h> +const CLOCKS_PER_SEC* = clong(100) +const CLOCK_REALTIME* = cint(1) +const TIMER_ABSTIME* = cint(4) +const CLOCK_MONOTONIC* = cint(4) + +# <unistd.h> +const F_OK* = cint(0) +const R_OK* = cint(4) +const W_OK* = cint(2) +const X_OK* = cint(1) +const F_LOCK* = cint(1) +const F_TEST* = cint(3) +const F_TLOCK* = cint(2) +const F_ULOCK* = cint(0) +const PC_2_SYMLINKS* = cint(13) +const PC_ALLOC_SIZE_MIN* = cint(15) +const PC_ASYNC_IO* = cint(9) +const PC_CHOWN_RESTRICTED* = cint(6) +const PC_FILESIZEBITS* = cint(12) +const PC_LINK_MAX* = cint(0) +const PC_MAX_CANON* = cint(1) +const PC_MAX_INPUT* = cint(2) +const PC_NAME_MAX* = cint(3) +const PC_NO_TRUNC* = cint(7) +const PC_PATH_MAX* = cint(4) +const PC_PIPE_BUF* = cint(5) +const PC_PRIO_IO* = cint(10) +const PC_REC_INCR_XFER_SIZE* = cint(16) +const PC_REC_MIN_XFER_SIZE* = cint(18) +const PC_REC_XFER_ALIGN* = cint(19) +const PC_SYMLINK_MAX* = cint(14) +const PC_SYNC_IO* = cint(11) +const PC_VDISABLE* = cint(8) +const SC_2_C_BIND* = cint(108) +const SC_2_C_DEV* = cint(109) +const SC_2_CHAR_TERM* = cint(107) +const SC_2_FORT_DEV* = cint(110) +const SC_2_FORT_RUN* = cint(111) +const SC_2_LOCALEDEF* = cint(112) +const SC_2_PBS* = cint(113) +const SC_2_PBS_ACCOUNTING* = cint(114) +const SC_2_PBS_CHECKPOINT* = cint(115) +const SC_2_PBS_LOCATE* = cint(116) +const SC_2_PBS_MESSAGE* = cint(117) +const SC_2_PBS_TRACK* = cint(118) +const SC_2_SW_DEV* = cint(119) +const SC_2_UPE* = cint(120) +const SC_2_VERSION* = cint(121) +const SC_ADVISORY_INFO* = cint(54) +const SC_AIO_LISTIO_MAX* = cint(34) +const SC_AIO_MAX* = cint(35) +const SC_AIO_PRIO_DELTA_MAX* = cint(36) +const SC_ARG_MAX* = cint(0) +const SC_ASYNCHRONOUS_IO* = cint(21) +const SC_ATEXIT_MAX* = cint(55) +const SC_BARRIERS* = cint(56) +const SC_BC_BASE_MAX* = cint(57) +const SC_BC_DIM_MAX* = cint(58) +const SC_BC_SCALE_MAX* = cint(59) +const SC_BC_STRING_MAX* = cint(60) +const SC_CHILD_MAX* = cint(1) +const SC_CLK_TCK* = cint(2) +const SC_CLOCK_SELECTION* = cint(61) +const SC_COLL_WEIGHTS_MAX* = cint(62) +const SC_CPUTIME* = cint(63) +const SC_DELAYTIMER_MAX* = cint(37) +const SC_EXPR_NEST_MAX* = cint(64) +const SC_FSYNC* = cint(22) +const SC_GETGR_R_SIZE_MAX* = cint(50) +const SC_GETPW_R_SIZE_MAX* = cint(51) +const SC_HOST_NAME_MAX* = cint(65) +const SC_IOV_MAX* = cint(66) +const SC_IPV6* = cint(67) +const SC_JOB_CONTROL* = cint(5) +const SC_LINE_MAX* = cint(68) +const SC_LOGIN_NAME_MAX* = cint(52) +const SC_MAPPED_FILES* = cint(23) +const SC_MEMLOCK* = cint(24) +const SC_MEMLOCK_RANGE* = cint(25) +const SC_MEMORY_PROTECTION* = cint(26) +const SC_MESSAGE_PASSING* = cint(27) +const SC_MONOTONIC_CLOCK* = cint(69) +const SC_MQ_OPEN_MAX* = cint(13) +const SC_MQ_PRIO_MAX* = cint(14) +const SC_NGROUPS_MAX* = cint(3) +const SC_OPEN_MAX* = cint(4) +const SC_PAGE_SIZE* = cint(8) +const SC_PRIORITIZED_IO* = cint(28) +const SC_PRIORITY_SCHEDULING* = cint(101) +const SC_RAW_SOCKETS* = cint(70) +const SC_RE_DUP_MAX* = cint(73) +const SC_READER_WRITER_LOCKS* = cint(71) +const SC_REALTIME_SIGNALS* = cint(29) +const SC_REGEXP* = cint(72) +const SC_RTSIG_MAX* = cint(15) +const SC_SAVED_IDS* = cint(6) +const SC_SEM_NSEMS_MAX* = cint(16) +const SC_SEM_VALUE_MAX* = cint(17) +const SC_SEMAPHORES* = cint(30) +const SC_SHARED_MEMORY_OBJECTS* = cint(31) +const SC_SHELL* = cint(74) +const SC_SIGQUEUE_MAX* = cint(18) +const SC_SPAWN* = cint(75) +const SC_SPIN_LOCKS* = cint(76) +const SC_SPORADIC_SERVER* = cint(77) +const SC_SS_REPL_MAX* = cint(78) +const SC_STREAM_MAX* = cint(100) +const SC_SYMLOOP_MAX* = cint(79) +const SC_SYNCHRONIZED_IO* = cint(32) +const SC_THREAD_ATTR_STACKADDR* = cint(43) +const SC_THREAD_ATTR_STACKSIZE* = cint(44) +const SC_THREAD_CPUTIME* = cint(80) +const SC_THREAD_DESTRUCTOR_ITERATIONS* = cint(53) +const SC_THREAD_KEYS_MAX* = cint(38) +const SC_THREAD_PRIO_INHERIT* = cint(46) +const SC_THREAD_PRIO_PROTECT* = cint(47) +const SC_THREAD_PRIORITY_SCHEDULING* = cint(45) +const SC_THREAD_PROCESS_SHARED* = cint(48) +const SC_THREAD_SAFE_FUNCTIONS* = cint(49) +const SC_THREAD_SPORADIC_SERVER* = cint(81) +const SC_THREAD_STACK_MIN* = cint(39) +const SC_THREAD_THREADS_MAX* = cint(40) +const SC_THREADS* = cint(42) +const SC_TIMEOUTS* = cint(82) +const SC_TIMER_MAX* = cint(19) +const SC_TIMERS* = cint(33) +const SC_TRACE* = cint(83) +const SC_TRACE_EVENT_FILTER* = cint(84) +const SC_TRACE_EVENT_NAME_MAX* = cint(85) +const SC_TRACE_INHERIT* = cint(86) +const SC_TRACE_LOG* = cint(87) +const SC_TRACE_NAME_MAX* = cint(88) +const SC_TRACE_SYS_MAX* = cint(89) +const SC_TRACE_USER_EVENT_MAX* = cint(90) +const SC_TTY_NAME_MAX* = cint(41) +const SC_TYPED_MEMORY_OBJECTS* = cint(91) +const SC_TZNAME_MAX* = cint(20) +const SC_V6_ILP32_OFF32* = cint(92) +const SC_V6_ILP32_OFFBIG* = cint(93) +const SC_V6_LP64_OFF64* = cint(94) +const SC_V6_LPBIG_OFFBIG* = cint(95) +const SC_VERSION* = cint(7) +const SC_XBS5_ILP32_OFF32* = cint(92) +const SC_XBS5_ILP32_OFFBIG* = cint(93) +const SC_XBS5_LP64_OFF64* = cint(94) +const SC_XBS5_LPBIG_OFFBIG* = cint(95) +const SC_XOPEN_CRYPT* = cint(96) +const SC_XOPEN_ENH_I18N* = cint(97) +const SC_XOPEN_LEGACY* = cint(98) +const SC_XOPEN_REALTIME* = cint(99) +const SC_XOPEN_REALTIME_THREADS* = cint(102) +const SC_XOPEN_SHM* = cint(103) +const SC_XOPEN_STREAMS* = cint(104) +const SC_XOPEN_UNIX* = cint(105) +const SC_XOPEN_VERSION* = cint(106) +const SC_NPROCESSORS_ONLN* = cint(10) +const SEEK_SET* = cint(0) +const SEEK_CUR* = cint(1) +const SEEK_END* = cint(2) diff --git a/lib/posix/posix_openbsd_amd64.nim b/lib/posix/posix_openbsd_amd64.nim new file mode 100644 index 000000000..184cd89c0 --- /dev/null +++ b/lib/posix/posix_openbsd_amd64.nim @@ -0,0 +1,565 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +const + hasSpawnH = true # should exist for every Posix system nowadays + hasAioH = false + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + ## A type representing a directory stream. + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct clonglong + + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: int ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + when defined(haiku): + d_dev*: Dev ## Device (not POSIX) + d_pdev*: Dev ## Parent device (only for queries) (not POSIX) + d_ino*: Ino ## File serial number. + when defined(dragonfly): + # DragonflyBSD doesn't have `d_reclen` field. + d_type*: uint8 + elif defined(linux) or defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or defined(genode): + d_reclen*: cshort ## Length of this record. (not POSIX) + d_type*: int8 ## Type of file; not supported by all filesystem types. + ## (not POSIX) + when defined(linux) or defined(openbsd): + d_off*: Off ## Not an offset. Value that `telldir()` would return. + elif defined(haiku): + d_pino*: Ino ## Parent inode (only for queries) (not POSIX) + d_reclen*: cushort ## Length of this record. (not POSIX) + + d_name*: array[0..255, char] ## Name of entry. + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + FTW* {.importc: "struct FTW", header: "<ftw.h>", final, pure.} = object + base*: cint + level*: cint + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: int ## Count of paths matched by pattern. + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_offs*: int ## Slots to reserve at the beginning of gl_pathv. + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>", final, pure.} = + object ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + currency_symbol*: cstring + decimal_point*: cstring + frac_digits*: char + grouping*: cstring + int_curr_symbol*: cstring + int_frac_digits*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_p_sign_posn*: char + mon_decimal_point*: cstring + mon_grouping*: cstring + mon_thousands_sep*: cstring + negative_sign*: cstring + n_cs_precedes*: char + n_sep_by_space*: char + n_sign_posn*: char + positive_sign*: cstring + p_cs_precedes*: char + p_sep_by_space*: char + p_sign_posn*: char + thousands_sep*: cstring + + Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = int + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = int32 + ## used for block sizes + Clock* {.importc: "clock_t", header: "<sys/types.h>".} = int + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = ( + when defined(freebsd): + uint32 + else: + int32) + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = uint32 + Id* {.importc: "id_t", header: "<sys/types.h>".} = int + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int + Key* {.importc: "key_t", header: "<sys/types.h>".} = int + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = uint32 + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = uint32 + Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 + Pthread_attr* {.importc: "pthread_attr_t", header: "<pthread.h>".} = int + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<pthread.h>".} = int + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<pthread.h>".} = int + Pthread_cond* {.importc: "pthread_cond_t", header: "<pthread.h>".} = int + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<pthread.h>".} = int + Pthread_key* {.importc: "pthread_key_t", header: "<pthread.h>".} = int + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<pthread.h>".} = int + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<pthread.h>".} = int + Pthread_once* {.importc: "pthread_once_t", header: "<pthread.h>".} = int + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<pthread.h>".} = int + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<pthread.h>".} = int + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<pthread.h>".} = int + Pthread* {.importc: "pthread_t", header: "<pthread.h>".} = int + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = int32 + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = int + Trace_attr* {.importc: "trace_attr_t", header: "<sys/types.h>".} = int + Trace_event_id* {.importc: "trace_event_id_t", + header: "<sys/types.h>".} = int + Trace_event_set* {.importc: "trace_event_set_t", + header: "<sys/types.h>".} = int + Trace_id* {.importc: "trace_id_t", header: "<sys/types.h>".} = int + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = uint32 + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = int + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*: array[0..255, char] ## Name of the hardware type on which the + ## system is running. + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + Ipc_perm* {.importc: "struct ipc_perm", + header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm + uid*: Uid ## Owner's user ID. + gid*: Gid ## Owner's group ID. + cuid*: Uid ## Creator's user ID. + cgid*: Gid ## Creator's group ID. + mode*: Mode ## Read/write permission. + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_mode*: Mode ## Mode of file (see below). + st_nlink*: Nlink ## Number of hard links to the file. + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + when StatHasNanoseconds: + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + else: + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: int ## File system block size. + f_frsize*: int ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: int ## File system ID. + f_flag*: int ## Bit mask of f_flag values. + f_namemax*: int ## Maximum filename length. + + Posix_typed_mem_info* {.importc: "struct posix_typed_mem_info", + header: "<sys/mman.h>", final, pure.} = object + posix_tmi_length*: int + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>", final, pure.} = object + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_notify*: cint ## Notification type. + sigev_signo*: cint ## Signal number. + sigev_value*: SigVal ## Signal value. + sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. + sigev_notify_attributes*: ptr Pthread_attr ## Notification attributes. + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_ptr*: pointer ## pointer signal value; + ## integer signal value not defined! + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_size*: int ## Stack size. + ss_flags*: cint ## Flags. + + SigStack* {.importc: "struct sigstack", + header: "<signal.h>", final, pure.} = object ## struct sigstack + ss_onstack*: cint ## Non-zero when signal stack is in use. + ss_sp*: pointer ## Signal stack pointer. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_code*: cint ## Signal code. + si_errno*: cint ## If non-zero, an errno value associated with + ## this signal, as defined in <errno.h>. + si_pid*: Pid ## Sending process ID. + si_uid*: Uid ## Real user ID of sending process. + si_addr*: pointer ## Address of faulting instruction. + si_status*: cint ## Exit value or signal. + si_band*: int ## Band event for SIGPOLL. + si_value*: SigVal ## Signal value. + + Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint + Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = cint + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + sched_ss_low_priority*: cint ## Low scheduling priority for + ## sporadic server. + sched_ss_repl_period*: Timespec ## Replenishment period for + ## sporadic server. + sched_ss_init_budget*: Timespec ## Initial budget for sporadic server. + sched_ss_max_repl*: cint ## Maximum pending replenishments for + ## sporadic server. + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + # the following aren't actually defined, but they're here to keep things happy for now + Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", + final, pure.} = object + Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", + final, pure.} = object ## ucontext_t + uc_link*: ptr Ucontext ## Pointer to the context that is resumed + ## when this context returns. + uc_sigmask*: Sigset ## The set of signals that are blocked when this + ## context is active. + uc_stack*: Stack ## The stack used by this context. + uc_mcontext*: Mcontext ## A machine-specific representation of the saved + ## context. + +when hasAioH: + type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_offset*: Off ## File offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: int ## Length of transfer. + aio_reqprio*: cint ## Request priority offset. + aio_sigevent*: SigEvent ## Signal number and value. + aio_lio_opcode: cint ## Operation to be performed. + +when hasSpawnH: + type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + +# according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html +# this is >=92 +const Sockaddr_un_path_length* = 92 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = uint8 + + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..255, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/uio.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: cint ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: SockLen ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: SockLen ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: uint32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: uint32 ## Set of interfaces for a scope. + + Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, + header: "<netinet/in.h>".} = object ## struct ipv6_mreq + ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. + ipv6mr_interface*: cint ## Interface index. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: uint32 ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_canonname*: cstring ## Canonical name of service location. + ai_next*: ptr AddrInfo ## Pointer to next in list. + + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc, header: "<time.h>".}: cint + timezone* {.importc, header: "<time.h>".}: int + +# Regenerate using detect.nim! +include posix_other_consts + +var + MAP_POPULATE*: cint = 0 + +when defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint + +var SOCK_CLOEXEC* {.importc, header: "<sys/socket.h>".}: cint + +var MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint + +when hasSpawnH: + # openbsd lacks this, so we define the constant to be 0 to not affect + # OR'ing of flags: + const POSIX_SPAWN_USEVFORK* = cint(0) + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Exit code, if WIFEXITED(s) +proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Termination signal, if WIFSIGNALED(s) +proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Stop signal, if WIFSTOPPED(s) +proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited normally. +proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited due to uncaught signal. +proc WIFSTOPPED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child is currently stopped. +proc WIFCONTINUED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child has been continued. + +when defined(nimHasStyleChecks): + {.pop.} diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim new file mode 100644 index 000000000..ea8731405 --- /dev/null +++ b/lib/posix/posix_other.nim @@ -0,0 +1,706 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +when defined(freertos) or defined(zephyr): + const + hasSpawnH = false # should exist for every Posix system nowadays + hasAioH = false +else: + const + hasSpawnH = true # should exist for every Posix system nowadays + hasAioH = defined(linux) + +when defined(linux) and not defined(android): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passl: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passl: "-lresolv".} + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + ## A type representing a directory stream. + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct ( + when defined(nimUse64BitCTime): + int64 + else: + clong + ) + + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: int ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + when defined(haiku): + d_dev*: Dev ## Device (not POSIX) + d_pdev*: Dev ## Parent device (only for queries) (not POSIX) + d_ino*: Ino ## File serial number. + when defined(dragonfly): + # DragonflyBSD doesn't have `d_reclen` field. + d_type*: uint8 + elif defined(linux) or defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or defined(genode): + d_reclen*: cshort ## Length of this record. (not POSIX) + d_type*: int8 ## Type of file; not supported by all filesystem types. + ## (not POSIX) + when defined(linux) or defined(openbsd): + d_off*: Off ## Not an offset. Value that `telldir()` would return. + elif defined(haiku): + d_pino*: Ino ## Parent inode (only for queries) (not POSIX) + d_reclen*: cushort ## Length of this record. (not POSIX) + + d_name*: array[0..255, char] ## Name of entry. + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + FTW* {.importc: "struct FTW", header: "<ftw.h>", final, pure.} = object + base*: cint + level*: cint + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: int ## Count of paths matched by pattern. + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_offs*: int ## Slots to reserve at the beginning of gl_pathv. + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>", final, pure.} = + object ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + currency_symbol*: cstring + decimal_point*: cstring + frac_digits*: char + grouping*: cstring + int_curr_symbol*: cstring + int_frac_digits*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_p_sign_posn*: char + mon_decimal_point*: cstring + mon_grouping*: cstring + mon_thousands_sep*: cstring + negative_sign*: cstring + n_cs_precedes*: char + n_sep_by_space*: char + n_sign_posn*: char + positive_sign*: cstring + p_cs_precedes*: char + p_sep_by_space*: char + p_sign_posn*: char + thousands_sep*: cstring + + Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = int + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = int + ## used for block sizes + Clock* {.importc: "clock_t", header: "<sys/types.h>".} = int + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = int + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = int + Id* {.importc: "id_t", header: "<sys/types.h>".} = int + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int + Key* {.importc: "key_t", header: "<sys/types.h>".} = int + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = ( + when defined(android) or defined(macos) or defined(macosx) or + (defined(bsd) and not defined(openbsd) and not defined(netbsd)): + uint16 + else: + uint32 + ) + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int + Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 + Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = int + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<sys/types.h>".} = int + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<sys/types.h>".} = int + Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>".} = int + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<sys/types.h>".} = int + Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = int + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>".} = int + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<sys/types.h>".} = int + Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = int + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<sys/types.h>".} = int + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<sys/types.h>".} = int + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<sys/types.h>".} = int + Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = int + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = int + Trace_attr* {.importc: "trace_attr_t", header: "<sys/types.h>".} = int + Trace_event_id* {.importc: "trace_event_id_t", + header: "<sys/types.h>".} = int + Trace_event_set* {.importc: "trace_event_set_t", + header: "<sys/types.h>".} = int + Trace_id* {.importc: "trace_id_t", header: "<sys/types.h>".} = int + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = int + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = int + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*: array[0..255, char] ## Name of the hardware type on which the + ## system is running. + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + Ipc_perm* {.importc: "struct ipc_perm", + header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm + uid*: Uid ## Owner's user ID. + gid*: Gid ## Owner's group ID. + cuid*: Uid ## Creator's user ID. + cgid*: Gid ## Creator's group ID. + mode*: Mode ## Read/write permission. + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_mode*: Mode ## Mode of file (see below). + st_nlink*: Nlink ## Number of hard links to the file. + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + when StatHasNanoseconds: + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + else: + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: int ## File system block size. + f_frsize*: int ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: int ## File system ID. + f_flag*: int ## Bit mask of f_flag values. + f_namemax*: int ## Maximum filename length. + + Posix_typed_mem_info* {.importc: "struct posix_typed_mem_info", + header: "<sys/mman.h>", final, pure.} = object + posix_tmi_length*: int + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>", final, pure.} = object + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_notify*: cint ## Notification type. + sigev_signo*: cint ## Signal number. + sigev_value*: SigVal ## Signal value. + sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. + sigev_notify_attributes*: ptr Pthread_attr ## Notification attributes. + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_ptr*: pointer ## pointer signal value; + ## integer signal value not defined! + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_size*: int ## Stack size. + ss_flags*: cint ## Flags. + + SigStack* {.importc: "struct sigstack", + header: "<signal.h>", final, pure.} = object ## struct sigstack + ss_onstack*: cint ## Non-zero when signal stack is in use. + ss_sp*: pointer ## Signal stack pointer. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_code*: cint ## Signal code. + si_errno*: cint ## If non-zero, an errno value associated with + ## this signal, as defined in <errno.h>. + si_pid*: Pid ## Sending process ID. + si_uid*: Uid ## Real user ID of sending process. + si_addr*: pointer ## Address of faulting instruction. + si_status*: cint ## Exit value or signal. + si_band*: int ## Band event for SIGPOLL. + si_value*: SigVal ## Signal value. + + Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint + Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = cint + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + sched_ss_low_priority*: cint ## Low scheduling priority for + ## sporadic server. + sched_ss_repl_period*: Timespec ## Replenishment period for + ## sporadic server. + sched_ss_init_budget*: Timespec ## Initial budget for sporadic server. + sched_ss_max_repl*: cint ## Maximum pending replenishments for + ## sporadic server. + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", + final, pure.} = object + Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", + final, pure.} = object ## ucontext_t + uc_link*: ptr Ucontext ## Pointer to the context that is resumed + ## when this context returns. + uc_sigmask*: Sigset ## The set of signals that are blocked when this + ## context is active. + uc_stack*: Stack ## The stack used by this context. + uc_mcontext*: Mcontext ## A machine-specific representation of the saved + ## context. + +when hasAioH: + type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_offset*: Off ## File offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: int ## Length of transfer. + aio_reqprio*: cint ## Request priority offset. + aio_sigevent*: SigEvent ## Signal number and value. + aio_lio_opcode: cint ## Operation to be performed. + +when hasSpawnH: + type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + +when defined(linux): + const Sockaddr_max_length* = 255 + # from sys/un.h + const Sockaddr_un_path_length* = 108 +elif defined(zephyr): + when defined(net_ipv6): + const Sockaddr_max_length* = 24 + elif defined(net_raw): + const Sockaddr_max_length* = 20 + elif defined(net_ipv4): + const Sockaddr_max_length* = 8 + else: + const Sockaddr_max_length* = 255 # just for compilation purposes + + const Sockaddr_un_path_length* = Sockaddr_max_length + # Zephyr is heavily customizable so it's easy to get to a state + # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. + when defined(net_ipv4) or defined(net_ipv6) or defined(net_raw): + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} +elif defined(freertos) or defined(lwip): + const Sockaddr_max_length* = 14 + const Sockaddr_un_path_length* = 108 +else: + const Sockaddr_max_length* = 255 + # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html + # this is >=92 + const Sockaddr_un_path_length* = 92 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cushort + +when defined(lwip): + type + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_len*: uint8 ## Address family. + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + s2_len*: uint8 ## Address family. + ss_family*: TSa_Family ## Address family. + s2_data1*: array[2, char] ## Address family. + s2_data2*: array[3, uint32] ## Address family. + when defined(lwip6) or defined(net_ipv6): + s2_data3*: array[3, uint32] ## Address family. +elif defined(zephyr): + type + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].} + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].} +else: + type + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + +type + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path + + +type + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/uio.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: cint ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: SockLen ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: SockLen ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + # TODO: Fixme for FreeRTOS/LwIP, these are incorrect + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: int32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: int32 ## Set of interfaces for a scope. + + Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, + header: "<netinet/in.h>".} = object ## struct ipv6_mreq + ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. + ipv6mr_interface*: cint ## Interface index. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: int32 ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_canonname*: cstring ## Canonical name of service location. + ai_next*: ptr AddrInfo ## Pointer to next in list. + +when not defined(lwip): + type + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + when defined(zephyr): + type + Tnfds* = distinct cint + else: + type + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc, header: "<time.h>".}: cint + timezone* {.importc, header: "<time.h>".}: int + +# Regenerate using detect.nim! +when defined(lwip): + include posix_freertos_consts +else: + include posix_other_consts + +when defined(linux): + var + MAP_POPULATE* {.importc, header: "<sys/mman.h>".}: cint + ## Populate (prefault) page tables for a mapping. +else: + var + MAP_POPULATE*: cint = 0 + +when defined(linux) or defined(nimdoc): + when defined(alpha) or defined(mips) or defined(mipsel) or + defined(mips64) or defined(mips64el) or defined(parisc) or + defined(sparc) or defined(sparc64) or defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) + ## Multiple binding: load balancing on incoming TCP connections + ## or UDP packets. (Requires Linux kernel > 3.9) + else: + const SO_REUSEPORT* = cint(15) +elif defined(nuttx): + # Not supported, use SO_REUSEADDR to avoid compilation errors. + var SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "<sys/socket.h>".}: cint +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint + +when defined(linux) or defined(bsd) or defined(nuttx): + var SOCK_CLOEXEC* {.importc, header: "<sys/socket.h>".}: cint + +when defined(macosx): + # We can't use the NOSIGNAL flag in the `send` function, it has no effect + # Instead we should use SO_NOSIGPIPE in setsockopt + const + MSG_NOSIGNAL* = 0'i32 + var + SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint +elif defined(solaris): + # Solaris doesn't have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0'i32 +elif defined(zephyr) or defined(freertos) or defined(lwip): + # LwIP/FreeRTOS doesn't have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0x20'i32 +else: + var + MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint + ## No SIGPIPE generated when an attempt to send is made on a stream-oriented socket that is no longer connected. + +when defined(haiku): + const + SIGKILLTHR* = 21 ## BeOS specific: Kill just the thread, not team + +when hasSpawnH: + when defined(linux): + # better be safe than sorry; Linux has this flag, macosx and NuttX don't, + # don't know about the other OSes + + # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we + # can't get the magic number from spawn.h + const POSIX_SPAWN_USEVFORK* = cint(0x40) + else: + # macosx and NuttX lack this, so we define the constant to be 0 to not affect + # OR'ing of flags: + const POSIX_SPAWN_USEVFORK* = cint(0) + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Exit code, if WIFEXITED(s) +proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Termination signal, if WIFSIGNALED(s) +proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Stop signal, if WIFSTOPPED(s) +proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited normally. +proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited due to uncaught signal. +proc WIFSTOPPED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child is currently stopped. +proc WIFCONTINUED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child has been continued. + +when defined(nimHasStyleChecks): + {.pop.} diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim new file mode 100644 index 000000000..d346b4150 --- /dev/null +++ b/lib/posix/posix_other_consts.nim @@ -0,0 +1,757 @@ +# Generated by detect.nim + +# <aio.h> +var AIO_ALLDONE* {.importc: "AIO_ALLDONE", header: "<aio.h>".}: cint +var AIO_CANCELED* {.importc: "AIO_CANCELED", header: "<aio.h>".}: cint +var AIO_NOTCANCELED* {.importc: "AIO_NOTCANCELED", header: "<aio.h>".}: cint +var LIO_NOP* {.importc: "LIO_NOP", header: "<aio.h>".}: cint +var LIO_NOWAIT* {.importc: "LIO_NOWAIT", header: "<aio.h>".}: cint +var LIO_READ* {.importc: "LIO_READ", header: "<aio.h>".}: cint +var LIO_WAIT* {.importc: "LIO_WAIT", header: "<aio.h>".}: cint +var LIO_WRITE* {.importc: "LIO_WRITE", header: "<aio.h>".}: cint + +# <dlfcn.h> +var RTLD_LAZY* {.importc: "RTLD_LAZY", header: "<dlfcn.h>".}: cint +var RTLD_NOW* {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: cint +var RTLD_GLOBAL* {.importc: "RTLD_GLOBAL", header: "<dlfcn.h>".}: cint +var RTLD_LOCAL* {.importc: "RTLD_LOCAL", header: "<dlfcn.h>".}: cint + +# <errno.h> +var E2BIG* {.importc: "E2BIG", header: "<errno.h>".}: cint +var EACCES* {.importc: "EACCES", header: "<errno.h>".}: cint +var EADDRINUSE* {.importc: "EADDRINUSE", header: "<errno.h>".}: cint +var EADDRNOTAVAIL* {.importc: "EADDRNOTAVAIL", header: "<errno.h>".}: cint +var EAFNOSUPPORT* {.importc: "EAFNOSUPPORT", header: "<errno.h>".}: cint +var EAGAIN* {.importc: "EAGAIN", header: "<errno.h>".}: cint +var EALREADY* {.importc: "EALREADY", header: "<errno.h>".}: cint +var EBADF* {.importc: "EBADF", header: "<errno.h>".}: cint +var EBADMSG* {.importc: "EBADMSG", header: "<errno.h>".}: cint +var EBUSY* {.importc: "EBUSY", header: "<errno.h>".}: cint +var ECANCELED* {.importc: "ECANCELED", header: "<errno.h>".}: cint +var ECHILD* {.importc: "ECHILD", header: "<errno.h>".}: cint +var ECONNABORTED* {.importc: "ECONNABORTED", header: "<errno.h>".}: cint +var ECONNREFUSED* {.importc: "ECONNREFUSED", header: "<errno.h>".}: cint +var ECONNRESET* {.importc: "ECONNRESET", header: "<errno.h>".}: cint +var EDEADLK* {.importc: "EDEADLK", header: "<errno.h>".}: cint +var EDESTADDRREQ* {.importc: "EDESTADDRREQ", header: "<errno.h>".}: cint +var EDOM* {.importc: "EDOM", header: "<errno.h>".}: cint +var EDQUOT* {.importc: "EDQUOT", header: "<errno.h>".}: cint +var EEXIST* {.importc: "EEXIST", header: "<errno.h>".}: cint +var EFAULT* {.importc: "EFAULT", header: "<errno.h>".}: cint +var EFBIG* {.importc: "EFBIG", header: "<errno.h>".}: cint +var EHOSTUNREACH* {.importc: "EHOSTUNREACH", header: "<errno.h>".}: cint +var EIDRM* {.importc: "EIDRM", header: "<errno.h>".}: cint +var EILSEQ* {.importc: "EILSEQ", header: "<errno.h>".}: cint +var EINPROGRESS* {.importc: "EINPROGRESS", header: "<errno.h>".}: cint +var EINTR* {.importc: "EINTR", header: "<errno.h>".}: cint +var EINVAL* {.importc: "EINVAL", header: "<errno.h>".}: cint +var EIO* {.importc: "EIO", header: "<errno.h>".}: cint +var EISCONN* {.importc: "EISCONN", header: "<errno.h>".}: cint +var EISDIR* {.importc: "EISDIR", header: "<errno.h>".}: cint +var ELOOP* {.importc: "ELOOP", header: "<errno.h>".}: cint +var EMFILE* {.importc: "EMFILE", header: "<errno.h>".}: cint +var EMLINK* {.importc: "EMLINK", header: "<errno.h>".}: cint +var EMSGSIZE* {.importc: "EMSGSIZE", header: "<errno.h>".}: cint +var EMULTIHOP* {.importc: "EMULTIHOP", header: "<errno.h>".}: cint +var ENAMETOOLONG* {.importc: "ENAMETOOLONG", header: "<errno.h>".}: cint +var ENETDOWN* {.importc: "ENETDOWN", header: "<errno.h>".}: cint +var ENETRESET* {.importc: "ENETRESET", header: "<errno.h>".}: cint +var ENETUNREACH* {.importc: "ENETUNREACH", header: "<errno.h>".}: cint +var ENFILE* {.importc: "ENFILE", header: "<errno.h>".}: cint +var ENOBUFS* {.importc: "ENOBUFS", header: "<errno.h>".}: cint +var ENODATA* {.importc: "ENODATA", header: "<errno.h>".}: cint +var ENODEV* {.importc: "ENODEV", header: "<errno.h>".}: cint +var ENOENT* {.importc: "ENOENT", header: "<errno.h>".}: cint +var ENOEXEC* {.importc: "ENOEXEC", header: "<errno.h>".}: cint +var ENOLCK* {.importc: "ENOLCK", header: "<errno.h>".}: cint +var ENOLINK* {.importc: "ENOLINK", header: "<errno.h>".}: cint +var ENOMEM* {.importc: "ENOMEM", header: "<errno.h>".}: cint +var ENOMSG* {.importc: "ENOMSG", header: "<errno.h>".}: cint +var ENOPROTOOPT* {.importc: "ENOPROTOOPT", header: "<errno.h>".}: cint +var ENOSPC* {.importc: "ENOSPC", header: "<errno.h>".}: cint +var ENOSR* {.importc: "ENOSR", header: "<errno.h>".}: cint +var ENOSTR* {.importc: "ENOSTR", header: "<errno.h>".}: cint +var ENOSYS* {.importc: "ENOSYS", header: "<errno.h>".}: cint +var ENOTCONN* {.importc: "ENOTCONN", header: "<errno.h>".}: cint +var ENOTDIR* {.importc: "ENOTDIR", header: "<errno.h>".}: cint +var ENOTEMPTY* {.importc: "ENOTEMPTY", header: "<errno.h>".}: cint +var ENOTSOCK* {.importc: "ENOTSOCK", header: "<errno.h>".}: cint +var ENOTSUP* {.importc: "ENOTSUP", header: "<errno.h>".}: cint +var ENOTTY* {.importc: "ENOTTY", header: "<errno.h>".}: cint +var ENXIO* {.importc: "ENXIO", header: "<errno.h>".}: cint +var EOPNOTSUPP* {.importc: "EOPNOTSUPP", header: "<errno.h>".}: cint +var EOVERFLOW* {.importc: "EOVERFLOW", header: "<errno.h>".}: cint +var EPERM* {.importc: "EPERM", header: "<errno.h>".}: cint +var EPIPE* {.importc: "EPIPE", header: "<errno.h>".}: cint +var EPROTO* {.importc: "EPROTO", header: "<errno.h>".}: cint +var EPROTONOSUPPORT* {.importc: "EPROTONOSUPPORT", header: "<errno.h>".}: cint +var EPROTOTYPE* {.importc: "EPROTOTYPE", header: "<errno.h>".}: cint +var ERANGE* {.importc: "ERANGE", header: "<errno.h>".}: cint +var EROFS* {.importc: "EROFS", header: "<errno.h>".}: cint +var ESPIPE* {.importc: "ESPIPE", header: "<errno.h>".}: cint +var ESRCH* {.importc: "ESRCH", header: "<errno.h>".}: cint +var ESTALE* {.importc: "ESTALE", header: "<errno.h>".}: cint +var ETIME* {.importc: "ETIME", header: "<errno.h>".}: cint +var ETIMEDOUT* {.importc: "ETIMEDOUT", header: "<errno.h>".}: cint +var ETXTBSY* {.importc: "ETXTBSY", header: "<errno.h>".}: cint +var EWOULDBLOCK* {.importc: "EWOULDBLOCK", header: "<errno.h>".}: cint +var EXDEV* {.importc: "EXDEV", header: "<errno.h>".}: cint + +# <fcntl.h> +var F_DUPFD* {.importc: "F_DUPFD", header: "<fcntl.h>".}: cint +when defined(nuttx): + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD_CLOEXEC", header: "<fcntl.h>".}: cint +else: + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "<fcntl.h>".}: cint +var F_GETFD* {.importc: "F_GETFD", header: "<fcntl.h>".}: cint +var F_SETFD* {.importc: "F_SETFD", header: "<fcntl.h>".}: cint +var F_GETFL* {.importc: "F_GETFL", header: "<fcntl.h>".}: cint +var F_SETFL* {.importc: "F_SETFL", header: "<fcntl.h>".}: cint +var F_GETLK* {.importc: "F_GETLK", header: "<fcntl.h>".}: cint +var F_SETLK* {.importc: "F_SETLK", header: "<fcntl.h>".}: cint +var F_SETLKW* {.importc: "F_SETLKW", header: "<fcntl.h>".}: cint +var F_GETOWN* {.importc: "F_GETOWN", header: "<fcntl.h>".}: cint +var F_SETOWN* {.importc: "F_SETOWN", header: "<fcntl.h>".}: cint +var FD_CLOEXEC* {.importc: "FD_CLOEXEC", header: "<fcntl.h>".}: cint +var F_RDLCK* {.importc: "F_RDLCK", header: "<fcntl.h>".}: cint +var F_UNLCK* {.importc: "F_UNLCK", header: "<fcntl.h>".}: cint +var F_WRLCK* {.importc: "F_WRLCK", header: "<fcntl.h>".}: cint +var O_CREAT* {.importc: "O_CREAT", header: "<fcntl.h>".}: cint +var O_EXCL* {.importc: "O_EXCL", header: "<fcntl.h>".}: cint +var O_NOCTTY* {.importc: "O_NOCTTY", header: "<fcntl.h>".}: cint +var O_TRUNC* {.importc: "O_TRUNC", header: "<fcntl.h>".}: cint +var O_APPEND* {.importc: "O_APPEND", header: "<fcntl.h>".}: cint +var O_DSYNC* {.importc: "O_DSYNC", header: "<fcntl.h>".}: cint +var O_NONBLOCK* {.importc: "O_NONBLOCK", header: "<fcntl.h>".}: cint +var O_RSYNC* {.importc: "O_RSYNC", header: "<fcntl.h>".}: cint +var O_SYNC* {.importc: "O_SYNC", header: "<fcntl.h>".}: cint +var O_ACCMODE* {.importc: "O_ACCMODE", header: "<fcntl.h>".}: cint +var O_RDONLY* {.importc: "O_RDONLY", header: "<fcntl.h>".}: cint +var O_RDWR* {.importc: "O_RDWR", header: "<fcntl.h>".}: cint +var O_WRONLY* {.importc: "O_WRONLY", header: "<fcntl.h>".}: cint +var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "<fcntl.h>".}: cint +when defined(nuttx): + var O_DIRECT* {.importc: "O_DIRECT", header: "<fcntl.h>".}: cint + var O_PATH* {.importc: "O_PATH", header: "<fcntl.h>".}: cint + var O_NOATIME* {.importc: "O_NOATIME", header: "<fcntl.h>".}: cint + var O_TMPFILE* {.importc: "O_TMPFILE", header: "<fcntl.h>".}: cint +var POSIX_FADV_NORMAL* {.importc: "POSIX_FADV_NORMAL", header: "<fcntl.h>".}: cint +var POSIX_FADV_SEQUENTIAL* {.importc: "POSIX_FADV_SEQUENTIAL", header: "<fcntl.h>".}: cint +var POSIX_FADV_RANDOM* {.importc: "POSIX_FADV_RANDOM", header: "<fcntl.h>".}: cint +var POSIX_FADV_WILLNEED* {.importc: "POSIX_FADV_WILLNEED", header: "<fcntl.h>".}: cint +var POSIX_FADV_DONTNEED* {.importc: "POSIX_FADV_DONTNEED", header: "<fcntl.h>".}: cint +var POSIX_FADV_NOREUSE* {.importc: "POSIX_FADV_NOREUSE", header: "<fcntl.h>".}: cint + +# <fenv.h> +var FE_DIVBYZERO* {.importc: "FE_DIVBYZERO", header: "<fenv.h>".}: cint +var FE_INEXACT* {.importc: "FE_INEXACT", header: "<fenv.h>".}: cint +var FE_INVALID* {.importc: "FE_INVALID", header: "<fenv.h>".}: cint +var FE_OVERFLOW* {.importc: "FE_OVERFLOW", header: "<fenv.h>".}: cint +var FE_UNDERFLOW* {.importc: "FE_UNDERFLOW", header: "<fenv.h>".}: cint +var FE_ALL_EXCEPT* {.importc: "FE_ALL_EXCEPT", header: "<fenv.h>".}: cint +var FE_DOWNWARD* {.importc: "FE_DOWNWARD", header: "<fenv.h>".}: cint +var FE_TONEAREST* {.importc: "FE_TONEAREST", header: "<fenv.h>".}: cint +var FE_TOWARDZERO* {.importc: "FE_TOWARDZERO", header: "<fenv.h>".}: cint +var FE_UPWARD* {.importc: "FE_UPWARD", header: "<fenv.h>".}: cint +var FE_DFL_ENV* {.importc: "FE_DFL_ENV", header: "<fenv.h>".}: cint + +# <fmtmsg.h> +var MM_HARD* {.importc: "MM_HARD", header: "<fmtmsg.h>".}: cint +var MM_SOFT* {.importc: "MM_SOFT", header: "<fmtmsg.h>".}: cint +var MM_FIRM* {.importc: "MM_FIRM", header: "<fmtmsg.h>".}: cint +var MM_APPL* {.importc: "MM_APPL", header: "<fmtmsg.h>".}: cint +var MM_UTIL* {.importc: "MM_UTIL", header: "<fmtmsg.h>".}: cint +var MM_OPSYS* {.importc: "MM_OPSYS", header: "<fmtmsg.h>".}: cint +var MM_RECOVER* {.importc: "MM_RECOVER", header: "<fmtmsg.h>".}: cint +var MM_NRECOV* {.importc: "MM_NRECOV", header: "<fmtmsg.h>".}: cint +var MM_HALT* {.importc: "MM_HALT", header: "<fmtmsg.h>".}: cint +var MM_ERROR* {.importc: "MM_ERROR", header: "<fmtmsg.h>".}: cint +var MM_WARNING* {.importc: "MM_WARNING", header: "<fmtmsg.h>".}: cint +var MM_INFO* {.importc: "MM_INFO", header: "<fmtmsg.h>".}: cint +var MM_NOSEV* {.importc: "MM_NOSEV", header: "<fmtmsg.h>".}: cint +var MM_PRINT* {.importc: "MM_PRINT", header: "<fmtmsg.h>".}: cint +var MM_CONSOLE* {.importc: "MM_CONSOLE", header: "<fmtmsg.h>".}: cint +var MM_OK* {.importc: "MM_OK", header: "<fmtmsg.h>".}: cint +var MM_NOTOK* {.importc: "MM_NOTOK", header: "<fmtmsg.h>".}: cint +var MM_NOMSG* {.importc: "MM_NOMSG", header: "<fmtmsg.h>".}: cint +var MM_NOCON* {.importc: "MM_NOCON", header: "<fmtmsg.h>".}: cint + +# <fnmatch.h> +var FNM_NOMATCH* {.importc: "FNM_NOMATCH", header: "<fnmatch.h>".}: cint +var FNM_PATHNAME* {.importc: "FNM_PATHNAME", header: "<fnmatch.h>".}: cint +var FNM_PERIOD* {.importc: "FNM_PERIOD", header: "<fnmatch.h>".}: cint +var FNM_NOESCAPE* {.importc: "FNM_NOESCAPE", header: "<fnmatch.h>".}: cint +var FNM_NOSYS* {.importc: "FNM_NOSYS", header: "<fnmatch.h>".}: cint + +# <ftw.h> +var FTW_F* {.importc: "FTW_F", header: "<ftw.h>".}: cint +var FTW_D* {.importc: "FTW_D", header: "<ftw.h>".}: cint +var FTW_DNR* {.importc: "FTW_DNR", header: "<ftw.h>".}: cint +var FTW_DP* {.importc: "FTW_DP", header: "<ftw.h>".}: cint +var FTW_NS* {.importc: "FTW_NS", header: "<ftw.h>".}: cint +var FTW_SL* {.importc: "FTW_SL", header: "<ftw.h>".}: cint +var FTW_SLN* {.importc: "FTW_SLN", header: "<ftw.h>".}: cint +var FTW_PHYS* {.importc: "FTW_PHYS", header: "<ftw.h>".}: cint +var FTW_MOUNT* {.importc: "FTW_MOUNT", header: "<ftw.h>".}: cint +var FTW_DEPTH* {.importc: "FTW_DEPTH", header: "<ftw.h>".}: cint +var FTW_CHDIR* {.importc: "FTW_CHDIR", header: "<ftw.h>".}: cint + +# <glob.h> +var GLOB_APPEND* {.importc: "GLOB_APPEND", header: "<glob.h>".}: cint +var GLOB_DOOFFS* {.importc: "GLOB_DOOFFS", header: "<glob.h>".}: cint +var GLOB_ERR* {.importc: "GLOB_ERR", header: "<glob.h>".}: cint +var GLOB_MARK* {.importc: "GLOB_MARK", header: "<glob.h>".}: cint +var GLOB_NOCHECK* {.importc: "GLOB_NOCHECK", header: "<glob.h>".}: cint +var GLOB_NOESCAPE* {.importc: "GLOB_NOESCAPE", header: "<glob.h>".}: cint +var GLOB_NOSORT* {.importc: "GLOB_NOSORT", header: "<glob.h>".}: cint +var GLOB_ABORTED* {.importc: "GLOB_ABORTED", header: "<glob.h>".}: cint +var GLOB_NOMATCH* {.importc: "GLOB_NOMATCH", header: "<glob.h>".}: cint +var GLOB_NOSPACE* {.importc: "GLOB_NOSPACE", header: "<glob.h>".}: cint +var GLOB_NOSYS* {.importc: "GLOB_NOSYS", header: "<glob.h>".}: cint + +# <langinfo.h> +var CODESET* {.importc: "CODESET", header: "<langinfo.h>".}: cint +var D_T_FMT* {.importc: "D_T_FMT", header: "<langinfo.h>".}: cint +var D_FMT* {.importc: "D_FMT", header: "<langinfo.h>".}: cint +var T_FMT* {.importc: "T_FMT", header: "<langinfo.h>".}: cint +var T_FMT_AMPM* {.importc: "T_FMT_AMPM", header: "<langinfo.h>".}: cint +var AM_STR* {.importc: "AM_STR", header: "<langinfo.h>".}: cint +var PM_STR* {.importc: "PM_STR", header: "<langinfo.h>".}: cint +var DAY_1* {.importc: "DAY_1", header: "<langinfo.h>".}: cint +var DAY_2* {.importc: "DAY_2", header: "<langinfo.h>".}: cint +var DAY_3* {.importc: "DAY_3", header: "<langinfo.h>".}: cint +var DAY_4* {.importc: "DAY_4", header: "<langinfo.h>".}: cint +var DAY_5* {.importc: "DAY_5", header: "<langinfo.h>".}: cint +var DAY_6* {.importc: "DAY_6", header: "<langinfo.h>".}: cint +var DAY_7* {.importc: "DAY_7", header: "<langinfo.h>".}: cint +var ABDAY_1* {.importc: "ABDAY_1", header: "<langinfo.h>".}: cint +var ABDAY_2* {.importc: "ABDAY_2", header: "<langinfo.h>".}: cint +var ABDAY_3* {.importc: "ABDAY_3", header: "<langinfo.h>".}: cint +var ABDAY_4* {.importc: "ABDAY_4", header: "<langinfo.h>".}: cint +var ABDAY_5* {.importc: "ABDAY_5", header: "<langinfo.h>".}: cint +var ABDAY_6* {.importc: "ABDAY_6", header: "<langinfo.h>".}: cint +var ABDAY_7* {.importc: "ABDAY_7", header: "<langinfo.h>".}: cint +var MON_1* {.importc: "MON_1", header: "<langinfo.h>".}: cint +var MON_2* {.importc: "MON_2", header: "<langinfo.h>".}: cint +var MON_3* {.importc: "MON_3", header: "<langinfo.h>".}: cint +var MON_4* {.importc: "MON_4", header: "<langinfo.h>".}: cint +var MON_5* {.importc: "MON_5", header: "<langinfo.h>".}: cint +var MON_6* {.importc: "MON_6", header: "<langinfo.h>".}: cint +var MON_7* {.importc: "MON_7", header: "<langinfo.h>".}: cint +var MON_8* {.importc: "MON_8", header: "<langinfo.h>".}: cint +var MON_9* {.importc: "MON_9", header: "<langinfo.h>".}: cint +var MON_10* {.importc: "MON_10", header: "<langinfo.h>".}: cint +var MON_11* {.importc: "MON_11", header: "<langinfo.h>".}: cint +var MON_12* {.importc: "MON_12", header: "<langinfo.h>".}: cint +var ABMON_1* {.importc: "ABMON_1", header: "<langinfo.h>".}: cint +var ABMON_2* {.importc: "ABMON_2", header: "<langinfo.h>".}: cint +var ABMON_3* {.importc: "ABMON_3", header: "<langinfo.h>".}: cint +var ABMON_4* {.importc: "ABMON_4", header: "<langinfo.h>".}: cint +var ABMON_5* {.importc: "ABMON_5", header: "<langinfo.h>".}: cint +var ABMON_6* {.importc: "ABMON_6", header: "<langinfo.h>".}: cint +var ABMON_7* {.importc: "ABMON_7", header: "<langinfo.h>".}: cint +var ABMON_8* {.importc: "ABMON_8", header: "<langinfo.h>".}: cint +var ABMON_9* {.importc: "ABMON_9", header: "<langinfo.h>".}: cint +var ABMON_10* {.importc: "ABMON_10", header: "<langinfo.h>".}: cint +var ABMON_11* {.importc: "ABMON_11", header: "<langinfo.h>".}: cint +var ABMON_12* {.importc: "ABMON_12", header: "<langinfo.h>".}: cint +var ERA* {.importc: "ERA", header: "<langinfo.h>".}: cint +var ERA_D_FMT* {.importc: "ERA_D_FMT", header: "<langinfo.h>".}: cint +var ERA_D_T_FMT* {.importc: "ERA_D_T_FMT", header: "<langinfo.h>".}: cint +var ERA_T_FMT* {.importc: "ERA_T_FMT", header: "<langinfo.h>".}: cint +var ALT_DIGITS* {.importc: "ALT_DIGITS", header: "<langinfo.h>".}: cint +var RADIXCHAR* {.importc: "RADIXCHAR", header: "<langinfo.h>".}: cint +var THOUSEP* {.importc: "THOUSEP", header: "<langinfo.h>".}: cint +var YESEXPR* {.importc: "YESEXPR", header: "<langinfo.h>".}: cint +var NOEXPR* {.importc: "NOEXPR", header: "<langinfo.h>".}: cint +var CRNCYSTR* {.importc: "CRNCYSTR", header: "<langinfo.h>".}: cint + +# <locale.h> +var LC_ALL* {.importc: "LC_ALL", header: "<locale.h>".}: cint +var LC_COLLATE* {.importc: "LC_COLLATE", header: "<locale.h>".}: cint +var LC_CTYPE* {.importc: "LC_CTYPE", header: "<locale.h>".}: cint +var LC_MESSAGES* {.importc: "LC_MESSAGES", header: "<locale.h>".}: cint +var LC_MONETARY* {.importc: "LC_MONETARY", header: "<locale.h>".}: cint +var LC_NUMERIC* {.importc: "LC_NUMERIC", header: "<locale.h>".}: cint +var LC_TIME* {.importc: "LC_TIME", header: "<locale.h>".}: cint + +# <netdb.h> +var IPPORT_RESERVED* {.importc: "IPPORT_RESERVED", header: "<netdb.h>".}: cint +var HOST_NOT_FOUND* {.importc: "HOST_NOT_FOUND", header: "<netdb.h>".}: cint +var NO_DATA* {.importc: "NO_DATA", header: "<netdb.h>".}: cint +var NO_RECOVERY* {.importc: "NO_RECOVERY", header: "<netdb.h>".}: cint +var TRY_AGAIN* {.importc: "TRY_AGAIN", header: "<netdb.h>".}: cint +var AI_PASSIVE* {.importc: "AI_PASSIVE", header: "<netdb.h>".}: cint +var AI_CANONNAME* {.importc: "AI_CANONNAME", header: "<netdb.h>".}: cint +var AI_NUMERICHOST* {.importc: "AI_NUMERICHOST", header: "<netdb.h>".}: cint +var AI_NUMERICSERV* {.importc: "AI_NUMERICSERV", header: "<netdb.h>".}: cint +var AI_V4MAPPED* {.importc: "AI_V4MAPPED", header: "<netdb.h>".}: cint +var AI_ALL* {.importc: "AI_ALL", header: "<netdb.h>".}: cint +var AI_ADDRCONFIG* {.importc: "AI_ADDRCONFIG", header: "<netdb.h>".}: cint +var NI_NOFQDN* {.importc: "NI_NOFQDN", header: "<netdb.h>".}: cint +var NI_NUMERICHOST* {.importc: "NI_NUMERICHOST", header: "<netdb.h>".}: cint +var NI_NAMEREQD* {.importc: "NI_NAMEREQD", header: "<netdb.h>".}: cint +var NI_NUMERICSERV* {.importc: "NI_NUMERICSERV", header: "<netdb.h>".}: cint +var NI_NUMERICSCOPE* {.importc: "NI_NUMERICSCOPE", header: "<netdb.h>".}: cint +var NI_DGRAM* {.importc: "NI_DGRAM", header: "<netdb.h>".}: cint +var EAI_AGAIN* {.importc: "EAI_AGAIN", header: "<netdb.h>".}: cint +var EAI_BADFLAGS* {.importc: "EAI_BADFLAGS", header: "<netdb.h>".}: cint +var EAI_FAIL* {.importc: "EAI_FAIL", header: "<netdb.h>".}: cint +var EAI_FAMILY* {.importc: "EAI_FAMILY", header: "<netdb.h>".}: cint +var EAI_MEMORY* {.importc: "EAI_MEMORY", header: "<netdb.h>".}: cint +var EAI_NONAME* {.importc: "EAI_NONAME", header: "<netdb.h>".}: cint +var EAI_SERVICE* {.importc: "EAI_SERVICE", header: "<netdb.h>".}: cint +var EAI_SOCKTYPE* {.importc: "EAI_SOCKTYPE", header: "<netdb.h>".}: cint +var EAI_SYSTEM* {.importc: "EAI_SYSTEM", header: "<netdb.h>".}: cint +var EAI_OVERFLOW* {.importc: "EAI_OVERFLOW", header: "<netdb.h>".}: cint + +# <net/if.h> +var IF_NAMESIZE* {.importc: "IF_NAMESIZE", header: "<net/if.h>".}: cint + +# <netinet/in.h> +var IPPROTO_IP* {.importc: "IPPROTO_IP", header: "<netinet/in.h>".}: cint +var IPPROTO_IPV6* {.importc: "IPPROTO_IPV6", header: "<netinet/in.h>".}: cint +var IPPROTO_ICMP* {.importc: "IPPROTO_ICMP", header: "<netinet/in.h>".}: cint +var IPPROTO_ICMPV6* {.importc: "IPPROTO_ICMPV6", header: "<netinet/in.h>".}: cint +var IPPROTO_RAW* {.importc: "IPPROTO_RAW", header: "<netinet/in.h>".}: cint +var IPPROTO_TCP* {.importc: "IPPROTO_TCP", header: "<netinet/in.h>".}: cint +var IPPROTO_UDP* {.importc: "IPPROTO_UDP", header: "<netinet/in.h>".}: cint +var INADDR_ANY* {.importc: "INADDR_ANY", header: "<netinet/in.h>".}: InAddrScalar +var INADDR_LOOPBACK* {.importc: "INADDR_LOOPBACK", header: "<netinet/in.h>".}: InAddrScalar +var INADDR_BROADCAST* {.importc: "INADDR_BROADCAST", header: "<netinet/in.h>".}: InAddrScalar +var INET_ADDRSTRLEN* {.importc: "INET_ADDRSTRLEN", header: "<netinet/in.h>".}: cint +var INET6_ADDRSTRLEN* {.importc: "INET6_ADDRSTRLEN", header: "<netinet/in.h>".}: cint +var IPV6_JOIN_GROUP* {.importc: "IPV6_JOIN_GROUP", header: "<netinet/in.h>".}: cint +var IPV6_LEAVE_GROUP* {.importc: "IPV6_LEAVE_GROUP", header: "<netinet/in.h>".}: cint +var IPV6_MULTICAST_HOPS* {.importc: "IPV6_MULTICAST_HOPS", header: "<netinet/in.h>".}: cint +var IPV6_MULTICAST_IF* {.importc: "IPV6_MULTICAST_IF", header: "<netinet/in.h>".}: cint +var IPV6_MULTICAST_LOOP* {.importc: "IPV6_MULTICAST_LOOP", header: "<netinet/in.h>".}: cint +var IPV6_UNICAST_HOPS* {.importc: "IPV6_UNICAST_HOPS", header: "<netinet/in.h>".}: cint +var IPV6_V6ONLY* {.importc: "IPV6_V6ONLY", header: "<netinet/in.h>".}: cint + +# <netinet/tcp.h> +var TCP_NODELAY* {.importc: "TCP_NODELAY", header: "<netinet/tcp.h>".}: cint + +# <nl_types.h> +var NL_SETD* {.importc: "NL_SETD", header: "<nl_types.h>".}: cint +var NL_CAT_LOCALE* {.importc: "NL_CAT_LOCALE", header: "<nl_types.h>".}: cint + +# <poll.h> +var POLLIN* {.importc: "POLLIN", header: "<poll.h>".}: cshort +var POLLRDNORM* {.importc: "POLLRDNORM", header: "<poll.h>".}: cshort +var POLLRDBAND* {.importc: "POLLRDBAND", header: "<poll.h>".}: cshort +var POLLPRI* {.importc: "POLLPRI", header: "<poll.h>".}: cshort +var POLLOUT* {.importc: "POLLOUT", header: "<poll.h>".}: cshort +var POLLWRNORM* {.importc: "POLLWRNORM", header: "<poll.h>".}: cshort +var POLLWRBAND* {.importc: "POLLWRBAND", header: "<poll.h>".}: cshort +var POLLERR* {.importc: "POLLERR", header: "<poll.h>".}: cshort +var POLLHUP* {.importc: "POLLHUP", header: "<poll.h>".}: cshort +var POLLNVAL* {.importc: "POLLNVAL", header: "<poll.h>".}: cshort + +# <pthread.h> +var PTHREAD_BARRIER_SERIAL_THREAD* {.importc: "PTHREAD_BARRIER_SERIAL_THREAD", header: "<pthread.h>".}: cint +var PTHREAD_CANCEL_ASYNCHRONOUS* {.importc: "PTHREAD_CANCEL_ASYNCHRONOUS", header: "<pthread.h>".}: cint +var PTHREAD_CANCEL_ENABLE* {.importc: "PTHREAD_CANCEL_ENABLE", header: "<pthread.h>".}: cint +var PTHREAD_CANCEL_DEFERRED* {.importc: "PTHREAD_CANCEL_DEFERRED", header: "<pthread.h>".}: cint +var PTHREAD_CANCEL_DISABLE* {.importc: "PTHREAD_CANCEL_DISABLE", header: "<pthread.h>".}: cint +var PTHREAD_CREATE_DETACHED* {.importc: "PTHREAD_CREATE_DETACHED", header: "<pthread.h>".}: cint +var PTHREAD_CREATE_JOINABLE* {.importc: "PTHREAD_CREATE_JOINABLE", header: "<pthread.h>".}: cint +var PTHREAD_EXPLICIT_SCHED* {.importc: "PTHREAD_EXPLICIT_SCHED", header: "<pthread.h>".}: cint +var PTHREAD_INHERIT_SCHED* {.importc: "PTHREAD_INHERIT_SCHED", header: "<pthread.h>".}: cint +var PTHREAD_MUTEX_DEFAULT* {.importc: "PTHREAD_MUTEX_DEFAULT", header: "<pthread.h>".}: cint +var PTHREAD_MUTEX_ERRORCHECK* {.importc: "PTHREAD_MUTEX_ERRORCHECK", header: "<pthread.h>".}: cint +var PTHREAD_MUTEX_NORMAL* {.importc: "PTHREAD_MUTEX_NORMAL", header: "<pthread.h>".}: cint +var PTHREAD_MUTEX_RECURSIVE* {.importc: "PTHREAD_MUTEX_RECURSIVE", header: "<pthread.h>".}: cint +var PTHREAD_PRIO_INHERIT* {.importc: "PTHREAD_PRIO_INHERIT", header: "<pthread.h>".}: cint +var PTHREAD_PRIO_NONE* {.importc: "PTHREAD_PRIO_NONE", header: "<pthread.h>".}: cint +var PTHREAD_PRIO_PROTECT* {.importc: "PTHREAD_PRIO_PROTECT", header: "<pthread.h>".}: cint +var PTHREAD_PROCESS_SHARED* {.importc: "PTHREAD_PROCESS_SHARED", header: "<pthread.h>".}: cint +var PTHREAD_PROCESS_PRIVATE* {.importc: "PTHREAD_PROCESS_PRIVATE", header: "<pthread.h>".}: cint +var PTHREAD_SCOPE_PROCESS* {.importc: "PTHREAD_SCOPE_PROCESS", header: "<pthread.h>".}: cint +var PTHREAD_SCOPE_SYSTEM* {.importc: "PTHREAD_SCOPE_SYSTEM", header: "<pthread.h>".}: cint + +# <sched.h> +var SCHED_FIFO* {.importc: "SCHED_FIFO", header: "<sched.h>".}: cint +var SCHED_RR* {.importc: "SCHED_RR", header: "<sched.h>".}: cint +var SCHED_SPORADIC* {.importc: "SCHED_SPORADIC", header: "<sched.h>".}: cint +var SCHED_OTHER* {.importc: "SCHED_OTHER", header: "<sched.h>".}: cint + +# <semaphore.h> +var SEM_FAILED* {.importc: "SEM_FAILED", header: "<semaphore.h>".}: pointer + +# <signal.h> +var SIGEV_NONE* {.importc: "SIGEV_NONE", header: "<signal.h>".}: cint +var SIGEV_SIGNAL* {.importc: "SIGEV_SIGNAL", header: "<signal.h>".}: cint +var SIGEV_THREAD* {.importc: "SIGEV_THREAD", header: "<signal.h>".}: cint +var SIGABRT* {.importc: "SIGABRT", header: "<signal.h>".}: cint +var SIGALRM* {.importc: "SIGALRM", header: "<signal.h>".}: cint +var SIGBUS* {.importc: "SIGBUS", header: "<signal.h>".}: cint +var SIGCHLD* {.importc: "SIGCHLD", header: "<signal.h>".}: cint +var SIGCONT* {.importc: "SIGCONT", header: "<signal.h>".}: cint +var SIGFPE* {.importc: "SIGFPE", header: "<signal.h>".}: cint +var SIGHUP* {.importc: "SIGHUP", header: "<signal.h>".}: cint +var SIGILL* {.importc: "SIGILL", header: "<signal.h>".}: cint +var SIGINT* {.importc: "SIGINT", header: "<signal.h>".}: cint +var SIGKILL* {.importc: "SIGKILL", header: "<signal.h>".}: cint +var SIGPIPE* {.importc: "SIGPIPE", header: "<signal.h>".}: cint +var SIGQUIT* {.importc: "SIGQUIT", header: "<signal.h>".}: cint +var SIGSEGV* {.importc: "SIGSEGV", header: "<signal.h>".}: cint +var SIGSTOP* {.importc: "SIGSTOP", header: "<signal.h>".}: cint +var SIGTERM* {.importc: "SIGTERM", header: "<signal.h>".}: cint +var SIGTSTP* {.importc: "SIGTSTP", header: "<signal.h>".}: cint +var SIGTTIN* {.importc: "SIGTTIN", header: "<signal.h>".}: cint +var SIGTTOU* {.importc: "SIGTTOU", header: "<signal.h>".}: cint +var SIGUSR1* {.importc: "SIGUSR1", header: "<signal.h>".}: cint +var SIGUSR2* {.importc: "SIGUSR2", header: "<signal.h>".}: cint +var SIGPOLL* {.importc: "SIGPOLL", header: "<signal.h>".}: cint +var SIGPROF* {.importc: "SIGPROF", header: "<signal.h>".}: cint +var SIGSYS* {.importc: "SIGSYS", header: "<signal.h>".}: cint +var SIGTRAP* {.importc: "SIGTRAP", header: "<signal.h>".}: cint +var SIGURG* {.importc: "SIGURG", header: "<signal.h>".}: cint +var SIGVTALRM* {.importc: "SIGVTALRM", header: "<signal.h>".}: cint +var SIGXCPU* {.importc: "SIGXCPU", header: "<signal.h>".}: cint +var SIGXFSZ* {.importc: "SIGXFSZ", header: "<signal.h>".}: cint +var SA_NOCLDSTOP* {.importc: "SA_NOCLDSTOP", header: "<signal.h>".}: cint +var SIG_BLOCK* {.importc: "SIG_BLOCK", header: "<signal.h>".}: cint +var SIG_UNBLOCK* {.importc: "SIG_UNBLOCK", header: "<signal.h>".}: cint +var SIG_SETMASK* {.importc: "SIG_SETMASK", header: "<signal.h>".}: cint +var SA_ONSTACK* {.importc: "SA_ONSTACK", header: "<signal.h>".}: cint +var SA_RESETHAND* {.importc: "SA_RESETHAND", header: "<signal.h>".}: cint +var SA_RESTART* {.importc: "SA_RESTART", header: "<signal.h>".}: cint +var SA_SIGINFO* {.importc: "SA_SIGINFO", header: "<signal.h>".}: cint +var SA_NOCLDWAIT* {.importc: "SA_NOCLDWAIT", header: "<signal.h>".}: cint +var SA_NODEFER* {.importc: "SA_NODEFER", header: "<signal.h>".}: cint +var SS_ONSTACK* {.importc: "SS_ONSTACK", header: "<signal.h>".}: cint +var SS_DISABLE* {.importc: "SS_DISABLE", header: "<signal.h>".}: cint +var MINSIGSTKSZ* {.importc: "MINSIGSTKSZ", header: "<signal.h>".}: cint +var SIGSTKSZ* {.importc: "SIGSTKSZ", header: "<signal.h>".}: cint +var SIG_HOLD* {.importc: "SIG_HOLD", header: "<signal.h>".}: Sighandler +var SIG_DFL* {.importc: "SIG_DFL", header: "<signal.h>".}: Sighandler +var SIG_ERR* {.importc: "SIG_ERR", header: "<signal.h>".}: Sighandler +var SIG_IGN* {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler + +# <sys/ipc.h> +var IPC_CREAT* {.importc: "IPC_CREAT", header: "<sys/ipc.h>".}: cint +var IPC_EXCL* {.importc: "IPC_EXCL", header: "<sys/ipc.h>".}: cint +var IPC_NOWAIT* {.importc: "IPC_NOWAIT", header: "<sys/ipc.h>".}: cint +var IPC_PRIVATE* {.importc: "IPC_PRIVATE", header: "<sys/ipc.h>".}: cint +var IPC_RMID* {.importc: "IPC_RMID", header: "<sys/ipc.h>".}: cint +var IPC_SET* {.importc: "IPC_SET", header: "<sys/ipc.h>".}: cint +var IPC_STAT* {.importc: "IPC_STAT", header: "<sys/ipc.h>".}: cint + +# <sys/mman.h> +var PROT_READ* {.importc: "PROT_READ", header: "<sys/mman.h>".}: cint +var PROT_WRITE* {.importc: "PROT_WRITE", header: "<sys/mman.h>".}: cint +var PROT_EXEC* {.importc: "PROT_EXEC", header: "<sys/mman.h>".}: cint +var PROT_NONE* {.importc: "PROT_NONE", header: "<sys/mman.h>".}: cint +var MAP_ANONYMOUS* {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint +var MAP_FIXED_NOREPLACE* {.importc: "MAP_FIXED_NOREPLACE", header: "<sys/mman.h>".}: cint +var MAP_NORESERVE* {.importc: "MAP_NORESERVE", header: "<sys/mman.h>".}: cint +var MAP_SHARED* {.importc: "MAP_SHARED", header: "<sys/mman.h>".}: cint +var MAP_PRIVATE* {.importc: "MAP_PRIVATE", header: "<sys/mman.h>".}: cint +var MAP_FIXED* {.importc: "MAP_FIXED", header: "<sys/mman.h>".}: cint +var MS_ASYNC* {.importc: "MS_ASYNC", header: "<sys/mman.h>".}: cint +var MS_SYNC* {.importc: "MS_SYNC", header: "<sys/mman.h>".}: cint +var MS_INVALIDATE* {.importc: "MS_INVALIDATE", header: "<sys/mman.h>".}: cint +var MCL_CURRENT* {.importc: "MCL_CURRENT", header: "<sys/mman.h>".}: cint +var MCL_FUTURE* {.importc: "MCL_FUTURE", header: "<sys/mman.h>".}: cint +var MAP_FAILED* {.importc: "MAP_FAILED", header: "<sys/mman.h>".}: pointer +var POSIX_MADV_NORMAL* {.importc: "POSIX_MADV_NORMAL", header: "<sys/mman.h>".}: cint +var POSIX_MADV_SEQUENTIAL* {.importc: "POSIX_MADV_SEQUENTIAL", header: "<sys/mman.h>".}: cint +var POSIX_MADV_RANDOM* {.importc: "POSIX_MADV_RANDOM", header: "<sys/mman.h>".}: cint +var POSIX_MADV_WILLNEED* {.importc: "POSIX_MADV_WILLNEED", header: "<sys/mman.h>".}: cint +var POSIX_MADV_DONTNEED* {.importc: "POSIX_MADV_DONTNEED", header: "<sys/mman.h>".}: cint +var POSIX_TYPED_MEM_ALLOCATE* {.importc: "POSIX_TYPED_MEM_ALLOCATE", header: "<sys/mman.h>".}: cint +var POSIX_TYPED_MEM_ALLOCATE_CONTIG* {.importc: "POSIX_TYPED_MEM_ALLOCATE_CONTIG", header: "<sys/mman.h>".}: cint +var POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc: "POSIX_TYPED_MEM_MAP_ALLOCATABLE", header: "<sys/mman.h>".}: cint + +# <sys/resource.h> +var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint +var RLIMIT_STACK* {.importc: "RLIMIT_STACK", header: "<sys/resource.h>".}: cint + +# <sys/select.h> +var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint +when defined(zephyr): + # Zephyr specific hardcoded value + var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "<sys/select.h>".}: cint + +# <sys/socket.h> +var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "<sys/socket.h>".}: cint +var MSG_DONTROUTE* {.importc: "MSG_DONTROUTE", header: "<sys/socket.h>".}: cint +var MSG_EOR* {.importc: "MSG_EOR", header: "<sys/socket.h>".}: cint +var MSG_OOB* {.importc: "MSG_OOB", header: "<sys/socket.h>".}: cint +var SCM_RIGHTS* {.importc: "SCM_RIGHTS", header: "<sys/socket.h>".}: cint +var SO_ACCEPTCONN* {.importc: "SO_ACCEPTCONN", header: "<sys/socket.h>".}: cint +var SO_BINDTODEVICE* {.importc: "SO_BINDTODEVICE", header: "<sys/socket.h>".}: cint +var SO_BROADCAST* {.importc: "SO_BROADCAST", header: "<sys/socket.h>".}: cint +var SO_DEBUG* {.importc: "SO_DEBUG", header: "<sys/socket.h>".}: cint +var SO_DONTROUTE* {.importc: "SO_DONTROUTE", header: "<sys/socket.h>".}: cint +var SO_ERROR* {.importc: "SO_ERROR", header: "<sys/socket.h>".}: cint +var SO_KEEPALIVE* {.importc: "SO_KEEPALIVE", header: "<sys/socket.h>".}: cint +var SO_LINGER* {.importc: "SO_LINGER", header: "<sys/socket.h>".}: cint +var SO_OOBINLINE* {.importc: "SO_OOBINLINE", header: "<sys/socket.h>".}: cint +var SO_RCVBUF* {.importc: "SO_RCVBUF", header: "<sys/socket.h>".}: cint +var SO_RCVLOWAT* {.importc: "SO_RCVLOWAT", header: "<sys/socket.h>".}: cint +var SO_RCVTIMEO* {.importc: "SO_RCVTIMEO", header: "<sys/socket.h>".}: cint +var SO_REUSEADDR* {.importc: "SO_REUSEADDR", header: "<sys/socket.h>".}: cint +var SO_SNDBUF* {.importc: "SO_SNDBUF", header: "<sys/socket.h>".}: cint +var SO_SNDLOWAT* {.importc: "SO_SNDLOWAT", header: "<sys/socket.h>".}: cint +var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "<sys/socket.h>".}: cint +var SO_TYPE* {.importc: "SO_TYPE", header: "<sys/socket.h>".}: cint +var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "<sys/socket.h>".}: cint +var SOCK_RAW* {.importc: "SOCK_RAW", header: "<sys/socket.h>".}: cint +when defined(zephyr): + const SOCK_SEQPACKET* = cint(5) + var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "<sys/socket.h>".}: cint +else: + var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint + var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint + +var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "<sys/socket.h>".}: cint +var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "<sys/socket.h>".}: cint +var MSG_PEEK* {.importc: "MSG_PEEK", header: "<sys/socket.h>".}: cint +var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "<sys/socket.h>".}: cint +var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "<sys/socket.h>".}: cint +var AF_INET* {.importc: "AF_INET", header: "<sys/socket.h>".}: cint +var AF_INET6* {.importc: "AF_INET6", header: "<sys/socket.h>".}: cint +var AF_UNIX* {.importc: "AF_UNIX", header: "<sys/socket.h>".}: cint +var AF_UNSPEC* {.importc: "AF_UNSPEC", header: "<sys/socket.h>".}: cint +var SHUT_RD* {.importc: "SHUT_RD", header: "<sys/socket.h>".}: cint +var SHUT_RDWR* {.importc: "SHUT_RDWR", header: "<sys/socket.h>".}: cint +var SHUT_WR* {.importc: "SHUT_WR", header: "<sys/socket.h>".}: cint + +# <sys/stat.h> +var S_IFBLK* {.importc: "S_IFBLK", header: "<sys/stat.h>".}: cint +var S_IFCHR* {.importc: "S_IFCHR", header: "<sys/stat.h>".}: cint +var S_IFDIR* {.importc: "S_IFDIR", header: "<sys/stat.h>".}: cint +var S_IFIFO* {.importc: "S_IFIFO", header: "<sys/stat.h>".}: cint +var S_IFLNK* {.importc: "S_IFLNK", header: "<sys/stat.h>".}: cint +var S_IFMT* {.importc: "S_IFMT", header: "<sys/stat.h>".}: cint +var S_IFREG* {.importc: "S_IFREG", header: "<sys/stat.h>".}: cint +var S_IFSOCK* {.importc: "S_IFSOCK", header: "<sys/stat.h>".}: cint +var S_IRGRP* {.importc: "S_IRGRP", header: "<sys/stat.h>".}: cint +var S_IROTH* {.importc: "S_IROTH", header: "<sys/stat.h>".}: cint +var S_IRUSR* {.importc: "S_IRUSR", header: "<sys/stat.h>".}: cint +var S_IRWXG* {.importc: "S_IRWXG", header: "<sys/stat.h>".}: cint +var S_IRWXO* {.importc: "S_IRWXO", header: "<sys/stat.h>".}: cint +var S_IRWXU* {.importc: "S_IRWXU", header: "<sys/stat.h>".}: cint +var S_ISGID* {.importc: "S_ISGID", header: "<sys/stat.h>".}: cint +var S_ISUID* {.importc: "S_ISUID", header: "<sys/stat.h>".}: cint +var S_ISVTX* {.importc: "S_ISVTX", header: "<sys/stat.h>".}: cint +var S_IWGRP* {.importc: "S_IWGRP", header: "<sys/stat.h>".}: cint +var S_IWOTH* {.importc: "S_IWOTH", header: "<sys/stat.h>".}: cint +var S_IWUSR* {.importc: "S_IWUSR", header: "<sys/stat.h>".}: cint +var S_IXGRP* {.importc: "S_IXGRP", header: "<sys/stat.h>".}: cint +var S_IXOTH* {.importc: "S_IXOTH", header: "<sys/stat.h>".}: cint +var S_IXUSR* {.importc: "S_IXUSR", header: "<sys/stat.h>".}: cint + +# <sys/statvfs.h> +var ST_RDONLY* {.importc: "ST_RDONLY", header: "<sys/statvfs.h>".}: cint +var ST_NOSUID* {.importc: "ST_NOSUID", header: "<sys/statvfs.h>".}: cint + +# <sys/wait.h> +var WNOHANG* {.importc: "WNOHANG", header: "<sys/wait.h>".}: cint +var WUNTRACED* {.importc: "WUNTRACED", header: "<sys/wait.h>".}: cint +var WEXITED* {.importc: "WEXITED", header: "<sys/wait.h>".}: cint +var WSTOPPED* {.importc: "WSTOPPED", header: "<sys/wait.h>".}: cint +var WCONTINUED* {.importc: "WCONTINUED", header: "<sys/wait.h>".}: cint +var WNOWAIT* {.importc: "WNOWAIT", header: "<sys/wait.h>".}: cint +var P_ALL* {.importc: "P_ALL", header: "<sys/wait.h>".}: cint +var P_PID* {.importc: "P_PID", header: "<sys/wait.h>".}: cint +var P_PGID* {.importc: "P_PGID", header: "<sys/wait.h>".}: cint + +# <spawn.h> +var POSIX_SPAWN_RESETIDS* {.importc: "POSIX_SPAWN_RESETIDS", header: "<spawn.h>".}: cint +var POSIX_SPAWN_SETPGROUP* {.importc: "POSIX_SPAWN_SETPGROUP", header: "<spawn.h>".}: cint +var POSIX_SPAWN_SETSCHEDPARAM* {.importc: "POSIX_SPAWN_SETSCHEDPARAM", header: "<spawn.h>".}: cint +var POSIX_SPAWN_SETSCHEDULER* {.importc: "POSIX_SPAWN_SETSCHEDULER", header: "<spawn.h>".}: cint +var POSIX_SPAWN_SETSIGDEF* {.importc: "POSIX_SPAWN_SETSIGDEF", header: "<spawn.h>".}: cint +var POSIX_SPAWN_SETSIGMASK* {.importc: "POSIX_SPAWN_SETSIGMASK", header: "<spawn.h>".}: cint + +# <stdio.h> +var IOFBF* {.importc: "_IOFBF", header: "<stdio.h>".}: cint +var IONBF* {.importc: "_IONBF", header: "<stdio.h>".}: cint + +# <time.h> +var CLOCKS_PER_SEC* {.importc: "CLOCKS_PER_SEC", header: "<time.h>".}: clong +var CLOCK_PROCESS_CPUTIME_ID* {.importc: "CLOCK_PROCESS_CPUTIME_ID", header: "<time.h>".}: cint +var CLOCK_THREAD_CPUTIME_ID* {.importc: "CLOCK_THREAD_CPUTIME_ID", header: "<time.h>".}: cint +var CLOCK_REALTIME* {.importc: "CLOCK_REALTIME", header: "<time.h>".}: cint +var TIMER_ABSTIME* {.importc: "TIMER_ABSTIME", header: "<time.h>".}: cint +var CLOCK_MONOTONIC* {.importc: "CLOCK_MONOTONIC", header: "<time.h>".}: cint + +# <unistd.h> +var POSIX_ASYNC_IO* {.importc: "_POSIX_ASYNC_IO", header: "<unistd.h>".}: cint +var POSIX_PRIO_IO* {.importc: "_POSIX_PRIO_IO", header: "<unistd.h>".}: cint +var POSIX_SYNC_IO* {.importc: "_POSIX_SYNC_IO", header: "<unistd.h>".}: cint +var F_OK* {.importc: "F_OK", header: "<unistd.h>".}: cint +var R_OK* {.importc: "R_OK", header: "<unistd.h>".}: cint +var W_OK* {.importc: "W_OK", header: "<unistd.h>".}: cint +var X_OK* {.importc: "X_OK", header: "<unistd.h>".}: cint +var CS_PATH* {.importc: "_CS_PATH", header: "<unistd.h>".}: cint +var CS_POSIX_V6_ILP32_OFF32_CFLAGS* {.importc: "_CS_POSIX_V6_ILP32_OFF32_CFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_ILP32_OFF32_LDFLAGS* {.importc: "_CS_POSIX_V6_ILP32_OFF32_LDFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_ILP32_OFF32_LIBS* {.importc: "_CS_POSIX_V6_ILP32_OFF32_LIBS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_ILP32_OFFBIG_CFLAGS* {.importc: "_CS_POSIX_V6_ILP32_OFFBIG_CFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS* {.importc: "_CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_ILP32_OFFBIG_LIBS* {.importc: "_CS_POSIX_V6_ILP32_OFFBIG_LIBS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_LP64_OFF64_CFLAGS* {.importc: "_CS_POSIX_V6_LP64_OFF64_CFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_LP64_OFF64_LDFLAGS* {.importc: "_CS_POSIX_V6_LP64_OFF64_LDFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_LP64_OFF64_LIBS* {.importc: "_CS_POSIX_V6_LP64_OFF64_LIBS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS* {.importc: "_CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS* {.importc: "_CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_LPBIG_OFFBIG_LIBS* {.importc: "_CS_POSIX_V6_LPBIG_OFFBIG_LIBS", header: "<unistd.h>".}: cint +var CS_POSIX_V6_WIDTH_RESTRICTED_ENVS* {.importc: "_CS_POSIX_V6_WIDTH_RESTRICTED_ENVS", header: "<unistd.h>".}: cint +var F_LOCK* {.importc: "F_LOCK", header: "<unistd.h>".}: cint +var F_TEST* {.importc: "F_TEST", header: "<unistd.h>".}: cint +var F_TLOCK* {.importc: "F_TLOCK", header: "<unistd.h>".}: cint +var F_ULOCK* {.importc: "F_ULOCK", header: "<unistd.h>".}: cint +var PC_2_SYMLINKS* {.importc: "_PC_2_SYMLINKS", header: "<unistd.h>".}: cint +var PC_ALLOC_SIZE_MIN* {.importc: "_PC_ALLOC_SIZE_MIN", header: "<unistd.h>".}: cint +var PC_ASYNC_IO* {.importc: "_PC_ASYNC_IO", header: "<unistd.h>".}: cint +var PC_CHOWN_RESTRICTED* {.importc: "_PC_CHOWN_RESTRICTED", header: "<unistd.h>".}: cint +var PC_FILESIZEBITS* {.importc: "_PC_FILESIZEBITS", header: "<unistd.h>".}: cint +var PC_LINK_MAX* {.importc: "_PC_LINK_MAX", header: "<unistd.h>".}: cint +var PC_MAX_CANON* {.importc: "_PC_MAX_CANON", header: "<unistd.h>".}: cint +var PC_MAX_INPUT* {.importc: "_PC_MAX_INPUT", header: "<unistd.h>".}: cint +var PC_NAME_MAX* {.importc: "_PC_NAME_MAX", header: "<unistd.h>".}: cint +var PC_NO_TRUNC* {.importc: "_PC_NO_TRUNC", header: "<unistd.h>".}: cint +var PC_PATH_MAX* {.importc: "_PC_PATH_MAX", header: "<unistd.h>".}: cint +var PC_PIPE_BUF* {.importc: "_PC_PIPE_BUF", header: "<unistd.h>".}: cint +var PC_PRIO_IO* {.importc: "_PC_PRIO_IO", header: "<unistd.h>".}: cint +var PC_REC_INCR_XFER_SIZE* {.importc: "_PC_REC_INCR_XFER_SIZE", header: "<unistd.h>".}: cint +var PC_REC_MIN_XFER_SIZE* {.importc: "_PC_REC_MIN_XFER_SIZE", header: "<unistd.h>".}: cint +var PC_REC_XFER_ALIGN* {.importc: "_PC_REC_XFER_ALIGN", header: "<unistd.h>".}: cint +var PC_SYMLINK_MAX* {.importc: "_PC_SYMLINK_MAX", header: "<unistd.h>".}: cint +var PC_SYNC_IO* {.importc: "_PC_SYNC_IO", header: "<unistd.h>".}: cint +var PC_VDISABLE* {.importc: "_PC_VDISABLE", header: "<unistd.h>".}: cint +var SC_2_C_BIND* {.importc: "_SC_2_C_BIND", header: "<unistd.h>".}: cint +var SC_2_C_DEV* {.importc: "_SC_2_C_DEV", header: "<unistd.h>".}: cint +var SC_2_CHAR_TERM* {.importc: "_SC_2_CHAR_TERM", header: "<unistd.h>".}: cint +var SC_2_FORT_DEV* {.importc: "_SC_2_FORT_DEV", header: "<unistd.h>".}: cint +var SC_2_FORT_RUN* {.importc: "_SC_2_FORT_RUN", header: "<unistd.h>".}: cint +var SC_2_LOCALEDEF* {.importc: "_SC_2_LOCALEDEF", header: "<unistd.h>".}: cint +var SC_2_PBS* {.importc: "_SC_2_PBS", header: "<unistd.h>".}: cint +var SC_2_PBS_ACCOUNTING* {.importc: "_SC_2_PBS_ACCOUNTING", header: "<unistd.h>".}: cint +var SC_2_PBS_CHECKPOINT* {.importc: "_SC_2_PBS_CHECKPOINT", header: "<unistd.h>".}: cint +var SC_2_PBS_LOCATE* {.importc: "_SC_2_PBS_LOCATE", header: "<unistd.h>".}: cint +var SC_2_PBS_MESSAGE* {.importc: "_SC_2_PBS_MESSAGE", header: "<unistd.h>".}: cint +var SC_2_PBS_TRACK* {.importc: "_SC_2_PBS_TRACK", header: "<unistd.h>".}: cint +var SC_2_SW_DEV* {.importc: "_SC_2_SW_DEV", header: "<unistd.h>".}: cint +var SC_2_UPE* {.importc: "_SC_2_UPE", header: "<unistd.h>".}: cint +var SC_2_VERSION* {.importc: "_SC_2_VERSION", header: "<unistd.h>".}: cint +var SC_ADVISORY_INFO* {.importc: "_SC_ADVISORY_INFO", header: "<unistd.h>".}: cint +var SC_AIO_LISTIO_MAX* {.importc: "_SC_AIO_LISTIO_MAX", header: "<unistd.h>".}: cint +var SC_AIO_MAX* {.importc: "_SC_AIO_MAX", header: "<unistd.h>".}: cint +var SC_AIO_PRIO_DELTA_MAX* {.importc: "_SC_AIO_PRIO_DELTA_MAX", header: "<unistd.h>".}: cint +var SC_ARG_MAX* {.importc: "_SC_ARG_MAX", header: "<unistd.h>".}: cint +var SC_ASYNCHRONOUS_IO* {.importc: "_SC_ASYNCHRONOUS_IO", header: "<unistd.h>".}: cint +var SC_ATEXIT_MAX* {.importc: "_SC_ATEXIT_MAX", header: "<unistd.h>".}: cint +var SC_BARRIERS* {.importc: "_SC_BARRIERS", header: "<unistd.h>".}: cint +var SC_BC_BASE_MAX* {.importc: "_SC_BC_BASE_MAX", header: "<unistd.h>".}: cint +var SC_BC_DIM_MAX* {.importc: "_SC_BC_DIM_MAX", header: "<unistd.h>".}: cint +var SC_BC_SCALE_MAX* {.importc: "_SC_BC_SCALE_MAX", header: "<unistd.h>".}: cint +var SC_BC_STRING_MAX* {.importc: "_SC_BC_STRING_MAX", header: "<unistd.h>".}: cint +var SC_CHILD_MAX* {.importc: "_SC_CHILD_MAX", header: "<unistd.h>".}: cint +var SC_CLK_TCK* {.importc: "_SC_CLK_TCK", header: "<unistd.h>".}: cint +var SC_CLOCK_SELECTION* {.importc: "_SC_CLOCK_SELECTION", header: "<unistd.h>".}: cint +var SC_COLL_WEIGHTS_MAX* {.importc: "_SC_COLL_WEIGHTS_MAX", header: "<unistd.h>".}: cint +var SC_CPUTIME* {.importc: "_SC_CPUTIME", header: "<unistd.h>".}: cint +var SC_DELAYTIMER_MAX* {.importc: "_SC_DELAYTIMER_MAX", header: "<unistd.h>".}: cint +var SC_EXPR_NEST_MAX* {.importc: "_SC_EXPR_NEST_MAX", header: "<unistd.h>".}: cint +var SC_FSYNC* {.importc: "_SC_FSYNC", header: "<unistd.h>".}: cint +var SC_GETGR_R_SIZE_MAX* {.importc: "_SC_GETGR_R_SIZE_MAX", header: "<unistd.h>".}: cint +var SC_GETPW_R_SIZE_MAX* {.importc: "_SC_GETPW_R_SIZE_MAX", header: "<unistd.h>".}: cint +var SC_HOST_NAME_MAX* {.importc: "_SC_HOST_NAME_MAX", header: "<unistd.h>".}: cint +var SC_IOV_MAX* {.importc: "_SC_IOV_MAX", header: "<unistd.h>".}: cint +var SC_IPV6* {.importc: "_SC_IPV6", header: "<unistd.h>".}: cint +var SC_JOB_CONTROL* {.importc: "_SC_JOB_CONTROL", header: "<unistd.h>".}: cint +var SC_LINE_MAX* {.importc: "_SC_LINE_MAX", header: "<unistd.h>".}: cint +var SC_LOGIN_NAME_MAX* {.importc: "_SC_LOGIN_NAME_MAX", header: "<unistd.h>".}: cint +var SC_MAPPED_FILES* {.importc: "_SC_MAPPED_FILES", header: "<unistd.h>".}: cint +var SC_MEMLOCK* {.importc: "_SC_MEMLOCK", header: "<unistd.h>".}: cint +var SC_MEMLOCK_RANGE* {.importc: "_SC_MEMLOCK_RANGE", header: "<unistd.h>".}: cint +var SC_MEMORY_PROTECTION* {.importc: "_SC_MEMORY_PROTECTION", header: "<unistd.h>".}: cint +var SC_MESSAGE_PASSING* {.importc: "_SC_MESSAGE_PASSING", header: "<unistd.h>".}: cint +var SC_MONOTONIC_CLOCK* {.importc: "_SC_MONOTONIC_CLOCK", header: "<unistd.h>".}: cint +var SC_MQ_OPEN_MAX* {.importc: "_SC_MQ_OPEN_MAX", header: "<unistd.h>".}: cint +var SC_MQ_PRIO_MAX* {.importc: "_SC_MQ_PRIO_MAX", header: "<unistd.h>".}: cint +var SC_NGROUPS_MAX* {.importc: "_SC_NGROUPS_MAX", header: "<unistd.h>".}: cint +var SC_OPEN_MAX* {.importc: "_SC_OPEN_MAX", header: "<unistd.h>".}: cint +var SC_PAGESIZE* {.importc: "_SC_PAGESIZE", header: "<unistd.h>".}: cint +var SC_PRIORITIZED_IO* {.importc: "_SC_PRIORITIZED_IO", header: "<unistd.h>".}: cint +var SC_PRIORITY_SCHEDULING* {.importc: "_SC_PRIORITY_SCHEDULING", header: "<unistd.h>".}: cint +var SC_RAW_SOCKETS* {.importc: "_SC_RAW_SOCKETS", header: "<unistd.h>".}: cint +var SC_RE_DUP_MAX* {.importc: "_SC_RE_DUP_MAX", header: "<unistd.h>".}: cint +var SC_READER_WRITER_LOCKS* {.importc: "_SC_READER_WRITER_LOCKS", header: "<unistd.h>".}: cint +var SC_REALTIME_SIGNALS* {.importc: "_SC_REALTIME_SIGNALS", header: "<unistd.h>".}: cint +var SC_REGEXP* {.importc: "_SC_REGEXP", header: "<unistd.h>".}: cint +var SC_RTSIG_MAX* {.importc: "_SC_RTSIG_MAX", header: "<unistd.h>".}: cint +var SC_SAVED_IDS* {.importc: "_SC_SAVED_IDS", header: "<unistd.h>".}: cint +var SC_SEM_NSEMS_MAX* {.importc: "_SC_SEM_NSEMS_MAX", header: "<unistd.h>".}: cint +var SC_SEM_VALUE_MAX* {.importc: "_SC_SEM_VALUE_MAX", header: "<unistd.h>".}: cint +var SC_SEMAPHORES* {.importc: "_SC_SEMAPHORES", header: "<unistd.h>".}: cint +var SC_SHARED_MEMORY_OBJECTS* {.importc: "_SC_SHARED_MEMORY_OBJECTS", header: "<unistd.h>".}: cint +var SC_SHELL* {.importc: "_SC_SHELL", header: "<unistd.h>".}: cint +var SC_SIGQUEUE_MAX* {.importc: "_SC_SIGQUEUE_MAX", header: "<unistd.h>".}: cint +var SC_SPAWN* {.importc: "_SC_SPAWN", header: "<unistd.h>".}: cint +var SC_SPIN_LOCKS* {.importc: "_SC_SPIN_LOCKS", header: "<unistd.h>".}: cint +var SC_SPORADIC_SERVER* {.importc: "_SC_SPORADIC_SERVER", header: "<unistd.h>".}: cint +var SC_SS_REPL_MAX* {.importc: "_SC_SS_REPL_MAX", header: "<unistd.h>".}: cint +var SC_STREAM_MAX* {.importc: "_SC_STREAM_MAX", header: "<unistd.h>".}: cint +var SC_SYMLOOP_MAX* {.importc: "_SC_SYMLOOP_MAX", header: "<unistd.h>".}: cint +var SC_SYNCHRONIZED_IO* {.importc: "_SC_SYNCHRONIZED_IO", header: "<unistd.h>".}: cint +var SC_THREAD_ATTR_STACKADDR* {.importc: "_SC_THREAD_ATTR_STACKADDR", header: "<unistd.h>".}: cint +var SC_THREAD_ATTR_STACKSIZE* {.importc: "_SC_THREAD_ATTR_STACKSIZE", header: "<unistd.h>".}: cint +var SC_THREAD_CPUTIME* {.importc: "_SC_THREAD_CPUTIME", header: "<unistd.h>".}: cint +var SC_THREAD_DESTRUCTOR_ITERATIONS* {.importc: "_SC_THREAD_DESTRUCTOR_ITERATIONS", header: "<unistd.h>".}: cint +var SC_THREAD_KEYS_MAX* {.importc: "_SC_THREAD_KEYS_MAX", header: "<unistd.h>".}: cint +var SC_THREAD_PRIO_INHERIT* {.importc: "_SC_THREAD_PRIO_INHERIT", header: "<unistd.h>".}: cint +var SC_THREAD_PRIO_PROTECT* {.importc: "_SC_THREAD_PRIO_PROTECT", header: "<unistd.h>".}: cint +var SC_THREAD_PRIORITY_SCHEDULING* {.importc: "_SC_THREAD_PRIORITY_SCHEDULING", header: "<unistd.h>".}: cint +var SC_THREAD_PROCESS_SHARED* {.importc: "_SC_THREAD_PROCESS_SHARED", header: "<unistd.h>".}: cint +var SC_THREAD_SAFE_FUNCTIONS* {.importc: "_SC_THREAD_SAFE_FUNCTIONS", header: "<unistd.h>".}: cint +var SC_THREAD_SPORADIC_SERVER* {.importc: "_SC_THREAD_SPORADIC_SERVER", header: "<unistd.h>".}: cint +var SC_THREAD_STACK_MIN* {.importc: "_SC_THREAD_STACK_MIN", header: "<unistd.h>".}: cint +var SC_THREAD_THREADS_MAX* {.importc: "_SC_THREAD_THREADS_MAX", header: "<unistd.h>".}: cint +var SC_THREADS* {.importc: "_SC_THREADS", header: "<unistd.h>".}: cint +var SC_TIMEOUTS* {.importc: "_SC_TIMEOUTS", header: "<unistd.h>".}: cint +var SC_TIMER_MAX* {.importc: "_SC_TIMER_MAX", header: "<unistd.h>".}: cint +var SC_TIMERS* {.importc: "_SC_TIMERS", header: "<unistd.h>".}: cint +var SC_TRACE* {.importc: "_SC_TRACE", header: "<unistd.h>".}: cint +var SC_TRACE_EVENT_FILTER* {.importc: "_SC_TRACE_EVENT_FILTER", header: "<unistd.h>".}: cint +var SC_TRACE_EVENT_NAME_MAX* {.importc: "_SC_TRACE_EVENT_NAME_MAX", header: "<unistd.h>".}: cint +var SC_TRACE_INHERIT* {.importc: "_SC_TRACE_INHERIT", header: "<unistd.h>".}: cint +var SC_TRACE_LOG* {.importc: "_SC_TRACE_LOG", header: "<unistd.h>".}: cint +var SC_TRACE_NAME_MAX* {.importc: "_SC_TRACE_NAME_MAX", header: "<unistd.h>".}: cint +var SC_TRACE_SYS_MAX* {.importc: "_SC_TRACE_SYS_MAX", header: "<unistd.h>".}: cint +var SC_TRACE_USER_EVENT_MAX* {.importc: "_SC_TRACE_USER_EVENT_MAX", header: "<unistd.h>".}: cint +var SC_TTY_NAME_MAX* {.importc: "_SC_TTY_NAME_MAX", header: "<unistd.h>".}: cint +var SC_TYPED_MEMORY_OBJECTS* {.importc: "_SC_TYPED_MEMORY_OBJECTS", header: "<unistd.h>".}: cint +var SC_TZNAME_MAX* {.importc: "_SC_TZNAME_MAX", header: "<unistd.h>".}: cint +var SC_V6_ILP32_OFF32* {.importc: "_SC_V6_ILP32_OFF32", header: "<unistd.h>".}: cint +var SC_V6_ILP32_OFFBIG* {.importc: "_SC_V6_ILP32_OFFBIG", header: "<unistd.h>".}: cint +var SC_V6_LP64_OFF64* {.importc: "_SC_V6_LP64_OFF64", header: "<unistd.h>".}: cint +var SC_V6_LPBIG_OFFBIG* {.importc: "_SC_V6_LPBIG_OFFBIG", header: "<unistd.h>".}: cint +var SC_VERSION* {.importc: "_SC_VERSION", header: "<unistd.h>".}: cint +var SC_XBS5_ILP32_OFF32* {.importc: "_SC_XBS5_ILP32_OFF32", header: "<unistd.h>".}: cint +var SC_XBS5_ILP32_OFFBIG* {.importc: "_SC_XBS5_ILP32_OFFBIG", header: "<unistd.h>".}: cint +var SC_XBS5_LP64_OFF64* {.importc: "_SC_XBS5_LP64_OFF64", header: "<unistd.h>".}: cint +var SC_XBS5_LPBIG_OFFBIG* {.importc: "_SC_XBS5_LPBIG_OFFBIG", header: "<unistd.h>".}: cint +var SC_XOPEN_CRYPT* {.importc: "_SC_XOPEN_CRYPT", header: "<unistd.h>".}: cint +var SC_XOPEN_ENH_I18N* {.importc: "_SC_XOPEN_ENH_I18N", header: "<unistd.h>".}: cint +var SC_XOPEN_LEGACY* {.importc: "_SC_XOPEN_LEGACY", header: "<unistd.h>".}: cint +var SC_XOPEN_REALTIME* {.importc: "_SC_XOPEN_REALTIME", header: "<unistd.h>".}: cint +var SC_XOPEN_REALTIME_THREADS* {.importc: "_SC_XOPEN_REALTIME_THREADS", header: "<unistd.h>".}: cint +var SC_XOPEN_SHM* {.importc: "_SC_XOPEN_SHM", header: "<unistd.h>".}: cint +var SC_XOPEN_STREAMS* {.importc: "_SC_XOPEN_STREAMS", header: "<unistd.h>".}: cint +var SC_XOPEN_UNIX* {.importc: "_SC_XOPEN_UNIX", header: "<unistd.h>".}: cint +var SC_XOPEN_VERSION* {.importc: "_SC_XOPEN_VERSION", header: "<unistd.h>".}: cint +var SC_NPROCESSORS_ONLN* {.importc: "_SC_NPROCESSORS_ONLN", header: "<unistd.h>".}: cint +var SEEK_SET* {.importc: "SEEK_SET", header: "<unistd.h>".}: cint +var SEEK_CUR* {.importc: "SEEK_CUR", header: "<unistd.h>".}: cint +var SEEK_END* {.importc: "SEEK_END", header: "<unistd.h>".}: cint + +# <nuttx/config.h> +when defined(nuttx): + var NEPOLL_MAX* {.importc: "CONFIG_FS_NEPOLL_DESCRIPTORS", header: "<nuttx/config.h>".}: cint diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim new file mode 100644 index 000000000..0c668246f --- /dev/null +++ b/lib/posix/posix_utils.nim @@ -0,0 +1,133 @@ +# +# Nim's Runtime Library +# (c) Copyright 2019 Federico Ceratto and other Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## A set of helpers for the POSIX module. +## Raw interfaces are in the other ``posix*.nim`` files. + +# Where possible, contribute OS-independent procs in `os <os.html>`_ instead. + +import std/[posix, parsecfg, os] +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/syncio + +type Uname* = object + sysname*, nodename*, release*, version*, machine*: string + +template charArrayToString(input: typed): string = + $cast[cstring](addr input) + +proc uname*(): Uname = + ## Provides system information in a `Uname` struct with sysname, nodename, + ## release, version and machine attributes. + + when defined(posix): + runnableExamples: + echo uname().nodename, uname().release, uname().version + doAssert uname().sysname.len != 0 + + var u: Utsname + if uname(u) != 0: + raiseOSError(OSErrorCode(errno)) + + result.sysname = charArrayToString u.sysname + result.nodename = charArrayToString u.nodename + result.release = charArrayToString u.release + result.version = charArrayToString u.version + result.machine = charArrayToString u.machine + +proc fsync*(fd: int) = + ## synchronize a file's buffer cache to the storage device + if fsync(fd.cint) != 0: + raiseOSError(OSErrorCode(errno)) + +proc stat*(path: string): Stat = + ## Returns file status in a `Stat` structure + if stat(path.cstring, result) != 0: + raiseOSError(OSErrorCode(errno)) + +proc memoryLock*(a1: pointer, a2: int) = + ## Locks pages starting from a1 for a1 bytes and prevent them from being swapped. + if mlock(a1, a2) != 0: + raiseOSError(OSErrorCode(errno)) + +proc memoryLockAll*(flags: int) = + ## Locks all memory for the running process to prevent swapping. + ## + ## example: + ## ```nim + ## memoryLockAll(MCL_CURRENT or MCL_FUTURE) + ## ``` + if mlockall(flags.cint) != 0: + raiseOSError(OSErrorCode(errno)) + +proc memoryUnlock*(a1: pointer, a2: int) = + ## Unlock pages starting from a1 for a1 bytes and allow them to be swapped. + if munlock(a1, a2) != 0: + raiseOSError(OSErrorCode(errno)) + +proc memoryUnlockAll*() = + ## Unlocks all memory for the running process to allow swapping. + if munlockall() != 0: + raiseOSError(OSErrorCode(errno)) + +proc sendSignal*(pid: Pid, signal: int) = + ## Sends a signal to a running process by calling `kill`. + ## Raise exception in case of failure e.g. process not running. + if kill(pid, signal.cint) != 0: + raiseOSError(OSErrorCode(errno)) + +proc mkstemp*(prefix: string, suffix=""): (string, File) = + ## Creates a unique temporary file from a prefix string. A six-character string + ## will be added. If suffix is provided it will be added to the string + ## The file is created with perms 0600. + ## Returns the filename and a file opened in r/w mode. + var tmpl = cstring(prefix & "XXXXXX" & suffix) + let fd = + if len(suffix) == 0: + when declared(mkostemp): + mkostemp(tmpl, O_CLOEXEC) + else: + mkstemp(tmpl) + else: + when declared(mkostemps): + mkostemps(tmpl, cint(len(suffix)), O_CLOEXEC) + else: + mkstemps(tmpl, cint(len(suffix))) + var f: File + if open(f, fd, fmReadWrite): + return ($tmpl, f) + raiseOSError(OSErrorCode(errno)) + +proc mkdtemp*(prefix: string): string = + ## Creates a unique temporary directory from a prefix string. Adds a six chars suffix. + ## The directory is created with permissions 0700. Returns the directory name. + var tmpl = cstring(prefix & "XXXXXX") + if mkdtemp(tmpl) == nil: + raiseOSError(OSErrorCode(errno)) + return $tmpl + +proc osReleaseFile*(): Config {.since: (1, 5).} = + ## Gets system identification from `os-release` file and returns it as a `parsecfg.Config`. + ## You also need to import the `parsecfg` module to gain access to this object. + ## The `os-release` file is an official Freedesktop.org open standard. + ## Available in Linux and BSD distributions, except Android and Android-based Linux. + ## `os-release` file is not available on Windows and OS X by design. + ## * https://www.freedesktop.org/software/systemd/man/os-release.html + runnableExamples: + import std/parsecfg + when defined(linux): + let data = osReleaseFile() + echo "OS name: ", data.getSectionValue("", "NAME") ## the data is up to each distro. + + # We do not use a {.strdefine.} because Standard says it *must* be that path. + for osReleaseFile in ["/etc/os-release", "/usr/lib/os-release"]: + if fileExists(osReleaseFile): + return loadConfig(osReleaseFile) + raise newException(IOError, "File not found: /etc/os-release, /usr/lib/os-release") diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index c3934c6a9..7fb6bb81c 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -7,200 +7,187 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} -import posix +import std/posix type Speed* = cuint Cflag* = cuint -{.deprecated: [Tcflag: Cflag].} const - NCCS* = 32 - -type - Termios* {.importc: "struct termios", header: "<termios.h>".} = object - c_iflag*: Cflag # input mode flags - c_oflag*: Cflag # output mode flags - c_cflag*: Cflag # control mode flags - c_lflag*: Cflag # local mode flags - c_cc*: array[NCCS, cuchar] # control characters + NCCS* = when defined(macosx): 20 else: 32 + +when defined(linux) and defined(amd64): + type + Termios* {.importc: "struct termios", header: "<termios.h>".} = object + c_iflag*: Cflag # input mode flags + c_oflag*: Cflag # output mode flags + c_cflag*: Cflag # control mode flags + c_lflag*: Cflag # local mode flags + c_line*: cuchar + c_cc*: array[NCCS, cuchar] # control characters + c_ispeed*: Speed + c_ospeed*: Speed +else: + type + Termios* {.importc: "struct termios", header: "<termios.h>".} = object + c_iflag*: Cflag # input mode flags + c_oflag*: Cflag # output mode flags + c_cflag*: Cflag # control mode flags + c_lflag*: Cflag # local mode flags + c_cc*: array[NCCS, cuchar] # control characters # cc characters -const - VINTR* = 0 - VQUIT* = 1 - VERASE* = 2 - VKILL* = 3 - VEOF* = 4 - VTIME* = 5 - VMIN* = 6 - VSWTC* = 7 - VSTART* = 8 - VSTOP* = 9 - VSUSP* = 10 - VEOL* = 11 - VREPRINT* = 12 - VDISCARD* = 13 - VWERASE* = 14 - VLNEXT* = 15 - VEOL2* = 16 +var + VINTR* {.importc, header: "<termios.h>".}: cint + VQUIT* {.importc, header: "<termios.h>".}: cint + VERASE* {.importc, header: "<termios.h>".}: cint + VKILL* {.importc, header: "<termios.h>".}: cint + VEOF* {.importc, header: "<termios.h>".}: cint + VTIME* {.importc, header: "<termios.h>".}: cint + VMIN* {.importc, header: "<termios.h>".}: cint + VSTART* {.importc, header: "<termios.h>".}: cint + VSTOP* {.importc, header: "<termios.h>".}: cint + VSUSP* {.importc, header: "<termios.h>".}: cint + VEOL* {.importc, header: "<termios.h>".}: cint # iflag bits -const - IGNBRK* = 1 - BRKINT* = 2 - IGNPAR* = 4 - PARMRK* = 10 - INPCK* = 20 - ISTRIP* = 40 - INLCR* = 100 - IGNCR* = 200 - ICRNL* = 400 - IUCLC* = 1000 - IXON* = 2000 - IXANY* = 4000 - IXOFF* = 10000 - IMAXBEL* = 20000 - IUTF8* = 40000 +var + IGNBRK* {.importc, header: "<termios.h>".}: Cflag + BRKINT* {.importc, header: "<termios.h>".}: Cflag + IGNPAR* {.importc, header: "<termios.h>".}: Cflag + PARMRK* {.importc, header: "<termios.h>".}: Cflag + INPCK* {.importc, header: "<termios.h>".}: Cflag + ISTRIP* {.importc, header: "<termios.h>".}: Cflag + INLCR* {.importc, header: "<termios.h>".}: Cflag + IGNCR* {.importc, header: "<termios.h>".}: Cflag + ICRNL* {.importc, header: "<termios.h>".}: Cflag + IUCLC* {.importc, header: "<termios.h>".}: Cflag + IXON* {.importc, header: "<termios.h>".}: Cflag + IXANY* {.importc, header: "<termios.h>".}: Cflag + IXOFF* {.importc, header: "<termios.h>".}: Cflag # oflag bits -const - OPOST* = 1 - OLCUC* = 2 - ONLCR* = 4 - OCRNL* = 10 - ONOCR* = 20 - ONLRET* = 40 - OFILL* = 100 - OFDEL* = 200 - NLDLY* = 400 - NL0* = 0 - NL1* = 400 - CRDLY* = 3000 - CR0* = 0 - CR1* = 1000 - CR2* = 2000 - CR3* = 3000 - TABDLY* = 14000 - TAB0* = 0 - TAB1* = 4000 - TAB2* = 10000 - TAB3* = 14000 - BSDLY* = 20000 - BS0* = 0 - BS1* = 20000 - FFDLY* = 0o000000100000 - FF0* = 0 - FF1* = 0o000000100000 - VTDLY* = 40000 - VT0* = 0 - VT1* = 40000 - XTABS* = 14000 +var + OPOST* {.importc, header: "<termios.h>".}: Cflag + ONLCR* {.importc, header: "<termios.h>".}: Cflag + OCRNL* {.importc, header: "<termios.h>".}: Cflag + ONOCR* {.importc, header: "<termios.h>".}: Cflag + ONLRET* {.importc, header: "<termios.h>".}: Cflag + OFILL* {.importc, header: "<termios.h>".}: Cflag + OFDEL* {.importc, header: "<termios.h>".}: Cflag + NLDLY* {.importc, header: "<termios.h>".}: Cflag + NL0* {.importc, header: "<termios.h>".}: Cflag + NL1* {.importc, header: "<termios.h>".}: Cflag + CRDLY* {.importc, header: "<termios.h>".}: Cflag + CR0* {.importc, header: "<termios.h>".}: Cflag + CR1* {.importc, header: "<termios.h>".}: Cflag + CR2* {.importc, header: "<termios.h>".}: Cflag + CR3* {.importc, header: "<termios.h>".}: Cflag + TABDLY* {.importc, header: "<termios.h>".}: Cflag + TAB0* {.importc, header: "<termios.h>".}: Cflag + TAB1* {.importc, header: "<termios.h>".}: Cflag + TAB2* {.importc, header: "<termios.h>".}: Cflag + TAB3* {.importc, header: "<termios.h>".}: Cflag + BSDLY* {.importc, header: "<termios.h>".}: Cflag + BS0* {.importc, header: "<termios.h>".}: Cflag + BS1* {.importc, header: "<termios.h>".}: Cflag + FFDLY* {.importc, header: "<termios.h>".}: Cflag + FF0* {.importc, header: "<termios.h>".}: Cflag + FF1* {.importc, header: "<termios.h>".}: Cflag + VTDLY* {.importc, header: "<termios.h>".}: Cflag + VT0* {.importc, header: "<termios.h>".}: Cflag + VT1* {.importc, header: "<termios.h>".}: Cflag # cflag bit meaning -const - CBAUD* = 10017 - B0* = 0 - B50* = 1 - B75* = 2 - B110* = 3 - B134* = 4 - B150* = 5 - B200* = 6 - B300* = 7 - B600* = 10 - B1200* = 11 - B1800* = 12 - B2400* = 13 - B4800* = 14 - B9600* = 15 - B19200* = 16 - B38400* = 17 - EXTA* = B19200 - EXTB* = B38400 - CSIZE* = 60 - CS5* = 0 - CS6* = 20 - CS7* = 40 - CS8* = 60 - CSTOPB* = 100 - CREAD* = 200 - PARENB* = 400 - PARODD* = 1000 - HUPCL* = 2000 - CLOCAL* = 4000 - CBAUDEX* = 10000 - B57600* = 10001 - B115200* = 10002 - B230400* = 10003 - B460800* = 10004 - B500000* = 10005 - B576000* = 10006 - B921600* = 10007 - B1000000* = 10010 - B1152000* = 10011 - B1500000* = 10012 - B2000000* = 10013 - B2500000* = 10014 - B3000000* = 10015 - B3500000* = 10016 - B4000000* = 10017 - MAX_BAUD* = B4000000 - CIBAUD* = 2003600000 - CMSPAR* = 0o010000000000 - CRTSCTS* = 0o020000000000 +var + B0* {.importc, header: "<termios.h>".}: Speed + B50* {.importc, header: "<termios.h>".}: Speed + B75* {.importc, header: "<termios.h>".}: Speed + B110* {.importc, header: "<termios.h>".}: Speed + B134* {.importc, header: "<termios.h>".}: Speed + B150* {.importc, header: "<termios.h>".}: Speed + B200* {.importc, header: "<termios.h>".}: Speed + B300* {.importc, header: "<termios.h>".}: Speed + B600* {.importc, header: "<termios.h>".}: Speed + B1200* {.importc, header: "<termios.h>".}: Speed + B1800* {.importc, header: "<termios.h>".}: Speed + B2400* {.importc, header: "<termios.h>".}: Speed + B4800* {.importc, header: "<termios.h>".}: Speed + B9600* {.importc, header: "<termios.h>".}: Speed + B19200* {.importc, header: "<termios.h>".}: Speed + B38400* {.importc, header: "<termios.h>".}: Speed + B57600* {.importc, header: "<termios.h>".}: Speed + B115200* {.importc, header: "<termios.h>".}: Speed + B230400* {.importc, header: "<termios.h>".}: Speed + B460800* {.importc, header: "<termios.h>".}: Speed + B500000* {.importc, header: "<termios.h>".}: Speed + B576000* {.importc, header: "<termios.h>".}: Speed + B921600* {.importc, header: "<termios.h>".}: Speed + B1000000* {.importc, header: "<termios.h>".}: Speed + B1152000* {.importc, header: "<termios.h>".}: Speed + B1500000* {.importc, header: "<termios.h>".}: Speed + B2000000* {.importc, header: "<termios.h>".}: Speed + B2500000* {.importc, header: "<termios.h>".}: Speed + B3000000* {.importc, header: "<termios.h>".}: Speed + B3500000* {.importc, header: "<termios.h>".}: Speed + B4000000* {.importc, header: "<termios.h>".}: Speed + EXTA* {.importc, header: "<termios.h>".}: Speed + EXTB* {.importc, header: "<termios.h>".}: Speed + CSIZE* {.importc, header: "<termios.h>".}: Cflag + CS5* {.importc, header: "<termios.h>".}: Cflag + CS6* {.importc, header: "<termios.h>".}: Cflag + CS7* {.importc, header: "<termios.h>".}: Cflag + CS8* {.importc, header: "<termios.h>".}: Cflag + CSTOPB* {.importc, header: "<termios.h>".}: Cflag + CREAD* {.importc, header: "<termios.h>".}: Cflag + PARENB* {.importc, header: "<termios.h>".}: Cflag + PARODD* {.importc, header: "<termios.h>".}: Cflag + HUPCL* {.importc, header: "<termios.h>".}: Cflag + CLOCAL* {.importc, header: "<termios.h>".}: Cflag # lflag bits -const - ISIG* = 1 - ICANON* = 2 - XCASE* = 4 - ECHO* = 10 - ECHOE* = 20 - ECHOK* = 40 - ECHONL* = 100 - NOFLSH* = 200 - TOSTOP* = 400 - ECHOCTL* = 1000 - ECHOPRT* = 2000 - ECHOKE* = 4000 - FLUSHO* = 10000 - PENDIN* = 40000 - IEXTEN* = 0o000000100000 - EXTPROC* = 0o000000200000 +var + ISIG* {.importc, header: "<termios.h>".}: Cflag + ICANON* {.importc, header: "<termios.h>".}: Cflag + ECHO* {.importc, header: "<termios.h>".}: Cflag + ECHOE* {.importc, header: "<termios.h>".}: Cflag + ECHOK* {.importc, header: "<termios.h>".}: Cflag + ECHONL* {.importc, header: "<termios.h>".}: Cflag + NOFLSH* {.importc, header: "<termios.h>".}: Cflag + TOSTOP* {.importc, header: "<termios.h>".}: Cflag + IEXTEN* {.importc, header: "<termios.h>".}: Cflag # tcflow() and TCXONC use these -const - TCOOFF* = 0 - TCOON* = 1 - TCIOFF* = 2 - TCION* = 3 +var + TCOOFF* {.importc, header: "<termios.h>".}: cint + TCOON* {.importc, header: "<termios.h>".}: cint + TCIOFF* {.importc, header: "<termios.h>".}: cint + TCION* {.importc, header: "<termios.h>".}: cint # tcflush() and TCFLSH use these -const - TCIFLUSH* = 0 - TCOFLUSH* = 1 - TCIOFLUSH* = 2 +var + TCIFLUSH* {.importc, header: "<termios.h>".}: cint + TCOFLUSH* {.importc, header: "<termios.h>".}: cint + TCIOFLUSH* {.importc, header: "<termios.h>".}: cint # tcsetattr uses these -const - TCSANOW* = 0 - TCSADRAIN* = 1 - TCSAFLUSH* = 2 +var + TCSANOW* {.importc, header: "<termios.h>".}: cint + TCSADRAIN* {.importc, header: "<termios.h>".}: cint + TCSAFLUSH* {.importc, header: "<termios.h>".}: cint # Compare a character C to a value VAL from the `cc' array in a # `struct termios'. If VAL is _POSIX_VDISABLE, no character can match it. -template cceq*(val, c: expr): expr = +template cceq*(val, c): untyped = c == val and val != POSIX_VDISABLE # Return the output baud rate stored in *TERMIOS_P. @@ -221,10 +208,6 @@ proc cfSetIspeed*(termios: ptr Termios; speed: Speed): cint {. importc: "cfsetispeed", header: "<termios.h>".} # Set both the input and output baud rates in *TERMIOS_OP to SPEED. -proc cfSetSpeed*(termios: ptr Termios; speed: Speed): cint {. - importc: "cfsetspeed", header: "<termios.h>".} -# Put the state of FD into *TERMIOS_P. - proc tcGetAttr*(fd: cint; termios: ptr Termios): cint {. importc: "tcgetattr", header: "<termios.h>".} # Set the state of FD to *TERMIOS_P. @@ -234,10 +217,6 @@ proc tcSetAttr*(fd: cint; optional_actions: cint; termios: ptr Termios): cint {. importc: "tcsetattr", header: "<termios.h>".} # Set *TERMIOS_P to indicate raw mode. -proc cfMakeRaw*(termios: ptr Termios) {.importc: "cfmakeraw", - header: "<termios.h>".} -# Send zero bits on FD. - proc tcSendBreak*(fd: cint; duration: cint): cint {.importc: "tcsendbreak", header: "<termios.h>".} # Wait for pending output to be written on FD. @@ -258,4 +237,20 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow", header: "<termios.h>".} # Get process group ID for session leader for controlling terminal FD. -proc tcGetSid*(fd: cint): Pid {.importc: "tcgetsid", header: "<termios.h>".} +# Window size ioctl. Solaris based systems have an uncommen place for this. +when defined(solaris) or defined(sunos): + var TIOCGWINSZ*{.importc, header: "<sys/termios.h>".}: culong +else: + var TIOCGWINSZ*{.importc, header: "<sys/ioctl.h>".}: culong + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +type IOctl_WinSize* = object + ws_row*, ws_col*, ws_xpixel*, ws_ypixel*: cushort + +when defined(nimHasStyleChecks): + {.pop.} + +proc ioctl*(fd: cint, request: culong, reply: ptr IOctl_WinSize): int {. + importc: "ioctl", header: "<stdio.h>", varargs.} diff --git a/lib/prelude.nim b/lib/prelude.nim deleted file mode 100644 index 940864207..000000000 --- a/lib/prelude.nim +++ /dev/null @@ -1,23 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This is an include file that simply imports common modules for your -## convenience: -## -## .. code-block:: nim -## include prelude -## -## Same as: -## -## .. code-block:: nim -## import os, strutils, times, parseutils, parseopt, hashes, tables, sets - -import os, strutils, times, parseutils, parseopt, hashes, tables, sets - - diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index c0acae138..b12ed7cdd 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -7,36 +7,112 @@ # distribution, for details about the copyright. # -## This module implements some common generic algorithms. +## This module implements some common generic algorithms on `openArray`s. +## +## Basic usage +## =========== +## -type - SortOrder* = enum ## sort order - Descending, Ascending +runnableExamples: + type People = tuple + year: int + name: string + + var a: seq[People] + + a.add((2000, "John")) + a.add((2005, "Marie")) + a.add((2010, "Jane")) + + # Sorting with default system.cmp + a.sort() + assert a == @[(year: 2000, name: "John"), (year: 2005, name: "Marie"), + (year: 2010, name: "Jane")] + + proc myCmp(x, y: People): int = + cmp(x.name, y.name) + + # Sorting with custom proc + a.sort(myCmp) + assert a == @[(year: 2010, name: "Jane"), (year: 2000, name: "John"), + (year: 2005, name: "Marie")] -{.deprecated: [TSortOrder: SortOrder].} +## See also +## ======== +## * `sequtils module<sequtils.html>`_ for working with the built-in seq type +## * `tables module<tables.html>`_ for sorting tables +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + + +type + SortOrder* = enum + Descending, Ascending proc `*`*(x: int, order: SortOrder): int {.inline.} = - ## flips `x` if ``order == Descending``; - ## if ``order == Ascending`` then `x` is returned. - ## `x` is supposed to be the result of a comparator, ie ``< 0`` for - ## *less than*, ``== 0`` for *equal*, ``> 0`` for *greater than*. + ## Flips the sign of `x` if `order == Descending`. + ## If `order == Ascending` then `x` is returned. + ## + ## `x` is supposed to be the result of a comparator, i.e. + ## | `< 0` for *less than*, + ## | `== 0` for *equal*, + ## | `> 0` for *greater than*. + runnableExamples: + assert -123 * Descending == 123 + assert 123 * Descending == -123 + assert -123 * Ascending == -123 + assert 123 * Ascending == 123 var y = order.ord - 1 result = (x xor y) - y -proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = - ## fills the array ``a[first..last]`` with `value`. +template fillImpl[T](a: var openArray[T], first, last: int, value: T) = var x = first while x <= last: a[x] = value inc(x) +proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = + ## Assigns `value` to all elements of the slice `a[first..last]`. + ## + ## If an invalid range is passed, it raises `IndexDefect`. + runnableExamples: + var a: array[6, int] + a.fill(1, 3, 9) + assert a == [0, 9, 9, 9, 0, 0] + a.fill(3, 5, 7) + assert a == [0, 9, 9, 7, 7, 7] + doAssertRaises(IndexDefect, a.fill(1, 7, 9)) + fillImpl(a, first, last, value) + proc fill*[T](a: var openArray[T], value: T) = - ## fills the array `a` with `value`. - fill(a, 0, a.high, value) + ## Assigns `value` to all elements of the container `a`. + runnableExamples: + var a: array[6, int] + a.fill(9) + assert a == [9, 9, 9, 9, 9, 9] + a.fill(4) + assert a == [4, 4, 4, 4, 4, 4] + fillImpl(a, 0, a.high, value) + proc reverse*[T](a: var openArray[T], first, last: Natural) = - ## reverses the array ``a[first..last]``. + ## Reverses the slice `a[first..last]`. + ## + ## If an invalid range is passed, it raises `IndexDefect`. + ## + ## **See also:** + ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a `seq[T]` + ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a `seq[T]` + runnableExamples: + var a = [1, 2, 3, 4, 5, 6] + a.reverse(1, 3) + assert a == [1, 4, 3, 2, 5, 6] + a.reverse(1, 3) + assert a == [1, 2, 3, 4, 5, 6] + doAssertRaises(IndexDefect, a.reverse(1, 7)) var x = first var y = last while x < y: @@ -45,65 +121,132 @@ proc reverse*[T](a: var openArray[T], first, last: Natural) = inc(x) proc reverse*[T](a: var openArray[T]) = - ## reverses the array `a`. - reverse(a, 0, a.high) - -proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = - ## returns the reverse of the array `a[first..last]`. - assert last >= first-1 - var i = last - first - var x = first.int - result = newSeq[T](i + 1) - while i >= 0: - result[i] = a[x] - dec(i) - inc(x) - -proc reversed*[T](a: openArray[T]): seq[T] = - ## returns the reverse of the array `a`. - reversed(a, 0, a.high) + ## Reverses the contents of the container `a`. + ## + ## **See also:** + ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a `seq[T]` + ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a `seq[T]` + runnableExamples: + var a = [1, 2, 3, 4, 5, 6] + a.reverse() + assert a == [6, 5, 4, 3, 2, 1] + a.reverse() + assert a == [1, 2, 3, 4, 5, 6] + # the max is needed, since a.high is -1 if a is empty + reverse(a, 0, max(0, a.high)) + +proc reversed*[T](a: openArray[T]): seq[T] {.inline.} = + ## Returns the elements of `a` in reverse order. + ## + ## **See also:** + ## * `reverse proc<#reverse,openArray[T]>`_ + runnableExamples: + assert [10, 11, 12].reversed == @[12, 11, 10] + assert seq[string].default.reversed == @[] + let n = a.len + result.setLen(n) + for i in 0..<n: result[i] = a[n - (i + 1)] + +proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] + {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = + reversed(toOpenArray(a, first, last)) + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +proc binarySearch*[T, K](a: openArray[T], key: K, + cmp: proc (x: T, y: K): int {.closure.}): int {.effectsOf: cmp.} = + ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. + ## Assumes that `a` is sorted according to `cmp`. + ## + ## `cmp` is the comparator function to use, the expected return values are + ## the same as those of system.cmp. + runnableExamples: + assert binarySearch(["a", "b", "c", "d"], "d", system.cmp[string]) == 3 + assert binarySearch(["a", "b", "c", "d"], "c", system.cmp[string]) == 2 + let len = a.len + + if len == 0: + return -1 + + if len == 1: + if cmp(a[0], key) == 0: + return 0 + else: + return -1 + + result = 0 + if (len and (len - 1)) == 0: + # when `len` is a power of 2, a faster shr can be used. + var step = len shr 1 + var cmpRes: int + while step > 0: + let i = result or step + cmpRes = cmp(a[i], key) + if cmpRes == 0: + return i + + if cmpRes < 0: + result = i + step = step shr 1 + if cmp(a[result], key) != 0: result = -1 + else: + var b = len + var cmpRes: int + while result < b: + var mid = (result + b) shr 1 + cmpRes = cmp(a[mid], key) + if cmpRes == 0: + return mid + + if cmpRes < 0: + result = mid + 1 + else: + b = mid + if result >= len or cmp(a[result], key) != 0: result = -1 proc binarySearch*[T](a: openArray[T], key: T): int = - ## binary search for `key` in `a`. Returns -1 if not found. - var b = len(a) - while result < b: - var mid = (result + b) div 2 - if a[mid] < key: result = mid + 1 - else: b = mid - if result >= len(a) or a[result] != key: result = -1 - -proc smartBinarySearch*[T](a: openArray[T], key: T): int = - ## ``a.len`` must be a power of 2 for this to work. - var step = a.len div 2 - while step > 0: - if a[result or step] <= key: - result = result or step - step = step shr 1 - if a[result] != key: result = -1 + ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. + ## Assumes that `a` is sorted. + runnableExamples: + assert binarySearch([0, 1, 2, 3, 4], 4) == 4 + assert binarySearch([0, 1, 2, 3, 4], 2) == 2 + binarySearch(a, key, cmp[T]) const onlySafeCode = true -proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}): int = - ## same as binarySearch except that if key is not in `a` then this - ## returns the location where `key` would be if it were. In other - ## words if you have a sorted sequence and you call - ## insert(thing, elm, lowerBound(thing, elm)) +proc lowerBound*[T, K](a: openArray[T], key: K, + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = + ## Returns the index of the first element in `a` that is not less than + ## (i.e. greater or equal to) `key`, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## `insert(thing, elm, lowerBound(thing, elm))` ## the sequence will still be sorted. + ## Assumes that `a` is sorted according to `cmp`. ## - ## `cmp` is the comparator function to use, the expected return values are - ## the same as that of system.cmp. + ## If an invalid range is passed, it raises `IndexDefect`. ## - ## example:: + ## This version uses `cmp` to compare the elements. + ## The expected return values are the same as those of `system.cmp`. ## - ## var arr = @[1,2,3,5,6,7,8,9] - ## arr.insert(4, arr.lowerBound(4)) - ## # after running the above arr is `[1,2,3,4,5,6,7,8,9]` + ## **See also:** + ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order + ## * `upperBound proc<#upperBound,openArray[T],T>`_ + runnableExamples: + var arr = @[1, 2, 3, 5, 6, 7, 8, 9] + assert arr.lowerBound(3, system.cmp[int]) == 2 + assert arr.lowerBound(4, system.cmp[int]) == 3 + assert arr.lowerBound(5, system.cmp[int]) == 3 + arr.insert(4, arr.lowerBound(4, system.cmp[int])) + assert arr == [1, 2, 3, 4, 5, 6, 7, 8, 9] result = a.low var count = a.high - a.low + 1 var step, pos: int while count != 0: - step = count div 2 + step = count shr 1 pos = result + step if cmp(a[pos], key) < 0: result = pos + 1 @@ -112,19 +255,83 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) count = step proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) -proc merge[T](a, b: var openArray[T], lo, m, hi: int, - cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = - template `<-` (a, b: expr) = - when false: - a = b - elif onlySafeCode: - shallowCopy(a, b) + ## Returns the index of the first element in `a` that is not less than + ## (i.e. greater or equal to) `key`, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## `insert(thing, elm, lowerBound(thing, elm))` + ## the sequence will still be sorted. + ## Assumes that `a` is sorted. + ## + ## This version uses the default comparison function `cmp`. + ## + ## **See also:** + ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order + ## * `upperBound proc<#upperBound,openArray[T],T>`_ + +proc upperBound*[T, K](a: openArray[T], key: K, + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = + ## Returns the index of the first element in `a` that is greater than + ## `key`, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## `insert(thing, elm, upperBound(thing, elm))` + ## the sequence will still be sorted. + ## Assumes that `a` is sorted according to `cmp`. + ## + ## If an invalid range is passed, it raises `IndexDefect`. + ## + ## This version uses `cmp` to compare the elements. The expected + ## return values are the same as those of `system.cmp`. + ## + ## **See also:** + ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order + ## * `lowerBound proc<#lowerBound,openArray[T],T>`_ + runnableExamples: + var arr = @[1, 2, 3, 5, 6, 7, 8, 9] + assert arr.upperBound(2, system.cmp[int]) == 2 + assert arr.upperBound(3, system.cmp[int]) == 3 + assert arr.upperBound(4, system.cmp[int]) == 3 + arr.insert(4, arr.upperBound(3, system.cmp[int])) + assert arr == [1, 2, 3, 4, 5, 6, 7, 8, 9] + result = a.low + var count = a.high - a.low + 1 + var step, pos: int + while count != 0: + step = count shr 1 + pos = result + step + if cmp(a[pos], key) <= 0: + result = pos + 1 + count -= step + 1 else: - copyMem(addr(a), addr(b), sizeof(T)) - # optimization: If max(left) <= min(right) there is nothing to do! - # 1 2 3 4 ## 5 6 7 8 + count = step + +proc upperBound*[T](a: openArray[T], key: T): int = upperBound(a, key, cmp[T]) + ## Returns the index of the first element in `a` that is greater than + ## `key`, or last if no such element is found. + ## In other words if you have a sorted sequence and you call + ## `insert(thing, elm, upperBound(thing, elm))` + ## the sequence will still be sorted. + ## Assumes that `a` is sorted. + ## + ## This version uses the default comparison function `cmp`. + ## + ## **See also:** + ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order + ## * `lowerBound proc<#lowerBound,openArray[T],T>`_ + +template `<-`(a, b) = + when defined(gcDestructors): + a = move b + elif onlySafeCode: + shallowCopy(a, b) + else: + copyMem(addr(a), addr(b), sizeof(T)) + +proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, + cmp: proc (x, y: T): int {.closure.}, order: SortOrder) {.effectsOf: cmp.} = + # Optimization: If max(left) <= min(right) there is nothing to do! + # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. - # On random data this safes up to 40% of merge calls + # On random data this saves up to 40% of mergeAlt calls. if cmp(a[m], a[m+1]) * order <= 0: return var j = lo # copy a[j..m] into b: @@ -158,80 +365,138 @@ proc merge[T](a, b: var openArray[T], lo, m, hi: int, else: if k < j: copyMem(addr(a[k]), addr(b[i]), sizeof(T)*(j-k)) -proc sort*[T](a: var openArray[T], +func sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Default Nim sort (an implementation of merge sort). The sorting - ## is guaranteed to be stable and the worst case is guaranteed to - ## be O(n log n). + ## is guaranteed to be stable (that is, equal elements stay in the same order) + ## and the worst case is guaranteed to be O(n log n). + ## Sorts by `cmp` in the specified `order`. + ## ## The current implementation uses an iterative ## mergesort to achieve this. It uses a temporary sequence of - ## length ``a.len div 2``. Currently Nim does not support a - ## sensible default argument for ``cmp``, so you have to provide one - ## of your own. However, the ``system.cmp`` procs can be used: - ## - ## .. code-block:: nim + ## length `a.len div 2`. If you do not wish to provide your own + ## `cmp`, you may use `system.cmp` or instead call the overloaded + ## version of `sort`, which uses `system.cmp`. ## - ## sort(myIntArray, system.cmp[int]) - ## - ## # do not use cmp[string] here as we want to use the specialized - ## # overload: - ## sort(myStrArray, system.cmp) + ## ```nim + ## sort(myIntArray, system.cmp[int]) + ## # do not use cmp[string] here as we want to use the specialized + ## # overload: + ## sort(myStrArray, system.cmp) + ## ``` ## ## You can inline adhoc comparison procs with the `do notation - ## <manual.html#do-notation>`_. Example: - ## - ## .. code-block:: nim + ## <manual.html#procedures-do-notation>`_. Example: ## + ## ```nim ## people.sort do (x, y: Person) -> int: ## result = cmp(x.surname, y.surname) ## if result == 0: ## result = cmp(x.name, y.name) + ## ``` + ## + ## **See also:** + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by `cmp` in the specified order + ## * `sorted proc<#sorted,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ + runnableExamples: + var d = ["boo", "fo", "barr", "qux"] + proc myCmp(x, y: string): int = + if x.len() > y.len() or x.len() == y.len(): 1 + else: -1 + sort(d, myCmp) + assert d == ["fo", "qux", "boo", "barr"] var n = a.len - var b: seq[T] - newSeq(b, n div 2) + var b = newSeq[T](n div 2) var s = 1 while s < n: var m = n-1-s while m >= 0: - merge(a, b, max(m-s+1, 0), m, m+s, cmp, order) + mergeAlt(a, b, max(m-s+1, 0), m, m+s, cmp, order) dec(m, s*2) s = s*2 +proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, + system.cmp[T], order) + ## Shortcut version of `sort` that uses `system.cmp[T]` as the comparison function. + ## + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by `cmp` in the specified order + ## * `sorted proc<#sorted,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ + proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): seq[T] = - ## returns `a` sorted by `cmp` in the specified `order`. + order = SortOrder.Ascending): seq[T] {.effectsOf: cmp.} = + ## Returns `a` sorted by `cmp` in the specified `order`. + ## + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = sorted(a, system.cmp[int]) + c = sorted(a, system.cmp[int], Descending) + d = sorted(["adam", "dande", "brian", "cat"], system.cmp[string]) + assert b == @[1, 2, 3, 4, 5] + assert c == @[5, 4, 3, 2, 1] + assert d == @["adam", "brian", "cat", "dande"] result = newSeq[T](a.len) for i in 0 .. a.high: result[i] = a[i] sort(result, cmp, order) -template sortedByIt*(seq1, op: expr): expr = - ## Convenience template around the ``sorted`` proc to reduce typing. - ## - ## The template injects the ``it`` variable which you can use directly in an - ## expression. Example: - ## - ## .. code-block:: nim +proc sorted*[T](a: openArray[T], order = SortOrder.Ascending): seq[T] = + ## Shortcut version of `sorted` that uses `system.cmp[T]` as the comparison function. ## - ## type Person = tuple[name: string, age: int] - ## var - ## p1: Person = (name: "p1", age: 60) - ## p2: Person = (name: "p2", age: 20) - ## p3: Person = (name: "p3", age: 30) - ## p4: Person = (name: "p4", age: 30) - ## people = @[p1,p2,p4,p3] + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = sorted(a) + c = sorted(a, Descending) + d = sorted(["adam", "dande", "brian", "cat"]) + assert b == @[1, 2, 3, 4, 5] + assert c == @[5, 4, 3, 2, 1] + assert d == @["adam", "brian", "cat", "dande"] + sorted[T](a, system.cmp[T], order) + +template sortedByIt*(seq1, op: untyped): untyped = + ## Convenience template around the `sorted` proc to reduce typing. ## - ## echo people.sortedByIt(it.name) + ## The template injects the `it` variable which you can use directly in an + ## expression. ## - ## Because the underlying ``cmp()`` is defined for tuples you can do - ## a nested sort like in the following example: + ## Because the underlying `cmp()` is defined for tuples you can also do + ## a nested sort. ## - ## .. code-block:: nim - ## - ## echo people.sortedByIt((it.age, it.name)) - ## - var result {.gensym.} = sorted(seq1, proc(x, y: type(seq1[0])): int = + ## **See also:** + ## * `sort func<#sort,openArray[T],proc(T,T)>`_ + ## * `sort proc<#sort,openArray[T]>`_ + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by `cmp` in the specified order + ## * `sorted proc<#sorted,openArray[T]>`_ + runnableExamples: + type Person = tuple[name: string, age: int] + var + p1: Person = (name: "p1", age: 60) + p2: Person = (name: "p2", age: 20) + p3: Person = (name: "p3", age: 30) + p4: Person = (name: "p4", age: 30) + people = @[p1, p2, p4, p3] + + assert people.sortedByIt(it.name) == @[(name: "p1", age: 60), (name: "p2", + age: 20), (name: "p3", age: 30), (name: "p4", age: 30)] + # Nested sort + assert people.sortedByIt((it.age, it.name)) == @[(name: "p2", age: 20), + (name: "p3", age: 30), (name: "p4", age: 30), (name: "p1", age: 60)] + var result = sorted(seq1, proc(x, y: typeof(items(seq1), typeOfIter)): int = var it {.inject.} = x let a = op it = y @@ -239,60 +504,203 @@ template sortedByIt*(seq1, op: expr): expr = result = cmp(a, b)) result -proc isSorted*[T](a: openarray[T], +func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): bool = + order = SortOrder.Ascending): bool {.effectsOf: cmp.} = ## Checks to see whether `a` is already sorted in `order` - ## using `cmp` for the comparison. Parameters identical - ## to `sort` + ## using `cmp` for the comparison. The parameters are identical + ## to `sort`. Requires O(n) time. + ## + ## **See also:** + ## * `isSorted proc<#isSorted,openArray[T]>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = [1, 2, 3, 4, 5] + c = [5, 4, 3, 2, 1] + d = ["adam", "brian", "cat", "dande"] + e = ["adam", "dande", "brian", "cat"] + assert isSorted(a) == false + assert isSorted(b) == true + assert isSorted(c) == false + assert isSorted(c, Descending) == true + assert isSorted(d) == true + assert isSorted(e) == false result = true for i in 0..<len(a)-1: - if cmp(a[i],a[i+1]) * order > 0: + if cmp(a[i], a[i+1]) * order > 0: return false +proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = + ## Shortcut version of `isSorted` that uses `system.cmp[T]` as the comparison function. + ## + ## **See also:** + ## * `isSorted func<#isSorted,openArray[T],proc(T,T)>`_ + runnableExamples: + let + a = [2, 3, 1, 5, 4] + b = [1, 2, 3, 4, 5] + c = [5, 4, 3, 2, 1] + d = ["adam", "brian", "cat", "dande"] + e = ["adam", "dande", "brian", "cat"] + assert isSorted(a) == false + assert isSorted(b) == true + assert isSorted(c) == false + assert isSorted(c, Descending) == true + assert isSorted(d) == true + assert isSorted(e) == false + isSorted(a, system.cmp[T], order) + +proc merge*[T]( + result: var seq[T], + x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} +) {.since: (1, 5, 1), effectsOf: cmp.} = + ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. + ## If you do not wish to provide your own `cmp`, + ## you may use `system.cmp` or instead call the overloaded + ## version of `merge`, which uses `system.cmp`. + ## + ## .. note:: The original data of `result` is not cleared, + ## new data is appended to `result`. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T]>`_ + runnableExamples: + let x = @[1, 3, 6] + let y = @[2, 3, 4] + + block: + var merged = @[7] # new data is appended to merged sequence + merged.merge(x, y, system.cmp[int]) + assert merged == @[7, 1, 2, 3, 3, 4, 6] + + block: + var merged = @[7] # if you only want new data, clear merged sequence first + merged.setLen(0) + merged.merge(x, y, system.cmp[int]) + assert merged.isSorted + assert merged == @[1, 2, 3, 3, 4, 6] + + import std/sugar + + var res: seq[(int, int)] + res.merge([(1, 1)], [(1, 2)], (a, b) => a[0] - b[0]) + assert res == @[(1, 1), (1, 2)] + + assert seq[int].default.dup(merge([1, 3], [2, 4])) == @[1, 2, 3, 4] + + let + sizeX = x.len + sizeY = y.len + oldLen = result.len + + result.setLen(oldLen + sizeX + sizeY) + + var + ix = 0 + iy = 0 + i = oldLen + + while true: + if ix == sizeX: + while iy < sizeY: + result[i] = y[iy] + inc i + inc iy + return + + if iy == sizeY: + while ix < sizeX: + result[i] = x[ix] + inc i + inc ix + return + + let itemX = x[ix] + let itemY = y[iy] + + if cmp(itemX, itemY) > 0: # to have a stable sort + result[i] = itemY + inc iy + else: + result[i] = itemX + inc ix + + inc i + +proc merge*[T](result: var seq[T], x, y: openArray[T]) {.inline, since: (1, 5, 1).} = + ## Shortcut version of `merge` that uses `system.cmp[T]` as the comparison function. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T],proc(T,T)>`_ + runnableExamples: + let x = [5, 10, 15, 20, 25] + let y = [50, 40, 30, 20, 10].sorted + + var merged: seq[int] + merged.merge(x, y) + assert merged.isSorted + assert merged == @[5, 10, 10, 15, 20, 20, 25, 30, 40, 50] + merge(result, x, y, system.cmp) + proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = - ## produces the Cartesian product of the array. Warning: complexity - ## may explode. + ## Produces the Cartesian product of the array. + ## Every element of the result is a combination of one element from each seq in `x`, + ## with the ith element coming from `x[i]`. + ## + ## .. warning:: complexity may explode. + runnableExamples: + assert product(@[@[1], @[2]]) == @[@[1, 2]] + assert product(@[@["A", "K"], @["Q"]]) == @[@["K", "Q"], @["A", "Q"]] + let xLen = x.len result = newSeq[seq[T]]() - if x.len == 0: + if xLen == 0: return - if x.len == 1: + if xLen == 1: result = @x return var - indexes = newSeq[int](x.len) - initial = newSeq[int](x.len) + indices = newSeq[int](xLen) + initial = newSeq[int](xLen) index = 0 - var next = newSeq[T]() - next.setLen(x.len) - for i in 0..(x.len-1): + var next = newSeq[T](xLen) + for i in 0 ..< xLen: if len(x[i]) == 0: return - initial[i] = len(x[i])-1 - indexes = initial + initial[i] = len(x[i]) - 1 + indices = initial while true: - while indexes[index] == -1: - indexes[index] = initial[index] + while indices[index] == -1: + indices[index] = initial[index] index += 1 - if index == x.len: return - indexes[index] -= 1 - for ni, i in indexes: + if index == xLen: return + indices[index] -= 1 + for ni, i in indices: next[ni] = x[ni][i] - var res: seq[T] - shallowCopy(res, next) - result.add(res) + result.add(next) index = 0 - indexes[index] -= 1 + indices[index] -= 1 -proc nextPermutation*[T](x: var openarray[T]): bool {.discardable.} = - ## Calculates the next lexicographic permutation, directly modifying ``x``. +proc nextPermutation*[T](x: var openArray[T]): bool {.discardable.} = + ## Calculates the next lexicographic permutation, directly modifying `x`. ## The result is whether a permutation happened, otherwise we have reached ## the last-ordered permutation. ## - ## .. code-block:: nim + ## If you start with an unsorted array/seq, the repeated permutations + ## will **not** give you all permutations but stop with the last. ## - ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - ## v.nextPermutation() - ## echo v # @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] + ## **See also:** + ## * `prevPermutation proc<#prevPermutation,openArray[T]>`_ + runnableExamples: + var v = @[0, 1, 2, 3] + assert v.nextPermutation() == true + assert v == @[0, 1, 3, 2] + assert v.nextPermutation() == true + assert v == @[0, 2, 1, 3] + assert v.prevPermutation() == true + assert v == @[0, 1, 3, 2] + v = @[3, 2, 1, 0] + assert v.nextPermutation() == false + assert v == @[3, 2, 1, 0] if x.len < 2: return false @@ -312,16 +720,21 @@ proc nextPermutation*[T](x: var openarray[T]): bool {.discardable.} = result = true -proc prevPermutation*[T](x: var openarray[T]): bool {.discardable.} = +proc prevPermutation*[T](x: var openArray[T]): bool {.discardable.} = ## Calculates the previous lexicographic permutation, directly modifying - ## ``x``. The result is whether a permutation happened, otherwise we have + ## `x`. The result is whether a permutation happened, otherwise we have ## reached the first-ordered permutation. ## - ## .. code-block:: nim - ## - ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] - ## v.prevPermutation() - ## echo v # @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ## **See also:** + ## * `nextPermutation proc<#nextPermutation,openArray[T]>`_ + runnableExamples: + var v = @[0, 1, 2, 3] + assert v.prevPermutation() == false + assert v == @[0, 1, 2, 3] + assert v.nextPermutation() == true + assert v == @[0, 1, 3, 2] + assert v.prevPermutation() == true + assert v == @[0, 1, 2, 3] if x.len < 2: return false @@ -342,30 +755,163 @@ proc prevPermutation*[T](x: var openarray[T]): bool {.discardable.} = result = true -when isMainModule: - # Tests for lowerBound - var arr = @[1,2,3,5,6,7,8,9] - assert arr.lowerBound(0) == 0 - assert arr.lowerBound(4) == 3 - assert arr.lowerBound(5) == 3 - assert arr.lowerBound(10) == 8 - arr = @[1,5,10] - assert arr.lowerBound(4) == 1 - assert arr.lowerBound(5) == 1 - assert arr.lowerBound(6) == 2 - # Tests for isSorted - var srt1 = [1,2,3,4,4,4,4,5] - var srt2 = ["iello","hello"] - var srt3 = [1.0,1.0,1.0] - var srt4: seq[int] = @[] - assert srt1.isSorted(cmp) == true - assert srt2.isSorted(cmp) == false - assert srt3.isSorted(cmp) == true - var srtseq = newSeq[int]() - assert srtseq.isSorted(cmp) == true - # Tests for reversed - var arr1 = @[0,1,2,3,4] - assert arr1.reversed() == @[4,3,2,1,0] - for i in 0 .. high(arr1): - assert arr1.reversed(0, i) == arr1.reversed()[high(arr1) - i .. high(arr1)] - assert arr1.reversed(i, high(arr1)) == arr1.reversed()[0 .. high(arr1) - i] +proc rotateInternal[T](arg: var openArray[T]; first, middle, last: int): int = + ## A port of std::rotate from C++. + ## Ported from [this reference](http://www.cplusplus.com/reference/algorithm/rotate/). + result = first + last - middle + + if first == middle or middle == last: + return + + assert first < middle + assert middle < last + + # m prefix for mutable + var + mFirst = first + mMiddle = middle + next = middle + + swap(arg[mFirst], arg[next]) + mFirst += 1 + next += 1 + if mFirst == mMiddle: + mMiddle = next + + while next != last: + swap(arg[mFirst], arg[next]) + mFirst += 1 + next += 1 + if mFirst == mMiddle: + mMiddle = next + + next = mMiddle + while next != last: + swap(arg[mFirst], arg[next]) + mFirst += 1 + next += 1 + if mFirst == mMiddle: + mMiddle = next + elif next == last: + next = mMiddle + +proc rotatedInternal[T](arg: openArray[T]; first, middle, last: int): seq[T] = + let argLen = arg.len + result = newSeq[T](argLen) + for i in 0 ..< first: + result[i] = arg[i] + let n = last - middle + let m = middle - first + for i in 0 ..< n: + result[first+i] = arg[middle+i] + for i in 0 ..< m: + result[first+n+i] = arg[first+i] + for i in last ..< argLen: + result[i] = arg[i] + +proc rotateLeft*[T](arg: var openArray[T]; slice: HSlice[int, int]; + dist: int): int {.discardable.} = + ## Performs a left rotation on a range of elements. If you want to rotate + ## right, use a negative `dist`. Specifically, `rotateLeft` rotates + ## the elements at `slice` by `dist` positions. + ## + ## | The element at index `slice.a + dist` will be at index `slice.a`. + ## | The element at index `slice.b` will be at `slice.a + dist - 1`. + ## | The element at index `slice.a` will be at `slice.b + 1 - dist`. + ## | The element at index `slice.a + dist - 1` will be at `slice.b`. + ## + ## Elements outside of `slice` will be left unchanged. + ## The time complexity is linear to `slice.b - slice.a + 1`. + ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. + ## + ## `slice` + ## : The indices of the element range that should be rotated. + ## + ## `dist` + ## : The distance in amount of elements that the data should be rotated. + ## Can be negative, can be any number. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],int>`_ for a version which rotates the whole container + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],HSlice[int,int],int>`_ for a version which returns a `seq[T]` + runnableExamples: + var a = [0, 1, 2, 3, 4, 5] + a.rotateLeft(1 .. 4, 3) + assert a == [0, 4, 1, 2, 3, 5] + a.rotateLeft(1 .. 4, 3) + assert a == [0, 3, 4, 1, 2, 5] + a.rotateLeft(1 .. 4, -3) + assert a == [0, 4, 1, 2, 3, 5] + doAssertRaises(IndexDefect, a.rotateLeft(1 .. 7, 2)) + let sliceLen = slice.b + 1 - slice.a + let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen + arg.rotateInternal(slice.a, slice.a + distLeft, slice.b + 1) + +proc rotateLeft*[T](arg: var openArray[T]; dist: int): int {.discardable.} = + ## Same as `rotateLeft`, but with default arguments for slice, + ## so that this procedure operates on the entire + ## `arg`, and not just on a part of it. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],HSlice[int,int],int>`_ for a version which rotates a range + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],int>`_ for a version which returns a `seq[T]` + runnableExamples: + var a = [1, 2, 3, 4, 5] + a.rotateLeft(2) + assert a == [3, 4, 5, 1, 2] + a.rotateLeft(4) + assert a == [2, 3, 4, 5, 1] + a.rotateLeft(-6) + assert a == [1, 2, 3, 4, 5] + let argLen = arg.len + let distLeft = ((dist mod argLen) + argLen) mod argLen + arg.rotateInternal(0, distLeft, argLen) + +proc rotatedLeft*[T](arg: openArray[T]; slice: HSlice[int, int], + dist: int): seq[T] = + ## Same as `rotateLeft`, just with the difference that it does + ## not modify the argument. It creates a new `seq` instead. + ## + ## Elements outside of `slice` will be left unchanged. + ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. + ## + ## `slice` + ## : The indices of the element range that should be rotated. + ## + ## `dist` + ## : The distance in amount of elements that the data should be rotated. + ## Can be negative, can be any number. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],HSlice[int,int],int>`_ for the in-place version of this proc + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],int>`_ for a version which rotates the whole container + runnableExamples: + var a = @[1, 2, 3, 4, 5] + a = rotatedLeft(a, 1 .. 4, 3) + assert a == @[1, 5, 2, 3, 4] + a = rotatedLeft(a, 1 .. 3, 2) + assert a == @[1, 3, 5, 2, 4] + a = rotatedLeft(a, 1 .. 3, -2) + assert a == @[1, 5, 2, 3, 4] + let sliceLen = slice.b + 1 - slice.a + let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen + arg.rotatedInternal(slice.a, slice.a + distLeft, slice.b + 1) + +proc rotatedLeft*[T](arg: openArray[T]; dist: int): seq[T] = + ## Same as `rotateLeft`, just with the difference that it does + ## not modify the argument. It creates a new `seq` instead. + ## + ## **See also:** + ## * `rotateLeft proc<#rotateLeft,openArray[T],int>`_ for the in-place version of this proc + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],HSlice[int,int],int>`_ for a version which rotates a range + runnableExamples: + var a = @[1, 2, 3, 4, 5] + a = rotatedLeft(a, 2) + assert a == @[3, 4, 5, 1, 2] + a = rotatedLeft(a, 4) + assert a == @[2, 3, 4, 5, 1] + a = rotatedLeft(a, -6) + assert a == @[1, 2, 3, 4, 5] + let argLen = arg.len + let distLeft = ((dist mod argLen) + argLen) mod argLen + arg.rotatedInternal(0, distLeft, argLen) diff --git a/lib/pure/async.nim b/lib/pure/async.nim new file mode 100644 index 000000000..e4d8d41c3 --- /dev/null +++ b/lib/pure/async.nim @@ -0,0 +1,9 @@ +## Exports [asyncmacro](asyncmacro.html) and [asyncfutures](asyncfutures.html) for native backends, +## and [asyncjs](asyncjs.html) on the JS backend. + +when defined(js): + import std/asyncjs + export asyncjs +else: + import std/[asyncmacro, asyncfutures] + export asyncmacro, asyncfutures diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cc337452f..126db7a7f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -7,39 +7,26 @@ # distribution, for details about the copyright. # -include "system/inclrtl" - -import os, oids, tables, strutils, macros, times - -import nativesockets, net - -export Port, SocketFlag - -#{.injectStmt: newGcInvariant().} - -## AsyncDispatch -## ************* -## ## This module implements asynchronous IO. This includes a dispatcher, -## a ``Future`` type implementation, and an ``async`` macro which allows -## asynchronous code to be written in a synchronous style with the ``await`` +## a `Future` type implementation, and an `async` macro which allows +## asynchronous code to be written in a synchronous style with the `await` ## keyword. ## -## The dispatcher acts as a kind of event loop. You must call ``poll`` on it -## (or a function which does so for you such as ``waitFor`` or ``runForever``) +## The dispatcher acts as a kind of event loop. You must call `poll` on it +## (or a function which does so for you such as `waitFor` or `runForever`) ## in order to poll for any outstanding events. The underlying implementation ## is based on epoll on Linux, IO Completion Ports on Windows and select on ## other operating systems. ## -## The ``poll`` function will not, on its own, return any events. Instead -## an appropriate ``Future`` object will be completed. A ``Future`` is a +## The `poll` function will not, on its own, return any events. Instead +## an appropriate `Future` object will be completed. A `Future` is a ## type which holds a value which is not yet available, but which *may* be ## available in the future. You can check whether a future is finished -## by using the ``finished`` function. When a future is finished it means that +## by using the `finished` function. When a future is finished it means that ## either the value that it holds is now available or it holds an error instead. ## The latter situation occurs when the operation to complete a future fails ## with an exception. You can distinguish between the two situations with the -## ``failed`` function. +## `failed` function. ## ## Future objects can also store a callback procedure which will be called ## automatically once the future completes. @@ -48,49 +35,50 @@ export Port, SocketFlag ## pattern. In this ## pattern you make a request for an action, and once that action is fulfilled ## a future is completed with the result of that action. Requests can be -## made by calling the appropriate functions. For example: calling the ``recv`` +## made by calling the appropriate functions. For example: calling the `recv` ## function will create a request for some data to be read from a socket. The -## future which the ``recv`` function returns will then complete once the +## future which the `recv` function returns will then complete once the ## requested amount of data is read **or** an exception occurs. ## ## Code to read some data from a socket may look something like this: +## ```Nim +## var future = socket.recv(100) +## future.addCallback( +## proc () = +## echo(future.read) +## ) +## ``` ## -## .. code-block::nim -## var future = socket.recv(100) -## future.callback = -## proc () = -## echo(future.read) -## -## All asynchronous functions returning a ``Future`` will not block. They +## All asynchronous functions returning a `Future` will not block. They ## will not however return immediately. An asynchronous function will have ## code which will be executed before an asynchronous request is made, in most ## cases this code sets up the request. ## -## In the above example, the ``recv`` function will return a brand new -## ``Future`` instance once the request for data to be read from the socket -## is made. This ``Future`` instance will complete once the requested amount +## In the above example, the `recv` function will return a brand new +## `Future` instance once the request for data to be read from the socket +## is made. This `Future` instance will complete once the requested amount ## of data is read, in this case it is 100 bytes. The second line sets a ## callback on this future which will be called once the future completes. -## All the callback does is write the data stored in the future to ``stdout``. -## The ``read`` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. +## All the callback does is write the data stored in the future to `stdout`. +## The `read` function is used for this and it checks whether the future +## completes with an error for you (if it did, it will simply raise the +## error), if there is no error, however, it returns the value of the future. ## ## Asynchronous procedures -## ----------------------- +## ======================= ## ## Asynchronous procedures remove the pain of working with callbacks. They do ## this by allowing you to write asynchronous code the same way as you would ## write synchronous code. ## -## An asynchronous procedure is marked using the ``{.async.}`` pragma. -## When marking a procedure with the ``{.async.}`` pragma it must have a -## ``Future[T]`` return type or no return type at all. If you do not specify -## a return type then ``Future[void]`` is assumed. +## An asynchronous procedure is marked using the `{.async.}` pragma. +## When marking a procedure with the `{.async.}` pragma it must have a +## `Future[T]` return type or no return type at all. If you do not specify +## a return type then `Future[void]` is assumed. ## -## Inside asynchronous procedures ``await`` can be used to call any +## Inside asynchronous procedures `await` can be used to call any ## procedures which return a -## ``Future``; this includes asynchronous procedures. When a procedure is +## `Future`; this includes asynchronous procedures. When a procedure is ## "awaited", the asynchronous procedure it is awaited in will ## suspend its execution ## until the awaited procedure's Future completes. At which point the @@ -98,315 +86,293 @@ export Port, SocketFlag ## when an asynchronous procedure is suspended other asynchronous procedures ## will be run by the dispatcher. ## -## The ``await`` call may be used in many contexts. It can be used on the right -## hand side of a variable declaration: ``var data = await socket.recv(100)``, +## The `await` call may be used in many contexts. It can be used on the right +## hand side of a variable declaration: `var data = await socket.recv(100)`, ## in which case the variable will be set to the value of the future -## automatically. It can be used to await a ``Future`` object, and it can -## be used to await a procedure returning a ``Future[void]``: -## ``await socket.send("foobar")``. +## automatically. It can be used to await a `Future` object, and it can +## be used to await a procedure returning a `Future[void]`: +## `await socket.send("foobar")`. +## +## If an awaited future completes with an error, then `await` will re-raise +## this error. To avoid this, you can use the `yield` keyword instead of +## `await`. The following section shows different ways that you can handle +## exceptions in async procs. +## +## .. caution:: +## Procedures marked {.async.} do not support mutable parameters such +## as `var int`. References such as `ref int` should be used instead. +## +## Handling Exceptions +## ------------------- +## +## You can handle exceptions in the same way as in ordinary Nim code; +## by using the try statement: +## +## ```Nim +## try: +## let data = await sock.recv(100) +## echo("Received ", data) +## except: +## # Handle exception +## ``` +## +## An alternative approach to handling exceptions is to use `yield` on a future +## then check the future's `failed` property. For example: +## +## ```Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception +## ``` +## ## ## Discarding futures -## ------------------ +## ================== +## +## Futures should **never** be discarded directly because they may contain +## errors. If you do not care for the result of a Future then you should use +## the `asyncCheck` procedure instead of the `discard` keyword. Note that this +## does not wait for completion, and you should use `waitFor` or `await` for that purpose. +## +## .. note:: `await` also checks if the future fails, so you can safely discard +## its result. ## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. +## Handling futures +## ================ +## +## There are many different operations that apply to a future. +## The three primary high-level operations are `asyncCheck`, +## `waitFor`, and `await`. +## +## * `asyncCheck`: Raises an exception if the future fails. It neither waits +## for the future to finish nor returns the result of the future. +## * `waitFor`: Polls the event loop and blocks the current thread until the +## future finishes. This is often used to call an async procedure from a +## synchronous context and should never be used in an `async` proc. +## * `await`: Pauses execution in the current async procedure until the future +## finishes. While the current procedure is paused, other async procedures will +## continue running. Should be used instead of `waitFor` in an async +## procedure. +## +## Here is a handy quick reference chart showing their high-level differences: +## ============== ===================== ======================= +## Procedure Context Blocking +## ============== ===================== ======================= +## `asyncCheck` non-async and async non-blocking +## `waitFor` non-async blocks current thread +## `await` async suspends current proc +## ============== ===================== ======================= ## ## Examples -## -------- +## ======== ## ## For examples take a look at the documentation for the modules implementing ## asynchronous IO. A good place to start is the ## `asyncnet module <asyncnet.html>`_. ## +## Investigating pending futures +## ============================= +## +## It's possible to get into a situation where an async proc, or more accurately +## a `Future[T]` gets stuck and +## never completes. This can happen for various reasons and can cause serious +## memory leaks. When this occurs it's hard to identify the procedure that is +## stuck. +## +## Thankfully there is a mechanism which tracks the count of each pending future. +## All you need to do to enable it is compile with `-d:futureLogging` and +## use the `getFuturesInProgress` procedure to get the list of pending futures +## together with the stack traces to the moment of their creation. +## +## You may also find it useful to use this +## `prometheus package <https://github.com/dom96/prometheus>`_ which will log +## the pending futures into prometheus, allowing you to analyse them via a nice +## graph. +## +## +## ## Limitations/Bugs -## ---------------- +## ================ ## -## * The effect system (``raises: []``) does not work with async procedures. -## * Can't await in a ``except`` body -## * Forward declarations for async procs are broken, -## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. -## * FutureVar[T] needs to be completed manually. - -# TODO: Check if yielded future is nil and throw a more meaningful exception - -# -- Futures - -type - FutureBase* = ref object of RootObj ## Untyped future. - cb: proc () {.closure,gcsafe.} - finished: bool - error*: ref Exception ## Stored exception - errorStackTrace*: string - when not defined(release): - stackTrace: string ## For debugging purposes only. - id: int - fromProc: string - - Future*[T] = ref object of FutureBase ## Typed future. - value: T ## Stored value - - FutureVar*[T] = distinct Future[T] - - FutureError* = object of Exception - cause*: FutureBase - -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - -when not defined(release): - var currentID = 0 -proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = - ## Creates a new future. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - new(result) - result.finished = false - when not defined(release): - result.stackTrace = getStackTrace() - result.id = currentID - result.fromProc = fromProc - currentID.inc() - -proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = - ## Create a new ``FutureVar``. This Future type is ideally suited for - ## situations where you want to avoid unnecessary allocations of Futures. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - result = FutureVar[T](newFuture[T](fromProc)) - -proc clean*[T](future: FutureVar[T]) = - ## Resets the ``finished`` status of ``future``. - Future[T](future).finished = false - Future[T](future).error = nil - -proc checkFinished[T](future: Future[T]) = - ## Checks whether `future` is finished. If it is then raises a - ## ``FutureError``. - when not defined(release): - if future.finished: - var msg = "" - msg.add("An attempt was made to complete a Future more than once. ") - msg.add("Details:") - msg.add("\n Future ID: " & $future.id) - msg.add("\n Created in proc: " & future.fromProc) - msg.add("\n Stack trace to moment of creation:") - msg.add("\n" & indent(future.stackTrace.strip(), 4)) - when T is string: - msg.add("\n Contents (string): ") - msg.add("\n" & indent(future.value.repr, 4)) - msg.add("\n Stack trace to moment of secondary completion:") - msg.add("\n" & indent(getStackTrace().strip(), 4)) - var err = newException(FutureError, msg) - err.cause = future - raise err - -proc complete*[T](future: Future[T], val: T) = - ## Completes ``future`` with value ``val``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.value = val - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*(future: Future[void]) = - ## Completes a void ``future``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*[T](future: FutureVar[T]) = - ## Completes a ``FutureVar``. - template fut: expr = Future[T](future) - checkFinished(fut) - assert(fut.error == nil) - fut.finished = true - if fut.cb != nil: - fut.cb() - -proc fail*[T](future: Future[T], error: ref Exception) = - ## Completes ``future`` with ``error``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - future.finished = true - future.error = error - future.errorStackTrace = - if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) - if future.cb != nil: - future.cb() - else: - # This is to prevent exceptions from being silently ignored when a future - # is discarded. - # TODO: This may turn out to be a bad idea. - # Turns out this is a bad idea. - #raise error - discard - -proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - ## - ## **Note**: You most likely want the other ``callback`` setter which - ## passes ``future`` as a param to the callback. - future.cb = cb - if future.finished: - future.cb() - -proc `callback=`*[T](future: Future[T], - cb: proc (future: Future[T]) {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - future.callback = proc () = cb(future) - -proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. - when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") - - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) - else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) - -proc read*[T](future: Future[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished: - if future.error != nil: - injectStacktrace(future) - raise future.error - when T isnot void: - return future.value - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - -proc readError*[T](future: Future[T]): ref Exception = - ## Retrieves the exception stored in ``future``. - ## - ## An ``ValueError`` exception will be thrown if no exception exists - ## in the specified Future. - if future.error != nil: return future.error - else: - raise newException(ValueError, "No error in future.") +## * The effect system (`raises: []`) does not work with async procedures. +## * Mutable parameters are not supported by async procedures. +## +## +## Multiple async backend support +## ============================== +## +## Thanks to its powerful macro support, Nim allows ``async``/``await`` to be +## implemented in libraries with only minimal support from the language - as +## such, multiple ``async`` libraries exist, including ``asyncdispatch`` and +## ``chronos``, and more may come to be developed in the future. +## +## Libraries built on top of async/await may wish to support multiple async +## backends - the best way to do so is to create separate modules for each backend +## that may be imported side-by-side. +## +## An alternative way is to select backend using a global compile flag - this +## method makes it difficult to compose applications that use both backends as may +## happen with transitive dependencies, but may be appropriate in some cases - +## libraries choosing this path should call the flag `asyncBackend`, allowing +## applications to choose the backend with `-d:asyncBackend=<backend_name>`. +## +## Known `async` backends include: +## +## * `-d:asyncBackend=none`: disable `async` support completely +## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html +## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/ +## +## ``none`` can be used when a library supports both a synchronous and +## asynchronous API, to disable the latter. -proc mget*[T](future: FutureVar[T]): var T = - ## Returns a mutable value stored in ``future``. - ## - ## Unlike ``read``, this function will not raise an exception if the - ## Future has not been finished. - result = Future[T](future).value +import std/[os, tables, strutils, times, heapqueue, options, asyncstreams] +import std/[math, monotimes] +import std/asyncfutures except callSoon -proc finished*[T](future: Future[T]): bool = - ## Determines whether ``future`` has completed. - ## - ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - future.finished +import std/[nativesockets, net, deques] -proc failed*(future: FutureBase): bool = - ## Determines whether ``future`` completed with an error. - return future.error != nil +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] -proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future - ## finished with an error. - ## - ## This should be used instead of ``discard`` to discard void futures. - future.callback = - proc () = - if future.failed: - injectStacktrace(future) - raise future.error - -proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once both ``fut1`` and ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`and`") - fut1.callback = - proc () = - if fut2.finished: retFuture.complete() - fut2.callback = - proc () = - if fut1.finished: retFuture.complete() - return retFuture +export Port, SocketFlag +export asyncfutures except callSoon +export asyncstreams -proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once either ``fut1`` or ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`or`") - proc cb() = - if not retFuture.finished: retFuture.complete() - fut1.callback = cb - fut2.callback = cb - return retFuture +# TODO: Check if yielded future is nil and throw a more meaningful exception type PDispatcherBase = ref object of RootRef - timers: seq[tuple[finishAt: float, fut: Future[void]]] - -proc processTimers(p: PDispatcherBase) = - var oldTimers = p.timers - p.timers = @[] - for t in oldTimers: - if epochTime() >= t.finishAt: - t.fut.complete() - else: - p.timers.add(t) + timers*: HeapQueue[tuple[finishAt: MonoTime, fut: Future[void]]] + callbacks*: Deque[proc () {.gcsafe.}] + +proc processTimers( + p: PDispatcherBase, didSomeWork: var bool +): Option[int] {.inline.} = + # Pop the timers in the order in which they will expire (smaller `finishAt`). + var count = p.timers.len + let t = getMonoTime() + while count > 0 and t >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + dec count + didSomeWork = true + + # Return the number of milliseconds in which the next timer will expire. + if p.timers.len == 0: return + + let millisecs = (p.timers[0].finishAt - getMonoTime()).inMilliseconds + return some(millisecs.int + 1) + +proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) = + while p.callbacks.len > 0: + var cb = p.callbacks.popFirst() + cb() + didSomeWork = true + +proc adjustTimeout( + p: PDispatcherBase, pollTimeout: int, nextTimer: Option[int] +): int {.inline.} = + if p.callbacks.len != 0: + return 0 + + if nextTimer.isNone() or pollTimeout == -1: + return pollTimeout + + result = max(nextTimer.get(), 0) + result = min(pollTimeout, result) + +proc runOnce(timeout: int): bool {.gcsafe.} + +proc callSoon*(cbproc: proc () {.gcsafe.}) {.gcsafe.} + ## Schedule `cbproc` to be called as soon as possible. + ## The callback is called when control returns to the event loop. + +proc initCallSoonProc = + if asyncfutures.getCallSoonProc().isNil: + asyncfutures.setCallSoonProc(callSoon) + +template implementSetInheritable() {.dirty.} = + when declared(setInheritable): + proc setInheritable*(fd: AsyncFD, inheritable: bool): bool = + ## Control whether a file handle can be inherited by child processes. + ## Returns `true` on success. + ## + ## This procedure is not guaranteed to be available for all platforms. + ## Test for availability with `declared() <system.html#declared,untyped>`_. + fd.FileHandle.setInheritable(inheritable) when defined(windows) or defined(nimdoc): - import winlean, sets, hashes + import std/[winlean, sets, hashes] type - CompletionKey = Dword + CompletionKey = ULONG_PTR CompletionData* = object - fd*: AsyncFD # TODO: Rename this. - cb*: proc (fd: AsyncFD, bytesTransferred: Dword, - errcode: OSErrorCode) {.closure,gcsafe.} + fd*: AsyncFD # TODO: Rename this. + cb*: owned(proc (fd: AsyncFD, bytesTransferred: DWORD, + errcode: OSErrorCode) {.closure, gcsafe.}) + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. PDispatcher* = ref object of PDispatcherBase ioPort: Handle - handles: HashSet[AsyncFD] + handles*: HashSet[AsyncFD] # Export handles so that an external library can register them. - CustomOverlapped = object of OVERLAPPED + CustomObj = object of OVERLAPPED data*: CompletionData - PCustomOverlapped* = ref CustomOverlapped + CustomRef* = ref CustomObj AsyncFD* = distinct int - {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, - TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: owned CustomRef + PostCallbackDataPtr = ptr PostCallbackData + + AsyncEventImpl = object + hEvent: Handle + hWaiter: Handle + pcd: PostCallbackDataPtr + AsyncEvent* = ptr AsyncEventImpl + + Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.} proc hash(x: AsyncFD): Hash {.borrow.} proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} - proc newDispatcher*(): PDispatcher = + proc newDispatcher*(): owned PDispatcher = ## Creates a new Dispatcher instance. new result result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) - result.handles = initSet[AsyncFD]() - result.timers = @[] + result.handles = initHashSet[AsyncFD]() + result.timers.clear() + result.callbacks = initDeque[proc () {.closure, gcsafe.}](64) + + var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + + proc setGlobalDispatcher*(disp: sink PDispatcher) = + if not gDisp.isNil: + assert gDisp.callbacks.len == 0 + gDisp = disp + initCallSoonProc() - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = - ## Retrieves the global thread-local dispatcher. - if gDisp.isNil: gDisp = newDispatcher() + if gDisp.isNil: + setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Handle = + ## Returns the underlying IO Completion Port handle (Windows) or selector + ## (Unix) for the specified dispatcher. + return disp.ioPort + proc register*(fd: AsyncFD) = - ## Registers ``fd`` with the dispatcher. + ## Registers `fd` with the dispatcher. let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, cast[CompletionKey](fd), 1) == 0: raiseOSError(osLastError()) @@ -414,28 +380,42 @@ when defined(windows) or defined(nimdoc): proc verifyPresence(fd: AsyncFD) = ## Ensures that file descriptor has been registered with the dispatcher. + ## Raises ValueError if `fd` has not been registered. let p = getGlobalDispatcher() if fd notin p.handles: raise newException(ValueError, "Operation performed on a socket which has not been registered with" & " the dispatcher yet.") - proc poll*(timeout = 500) = - ## Waits for completion events and processes them. + proc hasPendingOperations*(): bool = + ## Returns `true` if the global dispatcher has pending operations. + let p = getGlobalDispatcher() + p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 + + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() - if p.handles.len == 0 and p.timers.len == 0: + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") - let llTimeout = - if timeout == -1: winlean.INFINITE - else: timeout.int32 - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG - var customOverlapped: PCustomOverlapped + result = false + let nextTimer = processTimers(p, result) + let at = adjustTimeout(p, timeout, nextTimer) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: DWORD + var lpCompletionKey: ULONG_PTR + var customOverlapped: CustomRef let res = getQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, addr lpCompletionKey, cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + result = true + # For 'gcDestructors' the destructor of 'customOverlapped' will + # be called at the end and we are the only owner here. This means + # We do not have to 'GC_unref(customOverlapped)' because the destructor + # does that for us. # http://stackoverflow.com/a/12277264/492186 # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html @@ -445,162 +425,84 @@ when defined(windows) or defined(nimdoc): customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, OSErrorCode(-1)) - GC_unref(customOverlapped) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + + when not defined(gcDestructors): + GC_unref(customOverlapped) else: let errCode = osLastError() if customOverlapped != nil: assert customOverlapped.data.fd == lpCompletionKey.AsyncFD customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, errCode) - GC_unref(customOverlapped) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + when not defined(gcDestructors): + GC_unref(customOverlapped) else: if errCode.int32 == WAIT_TIMEOUT: # Timed out - discard + result = false else: raiseOSError(errCode) # Timer processing. - processTimers(p) + discard processTimers(p, result) + # Callback queue processing + processPendingCallbacks(p, result) - var connectExPtr: pointer = nil - var acceptExPtr: pointer = nil - var getAcceptExSockAddrsPtr: pointer = nil + + var acceptEx: WSAPROC_ACCEPTEX + var connectEx: WSAPROC_CONNECTEX + var getAcceptExSockAddrs: WSAPROC_GETACCEPTEXSOCKADDRS proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c - var bytesRet: Dword + var bytesRet: DWORD fun = nil result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, - sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, + sizeof(GUID).DWORD, addr fun, sizeof(pointer).DWORD, addr bytesRet, nil, nil) == 0 proc initAll() = - let dummySock = newNativeSocket() - if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX): + let dummySock = createNativeSocket() + if dummySock == INVALID_SOCKET: raiseOSError(osLastError()) - if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX): + var fun: pointer = nil + if not initPointer(dummySock, fun, WSAID_CONNECTEX): raiseOSError(osLastError()) - if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS): + connectEx = cast[WSAPROC_CONNECTEX](fun) + if not initPointer(dummySock, fun, WSAID_ACCEPTEX): raiseOSError(osLastError()) - - proc connectEx(s: SocketHandle, name: ptr SockAddr, namelen: cint, - lpSendBuffer: pointer, dwSendDataLength: Dword, - lpdwBytesSent: PDword, lpOverlapped: POVERLAPPED): bool = - if connectExPtr.isNil: raise newException(ValueError, "Need to initialise ConnectEx().") - let fun = - cast[proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, - lpSendBuffer: pointer, dwSendDataLength: Dword, - lpdwBytesSent: PDword, lpOverlapped: POVERLAPPED): bool {.stdcall,gcsafe.}](connectExPtr) - - result = fun(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent, - lpOverlapped) - - proc acceptEx(listenSock, acceptSock: SocketHandle, lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: Dword, lpdwBytesReceived: PDword, - lpOverlapped: POVERLAPPED): bool = - if acceptExPtr.isNil: raise newException(ValueError, "Need to initialise AcceptEx().") - let fun = - cast[proc (listenSock, acceptSock: SocketHandle, lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: Dword, lpdwBytesReceived: PDword, - lpOverlapped: POVERLAPPED): bool {.stdcall,gcsafe.}](acceptExPtr) - result = fun(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, - lpOverlapped) - - proc getAcceptExSockaddrs(lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: Dword, - LocalSockaddr: ptr ptr SockAddr, LocalSockaddrLength: LPInt, - RemoteSockaddr: ptr ptr SockAddr, RemoteSockaddrLength: LPInt) = - if getAcceptExSockAddrsPtr.isNil: - raise newException(ValueError, "Need to initialise getAcceptExSockAddrs().") - - let fun = - cast[proc (lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: Dword, LocalSockaddr: ptr ptr SockAddr, - LocalSockaddrLength: LPInt, RemoteSockaddr: ptr ptr SockAddr, - RemoteSockaddrLength: LPInt) {.stdcall,gcsafe.}](getAcceptExSockAddrsPtr) - - fun(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, - RemoteSockaddr, RemoteSockaddrLength) - - proc connect*(socket: AsyncFD, address: string, port: Port, - domain = nativesockets.AF_INET): Future[void] = - ## Connects ``socket`` to server at ``address:port``. - ## - ## Returns a ``Future`` which will complete when the connection succeeds - ## or an error occurs. - verifyPresence(socket) - var retFuture = newFuture[void]("connect") - # Apparently ``ConnectEx`` expects the socket to be initially bound: - var saddr: Sockaddr_in - saddr.sin_family = int16(toInt(domain)) - saddr.sin_port = 0 - saddr.sin_addr.s_addr = INADDR_ANY - if bindAddr(socket.SocketHandle, cast[ptr SockAddr](addr(saddr)), - sizeof(saddr).SockLen) < 0'i32: + acceptEx = cast[WSAPROC_ACCEPTEX](fun) + if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): raiseOSError(osLastError()) + getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) + close(dummySock) - var aiList = getAddrInfo(address, port, domain) - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - # "the OVERLAPPED structure must remain valid until the I/O completes" - # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - var ret = connectEx(socket.SocketHandle, it.ai_addr, - sizeof(Sockaddr_in).cint, nil, 0, nil, - cast[POVERLAPPED](ol)) - if ret: - # Request to connect completed immediately. - success = true - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - break - else: - lastError = osLastError() - if lastError.int32 == ERROR_IO_PENDING: - # In this case ``ol`` will be deallocated in ``poll``. - success = true - break - else: - GC_unref(ol) - success = false - it = it.ai_next - - dealloc(aiList) - if not success: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - return retFuture + proc newCustom*(): CustomRef = + result = CustomRef() # 0 + GC_ref(result) # 1 prevent destructor from doing a premature free. + # destructor of newCustom's caller --> 0. This means + # Windows holds a ref for us with RC == 0 (single owner). + # This is passed back to us in the IO completion port. proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - ## Reads **up to** ``size`` bytes from ``socket``. Returned future will + flags = {SocketFlag.SafeDisconn}): owned(Future[string]) = + ## Reads **up to** `size` bytes from `socket`. Returned future will ## complete once all the data requested is read, a part of the data has been ## read, or the socket has disconnected in which case the future will - ## complete with a value of ``""``. + ## complete with a value of `""`. ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + ## .. warning:: The `Peek` socket flag is not supported on Windows. # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very + # * When WSARecv completes immediately then `bytesReceived` is very # unreliable. # * Still need to implement message-oriented socket disconnection, # '\0' in the message currently signifies a socket disconnect. Who @@ -613,12 +515,11 @@ when defined(windows) or defined(nimdoc): dataBuf.buf = cast[cstring](alloc0(size)) dataBuf.len = size.ULONG - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) + var bytesReceived: DWORD + var flagsio = flags.toOSFlags().DWORD + var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): if bytesCount == 0 and dataBuf.buf[0] == '\0': @@ -632,7 +533,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dealloc dataBuf.buf dataBuf.buf = nil @@ -650,50 +551,32 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': - # We have to ensure that the buffer is empty because WSARecv will tell - # us immediately when it was disconnected, even when there is still - # data in the buffer. - # We want to give the user as much data as we can. So we only return - # the empty string (which signals a disconnection) when there is - # nothing left to read. - retFuture.complete("") - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." - # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx - else: - # Request to read completed immediately. - # From my tests bytesReceived isn't reliable. - let realSize = - if bytesReceived == 0: - size - else: - bytesReceived - var data = newString(realSize) - assert realSize <= size - copyMem(addr data[0], addr dataBuf.buf[0], realSize) - #dealloc dataBuf.buf - retFuture.complete($data) - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. + retFuture.fail(newOSError(err)) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") return retFuture - proc recvInto*(socket: AsyncFD, buf: cstring, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + proc recvInto*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = + ## Reads **up to** `size` bytes from `socket` into `buf`, which must ## at least be of that size. Returned future will complete once all the ## data requested is read, a part of the data has been read, or the socket ## has disconnected in which case the future will complete with a value of - ## ``0``. + ## `0`. ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + ## .. warning:: The `Peek` socket flag is not supported on Windows. # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very + # * When WSARecv completes immediately then `bytesReceived` is very # unreliable. # * Still need to implement message-oriented socket disconnection, # '\0' in the message currently signifies a socket disconnect. Who @@ -705,26 +588,22 @@ when defined(windows) or defined(nimdoc): #buf[] = '\0' var dataBuf: TWSABuf - dataBuf.buf = buf + dataBuf.buf = cast[cstring](buf) dataBuf.len = size.ULONG - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) + var bytesReceived: DWORD + var flagsio = flags.toOSFlags().DWORD + var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): - if bytesCount == 0 and dataBuf.buf[0] == '\0': - retFuture.complete(0) - else: - retFuture.complete(bytesCount) + retFuture.complete(bytesCount) else: if flags.isDisconnectionError(errcode): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dataBuf.buf = nil ) @@ -740,50 +619,35 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': - # We have to ensure that the buffer is empty because WSARecv will tell - # us immediately when it was disconnected, even when there is still - # data in the buffer. - # We want to give the user as much data as we can. So we only return - # the empty string (which signals a disconnection) when there is - # nothing left to read. - retFuture.complete(0) - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." - # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx - else: - # Request to read completed immediately. - # From my tests bytesReceived isn't reliable. - let realSize = - if bytesReceived == 0: - size - else: - bytesReceived - assert realSize <= size - retFuture.complete(realSize) - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. + retFuture.fail(newOSError(err)) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) return retFuture - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to ``socket``. The returned future will complete once all - ## data has been sent. + proc send*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = + ## Sends `size` bytes from `buf` to `socket`. The returned future + ## will complete once all data has been sent. + ## + ## .. warning:: Use it with caution. If `buf` refers to GC'ed object, + ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. verifyPresence(socket) var retFuture = newFuture[void]("send") var dataBuf: TWSABuf - dataBuf.buf = data # since this is not used in a callback, this is fine - dataBuf.len = data.len.ULONG + dataBuf.buf = cast[cstring](buf) + dataBuf.len = size.ULONG - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) + var bytesReceived, lowFlags: DWORD + var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): retFuture.complete() @@ -791,7 +655,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, @@ -803,16 +667,110 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) + else: + retFuture.complete() + # We don't deallocate `ol` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free `ol`. + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = + ## Sends `data` to specified destination `saddr`, using + ## socket `socket`. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.DWORD + var lowFlags = 0.DWORD + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = newCustom() + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newOSError(errcode)) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newOSError(err)) else: retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed + # We don't deallocate `ol` here because even though this completed # immediately poll will still be notified about its completion and it will - # free ``ol``. + # free `ol`. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = + ## Receives a datagram data from `socket` into `buf`, which must + ## be at least of size `size`, address of datagram's sender will be + ## stored into `saddr` and `saddrLen`. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.DWORD + var lowFlags = 0.DWORD + + var ol = newCustom() + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newOSError(errcode)) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newOSError(err)) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) return retFuture - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): + owned(Future[tuple[address: string, client: AsyncFD]]) {.gcsafe.} = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. @@ -820,44 +778,27 @@ when defined(windows) or defined(nimdoc): ## The resulting client socket is automatically registered to the ## dispatcher. ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## If `inheritable` is false (the default), the resulting client socket will + ## not be inheritable by child processes. + ## + ## The `accept` call may result in an error if the connecting socket + ## disconnects during the duration of the `accept`. If the `SafeDisconn` ## flag is specified then this error will not be raised and instead ## accept will be called again. verifyPresence(socket) var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - var clientSock = newNativeSocket() + var clientSock = createNativeSocket(inheritable = inheritable) if clientSock == osInvalidSocket: raiseOSError(osLastError()) const lpOutputLen = 1024 var lpOutputBuf = newString(lpOutputLen) - var dwBytesReceived: Dword - let dwReceiveDataLength = 0.Dword # We don't want any data to be read. - let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) - let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) + var dwBytesReceived: DWORD + let dwReceiveDataLength = 0.DWORD # We don't want any data to be read. + let dwLocalAddressLength = DWORD(sizeof(Sockaddr_in6) + 16) + let dwRemoteAddressLength = DWORD(sizeof(Sockaddr_in6) + 16) - template completeAccept(): stmt {.immediate, dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).SockLen) - if setoptRet != 0: raiseOSError(osLastError()) - - var localSockaddr, remoteSockaddr: ptr SockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr localSockaddr, addr localLen, - addr remoteSockaddr, addr remoteLen) - register(clientSock.AsyncFD) - # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 - retFuture.complete( - (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), - client: clientSock.AsyncFD) - ) - - template failAccept(errcode): stmt = + template failAccept(errcode) = if flags.isDisconnectionError(errcode): var newAcceptFut = acceptAddr(socket, flags) newAcceptFut.callback = @@ -867,12 +808,36 @@ when defined(windows) or defined(nimdoc): else: retFuture.complete(newAcceptFut.read) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) - var ol = PCustomOverlapped() - GC_ref(ol) + template completeAccept() {.dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: + let errcode = osLastError() + discard clientSock.closesocket() + failAccept(errcode) + else: + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockAddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + try: + let address = getAddrString(remoteSockaddr) + register(clientSock.AsyncFD) + retFuture.complete((address: address, client: clientSock.AsyncFD)) + except: + # getAddrString may raise + clientSock.close() + retFuture.fail(getCurrentException()) + + var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = if not retFuture.finished: if errcode == OSErrorCode(-1): completeAccept() @@ -894,25 +859,13 @@ when defined(windows) or defined(nimdoc): GC_unref(ol) else: completeAccept() - # We don't deallocate ``ol`` here because even though this completed + # We don't deallocate `ol` here because even though this completed # immediately poll will still be notified about its completion and it will - # free ``ol``. + # free `ol`. return retFuture - proc newAsyncNativeSocket*(domain, sockType, protocol: cint): AsyncFD = - ## Creates a new socket and registers it with the dispatcher implicitly. - result = newNativeSocket(domain, sockType, protocol).AsyncFD - result.SocketHandle.setBlocking(false) - register(result) - - proc newAsyncNativeSocket*(domain: Domain = nativesockets.AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): AsyncFD = - ## Creates a new socket and registers it with the dispatcher implicitly. - result = newNativeSocket(domain, sockType, protocol).AsyncFD - result.SocketHandle.setBlocking(false) - register(result) + implementSetInheritable() proc closeSocket*(socket: AsyncFD) = ## Closes a socket and ensures that it is unregistered. @@ -920,186 +873,586 @@ when defined(windows) or defined(nimdoc): getGlobalDispatcher().handles.excl(socket) proc unregister*(fd: AsyncFD) = - ## Unregisters ``fd``. + ## Unregisters `fd`. getGlobalDispatcher().handles.excl(fd) + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = + return fd in disp.handles + + {.push stackTrace: off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL) {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.DWORD, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: DWORD) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).DWORD + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = newCustom() + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(err) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + let err = osLastError() + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(err) + else: + # we incref `pcd.ovl` and `protect` callback one more time, + # because it will be unrefed and disposed in `poll()` after + # callback finishes. + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + let err = osLastError() + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(err) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + let err = osLastError() + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(err) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback `cb`. + ## + ## This is not `pure` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of unix-like libraries to be + ## asynchronous on Windows). + ## + ## If you use this function, you don't need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback `cb` returns `true`, if you want to remove + ## watch of `read` notifications, and `false`, if you want to continue + ## receiving notifications. + registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback `cb`. + ## + ## This is not `pure` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of unix-like libraries to be + ## asynchronous on Windows). + ## + ## If you use this function, you don't need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback `cb` returns `true`, if you want to remove + ## watch of `write` notifications, and `false`, if you want to continue + ## receiving notifications. + registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) + + template registerWaitableHandle(p, hEvent, flags, pcd, timeout, + handleCallback) = + let handleFD = AsyncFD(hEvent) + pcd.ioPort = p.ioPort + pcd.handleFd = handleFD + var ol = newCustom() + ol.data.fd = handleFD + ol.data.cb = handleCallback + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), timeout.DWORD, flags): + let err = osLastError() + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard closeHandle(hEvent) + raiseOSError(err) + p.handles.incl(handleFD) + + template closeWaitable(handle: untyped) = + let waitFd = pcd.waitFd + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + if unregisterWait(waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + discard closeHandle(handle) + raiseOSError(err) + if closeHandle(handle) == 0: + raiseOSError(osLastError()) + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Registers callback `cb` to be called when timer expired. + ## + ## Parameters: + ## + ## * `timeout` - timeout value in milliseconds. + ## * `oneshot` + ## * `true` - generate only one timeout event + ## * `false` - generate timeout events periodically + + doAssert(timeout > 0) + let p = getGlobalDispatcher() + + var hEvent = createEvent(nil, 1, 0, nil) + if hEvent == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.DWORD + if oneshot: flags = flags or WT_EXECUTEONLYONCE + + proc timercb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + let res = cb(fd) + if res or oneshot: + closeWaitable(hEvent) + else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + + registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) + + proc addProcess*(pid: int, cb: Callback) = + ## Registers callback `cb` to be called when process with process ID + ## `pid` exited. + const NULL = Handle(0) + let p = getGlobalDispatcher() + let procFlags = SYNCHRONIZE + var hProcess = openProcess(procFlags, 0, pid.DWORD) + if hProcess == NULL: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.DWORD or WT_EXECUTEONLYONCE.DWORD + + proc proccb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + closeWaitable(hProcess) + discard cb(fd) + + registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates a new thread-safe `AsyncEvent` object. + ## + ## New `AsyncEvent` object is not automatically registered with + ## dispatcher like `AsyncSocket`. + var sa = SECURITY_ATTRIBUTES( + nLength: sizeof(SECURITY_ATTRIBUTES).cint, + bInheritHandle: 1 + ) + var event = createEvent(addr(sa), 0'i32, 0'i32, nil) + if event == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + result.hEvent = event + + proc trigger*(ev: AsyncEvent) = + ## Set event `ev` to signaled state. + if setEvent(ev.hEvent) == 0: + raiseOSError(osLastError()) + + proc unregister*(ev: AsyncEvent) = + ## Unregisters event `ev`. + doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") + let p = getGlobalDispatcher() + p.handles.excl(AsyncFD(ev.hEvent)) + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(err) + ev.hWaiter = 0 + + proc close*(ev: AsyncEvent) = + ## Closes event `ev`. + let res = closeHandle(ev.hEvent) + deallocShared(cast[pointer](ev)) + if res == 0: + raiseOSError(osLastError()) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Registers callback `cb` to be called when `ev` will be signaled + doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") + + let p = getGlobalDispatcher() + let hEvent = ev.hEvent + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.DWORD + + proc eventcb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + if ev.hWaiter != 0: + if cb(fd): + # we need this check to avoid exception, if `unregister(event)` was + # called in callback. + deallocShared(cast[pointer](pcd)) + if ev.hWaiter != 0: + unregister(ev) + else: + # if callback returned `false`, then it wants to be called again, so + # we need to ref and protect `pcd.ovl` again, because it will be + # unrefed and disposed in `poll()`. + GC_ref(pcd.ovl) + pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) + else: + # if ev.hWaiter == 0, then event was unregistered before `poll()` call. + deallocShared(cast[pointer](pcd)) + + registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) + ev.hWaiter = pcd.waitFd + initAll() else: - import selectors - when defined(windows): - import winlean - const - EINTR = WSAEINPROGRESS - EINPROGRESS = WSAEINPROGRESS - EWOULDBLOCK = WSAEWOULDBLOCK - EAGAIN = EINPROGRESS - MSG_NOSIGNAL = 0 - else: - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, - MSG_NOSIGNAL - + import std/selectors + from std/posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL + when declared(posix.accept4): + from std/posix import accept4, SOCK_CLOEXEC + when defined(genode): + import genode/env # get the implicit Genode env + import genode/signals + + const + InitCallbackListSize = 4 # initial size of callbacks sequence, + # associated with file/socket descriptor. + InitDelayedCallbackListSize = 64 # initial size of delayed callbacks + # queue. type AsyncFD* = distinct cint - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.} - PData* = ref object of RootRef - fd: AsyncFD - readCBs: seq[Callback] - writeCBs: seq[Callback] + AsyncData = object + readList: seq[Callback] + writeList: seq[Callback] + + AsyncEvent* = distinct SelectEvent PDispatcher* = ref object of PDispatcherBase - selector: Selector - {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} + selector: Selector[AsyncData] + when defined(genode): + signalHandler: SignalHandler proc `==`*(x, y: AsyncFD): bool {.borrow.} + proc `==`*(x, y: AsyncEvent): bool {.borrow.} + + template newAsyncData(): AsyncData = + AsyncData( + readList: newSeqOfCap[Callback](InitCallbackListSize), + writeList: newSeqOfCap[Callback](InitCallbackListSize) + ) - proc newDispatcher*(): PDispatcher = + proc newDispatcher*(): owned(PDispatcher) = new result - result.selector = newSelector() - result.timers = @[] + result.selector = newSelector[AsyncData]() + result.timers.clear() + result.callbacks = initDeque[proc () {.closure, gcsafe.}](InitDelayedCallbackListSize) + when defined(genode): + let entrypoint = ep(cast[GenodeEnv](runtimeEnv)) + result.signalHandler = newSignalHandler(entrypoint): + discard runOnce(0) + + var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + + proc setGlobalDispatcher*(disp: owned PDispatcher) = + if not gDisp.isNil: + assert gDisp.callbacks.len == 0 + gDisp = disp + initCallSoonProc() - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: gDisp = newDispatcher() + if gDisp.isNil: + setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp - proc update(fd: AsyncFD, events: set[Event]) = - let p = getGlobalDispatcher() - assert fd.SocketHandle in p.selector - p.selector.update(fd.SocketHandle, events) + proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = + return disp.selector proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() - var data = PData(fd: fd, readCBs: @[], writeCBs: @[]) - p.selector.register(fd.SocketHandle, {}, data.RootRef) - - proc newAsyncNativeSocket*(domain: cint, sockType: cint, - protocol: cint): AsyncFD = - result = newNativeSocket(domain, sockType, protocol).AsyncFD - result.SocketHandle.setBlocking(false) - when defined(macosx): - result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) - register(result) - - proc newAsyncNativeSocket*(domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): AsyncFD = - result = newNativeSocket(domain, sockType, protocol).AsyncFD - result.SocketHandle.setBlocking(false) - when defined(macosx): - result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) - register(result) - - proc closeSocket*(sock: AsyncFD) = - let disp = getGlobalDispatcher() - disp.selector.unregister(sock.SocketHandle) - sock.SocketHandle.close() + var data = newAsyncData() + p.selector.registerHandle(fd.SocketHandle, {}, data) proc unregister*(fd: AsyncFD) = getGlobalDispatcher().selector.unregister(fd.SocketHandle) + proc unregister*(ev: AsyncEvent) = + getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = + return fd.SocketHandle in disp.selector + proc addRead*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() - if fd.SocketHandle notin p.selector: + var newEvents = {Event.Read} + withData(p.selector, fd.SocketHandle, adata) do: + adata.readList.add(cb) + newEvents.incl(Event.Read) + if len(adata.writeList) != 0: newEvents.incl(Event.Write) + do: raise newException(ValueError, "File descriptor not registered.") - p.selector[fd.SocketHandle].data.PData.readCBs.add(cb) - update(fd, p.selector[fd.SocketHandle].events + {EvRead}) + p.selector.updateHandle(fd.SocketHandle, newEvents) proc addWrite*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() - if fd.SocketHandle notin p.selector: + var newEvents = {Event.Write} + withData(p.selector, fd.SocketHandle, adata) do: + adata.writeList.add(cb) + newEvents.incl(Event.Write) + if len(adata.readList) != 0: newEvents.incl(Event.Read) + do: raise newException(ValueError, "File descriptor not registered.") - p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb) - update(fd, p.selector[fd.SocketHandle].events + {EvWrite}) + p.selector.updateHandle(fd.SocketHandle, newEvents) - proc poll*(timeout = 500) = + proc hasPendingOperations*(): bool = let p = getGlobalDispatcher() - for info in p.selector.select(timeout): - let data = PData(info.key.data) - assert data.fd == info.key.fd.AsyncFD - #echo("In poll ", data.fd.cint) - # There may be EvError here, but we handle them in callbacks, - # so that exceptions can be raised from `send(...)` and - # `recv(...)` routines. - - if EvRead in info.events: - # Callback may add items to ``data.readCBs`` which causes issues if - # we are iterating over ``data.readCBs`` at the same time. We therefore - # make a copy to iterate over. - let currentCBs = data.readCBs - data.readCBs = @[] - for cb in currentCBs: - if not cb(data.fd): - # Callback wants to be called again. - data.readCBs.add(cb) - - if EvWrite in info.events: - let currentCBs = data.writeCBs - data.writeCBs = @[] - for cb in currentCBs: - if not cb(data.fd): - # Callback wants to be called again. - data.writeCBs.add(cb) - - if info.key in p.selector: - var newEvents: set[Event] - if data.readCBs.len != 0: newEvents = {EvRead} - if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite} - if newEvents != info.key.events: - update(data.fd, newEvents) + not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 + + proc prependSeq(dest: var seq[Callback]; src: sink seq[Callback]) = + var old = move dest + dest = src + for i in 0..high(old): + dest.add(move old[i]) + + proc processBasicCallbacks( + fd: AsyncFD, event: Event + ): tuple[readCbListCount, writeCbListCount: int] = + # Process pending descriptor and AsyncEvent callbacks. + # + # Invoke every callback stored in `rwlist`, until one + # returns `false` (which means callback wants to stay + # alive). In such case all remaining callbacks will be added + # to `rwlist` again, in the order they have been inserted. + # + # `rwlist` associated with file descriptor MUST BE emptied before + # dispatching callback (See https://github.com/nim-lang/Nim/issues/5128), + # or it can be possible to fall into endless cycle. + var curList: seq[Callback] + + let selector = getGlobalDispatcher().selector + withData(selector, fd.int, fdData): + case event + of Event.Read: + #shallowCopy(curList, fdData.readList) + curList = move fdData.readList + fdData.readList = newSeqOfCap[Callback](InitCallbackListSize) + of Event.Write: + #shallowCopy(curList, fdData.writeList) + curList = move fdData.writeList + fdData.writeList = newSeqOfCap[Callback](InitCallbackListSize) else: - # FD no longer a part of the selector. Likely been closed - # (e.g. socket disconnected). - discard + assert false, "Cannot process callbacks for " & $event + + let newLength = max(len(curList), InitCallbackListSize) + var newList = newSeqOfCap[Callback](newLength) + + var eventsExtinguished = false + for cb in curList: + if eventsExtinguished: + newList.add(cb) + elif not cb(fd): + # Callback wants to be called again. + newList.add(cb) + # This callback has returned with EAGAIN, so we don't need to + # call any other callbacks as they are all waiting for the same event + # on the same fd. + # We do need to ensure they are called again though. + eventsExtinguished = true + + withData(selector, fd.int, fdData) do: + # Descriptor is still present in the queue. + case event + of Event.Read: prependSeq(fdData.readList, newList) + of Event.Write: prependSeq(fdData.writeList, newList) + else: + assert false, "Cannot process callbacks for " & $event + + result.readCbListCount = len(fdData.readList) + result.writeCbListCount = len(fdData.writeList) + do: + # Descriptor was unregistered in callback via `unregister()`. + result.readCbListCount = -1 + result.writeCbListCount = -1 + + proc processCustomCallbacks(p: PDispatcher; fd: AsyncFD) = + # Process pending custom event callbacks. Custom events are + # {Event.Timer, Event.Signal, Event.Process, Event.Vnode}. + # There can be only one callback registered with one descriptor, + # so there is no need to iterate over list. + var curList: seq[Callback] + + withData(p.selector, fd.int, adata) do: + curList = move adata.readList + adata.readList = newSeqOfCap[Callback](InitCallbackListSize) + + let newLength = len(curList) + var newList = newSeqOfCap[Callback](newLength) + + var cb = curList[0] + if not cb(fd): + newList.add(cb) + + withData(p.selector, fd.int, adata) do: + # descriptor still present in queue. + adata.readList = newList & adata.readList + if len(adata.readList) == 0: + # if no callbacks registered with descriptor, unregister it. + p.selector.unregister(fd.int) + do: + # descriptor was unregistered in callback via `unregister()`. + discard + + implementSetInheritable() - processTimers(p) + proc closeSocket*(sock: AsyncFD) = + let selector = getGlobalDispatcher().selector + if sock.SocketHandle notin selector: + raise newException(ValueError, "File descriptor not registered.") - proc connect*(socket: AsyncFD, address: string, port: Port, - domain = AF_INET): Future[void] = - var retFuture = newFuture[void]("connect") + let data = selector.getData(sock.SocketHandle) + sock.unregister() + sock.SocketHandle.close() + # We need to unblock the read and write callbacks which could still be + # waiting for the socket to become readable and/or writeable. + for cb in data.readList & data.writeList: + if not cb(sock): + raise newException( + ValueError, "Expecting async operations to stop when fd has closed." + ) + + proc runOnce(timeout: int): bool = + let p = getGlobalDispatcher() + if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + when defined(genode): + if timeout == 0: return + raise newException(ValueError, + "No handles or timers registered in dispatcher.") - proc cb(fd: AsyncFD): bool = - var ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) - if ret == 0: - # We have connected. - retFuture.complete() - return true - elif ret == EINTR: - # interrupted, keep waiting - return false - else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) - return true + result = false + var keys: array[64, ReadyKey] + let nextTimer = processTimers(p, result) + var count = + p.selector.selectInto(adjustTimeout(p, timeout, nextTimer), keys) + for i in 0..<count: + let fd = keys[i].fd.AsyncFD + let events = keys[i].events + var (readCbListCount, writeCbListCount) = (0, 0) + + if Event.Read in events or events == {Event.Error}: + (readCbListCount, writeCbListCount) = + processBasicCallbacks(fd, Event.Read) + result = true + + if Event.Write in events or events == {Event.Error}: + (readCbListCount, writeCbListCount) = + processBasicCallbacks(fd, Event.Write) + result = true + + var isCustomEvent = false + if Event.User in events: + (readCbListCount, writeCbListCount) = + processBasicCallbacks(fd, Event.Read) + isCustomEvent = true + if readCbListCount == 0: + p.selector.unregister(fd.int) + result = true + + when ioselSupportedPlatform: + const customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode} + if (customSet * events) != {}: + isCustomEvent = true + processCustomCallbacks(p, fd) + result = true - assert getSockDomain(socket.SocketHandle) == domain - var aiList = getAddrInfo(address, port, domain) - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - var ret = connect(socket.SocketHandle, it.ai_addr, it.ai_addrlen.Socklen) - if ret == 0: - # Request to connect completed immediately. - success = true - retFuture.complete() - break - else: - lastError = osLastError() - if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: - success = true - addWrite(socket, cb) - break - else: - success = false - it = it.ai_next + # because state `data` can be modified in callback we need to update + # descriptor events with currently registered callbacks. + if not isCustomEvent and (readCbListCount != -1 and writeCbListCount != -1): + var newEvents: set[Event] = {} + if readCbListCount > 0: incl(newEvents, Event.Read) + if writeCbListCount > 0: incl(newEvents, Event.Write) + p.selector.updateHandle(SocketHandle(fd), newEvents) - dealloc(aiList) - if not success: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - return retFuture + # Timer processing. + discard processTimers(p, result) + # Callback queue processing + processPendingCallbacks(p, result) proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = + flags = {SocketFlag.SafeDisconn}): owned(Future[string]) = var retFuture = newFuture[string]("recv") var readBuffer = newString(size) @@ -1110,11 +1463,12 @@ else: flags.toOSFlags()) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: if flags.isDisconnectionError(lastError): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -1128,8 +1482,8 @@ else: addRead(socket, cb) return retFuture - proc recvInto*(socket: AsyncFD, buf: cstring, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = + proc recvInto*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = var retFuture = newFuture[int]("recvInto") proc cb(sock: AsyncFD): bool = @@ -1138,11 +1492,12 @@ else: flags.toOSFlags()) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: if flags.isDisconnectionError(lastError): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1152,25 +1507,27 @@ else: addRead(socket, cb) return retFuture - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = + proc send*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = var retFuture = newFuture[void]("send") var written = 0 proc cb(sock: AsyncFD): bool = result = true - let netSize = data.len-written - var d = data.cstring + let netSize = size-written + var d = cast[cstring](buf) let res = send(sock.SocketHandle, addr d[written], netSize.cint, MSG_NOSIGNAL) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if lastError.int32 != EINTR and + lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: if flags.isDisconnectionError(lastError): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1184,47 +1541,427 @@ else: addWrite(socket, cb) return retFuture - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = + ## Sends `data` of size `size` in bytes to specified destination + ## (`saddr` of size `saddrLen` in bytes, using socket `socket`. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: + retFuture.fail(newOSError(lastError)) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = + ## Receives a datagram data from `socket` into `data`, which must + ## be at least of size `size` in bytes, address of datagram's sender + ## will be stored into `saddr` and `saddrLen`. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: + retFuture.fail(newOSError(lastError)) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): + owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage - var addrLen = sizeof(sockAddress).Socklen - var client = accept(sock.SocketHandle, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + var addrLen = sizeof(sockAddress).SockLen + var client = + when declared(accept4): + accept4(sock.SocketHandle, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen), if inheritable: 0 else: SOCK_CLOEXEC) + else: + accept(sock.SocketHandle, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + when declared(setInheritable) and not declared(accept4): + if client != osInvalidSocket and not setInheritable(client, inheritable): + # Set failure first because close() itself can fail, + # altering osLastError(). + retFuture.fail(newOSError(osLastError())) + close client + return false + if client == osInvalidSocket: let lastError = osLastError() - assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + assert lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN if lastError.int32 == EINTR: return false else: if flags.isDisconnectionError(lastError): return false else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: - register(client.AsyncFD) - retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), client.AsyncFD)) + try: + let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) + register(client.AsyncFD) + retFuture.complete((address, client.AsyncFD)) + except: + # getAddrString may raise + client.close() + retFuture.fail(getCurrentException()) addRead(socket, cb) return retFuture -proc sleepAsync*(ms: int): Future[void] = + when ioselSupportedPlatform: + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Start watching for timeout expiration, and then call the + ## callback `cb`. + ## `timeout` - time in milliseconds, + ## `oneshot` - if `true` only one event will be dispatched, + ## if `false` continuous events every `timeout` milliseconds. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerTimer(timeout, oneshot, data) + + proc addSignal*(signal: int, cb: Callback) = + ## Start watching signal `signal`, and when signal appears, call the + ## callback `cb`. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerSignal(signal, data) + + proc addProcess*(pid: int, cb: Callback) = + ## Start watching for process exit with pid `pid`, and then call + ## the callback `cb`. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerProcess(pid, data) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new `AsyncEvent`. + result = AsyncEvent(newSelectEvent()) + + proc trigger*(ev: AsyncEvent) = + ## Sets new `AsyncEvent` to signaled state. + trigger(SelectEvent(ev)) + + proc close*(ev: AsyncEvent) = + ## Closes `AsyncEvent` + close(SelectEvent(ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Start watching for event `ev`, and call callback `cb`, when + ## ev will be set to signaled state. + let p = getGlobalDispatcher() + var data = newAsyncData() + data.readList.add(cb) + p.selector.registerEvent(SelectEvent(ev), data) + +proc drain*(timeout = 500) = + ## Waits for completion of **all** events and processes them. Raises `ValueError` + ## if there are no pending operations. In contrast to `poll` this + ## processes as many events as are available until the timeout has elapsed. + var curTimeout = timeout + let start = now() + while hasPendingOperations(): + discard runOnce(curTimeout) + curTimeout -= (now() - start).inMilliseconds.int + if curTimeout < 0: + break + +proc poll*(timeout = 500) = + ## Waits for completion events and processes them. Raises `ValueError` + ## if there are no pending operations. This runs the underlying OS + ## `epoll`:idx: or `kqueue`:idx: primitive only once. + discard runOnce(timeout) + +template createAsyncNativeSocketImpl(domain, sockType, protocol: untyped, + inheritable = defined(nimInheritHandles)) = + let handle = createNativeSocket(domain, sockType, protocol, inheritable) + if handle == osInvalidSocket: + return osInvalidSocket.AsyncFD + handle.setBlocking(false) + when defined(macosx) and not defined(nimdoc): + handle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + result = handle.AsyncFD + register(result) + +proc createAsyncNativeSocket*(domain: cint, sockType: cint, + protocol: cint, + inheritable = defined(nimInheritHandles)): AsyncFD = + createAsyncNativeSocketImpl(domain, sockType, protocol, inheritable) + +proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, + inheritable = defined(nimInheritHandles)): AsyncFD = + createAsyncNativeSocketImpl(domain, sockType, protocol, inheritable) + +when defined(windows) or defined(nimdoc): + proc bindToDomain(handle: SocketHandle, domain: Domain) = + # Extracted into a separate proc, because connect() on Windows requires + # the socket to be initially bound. + template doBind(saddr) = + if bindAddr(handle, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + if domain == Domain.AF_INET6: + var saddr: Sockaddr_in6 + saddr.sin6_family = uint16(toInt(domain)) + doBind(saddr) + else: + var saddr: Sockaddr_in + saddr.sin_family = uint16(toInt(domain)) + doBind(saddr) + + proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) = + let retFuture = newFuture[void]("doConnect") + result = retFuture + + var ol = newCustom() + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + const SO_UPDATE_CONNECT_CONTEXT = 0x7010 + socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 + retFuture.complete() + else: + retFuture.fail(newOSError(errcode)) + ) + + let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr, + cint(addrInfo.ai_addrlen), nil, 0, nil, + cast[POVERLAPPED](ol)) + if ret: + # Request to connect completed immediately. + retFuture.complete() + # We don't deallocate `ol` here because even though this completed + # immediately poll will still be notified about its completion and it + # will free `ol`. + else: + let lastError = osLastError() + if lastError.int32 != ERROR_IO_PENDING: + # With ERROR_IO_PENDING `ol` will be deallocated in `poll`, + # and the future will be completed/failed there, too. + GC_unref(ol) + retFuture.fail(newOSError(lastError)) +else: + proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) = + let retFuture = newFuture[void]("doConnect") + result = retFuture + + proc cb(fd: AsyncFD): bool = + let ret = SocketHandle(fd).getSockOptInt( + cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + # We have connected. + retFuture.complete() + return true + elif ret == EINTR: + # interrupted, keep waiting + return false + else: + retFuture.fail(newOSError(OSErrorCode(ret))) + return true + + let ret = connect(socket.SocketHandle, + addrInfo.ai_addr, + addrInfo.ai_addrlen.SockLen) + if ret == 0: + # Request to connect completed immediately. + retFuture.complete() + else: + let lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + addWrite(socket, cb) + else: + retFuture.fail(newOSError(lastError)) + +template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, + protocol: Protocol = IPPROTO_RAW) = + ## Iterates through the AddrInfo linked list asynchronously + ## until the connection can be established. + const shouldCreateFd = not declared(fd) + + when shouldCreateFd: + let sockType = protocol.toSockType() + + var fdPerDomain: array[low(Domain).ord..high(Domain).ord, AsyncFD] + for i in low(fdPerDomain)..high(fdPerDomain): + fdPerDomain[i] = osInvalidSocket.AsyncFD + template closeUnusedFds(domainToKeep = -1) {.dirty.} = + for i, fd in fdPerDomain: + if fd != osInvalidSocket.AsyncFD and i != domainToKeep: + fd.closeSocket() + + var lastException: ref Exception + var curAddrInfo = addrInfo + var domain: Domain + when shouldCreateFd: + var curFd: AsyncFD + else: + var curFd = fd + proc tryNextAddrInfo(fut: Future[void]) {.gcsafe.} = + if fut == nil or fut.failed: + if fut != nil: + lastException = fut.readError() + + while curAddrInfo != nil: + let domainOpt = curAddrInfo.ai_family.toKnownDomain() + if domainOpt.isSome: + domain = domainOpt.unsafeGet() + break + curAddrInfo = curAddrInfo.ai_next + + if curAddrInfo == nil: + freeAddrInfo(addrInfo) + when shouldCreateFd: + closeUnusedFds() + if lastException != nil: + retFuture.fail(lastException) + else: + retFuture.fail(newException( + IOError, "Couldn't resolve address: " & address)) + return + + when shouldCreateFd: + curFd = fdPerDomain[ord(domain)] + if curFd == osInvalidSocket.AsyncFD: + try: + curFd = createAsyncNativeSocket(domain, sockType, protocol) + except: + freeAddrInfo(addrInfo) + closeUnusedFds() + raise getCurrentException() + when defined(windows): + curFd.SocketHandle.bindToDomain(domain) + fdPerDomain[ord(domain)] = curFd + + doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo + curAddrInfo = curAddrInfo.ai_next + else: + freeAddrInfo(addrInfo) + when shouldCreateFd: + closeUnusedFds(ord(domain)) + retFuture.complete(curFd) + else: + retFuture.complete() + + tryNextAddrInfo(nil) + +proc dial*(address: string, port: Port, + protocol: Protocol = IPPROTO_TCP): owned(Future[AsyncFD]) = + ## Establishes connection to the specified `address`:`port` pair via the + ## specified protocol. The procedure iterates through possible + ## resolutions of the `address` until it succeeds, meaning that it + ## seamlessly works with both IPv4 and IPv6. + ## Returns the async file descriptor, registered in the dispatcher of + ## the current thread, ready to send or receive data. + let retFuture = newFuture[AsyncFD]("dial") + result = retFuture + let sockType = protocol.toSockType() + + let aiList = getAddrInfo(address, port, Domain.AF_UNSPEC, sockType, protocol) + asyncAddrInfoLoop(aiList, noFD, protocol) + +proc connect*(socket: AsyncFD, address: string, port: Port, + domain = Domain.AF_INET): owned(Future[void]) = + let retFuture = newFuture[void]("connect") + result = retFuture + + when defined(windows): + verifyPresence(socket) + else: + assert getSockDomain(socket.SocketHandle) == domain + + let aiList = getAddrInfo(address, port, domain) + when defined(windows): + socket.SocketHandle.bindToDomain(domain) + asyncAddrInfoLoop(aiList, socket) + +proc sleepAsync*(ms: int | float): owned(Future[void]) = ## Suspends the execution of the current async procedure for the next - ## ``ms`` milliseconds. + ## `ms` milliseconds. var retFuture = newFuture[void]("sleepAsync") let p = getGlobalDispatcher() - p.timers.add((epochTime() + (ms / 1000), retFuture)) + when ms is int: + p.timers.push((getMonoTime() + initDuration(milliseconds = ms), retFuture)) + elif ms is float: + let ns = (ms * 1_000_000).int64 + p.timers.push((getMonoTime() + initDuration(nanoseconds = ns), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): owned(Future[bool]) = + ## Returns a future which will complete once `fut` completes or after + ## `timeout` milliseconds has elapsed. + ## + ## If `fut` completes first the returned future will hold true, + ## otherwise, if `timeout` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: + if fut.failed: + retFuture.fail(fut.error) + else: + retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) return retFuture proc accept*(socket: AsyncFD, - flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = + flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): owned(Future[AsyncFD]) = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection. + ## + ## If `inheritable` is false (the default), the resulting client socket + ## will not be inheritable by child processes. + ## ## The future will complete when the connection is successfully accepted. var retFut = newFuture[AsyncFD]("accept") - var fut = acceptAddr(socket, flags) + var fut = acceptAddr(socket, flags, inheritable) fut.callback = proc (future: Future[tuple[address: string, client: AsyncFD]]) = assert future.finished @@ -1234,382 +1971,45 @@ proc accept*(socket: AsyncFD, retFut.complete(future.read.client) return retFut -# -- Await Macro - -proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = - # Skips a nest of StmtList's. - result = node - if node[0].kind == nnkStmtList: - result = skipUntilStmtList(node[0]) - -proc skipStmtList(node: NimNode): NimNode {.compileTime.} = - result = node - if node[0].kind == nnkStmtList: - result = node[0] - -template createCb(retFutureSym, iteratorNameSym, - name: expr): stmt {.immediate.} = - var nameIterVar = iteratorNameSym - #{.push stackTrace: off.} - proc cb {.closure,gcsafe.} = - try: - if not nameIterVar.finished: - var next = nameIterVar() - if next == nil: - assert retFutureSym.finished, "Async procedure's (" & - name & ") return Future was not finished." - else: - next.callback = cb - except: - if retFutureSym.finished: - # Take a look at tasyncexceptions for the bug which this fixes. - # That test explains it better than I can here. - raise - else: - retFutureSym.fail(getCurrentException()) - cb() - #{.pop.} -proc generateExceptionCheck(futSym, - tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = - if tryStmt.kind == nnkNilLit: - result = rootReceiver - else: - var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] - let errorNode = newDotExpr(futSym, newIdentNode("error")) - for i in 1 .. <tryStmt.len: - let exceptBranch = tryStmt[i] - if exceptBranch[0].kind == nnkStmtList: - exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) - else: - var exceptIdentCount = 0 - var ifCond: NimNode - for i in 0 .. <exceptBranch.len: - let child = exceptBranch[i] - if child.kind == nnkIdent: - let cond = infix(errorNode, "of", child) - if exceptIdentCount == 0: - ifCond = cond - else: - ifCond = infix(ifCond, "or", cond) - else: - break - exceptIdentCount.inc - - expectKind(exceptBranch[exceptIdentCount], nnkStmtList) - exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) - # -> -> else: raise futSym.error - exceptionChecks.add((newIdentNode("true"), - newNimNode(nnkRaiseStmt).add(errorNode))) - # Read the future if there is no error. - # -> else: futSym.read - let elseNode = newNimNode(nnkElse, fromNode) - elseNode.add newNimNode(nnkStmtList, fromNode) - elseNode[0].add rootReceiver - - let ifBody = newStmtList() - ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) - ifBody.add newIfStmt(exceptionChecks) - ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) - - result = newIfStmt( - (newDotExpr(futSym, newIdentNode("failed")), ifBody) - ) - result.add elseNode - -template createVar(result: var NimNode, futSymName: string, - asyncProc: NimNode, - valueReceiver, rootReceiver: expr, - fromNode: NimNode) = - result = newNimNode(nnkStmtList, fromNode) - var futSym = genSym(nskVar, "future") - result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future<x> - valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read - result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) - -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - tryStmt: NimNode): NimNode {.compileTime.} = - #echo(node.treeRepr) - result = node - case node.kind - of nnkReturnStmt: - result = newNimNode(nnkStmtList, node) - if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) - else: - result.add newCall(newIdentNode("complete"), retFutureSym) - else: - result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) - - result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) - return # Don't process the children of this return stmt - of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": - case node[1].kind - of nnkIdent, nnkInfix: - # await x - result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x - of nnkCall, nnkCommand: - # await foo(p, x) - var futureValue: NimNode - result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, - futureValue, node) - else: - error("Invalid node kind in 'await', got: " & $node[1].kind) - elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": - # foo await x - var newCommand = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], - newCommand, node) - - of nnkVarSection, nnkLetSection: - case node[0][2].kind - of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": - # var x = await y - var newVarSection = node # TODO: Should this use copyNimNode? - result.createVar("future" & $node[0][0].ident, node[0][2][1], - newVarSection[0][2], newVarSection, node) - else: discard - of nnkAsgn: - case node[1].kind - of nnkCommand: - if node[1][0].ident == !"await": - # x = await y - var newAsgn = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) - else: discard - of nnkDiscardStmt: - # discard await x - if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": - var newDiscard = node - result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], - newDiscard[0], newDiscard, node) - of nnkTryStmt: - # try: await x; except: ... - result = newNimNode(nnkStmtList, node) - template wrapInTry(n, tryBody: expr) = - var temp = n - n[0] = tryBody - tryBody = temp - - # Transform ``except`` body. - # TODO: Could we perform some ``await`` transformation here to get it - # working in ``except``? - tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) - - proc processForTry(n: NimNode, i: var int, - res: NimNode): bool {.compileTime.} = - ## Transforms the body of the tryStmt. Does not transform the - ## body in ``except``. - ## Returns true if the tryStmt node was transformed into an ifStmt. - result = false - var skipped = n.skipStmtList() - while i < skipped.len: - var processed = processBody(skipped[i], retFutureSym, - subTypeIsVoid, n) - - # Check if we transformed the node into an exception check. - # This suggests skipped[i] contains ``await``. - if processed.kind != skipped[i].kind or processed.len != skipped[i].len: - processed = processed.skipUntilStmtList() - expectKind(processed, nnkStmtList) - expectKind(processed[2][1], nnkElse) - i.inc - - if not processForTry(n, i, processed[2][1][0]): - # We need to wrap the nnkElse nodes back into a tryStmt. - # As they are executed if an exception does not happen - # inside the awaited future. - # The following code will wrap the nodes inside the - # original tryStmt. - wrapInTry(n, processed[2][1][0]) - - res.add processed - result = true +proc keepAlive(x: string) = + discard "mark 'x' as escaping so that it is put into a closure for us to keep the data alive" + +proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = + ## Sends `data` to `socket`. The returned future will complete once all + ## data has been sent. + var retFuture = newFuture[void]("send") + if data.len > 0: + let sendFut = socket.send(unsafeAddr data[0], data.len, flags) + sendFut.callback = + proc () = + keepAlive(data) + if sendFut.failed: + retFuture.fail(sendFut.error) else: - res.add skipped[i] - i.inc - var i = 0 - if not processForTry(node, i, result): - # If the tryStmt hasn't been transformed we can just put the body - # back into it. - wrapInTry(node, result) - return - else: discard - - for i in 0 .. <result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) - -proc getName(node: NimNode): string {.compileTime.} = - case node.kind - of nnkPostfix: - return $node[1].ident - of nnkIdent: - return $node.ident - of nnkEmpty: - return "anonymous" - else: - error("Unknown name.") - -proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = - ## This macro transforms a single procedure into a closure iterator. - ## The ``async`` macro supports a stmtList holding multiple async procedures. - if prc.kind notin {nnkProcDef, nnkLambda}: - error("Cannot transform this node kind into an async proc." & - " Proc definition or lambda node expected.") - - hint("Processing " & prc[0].getName & " as an async proc.") - - let returnType = prc[3][0] - var baseType: NimNode - # Verify that the return type is a Future[T] - if returnType.kind == nnkBracketExpr: - let fut = repr(returnType[0]) - if fut != "Future": - error("Expected return type of 'Future' got '" & fut & "'") - baseType = returnType[1] - elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": - let fut = repr(returnType[1]) - if fut != "Future": - error("Expected return type of 'Future' got '" & fut & "'") - baseType = returnType[2] - elif returnType.kind == nnkEmpty: - baseType = returnType - else: - error("Expected return type of 'Future' got '" & repr(returnType) & "'") - - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].ident == !"void") - - var outerProcBody = newNimNode(nnkStmtList, prc[6]) - - # -> var retFuture = newFuture[T]() - var retFutureSym = genSym(nskVar, "retFuture") - var subRetType = - if returnType.kind == nnkEmpty: newIdentNode("void") - else: baseType - outerProcBody.add( - newVarStmt(retFutureSym, - newCall( - newNimNode(nnkBracketExpr, prc[6]).add( - newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. - subRetType), - newLit(prc[0].getName)))) # Get type from return type of this proc - - # -> iterator nameIter(): FutureBase {.closure.} = - # -> {.push warning[resultshadowed]: off.} - # -> var result: T - # -> {.pop.} - # -> <proc_body> - # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") - var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], - procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) - outerProcBody.add(closureIterator) - - # -> createCb(retFuture) - #var cbName = newIdentNode("cb") - var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) - outerProcBody.add procCb - - # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) - - result = prc - - # Remove the 'async' pragma. - for i in 0 .. <result[4].len: - if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": - result[4].del(i) - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result[3][0] = parseExpr("Future[void]") - - result[6] = outerProcBody - - #echo(treeRepr(result)) - #if prc[0].getName == "hubConnectionLoop": - # echo(toStrLit(result)) - -macro async*(prc: stmt): stmt {.immediate.} = - ## Macro which processes async procedures into the appropriate - ## iterators and yield statements. - if prc.kind == nnkStmtList: - for oneProc in prc: - result = newStmtList() - result.add asyncSingleProc(oneProc) + retFuture.complete() else: - result = asyncSingleProc(prc) + retFuture.complete() -proc recvLine*(socket: AsyncFD): Future[string] {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once - ## a full line is read or an error occurs. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. - ## The partial line **will be lost**. - ## - ## **Warning**: This assumes that lines are delimited by ``\r\L``. - ## - ## **Note**: This procedure is mostly used for testing. You likely want to - ## use ``asyncnet.recvLine`` instead. + return retFuture - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") +# -- Await Macro +import std/asyncmacro +export asyncmacro +proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} = + ## Returns a future that will complete when all the string data from the + ## specified future stream is retrieved. result = "" - var c = "" while true: - c = await recv(socket, 1) - if c.len == 0: - return "" - if c == "\r": - c = await recv(socket, 1) - assert c == "\l" - addNLIfEmpty() - return - elif c == "\L": - addNLIfEmpty() - return - add(result, c) + let (hasValue, value) = await future.read() + if hasValue: + result.add(value) + else: + break + +proc callSoon(cbproc: proc () {.gcsafe.}) = + getGlobalDispatcher().callbacks.addLast(cbproc) proc runForever*() = ## Begins a never ending global dispatcher poll loop. @@ -1622,3 +2022,44 @@ proc waitFor*[T](fut: Future[T]): T = poll() fut.read + +proc activeDescriptors*(): int {.inline.} = + ## Returns the current number of active file descriptors for the current + ## event loop. This is a cheap operation that does not involve a system call. + when defined(windows): + result = getGlobalDispatcher().handles.len + elif not defined(nimdoc): + result = getGlobalDispatcher().selector.count + +when defined(posix): + import std/posix + +when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): + proc maxDescriptors*(): int {.raises: OSError.} = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. + when defined(windows): + result = 16_700_000 + elif defined(zephyr) or defined(freertos): + result = FD_MAX + else: + var fdLim: RLimit + if getrlimit(RLIMIT_NOFILE, fdLim) < 0: + raiseOSError(osLastError()) + result = int(fdLim.rlim_cur) - 1 + +when defined(genode): + proc scheduleCallbacks*(): bool {.discardable.} = + ## *Genode only.* + ## Schedule callback processing and return immediately. + ## Returns `false` if there is nothing to schedule. + ## RPC servers should call this to dispatch `callSoon` + ## bodies after retiring an RPC to its client. + ## This is effectively a non-blocking `poll(…)` and is + ## equivalent to scheduling a momentary no-op timeout + ## but faster and with less overhead. + let dis = getGlobalDispatcher() + result = dis.callbacks.len > 0 + if result: submit(dis.signalHandler.cap) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c7b9fac18..0f6504342 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -9,29 +9,37 @@ ## This module implements asynchronous file reading and writing. ## -## .. code-block:: Nim -## import asyncfile, asyncdispatch, os +## ```Nim +## import std/[asyncfile, asyncdispatch, os] ## -## proc main() {.async.} = -## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) -## await file.write("test") -## file.setFilePos(0) -## let data = await file.readAll() -## doAssert data == "test" -## file.close() +## proc main() {.async.} = +## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) +## await file.write("test") +## file.setFilePos(0) +## let data = await file.readAll() +## doAssert data == "test" +## file.close() ## -## waitFor main() +## waitFor main() +## ``` -import asyncdispatch, os +import std/[asyncdispatch, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + when defined(windows) or defined(nimdoc): + import std/widestrs + +# TODO: Fix duplication introduced by PR #4683. when defined(windows) or defined(nimdoc): - import winlean + import std/winlean else: - import posix + import std/posix type AsyncFile* = ref object - fd: AsyncFd + fd: AsyncFD offset: int64 when defined(windows) or defined(nimdoc): @@ -48,56 +56,60 @@ when defined(windows) or defined(nimdoc): case mode of fmRead, fmReadWriteExisting: OPEN_EXISTING - of fmAppend, fmReadWrite, fmWrite: - if fileExists(filename): - OPEN_EXISTING - else: - CREATE_NEW + of fmReadWrite, fmWrite: + CREATE_ALWAYS + of fmAppend: + OPEN_ALWAYS else: proc getPosixFlags(mode: FileMode): cint = case mode of fmRead: result = O_RDONLY of fmWrite: - result = O_WRONLY or O_CREAT + result = O_WRONLY or O_CREAT or O_TRUNC of fmAppend: result = O_WRONLY or O_CREAT or O_APPEND of fmReadWrite: - result = O_RDWR or O_CREAT + result = O_RDWR or O_CREAT or O_TRUNC of fmReadWriteExisting: result = O_RDWR result = result or O_NONBLOCK -proc getFileSize(f: AsyncFile): int64 = +proc getFileSize*(f: AsyncFile): int64 = ## Retrieves the specified file's size. when defined(windows) or defined(nimdoc): - var high: DWord + var high: DWORD let low = getFileSize(f.fd.Handle, addr high) if low == INVALID_FILE_SIZE: raiseOSError(osLastError()) - return (high shl 32) or low + result = (high shl 32) or low + else: + let curPos = lseek(f.fd.cint, 0, SEEK_CUR) + result = lseek(f.fd.cint, 0, SEEK_END) + f.offset = lseek(f.fd.cint, curPos, SEEK_SET) + assert(f.offset == curPos) -proc openAsync*(filename: string, mode = fmRead): AsyncFile = - ## Opens a file specified by the path in ``filename`` using - ## the specified ``mode`` asynchronously. +proc newAsyncFile*(fd: AsyncFD): AsyncFile = + ## Creates `AsyncFile` with a previously opened file descriptor `fd`. new result + result.fd = fd + register(fd) + +proc openAsync*(filename: string, mode = fmRead): AsyncFile = + ## Opens a file specified by the path in `filename` using + ## the specified FileMode `mode` asynchronously. when defined(windows) or defined(nimdoc): let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - result.fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd - else: - result.fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) - if result.fd.Handle == INVALID_HANDLE_VALUE: + if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) - register(result.fd) + result = newAsyncFile(fd.AsyncFD) if mode == fmAppend: result.offset = getFileSize(result) @@ -106,27 +118,104 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = getPosixFlags(mode) # RW (Owner), RW (Group), R (Other) let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH - result.fd = open(filename, flags, perm).AsyncFD - if result.fd.cint == -1: + let fd = open(filename, flags, perm) + if fd == -1: raiseOSError(osLastError()) - register(result.fd) + result = newAsyncFile(fd.AsyncFD) -proc read*(f: AsyncFile, size: int): Future[string] = - ## Read ``size`` bytes from the specified file asynchronously starting at +proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = + ## Read `size` bytes from the specified file asynchronously starting at ## the current position of the file pointer. ## + ## If the file pointer is past the end of the file then zero is returned + ## and no bytes are read into `buf` + var retFuture = newFuture[int]("asyncfile.readBuffer") + + when defined(windows) or defined(nimdoc): + var ol = newCustom() + ol.data = CompletionData(fd: f.fd, cb: + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount > 0 + assert bytesCount <= size + f.offset.inc bytesCount + retFuture.complete(bytesCount) + else: + if errcode.int32 == ERROR_HANDLE_EOF: + retFuture.complete(0) + else: + retFuture.fail(newOSError(errcode)) + ) + ol.offset = DWORD(f.offset and 0xffffffff) + ol.offsetHigh = DWORD(f.offset shr 32) + + # According to MSDN we're supposed to pass nil to lpNumberOfBytesRead. + let ret = readFile(f.fd.Handle, buf, size.int32, nil, + cast[POVERLAPPED](ol)) + if not ret.bool: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + if err.int32 == ERROR_HANDLE_EOF: + # This happens in Windows Server 2003 + retFuture.complete(0) + else: + retFuture.fail(newOSError(err)) + else: + # Request completed immediately. + var bytesRead: DWORD + let overlappedRes = getOverlappedResult(f.fd.Handle, + cast[POVERLAPPED](ol), bytesRead, false.WINBOOL) + if not overlappedRes.bool: + let err = osLastError() + if err.int32 == ERROR_HANDLE_EOF: + retFuture.complete(0) + else: + retFuture.fail(newOSError(osLastError())) + else: + assert bytesRead > 0 + assert bytesRead <= size + f.offset.inc bytesRead + retFuture.complete(bytesRead) + else: + proc cb(fd: AsyncFD): bool = + result = true + let res = read(fd.cint, cast[cstring](buf), size.cint) + if res < 0: + let lastError = osLastError() + if lastError.int32 != EAGAIN: + retFuture.fail(newOSError(lastError)) + else: + result = false # We still want this callback to be called. + elif res == 0: + # EOF + retFuture.complete(0) + else: + f.offset.inc(res) + retFuture.complete(res) + + if not cb(f.fd): + addRead(f.fd, cb) + + return retFuture + +proc read*(f: AsyncFile, size: int): Future[string] = + ## Read `size` bytes from the specified file asynchronously starting at + ## the current position of the file pointer. `size` should be greater than zero. + ## ## If the file pointer is past the end of the file then an empty string is ## returned. + assert size > 0 var retFuture = newFuture[string]("asyncfile.read") when defined(windows) or defined(nimdoc): var buffer = alloc0(size) - var ol = PCustomOverlapped() - GC_ref(ol) + var ol = newCustom() ol.data = CompletionData(fd: f.fd, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): assert bytesCount > 0 @@ -139,13 +228,13 @@ proc read*(f: AsyncFile, size: int): Future[string] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil ) - ol.offset = DWord(f.offset and 0xffffffff) - ol.offsetHigh = DWord(f.offset shr 32) + ol.offset = DWORD(f.offset and 0xffffffff) + ol.offsetHigh = DWORD(f.offset shr 32) # According to MSDN we're supposed to pass nil to lpNumberOfBytesRead. let ret = readFile(f.fd.Handle, buffer, size.int32, nil, @@ -157,18 +246,23 @@ proc read*(f: AsyncFile, size: int): Future[string] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + + if err.int32 == ERROR_HANDLE_EOF: + # This happens in Windows Server 2003 + retFuture.complete("") + else: + retFuture.fail(newOSError(err)) else: # Request completed immediately. - var bytesRead: DWord + var bytesRead: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesRead, false.WinBool) + cast[POVERLAPPED](ol), bytesRead, false.WINBOOL) if not overlappedRes.bool: let err = osLastError() if err.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -185,11 +279,12 @@ proc read*(f: AsyncFile, size: int): Future[string] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: # EOF + f.offset = lseek(fd.cint, 0, SEEK_CUR) retFuture.complete("") else: readBuffer.setLen(res) @@ -206,6 +301,8 @@ proc readLine*(f: AsyncFile): Future[string] {.async.} = result = "" while true: var c = await read(f, 1) + if c.len == 0: + break if c[0] == '\c': c = await read(f, 1) break @@ -225,7 +322,7 @@ proc setFilePos*(f: AsyncFile, pos: int64) = ## operations. The file's first byte has the index zero. f.offset = pos when not defined(windows) and not defined(nimdoc): - let ret = lseek(f.fd.cint, pos, SEEK_SET) + let ret = lseek(f.fd.cint, pos.Off, SEEK_SET) if ret == -1: raiseOSError(osLastError()) @@ -238,8 +335,77 @@ proc readAll*(f: AsyncFile): Future[string] {.async.} = return result.add data +proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = + ## Writes `size` bytes from `buf` to the file specified asynchronously. + ## + ## The returned Future will complete once all data has been written to the + ## specified file. + var retFuture = newFuture[void]("asyncfile.writeBuffer") + when defined(windows) or defined(nimdoc): + var ol = newCustom() + ol.data = CompletionData(fd: f.fd, cb: + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount == size.int32 + retFuture.complete() + else: + retFuture.fail(newOSError(errcode)) + ) + # passing -1 here should work according to MSDN, but doesn't. For more + # information see + # http://stackoverflow.com/questions/33650899/does-asynchronous-file- + # appending-in-windows-preserve-order + ol.offset = DWORD(f.offset and 0xffffffff) + ol.offsetHigh = DWORD(f.offset shr 32) + f.offset.inc(size) + + # According to MSDN we're supposed to pass nil to lpNumberOfBytesWritten. + let ret = writeFile(f.fd.Handle, buf, size.int32, nil, + cast[POVERLAPPED](ol)) + if not ret.bool: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newOSError(err)) + else: + # Request completed immediately. + var bytesWritten: DWORD + let overlappedRes = getOverlappedResult(f.fd.Handle, + cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) + if not overlappedRes.bool: + retFuture.fail(newOSError(osLastError())) + else: + assert bytesWritten == size.int32 + retFuture.complete() + else: + var written = 0 + + proc cb(fd: AsyncFD): bool = + result = true + let remainderSize = size - written + var cbuf = cast[cstring](buf) + let res = write(fd.cint, addr cbuf[written], remainderSize.cint) + if res < 0: + let lastError = osLastError() + if lastError.int32 != EAGAIN: + retFuture.fail(newOSError(lastError)) + else: + result = false # We still want this callback to be called. + else: + written.inc res + f.offset.inc res + if res != remainderSize: + result = false # We still have data to write. + else: + retFuture.complete() + + if not cb(f.fd): + addWrite(f.fd, cb) + return retFuture + proc write*(f: AsyncFile, data: string): Future[void] = - ## Writes ``data`` to the file specified asynchronously. + ## Writes `data` to the file specified asynchronously. ## ## The returned Future will complete once all data has been written to the ## specified file. @@ -247,25 +413,24 @@ proc write*(f: AsyncFile, data: string): Future[void] = var copy = data when defined(windows) or defined(nimdoc): var buffer = alloc0(data.len) - copyMem(buffer, addr copy[0], data.len) + copyMem(buffer, copy.cstring, data.len) - var ol = PCustomOverlapped() - GC_ref(ol) + var ol = newCustom() ol.data = CompletionData(fd: f.fd, cb: - proc (fd: AsyncFD, bytesCount: DWord, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): assert bytesCount == data.len.int32 - f.offset.inc(data.len) retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil ) - ol.offset = DWord(f.offset and 0xffffffff) - ol.offsetHigh = DWord(f.offset shr 32) + ol.offset = DWORD(f.offset and 0xffffffff) + ol.offsetHigh = DWORD(f.offset shr 32) + f.offset.inc(data.len) # According to MSDN we're supposed to pass nil to lpNumberOfBytesWritten. let ret = writeFile(f.fd.Handle, buffer, data.len.int32, nil, @@ -277,29 +442,35 @@ proc write*(f: AsyncFile, data: string): Future[void] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. - var bytesWritten: DWord + var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesWritten, false.WinBool) + cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == data.len.int32 - f.offset.inc(data.len) retFuture.complete() else: var written = 0 proc cb(fd: AsyncFD): bool = result = true - let remainderSize = data.len-written - let res = write(fd.cint, addr copy[written], remainderSize.cint) + + let remainderSize = data.len - written + + let res = + if data.len == 0: + write(fd.cint, copy.cstring, 0) + else: + write(fd.cint, addr copy[written], remainderSize.cint) + if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -314,8 +485,26 @@ proc write*(f: AsyncFile, data: string): Future[void] = addWrite(f.fd, cb) return retFuture +proc setFileSize*(f: AsyncFile, length: int64) = + ## Set a file length. + when defined(windows) or defined(nimdoc): + var + high = (length shr 32).DWORD + let + low = (length and 0xffffffff).DWORD + status = setFilePointer(f.fd.Handle, low, addr high, 0) + lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + (setEndOfFile(f.fd.Handle) == 0): + raiseOSError(osLastError()) + else: + # will truncate if Off is a 32-bit type! + if ftruncate(f.fd.cint, length.Off) == -1: + raiseOSError(osLastError()) + proc close*(f: AsyncFile) = ## Closes the file specified. + unregister(f.fd) when defined(windows) or defined(nimdoc): if not closeHandle(f.fd.Handle).bool: raiseOSError(osLastError()) @@ -323,3 +512,26 @@ proc close*(f: AsyncFile) = if close(f.fd.cint) == -1: raiseOSError(osLastError()) +proc writeFromStream*(f: AsyncFile, fs: FutureStream[string]) {.async.} = + ## Reads data from the specified future stream until it is completed. + ## The data which is read is written to the file immediately and + ## freed from memory. + ## + ## This procedure is perfect for saving streamed data to a file without + ## wasting memory. + while true: + let (hasValue, value) = await fs.read() + if hasValue: + await f.write(value) + else: + break + +proc readToStream*(f: AsyncFile, fs: FutureStream[string]) {.async.} = + ## Writes data to the specified future stream as the file is read. + while true: + let data = await read(f, 4000) + if data.len == 0: + break + await fs.write(data) + + fs.complete() diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim deleted file mode 100644 index b806f4235..000000000 --- a/lib/pure/asyncftpclient.nim +++ /dev/null @@ -1,378 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements an asynchronous FTP client. It allows you to connect -## to an FTP server and perform operations on it such as for example: -## -## * The upload of new files. -## * The removal of existing files. -## * Download of files. -## * Changing of files' permissions. -## * Navigation through the FTP server's directories. -## -## Connecting to an FTP server -## ------------------------ -## -## In order to begin any sort of transfer of files you must first -## connect to an FTP server. You can do so with the ``connect`` procedure. -## -## .. code-block::nim -## import asyncdispatch, asyncftpclient -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## echo("Connected") -## waitFor(main()) -## -## A new ``main`` async procedure must be declared to allow the use of the -## ``await`` keyword. The connection will complete asynchronously and the -## client will be connected after the ``await ftp.connect()`` call. -## -## Uploading a new file -## -------------------- -## -## After a connection is made you can use the ``store`` procedure to upload -## a new file to the FTP server. Make sure to check you are in the correct -## working directory before you do so with the ``pwd`` procedure, you can also -## instead specify an absolute path. -## -## .. code-block::nim -## import asyncdispatch, asyncftpclient -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## let currentDir = await ftp.pwd() -## assert currentDir == "/home/user/" -## await ftp.store("file.txt", "file.txt") -## echo("File finished uploading") -## waitFor(main()) -## -## Checking the progress of a file transfer -## ---------------------------------------- -## -## The progress of either a file upload or a file download can be checked -## by specifying a ``onProgressChanged`` procedure to the ``store`` or -## ``retrFile`` procedures. -## -## .. code-block::nim -## import asyncdispatch, asyncftpclient -## -## proc onProgressChanged(total, progress: BiggestInt, -## speed: float): Future[void] = -## echo("Uploaded ", progress, " of ", total, " bytes") -## echo("Current speed: ", speed, " kb/s") -## -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged) -## echo("File finished uploading") -## waitFor(main()) - - -import asyncdispatch, asyncnet, strutils, parseutils, os, times - -from ftpclient import FtpBaseObj, ReplyError, FtpEvent -from net import BufferSize - -type - AsyncFtpClientObj* = FtpBaseObj[AsyncSocket] - AsyncFtpClient* = ref AsyncFtpClientObj - - ProgressChangedProc* = - proc (total, progress: BiggestInt, speed: float): - Future[void] {.closure, gcsafe.} - -const multiLineLimit = 10000 - -proc expectReply(ftp: AsyncFtpClient): Future[TaintedString] {.async.} = - result = await ftp.csock.recvLine() - var count = 0 - while result[3] == '-': - ## Multi-line reply. - let line = await ftp.csock.recvLine() - result.add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*(ftp: AsyncFtpClient, m: string): Future[TaintedString] {.async.} = - ## Send a message to the server, and wait for a primary reply. - ## ``\c\L`` is added for you. - ## - ## **Note:** The server may return multiple lines of coded replies. - await ftp.csock.send(m & "\c\L") - return await ftp.expectReply() - -proc assertReply(received: TaintedString, expected: varargs[string]) = - for i in items(expected): - if received.string.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received.string]) - -proc pasv(ftp: AsyncFtpClient) {.async.} = - ## Negotiate a data connection. - ftp.dsock = newAsyncSocket() - - var pasvMsg = (await ftp.send("PASV")).string.strip.TaintedString - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg.string, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0.. ^3] - var port = nums[^2.. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - await ftp.dsock.connect(ip.join("."), Port(properPort.toU16)) - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*(ftp: AsyncFtpClient) {.async.} = - ## Connect to the FTP server specified by ``ftp``. - await ftp.csock.connect(ftp.address, ftp.port) - - var reply = await ftp.expectReply() - if reply.startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = await ftp.expectReply() - - # Handle 220 messages from the server - assertReply(reply, "220") - - if ftp.user != "": - assertReply(await(ftp.send("USER " & ftp.user)), "230", "331") - - if ftp.pass != "": - assertReply(await(ftp.send("PASS " & ftp.pass)), "230") - -proc pwd*(ftp: AsyncFtpClient): Future[TaintedString] {.async.} = - ## Returns the current working directory. - let wd = await ftp.send("PWD") - assertReply wd, "257" - return wd.string.captureBetween('"').TaintedString # " - -proc cd*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Changes the current directory on the remote FTP server to ``dir``. - assertReply(await(ftp.send("CWD " & dir.normalizePathSep)), "250") - -proc cdup*(ftp: AsyncFtpClient) {.async.} = - ## Changes the current directory to the parent of the current directory. - assertReply(await(ftp.send("CDUP")), "200") - -proc getLines(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Downloads text data in ASCII mode - result = "" - assert ftp.dsockConnected - while ftp.dsockConnected: - let r = await ftp.dsock.recvLine() - if r.string == "": - ftp.dsockConnected = false - else: - result.add(r.string & "\n") - - assertReply(await(ftp.expectReply()), "226") - -proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = - ## Returns a list of filenames in the given directory. If ``dir`` is "", - ## the current directory is used. If ``async`` is true, this - ## function will return immediately and it will be your job to - ## use asyncio's ``poll`` to progress this operation. - await ftp.pasv() - - assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) - - result = splitLines(await ftp.getLines()) - -proc existsFile*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = - ## Determines whether ``file`` exists. - var files = await ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*(ftp: AsyncFtpClient, dir: string, recursive = false){.async.} = - ## Creates a directory ``dir``. If ``recursive`` is true, the topmost - ## subdirectory of ``dir`` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the ``dir`` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply(await(ftp.send("MKD " & dir.normalizePathSep)), "257") - else: - var reply = TaintedString"" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = await ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*(ftp: AsyncFtpClient, path: string, - permissions: set[FilePermission]) {.async.} = - ## Changes permission of ``path`` to ``permissions``. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply(await(ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep)), "200") - -proc list*(ftp: AsyncFtpClient, dir = ""): Future[string] {.async.} = - ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current - ## working directory. - await ftp.pasv() - - let reply = await ftp.send("LIST" & " " & dir.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc retrText*(ftp: AsyncFtpClient, file: string): Future[string] {.async.} = - ## Retrieves ``file``. File must be ASCII text. - await ftp.pasv() - let reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(1000) - var dataFut = ftp.dsock.recv(BufferSize) - while ftp.dsockConnected: - await dataFut or countdownFut - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, - progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(1000) - - if dataFut.finished: - let data = dataFut.read - if data != "": - progress.inc(data.len) - progressInSecond.inc(data.len) - file.write(data) - dataFut = ftp.dsock.recv(BufferSize) - else: - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - -proc defaultOnProgressChanged*(total, progress: BiggestInt, - speed: float): Future[void] {.nimcall,gcsafe.} = - ## Default FTP ``onProgressChanged`` handler. Does nothing. - result = newFuture[void]() - #echo(total, " ", progress, " ", speed) - result.complete() - -proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged = defaultOnProgressChanged) {.async.} = - ## Downloads ``file`` and saves it to ``dest``. - ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function - ## when the download is finished. The event's ``filename`` field will be equal - ## to ``file``. - var destFile = open(dest, mode = fmWrite) - await ftp.pasv() - var reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply.string: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - await getFile(ftp, destFile, fileSize, onProgressChanged) - -proc doUpload(ftp: AsyncFtpClient, file: File, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - - let total = file.getFileSize() - var data = newStringOfCap(4000) - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(1000) - var sendFut: Future[void] = nil - while ftp.dsockConnected: - if sendFut == nil or sendFut.finished: - progress.inc(data.len) - progressInSecond.inc(data.len) - # TODO: Async file reading. - let len = file.readBuffer(addr(data[0]), 4000) - setLen(data, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - else: - sendFut = ftp.dsock.send(data) - - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(1000) - - await countdownFut or sendFut - -proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged = defaultOnProgressChanged) {.async.} = - ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The ``EvStore`` event is passed to the specified ``handleEvent`` function - ## when the upload is finished, and the ``filename`` field will be - ## equal to ``file``. - var destFile = open(file) - await ftp.pasv() - - let reply = await ftp.send("STOR " & dest.normalizePathSep) - assertReply reply, ["125", "150"] - - await doUpload(ftp, destFile, onProgressChanged) - -proc newAsyncFtpClient*(address: string, port = Port(21), - user, pass = ""): AsyncFtpClient = - ## Creates a new ``AsyncFtpClient`` object. - new result - result.user = user - result.pass = pass - result.address = address - result.port = port - result.dsockConnected = false - result.csock = newAsyncSocket() - -when not defined(testing) and isMainModule: - var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") - proc main(ftp: AsyncFtpClient) {.async.} = - await ftp.connect() - echo await ftp.pwd() - echo await ftp.listDirs() - await ftp.store("payload.jpg", "payload.jpg") - await ftp.retrFile("payload.jpg", "payload2.jpg") - echo("Finished") - - waitFor main(ftp) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim new file mode 100644 index 000000000..29ebf8f89 --- /dev/null +++ b/lib/pure/asyncfutures.nim @@ -0,0 +1,527 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import std/[os, tables, strutils, times, heapqueue, options, deques, cstrutils, typetraits] + +import system/stacktraces + +when defined(nimPreviewSlimSystem): + import std/objectdollar # for StackTraceEntry + import std/assertions + +# TODO: This shouldn't need to be included, but should ideally be exported. +type + CallbackFunc = proc () {.closure, gcsafe.} + + CallbackList = object + function: CallbackFunc + next: owned(ref CallbackList) + + FutureBase* = ref object of RootObj ## Untyped future. + callbacks: CallbackList + + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release) or defined(futureLogging): + stackTrace: seq[StackTraceEntry] ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Defect + cause*: FutureBase + +when not defined(release): + var currentID = 0 + +const isFutureLoggingEnabled* = defined(futureLogging) + +const + NimAsyncContinueSuffix* = "NimAsyncContinue" ## For internal usage. Do not use. + +when isFutureLoggingEnabled: + import std/hashes + type + FutureInfo* = object + stackTrace*: seq[StackTraceEntry] + fromProc*: string + + var futuresInProgress {.threadvar.}: Table[FutureInfo, int] + + proc getFuturesInProgress*(): var Table[FutureInfo, int] = + return futuresInProgress + + proc hash(s: StackTraceEntry): Hash = + result = hash(s.procname) !& hash(s.line) !& + hash(s.filename) + result = !$result + + proc hash(fi: FutureInfo): Hash = + result = hash(fi.stackTrace) !& hash(fi.fromProc) + result = !$result + + proc getFutureInfo(fut: FutureBase): FutureInfo = + let info = FutureInfo( + stackTrace: fut.stackTrace, + fromProc: fut.fromProc + ) + return info + + proc logFutureStart(fut: FutureBase) = + let info = getFutureInfo(fut) + if info notin getFuturesInProgress(): + getFuturesInProgress()[info] = 0 + getFuturesInProgress()[info].inc() + + proc logFutureFinish(fut: FutureBase) = + getFuturesInProgress()[getFutureInfo(fut)].dec() + +var callSoonProc {.threadvar.}: proc (cbproc: proc ()) {.gcsafe.} + +proc getCallSoonProc*(): (proc(cbproc: proc ()) {.gcsafe.}) = + ## Get current implementation of `callSoon`. + return callSoonProc + +proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) = + ## Change current implementation of `callSoon`. This is normally called when dispatcher from `asyncdispatcher` is initialized. + callSoonProc = p + +proc callSoon*(cbproc: proc () {.gcsafe.}) = + ## Call `cbproc` "soon". + ## + ## If async dispatcher is running, `cbproc` will be executed during next dispatcher tick. + ## + ## If async dispatcher is not running, `cbproc` will be executed immediately. + if callSoonProc.isNil: + # Loop not initialized yet. Call the function directly to allow setup code to use futures. + cbproc() + else: + callSoonProc(cbproc) + +template setupFutureBase(fromProc: string) = + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTraceEntries() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFuture*[T](fromProc: string = "unspecified"): owned(Future[T]) = + ## Creates a new future. + ## + ## Specifying `fromProc`, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + setupFutureBase(fromProc) + when isFutureLoggingEnabled: logFutureStart(result) + +proc newFutureVar*[T](fromProc = "unspecified"): owned(FutureVar[T]) = + ## Create a new `FutureVar`. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying `fromProc`, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + let fo = newFuture[T](fromProc) + result = typeof(result)(fo) + when isFutureLoggingEnabled: logFutureStart(Future[T](result)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the `finished` status of `future`. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## `FutureError`. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(($future.stackTrace).strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent($future.value, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc call(callbacks: var CallbackList) = + var current = callbacks + while true: + if not current.function.isNil: + callSoon(current.function) + + if current.next.isNil: + break + else: + current = current.next[] + # callback will be called only once, let GC collect them now + callbacks.next = nil + callbacks.function = nil + +proc add(callbacks: var CallbackList, function: CallbackFunc) = + if callbacks.function.isNil: + callbacks.function = function + assert callbacks.next == nil + else: + let newCallback = new(ref CallbackList) + newCallback.function = function + newCallback.next = nil + + if callbacks.next == nil: + callbacks.next = newCallback + else: + var last = callbacks.next + while last.next != nil: + last = last.next + last.next = newCallback + +proc completeImpl[T, U](future: Future[T], val: sink U, isVoid: static bool) = + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + when not isVoid: + future.value = val + future.finished = true + future.callbacks.call() + when isFutureLoggingEnabled: logFutureFinish(future) + +proc complete*[T](future: Future[T], val: sink T) = + ## Completes `future` with value `val`. + completeImpl(future, val, false) + +proc complete*(future: Future[void], val = Future[void].default) = + completeImpl(future, (), true) + +proc complete*[T](future: FutureVar[T]) = + ## Completes a `FutureVar`. + template fut: untyped = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + fut.callbacks.call() + when isFutureLoggingEnabled: logFutureFinish(Future[T](future)) + +proc complete*[T](future: FutureVar[T], val: sink T) = + ## Completes a `FutureVar` with value `val`. + ## + ## Any previously stored value will be overwritten. + template fut: untyped = Future[T](future) + checkFinished(fut) + assert(fut.error.isNil()) + fut.finished = true + fut.value = val + fut.callbacks.call() + when isFutureLoggingEnabled: logFutureFinish(fut) + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes `future` with `error`. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + future.callbacks.call() + when isFutureLoggingEnabled: logFutureFinish(future) + +proc clearCallbacks*(future: FutureBase) = + future.callbacks.function = nil + future.callbacks.next = nil + +proc addCallback*(future: FutureBase, cb: proc() {.closure, gcsafe.}) = + ## Adds the callbacks proc to be called when the future completes. + ## + ## If future has already completed then `cb` will be called immediately. + assert cb != nil + if future.finished: + callSoon(cb) + else: + future.callbacks.add cb + +proc addCallback*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure, gcsafe.}) = + ## Adds the callbacks proc to be called when the future completes. + ## + ## If future has already completed then `cb` will be called immediately. + future.addCallback( + proc() = + cb(future) + ) + +proc `callback=`*(future: FutureBase, cb: proc () {.closure, gcsafe.}) = + ## Clears the list of callbacks and sets the callback proc to be called when the future completes. + ## + ## If future has already completed then `cb` will be called immediately. + ## + ## It's recommended to use `addCallback` or `then` instead. + future.clearCallbacks + future.addCallback cb + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure, gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then `cb` will be called immediately. + future.callback = proc () = cb(future) + +template getFilenameProcname(entry: StackTraceEntry): (string, string) = + when compiles(entry.filenameStr) and compiles(entry.procnameStr): + # We can't rely on "entry.filename" and "entry.procname" still being valid + # cstring pointers, because the "string.data" buffers they pointed to might + # be already garbage collected (this entry being a non-shallow copy, + # "entry.filename" no longer points to "entry.filenameStr.data", but to the + # buffer of the original object). + (entry.filenameStr, entry.procnameStr) + else: + ($entry.filename, $entry.procname) + +proc getHint(entry: StackTraceEntry): string = + ## We try to provide some hints about stack trace entries that the user + ## may not be familiar with, in particular calls inside the stdlib. + + let (filename, procname) = getFilenameProcname(entry) + + result = "" + if procname == "processPendingCallbacks": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: + return "Executes pending callbacks" + elif procname == "poll": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: + return "Processes asynchronous completion events" + + if procname.endsWith(NimAsyncContinueSuffix): + if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0: + return "Resumes an async procedure" + +proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = + when defined(nimStackTraceOverride): + let entries = addDebuggingInfo(stackTraceEntries) + else: + let entries = stackTraceEntries + + result = "" + # Find longest filename & line number combo for alignment purposes. + var longestLeft = 0 + for entry in entries: + let (filename, procname) = getFilenameProcname(entry) + + if procname == "": continue + + let leftLen = filename.len + len($entry.line) + if leftLen > longestLeft: + longestLeft = leftLen + + # Format the entries. + for entry in entries: + let (filename, procname) = getFilenameProcname(entry) + + if procname == "" and entry.line == reraisedFromBegin: + break + + let left = "$#($#)" % [filename, $entry.line] + result.add((spaces(2) & "$# $#\n") % [ + left, + procname + ]) + let hint = getHint(entry) + if hint.len > 0: + result.add(spaces(4) & "## " & hint & "\n") + +proc injectStacktrace[T](future: Future[T]) = + when not defined(release): + const header = "\nAsync traceback:\n" + + var exceptionMsg = future.error.msg + if header in exceptionMsg: + # This is messy: extract the original exception message from the msg + # containing the async traceback. + let start = exceptionMsg.find(header) + exceptionMsg = exceptionMsg[0..<start] + + + var newMsg = exceptionMsg & header + + let entries = getStackTraceEntries(future.error) + newMsg.add($entries) + + newMsg.add("Exception message: " & exceptionMsg & "\n") + + # # For debugging purposes + # newMsg.add("Exception type:") + # for entry in getStackTraceEntries(future.error): + # newMsg.add "\n" & $entry + future.error.msg = newMsg + +template readImpl(future, T) = + when future is Future[T]: + let fut {.cursor.} = future + else: + let fut {.cursor.} = Future[T](future) + if fut.finished: + if fut.error != nil: + injectStacktrace(fut) + raise fut.error + when T isnot void: + result = distinctBase(future).value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc read*[T](future: Future[T] | FutureVar[T]): lent T = + ## Retrieves the value of `future`. Future must be finished otherwise + ## this function will fail with a `ValueError` exception. + ## + ## If the result of the future is an error then that error will be raised. + readImpl(future, T) + +proc read*(future: Future[void] | FutureVar[void]) = + readImpl(future, void) + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in `future`. + ## + ## An `ValueError` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in `future`. + ## + ## Unlike `read`, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*(future: FutureBase | FutureVar): bool = + ## Determines whether `future` has completed. + ## + ## `True` may indicate an error or a value. Use `failed` to distinguish. + when future is FutureVar: + result = (FutureBase(future)).finished + else: + result = future.finished + +proc failed*(future: FutureBase): bool = + ## Determines whether `future` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on `future` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of `discard` to discard void futures, + ## or use `waitFor` if you need to wait for the future's completion. + assert(not future.isNil, "Future is nil") + # TODO: We can likely look at the stack trace here and inject the location + # where the `asyncCheck` was called to give a better error stack message. + proc asyncCheckCallback() = + if future.failed: + injectStacktrace(future) + raise future.error + future.callback = asyncCheckCallback + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both `fut1` and `fut2` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if not retFuture.finished: + if fut1.failed: retFuture.fail(fut1.error) + elif fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if not retFuture.finished: + if fut2.failed: retFuture.fail(fut2.error) + elif fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either `fut1` or `fut2` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb[X](fut: Future[X]) = + if not retFuture.finished: + if fut.failed: retFuture.fail(fut.error) + else: retFuture.complete() + fut1.callback = cb[T] + fut2.callback = cb[Y] + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in `futs` complete. + ## If the argument is empty, the returned future completes immediately. + ## + ## If the awaited futures are not `Future[void]`, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* `Future[void]`, + ## this proc returns `Future[void]`. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.addCallback proc (f: Future[T]) = + inc(completedFutures) + if not retFuture.finished: + if f.failed: + retFuture.fail(f.error) + else: + if completedFutures == totalFutures: + retFuture.complete() + + if totalFutures == 0: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.addCallback proc (f: Future[T]) = + inc(completedFutures) + if not retFuture.finished: + if f.failed: + retFuture.fail(f.error) + else: + retValues[i] = f.read() + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + if retValues.len == 0: + retFuture.complete(retValues) + + return retFuture diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 590b52c1a..39e945d5e 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -9,134 +9,141 @@ ## This module implements a high performance asynchronous HTTP server. ## -## Examples -## -------- -## -## This example will create an HTTP server on port 8080. The server will -## respond to all requests with a ``200 OK`` response code and "Hello World" -## as the response body. -## -## .. code-block::nim -## import asynchttpserver, asyncdispatch -## -## var server = newAsyncHttpServer() -## proc cb(req: Request) {.async.} = -## await req.respond(Http200, "Hello World") -## -## waitFor server.serve(Port(8080), cb) +## This HTTP server has not been designed to be used in production, but +## for testing applications locally. Because of this, when deploying your +## application in production you should use a reverse proxy (for example nginx) +## instead of allowing users to connect directly to this server. + +runnableExamples("-r:off"): + # This example will create an HTTP server on an automatically chosen port. + # It will respond to all requests with a `200 OK` response code and "Hello World" + # as the response body. + import std/asyncdispatch + proc main {.async.} = + var server = newAsyncHttpServer() + proc cb(req: Request) {.async.} = + echo (req.reqMethod, req.url, req.headers) + let headers = {"Content-type": "text/plain; charset=utf-8"} + await req.respond(Http200, "Hello World", headers.newHttpHeaders()) + + server.listen(Port(0)) # or Port(8080) to hardcode the standard HTTP port. + let port = server.getPort + echo "test this with: curl localhost:" & $port.uint16 & "/" + while true: + if server.shouldAcceptRequest(): + await server.acceptRequest(cb) + else: + # too many concurrent connections, `maxFDs` exceeded + # wait 500ms for FDs to be closed + await sleepAsync(500) + + waitFor main() + +import std/[asyncnet, asyncdispatch, parseutils, uri, strutils] +import std/httpcore +from std/nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions -import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils +export httpcore except parseHeader + +const + maxLine = 8*1024 + +# TODO: If it turns out that the decisions that asynchttpserver makes +# explicitly, about whether to close the client sockets or upgrade them are +# wrong, then add a return value which determines what to do for the callback. +# Also, maybe move `client` out of `Request` object and into the args for +# the proc. type Request* = object client*: AsyncSocket # TODO: Separate this into a Response object? - reqMethod*: string - headers*: StringTableRef + reqMethod*: HttpMethod + headers*: HttpHeaders protocol*: tuple[orig: string, major, minor: int] url*: Uri - hostname*: string ## The hostname of the client that made the request. + hostname*: string ## The hostname of the client that made the request. body*: string AsyncHttpServer* = ref object socket: AsyncSocket reuseAddr: bool + reusePort: bool + maxBody: int ## The maximum content-length that will be read for the body. + maxFDs: int + +proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} = + ## Returns the port `self` was bound to. + ## + ## Useful for identifying what port `self` is bound to, if it + ## was chosen automatically, for example via `listen(Port(0))`. + runnableExamples: + from std/nativesockets import Port + let server = newAsyncHttpServer() + server.listen(Port(0)) + assert server.getPort.uint16 > 0 + server.close() + result = getLocalAddr(self.socket)[1] + +proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, + maxBody = 8388608): AsyncHttpServer = + ## Creates a new `AsyncHttpServer` instance. + result = AsyncHttpServer(reuseAddr: reuseAddr, reusePort: reusePort, maxBody: maxBody) - HttpCode* = enum - Http100 = "100 Continue", - Http101 = "101 Switching Protocols", - Http200 = "200 OK", - Http201 = "201 Created", - Http202 = "202 Accepted", - Http204 = "204 No Content", - Http205 = "205 Reset Content", - Http206 = "206 Partial Content", - Http300 = "300 Multiple Choices", - Http301 = "301 Moved Permanently", - Http302 = "302 Found", - Http303 = "303 See Other", - Http304 = "304 Not Modified", - Http305 = "305 Use Proxy", - Http307 = "307 Temporary Redirect", - Http400 = "400 Bad Request", - Http401 = "401 Unauthorized", - Http403 = "403 Forbidden", - Http404 = "404 Not Found", - Http405 = "405 Method Not Allowed", - Http406 = "406 Not Acceptable", - Http407 = "407 Proxy Authentication Required", - Http408 = "408 Request Timeout", - Http409 = "409 Conflict", - Http410 = "410 Gone", - Http411 = "411 Length Required", - Http412 = "412 Precondition Failed", - Http413 = "413 Request Entity Too Large", - Http414 = "414 Request-URI Too Long", - Http415 = "415 Unsupported Media Type", - Http416 = "416 Requested Range Not Satisfiable", - Http417 = "417 Expectation Failed", - Http418 = "418 I'm a teapot", - Http500 = "500 Internal Server Error", - Http501 = "501 Not Implemented", - Http502 = "502 Bad Gateway", - Http503 = "503 Service Unavailable", - Http504 = "504 Gateway Timeout", - Http505 = "505 HTTP Version Not Supported" - - HttpVersion* = enum - HttpVer11, - HttpVer10 - -{.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, - THttpCode: HttpCode, THttpVersion: HttpVersion].} - -proc `==`*(protocol: tuple[orig: string, major, minor: int], - ver: HttpVersion): bool = - let major = - case ver - of HttpVer11, HttpVer10: 1 - let minor = - case ver - of HttpVer11: 1 - of HttpVer10: 0 - result = protocol.major == major and protocol.minor == minor - -proc newAsyncHttpServer*(reuseAddr = true): AsyncHttpServer = - ## Creates a new ``AsyncHttpServer`` instance. - new result - result.reuseAddr = reuseAddr - -proc addHeaders(msg: var string, headers: StringTableRef) = +proc addHeaders(msg: var string, headers: HttpHeaders) = for k, v in headers: msg.add(k & ": " & v & "\c\L") -proc sendHeaders*(req: Request, headers: StringTableRef): Future[void] = +proc sendHeaders*(req: Request, headers: HttpHeaders): Future[void] = ## Sends the specified headers to the requesting client. var msg = "" addHeaders(msg, headers) return req.client.send(msg) proc respond*(req: Request, code: HttpCode, content: string, - headers: StringTableRef = nil): Future[void] = - ## Responds to the request with the specified ``HttpCode``, headers and + headers: HttpHeaders = nil): Future[void] = + ## Responds to the request with the specified `HttpCode`, headers and ## content. ## ## This procedure will **not** close the client socket. + ## + ## Example: + ## ```Nim + ## import std/json + ## proc handler(req: Request) {.async.} = + ## if req.url.path == "/hello-world": + ## let msg = %* {"message": "Hello World"} + ## let headers = newHttpHeaders([("Content-Type","application/json")]) + ## await req.respond(Http200, $msg, headers) + ## else: + ## await req.respond(Http404, "Not Found") + ## ``` var msg = "HTTP/1.1 " & $code & "\c\L" if headers != nil: msg.addHeaders(headers) - msg.add("Content-Length: " & $content.len & "\c\L\c\L") + + # If the headers did not contain a Content-Length use our own + if headers.isNil() or not headers.hasKey("Content-Length"): + msg.add("Content-Length: ") + # this particular way saves allocations: + msg.addInt content.len + msg.add "\c\L" + + msg.add "\c\L" msg.add(content) result = req.client.send(msg) -proc parseHeader(line: string): tuple[key, value: string] = - var i = 0 - i = line.parseUntil(result.key, ':') - inc(i) # skip : - if i < len(line): - i += line.skipWhiteSpace(i) - i += line.parseUntil(result.value, {'\c', '\L'}, i) - else: - result.value = "" +proc respondError(req: Request, code: HttpCode): Future[void] = + ## Responds to the request with the specified `HttpCode`. + let content = $code + var msg = "HTTP/1.1 " & content & "\c\L" + + msg.add("Content-Length: " & $content.len & "\c\L\c\L") + msg.add(content) + result = req.client.send(msg) proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = var i = protocol.skipIgnoreCase("HTTP/") @@ -144,151 +151,290 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = raise newException(ValueError, "Invalid request protocol. Got: " & protocol) result.orig = protocol - i.inc protocol.parseInt(result.major, i) + i.inc protocol.parseSaturatedNatural(result.major, i) i.inc # Skip . - i.inc protocol.parseInt(result.minor, i) + i.inc protocol.parseSaturatedNatural(result.minor, i) proc sendStatus(client: AsyncSocket, status: string): Future[void] = - client.send("HTTP/1.1 " & status & "\c\L") + client.send("HTTP/1.1 " & status & "\c\L\c\L") -proc processClient(client: AsyncSocket, address: string, - callback: proc (request: Request): - Future[void] {.closure, gcsafe.}) {.async.} = - var request: Request - request.url = initUri() - request.headers = newStringTable(modeCaseInsensitive) - var lineFut = newFutureVar[string]("asynchttpserver.processClient") - lineFut.mget() = newStringOfCap(80) - var key, value = "" +func hasChunkedEncoding(request: Request): bool = + ## Searches for a chunked transfer encoding + const transferEncoding = "Transfer-Encoding" - while not client.isClosed: - # GET /path HTTP/1.1 - # Header: val - # \n - request.headers.clear(modeCaseInsensitive) - request.body = "" + if request.headers.hasKey(transferEncoding): + for encoding in seq[string](request.headers[transferEncoding]): + if "chunked" == encoding.strip: + # Returns true if it is both an HttpPost and has chunked encoding + return request.reqMethod == HttpPost + return false + +proc processRequest( + server: AsyncHttpServer, + req: FutureVar[Request], + client: AsyncSocket, + address: sink string, + lineFut: FutureVar[string], + callback: proc (request: Request): Future[void] {.closure, gcsafe.}, +): Future[bool] {.async.} = + + # Alias `request` to `req.mget()` so we don't have to write `mget` everywhere. + template request(): Request = + req.mget() + + # GET /path HTTP/1.1 + # Header: val + # \n + request.headers.clear() + request.body = "" + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + request.hostname = address + else: request.hostname.shallowCopy(address) - assert client != nil - request.client = client + assert client != nil + request.client = client - # First line - GET /path HTTP/1.1 + # We should skip at least one empty line before the request + # https://tools.ietf.org/html/rfc7230#section-3.5 + for i in 0..1: lineFut.mget().setLen(0) lineFut.clean() - await client.recvLineInto(lineFut) # TODO: Timeouts. + await client.recvLineInto(lineFut, maxLength = maxLine) # TODO: Timeouts. + if lineFut.mget == "": client.close() - return - - var i = 0 - for linePart in lineFut.mget.split(' '): - case i - of 0: request.reqMethod.shallowCopy(linePart.normalize) - of 1: parseUri(linePart, request.url) - of 2: - try: - request.protocol = parseProtocol(linePart) - except ValueError: - asyncCheck request.respond(Http400, - "Invalid request protocol. Got: " & linePart) - continue + return false + + if lineFut.mget.len > maxLine: + await request.respondError(Http413) + client.close() + return false + if lineFut.mget != "\c\L": + break + + # First line - GET /path HTTP/1.1 + var i = 0 + for linePart in lineFut.mget.split(' '): + case i + of 0: + case linePart + of "GET": request.reqMethod = HttpGet + of "POST": request.reqMethod = HttpPost + of "HEAD": request.reqMethod = HttpHead + of "PUT": request.reqMethod = HttpPut + of "DELETE": request.reqMethod = HttpDelete + of "PATCH": request.reqMethod = HttpPatch + of "OPTIONS": request.reqMethod = HttpOptions + of "CONNECT": request.reqMethod = HttpConnect + of "TRACE": request.reqMethod = HttpTrace + else: + asyncCheck request.respondError(Http400) + return true # Retry processing of request + of 1: + try: + parseUri(linePart, request.url) + except ValueError: + asyncCheck request.respondError(Http400) + return true + of 2: + try: + request.protocol = parseProtocol(linePart) + except ValueError: + asyncCheck request.respondError(Http400) + return true + else: + await request.respondError(Http400) + return true + inc i + + # Headers + while true: + i = 0 + lineFut.mget.setLen(0) + lineFut.clean() + await client.recvLineInto(lineFut, maxLength = maxLine) + + if lineFut.mget == "": + client.close(); return false + if lineFut.mget.len > maxLine: + await request.respondError(Http413) + client.close(); return false + if lineFut.mget == "\c\L": break + let (key, value) = parseHeader(lineFut.mget) + request.headers[key] = value + # Ensure the client isn't trying to DoS us. + if request.headers.len > headerLimit: + await client.sendStatus("400 Bad Request") + request.client.close() + return false + + if request.reqMethod == HttpPost: + # Check for Expect header + if request.headers.hasKey("Expect"): + if "100-continue" in request.headers["Expect"]: + await client.sendStatus("100 Continue") else: - await request.respond(Http400, "Invalid request. Got: " & lineFut.mget) - continue - inc i + await client.sendStatus("417 Expectation Failed") + + # Read the body + # - Check for Content-length header + if request.headers.hasKey("Content-Length"): + var contentLength = 0 + if parseSaturatedNatural(request.headers["Content-Length"], contentLength) == 0: + await request.respond(Http400, "Bad Request. Invalid Content-Length.") + return true + else: + if contentLength > server.maxBody: + await request.respondError(Http413) + return false + request.body = await client.recv(contentLength) + if request.body.len != contentLength: + await request.respond(Http400, "Bad Request. Content-Length does not match actual.") + return true + elif hasChunkedEncoding(request): + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding + var sizeOrData = 0 + var bytesToRead = 0 + request.body = "" - # Headers while true: - i = 0 lineFut.mget.setLen(0) lineFut.clean() - await client.recvLineInto(lineFut) - - if lineFut.mget == "": - client.close(); return - if lineFut.mget == "\c\L": break - let (key, value) = parseHeader(lineFut.mget) - request.headers[key] = value - - if request.reqMethod == "post": - # Check for Expect header - if request.headers.hasKey("Expect"): - if request.headers.getOrDefault("Expect").toLower == "100-continue": - await client.sendStatus("100 Continue") - else: - await client.sendStatus("417 Expectation Failed") - - # Read the body - # - Check for Content-length header - if request.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(request.headers.getOrDefault("Content-Length"), - contentLength) == 0: - await request.respond(Http400, "Bad Request. Invalid Content-Length.") - continue - else: - request.body = await client.recv(contentLength) - assert request.body.len == contentLength + + # The encoding format alternates between specifying a number of bytes to read + # and the data to be read, of the previously specified size + if sizeOrData mod 2 == 0: + # Expect a number of chars to read + await client.recvLineInto(lineFut, maxLength = maxLine) + try: + bytesToRead = lineFut.mget.parseHexInt + except ValueError: + # Malformed request + await request.respond(Http411, ("Invalid chunked transfer encoding - " & + "chunk data size must be hex encoded")) + return true else: - await request.respond(Http400, "Bad Request. No Content-Length.") - continue + if bytesToRead == 0: + # Done reading chunked data + break - case request.reqMethod - of "get", "post", "head", "put", "delete", "trace", "options", - "connect", "patch": - await callback(request) - else: - await request.respond(Http400, "Invalid request method. Got: " & - request.reqMethod) - - # Persistent connections - if (request.protocol == HttpVer11 and - request.headers.getOrDefault("connection").normalize != "close") or - (request.protocol == HttpVer10 and - request.headers.getOrDefault("connection").normalize == "keep-alive"): - # In HTTP 1.1 we assume that connection is persistent. Unless connection - # header states otherwise. - # In HTTP 1.0 we assume that the connection should not be persistent. - # Unless the connection header states otherwise. - discard - else: - request.client.close() + # Read bytesToRead and add to body + let chunk = await client.recv(bytesToRead) + request.body.add(chunk) + # Skip \r\n (chunk terminating bytes per spec) + let separator = await client.recv(2) + if separator != "\r\n": + await request.respond(Http400, "Bad Request. Encoding separator must be \\r\\n") + return true + + inc sizeOrData + elif request.reqMethod == HttpPost: + await request.respond(Http411, "Content-Length required.") + return true + + # Call the user's callback. + await callback(request) + + if "upgrade" in request.headers.getOrDefault("connection"): + return false + + # The request has been served, from this point on returning `true` means the + # connection will not be closed and will be kept in the connection pool. + + # Persistent connections + if (request.protocol == HttpVer11 and + cmpIgnoreCase(request.headers.getOrDefault("connection"), "close") != 0) or + (request.protocol == HttpVer10 and + cmpIgnoreCase(request.headers.getOrDefault("connection"), "keep-alive") == 0): + # In HTTP 1.1 we assume that connection is persistent. Unless connection + # header states otherwise. + # In HTTP 1.0 we assume that the connection should not be persistent. + # Unless the connection header states otherwise. + return true + else: + request.client.close() + return false + +proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string, + callback: proc (request: Request): + Future[void] {.closure, gcsafe.}) {.async.} = + var request = newFutureVar[Request]("asynchttpserver.processClient") + request.mget().url = initUri() + request.mget().headers = newHttpHeaders() + var lineFut = newFutureVar[string]("asynchttpserver.processClient") + lineFut.mget() = newStringOfCap(80) + + while not client.isClosed: + let retry = await processRequest( + server, request, client, address, lineFut, callback + ) + if not retry: + client.close() break -proc serve*(server: AsyncHttpServer, port: Port, - callback: proc (request: Request): Future[void] {.closure,gcsafe.}, - address = "") {.async.} = - ## Starts the process of listening for incoming HTTP connections on the - ## specified address and port. - ## - ## When a request is made by a client the specified callback will be called. - server.socket = newAsyncSocket() +const + nimMaxDescriptorsFallback* {.intdefine.} = 16_000 ## fallback value for \ + ## when `maxDescriptors` is not available. + ## This can be set on the command line during compilation + ## via `-d:nimMaxDescriptorsFallback=N` + +proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET) = + ## Listen to the given port and address. + when declared(maxDescriptors): + server.maxFDs = try: maxDescriptors() except: nimMaxDescriptorsFallback + else: + server.maxFDs = nimMaxDescriptorsFallback + server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() +proc shouldAcceptRequest*(server: AsyncHttpServer; + assumedDescriptorsPerRequest = 5): bool {.inline.} = + ## Returns true if the process's current number of opened file + ## descriptors is still within the maximum limit and so it's reasonable to + ## accept yet another request. + result = assumedDescriptorsPerRequest < 0 or + (activeDescriptors() + assumedDescriptorsPerRequest < server.maxFDs) + +proc acceptRequest*(server: AsyncHttpServer, + callback: proc (request: Request): Future[void] {.closure, gcsafe.}) {.async.} = + ## Accepts a single request. Write an explicit loop around this proc so that + ## errors can be handled properly. + var (address, client) = await server.socket.acceptAddr() + asyncCheck processClient(server, client, address, callback) + +proc serve*(server: AsyncHttpServer, port: Port, + callback: proc (request: Request): Future[void] {.closure, gcsafe.}, + address = ""; + assumedDescriptorsPerRequest = -1; + domain = AF_INET) {.async.} = + ## Starts the process of listening for incoming HTTP connections on the + ## specified address and port. + ## + ## When a request is made by a client the specified callback will be called. + ## + ## If `assumedDescriptorsPerRequest` is 0 or greater the server cares about + ## the process's maximum file descriptor limit. It then ensures that the + ## process still has the resources for `assumedDescriptorsPerRequest` + ## file descriptors before accepting a connection. + ## + ## You should prefer to call `acceptRequest` instead with a custom server + ## loop so that you're in control over the error handling and logging. + listen server, port, address, domain while true: - # TODO: Causes compiler crash. - #var (address, client) = await server.socket.acceptAddr() - var fut = await server.socket.acceptAddr() - asyncCheck processClient(fut.client, fut.address, callback) + if shouldAcceptRequest(server, assumedDescriptorsPerRequest): + var (address, client) = await server.socket.acceptAddr() + asyncCheck processClient(server, client, address, callback) + else: + poll() #echo(f.isNil) #echo(f.repr) proc close*(server: AsyncHttpServer) = ## Terminates the async http server instance. server.socket.close() - -when not defined(testing) and isMainModule: - proc main = - var server = newAsyncHttpServer() - proc cb(req: Request) {.async.} = - #echo(req.reqMethod, " ", req.url) - #echo(req.headers) - let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", - "Content-type": "text/plain; charset=utf-8"} - await req.respond(Http200, "Hello World", headers.newStringTable()) - - asyncCheck server.serve(Port(5555), cb) - runForever() - main() diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim new file mode 100644 index 000000000..d4e72c28a --- /dev/null +++ b/lib/pure/asyncmacro.nim @@ -0,0 +1,383 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Implements the `async` and `multisync` macros for `asyncdispatch`. + +import std/[macros, strutils, asyncfutures] + +type + Context = ref object + inTry: int + hasRet: bool + +# TODO: Ref https://github.com/nim-lang/Nim/issues/5617 +# TODO: Add more line infos +proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimNode]): NimNode = + result = newCall(theProc, args) + result.copyLineInfo(fromNode) + +template createCb(retFutureSym, iteratorNameSym, + strName, identName, futureVarCompletions: untyped) = + bind finished + var nameIterVar = iteratorNameSym + proc identName {.closure, stackTrace: off.} = + try: + if not nameIterVar.finished: + var next = nameIterVar() + # Continue while the yielded future is already finished. + while (not next.isNil) and next.finished: + next = nameIterVar() + if nameIterVar.finished: + break + + if next == nil: + if not retFutureSym.finished: + let msg = "Async procedure ($1) yielded `nil`, are you await'ing a `nil` Future?" + raise newException(AssertionDefect, msg % strName) + else: + {.gcsafe.}: + next.addCallback cast[proc() {.closure, gcsafe.}](identName) + except: + futureVarCompletions + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) + identName() + +proc createFutureVarCompletions(futureVarIdents: seq[NimNode], fromNode: NimNode): NimNode = + result = newNimNode(nnkStmtList, fromNode) + # Add calls to complete each FutureVar parameter. + for ident in futureVarIdents: + # Only complete them if they have not been completed already by the user. + # In the meantime, this was really useful for debugging :) + #result.add(newCall(newIdentNode("echo"), newStrLitNode(fromNode.lineinfo))) + result.add newIfStmt( + ( + newCall(newIdentNode("not"), + newDotExpr(ident, newIdentNode("finished"))), + newCallWithLineInfo(fromNode, newIdentNode("complete"), ident) + ) + ) + +proc processBody(ctx: Context; node, needsCompletionSym, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList, node) + + # As I've painfully found out, the order here really DOES matter. + result.add createFutureVarCompletions(futureVarIdents, node) + + ctx.hasRet = true + if node[0].kind == nnkEmpty: + if ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, newIdentNode("result")) + else: + result.add newAssignment(needsCompletionSym, newLit(true)) + else: + let x = processBody(ctx, node[0], needsCompletionSym, retFutureSym, futureVarIdents) + if x.kind == nnkYieldStmt: result.add x + elif ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, x) + else: + result.add newAssignment(newIdentNode("result"), x) + result.add newAssignment(needsCompletionSym, newLit(true)) + + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + return # Don't process the children of this return stmt + of RoutineNodes-{nnkTemplateDef}: + # skip all the nested procedure definitions + return + of nnkTryStmt: + if result[^1].kind == nnkFinally: + inc ctx.inTry + result[0] = processBody(ctx, result[0], needsCompletionSym, retFutureSym, futureVarIdents) + dec ctx.inTry + for i in 1 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + if ctx.inTry == 0 and ctx.hasRet: + let finallyNode = copyNimNode(result[^1]) + let stmtNode = newNimNode(nnkStmtList) + for child in result[^1]: + stmtNode.add child + stmtNode.add newIfStmt( + ( needsCompletionSym, + newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, + newIdentNode("result") + ) + ) + ) + finallyNode.add stmtNode + result[^1] = finallyNode + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + + # echo result.repr + +proc getName(node: NimNode): string = + case node.kind + of nnkPostfix: + return node[1].strVal + of nnkIdent, nnkSym: + return node.strVal + of nnkEmpty: + return "anonymous" + else: + error("Unknown name.", node) + +proc getFutureVarIdents(params: NimNode): seq[NimNode] = + result = @[] + for i in 1 ..< len(params): + expectKind(params[i], nnkIdentDefs) + if params[i][1].kind == nnkBracketExpr and + params[i][1][0].eqIdent(FutureVar.astToStr): + ## eqIdent: first char is case sensitive!!! + result.add(params[i][0]) + +proc isInvalidReturnType(typeName: string): bool = + return typeName notin ["Future"] #, "FutureStream"] + +proc verifyReturnType(typeName: string, node: NimNode = nil) = + if typeName.isInvalidReturnType: + error("Expected return type of 'Future' got '$1'" % + typeName, node) + +template await*(f: typed): untyped {.used.} = + static: + error "await expects Future[T], got " & $typeof(f) + +template await*[T](f: Future[T]): auto {.used.} = + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template yieldFuture {.redefine.} = yield FutureBase() + + when compiles(yieldFuture): + var internalTmpFuture: FutureBase = f + yield internalTmpFuture + (cast[typeof(f)](internalTmpFuture)).read() + else: + macro errorAsync(futureError: Future[T]) = + error( + "Can only 'await' inside a proc marked as 'async'. Use " & + "'waitFor' when calling an 'async' proc in a non-async scope instead", + futureError) + errorAsync(f) + +proc asyncSingleProc(prc: NimNode): NimNode = + ## This macro transforms a single procedure into a closure iterator. + ## The `async` macro supports a stmtList holding multiple async procedures. + if prc.kind == nnkProcTy: + result = prc + if prc[0][0].kind == nnkEmpty: + result[0][0] = quote do: Future[void] + return result + + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: + error("Cannot transform this node kind into an async proc." & + " proc/method definition or lambda node expected.", prc) + + if prc[4].kind != nnkEmpty: + for prag in prc[4]: + if prag.eqIdent("discardable"): + error("Cannot make async proc discardable. Futures have to be " & + "checked with `asyncCheck` instead of discarded", prag) + + let prcName = prc.name.getName + + var returnType = prc.params[0] + var baseType: NimNode + if returnType.kind in nnkCallKinds and returnType[0].eqIdent("owned") and + returnType.len == 2: + returnType = returnType[1] + # Verify that the return type is a Future[T] + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + verifyReturnType(fut, returnType[0]) + baseType = returnType[1] + elif returnType.kind in nnkCallKinds and returnType[0].eqIdent("[]"): + let fut = repr(returnType[1]) + verifyReturnType(fut, returnType[0]) + baseType = returnType[2] + elif returnType.kind == nnkEmpty: + baseType = returnType + else: + verifyReturnType(repr(returnType), returnType) + + let futureVarIdents = getFutureVarIdents(prc.params) + var outerProcBody = newNimNode(nnkStmtList, prc.body) + + # Extract the documentation comment from the original procedure declaration. + # Note that we're not removing it from the body in order not to make this + # transformation even more complex. + let body2 = extractDocCommentsAndRunnables(prc.body) + + # -> var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: baseType + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr, prc.body).add( + newIdentNode("newFuture"), + subRetType), + newLit(prcName)))) # Get type from return type of this proc + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} + # -> var result: T + # -> {.pop.} + # -> <proc_body> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") + var needsCompletionSym = genSym(nskVar, "needsCompletion") + var ctx = Context() + var procBody = processBody(ctx, prc.body, needsCompletionSym, retFutureSym, futureVarIdents) + # don't do anything with forward bodies (empty) + if procBody.kind != nnkEmpty: + # fix #13899, defer should not escape its original scope + let blockStmt = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) + procBody = newStmtList() + let resultIdent = ident"result" + procBody.add quote do: + # Check whether there is an implicit return + when typeof(`blockStmt`) is void: + `blockStmt` + else: + `resultIdent` = `blockStmt` + procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + procBody.insert(0): quote do: + {.push warning[resultshadowed]: off.} + when `subRetType` isnot void: + var `resultIdent`: `subRetType` + else: + var `resultIdent`: Future[void] + {.pop.} + + var `needsCompletionSym` = false + procBody.add quote do: + complete(`retFutureSym`, `resultIdent`) + + var closureIterator = newProc(iteratorNameSym, [quote do: owned(FutureBase)], + procBody, nnkIteratorDef) + closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom = prc.body) + closureIterator.addPragma(newIdentNode("closure")) + + # If proc has an explicit gcsafe pragma, we add it to iterator as well. + if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil: + closureIterator.addPragma(newIdentNode("gcsafe")) + outerProcBody.add(closureIterator) + + # -> createCb(retFuture) + # NOTE: The NimAsyncContinueSuffix is checked for in asyncfutures.nim to produce + # friendlier stack traces: + var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) + var procCb = getAst createCb(retFutureSym, iteratorNameSym, + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) + outerProcBody.add procCb + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) + + result = prc + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # xxx consider removing `owned`? it's inconsistent with non-void case + result.params[0] = quote do: owned(Future[void]) + + # based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 + if procBody.kind != nnkEmpty: + body2.add quote do: + `outerProcBody` + result.body = body2 + +macro async*(prc: untyped): untyped = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. + if prc.kind == nnkStmtList: + result = newStmtList() + for oneProc in prc: + result.add asyncSingleProc(oneProc) + else: + result = asyncSingleProc(prc) + when defined(nimDumpAsync): + echo repr result + +proc splitParamType(paramType: NimNode, async: bool): NimNode = + result = paramType + if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: + let firstAsync = "async" in paramType[1].toStrLit().strVal.normalize + let secondAsync = "async" in paramType[2].toStrLit().strVal.normalize + + if firstAsync: + result = paramType[if async: 1 else: 2] + elif secondAsync: + result = paramType[if async: 2 else: 1] + +proc stripReturnType(returnType: NimNode): NimNode = + # Strip out the 'Future' from 'Future[T]'. + result = returnType + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + verifyReturnType(fut, returnType) + result = returnType[1] + +proc splitProc(prc: NimNode): (NimNode, NimNode) = + ## Takes a procedure definition which takes a generic union of arguments, + ## for example: proc (socket: Socket | AsyncSocket). + ## It transforms them so that `proc (socket: Socket)` and + ## `proc (socket: AsyncSocket)` are returned. + + result[0] = prc.copyNimTree() + # Retrieve the `T` inside `Future[T]`. + let returnType = stripReturnType(result[0][3][0]) + result[0][3][0] = splitParamType(returnType, async = false) + for i in 1 ..< result[0][3].len: + # Sync proc (0) -> FormalParams (3) -> IdentDefs, the parameter (i) -> + # parameter type (1). + result[0][3][i][1] = splitParamType(result[0][3][i][1], async=false) + var multisyncAwait = quote: + template await(value: typed): untyped = + value + + result[0][^1] = nnkStmtList.newTree(multisyncAwait, result[0][^1]) + + result[1] = prc.copyNimTree() + if result[1][3][0].kind == nnkBracketExpr: + result[1][3][0][1] = splitParamType(result[1][3][0][1], async = true) + for i in 1 ..< result[1][3].len: + # Async proc (1) -> FormalParams (3) -> IdentDefs, the parameter (i) -> + # parameter type (1). + result[1][3][i][1] = splitParamType(result[1][3][i][1], async = true) + +macro multisync*(prc: untyped): untyped = + ## Macro which processes async procedures into both asynchronous and + ## synchronous procedures. + ## + ## The generated async procedures use the `async` macro, whereas the + ## generated synchronous procedures simply strip off the `await` calls. + let (sync, asyncPrc) = splitProc(prc) + result = newStmtList() + result.add(asyncSingleProc(asyncPrc)) + result.add(sync) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 6b19a48be..ee07e599e 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -1,47 +1,86 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta +# (c) Copyright 2017 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements a high-level asynchronous sockets API based on the -## asynchronous dispatcher defined in the ``asyncdispatch`` module. +## asynchronous dispatcher defined in the `asyncdispatch` module. +## +## Asynchronous IO in Nim +## ====================== +## +## Async IO in Nim consists of multiple layers (from highest to lowest): +## +## * `asyncnet` module +## +## * Async await +## +## * `asyncdispatch` module (event loop) +## +## * `selectors` module +## +## Each builds on top of the layers below it. The selectors module is an +## abstraction for the various system `select()` mechanisms such as epoll or +## kqueue. If you wish you can use it directly, and some people have done so +## `successfully <http://goran.krampe.se/2014/10/25/nim-socketserver/>`_. +## But you must be aware that on Windows it only supports +## `select()`. +## +## The async dispatcher implements the proactor pattern and also has an +## implementation of IOCP. It implements the proactor pattern for other +## OS' via the selectors module. Futures are also implemented here, and +## indeed all the procedures return a future. +## +## The final layer is the async await transformation. This allows you to +## write asynchronous code in a synchronous style and works similar to +## C#'s await. The transformation works by converting any async procedures +## into an iterator. +## +## This is all single threaded, fully non-blocking and does give you a +## lot of control. In theory you should be able to work with any of these +## layers interchangeably (as long as you only care about non-Windows +## platforms). +## +## For most applications using `asyncnet` is the way to go as it builds +## over all the layers, providing some extra features such as buffering. ## ## SSL -## --- +## === ## -## SSL can be enabled by compiling with the ``-d:ssl`` flag. +## SSL can be enabled by compiling with the `-d:ssl` flag. ## -## You must create a new SSL context with the ``newContext`` function defined -## in the ``net`` module. You may then call ``wrapSocket`` on your socket using +## You must create a new SSL context with the `newContext` function defined +## in the `net` module. You may then call `wrapSocket` on your socket using ## the newly created SSL context to get an SSL socket. ## ## Examples -## -------- +## ======== ## ## Chat server -## ^^^^^^^^^^^ +## ----------- ## ## The following example demonstrates a simple chat server. ## -## .. code-block::nim -## -## import asyncnet, asyncdispatch +## ```Nim +## import std/[asyncnet, asyncdispatch] ## ## var clients {.threadvar.}: seq[AsyncSocket] ## ## proc processClient(client: AsyncSocket) {.async.} = ## while true: ## let line = await client.recvLine() +## if line.len == 0: break ## for c in clients: ## await c.send(line & "\c\L") ## ## proc serve() {.async.} = ## clients = @[] ## var server = newAsyncSocket() +## server.setSockOpt(OptReuseAddr, true) ## server.bindAddr(Port(12345)) ## server.listen() ## @@ -53,52 +92,69 @@ ## ## asyncCheck serve() ## runForever() -## +## ``` + +import std/private/since -import asyncdispatch -import nativesockets -import net -import os +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +import std/[asyncdispatch, nativesockets, net, os] export SOBool -when defined(ssl): - import openssl +# TODO: Remove duplication introduced by PR #4683. + +const defineSsl = defined(ssl) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) + +when defineSsl: + import std/openssl type # TODO: I would prefer to just do: # AsyncSocket* {.borrow: `.`.} = distinct Socket. But that doesn't work. - AsyncSocketDesc = object + AsyncSocketDesc = object fd: SocketHandle - closed: bool ## determines whether this socket has been closed - case isBuffered: bool ## determines whether this socket is buffered. - of true: - buffer: array[0..BufferSize, char] - currPos: int # current index in buffer - bufLen: int # current length of buffer - of false: nil - case isSsl: bool - of true: - when defined(ssl): - sslHandle: SslPtr - sslContext: SslContext - bioIn: BIO - bioOut: BIO - of false: nil + closed: bool ## determines whether this socket has been closed + isBuffered: bool ## determines whether this socket is buffered. + buffer: array[0..BufferSize, char] + currPos: int # current index in buffer + bufLen: int # current length of buffer + isSsl: bool + when defineSsl: + sslHandle: SslPtr + sslContext: SslContext + bioIn: BIO + bioOut: BIO + sslNoShutdown: bool domain: Domain sockType: SockType protocol: Protocol AsyncSocket* = ref AsyncSocketDesc -{.deprecated: [PAsyncSocket: AsyncSocket].} - proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = - ## Creates a new ``AsyncSocket`` based on the supplied params. + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, + buffered = true, + inheritable = defined(nimInheritHandles)): owned(AsyncSocket) = + ## Creates a new `AsyncSocket` based on the supplied params. + ## + ## The supplied `fd`'s non-blocking state will be enabled implicitly. + ## + ## If `inheritable` is false (the default), the supplied `fd` will not + ## be inheritable by child processes. + ## + ## **Note**: This procedure will **NOT** register `fd` with the global + ## async dispatcher. You need to do this manually. If you have used + ## `newAsyncNativeSocket` to create `fd` then it's already registered. assert fd != osInvalidSocket.AsyncFD new(result) result.fd = fd.SocketHandle + fd.SocketHandle.setBlocking(false) + if not fd.SocketHandle.setInheritable(inheritable): + raiseOSError(osLastError()) result.isBuffered = buffered result.domain = domain result.sockType = sockType @@ -107,28 +163,54 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, result.currPos = 0 proc newAsyncSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = + protocol: Protocol = IPPROTO_TCP, buffered = true, + inheritable = defined(nimInheritHandles)): owned(AsyncSocket) = ## Creates a new asynchronous socket. ## ## This procedure will also create a brand new file descriptor for ## this socket. - result = newAsyncSocket(newAsyncNativeSocket(domain, sockType, protocol), - domain, sockType, protocol, buffered) + ## + ## If `inheritable` is false (the default), the new file descriptor will not + ## be inheritable by child processes. + let fd = createAsyncNativeSocket(domain, sockType, protocol, inheritable) + if fd.SocketHandle == osInvalidSocket: + raiseOSError(osLastError()) + result = newAsyncSocket(fd, domain, sockType, protocol, buffered, inheritable) + +proc getLocalAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's local address and port number. + ## + ## This is high-level interface for `getsockname`:idx:. + getLocalAddr(socket.fd, socket.domain) + +when not useNimNetLite: + proc getPeerAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc newAsyncSocket*(domain, sockType, protocol: cint, - buffered = true): AsyncSocket = + buffered = true, + inheritable = defined(nimInheritHandles)): owned(AsyncSocket) = ## Creates a new asynchronous socket. ## ## This procedure will also create a brand new file descriptor for ## this socket. - result = newAsyncSocket(newAsyncNativeSocket(domain, sockType, protocol), - Domain(domain), SockType(sockType), - Protocol(protocol), buffered) + ## + ## If `inheritable` is false (the default), the new file descriptor will not + ## be inheritable by child processes. + let fd = createAsyncNativeSocket(domain, sockType, protocol, inheritable) + if fd.SocketHandle == osInvalidSocket: + raiseOSError(osLastError()) + result = newAsyncSocket(fd, Domain(domain), SockType(sockType), + Protocol(protocol), buffered, inheritable) -when defined(ssl): - proc getSslError(handle: SslPtr, err: cint): cint = +when defineSsl: + proc getSslError(socket: AsyncSocket, err: cint): cint = + assert socket.isSsl assert err < 0 - var ret = SSLGetError(handle, err.cint) + var ret = SSL_get_error(socket.sslHandle, err.cint) case ret of SSL_ERROR_ZERO_RETURN: raiseSSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") @@ -139,6 +221,7 @@ when defined(ssl): of SSL_ERROR_WANT_X509_LOOKUP: raiseSSLError("Function for x509 lookup has been called.") of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + socket.sslNoShutdown = true raiseSSLError() else: raiseSSLError("Unknown Error") @@ -146,67 +229,107 @@ when defined(ssl): flags: set[SocketFlag]) {.async.} = let len = bioCtrlPending(socket.bioOut) if len > 0: - var data = newStringOfCap(len) - let read = bioRead(socket.bioOut, addr data[0], len) + var data = newString(len) + let read = bioRead(socket.bioOut, cast[cstring](addr data[0]), len) assert read != 0 if read < 0: - raiseSslError() + raiseSSLError() data.setLen(read) - await socket.fd.AsyncFd.send(data, flags) + await socket.fd.AsyncFD.send(data, flags) proc appeaseSsl(socket: AsyncSocket, flags: set[SocketFlag], - sslError: cint) {.async.} = + sslError: cint): owned(Future[bool]) {.async.} = + ## Returns `true` if `socket` is still connected, otherwise `false`. + result = true case sslError of SSL_ERROR_WANT_WRITE: await sendPendingSslData(socket, flags) of SSL_ERROR_WANT_READ: var data = await recv(socket.fd.AsyncFD, BufferSize, flags) - let ret = bioWrite(socket.bioIn, addr data[0], data.len.cint) - if ret < 0: - raiseSSLError() + let length = len(data) + if length > 0: + let ret = bioWrite(socket.bioIn, cast[cstring](addr data[0]), length.cint) + if ret < 0: + raiseSSLError() + elif length == 0: + # connection not properly closed by remote side or connection dropped + SSL_set_shutdown(socket.sslHandle, SSL_RECEIVED_SHUTDOWN) + result = false else: raiseSSLError("Cannot appease SSL.") template sslLoop(socket: AsyncSocket, flags: set[SocketFlag], - op: expr) = + op: untyped) = var opResult {.inject.} = -1.cint while opResult < 0: + ErrClearError() + # Call the desired operation. opResult = op - # Bit hackish here. - # TODO: Introduce an async template transformation pragma? - yield sendPendingSslData(socket, flags) + let err = + if opResult < 0: + getSslError(socket, opResult.cint) + else: + SSL_ERROR_NONE + # Send any remaining pending SSL data. + await sendPendingSslData(socket, flags) + + # If the operation failed, try to see if SSL has some data to read + # or write. if opResult < 0: - let err = getSslError(socket.sslHandle, opResult.cint) - yield appeaseSsl(socket, flags, err.cint) + let fut = appeaseSsl(socket, flags, err.cint) + yield fut + if not fut.read(): + # Socket disconnected. + if SocketFlag.SafeDisconn in flags: + opResult = 0.cint + break + else: + raiseSSLError("Socket has been disconnected") + +proc dial*(address: string, port: Port, protocol = IPPROTO_TCP, + buffered = true): owned(Future[AsyncSocket]) {.async.} = + ## Establishes connection to the specified `address`:`port` pair via the + ## specified protocol. The procedure iterates through possible + ## resolutions of the `address` until it succeeds, meaning that it + ## seamlessly works with both IPv4 and IPv6. + ## Returns AsyncSocket ready to send or receive data. + let asyncFd = await asyncdispatch.dial(address, port, protocol) + let sockType = protocol.toSockType() + let domain = getSockDomain(asyncFd.SocketHandle) + result = newAsyncSocket(asyncFd, domain, sockType, protocol, buffered) proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = - ## Connects ``socket`` to server at ``address:port``. + ## Connects `socket` to server at `address:port`. ## - ## Returns a ``Future`` which will complete when the connection succeeds + ## Returns a `Future` which will complete when the connection succeeds ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: - when defined(ssl): + when defineSsl: + if not isIpAddress(address): + # Set the SNI address for this connection. This call can fail if + # we're not using TLSv1+. + discard SSL_set_tlsext_host_name(socket.sslHandle, address) + let flags = {SocketFlag.SafeDisconn} sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) -template readInto(buf: cstring, size: int, socket: AsyncSocket, +template readInto(buf: pointer, size: int, socket: AsyncSocket, flags: set[SocketFlag]): int = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that + ## Reads **up to** `size` bytes from `socket` into `buf`. Note that ## this is a template and not a proc. + assert(not socket.closed, "Cannot `recv` on a closed socket") var res = 0 if socket.isSsl: - when defined(ssl): + when defineSsl: # SSL mode. sslLoop(socket, flags, - sslRead(socket.sslHandle, buf, size.cint)) + sslRead(socket.sslHandle, cast[cstring](buf), size.cint)) res = opResult else: - var recvIntoFut = recvInto(socket.fd.AsyncFD, buf, size, flags) - yield recvIntoFut # Not in SSL mode. - res = recvIntoFut.read() + res = await asyncdispatch.recvInto(socket.fd.AsyncFD, buf, size, flags) res template readIntoBuf(socket: AsyncSocket, @@ -216,12 +339,60 @@ template readIntoBuf(socket: AsyncSocket, socket.bufLen = size size +proc recvInto*(socket: AsyncSocket, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) {.async.} = + ## Reads **up to** `size` bytes from `socket` into `buf`. + ## + ## For buffered sockets this function will attempt to read all the requested + ## data. It will read this data in `BufferSize` chunks. + ## + ## For unbuffered sockets this function makes no effort to read + ## all the data requested. It will return as much data as the operating system + ## gives it. + ## + ## If socket is disconnected during the + ## recv operation then the future may complete with only a part of the + ## requested data. + ## + ## If socket is disconnected and no data is available + ## to be read then the future will complete with a value of `0`. + if socket.isBuffered: + let originalBufPos = socket.currPos + + if socket.bufLen == 0: + let res = socket.readIntoBuf(flags - {SocketFlag.Peek}) + if res == 0: + return 0 + + var read = 0 + var cbuf = cast[cstring](buf) + while read < size: + if socket.currPos >= socket.bufLen: + if SocketFlag.Peek in flags: + # We don't want to get another buffer if we're peeking. + break + let res = socket.readIntoBuf(flags - {SocketFlag.Peek}) + if res == 0: + break + + let chunk = min(socket.bufLen-socket.currPos, size-read) + copyMem(addr(cbuf[read]), addr(socket.buffer[socket.currPos]), chunk) + read.inc(chunk) + socket.currPos.inc(chunk) + + if SocketFlag.Peek in flags: + # Restore old buffer cursor position. + socket.currPos = originalBufPos + result = read + else: + result = readInto(buf, size, socket, flags) + proc recv*(socket: AsyncSocket, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = - ## Reads **up to** ``size`` bytes from ``socket``. + flags = {SocketFlag.SafeDisconn}): owned(Future[string]) {.async.} = + ## Reads **up to** `size` bytes from `socket`. ## ## For buffered sockets this function will attempt to read all the requested - ## data. It will read this data in ``BufferSize`` chunks. + ## data. It will read this data in `BufferSize` chunks. ## ## For unbuffered sockets this function makes no effort to read ## all the data requested. It will return as much data as the operating system @@ -232,10 +403,11 @@ proc recv*(socket: AsyncSocket, size: int, ## requested data. ## ## If socket is disconnected and no data is available - ## to be read then the future will complete with a value of ``""``. + ## to be read then the future will complete with a value of `""`. if socket.isBuffered: result = newString(size) - shallow(result) + when not defined(nimSeqsV2): + shallow(result) let originalBufPos = socket.currPos if socket.bufLen == 0: @@ -268,27 +440,46 @@ proc recv*(socket: AsyncSocket, size: int, let read = readInto(addr result[0], size, socket, flags) result.setLen(read) +proc send*(socket: AsyncSocket, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}) {.async.} = + ## Sends `size` bytes from `buf` to `socket`. The returned future will complete once all + ## data has been sent. + assert socket != nil + assert(not socket.closed, "Cannot `send` on a closed socket") + if socket.isSsl: + when defineSsl: + sslLoop(socket, flags, + sslWrite(socket.sslHandle, cast[cstring](buf), size.cint)) + await sendPendingSslData(socket, flags) + else: + await send(socket.fd.AsyncFD, buf, size, flags) + proc send*(socket: AsyncSocket, data: string, flags = {SocketFlag.SafeDisconn}) {.async.} = - ## Sends ``data`` to ``socket``. The returned future will complete once all + ## Sends `data` to `socket`. The returned future will complete once all ## data has been sent. assert socket != nil if socket.isSsl: - when defined(ssl): + when defineSsl: var copy = data sslLoop(socket, flags, - sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) + sslWrite(socket.sslHandle, cast[cstring](addr copy[0]), copy.len.cint)) await sendPendingSslData(socket, flags) else: await send(socket.fd.AsyncFD, data, flags) -proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncSocket]] = +proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): + owned(Future[tuple[address: string, client: AsyncSocket]]) = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. + ## + ## If `inheritable` is false (the default), the resulting client socket will + ## not be inheritable by child processes. + ## ## The future will complete when the connection is successfully accepted. var retFuture = newFuture[tuple[address: string, client: AsyncSocket]]("asyncnet.acceptAddr") - var fut = acceptAddr(socket.fd.AsyncFD, flags) + var fut = acceptAddr(socket.fd.AsyncFD, flags, inheritable) fut.callback = proc (future: Future[tuple[address: string, client: AsyncFD]]) = assert future.finished @@ -297,14 +488,16 @@ proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}): else: let resultTup = (future.read.address, newAsyncSocket(future.read.client, socket.domain, - socket.sockType, socket.protocol, socket.isBuffered)) + socket.sockType, socket.protocol, socket.isBuffered, inheritable)) retFuture.complete(resultTup) return retFuture proc accept*(socket: AsyncSocket, - flags = {SocketFlag.SafeDisconn}): Future[AsyncSocket] = + flags = {SocketFlag.SafeDisconn}): owned(Future[AsyncSocket]) = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection. + ## If `inheritable` is false (the default), the resulting client socket will + ## not be inheritable by child processes. ## The future will complete when the connection is successfully accepted. var retFut = newFuture[AsyncSocket]("asyncnet.accept") var fut = acceptAddr(socket, flags) @@ -318,36 +511,33 @@ proc accept*(socket: AsyncSocket, return retFut proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], - flags = {SocketFlag.SafeDisconn}) {.async.} = - ## Reads a line of data from ``socket`` into ``resString``. + flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {.async.} = + ## Reads a line of data from `socket` into `resString`. ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## If a full line is read `\r\L` is not + ## added to `line`, however if solely `\r\L` is read then `line` ## will be set to it. ## - ## If the socket is disconnected, ``line`` will be set to ``""``. + ## If the socket is disconnected, `line` will be set to `""`. ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. + ## If the socket is disconnected in the middle of a line (before `\r\L` + ## is read) then line will be set to `""`. ## The partial line **will be lost**. ## - ## **Warning**: The ``Peek`` flag is not yet implemented. + ## The `maxLength` parameter determines the maximum amount of characters + ## that can be read. `resString` will be truncated after that. ## - ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the - ## protocol uses ``\r\L`` to delimit a new line. + ## .. warning:: The `Peek` flag is not yet implemented. ## - ## **Warning**: ``recvLineInto`` currently uses a raw pointer to a string for - ## performance reasons. This will likely change soon to use FutureVars. + ## .. warning:: `recvLineInto` on unbuffered sockets assumes that the protocol uses `\r\L` to delimit a new line. assert SocketFlag.Peek notin flags ## TODO: - assert(not resString.mget.isNil(), - "String inside resString future needs to be initialised") result = newFuture[void]("asyncnet.recvLineInto") # TODO: Make the async transformation check for FutureVar params and complete # them when the result future is completed. # Can we replace the result future with the FutureVar? - template addNLIfEmpty(): stmt = + template addNLIfEmpty(): untyped = if resString.mget.len == 0: resString.mget.add("\c\L") @@ -384,18 +574,19 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], else: resString.mget.add socket.buffer[socket.currPos] socket.currPos.inc() + + # Verify that this isn't a DOS attack: #3847. + if resString.mget.len > maxLength: break else: var c = "" while true: - let recvFut = recv(socket, 1, flags) - c = recvFut.read() + c = await recv(socket, 1, flags) if c.len == 0: resString.mget.setLen(0) resString.complete() return if c == "\r": - let recvFut = recv(socket, 1, flags) # Skip \L - c = recvFut.read() + c = await recv(socket, 1, flags) # Skip \L assert c == "\L" addNLIfEmpty() resString.complete() @@ -405,100 +596,186 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], resString.complete() return resString.mget.add c + + # Verify that this isn't a DOS attack: #3847. + if resString.mget.len > maxLength: break resString.complete() proc recvLine*(socket: AsyncSocket, - flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once + flags = {SocketFlag.SafeDisconn}, + maxLength = MaxLineLength): owned(Future[string]) {.async.} = + ## Reads a line of data from `socket`. Returned future will complete once ## a full line is read or an error occurs. ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## If a full line is read `\r\L` is not + ## added to `line`, however if solely `\r\L` is read then `line` ## will be set to it. ## - ## If the socket is disconnected, ``line`` will be set to ``""``. + ## If the socket is disconnected, `line` will be set to `""`. ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. + ## If the socket is disconnected in the middle of a line (before `\r\L` + ## is read) then line will be set to `""`. ## The partial line **will be lost**. ## - ## **Warning**: The ``Peek`` flag is not yet implemented. + ## The `maxLength` parameter determines the maximum amount of characters + ## that can be read. The result is truncated after that. ## - ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol - ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") + ## .. warning:: The `Peek` flag is not yet implemented. + ## + ## .. warning:: `recvLine` on unbuffered sockets assumes that the protocol uses `\r\L` to delimit a new line. assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this var resString = newFutureVar[string]("asyncnet.recvLine") resString.mget() = "" - await socket.recvLineInto(resString, flags) + await socket.recvLineInto(resString, flags, maxLength) result = resString.mget() -proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the +proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ + ReadIOEffect].} = + ## Marks `socket` as accepting connections. + ## `Backlog` specifies the maximum length of the ## queue of pending connections. ## - ## Raises an EOS error upon failure. + ## Raises an OSError error upon failure. if listen(socket.fd, backlog) < 0'i32: raiseOSError(osLastError()) proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. tags: [ReadIOEffect].} = - ## Binds ``address``:``port`` to the socket. + ## Binds `address`:`port` to the socket. ## - ## If ``address`` is "" then ADDR_ANY will be bound. + ## If `address` is "" then ADDR_ANY will be bound. var realaddr = address if realaddr == "": case socket.domain of AF_INET6: realaddr = "::" - of AF_INET: realaddr = "0.0.0.0" + of AF_INET: realaddr = "0.0.0.0" else: - raiseOSError("Unknown socket address family and no address specified to bindAddr") + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") var aiList = getAddrInfo(realaddr, port, socket.domain) - if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: - dealloc(aiList) + if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: + freeAddrInfo(aiList) raiseOSError(osLastError()) - dealloc(aiList) + freeAddrInfo(aiList) + +proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = + ## Determines whether an AsyncSocket has data buffered. + # xxx dedup with std/net + s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen + +when defined(posix) and not useNimNetLite: + + proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + let retFuture = newFuture[void]("connectUnix") + result = retFuture + + proc cb(fd: AsyncFD): bool = + let ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + retFuture.complete() + return true + elif ret == EINTR: + return false + else: + retFuture.fail(newOSError(OSErrorCode(ret))) + return true + + var socketAddr = makeUnixAddr(path) + let ret = socket.fd.connect(cast[ptr SockAddr](addr socketAddr), + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) + if ret == 0: + # Request to connect completed immediately. + retFuture.complete() + else: + let lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + addWrite(AsyncFD(socket.fd), cb) + else: + retFuture.fail(newOSError(lastError)) + + proc bindUnix*(socket: AsyncSocket, path: string) {. + tags: [ReadIOEffect].} = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: + raiseOSError(osLastError()) + +elif defined(nimdoc): + + proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + discard + + proc bindUnix*(socket: AsyncSocket, path: string) = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + discard proc close*(socket: AsyncSocket) = ## Closes the socket. + if socket.closed: return + defer: socket.fd.AsyncFD.closeSocket() - when defined(ssl): - if socket.isSSL: - let res = SslShutdown(socket.sslHandle) - SSLFree(socket.sslHandle) + socket.closed = true # TODO: Add extra debugging checks for this. + + when defineSsl: + if socket.isSsl: + let res = + # Don't call SSL_shutdown if the connection has not been fully + # established, see: + # https://github.com/openssl/openssl/issues/710#issuecomment-253897666 + if not socket.sslNoShutdown and SSL_in_init(socket.sslHandle) == 0: + ErrClearError() + SSL_shutdown(socket.sslHandle) + else: + 0 + SSL_free(socket.sslHandle) if res == 0: discard elif res != 1: - raiseSslError() - socket.closed = true # TODO: Add extra debugging checks for this. + raiseSSLError() -when defined(ssl): +when defineSsl: + proc sslHandle*(self: AsyncSocket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns - ## ``socket`` into an SSL socket. + ## `socket` into an SSL socket. ## ## **Disclaimer**: This code is not well tested, may be very unsafe and ## prone to security vulnerabilities. socket.isSsl = true socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) + socket.sslHandle = SSL_new(socket.sslContext.context) if socket.sslHandle == nil: - raiseSslError() + raiseSSLError() - socket.bioIn = bioNew(bio_s_mem()) - socket.bioOut = bioNew(bio_s_mem()) + socket.bioIn = bioNew(bioSMem()) + socket.bioOut = bioNew(bioSMem()) sslSetBio(socket.sslHandle, socket.bioIn, socket.bioOut) + socket.sslNoShutdown = true + proc wrapConnectedSocket*(ctx: SslContext, socket: AsyncSocket, - handshake: SslHandshakeType) = + handshake: SslHandshakeType, + hostname: string = "") = ## Wraps a connected socket in an SSL context. This function effectively - ## turns ``socket`` into an SSL socket. + ## turns `socket` into an SSL socket. + ## `hostname` should be specified so that the client knows which hostname + ## the server certificate should be validated against. ## ## This should be called on a connected socket, and will perform ## an SSL handshake immediately. @@ -509,24 +786,39 @@ when defined(ssl): case handshake of handshakeAsClient: + if hostname.len > 0 and not isIpAddress(hostname): + # Set the SNI address for this connection. This call can fail if + # we're not using TLSv1+. + discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) sslSetConnectState(socket.sslHandle) of handshakeAsServer: sslSetAcceptState(socket.sslHandle) + proc getPeerCertificates*(socket: AsyncSocket): seq[Certificate] {.since: (1, 1).} = + ## Returns the certificate chain received by the peer we are connected to + ## through the given socket. + ## The handshake must have been completed and the certificate chain must + ## have been verified successfully or else an empty sequence is returned. + ## The chain is ordered from leaf certificate to root certificate. + if not socket.isSsl: + result = newSeq[Certificate]() + else: + result = getPeerCertificates(socket.sslHandle) + proc getSockOpt*(socket: AsyncSocket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = - ## Retrieves option ``opt`` as a boolean value. + ## Retrieves option `opt` as a boolean value. var res = getSockOptInt(socket.fd, cint(level), toCInt(opt)) result = res != 0 proc setSockOpt*(socket: AsyncSocket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = - ## Sets option ``opt`` to a boolean value specified by ``value``. + ## Sets option `opt` to a boolean value specified by `value`. var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) proc isSsl*(socket: AsyncSocket): bool = - ## Determines whether ``socket`` is a SSL socket. + ## Determines whether `socket` is a SSL socket. socket.isSsl proc getFd*(socket: AsyncSocket): SocketHandle = @@ -537,6 +829,130 @@ proc isClosed*(socket: AsyncSocket): bool = ## Determines whether the socket has been closed. return socket.closed +proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) + {.async, since: (1, 3).} = + ## This proc sends `data` to the specified `address`, which may be an IP + ## address or a hostname. If a hostname is specified this function will try + ## each IP of that hostname. The returned future will complete once all data + ## has been sent. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + assert(socket.protocol != IPPROTO_TCP, + "Cannot `sendTo` on a TCP socket. Use `send` instead") + assert(not socket.closed, "Cannot `sendTo` on a closed socket") + + let aiList = getAddrInfo(address, port, socket.domain, socket.sockType, + socket.protocol) + + var + it = aiList + success = false + lastException: ref Exception + + while it != nil: + let fut = sendTo(socket.fd.AsyncFD, cstring(data), len(data), it.ai_addr, + it.ai_addrlen.SockLen, flags) + + yield fut + + if not fut.failed: + success = true + + break + + lastException = fut.readError() + + it = it.ai_next + + freeAddrInfo(aiList) + + if not success: + if lastException != nil: + raise lastException + else: + raise newException(IOError, "Couldn't resolve address: " & address) + +proc recvFrom*(socket: AsyncSocket, data: FutureVar[string], size: int, + address: FutureVar[string], port: FutureVar[Port], + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) + {.async, since: (1, 3).} = + ## Receives a datagram data from `socket` into `data`, which must be at + ## least of size `size`. The address and port of datagram's sender will be + ## stored into `address` and `port`, respectively. Returned future will + ## complete once one datagram has been received, and will return size of + ## packet received. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + ## + ## **Notes** + ## * `data` must be initialized to the length of `size`. + ## * `address` must be initialized to 46 in length. + template adaptRecvFromToDomain(domain: Domain) = + var lAddr = sizeof(sAddr).SockLen + + result = await recvFromInto(AsyncFD(getFd(socket)), cstring(data.mget()), size, + cast[ptr SockAddr](addr sAddr), addr lAddr, + flags) + + data.mget().setLen(result) + data.complete() + + getAddrString(cast[ptr SockAddr](addr sAddr), address.mget()) + + address.complete() + + when domain == AF_INET6: + port.complete(ntohs(sAddr.sin6_port).Port) + else: + port.complete(ntohs(sAddr.sin_port).Port) + + assert(socket.protocol != IPPROTO_TCP, + "Cannot `recvFrom` on a TCP socket. Use `recv` or `recvInto` instead") + assert(not socket.closed, "Cannot `recvFrom` on a closed socket") + assert(size == len(data.mget()), + "`date` was not initialized correctly. `size` != `len(data.mget())`") + assert(46 == len(address.mget()), + "`address` was not initialized correctly. 46 != `len(address.mget())`") + + case socket.domain + of AF_INET6: + var sAddr: Sockaddr_in6 + adaptRecvFromToDomain(AF_INET6) + of AF_INET: + var sAddr: Sockaddr_in + adaptRecvFromToDomain(AF_INET) + else: + raise newException(ValueError, "Unknown socket address family") + +proc recvFrom*(socket: AsyncSocket, size: int, + flags = {SocketFlag.SafeDisconn}): + owned(Future[tuple[data: string, address: string, port: Port]]) + {.async, since: (1, 3).} = + ## Receives a datagram data from `socket`, which must be at least of size + ## `size`. Returned future will complete once one datagram has been received + ## and will return tuple with: data of packet received; and address and port + ## of datagram's sender. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + var + data = newFutureVar[string]() + address = newFutureVar[string]() + port = newFutureVar[Port]() + + data.mget().setLen(size) + address.mget().setLen(46) + + let read = await recvFrom(socket, data, size, address, port, flags) + + result = (data.mget(), address.mget(), port.mget()) + when not defined(testing) and isMainModule: type TestCases = enum diff --git a/lib/pure/asyncstreams.nim b/lib/pure/asyncstreams.nim new file mode 100644 index 000000000..c97b98d55 --- /dev/null +++ b/lib/pure/asyncstreams.nim @@ -0,0 +1,147 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Unstable API. + +import std/asyncfutures + +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/deques + +type + FutureStream*[T] = ref object ## Special future that acts as + ## a queue. Its API is still + ## experimental and so is + ## subject to change. + queue: Deque[T] + finished: bool + cb: proc () {.closure, gcsafe.} + error*: ref Exception + +proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] = + ## Create a new `FutureStream`. This future's callback is activated when + ## two events occur: + ## + ## * New data is written into the future stream. + ## * The future stream is completed (this means that no more data will be + ## written). + ## + ## Specifying `fromProc`, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + ## + ## **Note:** The API of FutureStream is still new and so has a higher + ## likelihood of changing in the future. + result = FutureStream[T](finished: false, cb: nil) + result.queue = initDeque[T]() + +proc complete*[T](future: FutureStream[T]) = + ## Completes a `FutureStream` signalling the end of data. + assert(future.error == nil, "Trying to complete failed stream") + future.finished = true + if not future.cb.isNil: + future.cb() + +proc fail*[T](future: FutureStream[T], error: ref Exception) = + ## Completes `future` with `error`. + assert(not future.finished) + future.finished = true + future.error = error + if not future.cb.isNil: + future.cb() + +proc `callback=`*[T](future: FutureStream[T], + cb: proc (future: FutureStream[T]) {.closure, gcsafe.}) = + ## Sets the callback proc to be called when data was placed inside the + ## future stream. + ## + ## The callback is also called when the future is completed. So you should + ## use `finished` to check whether data is available. + ## + ## If the future stream already has data or is finished then `cb` will be + ## called immediately. + proc named() = cb(future) + future.cb = named + if future.queue.len > 0 or future.finished: + callSoon(future.cb) + +proc finished*[T](future: FutureStream[T]): bool = + ## Check if a `FutureStream` is finished. `true` value means that + ## no more data will be placed inside the stream *and* that there is + ## no data waiting to be retrieved. + result = future.finished and future.queue.len == 0 + +proc failed*[T](future: FutureStream[T]): bool = + ## Determines whether `future` completed with an error. + return future.error != nil + +proc write*[T](future: FutureStream[T], value: T): Future[void] = + ## Writes the specified value inside the specified future stream. + ## + ## This will raise `ValueError` if `future` is finished. + result = newFuture[void]("FutureStream.put") + if future.finished: + let msg = "FutureStream is finished and so no longer accepts new data." + result.fail(newException(ValueError, msg)) + return + # TODO: Implement limiting of the streams storage to prevent it growing + # infinitely when no reads are occurring. + future.queue.addLast(value) + if not future.cb.isNil: future.cb() + result.complete() + +proc read*[T](future: FutureStream[T]): owned(Future[(bool, T)]) = + ## Returns a future that will complete when the `FutureStream` has data + ## placed into it. The future will be completed with the oldest + ## value stored inside the stream. The return value will also determine + ## whether data was retrieved, `false` means that the future stream was + ## completed and no data was retrieved. + ## + ## This function will remove the data that was returned from the underlying + ## `FutureStream`. + var resFut = newFuture[(bool, T)]("FutureStream.take") + let savedCb = future.cb + proc newCb(fs: FutureStream[T]) = + # Exit early if `resFut` is already complete. (See #8994). + if resFut.finished: return + + # We don't want this callback called again. + #future.cb = nil + + # The return value depends on whether the FutureStream has finished. + var res: (bool, T) + if finished(fs): + # Remember, this callback is called when the FutureStream is completed. + res[0] = false + else: + res[0] = true + res[1] = fs.queue.popFirst() + + if fs.failed: + resFut.fail(fs.error) + else: + resFut.complete(res) + + # If the saved callback isn't nil then let's call it. + if not savedCb.isNil: + if fs.queue.len > 0: + savedCb() + else: + future.cb = savedCb + + if future.queue.len > 0 or future.finished: + newCb(future) + else: + future.callback = newCb + return resFut + +proc len*[T](future: FutureStream[T]): int = + ## Returns the amount of data pieces inside the stream. + future.queue.len diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 32d37ce02..591d22cc0 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -9,157 +9,265 @@ ## This module implements a base64 encoder and decoder. ## -## Encoding data -## ------------- -## -## In order to encode some text simply call the ``encode`` procedure: +## Unstable API. ## -## .. code-block::nim -## import base64 -## let encoded = encode("Hello World") -## echo(encoded) # SGVsbG8gV29ybGQ= +## Base64 is an encoding and decoding technique used to convert binary +## data to an ASCII string format. +## Each Base64 digit represents exactly 6 bits of data. Three 8-bit +## bytes (i.e., a total of 24 bits) can therefore be represented by +## four 6-bit Base64 digits. + +##[ +# Basic usage +## Encoding data +]## + +runnableExamples: + let encoded = encode("Hello World") + assert encoded == "SGVsbG8gV29ybGQ=" + ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block::nim -## import base64 -## let encodedInts = encode([1,2,3]) -## echo(encodedInts) # AQID -## let encodedChars = encode(['h','e','y']) -## echo(encodedChars) # aGV5 -## -## The ``encode`` procedure takes an ``openarray`` so both arrays and sequences -## can be passed as parameters. -## + +runnableExamples: + let encodedInts = encode([1'u8,2,3]) + assert encodedInts == "AQID" + let encodedChars = encode(['h','e','y']) + assert encodedChars == "aGV5" + +##[ ## Decoding data -## ------------- -## -## To decode a base64 encoded data string simply call the ``decode`` -## procedure: +]## + +runnableExamples: + let decoded = decode("SGVsbG8gV29ybGQ=") + assert decoded == "Hello World" + +##[ +## URL Safe Base64 +]## + +runnableExamples: + assert encode("c\xf7>", safe = true) == "Y_c-" + assert encode("c\xf7>", safe = false) == "Y/c+" + +## See also +## ======== ## -## .. code-block::nim -## import base64 -## echo(decode("SGVsbG8gV29ybGQ=")) # Hello World +## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm +## * `sha1 module<sha1.html>`_ for the SHA-1 checksum algorithm + +template cbBase(a, b): untyped = [ + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b] const - cb64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" - -template encodeInternal(s: expr, lineLen: int, newLine: string): stmt {.immediate.} = - ## encodes `s` into base64 representation. After `lineLen` characters, a - ## `newline` is added. - var total = ((len(s) + 2) div 3) * 4 - var numLines = (total + lineLen - 1) div lineLen - if numLines > 0: inc(total, (numLines-1) * newLine.len) - - result = newString(total) - var i = 0 - var r = 0 - var currLine = 0 - while i < s.len - 2: - var a = ord(s[i]) - var b = ord(s[i+1]) - var c = ord(s[i+2]) - result[r] = cb64[a shr 2] - result[r+1] = cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)] - result[r+2] = cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)] - result[r+3] = cb64[c and 0x3F] - inc(r, 4) - inc(i, 3) - inc(currLine, 4) - if currLine >= lineLen and i != s.len-2: - for x in items(newLine): - result[r] = x - inc(r) - currLine = 0 - - if i < s.len-1: - var a = ord(s[i]) - var b = ord(s[i+1]) - result[r] = cb64[a shr 2] - result[r+1] = cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)] - result[r+2] = cb64[((b and 0x0F) shl 2)] - result[r+3] = '=' - if r+4 != result.len: - setLen(result, r+4) - elif i < s.len: - var a = ord(s[i]) - result[r] = cb64[a shr 2] - result[r+1] = cb64[(a and 3) shl 4] - result[r+2] = '=' - result[r+3] = '=' - if r+4 != result.len: - setLen(result, r+4) + cb64 = cbBase('+', '/') + cb64safe = cbBase('-', '_') + +const + invalidChar = 255 + +template encodeSize(size: int): int = (size * 4 div 3) + 6 + +template encodeInternal(s, alphabet: typed): untyped = + ## encodes `s` into base64 representation. + + result.setLen(encodeSize(s.len)) + + let + padding = s.len mod 3 + inputEnds = s.len - padding + + var + inputIndex = 0 + outputIndex = 0 + n: uint32 + b: uint32 + + template inputByte(exp: untyped) = + b = uint32(s[inputIndex]) + n = exp + inc inputIndex + + template outputChar(x: typed) = + result[outputIndex] = alphabet[x and 63] + inc outputIndex + + template outputChar(c: char) = + result[outputIndex] = c + inc outputIndex + + while inputIndex != inputEnds: + inputByte(b shl 16) + inputByte(n or b shl 8) + inputByte(n or b shl 0) + outputChar(n shr 18) + outputChar(n shr 12) + outputChar(n shr 6) + outputChar(n shr 0) + + if padding == 1: + inputByte(b shl 16) + outputChar(n shr 18) + outputChar(n shr 12) + outputChar('=') + outputChar('=') + + elif padding == 2: + inputByte(b shl 16) + inputByte(n or b shl 8) + outputChar(n shr 18) + outputChar(n shr 12) + outputChar(n shr 6) + outputChar('=') + + result.setLen(outputIndex) + +template encodeImpl() {.dirty.} = + if safe: + encodeInternal(s, cb64safe) else: - #assert(r == result.len) - discard + encodeInternal(s, cb64) -proc encode*[T:SomeInteger|char](s: openarray[T], lineLen = 75, newLine="\13\10"): string = - ## encodes `s` into base64 representation. After `lineLen` characters, a - ## `newline` is added. +proc encode*[T: byte|char](s: openArray[T], safe = false): string = + ## Encodes `s` into base64 representation. + ## + ## If `safe` is `true` then it will encode using the + ## URL-Safe and Filesystem-safe standard alphabet characters, + ## which substitutes `-` instead of `+` and `_` instead of `/`. + ## * https://en.wikipedia.org/wiki/Base64#URL_applications + ## * https://tools.ietf.org/html/rfc4648#page-7 ## - ## This procedure encodes an openarray (array or sequence) of either integers - ## or characters. - encodeInternal(s, lineLen, newLine) + ## **See also:** + ## * `decode proc<#decode,string>`_ for decoding a string + runnableExamples: + assert encode("Hello World") == "SGVsbG8gV29ybGQ=" + assert encode(['n', 'i', 'm']) == "bmlt" + assert encode(@['n', 'i', 'm']) == "bmlt" + assert encode([1'u8, 2, 3, 4, 5]) == "AQIDBAU=" + encodeImpl() -proc encode*(s: string, lineLen = 75, newLine="\13\10"): string = - ## encodes `s` into base64 representation. After `lineLen` characters, a - ## `newline` is added. +proc encode*[T: SomeInteger and not byte](s: openArray[T], safe = false): string + {.deprecated: "use `byte` or `char` instead".} = + encodeImpl() + +proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n", + safe = false): string = + ## Encodes `s` into base64 representation as lines. + ## Used in email MIME format, use `lineLen` and `newline`. + ## + ## This procedure encodes a string according to MIME spec. + ## + ## If `safe` is `true` then it will encode using the + ## URL-Safe and Filesystem-safe standard alphabet characters, + ## which substitutes `-` instead of `+` and `_` instead of `/`. + ## * https://en.wikipedia.org/wiki/Base64#URL_applications + ## * https://tools.ietf.org/html/rfc4648#page-7 ## - ## This procedure encodes a string. - encodeInternal(s, lineLen, newLine) + ## **See also:** + ## * `encode proc<#encode,openArray[T]>`_ for encoding an openArray + ## * `decode proc<#decode,string>`_ for decoding a string + runnableExamples: + assert encodeMime("Hello World", 4, "\n") == "SGVs\nbG8g\nV29y\nbGQ=" + template cpy(l, src, idx) = + b = l + while i < b: + result[i] = src[idx] + inc i + inc idx + + if s.len == 0: return + let e = encode(s, safe) + if e.len <= lineLen or newLine.len == 0: + return e + result = newString(e.len + newLine.len * ((e.len div lineLen) - int(e.len mod lineLen == 0))) + var i, j, k, b: int + let nd = e.len - lineLen + while j < nd: + cpy(i + lineLen, e, j) + cpy(i + newLine.len, newLine, k) + k = 0 + cpy(result.len, e, j) + +proc initDecodeTable*(): array[256, char] = + # computes a decode table at compile time + for i in 0 ..< 256: + let ch = char(i) + var code = invalidChar + if ch >= 'A' and ch <= 'Z': code = i - 0x00000041 + if ch >= 'a' and ch <= 'z': code = i - 0x00000047 + if ch >= '0' and ch <= '9': code = i + 0x00000004 + if ch == '+' or ch == '-': code = 0x0000003E + if ch == '/' or ch == '_': code = 0x0000003F + result[i] = char(code) -proc decodeByte(b: char): int {.inline.} = - case b - of '+': result = ord('>') - of '0'..'9': result = ord(b) + 4 - of 'A'..'Z': result = ord(b) - ord('A') - of 'a'..'z': result = ord(b) - 71 - else: result = 63 +const + decodeTable = initDecodeTable() proc decode*(s: string): string = - ## decodes a string in base64 representation back into its original form. - ## Whitespace is skipped. - const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} - var total = ((len(s) + 3) div 4) * 3 - # total is an upper bound, as we will skip arbitrary whitespace: - result = newString(total) - - var i = 0 - var r = 0 - while true: - while s[i] in Whitespace: inc(i) - if i < s.len-3: - var a = s[i].decodeByte - var b = s[i+1].decodeByte - var c = s[i+2].decodeByte - var d = s[i+3].decodeByte - - result[r] = chr((a shl 2) and 0xff or ((b shr 4) and 0x03)) - result[r+1] = chr((b shl 4) and 0xff or ((c shr 2) and 0x0F)) - result[r+2] = chr((c shl 6) and 0xff or (d and 0x3F)) - inc(r, 3) - inc(i, 4) - else: break - assert i == s.len - # adjust the length: - if i > 0 and s[i-1] == '=': - dec(r) - if i > 1 and s[i-2] == '=': dec(r) - setLen(result, r) - -when isMainModule: - assert encode("leasure.") == "bGVhc3VyZS4=" - assert encode("easure.") == "ZWFzdXJlLg==" - assert encode("asure.") == "YXN1cmUu" - assert encode("sure.") == "c3VyZS4=" - - const longText = """Man is distinguished, not only by his reason, but by this - singular passion from other animals, which is a lust of the mind, - that by a perseverance of delight in the continued and indefatigable - generation of knowledge, exceeds the short vehemence of any carnal - pleasure.""" - const tests = ["", "abc", "xyz", "man", "leasure.", "sure.", "easure.", - "asure.", longText] - for t in items(tests): - assert decode(encode(t)) == t + ## Decodes string `s` in base64 representation back into its original form. + ## The initial whitespace is skipped. + ## + ## **See also:** + ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray + runnableExamples: + assert decode("SGVsbG8gV29ybGQ=") == "Hello World" + assert decode(" SGVsbG8gV29ybGQ=") == "Hello World" + if s.len == 0: return + + proc decodeSize(size: int): int = + return (size * 3 div 4) + 6 + + template inputChar(x: untyped) = + let x = int decodeTable[ord(s[inputIndex])] + if x == invalidChar: + raise newException(ValueError, + "Invalid base64 format character `" & s[inputIndex] & + "` (ord " & $s[inputIndex].ord & ") at location " & $inputIndex & ".") + inc inputIndex + + template outputChar(x: untyped) = + result[outputIndex] = char(x and 255) + inc outputIndex + # pre allocate output string once + result.setLen(decodeSize(s.len)) + var + inputIndex = 0 + outputIndex = 0 + inputLen = s.len + inputEnds = 0 + # strip trailing characters + while inputLen > 0 and s[inputLen - 1] in {'\n', '\r', ' ', '='}: + dec inputLen + # hot loop: read 4 characters at at time + inputEnds = inputLen - 4 + while inputIndex <= inputEnds: + while s[inputIndex] in {'\n', '\r', ' '}: + inc inputIndex + inputChar(a) + inputChar(b) + inputChar(c) + inputChar(d) + outputChar(a shl 2 or b shr 4) + outputChar(b shl 4 or c shr 2) + outputChar(c shl 6 or d shr 0) + # do the last 2 or 3 characters + var leftLen = abs((inputIndex - inputLen) mod 4) + if leftLen == 2: + inputChar(a) + inputChar(b) + outputChar(a shl 2 or b shr 4) + elif leftLen == 3: + inputChar(a) + inputChar(b) + inputChar(c) + outputChar(a shl 2 or b shr 4) + outputChar(b shl 4 or c shr 2) + result.setLen(outputIndex) diff --git a/lib/pure/basic2d.nim b/lib/pure/basic2d.nim deleted file mode 100644 index 7d74424fa..000000000 --- a/lib/pure/basic2d.nim +++ /dev/null @@ -1,857 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Robert Persson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import math -import strutils - - -## Basic 2d support with vectors, points, matrices and some basic utilities. -## Vectors are implemented as direction vectors, ie. when transformed with a matrix -## the translation part of matrix is ignored. -## Operators `+` , `-` , `*` , `/` , `+=` , `-=` , `*=` and `/=` are implemented for vectors and scalars. -## -## Quick start example: -## -## .. code-block:: nim -## -## # Create a matrix which first rotates, then scales and at last translates -## -## var m:Matrix2d=rotate(DEG90) & scale(2.0) & move(100.0,200.0) -## -## # Create a 2d point at (100,0) and a vector (5,2) -## -## var pt:Point2d=point2d(100.0,0.0) -## -## var vec:Vector2d=vector2d(5.0,2.0) -## -## -## pt &= m # transforms pt in place -## -## var pt2:Point2d=pt & m #concatenates pt with m and returns a new point -## -## var vec2:Vector2d=vec & m #concatenates vec with m and returns a new vector - - -const - DEG360* = PI * 2.0 - ## 360 degrees in radians. - DEG270* = PI * 1.5 - ## 270 degrees in radians. - DEG180* = PI - ## 180 degrees in radians. - DEG90* = PI / 2.0 - ## 90 degrees in radians. - DEG60* = PI / 3.0 - ## 60 degrees in radians. - DEG45* = PI / 4.0 - ## 45 degrees in radians. - DEG30* = PI / 6.0 - ## 30 degrees in radians. - DEG15* = PI / 12.0 - ## 15 degrees in radians. - RAD2DEGCONST = 180.0 / PI - ## used internally by DegToRad and RadToDeg - -type - Matrix2d* = object - ## Implements a row major 2d matrix, which means - ## transformations are applied the order they are concatenated. - ## The rightmost column of the 3x3 matrix is left out since normally - ## not used for geometric transformations in 2d. - ax*,ay*,bx*,by*,tx*,ty*:float - Point2d* = object - ## Implements a non-homogeneous 2d point stored as - ## an `x` coordinate and an `y` coordinate. - x*,y*:float - Vector2d* = object - ## Implements a 2d **direction vector** stored as - ## an `x` coordinate and an `y` coordinate. Direction vector means, - ## that when transforming a vector with a matrix, the translational - ## part of the matrix is ignored. - x*,y*:float -{.deprecated: [TMatrix2d: Matrix2d, TPoint2d: Point2d, TVector2d: Vector2d].} - - -# Some forward declarations... -proc matrix2d*(ax,ay,bx,by,tx,ty:float):Matrix2d {.noInit.} - ## Creates a new matrix. - ## `ax`,`ay` is the local x axis - ## `bx`,`by` is the local y axis - ## `tx`,`ty` is the translation -proc vector2d*(x,y:float):Vector2d {.noInit,inline.} - ## Returns a new vector (`x`,`y`) -proc point2d*(x,y:float):Point2d {.noInit,inline.} - ## Returns a new point (`x`,`y`) - - - -let - IDMATRIX*:Matrix2d=matrix2d(1.0,0.0,0.0,1.0,0.0,0.0) - ## Quick access to an identity matrix - ORIGO*:Point2d=point2d(0.0,0.0) - ## Quick access to point (0,0) - XAXIS*:Vector2d=vector2d(1.0,0.0) - ## Quick access to an 2d x-axis unit vector - YAXIS*:Vector2d=vector2d(0.0,1.0) - ## Quick access to an 2d y-axis unit vector - - -# *************************************** -# Private utils -# *************************************** - -proc rtos(val:float):string= - return formatFloat(val,ffDefault,0) - -proc safeArccos(v:float):float= - ## assumes v is in range 0.0-1.0, but clamps - ## the value to avoid out of domain errors - ## due to rounding issues - return arccos(clamp(v,-1.0,1.0)) - - -template makeBinOpVector(s:expr)= - ## implements binary operators + , - , * and / for vectors - proc s*(a,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a.x,b.x),s(a.y,b.y)) - proc s*(a:Vector2d,b:float):Vector2d {.inline,noInit.} = vector2d(s(a.x,b),s(a.y,b)) - proc s*(a:float,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a,b.x),s(a,b.y)) - -template makeBinOpAssignVector(s:expr)= - ## implements inplace binary operators += , -= , /= and *= for vectors - proc s*(a:var Vector2d,b:Vector2d) {.inline.} = s(a.x,b.x) ; s(a.y,b.y) - proc s*(a:var Vector2d,b:float) {.inline.} = s(a.x,b) ; s(a.y,b) - - -# *************************************** -# Matrix2d implementation -# *************************************** - -proc setElements*(t:var Matrix2d,ax,ay,bx,by,tx,ty:float) {.inline.}= - ## Sets arbitrary elements in an existing matrix. - t.ax=ax - t.ay=ay - t.bx=bx - t.by=by - t.tx=tx - t.ty=ty - -proc matrix2d*(ax,ay,bx,by,tx,ty:float):Matrix2d = - result.setElements(ax,ay,bx,by,tx,ty) - -proc `&`*(a,b:Matrix2d):Matrix2d {.noInit.} = #concatenate matrices - ## Concatenates matrices returning a new matrix. - - # | a.AX a.AY 0 | | b.AX b.AY 0 | - # | a.BX a.BY 0 | * | b.BX b.BY 0 | - # | a.TX a.TY 1 | | b.TX b.TY 1 | - result.setElements( - a.ax * b.ax + a.ay * b.bx, - a.ax * b.ay + a.ay * b.by, - a.bx * b.ax + a.by * b.bx, - a.bx * b.ay + a.by * b.by, - a.tx * b.ax + a.ty * b.bx + b.tx, - a.tx * b.ay + a.ty * b.by + b.ty) - - -proc scale*(s:float):Matrix2d {.noInit.} = - ## Returns a new scale matrix. - result.setElements(s,0,0,s,0,0) - -proc scale*(s:float,org:Point2d):Matrix2d {.noInit.} = - ## Returns a new scale matrix using, `org` as scale origin. - result.setElements(s,0,0,s,org.x-s*org.x,org.y-s*org.y) - -proc stretch*(sx,sy:float):Matrix2d {.noInit.} = - ## Returns new a stretch matrix, which is a - ## scale matrix with non uniform scale in x and y. - result.setElements(sx,0,0,sy,0,0) - -proc stretch*(sx,sy:float,org:Point2d):Matrix2d {.noInit.} = - ## Returns a new stretch matrix, which is a - ## scale matrix with non uniform scale in x and y. - ## `org` is used as stretch origin. - result.setElements(sx,0,0,sy,org.x-sx*org.x,org.y-sy*org.y) - -proc move*(dx,dy:float):Matrix2d {.noInit.} = - ## Returns a new translation matrix. - result.setElements(1,0,0,1,dx,dy) - -proc move*(v:Vector2d):Matrix2d {.noInit.} = - ## Returns a new translation matrix from a vector. - result.setElements(1,0,0,1,v.x,v.y) - -proc rotate*(rad:float):Matrix2d {.noInit.} = - ## Returns a new rotation matrix, which - ## represents a rotation by `rad` radians - let - s=sin(rad) - c=cos(rad) - result.setElements(c,s,-s,c,0,0) - -proc rotate*(rad:float,org:Point2d):Matrix2d {.noInit.} = - ## Returns a new rotation matrix, which - ## represents a rotation by `rad` radians around - ## the origin `org` - let - s=sin(rad) - c=cos(rad) - result.setElements(c,s,-s,c,org.x+s*org.y-c*org.x,org.y-c*org.y-s*org.x) - -proc mirror*(v:Vector2d):Matrix2d {.noInit.} = - ## Returns a new mirror matrix, mirroring - ## around the line that passes through origo and - ## has the direction of `v` - let - sqx=v.x*v.x - sqy=v.y*v.y - nd=1.0/(sqx+sqy) #used to normalize invector - xy2=v.x*v.y*2.0*nd - sqd=nd*(sqx-sqy) - - if nd==Inf or nd==NegInf: - return IDMATRIX #mirroring around a zero vector is arbitrary=>just use identity - - result.setElements( - sqd,xy2, - xy2,-sqd, - 0.0,0.0) - -proc mirror*(org:Point2d,v:Vector2d):Matrix2d {.noInit.} = - ## Returns a new mirror matrix, mirroring - ## around the line that passes through `org` and - ## has the direction of `v` - let - sqx=v.x*v.x - sqy=v.y*v.y - nd=1.0/(sqx+sqy) #used to normalize invector - xy2=v.x*v.y*2.0*nd - sqd=nd*(sqx-sqy) - - if nd==Inf or nd==NegInf: - return IDMATRIX #mirroring around a zero vector is arbitrary=>just use identity - - result.setElements( - sqd,xy2, - xy2,-sqd, - org.x-org.y*xy2-org.x*sqd,org.y-org.x*xy2+org.y*sqd) - - - -proc skew*(xskew,yskew:float):Matrix2d {.noInit.} = - ## Returns a new skew matrix, which has its - ## x axis rotated `xskew` radians from the local x axis, and - ## y axis rotated `yskew` radians from the local y axis - result.setElements(cos(yskew),sin(yskew),-sin(xskew),cos(xskew),0,0) - - -proc `$`* (t:Matrix2d):string {.noInit.} = - ## Returns a string representation of the matrix - return rtos(t.ax) & "," & rtos(t.ay) & - "," & rtos(t.bx) & "," & rtos(t.by) & - "," & rtos(t.tx) & "," & rtos(t.ty) - -proc isUniform*(t:Matrix2d,tol=1.0e-6):bool= - ## Checks if the transform is uniform, that is - ## perpendicular axes of equal length, which means (for example) - ## it cannot transform a circle into an ellipse. - ## `tol` is used as tolerance for both equal length comparison - ## and perp. comparison. - - #dot product=0 means perpendicular coord. system: - if abs(t.ax*t.bx+t.ay*t.by)<=tol: - #subtract squared lengths of axes to check if uniform scaling: - if abs((t.ax*t.ax+t.ay*t.ay)-(t.bx*t.bx+t.by*t.by))<=tol: - return true - return false - -proc determinant*(t:Matrix2d):float= - ## Computes the determinant of the matrix. - - #NOTE: equivalent with perp.dot product for two 2d vectors - return t.ax*t.by-t.bx*t.ay - -proc isMirroring* (m:Matrix2d):bool= - ## Checks if the `m` is a mirroring matrix, - ## which means it will reverse direction of a curve transformed with it - return m.determinant<0.0 - -proc inverse*(m:Matrix2d):Matrix2d {.noInit.} = - ## Returns a new matrix, which is the inverse of the matrix - ## If the matrix is not invertible (determinant=0), an EDivByZero - ## will be raised. - let d=m.determinant - if d==0.0: - raise newException(DivByZeroError,"Cannot invert a zero determinant matrix") - - result.setElements( - m.by/d,-m.ay/d, - -m.bx/d,m.ax/d, - (m.bx*m.ty-m.by*m.tx)/d, - (m.ay*m.tx-m.ax*m.ty)/d) - -proc equals*(m1:Matrix2d,m2:Matrix2d,tol=1.0e-6):bool= - ## Checks if all elements of `m1`and `m2` is equal within - ## a given tolerance `tol`. - return - abs(m1.ax-m2.ax)<=tol and - abs(m1.ay-m2.ay)<=tol and - abs(m1.bx-m2.bx)<=tol and - abs(m1.by-m2.by)<=tol and - abs(m1.tx-m2.tx)<=tol and - abs(m1.ty-m2.ty)<=tol - -proc `=~`*(m1,m2:Matrix2d):bool= - ## Checks if `m1`and `m2` is approximately equal, using a - ## tolerance of 1e-6. - equals(m1,m2) - -proc isIdentity*(m:Matrix2d,tol=1.0e-6):bool= - ## Checks is a matrix is approximately an identity matrix, - ## using `tol` as tolerance for each element. - return equals(m,IDMATRIX,tol) - -proc apply*(m:Matrix2d,x,y:var float,translate=false)= - ## Applies transformation `m` onto `x`,`y`, optionally - ## using the translation part of the matrix. - if translate: # positional style transform - let newx=x*m.ax+y*m.bx+m.tx - y=x*m.ay+y*m.by+m.ty - x=newx - else: # delta style transform - let newx=x*m.ax+y*m.bx - y=x*m.ay+y*m.by - x=newx - - - -# *************************************** -# Vector2d implementation -# *************************************** -proc vector2d*(x,y:float):Vector2d = #forward decl. - result.x=x - result.y=y - -proc polarVector2d*(ang:float,len:float):Vector2d {.noInit.} = - ## Returns a new vector with angle `ang` and magnitude `len` - result.x=cos(ang)*len - result.y=sin(ang)*len - -proc slopeVector2d*(slope:float,len:float):Vector2d {.noInit.} = - ## Returns a new vector having slope (dy/dx) given by - ## `slope`, and a magnitude of `len` - let ang=arctan(slope) - result.x=cos(ang)*len - result.y=sin(ang)*len - -proc len*(v:Vector2d):float {.inline.}= - ## Returns the length of the vector. - sqrt(v.x*v.x+v.y*v.y) - -proc `len=`*(v:var Vector2d,newlen:float) {.noInit.} = - ## Sets the length of the vector, keeping its angle. - let fac=newlen/v.len - - if newlen==0.0: - v.x=0.0 - v.y=0.0 - return - - if fac==Inf or fac==NegInf: - #to short for float accuracy - #do as good as possible: - v.x=newlen - v.y=0.0 - else: - v.x*=fac - v.y*=fac - -proc sqrLen*(v:Vector2d):float {.inline.}= - ## Computes the squared length of the vector, which is - ## faster than computing the absolute length. - v.x*v.x+v.y*v.y - -proc angle*(v:Vector2d):float= - ## Returns the angle of the vector. - ## (The counter clockwise plane angle between posetive x axis and `v`) - result=arctan2(v.y,v.x) - if result<0.0: result+=DEG360 - -proc `$` *(v:Vector2d):string= - ## String representation of `v` - result=rtos(v.x) - result.add(",") - result.add(rtos(v.y)) - - -proc `&` *(v:Vector2d,m:Matrix2d):Vector2d {.noInit.} = - ## Concatenate vector `v` with a transformation matrix. - ## Transforming a vector ignores the translational part - ## of the matrix. - - # | AX AY 0 | - # | X Y 1 | * | BX BY 0 | - # | 0 0 1 | - result.x=v.x*m.ax+v.y*m.bx - result.y=v.x*m.ay+v.y*m.by - - -proc `&=`*(v:var Vector2d,m:Matrix2d) {.inline.}= - ## Applies transformation `m` onto `v` in place. - ## Transforming a vector ignores the translational part - ## of the matrix. - - # | AX AY 0 | - # | X Y 1 | * | BX BY 0 | - # | 0 0 1 | - let newx=v.x*m.ax+v.y*m.bx - v.y=v.x*m.ay+v.y*m.by - v.x=newx - - -proc tryNormalize*(v:var Vector2d):bool= - ## Modifies `v` to have a length of 1.0, keeping its angle. - ## If `v` has zero length (and thus no angle), it is left unmodified and - ## false is returned, otherwise true is returned. - - let mag=v.len - - if mag==0.0: - return false - - v.x/=mag - v.y/=mag - return true - - -proc normalize*(v:var Vector2d) {.inline.}= - ## Modifies `v` to have a length of 1.0, keeping its angle. - ## If `v` has zero length, an EDivByZero will be raised. - if not tryNormalize(v): - raise newException(DivByZeroError,"Cannot normalize zero length vector") - -proc transformNorm*(v:var Vector2d,t:Matrix2d)= - ## Applies a normal direction transformation `t` onto `v` in place. - ## The resulting vector is *not* normalized. Transforming a vector ignores the - ## translational part of the matrix. If the matrix is not invertible - ## (determinant=0), an EDivByZero will be raised. - - # transforming a normal is done by transforming - # by the transpose of the inverse of the original matrix - # this can be heavily optimized by precompute and inline - # | | AX AY 0 | ^-1| ^T - # | X Y 1 | * | | BX BY 0 | | - # | | 0 0 1 | | - let d=t.determinant - if(d==0.0): - raise newException(DivByZeroError,"Matrix is not invertible") - let newx = (t.by*v.x-t.ay*v.y)/d - v.y = (t.ax*v.y-t.bx*v.x)/d - v.x = newx - -proc transformInv*(v:var Vector2d,t:Matrix2d)= - ## Applies inverse of a transformation `t` to `v` in place. - ## This is faster than creating an inverse matrix and apply() it. - ## Transforming a vector ignores the translational part - ## of the matrix. If the matrix is not invertible (determinant=0), an EDivByZero - ## will be raised. - let d=t.determinant - - if(d==0.0): - raise newException(DivByZeroError,"Matrix is not invertible") - - let newx=(t.by*v.x-t.bx*v.y)/d - v.y = (t.ax*v.y-t.ay*v.x)/d - v.x = newx - -proc transformNormInv*(v:var Vector2d,t:Matrix2d)= - ## Applies an inverse normal direction transformation `t` onto `v` in place. - ## This is faster than creating an inverse - ## matrix and transformNorm(...) it. Transforming a vector ignores the - ## translational part of the matrix. - - # normal inverse transform is done by transforming - # by the inverse of the transpose of the inverse of the org. matrix - # which is equivalent with transforming with the transpose. - # | | | AX AY 0 |^-1|^T|^-1 | AX BX 0 | - # | X Y 1 | * | | | BX BY 0 | | | = | X Y 1 | * | AY BY 0 | - # | | | 0 0 1 | | | | 0 0 1 | - # This can be heavily reduced to: - let newx=t.ay*v.y+t.ax*v.x - v.y=t.by*v.y+t.bx*v.x - v.x=newx - -proc rotate90*(v:var Vector2d) {.inline.}= - ## Quickly rotates vector `v` 90 degrees counter clockwise, - ## without using any trigonometrics. - swap(v.x,v.y) - v.x= -v.x - -proc rotate180*(v:var Vector2d){.inline.}= - ## Quickly rotates vector `v` 180 degrees counter clockwise, - ## without using any trigonometrics. - v.x= -v.x - v.y= -v.y - -proc rotate270*(v:var Vector2d) {.inline.}= - ## Quickly rotates vector `v` 270 degrees counter clockwise, - ## without using any trigonometrics. - swap(v.x,v.y) - v.y= -v.y - -proc rotate*(v:var Vector2d,rad:float) = - ## Rotates vector `v` `rad` radians in place. - let - s=sin(rad) - c=cos(rad) - newx=c*v.x-s*v.y - v.y=c*v.y+s*v.x - v.x=newx - -proc scale*(v:var Vector2d,fac:float){.inline.}= - ## Scales vector `v` `rad` radians in place. - v.x*=fac - v.y*=fac - -proc stretch*(v:var Vector2d,facx,facy:float){.inline.}= - ## Stretches vector `v` `facx` times horizontally, - ## and `facy` times vertically. - v.x*=facx - v.y*=facy - -proc mirror*(v:var Vector2d,mirrvec:Vector2d)= - ## Mirrors vector `v` using `mirrvec` as mirror direction. - let - sqx=mirrvec.x*mirrvec.x - sqy=mirrvec.y*mirrvec.y - nd=1.0/(sqx+sqy) #used to normalize invector - xy2=mirrvec.x*mirrvec.y*2.0*nd - sqd=nd*(sqx-sqy) - - if nd==Inf or nd==NegInf: - return #mirroring around a zero vector is arbitrary=>keep as is is fastest - - let newx=xy2*v.y+sqd*v.x - v.y=v.x*xy2-sqd*v.y - v.x=newx - - -proc `-` *(v:Vector2d):Vector2d= - ## Negates a vector - result.x= -v.x - result.y= -v.y - -# declare templated binary operators -makeBinOpVector(`+`) -makeBinOpVector(`-`) -makeBinOpVector(`*`) -makeBinOpVector(`/`) -makeBinOpAssignVector(`+=`) -makeBinOpAssignVector(`-=`) -makeBinOpAssignVector(`*=`) -makeBinOpAssignVector(`/=`) - - -proc dot*(v1,v2:Vector2d):float= - ## Computes the dot product of two vectors. - ## Returns 0.0 if the vectors are perpendicular. - return v1.x*v2.x+v1.y*v2.y - -proc cross*(v1,v2:Vector2d):float= - ## Computes the cross product of two vectors, also called - ## the 'perpendicular dot product' in 2d. Returns 0.0 if the vectors - ## are parallel. - return v1.x*v2.y-v1.y*v2.x - -proc equals*(v1,v2:Vector2d,tol=1.0e-6):bool= - ## Checks if two vectors approximately equals with a tolerance. - return abs(v2.x-v1.x)<=tol and abs(v2.y-v1.y)<=tol - -proc `=~` *(v1,v2:Vector2d):bool= - ## Checks if two vectors approximately equals with a - ## hardcoded tolerance 1e-6 - equals(v1,v2) - -proc angleTo*(v1,v2:Vector2d):float= - ## Returns the smallest of the two possible angles - ## between `v1` and `v2` in radians. - var - nv1=v1 - nv2=v2 - if not nv1.tryNormalize or not nv2.tryNormalize: - return 0.0 # zero length vector has zero angle to any other vector - return safeArccos(dot(nv1,nv2)) - -proc angleCCW*(v1,v2:Vector2d):float= - ## Returns the counter clockwise plane angle from `v1` to `v2`, - ## in range 0 - 2*PI - let a=v1.angleTo(v2) - if v1.cross(v2)>=0.0: - return a - return DEG360-a - -proc angleCW*(v1,v2:Vector2d):float= - ## Returns the clockwise plane angle from `v1` to `v2`, - ## in range 0 - 2*PI - let a=v1.angleTo(v2) - if v1.cross(v2)<=0.0: - return a - return DEG360-a - -proc turnAngle*(v1,v2:Vector2d):float= - ## Returns the amount v1 should be rotated (in radians) to equal v2, - ## in range -PI to PI - let a=v1.angleTo(v2) - if v1.cross(v2)<=0.0: - return -a - return a - -proc bisect*(v1,v2:Vector2d):Vector2d {.noInit.}= - ## Computes the bisector between v1 and v2 as a normalized vector. - ## If one of the input vectors has zero length, a normalized version - ## of the other is returned. If both input vectors has zero length, - ## an arbitrary normalized vector is returned. - var - vmag1=v1.len - vmag2=v2.len - - # zero length vector equals arbitrary vector, just change to magnitude to one to - # avoid zero division - if vmag1==0.0: - if vmag2==0: #both are zero length return any normalized vector - return XAXIS - vmag1=1.0 - if vmag2==0.0: vmag2=1.0 - - let - x1=v1.x/vmag1 - y1=v1.y/vmag1 - x2=v2.x/vmag2 - y2=v2.y/vmag2 - - result.x=(x1 + x2) * 0.5 - result.y=(y1 + y2) * 0.5 - - if not result.tryNormalize(): - # This can happen if vectors are colinear. In this special case - # there are actually two bisectors, we select just - # one of them (x1,y1 rotated 90 degrees ccw). - result.x = -y1 - result.y = x1 - - - -# *************************************** -# Point2d implementation -# *************************************** - -proc point2d*(x,y:float):Point2d = - result.x=x - result.y=y - -proc sqrDist*(a,b:Point2d):float= - ## Computes the squared distance between `a` and `b` - let dx=b.x-a.x - let dy=b.y-a.y - result=dx*dx+dy*dy - -proc dist*(a,b:Point2d):float {.inline.}= - ## Computes the absolute distance between `a` and `b` - result=sqrt(sqrDist(a,b)) - -proc angle*(a,b:Point2d):float= - ## Computes the angle of the vector `b`-`a` - let dx=b.x-a.x - let dy=b.y-a.y - result=arctan2(dy,dx) - if result<0: - result += DEG360 - -proc `$` *(p:Point2d):string= - ## String representation of `p` - result=rtos(p.x) - result.add(",") - result.add(rtos(p.y)) - -proc `&`*(p:Point2d,t:Matrix2d):Point2d {.noInit,inline.} = - ## Concatenates a point `p` with a transform `t`, - ## resulting in a new, transformed point. - - # | AX AY 0 | - # | X Y 1 | * | BX BY 0 | - # | TX TY 1 | - result.x=p.x*t.ax+p.y*t.bx+t.tx - result.y=p.x*t.ay+p.y*t.by+t.ty - -proc `&=` *(p:var Point2d,t:Matrix2d) {.inline.}= - ## Applies transformation `t` onto `p` in place. - let newx=p.x*t.ax+p.y*t.bx+t.tx - p.y=p.x*t.ay+p.y*t.by+t.ty - p.x=newx - - -proc transformInv*(p:var Point2d,t:Matrix2d){.inline.}= - ## Applies the inverse of transformation `t` onto `p` in place. - ## If the matrix is not invertable (determinant=0) , EDivByZero will - ## be raised. - - # | AX AY 0 | ^-1 - # | X Y 1 | * | BX BY 0 | - # | TX TY 1 | - let d=t.determinant - if d==0.0: - raise newException(DivByZeroError,"Cannot invert a zero determinant matrix") - let - newx= (t.bx*t.ty-t.by*t.tx+p.x*t.by-p.y*t.bx)/d - p.y = -(t.ax*t.ty-t.ay*t.tx+p.x*t.ay-p.y*t.ax)/d - p.x=newx - - -proc `+`*(p:Point2d,v:Vector2d):Point2d {.noInit,inline.} = - ## Adds a vector `v` to a point `p`, resulting - ## in a new point. - result.x=p.x+v.x - result.y=p.y+v.y - -proc `+=`*(p:var Point2d,v:Vector2d) {.noInit,inline.} = - ## Adds a vector `v` to a point `p` in place. - p.x+=v.x - p.y+=v.y - -proc `-`*(p:Point2d,v:Vector2d):Point2d {.noInit,inline.} = - ## Subtracts a vector `v` from a point `p`, resulting - ## in a new point. - result.x=p.x-v.x - result.y=p.y-v.y - -proc `-`*(p1,p2:Point2d):Vector2d {.noInit,inline.} = - ## Subtracts `p2`from `p1` resulting in a difference vector. - result.x=p1.x-p2.x - result.y=p1.y-p2.y - -proc `-=`*(p:var Point2d,v:Vector2d) {.noInit,inline.} = - ## Subtracts a vector `v` from a point `p` in place. - p.x-=v.x - p.y-=v.y - -proc equals(p1,p2:Point2d,tol=1.0e-6):bool {.inline.}= - ## Checks if two points approximately equals with a tolerance. - return abs(p2.x-p1.x)<=tol and abs(p2.y-p1.y)<=tol - -proc `=~`*(p1,p2:Point2d):bool {.inline.}= - ## Checks if two vectors approximately equals with a - ## hardcoded tolerance 1e-6 - equals(p1,p2) - -proc polar*(p:Point2d,ang,dist:float):Point2d {.noInit.} = - ## Returns a point with a given angle and distance away from `p` - result.x=p.x+cos(ang)*dist - result.y=p.y+sin(ang)*dist - -proc rotate*(p:var Point2d,rad:float)= - ## Rotates a point in place `rad` radians around origo. - let - c=cos(rad) - s=sin(rad) - newx=p.x*c-p.y*s - p.y=p.y*c+p.x*s - p.x=newx - -proc rotate*(p:var Point2d,rad:float,org:Point2d)= - ## Rotates a point in place `rad` radians using `org` as - ## center of rotation. - let - c=cos(rad) - s=sin(rad) - newx=(p.x - org.x) * c - (p.y - org.y) * s + org.x - p.y=(p.y - org.y) * c + (p.x - org.x) * s + org.y - p.x=newx - -proc scale*(p:var Point2d,fac:float) {.inline.}= - ## Scales a point in place `fac` times with world origo as origin. - p.x*=fac - p.y*=fac - -proc scale*(p:var Point2d,fac:float,org:Point2d){.inline.}= - ## Scales the point in place `fac` times with `org` as origin. - p.x=(p.x - org.x) * fac + org.x - p.y=(p.y - org.y) * fac + org.y - -proc stretch*(p:var Point2d,facx,facy:float){.inline.}= - ## Scales a point in place non uniformly `facx` and `facy` times with - ## world origo as origin. - p.x*=facx - p.y*=facy - -proc stretch*(p:var Point2d,facx,facy:float,org:Point2d){.inline.}= - ## Scales the point in place non uniformly `facx` and `facy` times with - ## `org` as origin. - p.x=(p.x - org.x) * facx + org.x - p.y=(p.y - org.y) * facy + org.y - -proc move*(p:var Point2d,dx,dy:float){.inline.}= - ## Translates a point `dx`, `dy` in place. - p.x+=dx - p.y+=dy - -proc move*(p:var Point2d,v:Vector2d){.inline.}= - ## Translates a point with vector `v` in place. - p.x+=v.x - p.y+=v.y - -proc sgnArea*(a,b,c:Point2d):float= - ## Computes the signed area of the triangle thru points `a`,`b` and `c` - ## result>0.0 for counter clockwise triangle - ## result<0.0 for clockwise triangle - ## This is commonly used to determinate side of a point with respect to a line. - return ((b.x - c.x) * (b.y - a.y)-(b.y - c.y) * (b.x - a.x))*0.5 - -proc area*(a,b,c:Point2d):float= - ## Computes the area of the triangle thru points `a`,`b` and `c` - return abs(sgnArea(a,b,c)) - -proc closestPoint*(p:Point2d,pts:varargs[Point2d]):Point2d= - ## Returns a point selected from `pts`, that has the closest - ## euclidean distance to `p` - assert(pts.len>0) # must have at least one point - - var - bestidx=0 - bestdist=p.sqrDist(pts[0]) - curdist:float - - for idx in 1..high(pts): - curdist=p.sqrDist(pts[idx]) - if curdist<bestdist: - bestidx=idx - bestdist=curdist - - result=pts[bestidx] - - -# *************************************** -# Misc. math utilities that should -# probably be in another module. -# *************************************** -proc normAngle*(ang:float):float= - ## Returns an angle in radians, that is equal to `ang`, - ## but in the range 0 to <2*PI - if ang>=0.0 and ang<DEG360: - return ang - - return ang mod DEG360 - -proc degToRad*(deg:float):float {.inline.}= - ## converts `deg` degrees to radians - deg / RAD2DEGCONST - -proc radToDeg*(rad:float):float {.inline.}= - ## converts `rad` radians to degrees - rad * RAD2DEGCONST - - diff --git a/lib/pure/basic3d.nim b/lib/pure/basic3d.nim deleted file mode 100644 index 424c191f8..000000000 --- a/lib/pure/basic3d.nim +++ /dev/null @@ -1,1042 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Robert Persson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import math -import strutils -import times - - -## Basic 3d support with vectors, points, matrices and some basic utilities. -## Vectors are implemented as direction vectors, ie. when transformed with a matrix -## the translation part of matrix is ignored. The coordinate system used is -## right handed, because its compatible with 2d coordinate system (rotation around -## zaxis equals 2d rotation). -## Operators `+` , `-` , `*` , `/` , `+=` , `-=` , `*=` and `/=` are implemented -## for vectors and scalars. -## -## -## Quick start example: -## -## .. code-block:: nim -## -## # Create a matrix which first rotates, then scales and at last translates -## -## var m:Matrix3d=rotate(PI,vector3d(1,1,2.5)) & scale(2.0) & move(100.0,200.0,300.0) -## -## # Create a 3d point at (100,150,200) and a vector (5,2,3) -## -## var pt:Point3d=point3d(100.0,150.0,200.0) -## -## var vec:Vector3d=vector3d(5.0,2.0,3.0) -## -## -## pt &= m # transforms pt in place -## -## var pt2:Point3d=pt & m #concatenates pt with m and returns a new point -## -## var vec2:Vector3d=vec & m #concatenates vec with m and returns a new vector - - - -type - Matrix3d* =object - ## Implements a row major 3d matrix, which means - ## transformations are applied the order they are concatenated. - ## This matrix is stored as an 4x4 matrix: - ## [ ax ay az aw ] - ## [ bx by bz bw ] - ## [ cx cy cz cw ] - ## [ tx ty tz tw ] - ax*,ay*,az*,aw*, bx*,by*,bz*,bw*, cx*,cy*,cz*,cw*, tx*,ty*,tz*,tw*:float - Point3d* = object - ## Implements a non-homogeneous 3d point stored as - ## an `x` , `y` and `z` coordinate. - x*,y*,z*:float - Vector3d* = object - ## Implements a 3d **direction vector** stored as - ## an `x` , `y` and `z` coordinate. Direction vector means, - ## that when transforming a vector with a matrix, the translational - ## part of the matrix is ignored. - x*,y*,z*:float -{.deprecated: [TMatrix3d: Matrix3d, TPoint3d: Point3d, TVector3d: Vector3d].} - - -# Some forward declarations -proc matrix3d*(ax,ay,az,aw,bx,by,bz,bw,cx,cy,cz,cw,tx,ty,tz,tw:float):Matrix3d {.noInit.} - ## Creates a new 4x4 3d transformation matrix. - ## `ax` , `ay` , `az` is the local x axis. - ## `bx` , `by` , `bz` is the local y axis. - ## `cx` , `cy` , `cz` is the local z axis. - ## `tx` , `ty` , `tz` is the translation. -proc vector3d*(x,y,z:float):Vector3d {.noInit,inline.} - ## Returns a new 3d vector (`x`,`y`,`z`) -proc point3d*(x,y,z:float):Point3d {.noInit,inline.} - ## Returns a new 4d point (`x`,`y`,`z`) -proc tryNormalize*(v:var Vector3d):bool - ## Modifies `v` to have a length of 1.0, keeping its angle. - ## If `v` has zero length (and thus no angle), it is left unmodified and false is - ## returned, otherwise true is returned. - - - -let - IDMATRIX*:Matrix3d=matrix3d( - 1.0,0.0,0.0,0.0, - 0.0,1.0,0.0,0.0, - 0.0,0.0,1.0,0.0, - 0.0,0.0,0.0,1.0) - ## Quick access to a 3d identity matrix - ORIGO*:Point3d=point3d(0.0,0.0,0.0) - ## Quick access to point (0,0) - XAXIS*:Vector3d=vector3d(1.0,0.0,0.0) - ## Quick access to an 3d x-axis unit vector - YAXIS*:Vector3d=vector3d(0.0,1.0,0.0) - ## Quick access to an 3d y-axis unit vector - ZAXIS*:Vector3d=vector3d(0.0,0.0,1.0) - ## Quick access to an 3d z-axis unit vector - - - -# *************************************** -# Private utils -# *************************************** - -proc rtos(val:float):string= - return formatFloat(val,ffDefault,0) - -proc safeArccos(v:float):float= - ## assumes v is in range 0.0-1.0, but clamps - ## the value to avoid out of domain errors - ## due to rounding issues - return arccos(clamp(v,-1.0,1.0)) - -template makeBinOpVector(s:expr)= - ## implements binary operators + , - , * and / for vectors - proc s*(a,b:Vector3d):Vector3d {.inline,noInit.} = - vector3d(s(a.x,b.x),s(a.y,b.y),s(a.z,b.z)) - proc s*(a:Vector3d,b:float):Vector3d {.inline,noInit.} = - vector3d(s(a.x,b),s(a.y,b),s(a.z,b)) - proc s*(a:float,b:Vector3d):Vector3d {.inline,noInit.} = - vector3d(s(a,b.x),s(a,b.y),s(a,b.z)) - -template makeBinOpAssignVector(s:expr)= - ## implements inplace binary operators += , -= , /= and *= for vectors - proc s*(a:var Vector3d,b:Vector3d) {.inline.} = - s(a.x,b.x) ; s(a.y,b.y) ; s(a.z,b.z) - proc s*(a:var Vector3d,b:float) {.inline.} = - s(a.x,b) ; s(a.y,b) ; s(a.z,b) - - - -# *************************************** -# Matrix3d implementation -# *************************************** - -proc setElements*(t:var Matrix3d,ax,ay,az,aw,bx,by,bz,bw,cx,cy,cz,cw,tx,ty,tz,tw:float) {.inline.}= - ## Sets arbitrary elements in an exisitng matrix. - t.ax=ax - t.ay=ay - t.az=az - t.aw=aw - t.bx=bx - t.by=by - t.bz=bz - t.bw=bw - t.cx=cx - t.cy=cy - t.cz=cz - t.cw=cw - t.tx=tx - t.ty=ty - t.tz=tz - t.tw=tw - -proc matrix3d*(ax,ay,az,aw,bx,by,bz,bw,cx,cy,cz,cw,tx,ty,tz,tw:float):Matrix3d = - result.setElements(ax,ay,az,aw,bx,by,bz,bw,cx,cy,cz,cw,tx,ty,tz,tw) - -proc `&`*(a,b:Matrix3d):Matrix3d {.noinit.} = - ## Concatenates matrices returning a new matrix. - result.setElements( - a.aw*b.tx+a.az*b.cx+a.ay*b.bx+a.ax*b.ax, - a.aw*b.ty+a.az*b.cy+a.ay*b.by+a.ax*b.ay, - a.aw*b.tz+a.az*b.cz+a.ay*b.bz+a.ax*b.az, - a.aw*b.tw+a.az*b.cw+a.ay*b.bw+a.ax*b.aw, - - a.bw*b.tx+a.bz*b.cx+a.by*b.bx+a.bx*b.ax, - a.bw*b.ty+a.bz*b.cy+a.by*b.by+a.bx*b.ay, - a.bw*b.tz+a.bz*b.cz+a.by*b.bz+a.bx*b.az, - a.bw*b.tw+a.bz*b.cw+a.by*b.bw+a.bx*b.aw, - - a.cw*b.tx+a.cz*b.cx+a.cy*b.bx+a.cx*b.ax, - a.cw*b.ty+a.cz*b.cy+a.cy*b.by+a.cx*b.ay, - a.cw*b.tz+a.cz*b.cz+a.cy*b.bz+a.cx*b.az, - a.cw*b.tw+a.cz*b.cw+a.cy*b.bw+a.cx*b.aw, - - a.tw*b.tx+a.tz*b.cx+a.ty*b.bx+a.tx*b.ax, - a.tw*b.ty+a.tz*b.cy+a.ty*b.by+a.tx*b.ay, - a.tw*b.tz+a.tz*b.cz+a.ty*b.bz+a.tx*b.az, - a.tw*b.tw+a.tz*b.cw+a.ty*b.bw+a.tx*b.aw) - - -proc scale*(s:float):Matrix3d {.noInit.} = - ## Returns a new scaling matrix. - result.setElements(s,0,0,0, 0,s,0,0, 0,0,s,0, 0,0,0,1) - -proc scale*(s:float,org:Point3d):Matrix3d {.noInit.} = - ## Returns a new scaling matrix using, `org` as scale origin. - result.setElements(s,0,0,0, 0,s,0,0, 0,0,s,0, - org.x-s*org.x,org.y-s*org.y,org.z-s*org.z,1.0) - -proc stretch*(sx,sy,sz:float):Matrix3d {.noInit.} = - ## Returns new a stretch matrix, which is a - ## scale matrix with non uniform scale in x,y and z. - result.setElements(sx,0,0,0, 0,sy,0,0, 0,0,sz,0, 0,0,0,1) - -proc stretch*(sx,sy,sz:float,org:Point3d):Matrix3d {.noInit.} = - ## Returns a new stretch matrix, which is a - ## scale matrix with non uniform scale in x,y and z. - ## `org` is used as stretch origin. - result.setElements(sx,0,0,0, 0,sy,0,0, 0,0,sz,0, org.x-sx*org.x,org.y-sy*org.y,org.z-sz*org.z,1) - -proc move*(dx,dy,dz:float):Matrix3d {.noInit.} = - ## Returns a new translation matrix. - result.setElements(1,0,0,0, 0,1,0,0, 0,0,1,0, dx,dy,dz,1) - -proc move*(v:Vector3d):Matrix3d {.noInit.} = - ## Returns a new translation matrix from a vector. - result.setElements(1,0,0,0, 0,1,0,0, 0,0,1,0, v.x,v.y,v.z,1) - - -proc rotate*(angle:float,axis:Vector3d):Matrix3d {.noInit.}= - ## Creates a rotation matrix that rotates `angle` radians over - ## `axis`, which passes through origo. - - # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf - # for how this is computed - - var normax=axis - if not normax.tryNormalize: #simplifies matrix computation below a lot - raise newException(DivByZeroError,"Cannot rotate around zero length axis") - - let - cs=cos(angle) - si=sin(angle) - omc=1.0-cs - usi=normax.x*si - vsi=normax.y*si - wsi=normax.z*si - u2=normax.x*normax.x - v2=normax.y*normax.y - w2=normax.z*normax.z - uvomc=normax.x*normax.y*omc - uwomc=normax.x*normax.z*omc - vwomc=normax.y*normax.z*omc - - result.setElements( - u2+(1.0-u2)*cs, uvomc+wsi, uwomc-vsi, 0.0, - uvomc-wsi, v2+(1.0-v2)*cs, vwomc+usi, 0.0, - uwomc+vsi, vwomc-usi, w2+(1.0-w2)*cs, 0.0, - 0.0,0.0,0.0,1.0) - -proc rotate*(angle:float,org:Point3d,axis:Vector3d):Matrix3d {.noInit.}= - ## Creates a rotation matrix that rotates `angle` radians over - ## `axis`, which passes through `org`. - - # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf - # for how this is computed - - var normax=axis - if not normax.tryNormalize: #simplifies matrix computation below a lot - raise newException(DivByZeroError,"Cannot rotate around zero length axis") - - let - u=normax.x - v=normax.y - w=normax.z - u2=u*u - v2=v*v - w2=w*w - cs=cos(angle) - omc=1.0-cs - si=sin(angle) - a=org.x - b=org.y - c=org.z - usi=u*si - vsi=v*si - wsi=w*si - uvomc=normax.x*normax.y*omc - uwomc=normax.x*normax.z*omc - vwomc=normax.y*normax.z*omc - - result.setElements( - u2+(v2+w2)*cs, uvomc+wsi, uwomc-vsi, 0.0, - uvomc-wsi, v2+(u2+w2)*cs, vwomc+usi, 0.0, - uwomc+vsi, vwomc-usi, w2+(u2+v2)*cs, 0.0, - (a*(v2+w2)-u*(b*v+c*w))*omc+(b*w-c*v)*si, - (b*(u2+w2)-v*(a*u+c*w))*omc+(c*u-a*w)*si, - (c*(u2+v2)-w*(a*u+b*v))*omc+(a*v-b*u)*si,1.0) - - -proc rotateX*(angle:float):Matrix3d {.noInit.}= - ## Creates a matrix that rotates around the x-axis with `angle` radians, - ## which is also called a 'roll' matrix. - let - c=cos(angle) - s=sin(angle) - result.setElements( - 1,0,0,0, - 0,c,s,0, - 0,-s,c,0, - 0,0,0,1) - -proc rotateY*(angle:float):Matrix3d {.noInit.}= - ## Creates a matrix that rotates around the y-axis with `angle` radians, - ## which is also called a 'pitch' matrix. - let - c=cos(angle) - s=sin(angle) - result.setElements( - c,0,-s,0, - 0,1,0,0, - s,0,c,0, - 0,0,0,1) - -proc rotateZ*(angle:float):Matrix3d {.noInit.}= - ## Creates a matrix that rotates around the z-axis with `angle` radians, - ## which is also called a 'yaw' matrix. - let - c=cos(angle) - s=sin(angle) - result.setElements( - c,s,0,0, - -s,c,0,0, - 0,0,1,0, - 0,0,0,1) - -proc isUniform*(m:Matrix3d,tol=1.0e-6):bool= - ## Checks if the transform is uniform, that is - ## perpendicular axes of equal length, which means (for example) - ## it cannot transform a sphere into an ellipsoid. - ## `tol` is used as tolerance for both equal length comparison - ## and perpendicular comparison. - - #dot product=0 means perpendicular coord. system, check xaxis vs yaxis and xaxis vs zaxis - if abs(m.ax*m.bx+m.ay*m.by+m.az*m.bz)<=tol and # x vs y - abs(m.ax*m.cx+m.ay*m.cy+m.az*m.cz)<=tol and #x vs z - abs(m.bx*m.cx+m.by*m.cy+m.bz*m.cz)<=tol: #y vs z - - #subtract squared lengths of axes to check if uniform scaling: - let - sqxlen=(m.ax*m.ax+m.ay*m.ay+m.az*m.az) - sqylen=(m.bx*m.bx+m.by*m.by+m.bz*m.bz) - sqzlen=(m.cx*m.cx+m.cy*m.cy+m.cz*m.cz) - if abs(sqxlen-sqylen)<=tol and abs(sqxlen-sqzlen)<=tol: - return true - return false - - - -proc mirror*(planeperp:Vector3d):Matrix3d {.noInit.}= - ## Creates a matrix that mirrors over the plane that has `planeperp` as normal, - ## and passes through origo. `planeperp` does not need to be normalized. - - # https://en.wikipedia.org/wiki/Transformation_matrix - var n=planeperp - if not n.tryNormalize: - raise newException(DivByZeroError,"Cannot mirror over a plane with a zero length normal") - - let - a=n.x - b=n.y - c=n.z - ab=a*b - ac=a*c - bc=b*c - - result.setElements( - 1-2*a*a , -2*ab,-2*ac,0, - -2*ab , 1-2*b*b, -2*bc, 0, - -2*ac, -2*bc, 1-2*c*c,0, - 0,0,0,1) - - -proc mirror*(org:Point3d,planeperp:Vector3d):Matrix3d {.noInit.}= - ## Creates a matrix that mirrors over the plane that has `planeperp` as normal, - ## and passes through `org`. `planeperp` does not need to be normalized. - - # constructs a mirror M like the simpler mirror matrix constructor - # above but premultiplies with the inverse traslation of org - # and postmultiplies with the translation of org. - # With some fiddling this becomes reasonably simple: - var n=planeperp - if not n.tryNormalize: - raise newException(DivByZeroError,"Cannot mirror over a plane with a zero length normal") - - let - a=n.x - b=n.y - c=n.z - ab=a*b - ac=a*c - bc=b*c - aa=a*a - bb=b*b - cc=c*c - tx=org.x - ty=org.y - tz=org.z - - result.setElements( - 1-2*aa , -2*ab,-2*ac,0, - -2*ab , 1-2*bb, -2*bc, 0, - -2*ac, -2*bc, 1-2*cc,0, - 2*(ac*tz+ab*ty+aa*tx), - 2*(bc*tz+bb*ty+ab*tx), - 2*(cc*tz+bc*ty+ac*tx) ,1) - - -proc determinant*(m:Matrix3d):float= - ## Computes the determinant of matrix `m`. - - # This computation is gotten from ratsimp(optimize(determinant(m))) - # in maxima CAS - let - O1=m.cx*m.tw-m.cw*m.tx - O2=m.cy*m.tw-m.cw*m.ty - O3=m.cx*m.ty-m.cy*m.tx - O4=m.cz*m.tw-m.cw*m.tz - O5=m.cx*m.tz-m.cz*m.tx - O6=m.cy*m.tz-m.cz*m.ty - - return (O1*m.ay-O2*m.ax-O3*m.aw)*m.bz+ - (-O1*m.az+O4*m.ax+O5*m.aw)*m.by+ - (O2*m.az-O4*m.ay-O6*m.aw)*m.bx+ - (O3*m.az-O5*m.ay+O6*m.ax)*m.bw - - -proc inverse*(m:Matrix3d):Matrix3d {.noInit.}= - ## Computes the inverse of matrix `m`. If the matrix - ## determinant is zero, thus not invertible, a EDivByZero - ## will be raised. - - # this computation comes from optimize(invert(m)) in maxima CAS - - let - det=m.determinant - O2=m.cy*m.tw-m.cw*m.ty - O3=m.cz*m.tw-m.cw*m.tz - O4=m.cy*m.tz-m.cz*m.ty - O5=m.by*m.tw-m.bw*m.ty - O6=m.bz*m.tw-m.bw*m.tz - O7=m.by*m.tz-m.bz*m.ty - O8=m.by*m.cw-m.bw*m.cy - O9=m.bz*m.cw-m.bw*m.cz - O10=m.by*m.cz-m.bz*m.cy - O11=m.cx*m.tw-m.cw*m.tx - O12=m.cx*m.tz-m.cz*m.tx - O13=m.bx*m.tw-m.bw*m.tx - O14=m.bx*m.tz-m.bz*m.tx - O15=m.bx*m.cw-m.bw*m.cx - O16=m.bx*m.cz-m.bz*m.cx - O17=m.cx*m.ty-m.cy*m.tx - O18=m.bx*m.ty-m.by*m.tx - O19=m.bx*m.cy-m.by*m.cx - - if det==0.0: - raise newException(DivByZeroError,"Cannot normalize zero length vector") - - result.setElements( - (m.bw*O4+m.by*O3-m.bz*O2)/det , (-m.aw*O4-m.ay*O3+m.az*O2)/det, - (m.aw*O7+m.ay*O6-m.az*O5)/det , (-m.aw*O10-m.ay*O9+m.az*O8)/det, - (-m.bw*O12-m.bx*O3+m.bz*O11)/det , (m.aw*O12+m.ax*O3-m.az*O11)/det, - (-m.aw*O14-m.ax*O6+m.az*O13)/det , (m.aw*O16+m.ax*O9-m.az*O15)/det, - (m.bw*O17+m.bx*O2-m.by*O11)/det , (-m.aw*O17-m.ax*O2+m.ay*O11)/det, - (m.aw*O18+m.ax*O5-m.ay*O13)/det , (-m.aw*O19-m.ax*O8+m.ay*O15)/det, - (-m.bx*O4+m.by*O12-m.bz*O17)/det , (m.ax*O4-m.ay*O12+m.az*O17)/det, - (-m.ax*O7+m.ay*O14-m.az*O18)/det , (m.ax*O10-m.ay*O16+m.az*O19)/det) - - -proc equals*(m1:Matrix3d,m2:Matrix3d,tol=1.0e-6):bool= - ## Checks if all elements of `m1`and `m2` is equal within - ## a given tolerance `tol`. - return - abs(m1.ax-m2.ax)<=tol and - abs(m1.ay-m2.ay)<=tol and - abs(m1.az-m2.az)<=tol and - abs(m1.aw-m2.aw)<=tol and - abs(m1.bx-m2.bx)<=tol and - abs(m1.by-m2.by)<=tol and - abs(m1.bz-m2.bz)<=tol and - abs(m1.bw-m2.bw)<=tol and - abs(m1.cx-m2.cx)<=tol and - abs(m1.cy-m2.cy)<=tol and - abs(m1.cz-m2.cz)<=tol and - abs(m1.cw-m2.cw)<=tol and - abs(m1.tx-m2.tx)<=tol and - abs(m1.ty-m2.ty)<=tol and - abs(m1.tz-m2.tz)<=tol and - abs(m1.tw-m2.tw)<=tol - -proc `=~`*(m1,m2:Matrix3d):bool= - ## Checks if `m1` and `m2` is approximately equal, using a - ## tolerance of 1e-6. - equals(m1,m2) - -proc transpose*(m:Matrix3d):Matrix3d {.noInit.}= - ## Returns the transpose of `m` - result.setElements(m.ax,m.bx,m.cx,m.tx,m.ay,m.by,m.cy,m.ty,m.az,m.bz,m.cz,m.tz,m.aw,m.bw,m.cw,m.tw) - -proc getXAxis*(m:Matrix3d):Vector3d {.noInit.}= - ## Gets the local x axis of `m` - result.x=m.ax - result.y=m.ay - result.z=m.az - -proc getYAxis*(m:Matrix3d):Vector3d {.noInit.}= - ## Gets the local y axis of `m` - result.x=m.bx - result.y=m.by - result.z=m.bz - -proc getZAxis*(m:Matrix3d):Vector3d {.noInit.}= - ## Gets the local y axis of `m` - result.x=m.cx - result.y=m.cy - result.z=m.cz - - -proc `$`*(m:Matrix3d):string= - ## String representation of `m` - return rtos(m.ax) & "," & rtos(m.ay) & "," & rtos(m.az) & "," & rtos(m.aw) & - "\n" & rtos(m.bx) & "," & rtos(m.by) & "," & rtos(m.bz) & "," & rtos(m.bw) & - "\n" & rtos(m.cx) & "," & rtos(m.cy) & "," & rtos(m.cz) & "," & rtos(m.cw) & - "\n" & rtos(m.tx) & "," & rtos(m.ty) & "," & rtos(m.tz) & "," & rtos(m.tw) - -proc apply*(m:Matrix3d, x,y,z:var float, translate=false)= - ## Applies transformation `m` onto `x` , `y` , `z` , optionally - ## using the translation part of the matrix. - let - oldx=x - oldy=y - oldz=z - - x=m.cx*oldz+m.bx*oldy+m.ax*oldx - y=m.cy*oldz+m.by*oldy+m.ay*oldx - z=m.cz*oldz+m.bz*oldy+m.az*oldx - - if translate: - x+=m.tx - y+=m.ty - z+=m.tz - -# *************************************** -# Vector3d implementation -# *************************************** -proc vector3d*(x,y,z:float):Vector3d= - result.x=x - result.y=y - result.z=z - -proc len*(v:Vector3d):float= - ## Returns the length of the vector `v`. - sqrt(v.x*v.x+v.y*v.y+v.z*v.z) - -proc `len=`*(v:var Vector3d,newlen:float) {.noInit.} = - ## Sets the length of the vector, keeping its direction. - ## If the vector has zero length before changing it's length, - ## an arbitrary vector of the requested length is returned. - - let fac=newlen/v.len - - if newlen==0.0: - v.x=0.0 - v.y=0.0 - v.z=0.0 - return - - if fac==Inf or fac==NegInf: - #to short for float accuracy - #do as good as possible: - v.x=newlen - v.y=0.0 - v.z=0.0 - else: - v.x*=fac - v.y*=fac - v.z*=fac - - -proc sqrLen*(v:Vector3d):float {.inline.}= - ## Computes the squared length of the vector, which is - ## faster than computing the absolute length. - return v.x*v.x+v.y*v.y+v.z*v.z - -proc `$` *(v:Vector3d):string= - ## String representation of `v` - result=rtos(v.x) - result.add(",") - result.add(rtos(v.y)) - result.add(",") - result.add(rtos(v.z)) - -proc `&` *(v:Vector3d,m:Matrix3d):Vector3d {.noInit.} = - ## Concatenate vector `v` with a transformation matrix. - ## Transforming a vector ignores the translational part - ## of the matrix. - - # | AX AY AZ AW | - # | X Y Z 1 | * | BX BY BZ BW | - # | CX CY CZ CW | - # | 0 0 0 1 | - let - newx=m.cx*v.z+m.bx*v.y+m.ax*v.x - newy=m.cy*v.z+m.by*v.y+m.ay*v.x - result.z=m.cz*v.z+m.bz*v.y+m.az*v.x - result.y=newy - result.x=newx - - -proc `&=` *(v:var Vector3d,m:Matrix3d) {.noInit.} = - ## Applies transformation `m` onto `v` in place. - ## Transforming a vector ignores the translational part - ## of the matrix. - - # | AX AY AZ AW | - # | X Y Z 1 | * | BX BY BZ BW | - # | CX CY CZ CW | - # | 0 0 0 1 | - - let - newx=m.cx*v.z+m.bx*v.y+m.ax*v.x - newy=m.cy*v.z+m.by*v.y+m.ay*v.x - v.z=m.cz*v.z+m.bz*v.y+m.az*v.x - v.y=newy - v.x=newx - -proc transformNorm*(v:var Vector3d,m:Matrix3d)= - ## Applies a normal direction transformation `m` onto `v` in place. - ## The resulting vector is *not* normalized. Transforming a vector ignores the - ## translational part of the matrix. If the matrix is not invertible - ## (determinant=0), an EDivByZero will be raised. - - # transforming a normal is done by transforming - # by the transpose of the inverse of the original matrix - - # Major reason this simple function is here is that this function can be optimized in the future, - # (possibly by hardware) as well as having a consistent API with the 2d version. - v&=transpose(inverse(m)) - -proc transformInv*(v:var Vector3d,m:Matrix3d)= - ## Applies the inverse of `m` on vector `v`. Transforming a vector ignores - ## the translational part of the matrix. Transforming a vector ignores the - ## translational part of the matrix. - ## If the matrix is not invertible (determinant=0), an EDivByZero - ## will be raised. - - # Major reason this simple function is here is that this function can be optimized in the future, - # (possibly by hardware) as well as having a consistent API with the 2d version. - v&=m.inverse - -proc transformNormInv*(vec:var Vector3d,m:Matrix3d)= - ## Applies an inverse normal direction transformation `m` onto `v` in place. - ## This is faster than creating an inverse - ## matrix and transformNorm(...) it. Transforming a vector ignores the - ## translational part of the matrix. - - # see vector2d:s equivalent for a deeper look how/why this works - vec&=m.transpose - -proc tryNormalize*(v:var Vector3d):bool= - ## Modifies `v` to have a length of 1.0, keeping its angle. - ## If `v` has zero length (and thus no angle), it is left unmodified and false is - ## returned, otherwise true is returned. - let mag=v.len - - if mag==0.0: - return false - - v.x/=mag - v.y/=mag - v.z/=mag - - return true - -proc normalize*(v:var Vector3d) {.inline.}= - ## Modifies `v` to have a length of 1.0, keeping its angle. - ## If `v` has zero length, an EDivByZero will be raised. - if not tryNormalize(v): - raise newException(DivByZeroError,"Cannot normalize zero length vector") - -proc rotate*(vec:var Vector3d,angle:float,axis:Vector3d)= - ## Rotates `vec` in place, with `angle` radians over `axis`, which passes - ## through origo. - - # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf - # for how this is computed - - var normax=axis - if not normax.tryNormalize: - raise newException(DivByZeroError,"Cannot rotate around zero length axis") - - let - cs=cos(angle) - si=sin(angle) - omc=1.0-cs - u=normax.x - v=normax.y - w=normax.z - x=vec.x - y=vec.y - z=vec.z - uxyzomc=(u*x+v*y+w*z)*omc - - vec.x=u*uxyzomc+x*cs+(v*z-w*y)*si - vec.y=v*uxyzomc+y*cs+(w*x-u*z)*si - vec.z=w*uxyzomc+z*cs+(u*y-v*x)*si - -proc scale*(v:var Vector3d,s:float)= - ## Scales the vector in place with factor `s` - v.x*=s - v.y*=s - v.z*=s - -proc stretch*(v:var Vector3d,sx,sy,sz:float)= - ## Scales the vector non uniformly with factors `sx` , `sy` , `sz` - v.x*=sx - v.y*=sy - v.z*=sz - -proc mirror*(v:var Vector3d,planeperp:Vector3d)= - ## Computes the mirrored vector of `v` over the plane - ## that has `planeperp` as normal direction. - ## `planeperp` does not need to be normalized. - - var n=planeperp - n.normalize - - let - x=v.x - y=v.y - z=v.z - a=n.x - b=n.y - c=n.z - ac=a*c - ab=a*b - bc=b*c - - v.x= -2*(ac*z+ab*y+a*a*x)+x - v.y= -2*(bc*z+b*b*y+ab*x)+y - v.z= -2*(c*c*z+bc*y+ac*x)+z - - -proc `-` *(v:Vector3d):Vector3d= - ## Negates a vector - result.x= -v.x - result.y= -v.y - result.z= -v.z - -# declare templated binary operators -makeBinOpVector(`+`) -makeBinOpVector(`-`) -makeBinOpVector(`*`) -makeBinOpVector(`/`) -makeBinOpAssignVector(`+=`) -makeBinOpAssignVector(`-=`) -makeBinOpAssignVector(`*=`) -makeBinOpAssignVector(`/=`) - -proc dot*(v1,v2:Vector3d):float {.inline.}= - ## Computes the dot product of two vectors. - ## Returns 0.0 if the vectors are perpendicular. - return v1.x*v2.x+v1.y*v2.y+v1.z*v2.z - -proc cross*(v1,v2:Vector3d):Vector3d {.inline.}= - ## Computes the cross product of two vectors. - ## The result is a vector which is perpendicular - ## to the plane of `v1` and `v2`, which means - ## cross(xaxis,yaxis)=zaxis. The magnitude of the result is - ## zero if the vectors are colinear. - result.x = (v1.y * v2.z) - (v2.y * v1.z) - result.y = (v1.z * v2.x) - (v2.z * v1.x) - result.z = (v1.x * v2.y) - (v2.x * v1.y) - -proc equals*(v1,v2:Vector3d,tol=1.0e-6):bool= - ## Checks if two vectors approximately equals with a tolerance. - return abs(v2.x-v1.x)<=tol and abs(v2.y-v1.y)<=tol and abs(v2.z-v1.z)<=tol - -proc `=~` *(v1,v2:Vector3d):bool= - ## Checks if two vectors approximately equals with a - ## hardcoded tolerance 1e-6 - equals(v1,v2) - -proc angleTo*(v1,v2:Vector3d):float= - ## Returns the smallest angle between v1 and v2, - ## which is in range 0-PI - var - nv1=v1 - nv2=v2 - if not nv1.tryNormalize or not nv2.tryNormalize: - return 0.0 # zero length vector has zero angle to any other vector - return safeArccos(dot(nv1,nv2)) - -proc arbitraryAxis*(norm:Vector3d):Matrix3d {.noInit.}= - ## Computes the rotation matrix that would transform - ## world z vector into `norm`. The inverse of this matrix - ## is useful to transform a planar 3d object to 2d space. - ## This is the same algorithm used to interpret DXF and DWG files. - const lim=1.0/64.0 - var ax,ay,az:Vector3d - if abs(norm.x)<lim and abs(norm.y)<lim: - ax=cross(YAXIS,norm) - else: - ax=cross(ZAXIS,norm) - - ax.normalize() - ay=cross(norm,ax) - ay.normalize() - az=cross(ax,ay) - - result.setElements( - ax.x,ax.y,ax.z,0.0, - ay.x,ay.y,ay.z,0.0, - az.x,az.y,az.z,0.0, - 0.0,0.0,0.0,1.0) - -proc bisect*(v1,v2:Vector3d):Vector3d {.noInit.}= - ## Computes the bisector between v1 and v2 as a normalized vector. - ## If one of the input vectors has zero length, a normalized version - ## of the other is returned. If both input vectors has zero length, - ## an arbitrary normalized vector `v1` is returned. - var - vmag1=v1.len - vmag2=v2.len - - # zero length vector equals arbitrary vector, just change - # magnitude to one to avoid zero division - if vmag1==0.0: - if vmag2==0: #both are zero length return any normalized vector - return XAXIS - vmag1=1.0 - if vmag2==0.0: vmag2=1.0 - - let - x1=v1.x/vmag1 - y1=v1.y/vmag1 - z1=v1.z/vmag1 - x2=v2.x/vmag2 - y2=v2.y/vmag2 - z2=v2.z/vmag2 - - result.x=(x1 + x2) * 0.5 - result.y=(y1 + y2) * 0.5 - result.z=(z1 + z2) * 0.5 - - if not result.tryNormalize(): - # This can happen if vectors are colinear. In this special case - # there are actually inifinitely many bisectors, we select just - # one of them. - result=v1.cross(XAXIS) - if result.sqrLen<1.0e-9: - result=v1.cross(YAXIS) - if result.sqrLen<1.0e-9: - result=v1.cross(ZAXIS) # now we should be guaranteed to have succeeded - result.normalize - - - -# *************************************** -# Point3d implementation -# *************************************** -proc point3d*(x,y,z:float):Point3d= - result.x=x - result.y=y - result.z=z - -proc sqrDist*(a,b:Point3d):float= - ## Computes the squared distance between `a`and `b` - let dx=b.x-a.x - let dy=b.y-a.y - let dz=b.z-a.z - result=dx*dx+dy*dy+dz*dz - -proc dist*(a,b:Point3d):float {.inline.}= - ## Computes the absolute distance between `a`and `b` - result=sqrt(sqrDist(a,b)) - -proc `$` *(p:Point3d):string= - ## String representation of `p` - result=rtos(p.x) - result.add(",") - result.add(rtos(p.y)) - result.add(",") - result.add(rtos(p.z)) - -proc `&`*(p:Point3d,m:Matrix3d):Point3d= - ## Concatenates a point `p` with a transform `m`, - ## resulting in a new, transformed point. - result.z=m.cz*p.z+m.bz*p.y+m.az*p.x+m.tz - result.y=m.cy*p.z+m.by*p.y+m.ay*p.x+m.ty - result.x=m.cx*p.z+m.bx*p.y+m.ax*p.x+m.tx - -proc `&=` *(p:var Point3d,m:Matrix3d)= - ## Applies transformation `m` onto `p` in place. - let - x=p.x - y=p.y - z=p.z - p.x=m.cx*z+m.bx*y+m.ax*x+m.tx - p.y=m.cy*z+m.by*y+m.ay*x+m.ty - p.z=m.cz*z+m.bz*y+m.az*x+m.tz - -proc transformInv*(p:var Point3d,m:Matrix3d)= - ## Applies the inverse of transformation `m` onto `p` in place. - ## If the matrix is not invertable (determinant=0) , EDivByZero will - ## be raised. - - # can possibly be more optimized in the future so use this function when possible - p&=inverse(m) - - -proc `+`*(p:Point3d,v:Vector3d):Point3d {.noInit,inline.} = - ## Adds a vector `v` to a point `p`, resulting - ## in a new point. - result.x=p.x+v.x - result.y=p.y+v.y - result.z=p.z+v.z - -proc `+=`*(p:var Point3d,v:Vector3d) {.noInit,inline.} = - ## Adds a vector `v` to a point `p` in place. - p.x+=v.x - p.y+=v.y - p.z+=v.z - -proc `-`*(p:Point3d,v:Vector3d):Point3d {.noInit,inline.} = - ## Subtracts a vector `v` from a point `p`, resulting - ## in a new point. - result.x=p.x-v.x - result.y=p.y-v.y - result.z=p.z-v.z - -proc `-`*(p1,p2:Point3d):Vector3d {.noInit,inline.} = - ## Subtracts `p2`from `p1` resulting in a difference vector. - result.x=p1.x-p2.x - result.y=p1.y-p2.y - result.z=p1.z-p2.z - -proc `-=`*(p:var Point3d,v:Vector3d) {.noInit,inline.} = - ## Subtracts a vector `v` from a point `p` in place. - p.x-=v.x - p.y-=v.y - p.z-=v.z - -proc equals(p1,p2:Point3d,tol=1.0e-6):bool {.inline.}= - ## Checks if two points approximately equals with a tolerance. - return abs(p2.x-p1.x)<=tol and abs(p2.y-p1.y)<=tol and abs(p2.z-p1.z)<=tol - -proc `=~`*(p1,p2:Point3d):bool {.inline.}= - ## Checks if two vectors approximately equals with a - ## hardcoded tolerance 1e-6 - equals(p1,p2) - -proc rotate*(p:var Point3d,rad:float,axis:Vector3d)= - ## Rotates point `p` in place `rad` radians about an axis - ## passing through origo. - - var v=vector3d(p.x,p.y,p.z) - v.rotate(rad,axis) # reuse this code here since doing the same thing and quite complicated - p.x=v.x - p.y=v.y - p.z=v.z - -proc rotate*(p:var Point3d,angle:float,org:Point3d,axis:Vector3d)= - ## Rotates point `p` in place `rad` radians about an axis - ## passing through `org` - - # see PDF document http://inside.mines.edu/~gmurray/ArbitraryAxisRotation/ArbitraryAxisRotation.pdf - # for how this is computed - - var normax=axis - normax.normalize - - let - cs=cos(angle) - omc=1.0-cs - si=sin(angle) - u=normax.x - v=normax.y - w=normax.z - a=org.x - b=org.y - c=org.z - x=p.x - y=p.y - z=p.z - uu=u*u - vv=v*v - ww=w*w - ux=u*p.x - vy=v*p.y - wz=w*p.z - au=a*u - bv=b*v - cw=c*w - uxmvymwz=ux-vy-wz - - p.x=(a*(vv+ww)-u*(bv+cw-uxmvymwz))*omc + x*cs + (b*w+v*z-c*v-w*y)*si - p.y=(b*(uu+ww)-v*(au+cw-uxmvymwz))*omc + y*cs + (c*u-a*w+w*x-u*z)*si - p.z=(c*(uu+vv)-w*(au+bv-uxmvymwz))*omc + z*cs + (a*v+u*y-b*u-v*x)*si - -proc scale*(p:var Point3d,fac:float) {.inline.}= - ## Scales a point in place `fac` times with world origo as origin. - p.x*=fac - p.y*=fac - p.z*=fac - -proc scale*(p:var Point3d,fac:float,org:Point3d){.inline.}= - ## Scales the point in place `fac` times with `org` as origin. - p.x=(p.x - org.x) * fac + org.x - p.y=(p.y - org.y) * fac + org.y - p.z=(p.z - org.z) * fac + org.z - -proc stretch*(p:var Point3d,facx,facy,facz:float){.inline.}= - ## Scales a point in place non uniformly `facx` , `facy` , `facz` times - ## with world origo as origin. - p.x*=facx - p.y*=facy - p.z*=facz - -proc stretch*(p:var Point3d,facx,facy,facz:float,org:Point3d){.inline.}= - ## Scales the point in place non uniformly `facx` , `facy` , `facz` times - ## with `org` as origin. - p.x=(p.x - org.x) * facx + org.x - p.y=(p.y - org.y) * facy + org.y - p.z=(p.z - org.z) * facz + org.z - - -proc move*(p:var Point3d,dx,dy,dz:float){.inline.}= - ## Translates a point `dx` , `dy` , `dz` in place. - p.x+=dx - p.y+=dy - p.z+=dz - -proc move*(p:var Point3d,v:Vector3d){.inline.}= - ## Translates a point with vector `v` in place. - p.x+=v.x - p.y+=v.y - p.z+=v.z - -proc area*(a,b,c:Point3d):float {.inline.}= - ## Computes the area of the triangle thru points `a` , `b` and `c` - - # The area of a planar 3d quadliteral is the magnitude of the cross - # product of two edge vectors. Taking this time 0.5 gives the triangle area. - return cross(b-a,c-a).len*0.5 - diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim new file mode 100644 index 000000000..0d3351ee5 --- /dev/null +++ b/lib/pure/bitops.nim @@ -0,0 +1,883 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim Authors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a series of low level methods for bit manipulation. +## +## By default, compiler intrinsics are used where possible to improve performance +## on supported compilers: `GCC`, `LLVM_GCC`, `CLANG`, `VCC`, `ICC`. +## +## The module will fallback to pure nim procs in case the backend is not supported. +## You can also use the flag `noIntrinsicsBitOpts` to disable compiler intrinsics. +## +## This module is also compatible with other backends: `JavaScript`, `NimScript` +## as well as the `compiletime VM`. +## +## As a result of using optimized functions/intrinsics, some functions can return +## undefined results if the input is invalid. You can use the flag `noUndefinedBitOpts` +## to force predictable behaviour for all input, causing a small performance hit. +## +## At this time only `fastLog2`, `firstSetBit`, `countLeadingZeroBits` and `countTrailingZeroBits` +## may return undefined and/or platform dependent values if given invalid input. + +import std/macros +import std/private/since +from std/private/bitops_utils import forwardImpl, castToUnsigned + +func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} + ## Computes the `bitwise complement` of the integer `x`. + +func internalBitand[T: SomeInteger](x, y: T): T {.magic: "BitandI".} + +func internalBitor[T: SomeInteger](x, y: T): T {.magic: "BitorI".} + +func internalBitxor[T: SomeInteger](x, y: T): T {.magic: "BitxorI".} + +macro bitand*[T: SomeInteger](x, y: T; z: varargs[T]): T = + ## Computes the `bitwise and` of all arguments collectively. + let fn = bindSym("internalBitand") + result = newCall(fn, x, y) + for extra in z: + result = newCall(fn, result, extra) + +macro bitor*[T: SomeInteger](x, y: T; z: varargs[T]): T = + ## Computes the `bitwise or` of all arguments collectively. + let fn = bindSym("internalBitor") + result = newCall(fn, x, y) + for extra in z: + result = newCall(fn, result, extra) + +macro bitxor*[T: SomeInteger](x, y: T; z: varargs[T]): T = + ## Computes the `bitwise xor` of all arguments collectively. + let fn = bindSym("internalBitxor") + result = newCall(fn, x, y) + for extra in z: + result = newCall(fn, result, extra) + + +type BitsRange*[T] = range[0..sizeof(T)*8-1] + ## A range with all bit positions for type `T`. + +template typeMasked[T: SomeInteger](x: T): T = + when defined(js): + T(x and ((0xffffffff_ffffffff'u shr (64 - sizeof(T) * 8)))) + else: + x + +func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns an extracted (and shifted) slice of bits from `v`. + runnableExamples: + doAssert 0b10111.bitsliced(2 .. 4) == 0b101 + doAssert 0b11100.bitsliced(0 .. 2) == 0b100 + doAssert 0b11100.bitsliced(0 ..< 3) == 0b100 + + let + upmost = sizeof(T) * 8 - 1 + uv = v.castToUnsigned + ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T + +proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v` into an extracted (and shifted) slice of bits from `v`. + runnableExamples: + var x = 0b101110 + x.bitslice(2 .. 4) + doAssert x == 0b011 + + let + upmost = sizeof(T) * 8 - 1 + uv = v.castToUnsigned + v = ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T + +func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Creates a bitmask based on a slice of bits. + runnableExamples: + doAssert toMask[int32](1 .. 3) == 0b1110'i32 + doAssert toMask[int32](0 .. 3) == 0b1111'i32 + + let + upmost = sizeof(T) * 8 - 1 + bitmask = bitnot(0.T).castToUnsigned + ((bitmask shl (upmost - slice.b + slice.a)).typeMasked shr (upmost - slice.b)).T + +proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with only the `1` bits from `mask` matching those of + ## `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.masked(0b0000_1010'u8) == 0b0000_0010'u8 + + bitand(v, mask) + +func masked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with only the `1` bits in the range of `slice` + ## matching those of `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_1011'u8 + doAssert v.masked(1 .. 3) == 0b0000_1010'u8 + + bitand(v, toMask[T](slice)) + +proc mask*[T: SomeInteger](v: var T; mask: T) {.inline, since: (1, 3).} = + ## Mutates `v`, with only the `1` bits from `mask` matching those of + ## `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.mask(0b0000_1010'u8) + doAssert v == 0b0000_0010'u8 + + v = bitand(v, mask) + +proc mask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with only the `1` bits in the range of `slice` + ## matching those of `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_1011'u8 + v.mask(1 .. 3) + doAssert v == 0b0000_1010'u8 + + v = bitand(v, toMask[T](slice)) + +func setMasked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits from `mask` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.setMasked(0b0000_1010'u8) == 0b0000_1011'u8 + + bitor(v, mask) + +func setMasked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits in the range of `slice` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.setMasked(2 .. 3) == 0b0000_1111'u8 + + bitor(v, toMask[T](slice)) + +proc setMask*[T: SomeInteger](v: var T; mask: T) {.inline.} = + ## Mutates `v`, with all the `1` bits from `mask` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.setMask(0b0000_1010'u8) + doAssert v == 0b0000_1011'u8 + + v = bitor(v, mask) + +proc setMask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with all the `1` bits in the range of `slice` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.setMask(2 .. 3) + doAssert v == 0b0000_1111'u8 + + v = bitor(v, toMask[T](slice)) + +func clearMasked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits from `mask` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.clearMasked(0b0000_1010'u8) == 0b0000_0001'u8 + + bitand(v, bitnot(mask)) + +func clearMasked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits in the range of `slice` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.clearMasked(1 .. 3) == 0b0000_0001'u8 + + bitand(v, bitnot(toMask[T](slice))) + +proc clearMask*[T: SomeInteger](v: var T; mask: T) {.inline.} = + ## Mutates `v`, with all the `1` bits from `mask` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + var v = 0b0000_0011'u8 + v.clearMask(0b0000_1010'u8) + doAssert v == 0b0000_0001'u8 + + v = bitand(v, bitnot(mask)) + +proc clearMask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with all the `1` bits in the range of `slice` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + var v = 0b0000_0011'u8 + v.clearMask(1 .. 3) + doAssert v == 0b0000_0001'u8 + + v = bitand(v, bitnot(toMask[T](slice))) + +func flipMasked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits from `mask` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.flipMasked(0b0000_1010'u8) == 0b0000_1001'u8 + + bitxor(v, mask) + +func flipMasked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits in the range of `slice` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.flipMasked(1 .. 3) == 0b0000_1101'u8 + + bitxor(v, toMask[T](slice)) + +proc flipMask*[T: SomeInteger](v: var T; mask: T) {.inline.} = + ## Mutates `v`, with all the `1` bits from `mask` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.flipMask(0b0000_1010'u8) + doAssert v == 0b0000_1001'u8 + + v = bitxor(v, mask) + +proc flipMask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with all the `1` bits in the range of `slice` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.flipMask(1 .. 3) + doAssert v == 0b0000_1101'u8 + + v = bitxor(v, toMask[T](slice)) + +proc setBit*[T: SomeInteger](v: var T; bit: BitsRange[T]) {.inline.} = + ## Mutates `v`, with the bit at position `bit` set to 1. + runnableExamples: + var v = 0b0000_0011'u8 + v.setBit(5'u8) + doAssert v == 0b0010_0011'u8 + + v.setMask(1.T shl bit) + +proc clearBit*[T: SomeInteger](v: var T; bit: BitsRange[T]) {.inline.} = + ## Mutates `v`, with the bit at position `bit` set to 0. + runnableExamples: + var v = 0b0000_0011'u8 + v.clearBit(1'u8) + doAssert v == 0b0000_0001'u8 + + v.clearMask(1.T shl bit) + +proc flipBit*[T: SomeInteger](v: var T; bit: BitsRange[T]) {.inline.} = + ## Mutates `v`, with the bit at position `bit` flipped. + runnableExamples: + var v = 0b0000_0011'u8 + v.flipBit(1'u8) + doAssert v == 0b0000_0001'u8 + + v = 0b0000_0011'u8 + v.flipBit(2'u8) + doAssert v == 0b0000_0111'u8 + + v.flipMask(1.T shl bit) + +macro setBits*(v: typed; bits: varargs[typed]): untyped = + ## Mutates `v`, with the bits at positions `bits` set to 1. + runnableExamples: + var v = 0b0000_0011'u8 + v.setBits(3, 5, 7) + doAssert v == 0b1010_1011'u8 + + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("setBit", v, bit) + +macro clearBits*(v: typed; bits: varargs[typed]): untyped = + ## Mutates `v`, with the bits at positions `bits` set to 0. + runnableExamples: + var v = 0b1111_1111'u8 + v.clearBits(1, 3, 5, 7) + doAssert v == 0b0101_0101'u8 + + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("clearBit", v, bit) + +macro flipBits*(v: typed; bits: varargs[typed]): untyped = + ## Mutates `v`, with the bits at positions `bits` set to 0. + runnableExamples: + var v = 0b0000_1111'u8 + v.flipBits(1, 3, 5, 7) + doAssert v == 0b1010_0101'u8 + + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("flipBit", v, bit) + + +proc testBit*[T: SomeInteger](v: T; bit: BitsRange[T]): bool {.inline.} = + ## Returns true if the bit in `v` at positions `bit` is set to 1. + runnableExamples: + let v = 0b0000_1111'u8 + doAssert v.testBit(0) + doAssert not v.testBit(7) + + let mask = 1.T shl bit + return (v and mask) == mask + +# #### Pure Nim version #### + +func firstSetBitNim(x: uint32): int {.inline.} = + ## Returns the 1-based index of the least significant set bit of x, or if x is zero, returns zero. + # https://graphics.stanford.edu/%7Eseander/bithacks.html#ZerosOnRightMultLookup + const lookup: array[32, uint8] = [0'u8, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, + 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9] + let v = x.uint32 + let k = not v + 1 # get two's complement # cast[uint32](-cast[int32](v)) + result = 1 + lookup[uint32((v and k) * 0x077CB531'u32) shr 27].int + +func firstSetBitNim(x: uint64): int {.inline.} = + ## Returns the 1-based index of the least significant set bit of x, or if x is zero, returns zero. + # https://graphics.stanford.edu/%7Eseander/bithacks.html#ZerosOnRightMultLookup + let v = uint64(x) + var k = uint32(v and 0xFFFFFFFF'u32) + if k == 0: + k = uint32(v shr 32'u32) and 0xFFFFFFFF'u32 + result = 32 + else: + result = 0 + result += firstSetBitNim(k) + +func fastlog2Nim(x: uint32): int {.inline.} = + ## Quickly find the log base 2 of a 32-bit or less integer. + # https://graphics.stanford.edu/%7Eseander/bithacks.html#IntegerLogDeBruijn + # https://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers + const lookup: array[32, uint8] = [0'u8, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, + 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31] + var v = x.uint32 + v = v or v shr 1 # first round down to one less than a power of 2 + v = v or v shr 2 + v = v or v shr 4 + v = v or v shr 8 + v = v or v shr 16 + result = lookup[uint32(v * 0x07C4ACDD'u32) shr 27].int + +func fastlog2Nim(x: uint64): int {.inline.} = + ## Quickly find the log base 2 of a 64-bit integer. + # https://graphics.stanford.edu/%7Eseander/bithacks.html#IntegerLogDeBruijn + # https://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers + const lookup: array[64, uint8] = [0'u8, 58, 1, 59, 47, 53, 2, 60, 39, 48, 27, 54, + 33, 42, 3, 61, 51, 37, 40, 49, 18, 28, 20, 55, 30, 34, 11, 43, 14, 22, 4, 62, + 57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21, 56, 45, 25, 31, + 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5, 63] + var v = x.uint64 + v = v or v shr 1 # first round down to one less than a power of 2 + v = v or v shr 2 + v = v or v shr 4 + v = v or v shr 8 + v = v or v shr 16 + v = v or v shr 32 + result = lookup[(v * 0x03F6EAF2CD271461'u64) shr 58].int + +import system/countbits_impl + +const useBuiltinsRotate = (defined(amd64) or defined(i386)) and + (defined(gcc) or defined(clang) or defined(vcc) or + (defined(icl) and not defined(cpp))) and useBuiltins + +template parityImpl[T](value: T): int = + # formula id from: https://graphics.stanford.edu/%7Eseander/bithacks.html#ParityParallel + var v = value + when sizeof(T) == 8: + v = v xor (v shr 32) + when sizeof(T) >= 4: + v = v xor (v shr 16) + when sizeof(T) >= 2: + v = v xor (v shr 8) + v = v xor (v shr 4) + v = v and 0xf + ((0x6996'u shr v) and 1).int + + +when useGCC_builtins: + # Returns the bit parity in value + proc builtin_parity(x: cuint): cint {.importc: "__builtin_parity", cdecl.} + proc builtin_parityll(x: culonglong): cint {.importc: "__builtin_parityll", cdecl.} + + # Returns one plus the index of the least significant 1-bit of x, or if x is zero, returns zero. + proc builtin_ffs(x: cint): cint {.importc: "__builtin_ffs", cdecl.} + proc builtin_ffsll(x: clonglong): cint {.importc: "__builtin_ffsll", cdecl.} + + # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined. + proc builtin_clz(x: cuint): cint {.importc: "__builtin_clz", cdecl.} + proc builtin_clzll(x: culonglong): cint {.importc: "__builtin_clzll", cdecl.} + + # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined. + proc builtin_ctz(x: cuint): cint {.importc: "__builtin_ctz", cdecl.} + proc builtin_ctzll(x: culonglong): cint {.importc: "__builtin_ctzll", cdecl.} + +elif useVCC_builtins: + # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1). + func bitScanReverse(index: ptr culong, mask: culong): uint8 {. + importc: "_BitScanReverse", header: "<intrin.h>".} + func bitScanReverse64(index: ptr culong, mask: uint64): uint8 {. + importc: "_BitScanReverse64", header: "<intrin.h>".} + + # Search the mask data from least significant bit (LSB) to the most significant bit (MSB) for a set bit (1). + func bitScanForward(index: ptr culong, mask: culong): uint8 {. + importc: "_BitScanForward", header: "<intrin.h>".} + func bitScanForward64(index: ptr culong, mask: uint64): uint8 {. + importc: "_BitScanForward64", header: "<intrin.h>".} + + template vcc_scan_impl(fnc: untyped; v: untyped): int = + var index {.inject.}: culong = 0 + discard fnc(index.addr, v) + index.int + +elif useICC_builtins: + # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined. + func bitScanForward(p: ptr uint32, b: uint32): uint8 {. + importc: "_BitScanForward", header: "<immintrin.h>".} + func bitScanForward64(p: ptr uint32, b: uint64): uint8 {. + importc: "_BitScanForward64", header: "<immintrin.h>".} + + # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined. + func bitScanReverse(p: ptr uint32, b: uint32): uint8 {. + importc: "_BitScanReverse", header: "<immintrin.h>".} + func bitScanReverse64(p: ptr uint32, b: uint64): uint8 {. + importc: "_BitScanReverse64", header: "<immintrin.h>".} + + template icc_scan_impl(fnc: untyped; v: untyped): int = + var index: uint32 + discard fnc(index.addr, v) + index.int + +func countSetBits*(x: SomeInteger): int {.inline.} = + ## Counts the set bits in an integer (also called `Hamming weight`:idx:). + runnableExamples: + doAssert countSetBits(0b0000_0011'u8) == 2 + doAssert countSetBits(0b1010_1010'u8) == 4 + + result = countSetBitsImpl(x) + +func popcount*(x: SomeInteger): int {.inline.} = + ## Alias for `countSetBits <#countSetBits,SomeInteger>`_ (Hamming weight). + result = countSetBits(x) + +func parityBits*(x: SomeInteger): int {.inline.} = + ## Calculate the bit parity in an integer. If the number of 1-bits + ## is odd, the parity is 1, otherwise 0. + runnableExamples: + doAssert parityBits(0b0000_0000'u8) == 0 + doAssert parityBits(0b0101_0001'u8) == 1 + doAssert parityBits(0b0110_1001'u8) == 0 + doAssert parityBits(0b0111_1111'u8) == 1 + + # Can be used a base if creating ASM version. + # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd + let x = x.castToUnsigned + when nimvm: + result = forwardImpl(parityImpl, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_parity(x.uint32).int + else: result = builtin_parityll(x.uint64).int + else: + when sizeof(x) <= 4: result = parityImpl(x.uint32) + else: result = parityImpl(x.uint64) + +func firstSetBit*(x: SomeInteger): int {.inline.} = + ## Returns the 1-based index of the least significant set bit of `x`. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is 0, + ## otherwise the result is undefined. + runnableExamples: + doAssert firstSetBit(0b0000_0001'u8) == 1 + doAssert firstSetBit(0b0000_0010'u8) == 2 + doAssert firstSetBit(0b0000_0100'u8) == 3 + doAssert firstSetBit(0b0000_1000'u8) == 4 + doAssert firstSetBit(0b0000_1111'u8) == 1 + + # GCC builtin 'builtin_ffs' already handle zero input. + let x = x.castToUnsigned + when nimvm: + when noUndefined: + if x == 0: + return 0 + result = forwardImpl(firstSetBitNim, x) + else: + when noUndefined and not useGCC_builtins: + if x == 0: + return 0 + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_ffs(cast[cint](x.cuint)).int + else: result = builtin_ffsll(cast[clonglong](x.culonglong)).int + elif useVCC_builtins: + when sizeof(x) <= 4: + result = 1 + vcc_scan_impl(bitScanForward, x.culong) + elif arch64: + result = 1 + vcc_scan_impl(bitScanForward64, x.uint64) + else: + result = firstSetBitNim(x.uint64) + elif useICC_builtins: + when sizeof(x) <= 4: + result = 1 + icc_scan_impl(bitScanForward, x.uint32) + elif arch64: + result = 1 + icc_scan_impl(bitScanForward64, x.uint64) + else: + result = firstSetBitNim(x.uint64) + else: + when sizeof(x) <= 4: result = firstSetBitNim(x.uint32) + else: result = firstSetBitNim(x.uint64) + +func fastLog2*(x: SomeInteger): int {.inline.} = + ## Quickly find the log base 2 of an integer. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is -1, + ## otherwise the result is undefined. + runnableExamples: + doAssert fastLog2(0b0000_0001'u8) == 0 + doAssert fastLog2(0b0000_0010'u8) == 1 + doAssert fastLog2(0b0000_0100'u8) == 2 + doAssert fastLog2(0b0000_1000'u8) == 3 + doAssert fastLog2(0b0000_1111'u8) == 3 + + let x = x.castToUnsigned + when noUndefined: + if x == 0: + return -1 + when nimvm: + result = forwardImpl(fastlog2Nim, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = 31 - builtin_clz(x.uint32).int + else: result = 63 - builtin_clzll(x.uint64).int + elif useVCC_builtins: + when sizeof(x) <= 4: + result = vcc_scan_impl(bitScanReverse, x.culong) + elif arch64: + result = vcc_scan_impl(bitScanReverse64, x.uint64) + else: + result = fastlog2Nim(x.uint64) + elif useICC_builtins: + when sizeof(x) <= 4: + result = icc_scan_impl(bitScanReverse, x.uint32) + elif arch64: + result = icc_scan_impl(bitScanReverse64, x.uint64) + else: + result = fastlog2Nim(x.uint64) + else: + when sizeof(x) <= 4: result = fastlog2Nim(x.uint32) + else: result = fastlog2Nim(x.uint64) + +func countLeadingZeroBits*(x: SomeInteger): int {.inline.} = + ## Returns the number of leading zero bits in an integer. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is 0, + ## otherwise the result is undefined. + ## + ## **See also:** + ## * `countTrailingZeroBits proc <#countTrailingZeroBits,SomeInteger>`_ + runnableExamples: + doAssert countLeadingZeroBits(0b0000_0001'u8) == 7 + doAssert countLeadingZeroBits(0b0000_0010'u8) == 6 + doAssert countLeadingZeroBits(0b0000_0100'u8) == 5 + doAssert countLeadingZeroBits(0b0000_1000'u8) == 4 + doAssert countLeadingZeroBits(0b0000_1111'u8) == 4 + + let x = x.castToUnsigned + when noUndefined: + if x == 0: + return 0 + when nimvm: + result = sizeof(x)*8 - 1 - forwardImpl(fastlog2Nim, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_clz(x.uint32).int - (32 - sizeof(x)*8) + else: result = builtin_clzll(x.uint64).int + else: + when sizeof(x) <= 4: result = sizeof(x)*8 - 1 - fastlog2Nim(x.uint32) + else: result = sizeof(x)*8 - 1 - fastlog2Nim(x.uint64) + +func countTrailingZeroBits*(x: SomeInteger): int {.inline.} = + ## Returns the number of trailing zeros in an integer. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is 0, + ## otherwise the result is undefined. + ## + ## **See also:** + ## * `countLeadingZeroBits proc <#countLeadingZeroBits,SomeInteger>`_ + runnableExamples: + doAssert countTrailingZeroBits(0b0000_0001'u8) == 0 + doAssert countTrailingZeroBits(0b0000_0010'u8) == 1 + doAssert countTrailingZeroBits(0b0000_0100'u8) == 2 + doAssert countTrailingZeroBits(0b0000_1000'u8) == 3 + doAssert countTrailingZeroBits(0b0000_1111'u8) == 0 + + let x = x.castToUnsigned + when noUndefined: + if x == 0: + return 0 + when nimvm: + result = firstSetBit(x) - 1 + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_ctz(x.uint32).int + else: result = builtin_ctzll(x.uint64).int + else: + result = firstSetBit(x) - 1 + +when useBuiltinsRotate: + when defined(gcc): + # GCC was tested until version 4.8.1 and intrinsics were present. Not tested + # in previous versions. + func builtin_rotl8(value: uint8, shift: cint): uint8 + {.importc: "__rolb", header: "<x86intrin.h>".} + func builtin_rotl16(value: cushort, shift: cint): cushort + {.importc: "__rolw", header: "<x86intrin.h>".} + func builtin_rotl32(value: cuint, shift: cint): cuint + {.importc: "__rold", header: "<x86intrin.h>".} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: cint): culonglong + {.importc: "__rolq", header: "<x86intrin.h>".} + + func builtin_rotr8(value: uint8, shift: cint): uint8 + {.importc: "__rorb", header: "<x86intrin.h>".} + func builtin_rotr16(value: cushort, shift: cint): cushort + {.importc: "__rorw", header: "<x86intrin.h>".} + func builtin_rotr32(value: cuint, shift: cint): cuint + {.importc: "__rord", header: "<x86intrin.h>".} + when defined(amd64): + func builtin_rotr64(value: culonglong, shift: cint): culonglong + {.importc: "__rorq", header: "<x86intrin.h>".} + elif defined(clang): + # In CLANG, builtins have been present since version 8.0.0 and intrinsics + # since version 9.0.0. This implementation chose the builtins, as they have + # been around for longer. + # https://releases.llvm.org/8.0.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release + # https://releases.llvm.org/8.0.0/tools/clang/docs/LanguageExtensions.html#builtin-rotateleft + # source for correct declarations: https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/Builtins.def + func builtin_rotl8(value: uint8, shift: uint8): uint8 + {.importc: "__builtin_rotateleft8", nodecl.} + func builtin_rotl16(value: cushort, shift: cushort): cushort + {.importc: "__builtin_rotateleft16", nodecl.} + func builtin_rotl32(value: cuint, shift: cuint): cuint + {.importc: "__builtin_rotateleft32", nodecl.} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: culonglong): culonglong + {.importc: "__builtin_rotateleft64", nodecl.} + + func builtin_rotr8(value: uint8, shift: uint8): uint8 + {.importc: "__builtin_rotateright8", nodecl.} + func builtin_rotr16(value: cushort, shift: cushort): cushort + {.importc: "__builtin_rotateright16", nodecl.} + func builtin_rotr32(value: cuint, shift: cuint): cuint + {.importc: "__builtin_rotateright32", nodecl.} + when defined(amd64): + # shift is unsigned, refs https://github.com/llvm-mirror/clang/commit/892de415b7fde609dafc4e6c1643b7eaa0150a4d + func builtin_rotr64(value: culonglong, shift: culonglong): culonglong + {.importc: "__builtin_rotateright64", nodecl.} + elif defined(vcc): + # Tested on Microsoft (R) C/C++ Optimizing Compiler 19.28.29335 x64 and x86. + # Not tested in previous versions. + # https://docs.microsoft.com/en-us/cpp/intrinsics/rotl8-rotl16?view=msvc-160 + # https://docs.microsoft.com/en-us/cpp/intrinsics/rotr8-rotr16?view=msvc-160 + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/rotl-rotl64-rotr-rotr64?view=msvc-160 + func builtin_rotl8(value: uint8, shift: uint8): uint8 + {.importc: "_rotl8", header: "<intrin.h>".} + func builtin_rotl16(value: cushort, shift: uint8): cushort + {.importc: "_rotl16", header: "<intrin.h>".} + func builtin_rotl32(value: cuint, shift: cint): cuint + {.importc: "_rotl", header: "<stdlib.h>".} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: cint): culonglong + {.importc: "_rotl64", header: "<stdlib.h>".} + + func builtin_rotr8(value: uint8, shift: uint8): uint8 + {.importc: "_rotr8", header: "<intrin.h>".} + func builtin_rotr16(value: cushort, shift: uint8): cushort + {.importc: "_rotr16", header: "<intrin.h>".} + func builtin_rotr32(value: cuint, shift: cint): cuint + {.importc: "_rotr", header: "<stdlib.h>".} + when defined(amd64): + func builtin_rotr64(value: culonglong, shift: cint): culonglong + {.importc: "_rotr64", header: "<stdlib.h>".} + elif defined(icl): + # Tested on Intel(R) C++ Intel(R) 64 Compiler Classic Version 2021.1.2 Build + # 20201208_000000 x64 and x86. Not tested in previous versions. + func builtin_rotl8(value: uint8, shift: cint): uint8 + {.importc: "__rolb", header: "<immintrin.h>".} + func builtin_rotl16(value: cushort, shift: cint): cushort + {.importc: "__rolw", header: "<immintrin.h>".} + func builtin_rotl32(value: cuint, shift: cint): cuint + {.importc: "__rold", header: "<immintrin.h>".} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: cint): culonglong + {.importc: "__rolq", header: "<immintrin.h>".} + + func builtin_rotr8(value: uint8, shift: cint): uint8 + {.importc: "__rorb", header: "<immintrin.h>".} + func builtin_rotr16(value: cushort, shift: cint): cushort + {.importc: "__rorw", header: "<immintrin.h>".} + func builtin_rotr32(value: cuint, shift: cint): cuint + {.importc: "__rord", header: "<immintrin.h>".} + when defined(amd64): + func builtin_rotr64(value: culonglong, shift: cint): culonglong + {.importc: "__rorq", header: "<immintrin.h>".} + +func rotl[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} = + ## Left-rotate bits in a `value`. + # https://stackoverflow.com/a/776523 + const mask = 8 * sizeof(value) - 1 + let rot = rot and mask + (value shl rot) or (value shr ((-rot) and mask)) + +func rotr[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} = + ## Right-rotate bits in a `value`. + const mask = 8 * sizeof(value) - 1 + let rot = rot and mask + (value shr rot) or (value shl ((-rot) and mask)) + +func shiftTypeTo(size: static int, shift: int): auto {.inline.} = + ## Returns the `shift` for the rotation according to the compiler and the + ## `size`. + when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl): + cint(shift) + elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1): + uint8(shift) + elif defined(clang): + when size == 2: + cushort(shift) + elif size == 4: + cuint(shift) + elif size == 8: + culonglong(shift) + +func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Left-rotate bits in a `value`. + runnableExamples: + doAssert rotateLeftBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 + doAssert rotateLeftBits(0b00111100_11000011'u16, 8) == + 0b11000011_00111100'u16 + doAssert rotateLeftBits(0b0000111111110000_1111000000001111'u32, 16) == + 0b1111000000001111_0000111111110000'u32 + doAssert rotateLeftBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == + 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 + when nimvm: + rotl(value, shift.int32) + else: + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotl8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotl32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotl64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotl(value, shift.int32) + else: + rotl(value, shift.int32) + +func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Right-rotate bits in a `value`. + runnableExamples: + doAssert rotateRightBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 + doAssert rotateRightBits(0b00111100_11000011'u16, 8) == + 0b11000011_00111100'u16 + doAssert rotateRightBits(0b0000111111110000_1111000000001111'u32, 16) == + 0b1111000000001111_0000111111110000'u32 + doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == + 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 + when nimvm: + rotr(value, shift.int32) + else: + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotr8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotr32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotr64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotr(value, shift.int32) + else: + rotr(value, shift.int32) + +func repeatBits[T: SomeUnsignedInt](x: SomeUnsignedInt; retType: type[T]): T = + result = x + var i = 1 + while i != (sizeof(T) div sizeof(x)): + result = (result shl (sizeof(x)*8*i)) or result + i *= 2 + +func reverseBits*[T: SomeUnsignedInt](x: T): T = + ## Return the bit reversal of x. + runnableExamples: + doAssert reverseBits(0b10100100'u8) == 0b00100101'u8 + doAssert reverseBits(0xdd'u8) == 0xbb'u8 + doAssert reverseBits(0xddbb'u16) == 0xddbb'u16 + doAssert reverseBits(0xdeadbeef'u32) == 0xf77db57b'u32 + + template repeat(x: SomeUnsignedInt): T = repeatBits(x, T) + + result = x + result = + ((repeat(0x55u8) and result) shl 1) or + ((repeat(0xaau8) and result) shr 1) + result = + ((repeat(0x33u8) and result) shl 2) or + ((repeat(0xccu8) and result) shr 2) + when sizeof(T) == 1: + result = (result shl 4) or (result shr 4) + when sizeof(T) >= 2: + result = + ((repeat(0x0fu8) and result) shl 4) or + ((repeat(0xf0u8) and result) shr 4) + when sizeof(T) == 2: + result = (result shl 8) or (result shr 8) + when sizeof(T) >= 4: + result = + ((repeat(0x00ffu16) and result) shl 8) or + ((repeat(0xff00u16) and result) shr 8) + when sizeof(T) == 4: + result = (result shl 16) or (result shr 16) + when sizeof(T) == 8: + result = + ((repeat(0x0000ffffu32) and result) shl 16) or + ((repeat(0xffff0000u32) and result) shr 16) + result = (result shl 32) or (result shr 32) diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index c6a603318..59e2078df 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -9,40 +9,104 @@ ## This module implements a simple proc for opening URLs with the user's ## default browser. +## +## Unstable API. -import strutils +import std/private/since # used by the deprecated `openDefaultBrowser()` + +import std/strutils + +when defined(nimPreviewSlimSystem): + import std/assertions when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + from std/os import absolutePath else: - import os, osproc + import std/os + when not defined(osx): + import std/osproc + +const osOpenCmd* = + when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \ + ## Alias for the operating system specific *"open"* command, + ## `"open"` on OSX, MacOS and Windows, `"xdg-open"` on Linux, BSD, etc. + +proc prepare(s: string): string = + if s.contains("://"): + result = s + else: + result = "file://" & absolutePath(s) + +proc openDefaultBrowserRaw(url: string) = + ## note the url argument should be alreadly prepared, i.e. the url is passed "AS IS" -proc openDefaultBrowser*(url: string) = - ## opens `url` with the user's default browser. This does not block. - ## - ## Under Windows, ``ShellExecute`` is used. Under Mac OS X the ``open`` - ## command is used. Under Unix, it is checked if ``gnome-open`` exists and - ## used if it does. Next attempt is ``kde-open``, then ``xdg-open``. - ## Otherwise the environment variable ``BROWSER`` is used to determine the - ## default browser to use. when defined(windows): - when useWinUnicode: - var o = newWideCString("open") - var u = newWideCString(url) - discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) - else: - discard shellExecuteA(0'i32, "open", url, nil, nil, SW_SHOWNORMAL) + var o = newWideCString(osOpenCmd) + var u = newWideCString(url) + discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) elif defined(macosx): - discard execShellCmd("open " & quoteShell(url)) + discard execShellCmd(osOpenCmd & " " & quoteShell(url)) else: - const attempts = ["gnome-open ", "kde-open ", "xdg-open "] var u = quoteShell(url) - for a in items(attempts): - if execShellCmd(a & u) == 0: return - for b in getEnv("BROWSER").string.split(PathSep): + if execShellCmd(osOpenCmd & " " & u) == 0: return + for b in getEnv("BROWSER").split(PathSep): try: - # we use ``startProcess`` here because we don't want to block! - discard startProcess(command=b, args=[url], options={poUsePath}) + # we use `startProcess` here because we don't want to block! + discard startProcess(command = b, args = [url], options = {poUsePath}) return except OSError: discard + +proc openDefaultBrowser*(url: string) = + ## Opens `url` with the user's default browser. This does not block. + ## The URL must not be empty string, to open on a blank page see `openDefaultBrowser()`. + ## + ## Under Windows, `ShellExecute` is used. Under Mac OS X the `open` + ## command is used. Under Unix, it is checked if `xdg-open` exists and + ## used if it does. Otherwise the environment variable `BROWSER` is + ## used to determine the default browser to use. + ## + ## This proc doesn't raise an exception on error, beware. + ## + ## ```nim + ## block: openDefaultBrowser("https://nim-lang.org") + ## ``` + doAssert url.len > 0, "URL must not be empty string" + openDefaultBrowserRaw(url) + +proc openDefaultBrowser*() {.since: (1, 1), deprecated: + "not implemented, please open with a specific url instead".} = + ## Intends to open the user's default browser without any `url` (blank page). + ## This does not block. + ## Intends to implement IETF RFC-6694 Section 3, + ## ("about:blank" is reserved for a blank page). + ## + ## Beware that this intended behavior is **not** implemented and + ## considered not worthy to implement here. + ## + ## The following describes the behavior of current implementation: + ## + ## - Under Windows, this will only cause a pop-up dialog \ + ## asking the assocated application with `about` \ + ## (as Windows simply treats `about:` as a protocol like `http`). + ## - Under Mac OS X the `open "about:blank"` command is used. + ## - Under Unix, it is checked if `xdg-open` exists and used \ + ## if it does and open the application assocated with `text/html` mime \ + ## (not `x-scheme-handler/http`, so maybe html-viewer \ + ## other than your default browser is opened). \ + ## Otherwise the environment variable `BROWSER` is used \ + ## to determine the default browser to use. + ## + ## This proc doesn't raise an exception on error, beware. + ## + ## ```nim + ## block: openDefaultBrowser() + ## ``` + ## + ## **See also:** + ## + ## * https://tools.ietf.org/html/rfc6694#section-3 + openDefaultBrowserRaw("about:blank") # See IETF RFC-6694 Section 3. diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 200a4adf1..034f224ac 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -9,72 +9,32 @@ ## This module implements helper procs for CGI applications. Example: ## -## .. code-block:: Nim +## ```Nim +## import std/[strtabs, cgi] ## -## import strtabs, cgi -## -## # Fill the values when debugging: -## when debug: -## setTestData("name", "Klaus", "password", "123456") -## # read the data into `myData` -## var myData = readData() -## # check that the data's variable names are "name" or "password" -## validateData(myData, "name", "password") -## # start generating content: -## writeContentType() -## # generate content: -## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") -## write(stdout, "<html><head><title>Test</title></head><body>\n") -## writeLine(stdout, "your name: " & myData["name"]) -## writeLine(stdout, "your password: " & myData["password"]) -## writeLine(stdout, "</body></html>") - -import strutils, os, strtabs, cookies - -proc encodeUrl*(s: string): string = - ## Encodes a value to be HTTP safe: This means that characters in the set - ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, - ## a space is converted to ``'+'`` and every other character is encoded as - ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. - result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars - for i in 0..s.len-1: - case s[i] - of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) - of ' ': add(result, '+') - else: - add(result, '%') - add(result, toHex(ord(s[i]), 2)) - -proc handleHexChar(c: char, x: var int) {.inline.} = - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: assert(false) - -proc decodeUrl*(s: string): string = - ## Decodes a value from its HTTP representation: This means that a ``'+'`` - ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal - ## value) is converted to the character with ordinal number ``xx``, and - ## and every other character is carried over. - result = newString(s.len) - var i = 0 - var j = 0 - while i < s.len: - case s[i] - of '%': - var x = 0 - handleHexChar(s[i+1], x) - handleHexChar(s[i+2], x) - inc(i, 2) - result[j] = chr(x) - of '+': result[j] = ' ' - else: result[j] = s[i] - inc(i) - inc(j) - setLen(result, j) - -{.deprecated: [URLDecode: decodeUrl, URLEncode: encodeUrl].} +## # Fill the values when debugging: +## when debug: +## setTestData("name", "Klaus", "password", "123456") +## # read the data into `myData` +## var myData = readData() +## # check that the data's variable names are "name" or "password" +## validateData(myData, "name", "password") +## # start generating content: +## writeContentType() +## # generate content: +## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") +## write(stdout, "<html><head><title>Test</title></head><body>\n") +## writeLine(stdout, "your name: " & myData["name"]) +## writeLine(stdout, "your password: " & myData["password"]) +## writeLine(stdout, "</body></html>") +## ``` + +import std/[strutils, os, strtabs, cookies, uri] +export uri.encodeUrl, uri.decodeUrl + +when defined(nimPreviewSlimSystem): + import std/syncio + proc addXmlChar(dest: var string, c: char) {.inline.} = case c @@ -86,250 +46,215 @@ proc addXmlChar(dest: var string, c: char) {.inline.} = proc xmlEncode*(s: string): string = ## Encodes a value to be XML safe: - ## * ``"`` is replaced by ``"`` - ## * ``<`` is replaced by ``<`` - ## * ``>`` is replaced by ``>`` - ## * ``&`` is replaced by ``&`` + ## * `"` is replaced by `"` + ## * `<` is replaced by `<` + ## * `>` is replaced by `>` + ## * `&` is replaced by `&` ## * every other character is carried over. result = newStringOfCap(s.len + s.len shr 2) for i in 0..len(s)-1: addXmlChar(result, s[i]) type - CgiError* = object of IOError ## exception that is raised if a CGI error occurs - RequestMethod* = enum ## the used request method - methodNone, ## no REQUEST_METHOD environment variable - methodPost, ## query uses the POST method - methodGet ## query uses the GET method - -{.deprecated: [TRequestMethod: RequestMethod, ECgi: CgiError, - XMLencode: xmlEncode].} + CgiError* = object of IOError ## Exception that is raised if a CGI error occurs. + RequestMethod* = enum ## The used request method. + methodNone, ## no REQUEST_METHOD environment variable + methodPost, ## query uses the POST method + methodGet ## query uses the GET method proc cgiError*(msg: string) {.noreturn.} = - ## raises an ECgi exception with message `msg`. - var e: ref CgiError - new(e) - e.msg = msg - raise e + ## Raises a `CgiError` exception with message `msg`. + raise newException(CgiError, msg) proc getEncodedData(allowedMethods: set[RequestMethod]): string = - case getEnv("REQUEST_METHOD").string + case getEnv("REQUEST_METHOD") of "POST": if methodPost notin allowedMethods: cgiError("'REQUEST_METHOD' 'POST' is not supported") - var L = parseInt(getEnv("CONTENT_LENGTH").string) + var L = parseInt(getEnv("CONTENT_LENGTH")) + if L == 0: + return "" result = newString(L) if readBuffer(stdin, addr(result[0]), L) != L: cgiError("cannot read from stdin") of "GET": if methodGet notin allowedMethods: cgiError("'REQUEST_METHOD' 'GET' is not supported") - result = getEnv("QUERY_STRING").string + result = getEnv("QUERY_STRING") else: if methodNone notin allowedMethods: cgiError("'REQUEST_METHOD' must be 'POST' or 'GET'") -iterator decodeData*(data: string): tuple[key, value: TaintedString] = +iterator decodeData*(data: string): tuple[key, value: string] = ## Reads and decodes CGI data and yields the (name, value) pairs the ## data consists of. - var i = 0 - var name = "" - var value = "" - # decode everything in one pass: - while data[i] != '\0': - setLen(name, 0) # reuse memory - while true: - case data[i] - of '\0': break - of '%': - var x = 0 - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) - inc(i, 2) - add(name, chr(x)) - of '+': add(name, ' ') - of '=', '&': break - else: add(name, data[i]) - inc(i) - if data[i] != '=': cgiError("'=' expected") - inc(i) # skip '=' - setLen(value, 0) # reuse memory - while true: - case data[i] - of '%': - var x = 0 - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) - inc(i, 2) - add(value, chr(x)) - of '+': add(value, ' ') - of '&', '\0': break - else: add(value, data[i]) - inc(i) - yield (name.TaintedString, value.TaintedString) - if data[i] == '&': inc(i) - elif data[i] == '\0': break - else: cgiError("'&' expected") + for (key, value) in uri.decodeQuery(data): + yield (key, value) iterator decodeData*(allowedMethods: set[RequestMethod] = - {methodNone, methodPost, methodGet}): tuple[key, value: TaintedString] = + {methodNone, methodPost, methodGet}): tuple[key, value: string] = ## Reads and decodes CGI data and yields the (name, value) pairs the ## data consists of. If the client does not use a method listed in the - ## `allowedMethods` set, an `ECgi` exception is raised. - var data = getEncodedData(allowedMethods) - if not isNil(data): - for key, value in decodeData(data): - yield (key, value) + ## `allowedMethods` set, a `CgiError` exception is raised. + let data = getEncodedData(allowedMethods) + for (key, value) in uri.decodeQuery(data): + yield (key, value) proc readData*(allowedMethods: set[RequestMethod] = {methodNone, methodPost, methodGet}): StringTableRef = - ## Read CGI data. If the client does not use a method listed in the - ## `allowedMethods` set, an `ECgi` exception is raised. + ## Reads CGI data. If the client does not use a method listed in the + ## `allowedMethods` set, a `CgiError` exception is raised. result = newStringTable() for name, value in decodeData(allowedMethods): - result[name.string] = value.string + result[name] = value + +proc readData*(data: string): StringTableRef = + ## Reads CGI data from a string. + result = newStringTable() + for name, value in decodeData(data): + result[name] = value proc validateData*(data: StringTableRef, validKeys: varargs[string]) = - ## validates data; raises `ECgi` if this fails. This checks that each variable + ## Validates data; raises `CgiError` if this fails. This checks that each variable ## name of the CGI `data` occurs in the `validKeys` array. for key, val in pairs(data): if find(validKeys, key) < 0: cgiError("unknown variable name: " & key) proc getContentLength*(): string = - ## returns contents of the ``CONTENT_LENGTH`` environment variable - return getEnv("CONTENT_LENGTH").string + ## Returns contents of the `CONTENT_LENGTH` environment variable. + return getEnv("CONTENT_LENGTH") proc getContentType*(): string = - ## returns contents of the ``CONTENT_TYPE`` environment variable - return getEnv("CONTENT_Type").string + ## Returns contents of the `CONTENT_TYPE` environment variable. + return getEnv("CONTENT_Type") proc getDocumentRoot*(): string = - ## returns contents of the ``DOCUMENT_ROOT`` environment variable - return getEnv("DOCUMENT_ROOT").string + ## Returns contents of the `DOCUMENT_ROOT` environment variable. + return getEnv("DOCUMENT_ROOT") proc getGatewayInterface*(): string = - ## returns contents of the ``GATEWAY_INTERFACE`` environment variable - return getEnv("GATEWAY_INTERFACE").string + ## Returns contents of the `GATEWAY_INTERFACE` environment variable. + return getEnv("GATEWAY_INTERFACE") proc getHttpAccept*(): string = - ## returns contents of the ``HTTP_ACCEPT`` environment variable - return getEnv("HTTP_ACCEPT").string + ## Returns contents of the `HTTP_ACCEPT` environment variable. + return getEnv("HTTP_ACCEPT") proc getHttpAcceptCharset*(): string = - ## returns contents of the ``HTTP_ACCEPT_CHARSET`` environment variable - return getEnv("HTTP_ACCEPT_CHARSET").string + ## Returns contents of the `HTTP_ACCEPT_CHARSET` environment variable. + return getEnv("HTTP_ACCEPT_CHARSET") proc getHttpAcceptEncoding*(): string = - ## returns contents of the ``HTTP_ACCEPT_ENCODING`` environment variable - return getEnv("HTTP_ACCEPT_ENCODING").string + ## Returns contents of the `HTTP_ACCEPT_ENCODING` environment variable. + return getEnv("HTTP_ACCEPT_ENCODING") proc getHttpAcceptLanguage*(): string = - ## returns contents of the ``HTTP_ACCEPT_LANGUAGE`` environment variable - return getEnv("HTTP_ACCEPT_LANGUAGE").string + ## Returns contents of the `HTTP_ACCEPT_LANGUAGE` environment variable. + return getEnv("HTTP_ACCEPT_LANGUAGE") proc getHttpConnection*(): string = - ## returns contents of the ``HTTP_CONNECTION`` environment variable - return getEnv("HTTP_CONNECTION").string + ## Returns contents of the `HTTP_CONNECTION` environment variable. + return getEnv("HTTP_CONNECTION") proc getHttpCookie*(): string = - ## returns contents of the ``HTTP_COOKIE`` environment variable - return getEnv("HTTP_COOKIE").string + ## Returns contents of the `HTTP_COOKIE` environment variable. + return getEnv("HTTP_COOKIE") proc getHttpHost*(): string = - ## returns contents of the ``HTTP_HOST`` environment variable - return getEnv("HTTP_HOST").string + ## Returns contents of the `HTTP_HOST` environment variable. + return getEnv("HTTP_HOST") proc getHttpReferer*(): string = - ## returns contents of the ``HTTP_REFERER`` environment variable - return getEnv("HTTP_REFERER").string + ## Returns contents of the `HTTP_REFERER` environment variable. + return getEnv("HTTP_REFERER") proc getHttpUserAgent*(): string = - ## returns contents of the ``HTTP_USER_AGENT`` environment variable - return getEnv("HTTP_USER_AGENT").string + ## Returns contents of the `HTTP_USER_AGENT` environment variable. + return getEnv("HTTP_USER_AGENT") proc getPathInfo*(): string = - ## returns contents of the ``PATH_INFO`` environment variable - return getEnv("PATH_INFO").string + ## Returns contents of the `PATH_INFO` environment variable. + return getEnv("PATH_INFO") proc getPathTranslated*(): string = - ## returns contents of the ``PATH_TRANSLATED`` environment variable - return getEnv("PATH_TRANSLATED").string + ## Returns contents of the `PATH_TRANSLATED` environment variable. + return getEnv("PATH_TRANSLATED") proc getQueryString*(): string = - ## returns contents of the ``QUERY_STRING`` environment variable - return getEnv("QUERY_STRING").string + ## Returns contents of the `QUERY_STRING` environment variable. + return getEnv("QUERY_STRING") proc getRemoteAddr*(): string = - ## returns contents of the ``REMOTE_ADDR`` environment variable - return getEnv("REMOTE_ADDR").string + ## Returns contents of the `REMOTE_ADDR` environment variable. + return getEnv("REMOTE_ADDR") proc getRemoteHost*(): string = - ## returns contents of the ``REMOTE_HOST`` environment variable - return getEnv("REMOTE_HOST").string + ## Returns contents of the `REMOTE_HOST` environment variable. + return getEnv("REMOTE_HOST") proc getRemoteIdent*(): string = - ## returns contents of the ``REMOTE_IDENT`` environment variable - return getEnv("REMOTE_IDENT").string + ## Returns contents of the `REMOTE_IDENT` environment variable. + return getEnv("REMOTE_IDENT") proc getRemotePort*(): string = - ## returns contents of the ``REMOTE_PORT`` environment variable - return getEnv("REMOTE_PORT").string + ## Returns contents of the `REMOTE_PORT` environment variable. + return getEnv("REMOTE_PORT") proc getRemoteUser*(): string = - ## returns contents of the ``REMOTE_USER`` environment variable - return getEnv("REMOTE_USER").string + ## Returns contents of the `REMOTE_USER` environment variable. + return getEnv("REMOTE_USER") proc getRequestMethod*(): string = - ## returns contents of the ``REQUEST_METHOD`` environment variable - return getEnv("REQUEST_METHOD").string + ## Returns contents of the `REQUEST_METHOD` environment variable. + return getEnv("REQUEST_METHOD") proc getRequestURI*(): string = - ## returns contents of the ``REQUEST_URI`` environment variable - return getEnv("REQUEST_URI").string + ## Returns contents of the `REQUEST_URI` environment variable. + return getEnv("REQUEST_URI") proc getScriptFilename*(): string = - ## returns contents of the ``SCRIPT_FILENAME`` environment variable - return getEnv("SCRIPT_FILENAME").string + ## Returns contents of the `SCRIPT_FILENAME` environment variable. + return getEnv("SCRIPT_FILENAME") proc getScriptName*(): string = - ## returns contents of the ``SCRIPT_NAME`` environment variable - return getEnv("SCRIPT_NAME").string + ## Returns contents of the `SCRIPT_NAME` environment variable. + return getEnv("SCRIPT_NAME") proc getServerAddr*(): string = - ## returns contents of the ``SERVER_ADDR`` environment variable - return getEnv("SERVER_ADDR").string + ## Returns contents of the `SERVER_ADDR` environment variable. + return getEnv("SERVER_ADDR") proc getServerAdmin*(): string = - ## returns contents of the ``SERVER_ADMIN`` environment variable - return getEnv("SERVER_ADMIN").string + ## Returns contents of the `SERVER_ADMIN` environment variable. + return getEnv("SERVER_ADMIN") proc getServerName*(): string = - ## returns contents of the ``SERVER_NAME`` environment variable - return getEnv("SERVER_NAME").string + ## Returns contents of the `SERVER_NAME` environment variable. + return getEnv("SERVER_NAME") proc getServerPort*(): string = - ## returns contents of the ``SERVER_PORT`` environment variable - return getEnv("SERVER_PORT").string + ## Returns contents of the `SERVER_PORT` environment variable. + return getEnv("SERVER_PORT") proc getServerProtocol*(): string = - ## returns contents of the ``SERVER_PROTOCOL`` environment variable - return getEnv("SERVER_PROTOCOL").string + ## Returns contents of the `SERVER_PROTOCOL` environment variable. + return getEnv("SERVER_PROTOCOL") proc getServerSignature*(): string = - ## returns contents of the ``SERVER_SIGNATURE`` environment variable - return getEnv("SERVER_SIGNATURE").string + ## Returns contents of the `SERVER_SIGNATURE` environment variable. + return getEnv("SERVER_SIGNATURE") proc getServerSoftware*(): string = - ## returns contents of the ``SERVER_SOFTWARE`` environment variable - return getEnv("SERVER_SOFTWARE").string + ## Returns contents of the `SERVER_SOFTWARE` environment variable. + return getEnv("SERVER_SOFTWARE") proc setTestData*(keysvalues: varargs[string]) = - ## fills the appropriate environment variables to test your CGI application. + ## Fills the appropriate environment variables to test your CGI application. ## This can only simulate the 'GET' request method. `keysvalues` should ## provide embedded (name, value)-pairs. Example: - ## - ## .. code-block:: Nim - ## setTestData("name", "Hanz", "password", "12345") + ## ```Nim + ## setTestData("name", "Hanz", "password", "12345") + ## ``` putEnv("REQUEST_METHOD", "GET") var i = 0 var query = "" @@ -342,11 +267,11 @@ proc setTestData*(keysvalues: varargs[string]) = putEnv("QUERY_STRING", query) proc writeContentType*() = - ## call this before starting to send your HTML data to `stdout`. This + ## Calls this before starting to send your HTML data to `stdout`. This ## implements this part of the CGI protocol: - ## - ## .. code-block:: Nim - ## write(stdout, "Content-type: text/html\n\n") + ## ```Nim + ## write(stdout, "Content-type: text/html\n\n") + ## ``` write(stdout, "Content-type: text/html\n\n") proc resetForStacktrace() = @@ -372,11 +297,6 @@ proc setStackTraceStdout*() = ## Makes Nim output stacktraces to stdout, instead of server log. errorMessageWriter = writeErrorMessage -proc setStackTraceNewLine*() {.deprecated.} = - ## Makes Nim output stacktraces to stdout, instead of server log. - ## Depracated alias for setStackTraceStdout. - setStackTraceStdout() - proc setCookie*(name, value: string) = ## Sets a cookie. write(stdout, "Set-Cookie: ", name, "=", value, "\n") @@ -384,17 +304,12 @@ proc setCookie*(name, value: string) = var gcookies {.threadvar.}: StringTableRef -proc getCookie*(name: string): TaintedString = +proc getCookie*(name: string): string = ## Gets a cookie. If no cookie of `name` exists, "" is returned. if gcookies == nil: gcookies = parseCookies(getHttpCookie()) - result = TaintedString(gcookies.getOrDefault(name)) + result = gcookies.getOrDefault(name) proc existsCookie*(name: string): bool = ## Checks if a cookie of `name` exists. if gcookies == nil: gcookies = parseCookies(getHttpCookie()) result = hasKey(gcookies, name) - -when isMainModule: - const test1 = "abc\L+def xyz" - assert encodeUrl(test1) == "abc%0A%2Bdef+xyz" - assert decodeUrl(encodeUrl(test1)) == test1 diff --git a/lib/pure/collections/LockFreeHash.nim b/lib/pure/collections/LockFreeHash.nim deleted file mode 100644 index 1d4471b21..000000000 --- a/lib/pure/collections/LockFreeHash.nim +++ /dev/null @@ -1,609 +0,0 @@ -#nim c -t:-march=i686 --cpu:amd64 --threads:on -d:release lockfreehash.nim - -import unsigned, math, hashes - -#------------------------------------------------------------------------------ -## Memory Utility Functions - -proc newHeap*[T](): ptr T = - result = cast[ptr T](alloc0(sizeof(T))) - -proc copyNew*[T](x: var T): ptr T = - var - size = sizeof(T) - mem = alloc(size) - copyMem(mem, x.addr, size) - return cast[ptr T](mem) - -proc copyTo*[T](val: var T, dest: int) = - copyMem(pointer(dest), val.addr, sizeof(T)) - -proc allocType*[T](): pointer = alloc(sizeof(T)) - -proc newShared*[T](): ptr T = - result = cast[ptr T](allocShared0(sizeof(T))) - -proc copyShared*[T](x: var T): ptr T = - var - size = sizeof(T) - mem = allocShared(size) - copyMem(mem, x.addr, size) - return cast[ptr T](mem) - -#------------------------------------------------------------------------------ -## Pointer arithmetic - -proc `+`*(p: pointer, i: int): pointer {.inline.} = - cast[pointer](cast[int](p) + i) - -const - minTableSize = 8 - reProbeLimit = 12 - minCopyWork = 4096 - intSize = sizeof(int) - - - -when sizeof(int) == 4: # 32bit - type - Raw = range[0..1073741823] - ## The range of uint values that can be stored directly in a value slot - ## when on a 32 bit platform - {.deprecated: [TRaw: Raw].} -elif sizeof(int) == 8: # 64bit - type - Raw = range[0..4611686018427387903] - ## The range of uint values that can be stored directly in a value slot - ## when on a 64 bit platform - {.deprecated: [TRaw: Raw].} -else: - {.error: "unsupported platform".} - -type - Entry = tuple - key: int - value: int - - EntryArr = ptr array[0..10_000_000, Entry] - - PConcTable[K,V] = ptr object {.pure.} - len: int - used: int - active: int - copyIdx: int - copyDone: int - next: PConcTable[K,V] - data: EntryArr -{.deprecated: [TEntry: Entry, TEntryArr: EntryArr.} - -proc setVal[K,V](table: var PConcTable[K,V], key: int, val: int, - expVal: int, match: bool): int - -#------------------------------------------------------------------------------ - -# Create a new table -proc newLFTable*[K,V](size: int = minTableSize): PConcTable[K,V] = - let - dataLen = max(nextPowerOfTwo(size), minTableSize) - dataSize = dataLen*sizeof(Entry) - dataMem = allocShared0(dataSize) - tableSize = 7 * intSize - tableMem = allocShared0(tableSize) - table = cast[PConcTable[K,V]](tableMem) - table.len = dataLen - table.used = 0 - table.active = 0 - table.copyIdx = 0 - table.copyDone = 0 - table.next = nil - table.data = cast[EntryArr](dataMem) - result = table - -#------------------------------------------------------------------------------ - -# Delete a table -proc deleteConcTable[K,V](tbl: PConcTable[K,V]) = - deallocShared(tbl.data) - deallocShared(tbl) - -#------------------------------------------------------------------------------ - -proc `[]`[K,V](table: var PConcTable[K,V], i: int): var Entry {.inline.} = - table.data[i] - -#------------------------------------------------------------------------------ -# State flags stored in ptr - - -proc pack[T](x: T): int {.inline.} = - result = (cast[int](x) shl 2) - #echo("packKey ",cast[int](x) , " -> ", result) - -# Pop the flags off returning a 4 byte aligned ptr to our Key or Val -proc pop(x: int): int {.inline.} = - result = x and 0xFFFFFFFC'i32 - -# Pop the raw value off of our Key or Val -proc popRaw(x: int): int {.inline.} = - result = x shr 2 - -# Pop the flags off returning a 4 byte aligned ptr to our Key or Val -proc popPtr[V](x: int): ptr V {.inline.} = - result = cast[ptr V](pop(x)) - #echo("popPtr " & $x & " -> " & $cast[int](result)) - -# Ghost (sentinel) -# K or V is no longer valid use new table -const Ghost = 0xFFFFFFFC -proc isGhost(x: int): bool {.inline.} = - result = x == 0xFFFFFFFC - -# Tombstone -# applied to V = K is dead -proc isTomb(x: int): bool {.inline.} = - result = (x and 0x00000002) != 0 - -proc setTomb(x: int): int {.inline.} = - result = x or 0x00000002 - -# Prime -# K or V is in new table copied from old -proc isPrime(x: int): bool {.inline.} = - result = (x and 0x00000001) != 0 - -proc setPrime(x: int): int {.inline.} = - result = x or 0x00000001 - -#------------------------------------------------------------------------------ - -##This is for i32 only need to override for i64 -proc hashInt(x: int):int {.inline.} = - var h = uint32(x) #shr 2'u32 - h = h xor (h shr 16'u32) - h *= 0x85ebca6b'u32 - h = h xor (h shr 13'u32) - h *= 0xc2b2ae35'u32 - h = h xor (h shr 16'u32) - result = int(h) - -#------------------------------------------------------------------------------ - -proc resize[K,V](self: PConcTable[K,V]): PConcTable[K,V] = - var next = atomic_load_n(self.next.addr, ATOMIC_RELAXED) - #echo("next = " & $cast[int](next)) - if next != nil: - #echo("A new table already exists, copy in progress") - return next - var - oldLen = atomic_load_n(self.len.addr, ATOMIC_RELAXED) - newTable = newLFTable[K,V](oldLen*2) - success = atomic_compare_exchange_n(self.next.addr, next.addr, newTable, - false, ATOMIC_RELAXED, ATOMIC_RELAXED) - if not success: - echo("someone beat us to it! delete table we just created and return his " & $cast[int](next)) - deleteConcTable(newTable) - return next - else: - echo("Created New Table! " & $cast[int](newTable) & " Size = " & $newTable.len) - return newTable - - -#------------------------------------------------------------------------------ -#proc keyEQ[K](key1: ptr K, key2: ptr K): bool {.inline.} = -proc keyEQ[K](key1: int, key2: int): bool {.inline.} = - result = false - when K is Raw: - if key1 == key2: - result = true - else: - var - p1 = popPtr[K](key1) - p2 = popPtr[K](key2) - if p1 != nil and p2 != nil: - if cast[int](p1) == cast[int](p2): - return true - if p1[] == p2[]: - return true - -#------------------------------------------------------------------------------ - -#proc tableFull(self: var PConcTable[K,V]) : bool {.inline.} = - - -#------------------------------------------------------------------------------ - -proc copySlot[K,V](idx: int, oldTbl: var PConcTable[K,V], newTbl: var PConcTable[K,V]): bool = - #echo("Copy idx " & $idx) - var - oldVal = 0 - oldkey = 0 - ok = false - result = false - #Block the key so no other threads waste time here - while not ok: - ok = atomic_compare_exchange_n(oldTbl[idx].key.addr, oldKey.addr, - setTomb(oldKey), false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #echo("oldKey was = " & $oldKey & " set it to tomb " & $setTomb(oldKey)) - #Prevent new values from appearing in the old table by priming - oldVal = atomic_load_n(oldTbl[idx].value.addr, ATOMIC_RELAXED) - while not isPrime(oldVal): - var box = if oldVal == 0 or isTomb(oldVal) : oldVal.setTomb.setPrime - else: oldVal.setPrime - if atomic_compare_exchange_n(oldTbl[idx].value.addr, oldVal.addr, - box, false, ATOMIC_RELAXED, ATOMIC_RELAXED): - if isPrime(box) and isTomb(box): - return true - oldVal = box - break - #echo("oldVal was = ", oldVal, " set it to prime ", box) - if isPrime(oldVal) and isTomb(oldVal): - #when not (K is Raw): - # deallocShared(popPtr[K](oldKey)) - return false - if isTomb(oldVal): - echo("oldVal is Tomb!!!, should not happen") - if pop(oldVal) != 0: - result = setVal(newTbl, pop(oldKey), pop(oldVal), 0, true) == 0 - if result: - #echo("Copied a Slot! idx= " & $idx & " key= " & $oldKey & " val= " & $oldVal) - else: - #echo("copy slot failed") - # Our copy is done so we disable the old slot - while not ok: - ok = atomic_compare_exchange_n(oldTbl[idx].value.addr, oldVal.addr, - oldVal.setTomb.setPrime , false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #echo("disabled old slot") - #echo"---------------------" - -#------------------------------------------------------------------------------ - -proc promote[K,V](table: var PConcTable[K,V]) = - var - newData = atomic_load_n(table.next.data.addr, ATOMIC_RELAXED) - newLen = atomic_load_n(table.next.len.addr, ATOMIC_RELAXED) - newUsed = atomic_load_n(table.next.used.addr, ATOMIC_RELAXED) - - deallocShared(table.data) - atomic_store_n(table.data.addr, newData, ATOMIC_RELAXED) - atomic_store_n(table.len.addr, newLen, ATOMIC_RELAXED) - atomic_store_n(table.used.addr, newUsed, ATOMIC_RELAXED) - atomic_store_n(table.copyIdx.addr, 0, ATOMIC_RELAXED) - atomic_store_n(table.copyDone.addr, 0, ATOMIC_RELAXED) - deallocShared(table.next) - atomic_store_n(table.next.addr, nil, ATOMIC_RELAXED) - echo("new table swapped!") - -#------------------------------------------------------------------------------ - -proc checkAndPromote[K,V](table: var PConcTable[K,V], workDone: int): bool = - var - oldLen = atomic_load_n(table.len.addr, ATOMIC_RELAXED) - copyDone = atomic_load_n(table.copyDone.addr, ATOMIC_RELAXED) - ok: bool - result = false - if workDone > 0: - #echo("len to copy =" & $oldLen) - #echo("copyDone + workDone = " & $copyDone & " + " & $workDone) - while not ok: - ok = atomic_compare_exchange_n(table.copyDone.addr, copyDone.addr, - copyDone + workDone, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #if ok: echo("set copyDone") - # If the copy is done we can promote this table - if copyDone + workDone >= oldLen: - # Swap new data - #echo("work is done!") - table.promote - result = true - -#------------------------------------------------------------------------------ - -proc copySlotAndCheck[K,V](table: var PConcTable[K,V], idx: int): - PConcTable[K,V] = - var - newTable = cast[PConcTable[K,V]](atomic_load_n(table.next.addr, ATOMIC_RELAXED)) - result = newTable - if newTable != nil and copySlot(idx, table, newTable): - #echo("copied a single slot, idx = " & $idx) - if checkAndPromote(table, 1): return table - - -#------------------------------------------------------------------------------ - -proc helpCopy[K,V](table: var PConcTable[K,V]): PConcTable[K,V] = - var - newTable = cast[PConcTable[K,V]](atomic_load_n(table.next.addr, ATOMIC_RELAXED)) - result = newTable - if newTable != nil: - var - oldLen = atomic_load_n(table.len.addr, ATOMIC_RELAXED) - copyDone = atomic_load_n(table.copyDone.addr, ATOMIC_RELAXED) - copyIdx = 0 - work = min(oldLen, minCopyWork) - #panicStart = -1 - workDone = 0 - if copyDone < oldLen: - var ok: bool - while not ok: - ok = atomic_compare_exchange_n(table.copyIdx.addr, copyIdx.addr, - copyIdx + work, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #echo("copy idx = ", copyIdx) - for i in 0..work-1: - var idx = (copyIdx + i) and (oldLen - 1) - if copySlot(idx, table, newTable): - workDone += 1 - if workDone > 0: - #echo("did work ", workDone, " on thread ", cast[int](myThreadID[pointer]())) - if checkAndPromote(table, workDone): return table - # In case a thread finished all the work then got stalled before promotion - if checkAndPromote(table, 0): return table - - - -#------------------------------------------------------------------------------ - -proc setVal[K,V](table: var PConcTable[K,V], key: int, val: int, - expVal: int, match: bool): int = - #echo("-try set- in table ", " key = ", (popPtr[K](key)[]), " val = ", val) - when K is Raw: - var idx = hashInt(key) - else: - var idx = popPtr[K](key)[].hash - var - nextTable: PConcTable[K,V] - probes = 1 - # spin until we find a key slot or build and jump to next table - while true: - idx = idx and (table.len - 1) - #echo("try set idx = " & $idx & "for" & $key) - var - probedKey = 0 - openKey = atomic_compare_exchange_n(table[idx].key.addr, probedKey.addr, - key, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - if openKey: - if val.isTomb: - #echo("val was tomb, bail, no reason to set an open slot to tomb") - return val - #increment used slots - #echo("found an open slot, total used = " & - #$atomic_add_fetch(table.used.addr, 1, ATOMIC_RELAXED)) - discard atomic_add_fetch(table.used.addr, 1, ATOMIC_RELAXED) - break # We found an open slot - #echo("set idx ", idx, " key = ", key, " probed = ", probedKey) - if keyEQ[K](probedKey, key): - #echo("we found the matching slot") - break # We found a matching slot - if (not(expVal != 0 and match)) and (probes >= reProbeLimit or key.isTomb): - if key.isTomb: echo("Key is Tombstone") - #if probes >= reProbeLimit: echo("Too much probing " & $probes) - #echo("try to resize") - #create next bigger table - nextTable = resize(table) - #help do some copying - #echo("help copy old table to new") - nextTable = helpCopy(table) - #now setVal in the new table instead - #echo("jumping to next table to set val") - return setVal(nextTable, key, val, expVal, match) - else: - idx += 1 - probes += 1 - # Done spinning for a new slot - var oldVal = atomic_load_n(table[idx].value.addr, ATOMIC_RELAXED) - if val == oldVal: - #echo("this val is alredy in the slot") - return oldVal - nextTable = atomic_load_n(table.next.addr, ATOMIC_SEQ_CST) - if nextTable == nil and - ((oldVal == 0 and - (probes >= reProbeLimit or table.used / table.len > 0.8)) or - (isPrime(oldVal))): - if table.used / table.len > 0.8: echo("resize because usage ratio = " & - $(table.used / table.len)) - if isPrime(oldVal): echo("old val isPrime, should be a rare mem ordering event") - nextTable = resize(table) - if nextTable != nil: - #echo("tomb old slot then set in new table") - nextTable = copySlotAndCheck(table,idx) - return setVal(nextTable, key, val, expVal, match) - # Finally ready to add new val to table - while true: - if match and oldVal != expVal: - #echo("set failed, no match oldVal= " & $oldVal & " expVal= " & $expVal) - return oldVal - if atomic_compare_exchange_n(table[idx].value.addr, oldVal.addr, - val, false, ATOMIC_RELEASE, ATOMIC_RELAXED): - #echo("val set at table " & $cast[int](table)) - if expVal != 0: - if (oldVal == 0 or isTomb(oldVal)) and not isTomb(val): - discard atomic_add_fetch(table.active.addr, 1, ATOMIC_RELAXED) - elif not (oldVal == 0 or isTomb(oldVal)) and isTomb(val): - discard atomic_add_fetch(table.active.addr, -1, ATOMIC_RELAXED) - if oldVal == 0 and expVal != 0: - return setTomb(oldVal) - else: return oldVal - if isPrime(oldVal): - nextTable = copySlotAndCheck(table, idx) - return setVal(nextTable, key, val, expVal, match) - -#------------------------------------------------------------------------------ - -proc getVal[K,V](table: var PConcTable[K,V], key: int): int = - #echo("-try get- key = " & $key) - when K is Raw: - var idx = hashInt(key) - else: - var idx = popPtr[K](key)[].hash - #echo("get idx ", idx) - var - probes = 0 - val: int - while true: - idx = idx and (table.len - 1) - var - newTable: PConcTable[K,V] # = atomic_load_n(table.next.addr, ATOMIC_ACQUIRE) - probedKey = atomic_load_n(table[idx].key.addr, ATOMIC_SEQ_CST) - if keyEQ[K](probedKey, key): - #echo("found key after ", probes+1) - val = atomic_load_n(table[idx].value.addr, ATOMIC_ACQUIRE) - if not isPrime(val): - if isTomb(val): - #echo("val was tomb but not prime") - return 0 - else: - #echo("-GotIt- idx = ", idx, " key = ", key, " val ", val ) - return val - else: - newTable = copySlotAndCheck(table, idx) - return getVal(newTable, key) - else: - #echo("probe ", probes, " idx = ", idx, " key = ", key, " found ", probedKey ) - if probes >= reProbeLimit*4 or key.isTomb: - if newTable == nil: - #echo("too many probes and no new table ", key, " ", idx ) - return 0 - else: - newTable = helpCopy(table) - return getVal(newTable, key) - idx += 1 - probes += 1 - -#------------------------------------------------------------------------------ - -#proc set*(table: var PConcTable[Raw,Raw], key: Raw, val: Raw) = -# discard setVal(table, pack(key), pack(key), 0, false) - -#proc set*[V](table: var PConcTable[Raw,V], key: Raw, val: ptr V) = -# discard setVal(table, pack(key), cast[int](val), 0, false) - -proc set*[K,V](table: var PConcTable[K,V], key: var K, val: var V) = - when not (K is Raw): - var newKey = cast[int](copyShared(key)) - else: - var newKey = pack(key) - when not (V is Raw): - var newVal = cast[int](copyShared(val)) - else: - var newVal = pack(val) - var oldPtr = pop(setVal(table, newKey, newVal, 0, false)) - #echo("oldPtr = ", cast[int](oldPtr), " newPtr = ", cast[int](newPtr)) - when not (V is Raw): - if newVal != oldPtr and oldPtr != 0: - deallocShared(cast[ptr V](oldPtr)) - - - -proc get*[K,V](table: var PConcTable[K,V], key: var K): V = - when not (V is Raw): - when not (K is Raw): - return popPtr[V](getVal(table, cast[int](key.addr)))[] - else: - return popPtr[V](getVal(table, pack(key)))[] - else: - when not (K is Raw): - return popRaw(getVal(table, cast[int](key.addr))) - else: - return popRaw(getVal(table, pack(key))) - - - - - - - - - - - -#proc `[]`[K,V](table: var PConcTable[K,V], key: K): PEntry[K,V] {.inline.} = -# getVal(table, key) - -#proc `[]=`[K,V](table: var PConcTable[K,V], key: K, val: V): PEntry[K,V] {.inline.} = -# setVal(table, key, val) - - - - - - -#Tests ---------------------------- -when not defined(testing) and isMainModule: - import locks, times, mersenne - - const - numTests = 100000 - numThreads = 10 - - - - type - TestObj = tuple - thr: int - f0: int - f1: int - - Data = tuple[k: string,v: TestObj] - PDataArr = array[0..numTests-1, Data] - Dict = PConcTable[string,TestObj] - {.deprecated: [TTestObj: TestObj, TData: Data].} - - var - thr: array[0..numThreads-1, Thread[Dict]] - - table = newLFTable[string,TestObj](8) - rand = newMersenneTwister(2525) - - proc createSampleData(len: int): PDataArr = - #result = cast[PDataArr](allocShared0(sizeof(Data)*numTests)) - for i in 0..len-1: - result[i].k = "mark" & $(i+1) - #echo("mark" & $(i+1), " ", hash("mark" & $(i+1))) - result[i].v.thr = 0 - result[i].v.f0 = i+1 - result[i].v.f1 = 0 - #echo("key = " & $(i+1) & " Val ptr = " & $cast[int](result[i].v.addr)) - - - - proc threadProc(tp: Dict) {.thread.} = - var t = cpuTime(); - for i in 1..numTests: - var key = "mark" & $(i) - var got = table.get(key) - got.thr = cast[int](myThreadID[pointer]()) - got.f1 = got.f1 + 1 - table.set(key, got) - t = cpuTime() - t - echo t - - - var testData = createSampleData(numTests) - - for i in 0..numTests-1: - table.set(testData[i].k, testData[i].v) - - var i = 0 - while i < numThreads: - createThread(thr[i], threadProc, table) - i += 1 - - joinThreads(thr) - - - - - - var fails = 0 - - for i in 0..numTests-1: - var got = table.get(testData[i].k) - if got.f0 != i+1 or got.f1 != numThreads: - fails += 1 - echo(got) - - echo("Failed read or write = ", fails) - - - #for i in 1..numTests: - # echo(i, " = ", hashInt(i) and 8191) - - deleteConcTable(table) diff --git a/lib/pure/collections/chains.nim b/lib/pure/collections/chains.nim new file mode 100644 index 000000000..6b2ecd272 --- /dev/null +++ b/lib/pure/collections/chains.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Template based implementation of singly and doubly linked lists. +## The involved types should have 'prev' or 'next' fields and the +## list header should have 'head' or 'tail' fields. + +template prepend*(header, node) = + when compiles(header.head): + when compiles(node.prev): + if header.head != nil: + header.head.prev = node + node.next = header.head + header.head = node + when compiles(header.tail): + if header.tail == nil: + header.tail = node + +template append*(header, node) = + when compiles(header.head): + if header.head == nil: + header.head = node + when compiles(header.tail): + when compiles(node.prev): + node.prev = header.tail + if header.tail != nil: + header.tail.next = node + header.tail = node + +template unlink*(header, node) = + if node.next != nil: + node.next.prev = node.prev + if node.prev != nil: + node.prev.next = node.next + if header.head == node: + header.head = node.prev + if header.tail == node: + header.tail = node.next diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index bb234565b..24257dacb 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -8,15 +8,41 @@ # ## This module implements a `crit bit tree`:idx: which is an efficient -## container for a set or a mapping of strings. Based on the excellent paper -## by Adam Langley. - -include "system/inclrtl" +## container for a sorted set of strings, or for a sorted mapping of strings. Based on the +## [excellent paper by Adam Langley](https://www.imperialviolet.org/binary/critbit.pdf). +## (A crit bit tree is a form of `radix tree`:idx: or `patricia trie`:idx:.) + +runnableExamples: + from std/sequtils import toSeq + + var critbitAsSet: CritBitTree[void] = ["kitten", "puppy"].toCritBitTree + doAssert critbitAsSet.len == 2 + critbitAsSet.incl("") + doAssert "" in critbitAsSet + critbitAsSet.excl("") + doAssert "" notin critbitAsSet + doAssert toSeq(critbitAsSet.items) == @["kitten", "puppy"] + let same = ["puppy", "kitten", "puppy"].toCritBitTree + doAssert toSeq(same.keys) == toSeq(critbitAsSet.keys) + + var critbitAsDict: CritBitTree[int] = {"key1": 42}.toCritBitTree + doAssert critbitAsDict.len == 1 + critbitAsDict["key2"] = 0 + doAssert "key2" in critbitAsDict + doAssert critbitAsDict["key2"] == 0 + critbitAsDict.excl("key1") + doAssert "key1" notin critbitAsDict + doAssert toSeq(critbitAsDict.pairs) == @[("key2", 0)] + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions type - NodeObj[T] = object {.acyclic.} + NodeObj[T] {.acyclic.} = object byte: int ## byte index of the difference - otherbits: char + otherBits: char case isLeaf: bool of false: child: array[0..1, ref NodeObj[T]] of true: @@ -27,15 +53,17 @@ type Node[T] = ref NodeObj[T] CritBitTree*[T] = object ## The crit bit tree can either be used ## as a mapping from strings to - ## some type ``T`` or as a set of - ## strings if ``T`` is void. + ## some type `T` or as a set of + ## strings if `T` is `void`. root: Node[T] count: int -{.deprecated: [TCritBitTree: CritBitTree].} +func len*[T](c: CritBitTree[T]): int {.inline.} = + ## Returns the number of elements in `c` in O(1). + runnableExamples: + let c = ["key1", "key2"].toCritBitTree + doAssert c.len == 2 -proc len*[T](c: CritBitTree[T]): int = - ## returns the number of elements in `c` in O(1). result = c.count proc rawGet[T](c: CritBitTree[T], key: string): Node[T] = @@ -48,19 +76,22 @@ proc rawGet[T](c: CritBitTree[T], key: string): Node[T] = else: return if it.key == key: it else: nil -proc contains*[T](c: CritBitTree[T], key: string): bool {.inline.} = - ## returns true iff `c` contains the given `key`. +func contains*[T](c: CritBitTree[T], key: string): bool {.inline.} = + ## Returns true if `c` contains the given `key`. + runnableExamples: + var c: CritBitTree[void] + incl(c, "key") + doAssert c.contains("key") + result = rawGet(c, key) != nil -proc hasKey*[T](c: CritBitTree[T], key: string): bool {.inline.} = - ## alias for `contains`. +func hasKey*[T](c: CritBitTree[T], key: string): bool {.inline.} = + ## Alias for `contains <#contains,CritBitTree[T],string>`_. result = rawGet(c, key) != nil proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = if c.root == nil: - new c.root - c.root.isleaf = true - c.root.key = key + c.root = Node[T](isleaf: true, key: key) result = c.root else: var it = c.root @@ -72,26 +103,25 @@ proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = var newOtherBits = 0 var newByte = 0 block blockX: - while newbyte < key.len: - if it.key[newbyte] != key[newbyte]: - newotherbits = it.key[newbyte].ord xor key[newbyte].ord + while newByte < key.len: + let ch = if newByte < it.key.len: it.key[newByte] else: '\0' + if ch != key[newByte]: + newOtherBits = ch.ord xor key[newByte].ord break blockX - inc newbyte - if it.key[newbyte] != '\0': - newotherbits = it.key[newbyte].ord + inc newByte + if newByte < it.key.len: + newOtherBits = it.key[newByte].ord else: return it while (newOtherBits and (newOtherBits-1)) != 0: newOtherBits = newOtherBits and (newOtherBits-1) newOtherBits = newOtherBits xor 255 - let ch = it.key[newByte] + let ch = if newByte < it.key.len: it.key[newByte] else: '\0' let dir = (1 + (ord(ch) or newOtherBits)) shr 8 var inner: Node[T] new inner - new result - result.isLeaf = true - result.key = key + result = Node[T](isLeaf: true, key: key) inner.otherBits = chr(newOtherBits) inner.byte = newByte inner.child[1 - dir] = result @@ -109,9 +139,87 @@ proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = wherep[] = inner inc c.count -proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = - ## returns true iff `c` contains the given `key`. If the key does not exist - ## ``c[key] = val`` is performed. +func exclImpl[T](c: var CritBitTree[T], key: string): int = + var p = c.root + var wherep = addr(c.root) + var whereq: ptr Node[T] = nil + if p == nil: return c.count + var dir = 0 + var q: Node[T] + while not p.isLeaf: + whereq = wherep + q = p + let ch = if p.byte < key.len: key[p.byte] else: '\0' + dir = (1 + (ch.ord or p.otherBits.ord)) shr 8 + wherep = addr(p.child[dir]) + p = wherep[] + if p.key == key: + # else: not in tree at all + if whereq == nil: + c.root = nil + else: + whereq[] = q.child[1 - dir] + dec c.count + + return c.count + +proc excl*[T](c: var CritBitTree[T], key: string) = + ## Removes `key` (and its associated value) from the set `c`. + ## If the `key` does not exist, nothing happens. + ## + ## **See also:** + ## * `incl proc <#incl,CritBitTree[void],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + runnableExamples: + var c: CritBitTree[void] + incl(c, "key") + excl(c, "key") + doAssert not c.contains("key") + + discard exclImpl(c, key) + +proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = + ## Returns true if `c` does not contain the given `key`. If the key + ## does exist, `c.excl(key)` is performed. + ## + ## **See also:** + ## * `excl proc <#excl,CritBitTree[T],string>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[T],string,T>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[void],string>`_ + runnableExamples: + block: + var c: CritBitTree[void] + doAssert c.missingOrExcl("key") + block: + var c: CritBitTree[void] + incl(c, "key") + doAssert not c.missingOrExcl("key") + doAssert not c.contains("key") + + let oldCount = c.count + discard exclImpl(c, key) + result = c.count == oldCount + +proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: sink T): bool = + ## Returns true if `c` contains the given `key`. If the key does not exist, + ## `c[key] = val` is performed. + ## + ## **See also:** + ## * `incl proc <#incl,CritBitTree[void],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[void],string>`_ + ## * `missingOrExcl proc <#missingOrExcl,CritBitTree[T],string>`_ + runnableExamples: + block: + var c: CritBitTree[int] + doAssert not c.containsOrIncl("key", 42) + doAssert c.contains("key") + block: + var c: CritBitTree[int] + incl(c, "key", 21) + doAssert c.containsOrIncl("key", 42) + doAssert c["key"] == 21 + let oldCount = c.count var n = rawInsert(c, key) result = c.count == oldCount @@ -119,80 +227,101 @@ proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = if not result: n.val = val proc containsOrIncl*(c: var CritBitTree[void], key: string): bool = - ## returns true iff `c` contains the given `key`. If the key does not exist + ## Returns true if `c` contains the given `key`. If the key does not exist, ## it is inserted into `c`. + ## + ## **See also:** + ## * `incl proc <#incl,CritBitTree[void],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[T],string,T>`_ + ## * `missingOrExcl proc <#missingOrExcl,CritBitTree[T],string>`_ + runnableExamples: + block: + var c: CritBitTree[void] + doAssert not c.containsOrIncl("key") + doAssert c.contains("key") + block: + var c: CritBitTree[void] + incl(c, "key") + doAssert c.containsOrIncl("key") + let oldCount = c.count - var n = rawInsert(c, key) + discard rawInsert(c, key) result = c.count == oldCount -proc inc*(c: var CritBitTree[int]; key: string) = - ## counts the 'key'. - let oldCount = c.count +proc inc*(c: var CritBitTree[int]; key: string, val: int = 1) = + ## Increments `c[key]` by `val`. + runnableExamples: + var c: CritBitTree[int] + c["key"] = 1 + inc(c, "key") + doAssert c["key"] == 2 + var n = rawInsert(c, key) - if c.count == oldCount: - # not a new key: - inc n.val + inc n.val, val proc incl*(c: var CritBitTree[void], key: string) = - ## includes `key` in `c`. + ## Includes `key` in `c`. + ## + ## **See also:** + ## * `excl proc <#excl,CritBitTree[T],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + runnableExamples: + var c: CritBitTree[void] + incl(c, "key") + doAssert c.hasKey("key") + discard rawInsert(c, key) -proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = - ## puts a (key, value)-pair into `t`. +proc incl*[T](c: var CritBitTree[T], key: string, val: sink T) = + ## Inserts `key` with value `val` into `c`. + ## + ## **See also:** + ## * `excl proc <#excl,CritBitTree[T],string>`_ + ## * `incl proc <#incl,CritBitTree[void],string>`_ + runnableExamples: + var c: CritBitTree[int] + incl(c, "key", 42) + doAssert c["key"] == 42 + + var n = rawInsert(c, key) + n.val = val + +proc `[]=`*[T](c: var CritBitTree[T], key: string, val: sink T) = + ## Alias for `incl <#incl,CritBitTree[T],string,T>`_. + ## + ## **See also:** + ## * `[] proc <#[],CritBitTree[T],string>`_ + ## * `[] proc <#[],CritBitTree[T],string_2>`_ var n = rawInsert(c, key) n.val = val -template get[T](c: CritBitTree[T], key: string): T {.immediate.} = +template get[T](c: CritBitTree[T], key: string): T = let n = rawGet(c, key) - if n != nil: result = n.val - else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + if n == nil: + raise newException(KeyError, "key not found: " & key) -proc `[]`*[T](c: CritBitTree[T], key: string): T {.inline, deprecatedGet.} = - ## retrieves the value at ``c[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. - get(c, key) + n.val -proc `[]`*[T](c: var CritBitTree[T], key: string): var T {.inline, - deprecatedGet.} = - ## retrieves the value at ``c[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. +func `[]`*[T](c: CritBitTree[T], key: string): lent T {.inline.} = + ## Retrieves the value at `c[key]`. If `key` is not in `t`, the + ## `KeyError` exception is raised. One can check with `hasKey` whether + ## the key exists. + ## + ## **See also:** + ## * `[] proc <#[],CritBitTree[T],string_2>`_ + ## * `[]= proc <#[]=,CritBitTree[T],string,T>`_ get(c, key) -proc mget*[T](c: var CritBitTree[T], key: string): var T {.inline, deprecated.} = - ## retrieves the value at ``c[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. +func `[]`*[T](c: var CritBitTree[T], key: string): var T {.inline.} = + ## Retrieves the value at `c[key]`. The value can be modified. + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## + ## **See also:** + ## * `[] proc <#[],CritBitTree[T],string>`_ + ## * `[]= proc <#[]=,CritBitTree[T],string,T>`_ get(c, key) -proc excl*[T](c: var CritBitTree[T], key: string) = - ## removes `key` (and its associated value) from the set `c`. - ## If the `key` does not exist, nothing happens. - var p = c.root - var wherep = addr(c.root) - var whereq: ptr Node[T] = nil - if p == nil: return - var dir = 0 - var q: Node[T] - while not p.isLeaf: - whereq = wherep - q = p - let ch = if p.byte < key.len: key[p.byte] else: '\0' - dir = (1 + (ch.ord or p.otherBits.ord)) shr 8 - wherep = addr(p.child[dir]) - p = wherep[] - if p.key == key: - # else: not in tree at all - if whereq == nil: - c.root = nil - else: - whereq[] = q.child[1 - dir] - dec c.count - iterator leaves[T](n: Node[T]): Node[T] = if n != nil: # XXX actually we could compute the necessary stack size in advance: @@ -207,32 +336,64 @@ iterator leaves[T](n: Node[T]): Node[T] = yield it iterator keys*[T](c: CritBitTree[T]): string = - ## yields all keys in lexicographical order. + ## Yields all keys in lexicographical order. + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 1, "key2": 2}.toCritBitTree + doAssert toSeq(c.keys) == @["key1", "key2"] + for x in leaves(c.root): yield x.key -iterator values*[T](c: CritBitTree[T]): T = - ## yields all values of `c` in the lexicographical order of the +iterator values*[T](c: CritBitTree[T]): lent T = + ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. + ## + ## **See also:** + ## * `mvalues iterator <#mvalues.i,CritBitTree[T]>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 1, "key2": 2}.toCritBitTree + doAssert toSeq(c.values) == @[1, 2] + for x in leaves(c.root): yield x.val iterator mvalues*[T](c: var CritBitTree[T]): var T = - ## yields all values of `c` in the lexicographical order of the + ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. The values can be modified. + ## + ## **See also:** + ## * `values iterator <#values.i,CritBitTree[T]>`_ for x in leaves(c.root): yield x.val iterator items*[T](c: CritBitTree[T]): string = - ## yields all keys in lexicographical order. + ## Alias for `keys <#keys.i,CritBitTree[T]>`_. for x in leaves(c.root): yield x.key iterator pairs*[T](c: CritBitTree[T]): tuple[key: string, val: T] = - ## yields all (key, value)-pairs of `c`. + ## Yields all `(key, value)`-pairs of `c` in the lexicographical order of the + ## corresponding keys. + ## + ## **See also:** + ## * `mpairs iterator <#mpairs.i,CritBitTree[T]>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 1, "key2": 2}.toCritBitTree + doAssert toSeq(c.pairs) == @[(key: "key1", val: 1), (key: "key2", val: 2)] + for x in leaves(c.root): yield (x.key, x.val) iterator mpairs*[T](c: var CritBitTree[T]): tuple[key: string, val: var T] = - ## yields all (key, value)-pairs of `c`. The yielded values can be modified. + ## Yields all `(key, value)`-pairs of `c` in the lexicographical order of the + ## corresponding keys. The yielded values can be modified. + ## + ## **See also:** + ## * `pairs iterator <#pairs.i,CritBitTree[T]>`_ for x in leaves(c.root): yield (x.key, x.val) -proc allprefixedAux[T](c: CritBitTree[T], key: string; longestMatch: bool): Node[T] = +proc allprefixedAux[T](c: CritBitTree[T], key: string): Node[T] = var p = c.root var top = p if p != nil: @@ -242,58 +403,83 @@ proc allprefixedAux[T](c: CritBitTree[T], key: string; longestMatch: bool): Node let dir = (1 + (ch.ord or p.otherBits.ord)) shr 8 p = p.child[dir] if q.byte < key.len: top = p - if not longestMatch: - for i in 0 .. <key.len: - if p.key[i] != key[i]: return + for i in 0 ..< key.len: + if i >= p.key.len or p.key[i] != key[i]: return result = top -iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string; - longestMatch=false): string = - ## yields all keys starting with `prefix`. If `longestMatch` is true, - ## the longest match is returned, it doesn't have to be a complete match then. - let top = allprefixedAux(c, prefix, longestMatch) - for x in leaves(top): yield x.key +iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = + ## Yields all keys starting with `prefix`. + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 42, "key2": 43}.toCritBitTree + doAssert toSeq(c.keysWithPrefix("key")) == @["key1", "key2"] -iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string; - longestMatch=false): string = - ## yields all keys starting with `prefix`. - let top = allprefixedAux(c, prefix, longestMatch) + let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string; - longestMatch=false): T = - ## yields all values of `c` starting with `prefix` of the +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): lent T = + ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. - let top = allprefixedAux(c, prefix, longestMatch) + ## + ## **See also:** + ## * `mvaluesWithPrefix iterator <#mvaluesWithPrefix.i,CritBitTree[T],string>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 42, "key2": 43}.toCritBitTree + doAssert toSeq(c.valuesWithPrefix("key")) == @[42, 43] + + let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.val -iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string; - longestMatch=false): var T = - ## yields all values of `c` starting with `prefix` of the +iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string): var T = + ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. The values can be modified. - let top = allprefixedAux(c, prefix, longestMatch) + ## + ## **See also:** + ## * `valuesWithPrefix iterator <#valuesWithPrefix.i,CritBitTree[T],string>`_ + let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.val +iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string): string = + ## Alias for `keysWithPrefix <#keysWithPrefix.i,CritBitTree[T],string>`_. + let top = allprefixedAux(c, prefix) + for x in leaves(top): yield x.key + iterator pairsWithPrefix*[T](c: CritBitTree[T], - prefix: string; - longestMatch=false): tuple[key: string, val: T] = - ## yields all (key, value)-pairs of `c` starting with `prefix`. - let top = allprefixedAux(c, prefix, longestMatch) + prefix: string): tuple[key: string, val: T] = + ## Yields all (key, value)-pairs of `c` starting with `prefix`. + ## + ## **See also:** + ## * `mpairsWithPrefix iterator <#mpairsWithPrefix.i,CritBitTree[T],string>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 42, "key2": 43}.toCritBitTree + doAssert toSeq(c.pairsWithPrefix("key")) == @[(key: "key1", val: 42), (key: "key2", val: 43)] + + let top = allprefixedAux(c, prefix) for x in leaves(top): yield (x.key, x.val) iterator mpairsWithPrefix*[T](c: var CritBitTree[T], - prefix: string; - longestMatch=false): tuple[key: string, val: var T] = - ## yields all (key, value)-pairs of `c` starting with `prefix`. + prefix: string): tuple[key: string, val: var T] = + ## Yields all (key, value)-pairs of `c` starting with `prefix`. ## The yielded values can be modified. - let top = allprefixedAux(c, prefix, longestMatch) + ## + ## **See also:** + ## * `pairsWithPrefix iterator <#pairsWithPrefix.i,CritBitTree[T],string>`_ + let top = allprefixedAux(c, prefix) for x in leaves(top): yield (x.key, x.val) -proc `$`*[T](c: CritBitTree[T]): string = - ## turns `c` into a string representation. Example outputs: - ## ``{keyA: value, keyB: value}``, ``{:}`` - ## If `T` is void the outputs look like: - ## ``{keyA, keyB}``, ``{}``. +func `$`*[T](c: CritBitTree[T]): string = + ## Turns `c` into a string representation. + runnableExamples: + doAssert $CritBitTree[int].default == "{:}" + doAssert $toCritBitTree({"key1": 1, "key2": 2}) == """{"key1": 1, "key2": 2}""" + doAssert $CritBitTree[void].default == "{}" + doAssert $toCritBitTree(["key1", "key2"]) == """{"key1", "key2"}""" + if c.len == 0: when T is void: result = "{}" @@ -307,28 +493,45 @@ proc `$`*[T](c: CritBitTree[T]): string = const avgItemLen = 16 result = newStringOfCap(c.count * avgItemLen) result.add("{") - for key, val in pairs(c): - if result.len > 1: result.add(", ") - result.add($key) - when T isnot void: + when T is void: + for key in keys(c): + if result.len > 1: result.add(", ") + result.addQuoted(key) + else: + for key, val in pairs(c): + if result.len > 1: result.add(", ") + result.addQuoted(key) result.add(": ") - result.add($val) + result.addQuoted(val) result.add("}") -when isMainModule: - import sequtils - - var r: CritBitTree[void] - r.incl "abc" - r.incl "xyz" - r.incl "def" - r.incl "definition" - r.incl "prefix" - - doAssert r.contains"def" - - r.excl "def" - - assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"] - - assert toSeq(r.itemsWithPrefix("de")) == @["definition"] +func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} = + ## Returns the length of the longest common prefix of all keys in `c`. + ## If `c` is empty, returns 0. + runnableExamples: + var c: CritBitTree[void] + doAssert c.commonPrefixLen == 0 + incl(c, "key1") + doAssert c.commonPrefixLen == 4 + incl(c, "key2") + doAssert c.commonPrefixLen == 3 + + if c.root != nil: + if c.root.isLeaf: len(c.root.key) + else: c.root.byte + else: 0 + +proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = + ## Creates a new `CritBitTree` that contains the given `pairs`. + runnableExamples: + doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string] + doAssert {"a": 0, "b": 1, "c": 2}.toCritBitTree is CritBitTree[int] + + for item in pairs: result.incl item[0], item[1] + +proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} = + ## Creates a new `CritBitTree` that contains the given `items`. + runnableExamples: + doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void] + + for item in items: result.incl item diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim new file mode 100644 index 000000000..d2b0099f2 --- /dev/null +++ b/lib/pure/collections/deques.nim @@ -0,0 +1,480 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## An implementation of a `deque`:idx: (double-ended queue). +## The underlying implementation uses a `seq`. +## +## .. note:: None of the procs that get an individual value from the deque should be used +## on an empty deque. +## +## If compiled with the `boundChecks` option, those procs will raise an `IndexDefect` +## on such access. This should not be relied upon, as `-d:danger` or `--checks:off` will +## disable those checks and then the procs may return garbage or crash the program. +## +## As such, a check to see if the deque is empty is needed before any +## access, unless your program logic guarantees it indirectly. + +runnableExamples: + var a = [10, 20, 30, 40].toDeque + + doAssertRaises(IndexDefect, echo a[4]) + + a.addLast(50) + assert $a == "[10, 20, 30, 40, 50]" + + assert a.peekFirst == 10 + assert a.peekLast == 50 + assert len(a) == 5 + + assert a.popFirst == 10 + assert a.popLast == 50 + assert len(a) == 3 + + a.addFirst(11) + a.addFirst(22) + a.addFirst(33) + assert $a == "[33, 22, 11, 20, 30, 40]" + + a.shrink(fromFirst = 1, fromLast = 2) + assert $a == "[22, 11, 20]" + +## See also +## ======== +## * `lists module <lists.html>`_ for singly and doubly linked lists and rings + +import std/private/since + +import std/[assertions, hashes, math] + +type + Deque*[T] = object + ## A double-ended queue backed with a ringed `seq` buffer. + ## + ## To initialize an empty deque, + ## use the `initDeque proc <#initDeque,int>`_. + data: seq[T] + + # `head` and `tail` are masked only when accessing an element of `data` + # so that `tail - head == data.len` when the deque is full. + # They are uint so that incrementing/decrementing them doesn't cause + # over/underflow. You can get a number of items with `tail - head` + # even if `tail` or `head` is wraps around and `tail < head`, because + # `tail - head == (uint.high + 1 + tail) - head` when `tail < head`. + head, tail: uint + +const + defaultInitialSize* = 4 + +template initImpl(result: typed, initialSize: int) = + let correctSize = nextPowerOfTwo(initialSize) + newSeq(result.data, correctSize) + +template checkIfInitialized(deq: typed) = + if deq.data.len == 0: + initImpl(deq, defaultInitialSize) + +func mask[T](deq: Deque[T]): uint {.inline.} = + uint(deq.data.len) - 1 + +proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = + ## Creates a new empty deque. + ## + ## Optionally, the initial capacity can be reserved via `initialSize` + ## as a performance optimization + ## (default: `defaultInitialSize <#defaultInitialSize>`_). + ## The length of a newly created deque will still be 0. + ## + ## **See also:** + ## * `toDeque proc <#toDeque,openArray[T]>`_ + result.initImpl(initialSize) + +func len*[T](deq: Deque[T]): int {.inline.} = + ## Returns the number of elements of `deq`. + int(deq.tail - deq.head) + +template emptyCheck(deq) = + # Bounds check for the regular deque access. + when compileOption("boundChecks"): + if unlikely(deq.len < 1): + raise newException(IndexDefect, "Empty deque.") + +template xBoundsCheck(deq, i) = + # Bounds check for the array like accesses. + when compileOption("boundChecks"): # `-d:danger` or `--checks:off` should disable this. + if unlikely(i >= deq.len): # x < deq.low is taken care by the Natural parameter + raise newException(IndexDefect, + "Out of bounds: " & $i & " > " & $(deq.len - 1)) + if unlikely(i < 0): # when used with BackwardsIndex + raise newException(IndexDefect, + "Out of bounds: " & $i & " < 0") + +proc `[]`*[T](deq: Deque[T], i: Natural): lent T {.inline.} = + ## Accesses the `i`-th element of `deq`. + runnableExamples: + let a = [10, 20, 30, 40, 50].toDeque + assert a[0] == 10 + assert a[3] == 40 + doAssertRaises(IndexDefect, echo a[8]) + + xBoundsCheck(deq, i) + return deq.data[(deq.head + i.uint) and deq.mask] + +proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = + ## Accesses the `i`-th element of `deq` and returns a mutable + ## reference to it. + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + inc(a[0]) + assert a[0] == 11 + + xBoundsCheck(deq, i) + return deq.data[(deq.head + i.uint) and deq.mask] + +proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = + ## Sets the `i`-th element of `deq` to `val`. + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + a[0] = 99 + a[3] = 66 + assert $a == "[99, 20, 30, 66, 50]" + + checkIfInitialized(deq) + xBoundsCheck(deq, i) + deq.data[(deq.head + i.uint) and deq.mask] = val + +proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): lent T {.inline.} = + ## Accesses the backwards indexed `i`-th element. + ## + ## `deq[^1]` is the last element. + runnableExamples: + let a = [10, 20, 30, 40, 50].toDeque + assert a[^1] == 50 + assert a[^4] == 20 + doAssertRaises(IndexDefect, echo a[^9]) + + xBoundsCheck(deq, deq.len - int(i)) + return deq[deq.len - int(i)] + +proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = + ## Accesses the backwards indexed `i`-th element and returns a mutable + ## reference to it. + ## + ## `deq[^1]` is the last element. + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + inc(a[^1]) + assert a[^1] == 51 + + xBoundsCheck(deq, deq.len - int(i)) + return deq[deq.len - int(i)] + +proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: sink T) {.inline.} = + ## Sets the backwards indexed `i`-th element of `deq` to `x`. + ## + ## `deq[^1]` is the last element. + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + a[^1] = 99 + a[^3] = 77 + assert $a == "[10, 20, 77, 40, 99]" + + checkIfInitialized(deq) + xBoundsCheck(deq, deq.len - int(i)) + deq[deq.len - int(i)] = x + +iterator items*[T](deq: Deque[T]): lent T = + ## Yields every element of `deq`. + ## + ## **See also:** + ## * `mitems iterator <#mitems.i,Deque[T]>`_ + runnableExamples: + from std/sequtils import toSeq + + let a = [10, 20, 30, 40, 50].toDeque + assert toSeq(a.items) == @[10, 20, 30, 40, 50] + + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] + +iterator mitems*[T](deq: var Deque[T]): var T = + ## Yields every element of `deq`, which can be modified. + ## + ## **See also:** + ## * `items iterator <#items.i,Deque[T]>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + for x in mitems(a): + x = 5 * x - 1 + assert $a == "[49, 99, 149, 199, 249]" + + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] + +iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = + ## Yields every `(position, value)`-pair of `deq`. + runnableExamples: + from std/sequtils import toSeq + + let a = [10, 20, 30].toDeque + assert toSeq(a.pairs) == @[(0, 10), (1, 20), (2, 30)] + + for c in 0 ..< deq.len: + yield (c, deq.data[(deq.head + c.uint) and deq.mask]) + +proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = + ## Returns true if `item` is in `deq` or false if not found. + ## + ## Usually used via the `in` operator. + ## It is the equivalent of `deq.find(item) >= 0`. + runnableExamples: + let q = [7, 9].toDeque + assert 7 in q + assert q.contains(7) + assert 8 notin q + + for e in deq: + if e == item: return true + return false + +proc expandIfNeeded[T](deq: var Deque[T]) = + checkIfInitialized(deq) + let cap = deq.data.len + assert deq.len <= cap + if unlikely(deq.len == cap): + var n = newSeq[T](cap * 2) + var i = 0 + for x in mitems(deq): + when nimvm: n[i] = x # workaround for VM bug + else: n[i] = move(x) + inc i + deq.data = move(n) + deq.tail = cap.uint + deq.head = 0 + +proc addFirst*[T](deq: var Deque[T], item: sink T) = + ## Adds an `item` to the beginning of `deq`. + ## + ## **See also:** + ## * `addLast proc <#addLast,Deque[T],sinkT>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addFirst(10 * i) + assert $a == "[50, 40, 30, 20, 10]" + + expandIfNeeded(deq) + dec deq.head + deq.data[deq.head and deq.mask] = item + +proc addLast*[T](deq: var Deque[T], item: sink T) = + ## Adds an `item` to the end of `deq`. + ## + ## **See also:** + ## * `addFirst proc <#addFirst,Deque[T],sinkT>`_ + runnableExamples: + var a = initDeque[int]() + for i in 1 .. 5: + a.addLast(10 * i) + assert $a == "[10, 20, 30, 40, 50]" + + expandIfNeeded(deq) + deq.data[deq.tail and deq.mask] = item + inc deq.tail + +proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = + ## Creates a new deque that contains the elements of `x` (in the same order). + ## + ## **See also:** + ## * `initDeque proc <#initDeque,int>`_ + runnableExamples: + let a = toDeque([7, 8, 9]) + assert len(a) == 3 + assert $a == "[7, 8, 9]" + + result.initImpl(x.len) + for item in items(x): + result.addLast(item) + +proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = + ## Returns the first element of `deq`, but does not remove it from the deque. + ## + ## **See also:** + ## * `peekFirst proc <#peekFirst,Deque[T]_2>`_ which returns a mutable reference + ## * `peekLast proc <#peekLast,Deque[T]>`_ + runnableExamples: + let a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + assert a.peekFirst == 10 + assert len(a) == 5 + + emptyCheck(deq) + result = deq.data[deq.head and deq.mask] + +proc peekLast*[T](deq: Deque[T]): lent T {.inline.} = + ## Returns the last element of `deq`, but does not remove it from the deque. + ## + ## **See also:** + ## * `peekLast proc <#peekLast,Deque[T]_2>`_ which returns a mutable reference + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + runnableExamples: + let a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + assert a.peekLast == 50 + assert len(a) == 5 + + emptyCheck(deq) + result = deq.data[(deq.tail - 1) and deq.mask] + +proc peekFirst*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = + ## Returns a mutable reference to the first element of `deq`, + ## but does not remove it from the deque. + ## + ## **See also:** + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `peekLast proc <#peekLast,Deque[T]_2>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + a.peekFirst() = 99 + assert $a == "[99, 20, 30, 40, 50]" + + emptyCheck(deq) + result = deq.data[deq.head and deq.mask] + +proc peekLast*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = + ## Returns a mutable reference to the last element of `deq`, + ## but does not remove it from the deque. + ## + ## **See also:** + ## * `peekFirst proc <#peekFirst,Deque[T]_2>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + a.peekLast() = 99 + assert $a == "[10, 20, 30, 40, 99]" + + emptyCheck(deq) + result = deq.data[(deq.tail - 1) and deq.mask] + +template destroy(x: untyped) = + reset(x) + +proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = + ## Removes and returns the first element of the `deq`. + ## + ## See also: + ## * `popLast proc <#popLast,Deque[T]>`_ + ## * `shrink proc <#shrink,Deque[T],int,int>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + assert a.popFirst == 10 + assert $a == "[20, 30, 40, 50]" + + emptyCheck(deq) + result = move deq.data[deq.head and deq.mask] + inc deq.head + +proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = + ## Removes and returns the last element of the `deq`. + ## + ## **See also:** + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `shrink proc <#shrink,Deque[T],int,int>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + assert a.popLast == 50 + assert $a == "[10, 20, 30, 40]" + + emptyCheck(deq) + dec deq.tail + result = move deq.data[deq.tail and deq.mask] + +proc clear*[T](deq: var Deque[T]) {.inline.} = + ## Resets the deque so that it is empty. + ## + ## **See also:** + ## * `shrink proc <#shrink,Deque[T],int,int>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + clear(a) + assert len(a) == 0 + + for el in mitems(deq): destroy(el) + deq.tail = deq.head + +proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = + ## Removes `fromFirst` elements from the front of the deque and + ## `fromLast` elements from the back. + ## + ## If the supplied number of elements exceeds the total number of elements + ## in the deque, the deque will remain empty. + ## + ## **See also:** + ## * `clear proc <#clear,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" + a.shrink(fromFirst = 2, fromLast = 1) + assert $a == "[30, 40]" + + if fromFirst + fromLast > deq.len: + clear(deq) + return + + for i in 0 ..< fromFirst: + destroy(deq.data[deq.head and deq.mask]) + inc deq.head + + for i in 0 ..< fromLast: + destroy(deq.data[(deq.tail - 1) and deq.mask]) + dec deq.tail + +proc `$`*[T](deq: Deque[T]): string = + ## Turns a deque into its string representation. + runnableExamples: + let a = [10, 20, 30].toDeque + assert $a == "[10, 20, 30]" + + result = "[" + for x in deq: + if result.len > 1: result.add(", ") + result.addQuoted(x) + result.add("]") + +func `==`*[T](deq1, deq2: Deque[T]): bool = + ## The `==` operator for Deque. + ## Returns `true` if both deques contains the same values in the same order. + runnableExamples: + var a, b = initDeque[int]() + a.addFirst(2) + a.addFirst(1) + b.addLast(1) + b.addLast(2) + doAssert a == b + + if deq1.len != deq2.len: + return false + + for i in 0 ..< deq1.len: + if deq1.data[(deq1.head + i.uint) and deq1.mask] != deq2.data[(deq2.head + i.uint) and deq2.mask]: + return false + + true + +func hash*[T](deq: Deque[T]): Hash = + ## Hashing of Deque. + var h: Hash = 0 + for x in deq: + h = h !& hash(x) + !$h diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim new file mode 100644 index 000000000..17785c8c7 --- /dev/null +++ b/lib/pure/collections/hashcommon.nim @@ -0,0 +1,76 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# An `include` file which contains common code for +# hash sets and tables. + +when defined(nimPreviewSlimSystem): + import std/assertions + +import std / outparams + +const + growthFactor = 2 + +# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These +# two procs retain clarity of that encoding without the space cost of an enum. +proc isEmpty(hcode: Hash): bool {.inline.} = + result = hcode == 0 + +proc isFilled(hcode: Hash): bool {.inline.} = + result = hcode != 0 + +proc nextTry(h, maxHash: Hash): Hash {.inline.} = + result = (h + 1) and maxHash + +proc mustRehash[T](t: T): bool {.inline.} = + # If this is changed, make sure to synchronize it with `slotsNeeded` below + assert(t.dataLen > t.counter) + result = (t.dataLen * 2 < t.counter * 3) or (t.dataLen - t.counter < 4) + +proc slotsNeeded(count: Natural): int {.inline.} = + # Make sure to synchronize with `mustRehash` above + result = nextPowerOfTwo(count * 3 div 2 + 4) + +template rawGetKnownHCImpl() {.dirty.} = + if t.dataLen == 0: + return -1 + var h: Hash = hc and maxHash(t) # start with real hash value + while isFilled(t.data[h].hcode): + # Compare hc THEN key with boolean short circuit. This makes the common case + # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. + # It does slow down succeeding lookups by one extra Hash cmp&and..usually + # just a few clock cycles, generally worth it for any non-integer-like A. + if t.data[h].hcode == hc and t.data[h].key == key: + return h + h = nextTry(h, maxHash(t)) + result = -1 - h # < 0 => MISSING; insert idx = -1 - result + +proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} = + rawGetKnownHCImpl() + +template genHashImpl(key, hc: typed) = + hc = hash(key) + if hc == 0: # This almost never taken branch should be very predictable. + when sizeof(int) < 4: + hc = 31415 # Value doesn't matter; Any non-zero favorite is fine <= 16-bit. + else: + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + +template genHash(key: typed): Hash = + var res: Hash + genHashImpl(key, res) + res + +template rawGetImpl() {.dirty.} = + genHashImpl(key, hc) + rawGetKnownHCImpl() + +proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = + rawGetImpl() diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim new file mode 100644 index 000000000..96f9b4430 --- /dev/null +++ b/lib/pure/collections/heapqueue.nim @@ -0,0 +1,266 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Yuriy Glukhov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + + +## The `heapqueue` module implements a +## `binary heap data structure<https://en.wikipedia.org/wiki/Binary_heap>`_ +## that can be used as a `priority queue<https://en.wikipedia.org/wiki/Priority_queue>`_. +## They are represented as arrays for which `a[k] <= a[2*k+1]` and `a[k] <= a[2*k+2]` +## for all indices `k` (counting elements from 0). The interesting property of a heap is that +## `a[0]` is always its smallest element. +## +## Basic usage +## ----------- +## +runnableExamples: + var heap = [8, 2].toHeapQueue + heap.push(5) + # the first element is the lowest element + assert heap[0] == 2 + # remove and return the lowest element + assert heap.pop() == 2 + # the lowest element remaining is 5 + assert heap[0] == 5 + +## Usage with custom objects +## ------------------------- +## To use a `HeapQueue` with a custom object, the `<` operator must be +## implemented. + +runnableExamples: + type Job = object + priority: int + + proc `<`(a, b: Job): bool = a.priority < b.priority + + var jobs = initHeapQueue[Job]() + jobs.push(Job(priority: 1)) + jobs.push(Job(priority: 2)) + + assert jobs[0].priority == 1 + + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + +type HeapQueue*[T] = object + ## A heap queue, commonly known as a priority queue. + data: seq[T] + +proc initHeapQueue*[T](): HeapQueue[T] = + ## Creates a new empty heap. + ## + ## Heaps are initialized by default, so it is not necessary to call + ## this function explicitly. + ## + ## **See also:** + ## * `toHeapQueue proc <#toHeapQueue,openArray[T]>`_ + result = default(HeapQueue[T]) + +proc len*[T](heap: HeapQueue[T]): int {.inline.} = + ## Returns the number of elements of `heap`. + runnableExamples: + let heap = [9, 5, 8].toHeapQueue + assert heap.len == 3 + + heap.data.len + +proc `[]`*[T](heap: HeapQueue[T], i: Natural): lent T {.inline.} = + ## Accesses the i-th element of `heap`. + heap.data[i] + +iterator items*[T](heap: HeapQueue[T]): lent T {.inline, since: (2, 1, 1).} = + ## Iterates over each item of `heap`. + let L = len(heap) + for i in 0 .. high(heap.data): + yield heap.data[i] + assert(len(heap) == L, "the length of the HeapQueue changed while iterating over it") + +proc heapCmp[T](x, y: T): bool {.inline.} = x < y + +proc siftup[T](heap: var HeapQueue[T], startpos, p: int) = + ## `heap` is a heap at all indices >= `startpos`, except possibly for `p`. `p` + ## is the index of a leaf with a possibly out-of-order value. Restores the + ## heap invariant. + var pos = p + let newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + let parentpos = (pos - 1) shr 1 + let parent = heap[parentpos] + if heapCmp(newitem, parent): + heap.data[pos] = parent + pos = parentpos + else: + break + heap.data[pos] = newitem + +proc siftdownToBottom[T](heap: var HeapQueue[T], p: int) = + # This is faster when the element should be close to the bottom. + let endpos = len(heap) + var pos = p + let startpos = pos + let newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + var childpos = 2 * pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + let rightpos = childpos + 1 + if rightpos < endpos and not heapCmp(heap[childpos], heap[rightpos]): + childpos = rightpos + # Move the smaller child up. + heap.data[pos] = heap[childpos] + pos = childpos + childpos = 2 * pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap.data[pos] = newitem + siftup(heap, startpos, pos) + +proc siftdown[T](heap: var HeapQueue[T], p: int) = + let endpos = len(heap) + var pos = p + let newitem = heap[pos] + var childpos = 2 * pos + 1 + while childpos < endpos: + let rightpos = childpos + 1 + if rightpos < endpos and not heapCmp(heap[childpos], heap[rightpos]): + childpos = rightpos + if not heapCmp(heap[childpos], newitem): + break + heap.data[pos] = heap[childpos] + pos = childpos + childpos = 2 * pos + 1 + heap.data[pos] = newitem + +proc push*[T](heap: var HeapQueue[T], item: sink T) = + ## Pushes `item` onto `heap`, maintaining the heap invariant. + heap.data.add(item) + siftup(heap, 0, len(heap) - 1) + +proc toHeapQueue*[T](x: openArray[T]): HeapQueue[T] {.since: (1, 3).} = + ## Creates a new HeapQueue that contains the elements of `x`. + ## + ## **See also:** + ## * `initHeapQueue proc <#initHeapQueue>`_ + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + assert heap.pop() == 5 + assert heap[0] == 8 + + # see https://en.wikipedia.org/wiki/Binary_heap#Building_a_heap + result.data = @x + for i in countdown(x.len div 2 - 1, 0): + siftdown(result, i) + +proc pop*[T](heap: var HeapQueue[T]): T = + ## Pops and returns the smallest item from `heap`, + ## maintaining the heap invariant. + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + assert heap.pop() == 5 + + let lastelt = heap.data.pop() + if heap.len > 0: + result = heap[0] + heap.data[0] = lastelt + siftdownToBottom(heap, 0) + else: + result = lastelt + +proc find*[T](heap: HeapQueue[T], x: T): int {.since: (1, 3).} = + ## Linear scan to find the index of the item `x` or -1 if not found. + runnableExamples: + let heap = [9, 5, 8].toHeapQueue + assert heap.find(5) == 0 + assert heap.find(9) == 1 + assert heap.find(777) == -1 + + result = -1 + for i in 0 ..< heap.len: + if heap[i] == x: return i + +proc contains*[T](heap: HeapQueue[T], x: T): bool {.since: (2, 1, 1).} = + ## Returns true if `x` is in `heap` or false if not found. This is a shortcut + ## for `find(heap, x) >= 0`. + result = find(heap, x) >= 0 + +proc del*[T](heap: var HeapQueue[T], index: Natural) = + ## Removes the element at `index` from `heap`, maintaining the heap invariant. + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + heap.del(1) + assert heap[0] == 5 + assert heap[1] == 8 + + swap(heap.data[^1], heap.data[index]) + let newLen = heap.len - 1 + heap.data.setLen(newLen) + if index < newLen: + siftdownToBottom(heap, index) + +proc replace*[T](heap: var HeapQueue[T], item: sink T): T = + ## Pops and returns the current smallest value, and add the new item. + ## This is more efficient than `pop()` followed by `push()`, and can be + ## more appropriate when using a fixed-size heap. Note that the value + ## returned may be larger than `item`! That constrains reasonable uses of + ## this routine unless written as part of a conditional replacement. + ## + ## **See also:** + ## * `pushpop proc <#pushpop,HeapQueue[T],sinkT>`_ + runnableExamples: + var heap = [5, 12].toHeapQueue + assert heap.replace(6) == 5 + assert heap.len == 2 + assert heap[0] == 6 + assert heap.replace(4) == 6 + + result = heap[0] + heap.data[0] = item + siftdown(heap, 0) + +proc pushpop*[T](heap: var HeapQueue[T], item: sink T): T = + ## Fast version of a `push()` followed by a `pop()`. + ## + ## **See also:** + ## * `replace proc <#replace,HeapQueue[T],sinkT>`_ + runnableExamples: + var heap = [5, 12].toHeapQueue + assert heap.pushpop(6) == 5 + assert heap.len == 2 + assert heap[0] == 6 + assert heap.pushpop(4) == 4 + + result = item + if heap.len > 0 and heapCmp(heap.data[0], result): + swap(result, heap.data[0]) + siftdown(heap, 0) + +proc clear*[T](heap: var HeapQueue[T]) = + ## Removes all elements from `heap`, making it empty. + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + heap.clear() + assert heap.len == 0 + + heap.data.setLen(0) + +proc `$`*[T](heap: HeapQueue[T]): string = + ## Turns a heap into its string representation. + runnableExamples: + let heap = [1, 2].toHeapQueue + assert $heap == "[1, 2]" + + result = "[" + for x in heap.data: + if result.len > 1: result.add(", ") + result.addQuoted(x) + result.add("]") diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 603a4b595..765a23e97 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -7,214 +7,17 @@ # distribution, for details about the copyright. # -## The ``intsets`` module implements an efficient int set implemented as a -## `sparse bit set`:idx:. -## **Note**: Since Nim currently does not allow the assignment operator to -## be overloaded, ``=`` for int sets performs some rather meaningless shallow -## copy; use ``assign`` to get a deep copy. +## Specialization of the generic `packedsets module <packedsets.html>`_ +## (see its documentation for more examples) for ordinal sparse sets. -import - hashes, math +import std/private/since +import std/packedsets +export packedsets type - BitScalar = int + IntSet* = PackedSet[int] -const - InitIntSetSize = 8 # must be a power of two! - TrunkShift = 9 - BitsPerTrunk = 1 shl TrunkShift # needs to be a power of 2 and - # divisible by 64 - TrunkMask = BitsPerTrunk - 1 - IntsPerTrunk = BitsPerTrunk div (sizeof(BitScalar) * 8) - IntShift = 5 + ord(sizeof(BitScalar) == 8) # 5 or 6, depending on int width - IntMask = 1 shl IntShift - 1 +proc toIntSet*(x: openArray[int]): IntSet {.since: (1, 3), inline.} = toPackedSet[int](x) -type - PTrunk = ref Trunk - Trunk {.final.} = object - next: PTrunk # all nodes are connected with this pointer - key: int # start address at bit 0 - bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector - - TrunkSeq = seq[PTrunk] - IntSet* = object ## an efficient set of 'int' implemented as a sparse bit set - counter, max: int - head: PTrunk - data: TrunkSeq - -{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkSeq: TrunkSeq].} - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash - -proc intSetGet(t: IntSet, key: int): PTrunk = - var h = key and t.max - while t.data[h] != nil: - if t.data[h].key == key: - return t.data[h] - h = nextTry(h, t.max) - result = nil - -proc intSetRawInsert(t: IntSet, data: var TrunkSeq, desc: PTrunk) = - var h = desc.key and t.max - while data[h] != nil: - assert(data[h] != desc) - h = nextTry(h, t.max) - assert(data[h] == nil) - data[h] = desc - -proc intSetEnlarge(t: var IntSet) = - var n: TrunkSeq - var oldMax = t.max - t.max = ((t.max + 1) * 2) - 1 - newSeq(n, t.max + 1) - for i in countup(0, oldMax): - if t.data[i] != nil: intSetRawInsert(t, n, t.data[i]) - swap(t.data, n) - -proc intSetPut(t: var IntSet, key: int): PTrunk = - var h = key and t.max - while t.data[h] != nil: - if t.data[h].key == key: - return t.data[h] - h = nextTry(h, t.max) - if mustRehash(t.max + 1, t.counter): intSetEnlarge(t) - inc(t.counter) - h = key and t.max - while t.data[h] != nil: h = nextTry(h, t.max) - assert(t.data[h] == nil) - new(result) - result.next = t.head - result.key = key - t.head = result - t.data[h] = result - -proc contains*(s: IntSet, key: int): bool = - ## returns true iff `key` is in `s`. - var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: - var u = key and TrunkMask - result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0 - else: - result = false - -proc incl*(s: var IntSet, key: int) = - ## includes an element `key` in `s`. - var t = intSetPut(s, `shr`(key, TrunkShift)) - var u = key and TrunkMask - t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or - `shl`(1, u and IntMask) - -proc excl*(s: var IntSet, key: int) = - ## excludes `key` from the set `s`. - var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: - var u = key and TrunkMask - t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and - not `shl`(1, u and IntMask) - -proc containsOrIncl*(s: var IntSet, key: int): bool = - ## returns true if `s` contains `key`, otherwise `key` is included in `s` - ## and false is returned. - var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: - var u = key and TrunkMask - result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0 - if not result: - t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or - `shl`(1, u and IntMask) - else: - incl(s, key) - result = false - -proc initIntSet*: IntSet = - ## creates a new int set that is empty. - newSeq(result.data, InitIntSetSize) - result.max = InitIntSetSize-1 - result.counter = 0 - result.head = nil - -proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil - -proc assign*(dest: var IntSet, src: IntSet) = - ## copies `src` to `dest`. `dest` does not need to be initialized by - ## `initIntSet`. - dest.counter = src.counter - dest.max = src.max - newSeq(dest.data, src.data.len) - - var it = src.head - while it != nil: - - var h = it.key and dest.max - while dest.data[h] != nil: h = nextTry(h, dest.max) - assert(dest.data[h] == nil) - - var n: PTrunk - new(n) - n.next = dest.head - n.key = it.key - n.bits = it.bits - dest.head = n - dest.data[h] = n - - it = it.next - -iterator items*(s: IntSet): int {.inline.} = - ## iterates over any included element of `s`. - var r = s.head - while r != nil: - var i = 0 - while i <= high(r.bits): - var w = r.bits[i] - # taking a copy of r.bits[i] here is correct, because - # modifying operations are not allowed during traversation - var j = 0 - while w != 0: # test all remaining bits for zero - if (w and 1) != 0: # the bit is set! - yield (r.key shl TrunkShift) or (i shl IntShift +% j) - inc(j) - w = w shr 1 - inc(i) - r = r.next - -template dollarImpl(): stmt = - result = "{" - for key in items(s): - if result.len > 1: result.add(", ") - result.add($key) - result.add("}") - -proc `$`*(s: IntSet): string = - ## The `$` operator for int sets. - dollarImpl() - -proc empty*(s: IntSet): bool {.inline, deprecated.} = - ## returns true if `s` is empty. This is safe to call even before - ## the set has been initialized with `initIntSet`. Note this never - ## worked reliably and so is deprecated. - result = s.counter == 0 - -when isMainModule: - import sequtils, algorithm - - var x = initIntSet() - x.incl(1) - x.incl(2) - x.incl(7) - x.incl(1056) - - var xs = toSeq(items(x)) - xs.sort(cmp[int]) - assert xs == @[1, 2, 7, 1056] - - var y: IntSet - assign(y, x) - var ys = toSeq(items(y)) - ys.sort(cmp[int]) - assert ys == @[1, 2, 7, 1056] +proc initIntSet*(): IntSet {.inline.} = initPackedSet[int]() diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index f847ddd58..6b88747ef 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -7,232 +7,531 @@ # distribution, for details about the copyright. # -## Implementation of singly and doubly linked lists. Because it makes no sense -## to do so, the 'next' and 'prev' pointers are not hidden from you and can -## be manipulated directly for efficiency. - -when not defined(nimhygiene): - {.pragma: dirty.} +## Implementation of: +## * `singly linked lists <#SinglyLinkedList>`_ +## * `doubly linked lists <#DoublyLinkedList>`_ +## * `singly linked rings <#SinglyLinkedRing>`_ (circular lists) +## * `doubly linked rings <#DoublyLinkedRing>`_ (circular lists) +## +## # Basic Usage +## Because it makes no sense to do otherwise, the `next` and `prev` pointers +## are not hidden from you and can be manipulated directly for efficiency. +## +## ## Lists +runnableExamples: + var list = initDoublyLinkedList[int]() + let + a = newDoublyLinkedNode[int](3) + b = newDoublyLinkedNode[int](7) + c = newDoublyLinkedNode[int](9) + + list.add(a) + list.add(b) + list.prepend(c) + + assert a.next == b + assert a.prev == c + assert c.next == a + assert c.next.next == b + assert c.prev == nil + assert b.next == nil + +## ## Rings +runnableExamples: + var ring = initSinglyLinkedRing[int]() + let + a = newSinglyLinkedNode[int](3) + b = newSinglyLinkedNode[int](7) + c = newSinglyLinkedNode[int](9) + + ring.add(a) + ring.add(b) + ring.prepend(c) + + assert c.next == a + assert a.next == b + assert c.next.next == b + assert b.next == c + assert c.next.next.next == c + +## # See also +## * `deques module <deques.html>`_ for double-ended queues + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions type - DoublyLinkedNodeObj*[T] = object ## a node a doubly linked list consists of - next*, prev*: ref DoublyLinkedNodeObj[T] + DoublyLinkedNodeObj*[T] = object + ## A node of a doubly linked list. + ## + ## It consists of a `value` field, and pointers to `next` and `prev`. + next*: DoublyLinkedNode[T] + prev* {.cursor.}: DoublyLinkedNode[T] value*: T DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T] - SinglyLinkedNodeObj*[T] = object ## a node a singly linked list consists of - next*: ref SinglyLinkedNodeObj[T] + SinglyLinkedNodeObj*[T] = object + ## A node of a singly linked list. + ## + ## It consists of a `value` field, and a pointer to `next`. + next*: SinglyLinkedNode[T] value*: T SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T] - SinglyLinkedList*[T] = object ## a singly linked list - head*, tail*: SinglyLinkedNode[T] + SinglyLinkedList*[T] = object + ## A singly linked list. + head*: SinglyLinkedNode[T] + tail* {.cursor.}: SinglyLinkedNode[T] - DoublyLinkedList*[T] = object ## a doubly linked list - head*, tail*: DoublyLinkedNode[T] + DoublyLinkedList*[T] = object + ## A doubly linked list. + head*: DoublyLinkedNode[T] + tail* {.cursor.}: DoublyLinkedNode[T] - SinglyLinkedRing*[T] = object ## a singly linked ring - head*, tail*: SinglyLinkedNode[T] + SinglyLinkedRing*[T] = object + ## A singly linked ring. + head*: SinglyLinkedNode[T] + tail* {.cursor.}: SinglyLinkedNode[T] - DoublyLinkedRing*[T] = object ## a doubly linked ring + DoublyLinkedRing*[T] = object + ## A doubly linked ring. head*: DoublyLinkedNode[T] -{.deprecated: [TDoublyLinkedNode: DoublyLinkedNodeObj, - PDoublyLinkedNode: DoublyLinkedNode, - TSinglyLinkedNode: SinglyLinkedNodeObj, - PSinglyLinkedNode: SinglyLinkedNode, - TDoublyLinkedList: DoublyLinkedList, - TSinglyLinkedRing: SinglyLinkedRing, - TDoublyLinkedRing: DoublyLinkedRing, - TSinglyLinkedList: SinglyLinkedList].} + SomeLinkedList*[T] = SinglyLinkedList[T] | DoublyLinkedList[T] + + SomeLinkedRing*[T] = SinglyLinkedRing[T] | DoublyLinkedRing[T] + + SomeLinkedCollection*[T] = SomeLinkedList[T] | SomeLinkedRing[T] + + SomeLinkedNode*[T] = SinglyLinkedNode[T] | DoublyLinkedNode[T] proc initSinglyLinkedList*[T](): SinglyLinkedList[T] = - ## creates a new singly linked list that is empty. + ## Creates a new singly linked list that is empty. + ## + ## Singly linked lists are initialized by default, so it is not necessary to + ## call this function explicitly. + runnableExamples: + let a = initSinglyLinkedList[int]() + discard proc initDoublyLinkedList*[T](): DoublyLinkedList[T] = - ## creates a new doubly linked list that is empty. + ## Creates a new doubly linked list that is empty. + ## + ## Doubly linked lists are initialized by default, so it is not necessary to + ## call this function explicitly. + runnableExamples: + let a = initDoublyLinkedList[int]() + discard proc initSinglyLinkedRing*[T](): SinglyLinkedRing[T] = - ## creates a new singly linked ring that is empty. + ## Creates a new singly linked ring that is empty. + ## + ## Singly linked rings are initialized by default, so it is not necessary to + ## call this function explicitly. + runnableExamples: + let a = initSinglyLinkedRing[int]() + discard proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] = - ## creates a new doubly linked ring that is empty. + ## Creates a new doubly linked ring that is empty. + ## + ## Doubly linked rings are initialized by default, so it is not necessary to + ## call this function explicitly. + runnableExamples: + let a = initDoublyLinkedRing[int]() + discard proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] = - ## creates a new doubly linked node with the given `value`. + ## Creates a new doubly linked node with the given `value`. + runnableExamples: + let n = newDoublyLinkedNode[int](5) + assert n.value == 5 + new(result) result.value = value proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = - ## creates a new singly linked node with the given `value`. + ## Creates a new singly linked node with the given `value`. + runnableExamples: + let n = newSinglyLinkedNode[int](5) + assert n.value == 5 + new(result) result.value = value template itemsListImpl() {.dirty.} = - var it = L.head + var it {.cursor.} = L.head while it != nil: yield it.value it = it.next template itemsRingImpl() {.dirty.} = - var it = L.head + var it {.cursor.} = L.head if it != nil: while true: yield it.value it = it.next if it == L.head: break -template nodesListImpl() {.dirty.} = - var it = L.head - while it != nil: - var nxt = it.next - yield it - it = nxt - -template nodesRingImpl() {.dirty.} = - var it = L.head - if it != nil: - while true: - var nxt = it.next - yield it - it = nxt - if it == L.head: break - -template findImpl() {.dirty.} = - for x in nodes(L): - if x.value == value: return x +iterator items*[T](L: SomeLinkedList[T]): T = + ## Yields every value of `L`. + ## + ## **See also:** + ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedList[T]>`_ + runnableExamples: + from std/sugar import collect + from std/sequtils import toSeq + let a = collect(initSinglyLinkedList): + for i in 1..3: 10 * i + assert toSeq(items(a)) == toSeq(a) + assert toSeq(a) == @[10, 20, 30] -iterator items*[T](L: DoublyLinkedList[T]): T = - ## yields every value of `L`. itemsListImpl() -iterator items*[T](L: SinglyLinkedList[T]): T = - ## yields every value of `L`. - itemsListImpl() +iterator items*[T](L: SomeLinkedRing[T]): T = + ## Yields every value of `L`. + ## + ## **See also:** + ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedRing[T]>`_ + runnableExamples: + from std/sugar import collect + from std/sequtils import toSeq + let a = collect(initSinglyLinkedRing): + for i in 1..3: 10 * i + assert toSeq(items(a)) == toSeq(a) + assert toSeq(a) == @[10, 20, 30] -iterator items*[T](L: SinglyLinkedRing[T]): T = - ## yields every value of `L`. itemsRingImpl() -iterator items*[T](L: DoublyLinkedRing[T]): T = - ## yields every value of `L`. - itemsRingImpl() +iterator mitems*[T](L: var SomeLinkedList[T]): var T = + ## Yields every value of `L` so that you can modify it. + ## + ## **See also:** + ## * `items iterator <#items.i,SomeLinkedList[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedList[T]>`_ + runnableExamples: + var a = initSinglyLinkedList[int]() + for i in 1..5: + a.add(10 * i) + assert $a == "[10, 20, 30, 40, 50]" + for x in mitems(a): + x = 5 * x - 1 + assert $a == "[49, 99, 149, 199, 249]" -iterator mitems*[T](L: var DoublyLinkedList[T]): var T = - ## yields every value of `L` so that you can modify it. itemsListImpl() -iterator mitems*[T](L: var SinglyLinkedList[T]): var T = - ## yields every value of `L` so that you can modify it. - itemsListImpl() +iterator mitems*[T](L: var SomeLinkedRing[T]): var T = + ## Yields every value of `L` so that you can modify it. + ## + ## **See also:** + ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ + ## * `nodes iterator <#nodes.i,SomeLinkedRing[T]>`_ + runnableExamples: + var a = initSinglyLinkedRing[int]() + for i in 1..5: + a.add(10 * i) + assert $a == "[10, 20, 30, 40, 50]" + for x in mitems(a): + x = 5 * x - 1 + assert $a == "[49, 99, 149, 199, 249]" -iterator mitems*[T](L: var SinglyLinkedRing[T]): var T = - ## yields every value of `L` so that you can modify it. itemsRingImpl() -iterator mitems*[T](L: var DoublyLinkedRing[T]): var T = - ## yields every value of `L` so that you can modify it. - itemsRingImpl() - -iterator nodes*[T](L: SinglyLinkedList[T]): SinglyLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the - ## list during traversal is supported. - nodesListImpl() - -iterator nodes*[T](L: DoublyLinkedList[T]): DoublyLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the +iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = + ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. - nodesListImpl() + ## + ## **See also:** + ## * `items iterator <#items.i,SomeLinkedList[T]>`_ + ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ + runnableExamples: + var a = initDoublyLinkedList[int]() + for i in 1..5: + a.add(10 * i) + assert $a == "[10, 20, 30, 40, 50]" + for x in nodes(a): + if x.value == 30: + a.remove(x) + else: + x.value = 5 * x.value - 1 + assert $a == "[49, 99, 199, 249]" + + var it {.cursor.} = L.head + while it != nil: + let nxt = it.next + yield it + it = nxt -iterator nodes*[T](L: SinglyLinkedRing[T]): SinglyLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the +iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = + ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. - nodesRingImpl() + ## + ## **See also:** + ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ + ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ + runnableExamples: + var a = initDoublyLinkedRing[int]() + for i in 1..5: + a.add(10 * i) + assert $a == "[10, 20, 30, 40, 50]" + for x in nodes(a): + if x.value == 30: + a.remove(x) + else: + x.value = 5 * x.value - 1 + assert $a == "[49, 99, 199, 249]" + + var it {.cursor.} = L.head + if it != nil: + while true: + let nxt = it.next + yield it + it = nxt + if it == L.head: break -iterator nodes*[T](L: DoublyLinkedRing[T]): DoublyLinkedNode[T] = - ## iterates over every node of `x`. Removing the current node from the - ## list during traversal is supported. - nodesRingImpl() +proc `$`*[T](L: SomeLinkedCollection[T]): string = + ## Turns a list into its string representation for logging and printing. + runnableExamples: + let a = [1, 2, 3, 4].toSinglyLinkedList + assert $a == "[1, 2, 3, 4]" -template dollarImpl() {.dirty.} = result = "[" for x in nodes(L): if result.len > 1: result.add(", ") - result.add($x.value) + result.addQuoted(x.value) result.add("]") -proc `$`*[T](L: SinglyLinkedList[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc `$`*[T](L: DoublyLinkedList[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc `$`*[T](L: SinglyLinkedRing[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc `$`*[T](L: DoublyLinkedRing[T]): string = - ## turns a list into its string representation. - dollarImpl() - -proc find*[T](L: SinglyLinkedList[T], value: T): SinglyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not +proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = + ## Searches in the list for a value. Returns `nil` if the value does not ## exist. - findImpl() + ## + ## **See also:** + ## * `contains proc <#contains,SomeLinkedCollection[T],T>`_ + runnableExamples: + let a = [9, 8].toSinglyLinkedList + assert a.find(9).value == 9 + assert a.find(1) == nil -proc find*[T](L: DoublyLinkedList[T], value: T): DoublyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not - ## exist. - findImpl() + for x in nodes(L): + if x.value == value: return x -proc find*[T](L: SinglyLinkedRing[T], value: T): SinglyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not - ## exist. - findImpl() +proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = + ## Searches in the list for a value. Returns `false` if the value does not + ## exist, `true` otherwise. This allows the usage of the `in` and `notin` + ## operators. + ## + ## **See also:** + ## * `find proc <#find,SomeLinkedCollection[T],T>`_ + runnableExamples: + let a = [9, 8].toSinglyLinkedList + assert a.contains(9) + assert 8 in a + assert(not a.contains(1)) + assert 2 notin a -proc find*[T](L: DoublyLinkedRing[T], value: T): DoublyLinkedNode[T] = - ## searches in the list for a value. Returns nil if the value does not - ## exist. - findImpl() - -proc contains*[T](L: SinglyLinkedList[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. result = find(L, value) != nil -proc contains*[T](L: DoublyLinkedList[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. - result = find(L, value) != nil +proc prepend*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = + ## Prepends a shallow copy of `b` to the beginning of `a`. + ## + ## **See also:** + ## * `prependMoved proc <#prependMoved,T,T>`_ + ## for moving the second list instead of copying + runnableExamples: + from std/sequtils import toSeq + var a = [4, 5].toSinglyLinkedList + let b = [1, 2, 3].toSinglyLinkedList + a.prepend(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [1, 2, 3] + a.prepend(a) + assert a.toSeq == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] + + var tmp = b.copy + tmp.addMoved(a) + a = tmp + +proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = + ## Moves `b` before the head of `a`. Efficiency: O(1). + ## Note that `b` becomes empty after the operation unless it has the same address as `a`. + ## Self-prepending results in a cycle. + ## + ## **See also:** + ## * `prepend proc <#prepend,T,T>`_ + ## for prepending a copy of a list + runnableExamples: + import std/[sequtils, enumerate, sugar] + var + a = [4, 5].toSinglyLinkedList + b = [1, 2, 3].toSinglyLinkedList + c = [0, 1].toSinglyLinkedList + a.prependMoved(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [] + c.prependMoved(c) + let s = collect: + for i, ci in enumerate(c): + if i == 6: break + ci + assert s == [0, 1, 0, 1, 0, 1] + + b.addMoved(a) + swap a, b + +proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedList[int]() + let n = newSinglyLinkedNode[int](9) + a.add(n) + assert a.contains(9) -proc contains*[T](L: SinglyLinkedRing[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. - result = find(L, value) != nil + n.next = nil + if L.tail != nil: + assert(L.tail.next == nil) + L.tail.next = n + L.tail = n + if L.head == nil: L.head = n -proc contains*[T](L: DoublyLinkedRing[T], value: T): bool {.inline.} = - ## searches in the list for a value. Returns false if the value does not - ## exist, true otherwise. - result = find(L, value) != nil +proc add*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedList[int]() + a.add(9) + a.add(8) + assert a.contains(9) + + add(L, newSinglyLinkedNode(value)) proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = - ## prepends a node to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedList[int]() + let n = newSinglyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + n.next = L.head L.head = n + if L.tail == nil: L.tail = n proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = - ## prepends a node to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + runnableExamples: + var a = initSinglyLinkedList[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) + prepend(L, newSinglyLinkedNode(value)) -proc append*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## appends a node `n` to `L`. Efficiency: O(1). +func copy*[T](a: SinglyLinkedList[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a shallow copy of `a`. + runnableExamples: + from std/sequtils import toSeq + type Foo = ref object + x: int + var + f = Foo(x: 1) + a = [f].toSinglyLinkedList + let b = a.copy + a.add([f].toSinglyLinkedList) + assert a.toSeq == [f, f] + assert b.toSeq == [f] # b isn't modified... + f.x = 42 + assert a.head.value.x == 42 + assert b.head.value.x == 42 # ... but the elements are not deep copied + + let c = [1, 2, 3].toSinglyLinkedList + assert $c == $c.copy + + result = initSinglyLinkedList[T]() + for x in a.items: + result.add(x) + +proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} = + ## Moves `b` to the end of `a`. Efficiency: O(1). + ## Note that `b` becomes empty after the operation unless it has the same address as `a`. + ## Self-adding results in a cycle. + ## + ## **See also:** + ## * `add proc <#add,T,T>`_ for adding a copy of a list + runnableExamples: + import std/[sequtils, enumerate, sugar] + var + a = [1, 2, 3].toSinglyLinkedList + b = [4, 5].toSinglyLinkedList + c = [0, 1].toSinglyLinkedList + a.addMoved(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [] + c.addMoved(c) + let s = collect: + for i, ci in enumerate(c): + if i == 6: break + ci + assert s == [0, 1, 0, 1, 0, 1] + + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + a.tail.next = b.head + a.tail = b.tail + if a.addr != b.addr: + b.head = nil + b.tail = nil + +proc add*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedList[int]() + let n = newDoublyLinkedNode[int](9) + a.add(n) + assert a.contains(9) + n.next = nil n.prev = L.tail if L.tail != nil: @@ -241,12 +540,41 @@ proc append*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = L.tail = n if L.head == nil: L.head = n -proc append*[T](L: var DoublyLinkedList[T], value: T) = - ## appends a value to `L`. Efficiency: O(1). - append(L, newDoublyLinkedNode(value)) +proc add*[T](L: var DoublyLinkedList[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedList[int]() + a.add(9) + a.add(8) + assert a.contains(9) + + add(L, newDoublyLinkedNode(value)) proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## prepends a node `n` to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedList[int]() + let n = newDoublyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + n.prev = nil n.next = L.head if L.head != nil: @@ -256,35 +584,224 @@ proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = if L.tail == nil: L.tail = n proc prepend*[T](L: var DoublyLinkedList[T], value: T) = - ## prepends a value to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedList[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) + prepend(L, newDoublyLinkedNode(value)) +func copy*[T](a: DoublyLinkedList[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a shallow copy of `a`. + runnableExamples: + from std/sequtils import toSeq + type Foo = ref object + x: int + var + f = Foo(x: 1) + a = [f].toDoublyLinkedList + let b = a.copy + a.add([f].toDoublyLinkedList) + assert a.toSeq == [f, f] + assert b.toSeq == [f] # b isn't modified... + f.x = 42 + assert a.head.value.x == 42 + assert b.head.value.x == 42 # ... but the elements are not deep copied + + let c = [1, 2, 3].toDoublyLinkedList + assert $c == $c.copy + + result = initDoublyLinkedList[T]() + for x in a.items: + result.add(x) + +proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} = + ## Moves `b` to the end of `a`. Efficiency: O(1). + ## Note that `b` becomes empty after the operation unless it has the same address as `a`. + ## Self-adding results in a cycle. + ## + ## **See also:** + ## * `add proc <#add,T,T>`_ + ## for adding a copy of a list + runnableExamples: + import std/[sequtils, enumerate, sugar] + var + a = [1, 2, 3].toDoublyLinkedList + b = [4, 5].toDoublyLinkedList + c = [0, 1].toDoublyLinkedList + a.addMoved(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [] + c.addMoved(c) + let s = collect: + for i, ci in enumerate(c): + if i == 6: break + ci + assert s == [0, 1, 0, 1, 0, 1] + + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + b.head.prev = a.tail + a.tail.next = b.head + a.tail = b.tail + if a.addr != b.addr: + b.head = nil + b.tail = nil + +proc add*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = + ## Appends a shallow copy of `b` to the end of `a`. + ## + ## **See also:** + ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ + ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ + ## for moving the second list instead of copying + runnableExamples: + from std/sequtils import toSeq + var a = [1, 2, 3].toSinglyLinkedList + let b = [4, 5].toSinglyLinkedList + a.add(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [4, 5] + a.add(a) + assert a.toSeq == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] + + var tmp = b.copy + a.addMoved(tmp) + +proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} = + ## Removes a node `n` from `L`. + ## Returns `true` if `n` was found in `L`. + ## Efficiency: O(n); the list is traversed until `n` is found. + ## Attempting to remove an element not contained in the list is a no-op. + ## When the list is cyclic, the cycle is preserved after removal. + runnableExamples: + import std/[sequtils, enumerate, sugar] + var a = [0, 1, 2].toSinglyLinkedList + let n = a.head.next + assert n.value == 1 + assert a.remove(n) == true + assert a.toSeq == [0, 2] + assert a.remove(n) == false + assert a.toSeq == [0, 2] + a.addMoved(a) # cycle: [0, 2, 0, 2, ...] + a.remove(a.head) + let s = collect: + for i, ai in enumerate(a): + if i == 4: break + ai + assert s == [2, 2, 2, 2] + + if n == L.head: + L.head = n.next + if L.tail.next == n: + L.tail.next = L.head # restore cycle + else: + var prev {.cursor.} = L.head + while prev.next != n and prev.next != nil: + prev = prev.next + if prev.next == nil: + return false + prev.next = n.next + if L.tail == n: + L.tail = prev # update tail if we removed the last node + true + proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = - ## removes `n` from `L`. Efficiency: O(1). + ## Removes a node `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, + ## otherwise the effects are undefined. + ## When the list is cyclic, the cycle is preserved after removal. + runnableExamples: + import std/[sequtils, enumerate, sugar] + var a = [0, 1, 2].toSinglyLinkedList + let n = a.head.next + assert n.value == 1 + a.remove(n) + assert a.toSeq == [0, 2] + a.remove(n) + assert a.toSeq == [0, 2] + a.addMoved(a) # cycle: [0, 2, 0, 2, ...] + a.remove(a.head) + let s = collect: + for i, ai in enumerate(a): + if i == 4: break + ai + assert s == [2, 2, 2, 2] + if n == L.tail: L.tail = n.prev if n == L.head: L.head = n.next if n.next != nil: n.next.prev = n.prev if n.prev != nil: n.prev.next = n.next -proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## appends a node `n` to `L`. Efficiency: O(1). + +proc add*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedRing[int]() + let n = newSinglyLinkedNode[int](9) + a.add(n) + assert a.contains(9) + if L.head != nil: n.next = L.head assert(L.tail != nil) L.tail.next = n - L.tail = n else: n.next = n L.head = n - L.tail = n + L.tail = n -proc append*[T](L: var SinglyLinkedRing[T], value: T) = - ## appends a value to `L`. Efficiency: O(1). - append(L, newSinglyLinkedNode(value)) +proc add*[T](L: var SinglyLinkedRing[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedRing[int]() + a.add(9) + a.add(8) + assert a.contains(9) + + add(L, newSinglyLinkedNode(value)) proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = - ## prepends a node `n` to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value + runnableExamples: + var a = initSinglyLinkedRing[int]() + let n = newSinglyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + if L.head != nil: n.next = L.head assert(L.tail != nil) @@ -295,11 +812,40 @@ proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = L.head = n proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = - ## prepends a value to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## for prepending a node + runnableExamples: + var a = initSinglyLinkedRing[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) + prepend(L, newSinglyLinkedNode(value)) -proc append*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## appends a node `n` to `L`. Efficiency: O(1). + + +proc add*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = + ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedRing[int]() + let n = newDoublyLinkedNode[int](9) + a.add(n) + assert a.contains(9) + if L.head != nil: n.next = L.head n.prev = L.head.prev @@ -310,12 +856,41 @@ proc append*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = n.next = n L.head = n -proc append*[T](L: var DoublyLinkedRing[T], value: T) = - ## appends a value to `L`. Efficiency: O(1). - append(L, newDoublyLinkedNode(value)) +proc add*[T](L: var DoublyLinkedRing[T], value: T) = + ## Appends (adds to the end) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedRing[int]() + a.add(9) + a.add(8) + assert a.contains(9) + + add(L, newDoublyLinkedNode(value)) proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## prepends a node `n` to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedRing[int]() + let n = newDoublyLinkedNode[int](9) + a.prepend(n) + assert a.contains(9) + if L.head != nil: n.next = L.head n.prev = L.head.prev @@ -327,17 +902,114 @@ proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = L.head = n proc prepend*[T](L: var DoublyLinkedRing[T], value: T) = - ## prepends a value to `L`. Efficiency: O(1). + ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for appending a node + ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value + ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for prepending a node + ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## for removing a node + runnableExamples: + var a = initDoublyLinkedRing[int]() + a.prepend(9) + a.prepend(8) + assert a.contains(9) + prepend(L, newDoublyLinkedNode(value)) proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = - ## removes `n` from `L`. Efficiency: O(1). + ## Removes `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, + ## otherwise the effects are undefined. + runnableExamples: + var a = initDoublyLinkedRing[int]() + let n = newDoublyLinkedNode[int](5) + a.add(n) + assert 5 in a + a.remove(n) + assert 5 notin a + n.next.prev = n.prev n.prev.next = n.next if n == L.head: - var p = L.head.prev + let p = L.head.prev if p == L.head: # only one element left: L.head = nil else: - L.head = L.head.prev + L.head = p + +proc append*[T](a: var (SinglyLinkedList[T] | SinglyLinkedRing[T]), + b: SinglyLinkedList[T] | SinglyLinkedNode[T] | T) = + ## Alias for `a.add(b)`. + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## * `add proc <#add,SinglyLinkedList[T],T>`_ + ## * `add proc <#add,T,T>`_ + a.add(b) + +proc append*[T](a: var (DoublyLinkedList[T] | DoublyLinkedRing[T]), + b: DoublyLinkedList[T] | DoublyLinkedNode[T] | T) = + ## Alias for `a.add(b)`. + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## * `add proc <#add,DoublyLinkedList[T],T>`_ + ## * `add proc <#add,T,T>`_ + a.add(b) + +proc appendMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = + ## Alias for `a.addMoved(b)`. + ## + ## **See also:** + ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ + ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ + a.addMoved(b) + +func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `SinglyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toSinglyLinkedRing*[T](elems: openArray[T]): SinglyLinkedRing[T] = + ## Creates a new `SinglyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedRing[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `DoublyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedRing*[T](elems: openArray[T]): DoublyLinkedRing[T] = + ## Creates a new `DoublyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedRing[T]() + for elem in elems.items: + result.add(elem) diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim deleted file mode 100644 index c35a2dc29..000000000 --- a/lib/pure/collections/queues.nim +++ /dev/null @@ -1,102 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implementation of a `queue`:idx:. The underlying implementation uses a ``seq``. -## Note: For inter thread communication use -## a `TChannel <channels.html>`_ instead. - -import math - -type - Queue*[T] = object ## a queue - data: seq[T] - rd, wr, count, mask: int - -{.deprecated: [TQueue: Queue].} - -proc initQueue*[T](initialSize=4): Queue[T] = - ## creates a new queue. `initialSize` needs to be a power of 2. - assert isPowerOfTwo(initialSize) - result.mask = initialSize-1 - newSeq(result.data, initialSize) - -proc len*[T](q: Queue[T]): int = - ## returns the number of elements of `q`. - result = q.count - -iterator items*[T](q: Queue[T]): T = - ## yields every element of `q`. - var i = q.rd - var c = q.count - while c > 0: - dec c - yield q.data[i] - i = (i + 1) and q.mask - -iterator mitems*[T](q: var Queue[T]): var T = - ## yields every element of `q`. - var i = q.rd - var c = q.count - while c > 0: - dec c - yield q.data[i] - i = (i + 1) and q.mask - -proc add*[T](q: var Queue[T], item: T) = - ## adds an `item` to the end of the queue `q`. - var cap = q.mask+1 - if q.count >= cap: - var n: seq[T] - newSeq(n, cap*2) - var i = 0 - for x in items(q): - shallowCopy(n[i], x) - inc i - shallowCopy(q.data, n) - q.mask = cap*2 - 1 - q.wr = q.count - q.rd = 0 - inc q.count - q.data[q.wr] = item - q.wr = (q.wr + 1) and q.mask - -proc enqueue*[T](q: var Queue[T], item: T) = - ## alias for the ``add`` operation. - add(q, item) - -proc dequeue*[T](q: var Queue[T]): T = - ## removes and returns the first element of the queue `q`. - assert q.count > 0 - dec q.count - result = q.data[q.rd] - q.rd = (q.rd + 1) and q.mask - -proc `$`*[T](q: Queue[T]): string = - ## turns a queue into its string representation. - result = "[" - for x in items(q): - if result.len > 1: result.add(", ") - result.add($x) - result.add("]") - -when isMainModule: - var q = initQueue[int]() - q.add(123) - q.add(9) - q.add(4) - var first = q.dequeue - q.add(56) - q.add(6) - var second = q.dequeue - q.add(789) - - assert first == 123 - assert second == 9 - assert($q == "[4, 56, 6, 789]") - diff --git a/lib/pure/collections/rtarrays.nim b/lib/pure/collections/rtarrays.nim index 9d8085643..3c3ffda7c 100644 --- a/lib/pure/collections/rtarrays.nim +++ b/lib/pure/collections/rtarrays.nim @@ -10,18 +10,19 @@ ## Module that implements a fixed length array whose size ## is determined at runtime. Note: This is not ready for other people to use! +## +## Unstable API. const ArrayPartSize = 10 type - RtArray*[T] = object ## + RtArray*[T] = object ## L: Natural spart: seq[T] - apart: array [ArrayPartSize, T] - UncheckedArray* {.unchecked.}[T] = array[0..100_000_000, T] + apart: array[ArrayPartSize, T] -template usesSeqPart(x): expr = x.L > ArrayPartSize +template usesSeqPart(x): untyped = x.L > ArrayPartSize proc initRtArray*[T](len: Natural): RtArray[T] = result.L = len diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index b72face91..3c0d8dc0e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -7,34 +7,131 @@ # distribution, for details about the copyright. # -## :Author: Alexander Mitchell-Robinson (Amrykid) +## Although this module has `seq` in its name, it implements operations +## not only for the `seq`:idx: type, but for three built-in container types +## under the `openArray` umbrella: +## * sequences +## * strings +## * array ## -## This module implements operations for the built-in `seq`:idx: type which -## were inspired by functional programming languages. +## The `system` module defines several common functions, such as: +## * `newSeq[T]` for creating new sequences of type `T` +## * `@` for converting arrays and strings to sequences +## * `add` for adding new elements to strings and sequences +## * `&` for string and seq concatenation +## * `in` (alias for `contains`) and `notin` for checking if an item is +## in a container ## -## For functional style programming you may want to pass `anonymous procs -## <manual.html#anonymous-procs>`_ to procs like ``filter`` to reduce typing. -## Anonymous procs can use `the special do notation <manual.html#do-notation>`_ -## which is more convenient in certain situations. +## This module builds upon that, providing additional functionality in form of +## procs, iterators and templates inspired by functional programming +## languages. ## -## **Note**: This interface will change as soon as the compiler supports -## closures and proper coroutines. - -when not defined(nimhygiene): - {.pragma: dirty.} - -proc concat*[T](seqs: varargs[seq[T]]): seq[T] = +## For functional style programming you have different options at your disposal: +## * the `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ +## * pass an `anonymous proc<manual.html#procedures-anonymous-procs>`_ +## * import the `sugar module<sugar.html>`_ and use +## the `=> macro<sugar.html#%3D>.m,untyped,untyped>`_ +## * use `...It templates<#18>`_ +## (`mapIt<#mapIt.t,typed,untyped>`_, +## `filterIt<#filterIt.t,untyped,untyped>`_, etc.) +## +## Chaining of functions is possible thanks to the +## `method call syntax<manual.html#procedures-method-call-syntax>`_. + +runnableExamples: + import std/sugar + + # Creating a sequence from 1 to 10, multiplying each member by 2, + # keeping only the members which are not divisible by 6. + let + foo = toSeq(1..10).map(x => x * 2).filter(x => x mod 6 != 0) + bar = toSeq(1..10).mapIt(it * 2).filterIt(it mod 6 != 0) + baz = collect: + for i in 1..10: + let j = 2 * i + if j mod 6 != 0: + j + + doAssert foo == bar + doAssert foo == baz + doAssert foo == @[2, 4, 8, 10, 14, 16, 20] + + doAssert foo.any(x => x > 17) + doAssert not bar.allIt(it < 20) + doAssert foo.foldl(a + b) == 74 # sum of all members + + +runnableExamples: + from std/strutils import join + + let + vowels = @"aeiou" + foo = "sequtils is an awesome module" + + doAssert (vowels is seq[char]) and (vowels == @['a', 'e', 'i', 'o', 'u']) + doAssert foo.filterIt(it notin vowels).join == "sqtls s n wsm mdl" + +## See also +## ======== +## * `strutils module<strutils.html>`_ for common string functions +## * `sugar module<sugar.html>`_ for syntactic sugar macros +## * `algorithm module<algorithm.html>`_ for common generic algorithms +## * `json module<json.html>`_ for a structure which allows +## heterogeneous members + + +import std/private/since + +import std/macros +from std/typetraits import supportsCopyMem + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +macro evalOnceAs(expAlias, exp: untyped, + letAssigneable: static[bool]): untyped = + ## Injects `expAlias` in caller scope, to avoid bugs involving multiple + ## substitution in macro arguments such as + ## https://github.com/nim-lang/Nim/issues/7187. + ## `evalOnceAs(myAlias, myExp)` will behave as `let myAlias = myExp` + ## except when `letAssigneable` is false (e.g. to handle openArray) where + ## it just forwards `exp` unchanged. + expectKind(expAlias, nnkIdent) + var val = exp + + result = newStmtList() + # If `exp` is not a symbol we evaluate it once here and then use the temporary + # symbol as alias + if exp.kind != nnkSym and letAssigneable: + val = genSym() + result.add(newLetStmt(val, exp)) + + result.add( + newProc(name = genSym(nskTemplate, $expAlias), params = [getType(untyped)], + body = val, procType = nnkTemplateDef)) + +func concat*[T](seqs: varargs[seq[T]]): seq[T] = ## Takes several sequences' items and returns them inside a new sequence. + ## All sequences must be of the same type. ## - ## Example: + ## **See also:** + ## * `distribute func<#distribute,seq[T],Positive>`_ for a reverse + ## operation ## - ## .. code-block:: - ## let - ## s1 = @[1, 2, 3] - ## s2 = @[4, 5] - ## s3 = @[6, 7] - ## total = concat(s1, s2, s3) - ## assert total == @[1, 2, 3, 4, 5, 6, 7] + runnableExamples: + let + s1 = @[1, 2, 3] + s2 = @[4, 5] + s3 = @[6, 7] + total = concat(s1, s2, s3) + assert total == @[1, 2, 3, 4, 5, 6, 7] + var L = 0 for seqitm in items(seqs): inc(L, len(seqitm)) newSeq(result, L) @@ -44,110 +141,226 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) -proc cycle*[T](s: seq[T], n: Natural): seq[T] = - ## Returns a new sequence with the items of `s` repeated `n` times. - ## - ## Example: - ## - ## .. code-block: +func addUnique*[T](s: var seq[T], x: sink T) = + ## Adds `x` to the container `s` if it is not already present. + ## Uses `==` to check if the item is already present. + runnableExamples: + var a = @[1, 2, 3] + a.addUnique(4) + a.addUnique(4) + assert a == @[1, 2, 3, 4] + + for i in 0..high(s): + if s[i] == x: return + when declared(ensureMove): + s.add ensureMove(x) + else: + s.add x + +func count*[T](s: openArray[T], x: T): int = + ## Returns the number of occurrences of the item `x` in the container `s`. ## - ## let - ## s = @[1, 2, 3] - ## total = s.cycle(3) - ## assert total == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + runnableExamples: + let + a = @[1, 2, 2, 3, 2, 4, 2] + b = "abracadabra" + assert count(a, 2) == 4 + assert count(a, 99) == 0 + assert count(b, 'r') == 2 + + for itm in items(s): + if itm == x: + inc result + +func cycle*[T](s: openArray[T], n: Natural): seq[T] = + ## Returns a new sequence with the items of the container `s` repeated + ## `n` times. + ## `n` must be a non-negative number (zero or more). + ## + runnableExamples: + let + s = @[1, 2, 3] + total = s.cycle(3) + assert total == @[1, 2, 3, 1, 2, 3, 1, 2, 3] + result = newSeq[T](n * s.len) var o = 0 - for x in 0..<n: + for x in 0 ..< n: for e in s: result[o] = e inc o proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. + ## `n` must be a non-negative number (zero or more). ## - ## Example: - ## - ## .. code-block: - ## - ## let - ## total = repeat(5, 3) - ## assert total == @[5, 5, 5] + runnableExamples: + let + total = repeat(5, 3) + assert total == @[5, 5, 5] + result = newSeq[T](n) - for i in 0..<n: + for i in 0 ..< n: result[i] = x -proc deduplicate*[T](seq1: seq[T]): seq[T] = +func deduplicate*[T](s: openArray[T], isSorted: bool = false): seq[T] = ## Returns a new sequence without duplicates. ## - ## .. code-block:: - ## let - ## dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] - ## dup2 = @["a", "a", "c", "d", "d"] - ## unique1 = deduplicate(dup1) - ## unique2 = deduplicate(dup2) - ## assert unique1 == @[1, 3, 4, 2, 8] - ## assert unique2 == @["a", "c", "d"] + ## Setting the optional argument `isSorted` to true (default: false) + ## uses a faster algorithm for deduplication. + ## + runnableExamples: + let + dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] + dup2 = @["a", "a", "c", "d", "d"] + unique1 = deduplicate(dup1) + unique2 = deduplicate(dup2, isSorted = true) + assert unique1 == @[1, 3, 4, 2, 8] + assert unique2 == @["a", "c", "d"] + result = @[] - for itm in items(seq1): - if not result.contains(itm): result.add(itm) - -{.deprecated: [distnct: deduplicate].} - -proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = - ## Returns a new sequence with a combination of the two input sequences. - ## - ## For convenience you can access the returned tuples through the named - ## fields `a` and `b`. If one sequence is shorter, the remaining items in the - ## longer sequence are discarded. Example: - ## - ## .. code-block:: - ## let - ## short = @[1, 2, 3] - ## long = @[6, 5, 4, 3, 2, 1] - ## words = @["one", "two", "three"] - ## zip1 = zip(short, long) - ## zip2 = zip(short, words) - ## assert zip1 == @[(1, 6), (2, 5), (3, 4)] - ## assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] - ## assert zip1[2].b == 4 - ## assert zip2[2].b == "three" - var m = min(seq1.len, seq2.len) - newSeq(result, m) - for i in 0 .. m-1: result[i] = (seq1[i], seq2[i]) - -proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = - ## Splits and distributes a sequence `s` into `num` sub sequences. - ## - ## Returns a sequence of `num` sequences. For some input values this is the - ## inverse of the `concat <#concat>`_ proc. The proc will assert in debug - ## builds if `s` is nil or `num` is less than one, and will likely crash on - ## release builds. The input sequence `s` can be empty, which will produce + if s.len > 0: + if isSorted: + var prev = s[0] + result.add(prev) + for i in 1..s.high: + if s[i] != prev: + prev = s[i] + result.add(prev) + else: + for itm in items(s): + if not result.contains(itm): result.add(itm) + +func minIndex*[T](s: openArray[T]): int {.since: (1, 1).} = + ## Returns the index of the minimum value of `s`. + ## `T` needs to have a `<` operator. + runnableExamples: + let + a = @[1, 2, 3, 4] + b = @[6, 5, 4, 3] + c = [2, -7, 8, -5] + d = "ziggy" + assert minIndex(a) == 0 + assert minIndex(b) == 3 + assert minIndex(c) == 1 + assert minIndex(d) == 2 + + for i in 1..high(s): + if s[i] < s[result]: result = i + +func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = + ## Returns the index of the maximum value of `s`. + ## `T` needs to have a `<` operator. + runnableExamples: + let + a = @[1, 2, 3, 4] + b = @[6, 5, 4, 3] + c = [2, -7, 8, -5] + d = "ziggy" + assert maxIndex(a) == 3 + assert maxIndex(b) == 0 + assert maxIndex(c) == 2 + assert maxIndex(d) == 0 + + for i in 1..high(s): + if s[i] > s[result]: result = i + +func minmax*[T](x: openArray[T]): (T, T) = + ## The minimum and maximum values of `x`. `T` needs to have a `<` operator. + var l = x[0] + var h = x[0] + for i in 1..high(x): + if x[i] < l: l = x[i] + if h < x[i]: h = x[i] + result = (l, h) + + +template zipImpl(s1, s2, retType: untyped): untyped = + proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = + ## Returns a new sequence with a combination of the two input containers. + ## + ## The input containers can be of different types. + ## If one container is shorter, the remaining items in the longer container + ## are discarded. + ## + ## **Note**: For Nim 1.0.x and older version, `zip` returned a seq of + ## named tuples with fields `a` and `b`. For Nim versions 1.1.x and newer, + ## `zip` returns a seq of unnamed tuples. + runnableExamples: + let + short = @[1, 2, 3] + long = @[6, 5, 4, 3, 2, 1] + words = @["one", "two", "three"] + letters = "abcd" + zip1 = zip(short, long) + zip2 = zip(short, words) + assert zip1 == @[(1, 6), (2, 5), (3, 4)] + assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] + assert zip1[2][0] == 3 + assert zip2[1][1] == "two" + when (NimMajor, NimMinor) <= (1, 0): + let + zip3 = zip(long, letters) + assert zip3 == @[(a: 6, b: 'a'), (5, 'b'), (4, 'c'), (3, 'd')] + assert zip3[0].b == 'a' + else: + let + zip3: seq[tuple[num: int, letter: char]] = zip(long, letters) + assert zip3 == @[(6, 'a'), (5, 'b'), (4, 'c'), (3, 'd')] + assert zip3[0].letter == 'a' + + var m = min(s1.len, s2.len) + newSeq(result, m) + for i in 0 ..< m: + result[i] = (s1[i], s2[i]) + +when (NimMajor, NimMinor) <= (1, 0): + zipImpl(s1, s2, seq[tuple[a: S, b: T]]) +else: + zipImpl(s1, s2, seq[(S, T)]) + +proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = + ## Returns a tuple of two sequences split out from a sequence of 2-field tuples. + runnableExamples: + let + zipped = @[(1, 'a'), (2, 'b'), (3, 'c')] + unzipped1 = @[1, 2, 3] + unzipped2 = @['a', 'b', 'c'] + assert zipped.unzip() == (unzipped1, unzipped2) + assert zip(unzipped1, unzipped2).unzip() == (unzipped1, unzipped2) + result = (newSeq[S](s.len), newSeq[T](s.len)) + for i in 0..<s.len: + result[0][i] = s[i][0] + result[1][i] = s[i][1] + +func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = + ## Splits and distributes a sequence `s` into `num` sub-sequences. + ## + ## Returns a sequence of `num` sequences. For *some* input values this is the + ## inverse of the `concat <#concat,varargs[seq[T]]>`_ func. + ## The input sequence `s` can be empty, which will produce ## `num` empty sequences. ## ## If `spread` is false and the length of `s` is not a multiple of `num`, the - ## proc will max out the first sub sequences with ``1 + len(s) div num`` + ## func will max out the first sub-sequence with `1 + len(s) div num` ## entries, leaving the remainder of elements to the last sequence. ## - ## On the other hand, if `spread` is true, the proc will distribute evenly + ## On the other hand, if `spread` is true, the func will distribute evenly ## the remainder of the division across all sequences, which makes the result ## more suited to multithreading where you are passing equal sized work units ## to a thread pool and want to maximize core usage. ## - ## Example: - ## - ## .. code-block:: - ## let numbers = @[1, 2, 3, 4, 5, 6, 7] - ## assert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - ## assert numbers.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - ## assert numbers.distribute(6)[0] == @[1, 2] - ## assert numbers.distribute(6)[5] == @[7] - assert(not s.isNil, "`s` can't be nil") + runnableExamples: + let numbers = @[1, 2, 3, 4, 5, 6, 7] + assert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + assert numbers.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + assert numbers.distribute(6)[0] == @[1, 2] + assert numbers.distribute(6)[1] == @[3] + if num < 2: result = @[s] return - let num = int(num) # XXX probably only needed because of .. bug - # Create the result and calculate the stride size and the remainder if any. result = newSeq[seq[T]](num) var @@ -159,193 +372,266 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = if extra == 0 or spread == false: # Use an algorithm which overcounts the stride and minimizes reading limits. if extra > 0: inc(stride) - - for i in 0 .. <num: + for i in 0 ..< num: result[i] = newSeq[T]() - for g in first .. <min(s.len, first + stride): + for g in first ..< min(s.len, first + stride): result[i].add(s[g]) first += stride - else: # Use an undercounting algorithm which *adds* the remainder each iteration. - for i in 0 .. <num: + for i in 0 ..< num: last = first + stride if extra > 0: extra -= 1 inc(last) - result[i] = newSeq[T]() - for g in first .. <last: + for g in first ..< last: result[i].add(s[g]) first = last - -proc map*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): - seq[S]{.inline.} = - ## Returns a new sequence with the results of `op` applied to every item in - ## `data`. - ## - ## Since the input is not modified you can use this version of ``map`` to - ## transform the type of the elements in the input sequence. Example: - ## - ## .. code-block:: nim - ## let - ## a = @[1, 2, 3, 4] - ## b = map(a, proc(x: int): string = $x) - ## assert b == @["1", "2", "3", "4"] - newSeq(result, data.len) - for i in 0..data.len-1: result[i] = op(data[i]) - -proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) - {.deprecated.} = - ## Applies `op` to every item in `data` modifying it directly. - ## - ## Note that this version of ``map`` requires your input and output types to - ## be the same, since they are modified in-place. Example: - ## - ## .. code-block:: nim - ## var a = @["1", "2", "3", "4"] - ## echo repr(a) - ## # --> ["1", "2", "3", "4"] - ## map(a, proc(x: var string) = x &= "42") - ## echo repr(a) - ## # --> ["142", "242", "342", "442"] - ## **Deprecated since version 0.12.0:** Use the ``apply`` proc instead. - for i in 0..data.len-1: op(data[i]) - -proc apply*[T](data: var seq[T], op: proc (x: var T) {.closure.}) - {.inline.} = - ## Applies `op` to every item in `data` modifying it directly. - ## - ## Note that this requires your input and output types to - ## be the same, since they are modified in-place. - ## The parameter function takes a ``var T`` type parameter. - ## Example: - ## - ## .. code-block:: nim - ## var a = @["1", "2", "3", "4"] - ## echo repr(a) - ## # --> ["1", "2", "3", "4"] - ## map(a, proc(x: var string) = x &= "42") - ## echo repr(a) - ## # --> ["142", "242", "342", "442"] - ## - for i in 0..data.len-1: op(data[i]) - -proc apply*[T](data: var seq[T], op: proc (x: T): T {.closure.}) - {.inline.} = - ## Applies `op` to every item in `data` modifying it directly. - ## - ## Note that this requires your input and output types to - ## be the same, since they are modified in-place. - ## The parameter function takes and returns a ``T`` type variable. - ## Example: - ## - ## .. code-block:: nim - ## var a = @["1", "2", "3", "4"] - ## echo repr(a) - ## # --> ["1", "2", "3", "4"] - ## map(a, proc(x: string): string = x & "42") - ## echo repr(a) - ## # --> ["142", "242", "342", "442"] - ## - for i in 0..data.len-1: data[i] = op(data[i]) - - -iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T = - ## Iterates through a sequence and yields every item that fulfills the - ## predicate. +proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): + seq[S] {.inline, effectsOf: op.} = + ## Returns a new sequence with the results of the `op` proc applied to every + ## item in the container `s`. + ## + ## Since the input is not modified, you can use it to + ## transform the type of the elements in the input container. ## - ## Example: - ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): - ## echo($n) - ## # echoes 4, 8, 4 in separate lines - for i in 0..<seq1.len: - if pred(seq1[i]): - yield seq1[i] - -proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T] - {.inline.} = - ## Returns a new sequence with all the items that fulfilled the predicate. - ## - ## Example: - ## - ## .. code-block:: - ## let - ## colors = @["red", "yellow", "black"] - ## f1 = filter(colors, proc(x: string): bool = x.len < 6) - ## f2 = filter(colors) do (x: string) -> bool : x.len > 5 - ## assert f1 == @["red", "black"] - ## assert f2 == @["yellow"] + ## Instead of using `map` and `filter`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `mapIt template<#mapIt.t,typed,untyped>`_ + ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ for the in-place version + ## + runnableExamples: + let + a = @[1, 2, 3, 4] + b = map(a, proc(x: int): string = $x) + assert b == @["1", "2", "3", "4"] + + newSeq(result, s.len) + for i in 0 ..< s.len: + result[i] = op(s[i]) + +proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) + {.inline, effectsOf: op.} = + ## Applies `op` to every item in `s`, modifying it directly. + ## + ## Note that the container `s` must be declared as a `var`, + ## since `s` is modified in-place. + ## The parameter function takes a `var T` type parameter. + ## + ## **See also:** + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ + ## * `map proc<#map,openArray[T],proc(T)>`_ + ## + runnableExamples: + var a = @["1", "2", "3", "4"] + apply(a, proc(x: var string) = x &= "42") + assert a == @["142", "242", "342", "442"] + + for i in 0 ..< s.len: op(s[i]) + +proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) + {.inline, effectsOf: op.} = + ## Applies `op` to every item in `s` modifying it directly. + ## + ## Note that the container `s` must be declared as a `var` + ## and it is required for your input and output types to + ## be the same, since `s` is modified in-place. + ## The parameter function takes and returns a `T` type variable. + ## + ## **See also:** + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ + ## * `map proc<#map,openArray[T],proc(T)>`_ + ## + runnableExamples: + var a = @["1", "2", "3", "4"] + apply(a, proc(x: string): string = x & "42") + assert a == @["142", "242", "342", "442"] + + for i in 0 ..< s.len: s[i] = op(s[i]) + +proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3), effectsOf: op.} = + ## Same as `apply` but for a proc that does not return anything + ## and does not mutate `s` directly. + runnableExamples: + var message: string + apply([0, 1, 2, 3, 4], proc(item: int) = message.addInt item) + assert message == "01234" + for i in 0 ..< s.len: op(s[i]) + +iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T {.effectsOf: pred.} = + ## Iterates through a container `s` and yields every item that fulfills the + ## predicate `pred` (a function that returns a `bool`). + ## + ## Instead of using `map` and `filter`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `filter proc<#filter,openArray[T],proc(T)>`_ + ## * `filterIt template<#filterIt.t,untyped,untyped>`_ + ## + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + var evens = newSeq[int]() + for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): + evens.add(n) + assert evens == @[4, 8, 4] + + for i in 0 ..< s.len: + if pred(s[i]): + yield s[i] + +proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] + {.inline, effectsOf: pred.} = + ## Returns a new sequence with all the items of `s` that fulfill the + ## predicate `pred` (a function that returns a `bool`). + ## + ## Instead of using `map` and `filter`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `filterIt template<#filterIt.t,untyped,untyped>`_ + ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ + ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ for the in-place version + ## + runnableExamples: + let + colors = @["red", "yellow", "black"] + f1 = filter(colors, proc(x: string): bool = x.len < 6) + f2 = filter(colors, proc(x: string): bool = x.contains('y')) + assert f1 == @["red", "black"] + assert f2 == @["yellow"] + result = newSeq[T]() - for i in 0..<seq1.len: - if pred(seq1[i]): - result.add(seq1[i]) + for i in 0 ..< s.len: + if pred(s[i]): + result.add(s[i]) -proc keepIf*[T](seq1: var seq[T], pred: proc(item: T): bool {.closure.}) - {.inline.} = - ## Keeps the items in the passed sequence if they fulfilled the predicate. - ## Same as the ``filter`` proc, but modifies the sequence directly. +proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) + {.inline, effectsOf: pred.} = + ## Keeps the items in the passed sequence `s` if they fulfill the + ## predicate `pred` (a function that returns a `bool`). ## - ## Example: + ## Note that `s` must be declared as a `var`. ## - ## .. code-block:: - ## var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] - ## keepIf(floats, proc(x: float): bool = x > 10) - ## assert floats == @[13.0, 12.5, 10.1] - var pos = 0 - for i in 0 .. <len(seq1): - if pred(seq1[i]): - if pos != i: - shallowCopy(seq1[pos], seq1[i]) - inc(pos) - setLen(seq1, pos) - -proc delete*[T](s: var seq[T]; first, last: Natural) = - ## Deletes in `s` the items at position `first` .. `last`. This modifies - ## `s` itself, it does not return a copy. + ## Similar to the `filter proc<#filter,openArray[T],proc(T)>`_, + ## but modifies the sequence directly. ## - ## Example: + ## **See also:** + ## * `keepItIf template<#keepItIf.t,seq,untyped>`_ + ## * `filter proc<#filter,openArray[T],proc(T)>`_ ## - ##.. code-block:: - ## let outcome = @[1,1,1,1,1,1,1,1] - ## var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - ## dest.delete(3, 8) - ## assert outcome == dest + runnableExamples: + var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] + keepIf(floats, proc(x: float): bool = x > 10) + assert floats == @[13.0, 12.5, 10.1] + var pos = 0 + for i in 0 ..< len(s): + if pred(s[i]): + if pos != i: + when defined(gcDestructors): + s[pos] = move(s[i]) + else: + shallowCopy(s[pos], s[i]) + inc(pos) + setLen(s, pos) + +func delete*[T](s: var seq[T]; slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. + ## + ## This operation moves all elements after `s[slice]` in linear time. + runnableExamples: + var a = @[10, 11, 12, 13, 14] + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == @[10, 11, 12, 13, 14] + a.delete(4..4) + assert a == @[10, 11, 12, 13] + a.delete(1..2) + assert a == @[10, 13] + a.delete(1..<1) # empty slice + assert a == @[10, 13] + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + template defaultImpl = + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + while i < newLen: + when defined(gcDestructors): + s[i] = move(s[j]) + else: + s[i].shallowCopy(s[j]) + inc(i) + inc(j) + setLen(s, newLen) + when nimvm: defaultImpl() + else: + when defined(js): + let n = slice.b - slice.a + 1 + let first = slice.a + {.emit: "`s`.splice(`first`, `n`);".} + else: + defaultImpl() + +func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete(s, first..last)`".} = + ## Deletes the items of a sequence `s` at positions `first..last` + ## (including both ends of the range). + ## This modifies `s` itself, it does not return a copy. + runnableExamples("--warning:deprecated:off"): + let outcome = @[1, 1, 1, 1, 1, 1, 1, 1] + var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] + dest.delete(3, 8) + assert outcome == dest + doAssert first <= last + if first >= s.len: + return var i = first - var j = last+1 - var newLen = len(s)-j+i + var j = min(len(s), last + 1) + var newLen = len(s) - j + i while i < newLen: - s[i].shallowCopy(s[j]) + when defined(gcDestructors): + s[i] = move(s[j]) + else: + s[i].shallowCopy(s[j]) inc(i) inc(j) setLen(s, newLen) -proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = +func insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = ## Inserts items from `src` into `dest` at position `pos`. This modifies ## `dest` itself, it does not return a copy. ## - ## Example: + ## Note that the elements of `src` and `dest` must be of the same type. ## - ##.. code-block:: - ## var dest = @[1,1,1,1,1,1,1,1] - ## let - ## src = @[2,2,2,2,2,2] - ## outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - ## dest.insert(src, 3) - ## assert dest == outcome + runnableExamples: + var dest = @[1, 1, 1, 1, 1, 1, 1, 1] + let + src = @[2, 2, 2, 2, 2, 2] + outcome = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] + dest.insert(src, 3) + assert dest == outcome var j = len(dest) - 1 - var i = len(dest) + len(src) - 1 + var i = j + len(src) + if i == j: return dest.setLen(i + 1) # Move items after `pos` to the end of the sequence. while j >= pos: - dest[i].shallowCopy(dest[j]) + when defined(gcDestructors): + dest[i] = move(dest[j]) + else: + dest[i].shallowCopy(dest[j]) dec(i) dec(j) # Insert items from `dest` into `dest` at `pos` @@ -355,405 +641,342 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = inc(j) -template filterIt*(seq1, pred: expr): expr = - ## Returns a new sequence with all the items that fulfilled the predicate. +template filterIt*(s, pred: untyped): untyped = + ## Returns a new sequence with all the items of `s` that fulfill the + ## predicate `pred`. + ## + ## Unlike the `filter proc<#filter,openArray[T],proc(T)>`_ and + ## `filter iterator<#filter.i,openArray[T],proc(T)>`_, + ## the predicate needs to be an expression using the `it` variable + ## for testing, like: `filterIt("abcxyz", it == 'x')`. ## - ## Unlike the `proc` version, the predicate needs to be an expression using - ## the ``it`` variable for testing, like: ``filterIt("abcxyz", it == 'x')``. - ## Example: + ## Instead of using `mapIt` and `filterIt`, consider using the `collect` macro + ## from the `sugar` module. ## - ## .. code-block:: - ## let - ## temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] - ## acceptable = filterIt(temperatures, it < 50 and it > -10) - ## notAcceptable = filterIt(temperatures, it > 50 or it < -10) - ## assert acceptable == @[-2.0, 24.5, 44.31] - ## assert notAcceptable == @[-272.15, 99.9, -113.44] - var result {.gensym.} = newSeq[type(seq1[0])]() - for it {.inject.} in items(seq1): + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `filter proc<#filter,openArray[T],proc(T)>`_ + ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ + ## + runnableExamples: + let + temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] + acceptable = temperatures.filterIt(it < 50 and it > -10) + notAcceptable = temperatures.filterIt(it > 50 or it < -10) + assert acceptable == @[-2.0, 24.5, 44.31] + assert notAcceptable == @[-272.15, 99.9, -113.44] + + var result = newSeq[typeof(s[0])]() + for it {.inject.} in items(s): if pred: result.add(it) result -template keepItIf*(varSeq: seq, pred: expr) = - ## Convenience template around the ``keepIf`` proc to reduce typing. +template keepItIf*(varSeq: seq, pred: untyped) = + ## Keeps the items in the passed sequence (must be declared as a `var`) + ## if they fulfill the predicate. + ## + ## Unlike the `keepIf proc<#keepIf,seq[T],proc(T)>`_, + ## the predicate needs to be an expression using + ## the `it` variable for testing, like: `keepItIf("abcxyz", it == 'x')`. ## - ## Unlike the `proc` version, the predicate needs to be an expression using - ## the ``it`` variable for testing, like: ``keepItIf("abcxyz", it == 'x')``. - ## Example: + ## **See also:** + ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ + ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## - ## .. code-block:: - ## var candidates = @["foo", "bar", "baz", "foobar"] - ## keepItIf(candidates, it.len == 3 and it[0] == 'b') - ## assert candidates == @["bar", "baz"] + runnableExamples: + var candidates = @["foo", "bar", "baz", "foobar"] + candidates.keepItIf(it.len == 3 and it[0] == 'b') + assert candidates == @["bar", "baz"] + var pos = 0 - for i in 0 .. <len(varSeq): + for i in 0 ..< len(varSeq): let it {.inject.} = varSeq[i] if pred: if pos != i: - shallowCopy(varSeq[pos], varSeq[i]) + when defined(gcDestructors): + varSeq[pos] = move(varSeq[i]) + else: + shallowCopy(varSeq[pos], varSeq[i]) inc(pos) setLen(varSeq, pos) -proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = - ## Iterates through a sequence and checks if every item fulfills the +since (1, 1): + template countIt*(s, pred: untyped): int = + ## Returns a count of all the items that fulfill the predicate. + ## + ## The predicate needs to be an expression using + ## the `it` variable for testing, like: `countIt(@[1, 2, 3], it > 2)`. + ## + runnableExamples: + let numbers = @[-3, -2, -1, 0, 1, 2, 3, 4, 5, 6] + iterator iota(n: int): int = + for i in 0..<n: yield i + assert numbers.countIt(it < 0) == 3 + assert countIt(iota(10), it < 2) == 2 + + var result = 0 + for it {.inject.} in s: + if pred: result += 1 + result + +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = + ## Iterates through a container and checks if every item fulfills the ## predicate. ## - ## Example: + ## **See also:** + ## * `allIt template<#allIt.t,untyped,untyped>`_ + ## * `any proc<#any,openArray[T],proc(T)>`_ ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert all(numbers, proc (x: int): bool = return x < 10) == true - ## assert all(numbers, proc (x: int): bool = return x < 9) == false - for i in seq1: + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert all(numbers, proc (x: int): bool = x < 10) == true + assert all(numbers, proc (x: int): bool = x < 9) == false + + for i in s: if not pred(i): return false - return true + true -template allIt*(seq1, pred: expr): bool {.immediate.} = - ## Checks if every item fulfills the predicate. +template allIt*(s, pred: untyped): bool = + ## Iterates through a container and checks if every item fulfills the + ## predicate. ## - ## Example: + ## Unlike the `all proc<#all,openArray[T],proc(T)>`_, + ## the predicate needs to be an expression using + ## the `it` variable for testing, like: `allIt("abba", it == 'a')`. ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert allIt(numbers, it < 10) == true - ## assert allIt(numbers, it < 9) == false - var result {.gensym.} = true - for it {.inject.} in items(seq1): + ## **See also:** + ## * `all proc<#all,openArray[T],proc(T)>`_ + ## * `anyIt template<#anyIt.t,untyped,untyped>`_ + ## + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert numbers.allIt(it < 10) == true + assert numbers.allIt(it < 9) == false + + var result = true + for it {.inject.} in items(s): if not pred: result = false break result -proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = - ## Iterates through a sequence and checks if some item fulfills the - ## predicate. +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = + ## Iterates through a container and checks if at least one item + ## fulfills the predicate. ## - ## Example: + ## **See also:** + ## * `anyIt template<#anyIt.t,untyped,untyped>`_ + ## * `all proc<#all,openArray[T],proc(T)>`_ ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert any(numbers, proc (x: int): bool = return x > 8) == true - ## assert any(numbers, proc (x: int): bool = return x > 9) == false - for i in seq1: + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert any(numbers, proc (x: int): bool = x > 8) == true + assert any(numbers, proc (x: int): bool = x > 9) == false + + for i in s: if pred(i): return true - return false + false -template anyIt*(seq1, pred: expr): bool {.immediate.} = - ## Checks if some item fulfills the predicate. +template anyIt*(s, pred: untyped): bool = + ## Iterates through a container and checks if at least one item + ## fulfills the predicate. + ## + ## Unlike the `any proc<#any,openArray[T],proc(T)>`_, + ## the predicate needs to be an expression using + ## the `it` variable for testing, like: `anyIt("abba", it == 'a')`. ## - ## Example: + ## **See also:** + ## * `any proc<#any,openArray[T],proc(T)>`_ + ## * `allIt template<#allIt.t,untyped,untyped>`_ ## - ## .. code-block:: - ## let numbers = @[1, 4, 5, 8, 9, 7, 4] - ## assert anyIt(numbers, it > 8) == true - ## assert anyIt(numbers, it > 9) == false - var result {.gensym.} = false - for it {.inject.} in items(seq1): + runnableExamples: + let numbers = @[1, 4, 5, 8, 9, 7, 4] + assert numbers.anyIt(it > 8) == true + assert numbers.anyIt(it > 9) == false + + var result = false + for it {.inject.} in items(s): if pred: result = true break result -template toSeq*(iter: expr): expr {.immediate.} = - ## Transforms any iterator into a sequence. - ## - ## Example: - ## - ## .. code-block:: - ## let - ## numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9] - ## odd_numbers = toSeq(filter(numeric) do (x: int) -> bool: - ## if x mod 2 == 1: - ## result = true) - ## assert odd_numbers == @[1, 3, 5, 7, 9] +template toSeq1(s: not iterator): untyped = + # overload for typed but not iterator + type OutType = typeof(items(s)) + when compiles(s.len): + block: + evalOnceAs(s2, s, compiles((let _ = s))) + var i = 0 + var result = newSeq[OutType](s2.len) + for it in s2: + result[i] = it + i += 1 + result + else: + var result: seq[OutType]# = @[] + for it in s: + result.add(it) + result - when compiles(iter.len): +template toSeq2(iter: iterator): untyped = + # overload for iterator + evalOnceAs(iter2, iter(), false) + when compiles(iter2.len): var i = 0 - var result = newSeq[type(iter)](iter.len) - for x in iter: + var result = newSeq[typeof(iter2)](iter2.len) + for x in iter2: result[i] = x inc i result else: - var result: seq[type(iter)] = @[] - for x in iter: - result.add(x) + type OutType = typeof(iter2()) + var result: seq[OutType]# = @[] + when compiles(iter2()): + evalOnceAs(iter4, iter, false) + let iter3 = iter4() + for x in iter3(): + result.add(x) + else: + for x in iter2(): + result.add(x) result -template foldl*(sequence, operation: expr): expr = +template toSeq*(iter: untyped): untyped = + ## Transforms any iterable (anything that can be iterated over, e.g. with + ## a for-loop) into a sequence. + ## + runnableExamples: + let + myRange = 1..5 + mySet: set[int8] = {5'i8, 3, 1} + assert typeof(myRange) is HSlice[system.int, system.int] + assert typeof(mySet) is set[int8] + + let + mySeq1 = toSeq(myRange) + mySeq2 = toSeq(mySet) + assert mySeq1 == @[1, 2, 3, 4, 5] + assert mySeq2 == @[1'i8, 3, 5] + + when compiles(toSeq1(iter)): + toSeq1(iter) + elif compiles(toSeq2(iter)): + toSeq2(iter) + else: + # overload for untyped, e.g.: `toSeq(myInlineIterator(3))` + when compiles(iter.len): + block: + evalOnceAs(iter2, iter, true) + var result = newSeq[typeof(iter)](iter2.len) + var i = 0 + for x in iter2: + result[i] = x + inc i + result + else: + var result: seq[typeof(iter)] = @[] + for x in iter: + result.add(x) + result + +template foldl*(sequence, operation: untyped): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## ## The sequence is required to have at least a single element. Debug versions ## of your program will assert in this situation but release versions will ## happily go ahead. If the sequence has a single element it will be returned - ## without applying ``operation``. + ## without applying `operation`. ## - ## The ``operation`` parameter should be an expression which uses the - ## variables ``a`` and ``b`` for each step of the fold. Since this is a left + ## The `operation` parameter should be an expression which uses the + ## variables `a` and `b` for each step of the fold. Since this is a left ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (((1) - 2) - - ## 3). Example: - ## - ## .. code-block:: - ## let - ## numbers = @[5, 9, 11] - ## addition = foldl(numbers, a + b) - ## subtraction = foldl(numbers, a - b) - ## multiplication = foldl(numbers, a * b) - ## words = @["nim", "is", "cool"] - ## concatenation = foldl(words, a & b) - ## assert addition == 25, "Addition is (((5)+9)+11)" - ## assert subtraction == -15, "Subtraction is (((5)-9)-11)" - ## assert multiplication == 495, "Multiplication is (((5)*9)*11)" - ## assert concatenation == "nimiscool" - assert sequence.len > 0, "Can't fold empty sequences" - var result {.gensym.}: type(sequence[0]) - result = sequence[0] - for i in 1..<sequence.len: - let - a {.inject.} = result - b {.inject.} = sequence[i] - result = operation - result - -template foldr*(sequence, operation: expr): expr = - ## Template to fold a sequence from right to left, returning the accumulation. + ## 3). ## - ## The sequence is required to have at least a single element. Debug versions - ## of your program will assert in this situation but release versions will - ## happily go ahead. If the sequence has a single element it will be returned - ## without applying ``operation``. + ## **See also:** + ## * `foldl template<#foldl.t,,,>`_ with a starting parameter + ## * `foldr template<#foldr.t,untyped,untyped>`_ ## - ## The ``operation`` parameter should be an expression which uses the - ## variables ``a`` and ``b`` for each step of the fold. Since this is a right - ## fold, for non associative binary operations like subtraction think that - ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 - - ## (3))). Example: - ## - ## .. code-block:: - ## let - ## numbers = @[5, 9, 11] - ## addition = foldr(numbers, a + b) - ## subtraction = foldr(numbers, a - b) - ## multiplication = foldr(numbers, a * b) - ## words = @["nim", "is", "cool"] - ## concatenation = foldr(words, a & b) - ## assert addition == 25, "Addition is (5+(9+(11)))" - ## assert subtraction == 7, "Subtraction is (5-(9-(11)))" - ## assert multiplication == 495, "Multiplication is (5*(9*(11)))" - ## assert concatenation == "nimiscool" - assert sequence.len > 0, "Can't fold empty sequences" - var result {.gensym.}: type(sequence[0]) - result = sequence[sequence.len - 1] - for i in countdown(sequence.len - 2, 0): + runnableExamples: let - a {.inject.} = sequence[i] - b {.inject.} = result - result = operation - result + numbers = @[5, 9, 11] + addition = foldl(numbers, a + b) + subtraction = foldl(numbers, a - b) + multiplication = foldl(numbers, a * b) + words = @["nim", "is", "cool"] + concatenation = foldl(words, a & b) + procs = @["proc", "Is", "Also", "Fine"] -template mapIt*(seq1, typ, op: expr): expr {.deprecated.}= - ## Convenience template around the ``map`` proc to reduce typing. - ## - ## The template injects the ``it`` variable which you can use directly in an - ## expression. You also need to pass as `typ` the type of the expression, - ## since the new returned sequence can have a different type than the - ## original. Example: - ## - ## .. code-block:: - ## let - ## nums = @[1, 2, 3, 4] - ## strings = nums.mapIt(string, $(4 * it)) - ## assert strings == @["4", "8", "12", "16"] - ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)`` - ## template instead. - var result {.gensym.}: seq[typ] = @[] - for it {.inject.} in items(seq1): - result.add(op) - result + func foo(acc, cur: string): string = + result = acc & cur -template mapIt*(seq1, op: expr): expr = - ## Convenience template around the ``map`` proc to reduce typing. - ## - ## The template injects the ``it`` variable which you can use directly in an - ## expression. Example: - ## - ## .. code-block:: - ## let - ## nums = @[1, 2, 3, 4] - ## strings = nums.mapIt($(4 * it)) - ## assert strings == @["4", "8", "12", "16"] - type outType = type(( - block: - var it{.inject.}: type(items(seq1)); - op)) - var result: seq[outType] - when compiles(seq1.len): - let s = seq1 - var i = 0 - result = newSeq[outType](s.len) - for it {.inject.} in s: - result[i] = op - i += 1 - else: - result = @[] - for it {.inject.} in seq1: - result.add(op) + assert addition == 25, "Addition is (((5)+9)+11)" + assert subtraction == -15, "Subtraction is (((5)-9)-11)" + assert multiplication == 495, "Multiplication is (((5)*9)*11)" + assert concatenation == "nimiscool" + assert foldl(procs, foo(a, b)) == "procIsAlsoFine" + + let s = sequence + assert s.len > 0, "Can't fold empty sequences" + var result: typeof(s[0]) + result = s[0] + for i in 1..<s.len: + let + a {.inject.} = result + b {.inject.} = s[i] + result = operation result -template applyIt*(varSeq, op: expr) = - ## Convenience template around the mutable ``apply`` proc to reduce typing. +template foldl*(sequence, operation, first): untyped = + ## Template to fold a sequence from left to right, returning the accumulation. ## - ## The template injects the ``it`` variable which you can use directly in an - ## expression. The expression has to return the same type as the sequence you - ## are mutating. Example: + ## This version of `foldl` gets a **starting parameter**. This makes it possible + ## to accumulate the sequence into a different type than the sequence elements. ## - ## .. code-block:: - ## var nums = @[1, 2, 3, 4] - ## nums.applyIt(it * 3) - ## assert nums[0] + nums[3] == 15 - for i in 0 .. <varSeq.len: - let it {.inject.} = varSeq[i] - varSeq[i] = op - - - -template newSeqWith*(len: int, init: expr): expr = - ## creates a new sequence, calling `init` to initialize each value. Example: + ## The `operation` parameter should be an expression which uses the variables + ## `a` and `b` for each step of the fold. The `first` parameter is the + ## start value (the first `a`) and therefore defines the type of the result. ## - ## .. code-block:: - ## var seq2D = newSeqWith(20, newSeq[bool](10)) - ## seq2D[0][0] = true - ## seq2D[1][0] = true - ## seq2D[0][1] = true + ## **See also:** + ## * `foldr template<#foldr.t,untyped,untyped>`_ ## - ## import math - ## var seqRand = newSeqWith(20, random(10)) - ## echo seqRand - var result {.gensym.} = newSeq[type(init)](len) - for i in 0 .. <len: - result[i] = init - result - -when isMainModule: - import strutils - block: # concat test + runnableExamples: let - s1 = @[1, 2, 3] - s2 = @[4, 5] - s3 = @[6, 7] - total = concat(s1, s2, s3) - assert total == @[1, 2, 3, 4, 5, 6, 7] + numbers = @[0, 8, 1, 5] + digits = foldl(numbers, a & (chr(b + ord('0'))), "") + assert digits == "0815" - block: # duplicates test + var result: typeof(first) = first + for x in items(sequence): let - dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] - dup2 = @["a", "a", "c", "d", "d"] - unique1 = deduplicate(dup1) - unique2 = deduplicate(dup2) - assert unique1 == @[1, 3, 4, 2, 8] - assert unique2 == @["a", "c", "d"] - - block: # zip test - let - short = @[1, 2, 3] - long = @[6, 5, 4, 3, 2, 1] - words = @["one", "two", "three"] - zip1 = zip(short, long) - zip2 = zip(short, words) - assert zip1 == @[(1, 6), (2, 5), (3, 4)] - assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] - assert zip1[2].b == 4 - assert zip2[2].b == "three" - - block: # filter proc test - let - colors = @["red", "yellow", "black"] - f1 = filter(colors, proc(x: string): bool = x.len < 6) - f2 = filter(colors) do (x: string) -> bool : x.len > 5 - assert f1 == @["red", "black"] - assert f2 == @["yellow"] - - block: # filter iterator test - let numbers = @[1, 4, 5, 8, 9, 7, 4] - assert toSeq(filter(numbers, proc (x: int): bool = x mod 2 == 0)) == - @[4, 8, 4] - - block: # keepIf test - var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] - keepIf(floats, proc(x: float): bool = x > 10) - assert floats == @[13.0, 12.5, 10.1] - - block: # filterIt test - let - temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] - acceptable = filterIt(temperatures, it < 50 and it > -10) - notAcceptable = filterIt(temperatures, it > 50 or it < -10) - assert acceptable == @[-2.0, 24.5, 44.31] - assert notAcceptable == @[-272.15, 99.9, -113.44] - - block: # keepItIf test - var candidates = @["foo", "bar", "baz", "foobar"] - keepItIf(candidates, it.len == 3 and it[0] == 'b') - assert candidates == @["bar", "baz"] - - block: # any - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - len0seq : seq[int] = @[] - assert any(numbers, proc (x: int): bool = return x > 8) == true - assert any(numbers, proc (x: int): bool = return x > 9) == false - assert any(len0seq, proc (x: int): bool = return true) == false - - block: # anyIt - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - len0seq : seq[int] = @[] - assert anyIt(numbers, it > 8) == true - assert anyIt(numbers, it > 9) == false - assert anyIt(len0seq, true) == false - - block: # all - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - len0seq : seq[int] = @[] - assert all(numbers, proc (x: int): bool = return x < 10) == true - assert all(numbers, proc (x: int): bool = return x < 9) == false - assert all(len0seq, proc (x: int): bool = return false) == true - - block: # allIt - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - len0seq : seq[int] = @[] - assert allIt(numbers, it < 10) == true - assert allIt(numbers, it < 9) == false - assert allIt(len0seq, false) == true - - block: # toSeq test - let - numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9] - odd_numbers = toSeq(filter(numeric) do (x: int) -> bool: - if x mod 2 == 1: - result = true) - assert odd_numbers == @[1, 3, 5, 7, 9] - - block: # foldl tests - let - numbers = @[5, 9, 11] - addition = foldl(numbers, a + b) - subtraction = foldl(numbers, a - b) - multiplication = foldl(numbers, a * b) - words = @["nim", "is", "cool"] - concatenation = foldl(words, a & b) - assert addition == 25, "Addition is (((5)+9)+11)" - assert subtraction == -15, "Subtraction is (((5)-9)-11)" - assert multiplication == 495, "Multiplication is (((5)*9)*11)" - assert concatenation == "nimiscool" + a {.inject.} = result + b {.inject.} = x + result = operation + result - block: # foldr tests +template foldr*(sequence, operation: untyped): untyped = + ## Template to fold a sequence from right to left, returning the accumulation. + ## + ## The sequence is required to have at least a single element. Debug versions + ## of your program will assert in this situation but release versions will + ## happily go ahead. If the sequence has a single element it will be returned + ## without applying `operation`. + ## + ## The `operation` parameter should be an expression which uses the + ## variables `a` and `b` for each step of the fold. Since this is a right + ## fold, for non associative binary operations like subtraction think that + ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 - + ## (3))). + ## + ## **See also:** + ## * `foldl template<#foldl.t,untyped,untyped>`_ + ## * `foldl template<#foldl.t,,,>`_ with a starting parameter + ## + runnableExamples: let numbers = @[5, 9, 11] addition = foldr(numbers, a + b) @@ -766,77 +989,174 @@ when isMainModule: assert multiplication == 495, "Multiplication is (5*(9*(11)))" assert concatenation == "nimiscool" - block: # delete tests - let outcome = @[1,1,1,1,1,1,1,1] - var dest = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - dest.delete(3, 8) - assert outcome == dest, """\ - Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1] - is [1,1,1,1,1,1,1,1]""" - - block: # insert tests - var dest = @[1,1,1,1,1,1,1,1] + let s = sequence # xxx inefficient, use {.evalonce.} pending #13750 + let n = s.len + assert n > 0, "Can't fold empty sequences" + var result = s[n - 1] + for i in countdown(n - 2, 0): let - src = @[2,2,2,2,2,2] - outcome = @[1,1,1,2,2,2,2,2,2,1,1,1,1,1] - dest.insert(src, 3) - assert dest == outcome, """\ - Inserting [2,2,2,2,2,2] into [1,1,1,1,1,1,1,1] - at 3 is [1,1,1,2,2,2,2,2,2,1,1,1,1,1]""" + a {.inject.} = s[i] + b {.inject.} = result + result = operation + result - block: # mapIt tests - var +template mapIt*(s: typed, op: untyped): untyped = + ## Returns a new sequence with the results of the `op` proc applied to every + ## item in the container `s`. + ## + ## Since the input is not modified you can use it to + ## transform the type of the elements in the input container. + ## + ## The template injects the `it` variable which you can use directly in an + ## expression. + ## + ## Instead of using `mapIt` and `filterIt`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `map proc<#map,openArray[T],proc(T)>`_ + ## * `applyIt template<#applyIt.t,untyped,untyped>`_ for the in-place version + ## + runnableExamples: + let nums = @[1, 2, 3, 4] strings = nums.mapIt($(4 * it)) + assert strings == @["4", "8", "12", "16"] + + type OutType = typeof(( + block: + var it{.inject.}: typeof(items(s), typeOfIter); + op), typeOfProc) + when OutType is not (proc): + # Here, we avoid to create closures in loops. + # This avoids https://github.com/nim-lang/Nim/issues/12625 + when compiles(s.len): + block: # using a block avoids https://github.com/nim-lang/Nim/issues/8580 + + # BUG: `evalOnceAs(s2, s, false)` would lead to C compile errors + # (`error: use of undeclared identifier`) instead of Nim compile errors + evalOnceAs(s2, s, compiles((let _ = s))) + + var i = 0 + var result = newSeq[OutType](s2.len) + for it {.inject.} in s2: + result[i] = op + i += 1 + result + else: + var result: seq[OutType]# = @[] + # use `items` to avoid https://github.com/nim-lang/Nim/issues/12639 + for it {.inject.} in items(s): + result.add(op) + result + else: + # `op` is going to create closures in loops, let's fallback to `map`. + # NOTE: Without this fallback, developers have to define a helper function and + # call `map`: + # [1, 2].map((it) => ((x: int) => it + x)) + # With this fallback, above code can be simplified to: + # [1, 2].mapIt((x: int) => it + x) + # In this case, `mapIt` is just syntax sugar for `map`. + type InType = typeof(items(s), typeOfIter) + # Use a help proc `f` to create closures for each element in `s` + let f = proc (x: InType): OutType = + let it {.inject.} = x + op + map(s, f) + +template applyIt*(varSeq, op: untyped) = + ## Convenience template around the mutable `apply` proc to reduce typing. + ## + ## The template injects the `it` variable which you can use directly in an + ## expression. The expression has to return the same type as the elements + ## of the sequence you are mutating. + ## + ## **See also:** + ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ + ## * `mapIt template<#mapIt.t,typed,untyped>`_ + ## + runnableExamples: + var nums = @[1, 2, 3, 4] nums.applyIt(it * 3) assert nums[0] + nums[3] == 15 - block: # distribute tests - let numbers = @[1, 2, 3, 4, 5, 6, 7] - doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert numbers.distribute(6)[0] == @[1, 2] - doAssert numbers.distribute(6)[5] == @[7] - let a = @[1, 2, 3, 4, 5, 6, 7] - doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] - doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] - doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] - doAssert a.distribute(6, false) == @[ - @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] - doAssert a.distribute(8, false) == a.distribute(8, true) - doAssert a.distribute(90, false) == a.distribute(90, true) - var b = @[0] - for f in 1 .. 25: b.add(f) - doAssert b.distribute(5, true)[4].len == 5 - doAssert b.distribute(5, false)[4].len == 2 - - block: # newSeqWith tests - var seq2D = newSeqWith(4, newSeq[bool](2)) - seq2D[0][0] = true - seq2D[1][0] = true - seq2D[0][1] = true - doAssert seq2D == @[@[true, true], @[true, false], @[false, false], @[false, false]] - - block: # cycle tests - let - a = @[1, 2, 3] - b: seq[int] = @[] - - doAssert a.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] - doAssert a.cycle(0) == @[] - #doAssert a.cycle(-1) == @[] # will not compile! - doAssert b.cycle(3) == @[] + for i in low(varSeq) .. high(varSeq): + let it {.inject.} = varSeq[i] + varSeq[i] = op - block: # repeat tests - assert repeat(10, 5) == @[10, 10, 10, 10, 10] - assert repeat(@[1,2,3], 2) == @[@[1,2,3], @[1,2,3]] - when not defined(testing): - echo "Finished doc tests" +template newSeqWith*(len: int, init: untyped): untyped = + ## Creates a new `seq` of length `len`, calling `init` to initialize + ## each value of the seq. + ## + ## Useful for creating "2D" seqs - seqs containing other seqs + ## or to populate fields of the created seq. + runnableExamples: + ## Creates a seq containing 5 bool seqs, each of length of 3. + var seq2D = newSeqWith(5, newSeq[bool](3)) + assert seq2D.len == 5 + assert seq2D[0].len == 3 + assert seq2D[4][2] == false + + ## Creates a seq with random numbers + import std/random + var seqRand = newSeqWith(20, rand(1.0)) + assert seqRand[0] != seqRand[1] + type T = typeof(init) + let newLen = len + when supportsCopyMem(T) and declared(newSeqUninit): + var result = newSeqUninit[T](newLen) + else: # TODO: use `newSeqUnsafe` when that's available + var result = newSeq[T](newLen) + for i in 0 ..< newLen: + result[i] = init + move(result) # refs bug #7295 + +func mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; + filter = nnkLiterals): NimNode = + if constructor.kind in filter: + result = newNimNode(nnkCall, lineInfoFrom = constructor) + result.add op + result.add constructor + else: + result = copyNimNode(constructor) + for v in constructor: + if nested or v.kind in filter: + result.add mapLitsImpl(v, op, nested, filter) + else: + result.add v + +macro mapLiterals*(constructor, op: untyped; + nested = true): untyped = + ## Applies `op` to each of the **atomic** literals like `3` + ## or `"abc"` in the specified `constructor` AST. This can + ## be used to map every array element to some target type: + runnableExamples: + let x = mapLiterals([0.1, 1.2, 2.3, 3.4], int) + doAssert x is array[4, int] + doAssert x == [int(0.1), int(1.2), int(2.3), int(3.4)] + ## If `nested` is true (which is the default), the literals are replaced + ## everywhere in the `constructor` AST, otherwise only the first level + ## is considered: + runnableExamples: + let a = mapLiterals((1.2, (2.3, 3.4), 4.8), int) + let b = mapLiterals((1.2, (2.3, 3.4), 4.8), int, nested=false) + assert a == (1, (2, 3), 4) + assert b == (1, (2.3, 3.4), 4) + + let c = mapLiterals((1, (2, 3), 4, (5, 6)), `$`) + let d = mapLiterals((1, (2, 3), 4, (5, 6)), `$`, nested=false) + assert c == ("1", ("2", "3"), "4", ("5", "6")) + assert d == ("1", (2, 3), "4", (5, 6)) + ## There are no constraints for the `constructor` AST, it + ## works for nested tuples of arrays of sets etc. + result = mapLitsImpl(constructor, op, nested.boolVal) + +iterator items*[T](xs: iterator: T): T = + ## Iterates over each element yielded by a closure iterator. This may + ## not seem particularly useful on its own, but this allows closure + ## iterators to be used by the mapIt, filterIt, allIt, anyIt, etc. + ## templates. + for x in xs(): + yield x diff --git a/lib/pure/collections/setimpl.nim b/lib/pure/collections/setimpl.nim new file mode 100644 index 000000000..360a075d6 --- /dev/null +++ b/lib/pure/collections/setimpl.nim @@ -0,0 +1,156 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# An `include` file for the different hash set implementations. + + +template maxHash(t): untyped = high(t.data) +template dataLen(t): untyped = len(t.data) + +include hashcommon + +template initImpl(s: typed, size: int) = + let correctSize = slotsNeeded(size) + when s is OrderedSet: + s.first = -1 + s.last = -1 + s.counter = 0 + newSeq(s.data, correctSize) + +template rawInsertImpl() {.dirty.} = + if data.len == 0: + initImpl(s, defaultInitialSize) + data[h].key = key + data[h].hcode = hc + +proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A, + hc: Hash, h: Hash) = + rawInsertImpl() + +proc enlarge[A](s: var HashSet[A]) = + var n: KeyValuePairSeq[A] + newSeq(n, len(s.data) * growthFactor) + swap(s.data, n) # n is now old seq + for i in countup(0, high(n)): + if isFilled(n[i].hcode): + var j = -1 - rawGetKnownHC(s, n[i].key, n[i].hcode) + rawInsert(s, s.data, n[i].key, n[i].hcode, j) + +template inclImpl() {.dirty.} = + if s.data.len == 0: + initImpl(s, defaultInitialSize) + var hc: Hash + var index = rawGet(s, key, hc) + if index < 0: + if mustRehash(s): + enlarge(s) + index = rawGetKnownHC(s, key, hc) + rawInsert(s, s.data, key, hc, -1 - index) + inc(s.counter) + +template containsOrInclImpl() {.dirty.} = + if s.data.len == 0: + initImpl(s, defaultInitialSize) + var hc: Hash + var index = rawGet(s, key, hc) + if index >= 0: + result = true + else: + result = false + if mustRehash(s): + enlarge(s) + index = rawGetKnownHC(s, key, hc) + rawInsert(s, s.data, key, hc, -1 - index) + inc(s.counter) + +template doWhile(a, b) = + while true: + b + if not a: break + +proc exclImpl[A](s: var HashSet[A], key: A): bool {.inline.} = + var hc: Hash + var i = rawGet(s, key, hc) + var msk = high(s.data) + result = true + + if i >= 0: + result = false + dec(s.counter) + while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 + var j = i # The correctness of this depends on (h+1) in nextTry, + var r = j # though may be adaptable to other simple sequences. + s.data[i].hcode = 0 # mark current EMPTY + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} + doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + i = (i + 1) and msk # increment mod table size + if isEmpty(s.data[i].hcode): # end of collision cluster; So all done + return + r = s.data[i].hcode and msk # "home" location of key@i + s.data[j] = move(s.data[i]) # data[i] will be marked EMPTY next loop + +template dollarImpl() {.dirty.} = + result = "{" + for key in items(s): + if result.len > 1: result.add(", ") + result.addQuoted(key) + result.add("}") + + + +# --------------------------- OrderedSet ------------------------------ + +proc rawGet[A](t: OrderedSet[A], key: A, hc: var Hash): int {.inline.} = + rawGetImpl() + +proc rawInsert[A](s: var OrderedSet[A], data: var OrderedKeyValuePairSeq[A], + key: A, hc: Hash, h: Hash) = + rawInsertImpl() + data[h].next = -1 + if s.first < 0: s.first = h + if s.last >= 0: data[s.last].next = h + s.last = h + +proc enlarge[A](s: var OrderedSet[A]) = + var n: OrderedKeyValuePairSeq[A] + newSeq(n, len(s.data) * growthFactor) + var h = s.first + s.first = -1 + s.last = -1 + swap(s.data, n) + while h >= 0: + var nxt = n[h].next + if isFilled(n[h].hcode): + var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode) + rawInsert(s, s.data, n[h].key, n[h].hcode, j) + h = nxt + +proc exclImpl[A](s: var OrderedSet[A], key: A): bool {.inline.} = + if len(s.data) == 0: + return true + var n: OrderedKeyValuePairSeq[A] + newSeq(n, len(s.data)) + var h = s.first + s.first = -1 + s.last = -1 + swap(s.data, n) + let hc = genHash(key) + result = true + while h >= 0: + var nxt = n[h].next + if isFilled(n[h].hcode): + if n[h].hcode == hc and n[h].key == key: + dec s.counter + result = false + else: + var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode) + rawInsert(s, s.data, n[h].key, n[h].hcode, j) + h = nxt diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index abe9cf85e..af13135aa 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -7,23 +7,55 @@ # distribution, for details about the copyright. # -## The ``sets`` module implements an efficient `hash set`:idx: and +## The `sets` module implements an efficient `hash set`:idx: and ## ordered hash set. ## ## Hash sets are different from the `built in set type -## <manual.html#set-type>`_. Sets allow you to store any value that can be +## <manual.html#types-set-type>`_. Sets allow you to store any value that can be ## `hashed <hashes.html>`_ and they don't contain duplicate entries. ## -## **Note**: The data types declared here have *value semantics*: This means -## that ``=`` performs a copy of the set. +## Common usages of sets: +## * removing duplicates from a container by converting it with `toHashSet proc +## <#toHashSet,openArray[A]>`_ (see also `sequtils.deduplicate func +## <sequtils.html#deduplicate,openArray[T],bool>`_) +## * membership testing +## * mathematical operations on two sets, such as +## `union <#union,HashSet[A],HashSet[A]>`_, +## `intersection <#intersection,HashSet[A],HashSet[A]>`_, +## `difference <#difference,HashSet[A],HashSet[A]>`_, and +## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ +## +## **Examples:** +## +## ```Nim +## echo toHashSet([9, 5, 1]) # {9, 1, 5} +## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} +## +## let +## s1 = toHashSet([9, 5, 1]) +## s2 = toHashSet([3, 5, 7]) +## +## echo s1 + s2 # {9, 1, 3, 5, 7} +## echo s1 - s2 # {1, 9} +## echo s1 * s2 # {5} +## echo s1 -+- s2 # {9, 1, 3, 7} +## ``` +## +## Note: The data types declared here have *value semantics*: This means +## that `=` performs a copy of the set. +## +## **See also:** +## * `intsets module <intsets.html>`_ for efficient int sets +## * `tables module <tables.html>`_ for hash tables + import - os, hashes, math + std/[hashes, math] -{.pragma: myShallow.} -when not defined(nimhygiene): - {.pragma: dirty.} +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} +{.pragma: myShallow.} # For "integer-like A" that are too big for intsets/bit-vectors to be practical, # it would be best to shrink hcode to the same size as the integer. Larger # codes should never be needed, and this can pack more entries per cache-line. @@ -31,413 +63,403 @@ when not defined(nimhygiene): type KeyValuePair[A] = tuple[hcode: Hash, key: A] KeyValuePairSeq[A] = seq[KeyValuePair[A]] - HashSet* {.myShallow.}[A] = object ## \ + HashSet*[A] {.myShallow.} = object ## \ ## A generic hash set. ## - ## Use `init() <#init,HashSet[A],int>`_ or `initSet[type]() <#initSet>`_ + ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int -{.deprecated: [TSet: HashSet].} - -# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These -# two procs retain clarity of that encoding without the space cost of an enum. -proc isEmpty(hcode: Hash): bool {.inline.} = - result = hcode == 0 - -proc isFilled(hcode: Hash): bool {.inline.} = - result = hcode != 0 - -proc isValid*[A](s: HashSet[A]): bool = - ## Returns `true` if the set has been initialized with `initSet <#initSet>`_. - ## - ## Most operations over an uninitialized set will crash at runtime and - ## `assert <system.html#assert>`_ in debug builds. You can use this proc in - ## your own procs to verify that sets passed to your procs are correctly - ## initialized. Example: - ## - ## .. code-block :: - ## proc savePreferences(options: Set[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! - result = not s.data.isNil - -proc len*[A](s: HashSet[A]): int = - ## Returns the number of keys in `s`. - ## - ## Due to an implementation detail you can call this proc on variables which - ## have not been initialized yet. The proc will return zero as the length - ## then. Example: - ## - ## .. code-block:: - ## - ## var values: Set[int] - ## assert(not values.isValid) - ## assert values.len == 0 - result = s.counter +type + OrderedKeyValuePair[A] = tuple[ + hcode: Hash, next: int, key: A] + OrderedKeyValuePairSeq[A] = seq[OrderedKeyValuePair[A]] + OrderedSet*[A] {.myShallow.} = object ## \ + ## A generic hash set that remembers insertion order. + ## + ## Use `init proc <#init,OrderedSet[A]>`_ or `initOrderedSet proc + ## <#initOrderedSet>`_ before calling other procs on it. + data: OrderedKeyValuePairSeq[A] + counter, first, last: int + SomeSet*[A] = HashSet[A] | OrderedSet[A] + ## Type union representing `HashSet` or `OrderedSet`. -proc card*[A](s: HashSet[A]): int = - ## Alias for `len() <#len,TSet[A]>`_. - ## - ## Card stands for the `cardinality - ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. - result = s.counter +const + defaultInitialSize* = 64 -iterator items*[A](s: HashSet[A]): A = - ## Iterates over keys in the set `s`. - ## - ## If you need a sequence with the keys you can use `sequtils.toSeq() - ## <sequtils.html#toSeq>`_ on the iterator. Usage example: - ## - ## .. code-block:: - ## type - ## pair = tuple[a, b: int] - ## var - ## a, b = initSet[pair]() - ## a.incl((2, 3)) - ## a.incl((3, 2)) - ## a.incl((2, 3)) - ## for x, y in a.items: - ## b.incl((x - 2, y + 1)) - ## assert a.len == 2 - ## echo b - ## # --> {(a: 1, b: 3), (a: 0, b: 4)} - assert s.isValid, "The set needs to be initialized." - for h in 0..high(s.data): - if isFilled(s.data[h].hcode): yield s.data[h].key +include setimpl -const - growthFactor = 2 +# --------------------------------------------------------------------- +# ------------------------------ HashSet ------------------------------ +# --------------------------------------------------------------------- -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) -proc rightSize*(count: Natural): int {.inline.} = - ## Return the value of `initialSize` to support `count` items. +proc init*[A](s: var HashSet[A], initialSize = defaultInitialSize) = + ## Initializes a hash set. ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. + ## Starting from Nim v0.20, sets are initialized by default and it is + ## not necessary to call this function explicitly. ## - ## Internally, we want mustRehash(rightSize(x), x) == false. - result = nextPowerOfTwo(count * 3 div 2 + 4) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = (h + 1) and maxHash - -template rawGetKnownHCImpl() {.dirty.} = - var h: Hash = hc and high(s.data) # start with real hash value - while isFilled(s.data[h].hcode): - # Compare hc THEN key with boolean short circuit. This makes the common case - # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. - # It does slow down succeeding lookups by one extra Hash cmp&and..usually - # just a few clock cycles, generally worth it for any non-integer-like A. - if s.data[h].hcode == hc and s.data[h].key == key: # compare hc THEN key - return h - h = nextTry(h, high(s.data)) - result = -1 - h # < 0 => MISSING; insert idx = -1 - result - -template rawGetImpl() {.dirty.} = - hc = hash(key) - if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. - rawGetKnownHCImpl() - -template rawInsertImpl() {.dirty.} = - data[h].key = key - data[h].hcode = hc - -proc rawGetKnownHC[A](s: HashSet[A], key: A, hc: Hash): int {.inline.} = - rawGetKnownHCImpl() - -proc rawGet[A](s: HashSet[A], key: A, hc: var Hash): int {.inline.} = - rawGetImpl() + ## You can call this proc on a previously initialized hash set, which will + ## discard all its values. This might be more convenient than iterating over + ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. + ## + ## See also: + ## * `initHashSet proc <#initHashSet>`_ + ## * `toHashSet proc <#toHashSet,openArray[A]>`_ + runnableExamples: + var a: HashSet[int] + init(a) + + initImpl(s, initialSize) + +proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] = + ## Wrapper around `init proc <#init,HashSet[A]>`_ for initialization of + ## hash sets. + ## + ## Returns an empty hash set you can assign directly in `var` blocks in a + ## single line. + ## + ## Starting from Nim v0.20, sets are initialized by default and it is + ## not necessary to call this function explicitly. + ## + ## See also: + ## * `toHashSet proc <#toHashSet,openArray[A]>`_ + runnableExamples: + var a = initHashSet[int]() + a.incl(3) + assert len(a) == 1 + result = default(HashSet[A]) + result.init(initialSize) proc `[]`*[A](s: var HashSet[A], key: A): var A = - ## returns the element that is actually stored in 's' which has the same - ## value as 'key' or raises the ``KeyError`` exception. This is useful - ## when one overloaded 'hash' and '==' but still needs reference semantics - ## for sharing. - assert s.isValid, "The set needs to be initialized." + ## Returns the element that is actually stored in `s` which has the same + ## value as `key` or raises the `KeyError` exception. + ## + ## This is useful when one overloaded `hash` and `==` but still needs + ## reference semantics for sharing. var hc: Hash var index = rawGet(s, key, hc) if index >= 0: result = s.data[index].key - else: raise newException(KeyError, "key not found: " & $key) - -proc mget*[A](s: var HashSet[A], key: A): var A {.deprecated.} = - ## returns the element that is actually stored in 's' which has the same - ## value as 'key' or raises the ``KeyError`` exception. This is useful - ## when one overloaded 'hash' and '==' but still needs reference semantics - ## for sharing. Use ```[]``` instead. - s[key] + else: + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") proc contains*[A](s: HashSet[A], key: A): bool = - ## Returns true iff `key` is in `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## assert(not values.contains(2)) - ## values.incl(2) - ## assert values.contains(2) - ## values.excl(2) - ## assert(not values.contains(2)) - assert s.isValid, "The set needs to be initialized." + ## Returns true if `key` is in `s`. + ## + ## This allows the usage of `in` operator. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],A>`_ + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var values = initHashSet[int]() + assert(not values.contains(2)) + assert 2 notin values + + values.incl(2) + assert values.contains(2) + assert 2 in values + var hc: Hash var index = rawGet(s, key, hc) result = index >= 0 -proc rawInsert[A](s: var HashSet[A], data: var KeyValuePairSeq[A], key: A, - hc: Hash, h: Hash) = - rawInsertImpl() +proc len*[A](s: HashSet[A]): int = + ## Returns the number of elements in `s`. + ## + ## Due to an implementation detail you can call this proc on variables which + ## have not been initialized yet. The proc will return zero as the length + ## then. + runnableExamples: + var a: HashSet[string] + assert len(a) == 0 + let s = toHashSet([3, 5, 7]) + assert len(s) == 3 -proc enlarge[A](s: var HashSet[A]) = - var n: KeyValuePairSeq[A] - newSeq(n, len(s.data) * growthFactor) - swap(s.data, n) # n is now old seq - for i in countup(0, high(n)): - if isFilled(n[i].hcode): - var j = -1 - rawGetKnownHC(s, n[i].key, n[i].hcode) - rawInsert(s, s.data, n[i].key, n[i].hcode, j) + result = s.counter -template inclImpl() {.dirty.} = - var hc: Hash - var index = rawGet(s, key, hc) - if index < 0: - if mustRehash(len(s.data), s.counter): - enlarge(s) - index = rawGetKnownHC(s, key, hc) - rawInsert(s, s.data, key, hc, -1 - index) - inc(s.counter) - -template containsOrInclImpl() {.dirty.} = - var hc: Hash - var index = rawGet(s, key, hc) - if index >= 0: - result = true - else: - if mustRehash(len(s.data), s.counter): - enlarge(s) - index = rawGetKnownHC(s, key, hc) - rawInsert(s, s.data, key, hc, -1 - index) - inc(s.counter) +proc card*[A](s: HashSet[A]): int = + ## Alias for `len() <#len,HashSet[A]>`_. + ## + ## Card stands for the `cardinality + ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. + result = s.counter proc incl*[A](s: var HashSet[A], key: A) = ## Includes an element `key` in `s`. ## - ## This doesn't do anything if `key` is already in `s`. Example: + ## This doesn't do anything if `key` is already in `s`. ## - ## .. code-block:: - ## var values = initSet[int]() - ## values.incl(2) - ## values.incl(2) - ## assert values.len == 1 - assert s.isValid, "The set needs to be initialized." + ## See also: + ## * `excl proc <#excl,HashSet[A],A>`_ for excluding an element + ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var values = initHashSet[int]() + values.incl(2) + values.incl(2) + assert values.len == 1 + inclImpl() proc incl*[A](s: var HashSet[A], other: HashSet[A]) = - ## Includes all elements from `other` into `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## values.incl(2) - ## var others = toSet([6, 7]) - ## values.incl(others) - ## assert values.len == 3 - assert s.isValid, "The set `s` needs to be initialized." - assert other.isValid, "The set `other` needs to be initialized." - for item in other: incl(s, item) + ## Includes all elements from `other` set into `s` (must be declared as `var`). + ## + ## This is the in-place version of `s + other <#+,HashSet[A],HashSet[A]>`_. + ## + ## See also: + ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set + ## * `incl proc <#incl,HashSet[A],A>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var + values = toHashSet([1, 2, 3]) + others = toHashSet([3, 4, 5]) + values.incl(others) + assert values.len == 5 -template doWhile(a: expr, b: stmt): stmt = - while true: - b - if not a: break + for item in other: incl(s, item) -proc excl*[A](s: var HashSet[A], key: A) = - ## Excludes `key` from the set `s`. +proc toHashSet*[A](keys: openArray[A]): HashSet[A] = + ## Creates a new hash set that contains the members of the given + ## collection (seq, array, or string) `keys`. ## - ## This doesn't do anything if `key` is not found in `s`. Example: + ## Duplicates are removed. ## - ## .. code-block:: - ## var s = toSet([2, 3, 6, 7]) - ## s.excl(2) - ## s.excl(2) - ## assert s.len == 3 - assert s.isValid, "The set needs to be initialized." - var hc: Hash - var i = rawGet(s, key, hc) - var msk = high(s.data) - if i >= 0: - s.data[i].hcode = 0 - dec(s.counter) - while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 - var j = i # The correctness of this depends on (h+1) in nextTry, - var r = j # though may be adaptable to other simple sequences. - s.data[i].hcode = 0 # mark current EMPTY - doWhile ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): - i = (i + 1) and msk # increment mod table size - if isEmpty(s.data[i].hcode): # end of collision cluster; So all done - return - r = s.data[i].hcode and msk # "home" location of key@i - shallowCopy(s.data[j], s.data[i]) # data[j] will be marked EMPTY next loop + ## See also: + ## * `initHashSet proc <#initHashSet>`_ + runnableExamples: + let + a = toHashSet([5, 3, 2]) + b = toHashSet("abracadabra") + assert len(a) == 3 + ## a == {2, 3, 5} + assert len(b) == 5 + ## b == {'a', 'b', 'c', 'd', 'r'} -proc excl*[A](s: var HashSet[A], other: HashSet[A]) = - ## Excludes everything in `other` from `s`. + result = initHashSet[A](keys.len) + for key in items(keys): result.incl(key) + +iterator items*[A](s: HashSet[A]): A = + ## Iterates over elements of the set `s`. ## - ## Example: + ## If you need a sequence with the elements you can use `sequtils.toSeq + ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim + ## type + ## pair = tuple[a, b: int] ## var - ## numbers = toSet([1, 2, 3, 4, 5]) - ## even = toSet([2, 4, 6, 8]) - ## numbers.excl(even) - ## echo numbers - ## # --> {1, 3, 5} - assert s.isValid, "The set `s` needs to be initialized." - assert other.isValid, "The set `other` needs to be initialized." - for item in other: excl(s, item) + ## a, b = initHashSet[pair]() + ## a.incl((2, 3)) + ## a.incl((3, 2)) + ## a.incl((2, 3)) + ## for x, y in a.items: + ## b.incl((x - 2, y + 1)) + ## assert a.len == 2 + ## echo b + ## # --> {(a: 1, b: 3), (a: 0, b: 4)} + ## ``` + let length = s.len + for h in 0 .. high(s.data): + if isFilled(s.data[h].hcode): + yield s.data[h].key + assert(len(s) == length, "the length of the HashSet changed while iterating over it") proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = - ## Includes `key` in the set `s` and tells if `key` was added to `s`. - ## - ## The difference with regards to the `incl() <#incl,TSet[A],A>`_ proc is - ## that this proc returns `true` if `key` was already present in `s`. The - ## proc will return false if `key` was added as a new value to `s` during - ## this call. Example: - ## - ## .. code-block:: - ## var values = initSet[int]() - ## assert values.containsOrIncl(2) == false - ## assert values.containsOrIncl(2) == true - assert s.isValid, "The set needs to be initialized." + ## Includes `key` in the set `s` and tells if `key` was already in `s`. + ## + ## The difference with regards to the `incl proc <#incl,HashSet[A],A>`_ is + ## that this proc returns `true` if `s` already contained `key`. The + ## proc will return `false` if `key` was added as a new value to `s` during + ## this call. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],A>`_ for including an element + ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set + ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ + runnableExamples: + var values = initHashSet[int]() + assert values.containsOrIncl(2) == false + assert values.containsOrIncl(2) == true + assert values.containsOrIncl(3) == false + containsOrInclImpl() -proc init*[A](s: var HashSet[A], initialSize=64) = - ## Initializes a hash set. +proc excl*[A](s: var HashSet[A], key: A) = + ## Excludes `key` from the set `s`. ## - ## The `initialSize` parameter needs to be a power of two. You can use - ## `math.nextPowerOfTwo() <math.html#nextPowerOfTwo>`_ or `rightSize` to - ## guarantee that at runtime. All set variables must be initialized before - ## use with other procs from this module with the exception of `isValid() - ## <#isValid,TSet[A]>`_ and `len() <#len,TSet[A]>`_. + ## This doesn't do anything if `key` is not found in `s`. ## - ## You can call this proc on a previously initialized hash set, which will - ## discard all its values. This might be more convenient than iterating over - ## existing values and calling `excl() <#excl,TSet[A],A>`_ on them. Example: - ## - ## .. code-block :: - ## var a: Set[int] - ## a.init(4) - ## a.incl(2) - ## a.init - ## assert a.len == 0 and a.isValid - assert isPowerOfTwo(initialSize) - s.counter = 0 - newSeq(s.data, initialSize) + ## See also: + ## * `incl proc <#incl,HashSet[A],A>`_ for including an element + ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set + ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ + runnableExamples: + var s = toHashSet([2, 3, 6, 7]) + s.excl(2) + s.excl(2) + assert s.len == 3 -proc initSet*[A](initialSize=64): HashSet[A] = - ## Wrapper around `init() <#init,TSet[A],int>`_ for initialization of hash - ## sets. - ## - ## Returns an empty hash set you can assign directly in ``var`` blocks in a - ## single line. Example: - ## - ## .. code-block :: - ## var a = initSet[int](4) - ## a.incl(2) - result.init(initialSize) + discard exclImpl(s, key) -proc toSet*[A](keys: openArray[A]): HashSet[A] = - ## Creates a new hash set that contains the given `keys`. - ## - ## Example: - ## - ## .. code-block:: - ## var numbers = toSet([1, 2, 3, 4, 5]) - ## assert numbers.contains(2) - ## assert numbers.contains(4) - result = initSet[A](rightSize(keys.len)) - for key in items(keys): result.incl(key) +proc excl*[A](s: var HashSet[A], other: HashSet[A]) = + ## Excludes all elements of `other` set from `s`. + ## + ## This is the in-place version of `s - other <#-,HashSet[A],HashSet[A]>`_. + ## + ## See also: + ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set + ## * `excl proc <#excl,HashSet[A],A>`_ for excluding an element + ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ + runnableExamples: + var + numbers = toHashSet([1, 2, 3, 4, 5]) + even = toHashSet([2, 4, 6, 8]) + numbers.excl(even) + assert len(numbers) == 3 + ## numbers == {1, 3, 5} + + for item in other: discard exclImpl(s, item) + +proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = + ## Excludes `key` in the set `s` and tells if `key` was already missing from `s`. + ## + ## The difference with regards to the `excl proc <#excl,HashSet[A],A>`_ is + ## that this proc returns `true` if `key` was missing from `s`. + ## The proc will return `false` if `key` was in `s` and it was removed + ## during this call. + ## + ## See also: + ## * `excl proc <#excl,HashSet[A],A>`_ for excluding an element + ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set + ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ + runnableExamples: + var s = toHashSet([2, 3, 6, 7]) + assert s.missingOrExcl(4) == true + assert s.missingOrExcl(6) == false + assert s.missingOrExcl(6) == true + + exclImpl(s, key) + +proc pop*[A](s: var HashSet[A]): A = + ## Removes and returns an arbitrary element from the set `s`. + ## + ## Raises `KeyError` if the set `s` is empty. + ## + ## See also: + ## * `clear proc <#clear,HashSet[A]>`_ + runnableExamples: + var s = toHashSet([2, 1]) + assert [s.pop, s.pop] in [[1, 2], [2,1]] # order unspecified + doAssertRaises(KeyError, echo s.pop) + + for h in 0 .. high(s.data): + if isFilled(s.data[h].hcode): + result = s.data[h].key + excl(s, result) + return result + raise newException(KeyError, "set is empty") + +proc clear*[A](s: var HashSet[A]) = + ## Clears the HashSet back to an empty state, without shrinking + ## any of the existing storage. + ## + ## `O(n)` operation, where `n` is the size of the hash bucket. + ## + ## See also: + ## * `pop proc <#pop,HashSet[A]>`_ + runnableExamples: + var s = toHashSet([3, 5, 7]) + clear(s) + assert len(s) == 0 -template dollarImpl(): stmt {.dirty.} = - result = "{" - for key in items(s): - if result.len > 1: result.add(", ") - result.add($key) - result.add("}") + s.counter = 0 + for i in 0 ..< s.data.len: + s.data[i].hcode = 0 + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} -proc `$`*[A](s: HashSet[A]): string = - ## Converts the set `s` to a string, mostly for logging purposes. - ## - ## Don't use this proc for serialization, the representation may change at - ## any moment and values are not escaped. Example: - ## - ## Example: - ## - ## .. code-block:: - ## echo toSet([2, 4, 5]) - ## # --> {2, 4, 5} - ## echo toSet(["no", "esc'aping", "is \" provided"]) - ## # --> {no, esc'aping, is " provided} - assert s.isValid, "The set needs to be initialized." - dollarImpl() proc union*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the union of the sets `s1` and `s2`. ## - ## The union of two sets is represented mathematically as *A ∪ B* and is the - ## set of all objects that are members of `s1`, `s2` or both. Example: + ## The same as `s1 + s2 <#+,HashSet[A],HashSet[A]>`_. ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = union(a, b) - ## assert c == toSet(["a", "b", "c"]) - assert s1.isValid, "The set `s1` needs to be initialized." - assert s2.isValid, "The set `s2` needs to be initialized." + ## The union of two sets is represented mathematically as *A ∪ B* and is the + ## set of all objects that are members of `s1`, `s2` or both. + ## + ## See also: + ## * `intersection proc <#intersection,HashSet[A],HashSet[A]>`_ + ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ + ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + c = union(a, b) + assert c == toHashSet(["a", "b", "c"]) + result = s1 incl(result, s2) proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the intersection of the sets `s1` and `s2`. ## + ## The same as `s1 * s2 <#*,HashSet[A],HashSet[A]>`_. + ## ## The intersection of two sets is represented mathematically as *A ∩ B* and ## is the set of all objects that are members of `s1` and `s2` at the same - ## time. Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = intersection(a, b) - ## assert c == toSet(["b"]) - assert s1.isValid, "The set `s1` needs to be initialized." - assert s2.isValid, "The set `s2` needs to be initialized." - result = initSet[A](min(s1.data.len, s2.data.len)) - for item in s1: - if item in s2: incl(result, item) + ## time. + ## + ## See also: + ## * `union proc <#union,HashSet[A],HashSet[A]>`_ + ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ + ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + c = intersection(a, b) + assert c == toHashSet(["b"]) + + result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) + + # iterate over the elements of the smaller set + if s1.data.len < s2.data.len: + for item in s1: + if item in s2: incl(result, item) + else: + for item in s2: + if item in s1: incl(result, item) + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. ## - ## The difference of two sets is represented mathematically as *A \ B* and is + ## The same as `s1 - s2 <#-,HashSet[A],HashSet[A]>`_. + ## + ## The difference of two sets is represented mathematically as *A ∖ B* and is ## the set of all objects that are members of `s1` and not members of `s2`. - ## Example: ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = difference(a, b) - ## assert c == toSet(["a"]) - assert s1.isValid, "The set `s1` needs to be initialized." - assert s2.isValid, "The set `s2` needs to be initialized." - result = initSet[A]() + ## See also: + ## * `union proc <#union,HashSet[A],HashSet[A]>`_ + ## * `intersection proc <#intersection,HashSet[A],HashSet[A]>`_ + ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + c = difference(a, b) + assert c == toHashSet(["a"]) + + result = initHashSet[A]() for item in s1: if not contains(s2, item): incl(result, item) @@ -445,51 +467,53 @@ proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = proc symmetricDifference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the symmetric difference of the sets `s1` and `s2`. ## + ## The same as `s1 -+- s2 <#-+-,HashSet[A],HashSet[A]>`_. + ## ## The symmetric difference of two sets is represented mathematically as *A △ ## B* or *A ⊖ B* and is the set of all objects that are members of `s1` or - ## `s2` but not both at the same time. Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = symmetricDifference(a, b) - ## assert c == toSet(["a", "c"]) - assert s1.isValid, "The set `s1` needs to be initialized." - assert s2.isValid, "The set `s2` needs to be initialized." + ## `s2` but not both at the same time. + ## + ## See also: + ## * `union proc <#union,HashSet[A],HashSet[A]>`_ + ## * `intersection proc <#intersection,HashSet[A],HashSet[A]>`_ + ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + c = symmetricDifference(a, b) + assert c == toHashSet(["a", "c"]) + result = s1 for item in s2: if containsOrIncl(result, item): excl(result, item) proc `+`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `union(s1, s2) <#union>`_. + ## Alias for `union(s1, s2) <#union,HashSet[A],HashSet[A]>`_. result = union(s1, s2) proc `*`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `intersection(s1, s2) <#intersection>`_. + ## Alias for `intersection(s1, s2) <#intersection,HashSet[A],HashSet[A]>`_. result = intersection(s1, s2) proc `-`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `difference(s1, s2) <#difference>`_. + ## Alias for `difference(s1, s2) <#difference,HashSet[A],HashSet[A]>`_. result = difference(s1, s2) proc `-+-`*[A](s1, s2: HashSet[A]): HashSet[A] {.inline.} = - ## Alias for `symmetricDifference(s1, s2) <#symmetricDifference>`_. + ## Alias for `symmetricDifference(s1, s2) + ## <#symmetricDifference,HashSet[A],HashSet[A]>`_. result = symmetricDifference(s1, s2) proc disjoint*[A](s1, s2: HashSet[A]): bool = - ## Returns true iff the sets `s1` and `s2` have no items in common. - ## - ## Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## assert disjoint(a, b) == false - ## assert disjoint(a, b - a) == true - assert s1.isValid, "The set `s1` needs to be initialized." - assert s2.isValid, "The set `s2` needs to be initialized." + ## Returns `true` if the sets `s1` and `s2` have no items in common. + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + assert disjoint(a, b) == false + assert disjoint(a, b - a) == true + for item in s1: if item in s2: return false return true @@ -498,306 +522,344 @@ proc `<`*[A](s, t: HashSet[A]): bool = ## Returns true if `s` is a strict or proper subset of `t`. ## ## A strict or proper subset `s` has all of its members in `t` but `t` has - ## more elements than `s`. Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = intersection(a, b) - ## assert c < a and c < b - ## assert((a < a) == false) + ## more elements than `s`. + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + c = intersection(a, b) + assert c < a and c < b + assert(not (a < a)) + s.counter != t.counter and s <= t proc `<=`*[A](s, t: HashSet[A]): bool = - ## Returns true if `s` is subset of `t`. + ## Returns true if `s` is a subset of `t`. ## ## A subset `s` has all of its members in `t` and `t` doesn't necessarily - ## have more members than `s`. That is, `s` can be equal to `t`. Example: - ## - ## .. code-block:: - ## var - ## a = toSet(["a", "b"]) - ## b = toSet(["b", "c"]) - ## c = intersection(a, b) - ## assert c <= a and c <= b - ## assert((a <= a)) + ## have more members than `s`. That is, `s` can be equal to `t`. + runnableExamples: + let + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) + c = intersection(a, b) + assert c <= a and c <= b + assert a <= a + result = false if s.counter > t.counter: return result = true - for item in s: + for item in items(s): if not(t.contains(item)): result = false return proc `==`*[A](s, t: HashSet[A]): bool = ## Returns true if both `s` and `t` have the same members and set size. - ## - ## Example: - ## - ## .. code-block:: - ## var - ## a = toSet([1, 2]) - ## b = toSet([1]) - ## b.incl(2) - ## assert a == b + runnableExamples: + var + a = toHashSet([1, 2]) + b = toHashSet([2, 1]) + assert a == b + s.counter == t.counter and s <= t -proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = - ## Returns a new set after applying `op` on each of the elements of `data`. +proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] {.effectsOf: op.} = + ## Returns a new set after applying `op` proc on each of the elements of + ##`data` set. ## - ## You can use this proc to transform the elements from a set. Example: - ## - ## .. code-block:: - ## var a = toSet([1, 2, 3]) - ## var b = a.map(proc (x: int): string = $x) - ## assert b == toSet(["1", "2", "3"]) - result = initSet[B]() - for item in data: result.incl(op(item)) - -# ------------------------------ ordered set ------------------------------ + ## You can use this proc to transform the elements from a set. + runnableExamples: + let + a = toHashSet([1, 2, 3]) + b = a.map(proc (x: int): string = $x) + assert b == toHashSet(["1", "2", "3"]) -type - OrderedKeyValuePair[A] = tuple[ - hcode: Hash, next: int, key: A] - OrderedKeyValuePairSeq[A] = seq[OrderedKeyValuePair[A]] - OrderedSet* {.myShallow.}[A] = object ## \ - ## A generic hash set that remembers insertion order. - ## - ## Use `init() <#init,OrderedSet[A],int>`_ or `initOrderedSet[type]() - ## <#initOrderedSet>`_ before calling other procs on it. - data: OrderedKeyValuePairSeq[A] - counter, first, last: int + result = initHashSet[B]() + for item in items(data): result.incl(op(item)) -{.deprecated: [TOrderedSet: OrderedSet].} +proc hash*[A](s: HashSet[A]): Hash = + ## Hashing of HashSet. + for h in 0 .. high(s.data): + result = result xor s.data[h].hcode + result = !$result -proc isValid*[A](s: OrderedSet[A]): bool = - ## Returns `true` if the ordered set has been initialized with `initSet - ## <#initOrderedSet>`_. +proc `$`*[A](s: HashSet[A]): string = + ## Converts the set `s` to a string, mostly for logging and printing purposes. ## - ## Most operations over an uninitialized ordered set will crash at runtime - ## and `assert <system.html#assert>`_ in debug builds. You can use this proc - ## in your own procs to verify that ordered sets passed to your procs are - ## correctly initialized. Example: + ## Don't use this proc for serialization, the representation may change at + ## any moment and values are not escaped. ## - ## .. code-block:: - ## proc saveTarotCards(cards: OrderedSet[int]) = - ## assert cards.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! - result = not s.data.isNil + ## **Examples:** + ## ```Nim + ## echo toHashSet([2, 4, 5]) + ## # --> {2, 4, 5} + ## echo toHashSet(["no", "esc'aping", "is \" provided"]) + ## # --> {no, esc'aping, is " provided} + ## ``` + dollarImpl() -proc len*[A](s: OrderedSet[A]): int {.inline.} = - ## Returns the number of keys in `s`. + +proc initSet*[A](initialSize = defaultInitialSize): HashSet[A] {.deprecated: + "Deprecated since v0.20, use 'initHashSet'".} = initHashSet[A](initialSize) + +proc toSet*[A](keys: openArray[A]): HashSet[A] {.deprecated: + "Deprecated since v0.20, use 'toHashSet'".} = toHashSet[A](keys) + +proc isValid*[A](s: HashSet[A]): bool {.deprecated: + "Deprecated since v0.20; sets are initialized by default".} = + ## Returns `true` if the set has been initialized (with `initHashSet proc + ## <#initHashSet>`_ or `init proc <#init,HashSet[A]>`_). ## - ## Due to an implementation detail you can call this proc on variables which - ## have not been initialized yet. The proc will return zero as the length - ## then. Example: + runnableExamples: + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + # Do stuff here, may crash in release builds! + result = s.data.len > 0 + + + +# --------------------------------------------------------------------- +# --------------------------- OrderedSet ------------------------------ +# --------------------------------------------------------------------- + +template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = + if s.data.len > 0: + var h = s.first + var idx = 0 + while h >= 0: + var nxt = s.data[h].next + if isFilled(s.data[h].hcode): + yieldStmt + inc(idx) + h = nxt + + +proc init*[A](s: var OrderedSet[A], initialSize = defaultInitialSize) = + ## Initializes an ordered hash set. ## - ## .. code-block:: + ## Starting from Nim v0.20, sets are initialized by default and it is + ## not necessary to call this function explicitly. ## - ## var values: OrderedSet[int] - ## assert(not values.isValid) - ## assert values.len == 0 - result = s.counter - -proc card*[A](s: OrderedSet[A]): int {.inline.} = - ## Alias for `len() <#len,TOrderedSet[A]>`_. + ## You can call this proc on a previously initialized hash set, which will + ## discard all its values. This might be more convenient than iterating over + ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. ## - ## Card stands for the `cardinality - ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. - result = s.counter + ## See also: + ## * `initOrderedSet proc <#initOrderedSet>`_ + ## * `toOrderedSet proc <#toOrderedSet,openArray[A]>`_ + runnableExamples: + var a: OrderedSet[int] + init(a) -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = - var h = s.first - while h >= 0: - var nxt = s.data[h].next - if isFilled(s.data[h].hcode): yieldStmt - h = nxt + initImpl(s, initialSize) -iterator items*[A](s: OrderedSet[A]): A = - ## Iterates over keys in the ordered set `s` in insertion order. +proc initOrderedSet*[A](initialSize = defaultInitialSize): OrderedSet[A] = + ## Wrapper around `init proc <#init,OrderedSet[A]>`_ for initialization of + ## ordered hash sets. ## - ## If you need a sequence with the keys you can use `sequtils.toSeq() - ## <sequtils.html#toSeq>`_ on the iterator. Usage example: + ## Returns an empty ordered hash set you can assign directly in `var` blocks + ## in a single line. ## - ## .. code-block:: - ## var a = initOrderedSet[int]() - ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: - ## a.incl(value) - ## for value in a.items: - ## echo "Got ", value - ## # --> Got 9 - ## # --> Got 2 - ## # --> Got 1 - ## # --> Got 5 - ## # --> Got 8 - ## # --> Got 4 - assert s.isValid, "The set needs to be initialized." - forAllOrderedPairs: - yield s.data[h].key + ## Starting from Nim v0.20, sets are initialized by default and it is + ## not necessary to call this function explicitly. + ## + ## See also: + ## * `toOrderedSet proc <#toOrderedSet,openArray[A]>`_ + runnableExamples: + var a = initOrderedSet[int]() + a.incl(3) + assert len(a) == 1 -proc rawGetKnownHC[A](s: OrderedSet[A], key: A, hc: Hash): int {.inline.} = - rawGetKnownHCImpl() + result.init(initialSize) -proc rawGet[A](s: OrderedSet[A], key: A, hc: var Hash): int {.inline.} = - rawGetImpl() +proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = + ## Creates a new hash set that contains the members of the given + ## collection (seq, array, or string) `keys`. + ## + ## Duplicates are removed. + ## + ## See also: + ## * `initOrderedSet proc <#initOrderedSet>`_ + runnableExamples: + let + a = toOrderedSet([5, 3, 2]) + b = toOrderedSet("abracadabra") + assert len(a) == 3 + ## a == {5, 3, 2} # different than in HashSet + assert len(b) == 5 + ## b == {'a', 'b', 'r', 'c', 'd'} # different than in HashSet + + result = initOrderedSet[A](keys.len) + for key in items(keys): result.incl(key) proc contains*[A](s: OrderedSet[A], key: A): bool = - ## Returns true iff `key` is in `s`. + ## Returns true if `key` is in `s`. ## - ## Example: + ## This allows the usage of `in` operator. ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## assert(not values.contains(2)) - ## values.incl(2) - ## assert values.contains(2) - assert s.isValid, "The set needs to be initialized." + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var values = initOrderedSet[int]() + assert(not values.contains(2)) + assert 2 notin values + + values.incl(2) + assert values.contains(2) + assert 2 in values + var hc: Hash var index = rawGet(s, key, hc) result = index >= 0 -proc rawInsert[A](s: var OrderedSet[A], data: var OrderedKeyValuePairSeq[A], - key: A, hc: Hash, h: Hash) = - rawInsertImpl() - data[h].next = -1 - if s.first < 0: s.first = h - if s.last >= 0: data[s.last].next = h - s.last = h - -proc enlarge[A](s: var OrderedSet[A]) = - var n: OrderedKeyValuePairSeq[A] - newSeq(n, len(s.data) * growthFactor) - var h = s.first - s.first = -1 - s.last = -1 - swap(s.data, n) - while h >= 0: - var nxt = n[h].next - if isFilled(n[h].hcode): - var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode) - rawInsert(s, s.data, n[h].key, n[h].hcode, j) - h = nxt - proc incl*[A](s: var OrderedSet[A], key: A) = ## Includes an element `key` in `s`. ## - ## This doesn't do anything if `key` is already in `s`. Example: + ## This doesn't do anything if `key` is already in `s`. ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## values.incl(2) - ## values.incl(2) - ## assert values.len == 1 - assert s.isValid, "The set needs to be initialized." + ## See also: + ## * `excl proc <#excl,OrderedSet[A],A>`_ for excluding an element + ## * `incl proc <#incl,HashSet[A],OrderedSet[A]>`_ for including other set + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var values = initOrderedSet[int]() + values.incl(2) + values.incl(2) + assert values.len == 1 + inclImpl() proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) = - ## Includes all elements from `other` into `s`. - ## - ## Example: - ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## values.incl(2) - ## var others = toOrderedSet([6, 7]) - ## values.incl(others) - ## assert values.len == 3 - assert s.isValid, "The set `s` needs to be initialized." - assert other.isValid, "The set `other` needs to be initialized." - for item in other: incl(s, item) + ## Includes all elements from the OrderedSet `other` into + ## HashSet `s` (must be declared as `var`). + ## + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var + values = toHashSet([1, 2, 3]) + others = toOrderedSet([3, 4, 5]) + values.incl(others) + assert values.len == 5 + + for item in items(other): incl(s, item) proc containsOrIncl*[A](s: var OrderedSet[A], key: A): bool = - ## Includes `key` in the set `s` and tells if `key` was added to `s`. + ## Includes `key` in the set `s` and tells if `key` was already in `s`. ## - ## The difference with regards to the `incl() <#incl,TOrderedSet[A],A>`_ proc - ## is that this proc returns `true` if `key` was already present in `s`. The + ## The difference with regards to the `incl proc <#incl,OrderedSet[A],A>`_ is + ## that this proc returns `true` if `s` already contained `key`. The ## proc will return false if `key` was added as a new value to `s` during - ## this call. Example: + ## this call. ## - ## .. code-block:: - ## var values = initOrderedSet[int]() - ## assert values.containsOrIncl(2) == false - ## assert values.containsOrIncl(2) == true - assert s.isValid, "The set needs to be initialized." - containsOrInclImpl() + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,OrderedSet[A],A>`_ + runnableExamples: + var values = initOrderedSet[int]() + assert values.containsOrIncl(2) == false + assert values.containsOrIncl(2) == true + assert values.containsOrIncl(3) == false -proc init*[A](s: var OrderedSet[A], initialSize=64) = - ## Initializes an ordered hash set. - ## - ## The `initialSize` parameter needs to be a power of two. You can use - ## `math.nextPowerOfTwo() <math.html#nextPowerOfTwo>`_ or `rightSize` to - ## guarantee that at runtime. All set variables must be initialized before - ## use with other procs from this module with the exception of `isValid() - ## <#isValid,TOrderedSet[A]>`_ and `len() <#len,TOrderedSet[A]>`_. - ## - ## You can call this proc on a previously initialized ordered hash set to - ## discard its values. At the moment this is the only proc to remove elements - ## from an ordered hash set. Example: - ## - ## .. code-block :: - ## var a: OrderedSet[int] - ## a.init(4) - ## a.incl(2) - ## a.init - ## assert a.len == 0 and a.isValid - assert isPowerOfTwo(initialSize) - s.counter = 0 - s.first = -1 - s.last = -1 - newSeq(s.data, initialSize) + containsOrInclImpl() -proc initOrderedSet*[A](initialSize=64): OrderedSet[A] = - ## Wrapper around `init() <#init,TOrderedSet[A],int>`_ for initialization of - ## ordered hash sets. +proc excl*[A](s: var OrderedSet[A], key: A) = + ## Excludes `key` from the set `s`. Efficiency: `O(n)`. ## - ## Returns an empty ordered hash set you can assign directly in ``var`` - ## blocks in a single line. Example: + ## This doesn't do anything if `key` is not found in `s`. ## - ## .. code-block :: - ## var a = initOrderedSet[int](4) - ## a.incl(2) - result.init(initialSize) + ## See also: + ## * `incl proc <#incl,OrderedSet[A],A>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,OrderedSet[A],A>`_ + runnableExamples: + var s = toOrderedSet([2, 3, 6, 7]) + s.excl(2) + s.excl(2) + assert s.len == 3 -proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = - ## Creates a new ordered hash set that contains the given `keys`. + discard exclImpl(s, key) + +proc missingOrExcl*[A](s: var OrderedSet[A], key: A): bool = + ## Excludes `key` in the set `s` and tells if `key` was already missing from `s`. + ## Efficiency: O(n). ## - ## Example: + ## The difference with regards to the `excl proc <#excl,OrderedSet[A],A>`_ is + ## that this proc returns `true` if `key` was missing from `s`. + ## The proc will return `false` if `key` was in `s` and it was removed + ## during this call. ## - ## .. code-block:: - ## var numbers = toOrderedSet([1, 2, 3, 4, 5]) - ## assert numbers.contains(2) - ## assert numbers.contains(4) - result = initOrderedSet[A](rightSize(keys.len)) - for key in items(keys): result.incl(key) + ## See also: + ## * `excl proc <#excl,OrderedSet[A],A>`_ + ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ + runnableExamples: + var s = toOrderedSet([2, 3, 6, 7]) + assert s.missingOrExcl(4) == true + assert s.missingOrExcl(6) == false + assert s.missingOrExcl(6) == true -proc `$`*[A](s: OrderedSet[A]): string = - ## Converts the ordered hash set `s` to a string, mostly for logging purposes. + exclImpl(s, key) + +proc clear*[A](s: var OrderedSet[A]) = + ## Clears the OrderedSet back to an empty state, without shrinking + ## any of the existing storage. ## - ## Don't use this proc for serialization, the representation may change at - ## any moment and values are not escaped. Example: + ## `O(n)` operation where `n` is the size of the hash bucket. + runnableExamples: + var s = toOrderedSet([3, 5, 7]) + clear(s) + assert len(s) == 0 + + s.counter = 0 + s.first = -1 + s.last = -1 + for i in 0 ..< s.data.len: + s.data[i].hcode = 0 + s.data[i].next = 0 + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} + +proc len*[A](s: OrderedSet[A]): int {.inline.} = + ## Returns the number of elements in `s`. ## - ## Example: + ## Due to an implementation detail you can call this proc on variables which + ## have not been initialized yet. The proc will return zero as the length + ## then. + runnableExamples: + var a: OrderedSet[string] + assert len(a) == 0 + let s = toHashSet([3, 5, 7]) + assert len(s) == 3 + + result = s.counter + +proc card*[A](s: OrderedSet[A]): int {.inline.} = + ## Alias for `len() <#len,OrderedSet[A]>`_. ## - ## .. code-block:: - ## echo toOrderedSet([2, 4, 5]) - ## # --> {2, 4, 5} - ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) - ## # --> {no, esc'aping, is " provided} - assert s.isValid, "The set needs to be initialized." - dollarImpl() + ## Card stands for the `cardinality + ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. + result = s.counter proc `==`*[A](s, t: OrderedSet[A]): bool = ## Equality for ordered sets. + runnableExamples: + let + a = toOrderedSet([1, 2]) + b = toOrderedSet([2, 1]) + assert(not (a == b)) + if s.counter != t.counter: return false var h = s.first - var g = s.first + var g = t.first var compared = 0 while h >= 0 and g >= 0: var nxh = s.data[h].next var nxg = t.data[g].next - if isFilled(s.data[h].hcode) and isFilled(s.data[g].hcode): - if s.data[h].key == s.data[g].key: + if isFilled(s.data[h].hcode) and isFilled(t.data[g].hcode): + if s.data[h].key == t.data[g].key: inc compared else: return false @@ -805,179 +867,64 @@ proc `==`*[A](s, t: OrderedSet[A]): bool = g = nxg result = compared == s.counter -when isMainModule and not defined(release): - proc testModule() = - ## Internal micro test to validate docstrings and such. - block isValidTest: - var options: HashSet[string] - proc savePreferences(options: HashSet[string]) = - assert options.isValid, "Pass an initialized set!" - options = initSet[string]() - options.savePreferences - - block lenTest: - var values: HashSet[int] - assert(not values.isValid) - assert values.len == 0 - assert values.card == 0 - - block setIterator: - type pair = tuple[a, b: int] - var a, b = initSet[pair]() - a.incl((2, 3)) - a.incl((3, 2)) - a.incl((2, 3)) - for x, y in a.items: - b.incl((x - 2, y + 1)) - assert a.len == b.card - assert a.len == 2 - #echo b - - block setContains: - var values = initSet[int]() - assert(not values.contains(2)) - values.incl(2) - assert values.contains(2) - values.excl(2) - assert(not values.contains(2)) - - values.incl(4) - var others = toSet([6, 7]) - values.incl(others) - assert values.len == 3 - - values.init - assert values.containsOrIncl(2) == false - assert values.containsOrIncl(2) == true - var - a = toSet([1, 2]) - b = toSet([1]) - b.incl(2) - assert a == b - - block exclusions: - var s = toSet([2, 3, 6, 7]) - s.excl(2) - s.excl(2) - assert s.len == 3 - - var - numbers = toSet([1, 2, 3, 4, 5]) - even = toSet([2, 4, 6, 8]) - numbers.excl(even) - #echo numbers - # --> {1, 3, 5} - - block toSeqAndString: - var a = toSet([2, 4, 5]) - var b = initSet[int]() - for x in [2, 4, 5]: b.incl(x) - assert($a == $b) - #echo a - #echo toSet(["no", "esc'aping", "is \" provided"]) - - #block orderedToSeqAndString: - # echo toOrderedSet([2, 4, 5]) - # echo toOrderedSet(["no", "esc'aping", "is \" provided"]) - - block setOperations: - var - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) - c = union(a, b) - assert c == toSet(["a", "b", "c"]) - var d = intersection(a, b) - assert d == toSet(["b"]) - var e = difference(a, b) - assert e == toSet(["a"]) - var f = symmetricDifference(a, b) - assert f == toSet(["a", "c"]) - assert d < a and d < b - assert((a < a) == false) - assert d <= a and d <= b - assert((a <= a)) - # Alias test. - assert a + b == toSet(["a", "b", "c"]) - assert a * b == toSet(["b"]) - assert a - b == toSet(["a"]) - assert a -+- b == toSet(["a", "c"]) - assert disjoint(a, b) == false - assert disjoint(a, b - a) == true - - block mapSet: - var a = toSet([1, 2, 3]) - var b = a.map(proc (x: int): string = $x) - assert b == toSet(["1", "2", "3"]) - - block isValidTest: - var cards: OrderedSet[string] - proc saveTarotCards(cards: OrderedSet[string]) = - assert cards.isValid, "Pass an initialized set!" - cards = initOrderedSet[string]() - cards.saveTarotCards - - block lenTest: - var values: OrderedSet[int] - assert(not values.isValid) - assert values.len == 0 - assert values.card == 0 - - block setIterator: - type pair = tuple[a, b: int] - var a, b = initOrderedSet[pair]() - a.incl((2, 3)) - a.incl((3, 2)) - a.incl((2, 3)) - for x, y in a.items: - b.incl((x - 2, y + 1)) - assert a.len == b.card - assert a.len == 2 - - #block orderedSetIterator: - # var a = initOrderedSet[int]() - # for value in [9, 2, 1, 5, 1, 8, 4, 2]: - # a.incl(value) - # for value in a.items: - # echo "Got ", value - - block setContains: - var values = initOrderedSet[int]() - assert(not values.contains(2)) - values.incl(2) - assert values.contains(2) - - block toSeqAndString: - var a = toOrderedSet([2, 4, 5]) - var b = initOrderedSet[int]() - for x in [2, 4, 5]: b.incl(x) - assert($a == $b) - assert(a == b) # https://github.com/Araq/Nimrod/issues/1413 - - block initBlocks: - var a: OrderedSet[int] - a.init(4) - a.incl(2) - a.init - assert a.len == 0 and a.isValid - a = initOrderedSet[int](4) - a.incl(2) - assert a.len == 1 - - var b: HashSet[int] - b.init(4) - b.incl(2) - b.init - assert b.len == 0 and b.isValid - b = initSet[int](4) - b.incl(2) - assert b.len == 1 - - for i in 0 .. 32: - var s = rightSize(i) - if s <= i or mustRehash(s, i): - echo "performance issue: rightSize() will not elide enlarge() at ", i - - when not defined(testing): - echo "Micro tests run successfully." - - testModule() +proc hash*[A](s: OrderedSet[A]): Hash = + ## Hashing of OrderedSet. + forAllOrderedPairs: + result = result !& s.data[h].hcode + result = !$result + +proc `$`*[A](s: OrderedSet[A]): string = + ## Converts the ordered hash set `s` to a string, mostly for logging and + ## printing purposes. + ## + ## Don't use this proc for serialization, the representation may change at + ## any moment and values are not escaped. + ## + ## **Examples:** + ## ```Nim + ## echo toOrderedSet([2, 4, 5]) + ## # --> {2, 4, 5} + ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) + ## # --> {no, esc'aping, is " provided} + ## ``` + dollarImpl() + + + +iterator items*[A](s: OrderedSet[A]): A = + ## Iterates over keys in the ordered set `s` in insertion order. + ## + ## If you need a sequence with the elements you can use `sequtils.toSeq + ## template <sequtils.html#toSeq.t,untyped>`_. + ## + ## ```Nim + ## var a = initOrderedSet[int]() + ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: + ## a.incl(value) + ## for value in a.items: + ## echo "Got ", value + ## # --> Got 9 + ## # --> Got 2 + ## # --> Got 1 + ## # --> Got 5 + ## # --> Got 8 + ## # --> Got 4 + ## ``` + let length = s.len + forAllOrderedPairs: + yield s.data[h].key + assert(len(s) == length, "the length of the OrderedSet changed while iterating over it") + +iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = + ## Iterates through (position, value) tuples of OrderedSet `s`. + runnableExamples: + let a = toOrderedSet("abracadabra") + var p = newSeq[(int, char)]() + for x in pairs(a): + p.add(x) + assert p == @[(0, 'a'), (1, 'b'), (2, 'r'), (3, 'c'), (4, 'd')] + + let length = s.len + forAllOrderedPairs: + yield (idx, s.data[h].key) + assert(len(s) == length, "the length of the OrderedSet changed while iterating over it") diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim new file mode 100644 index 000000000..ec8f1cd86 --- /dev/null +++ b/lib/pure/collections/sharedlist.nim @@ -0,0 +1,105 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Shared list support. +## +## Unstable API. + +{.deprecated.} + +{.push stackTrace: off.} + +import + std/locks + +const + ElemsPerNode = 100 + +type + SharedListNode[A] = ptr object + next: SharedListNode[A] + dataLen: int + d: array[ElemsPerNode, A] + + SharedList*[A] = object ## generic shared list + head, tail: SharedListNode[A] + lock*: Lock + +template withLock(t, x: untyped) = + acquire(t.lock) + x + release(t.lock) + +proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = + ## Iterates over the list. If `action` returns true, the + ## current item is removed from the list. + ## + ## .. warning:: It may not preserve the element order after some modifications. + withLock(x): + var n = x.head + while n != nil: + var i = 0 + while i < n.dataLen: + # action can add new items at the end, so release the lock: + release(x.lock) + if action(n.d[i]): + acquire(x.lock) + let t = x.tail + dec t.dataLen # TODO considering t.dataLen == 0, + # probably the module should be refactored using doubly linked lists + n.d[i] = t.d[t.dataLen] + else: + acquire(x.lock) + inc i + n = n.next + +iterator items*[A](x: var SharedList[A]): A = + withLock(x): + var it = x.head + while it != nil: + for i in 0..it.dataLen-1: + yield it.d[i] + it = it.next + +proc add*[A](x: var SharedList[A]; y: A) = + withLock(x): + var node: SharedListNode[A] + if x.tail == nil: + node = cast[typeof node](allocShared0(sizeof(node[]))) + x.tail = node + x.head = node + elif x.tail.dataLen == ElemsPerNode: + node = cast[typeof node](allocShared0(sizeof(node[]))) + x.tail.next = node + x.tail = node + else: + node = x.tail + node.d[node.dataLen] = y + inc(node.dataLen) + +proc init*[A](t: var SharedList[A]) = + initLock t.lock + t.head = nil + t.tail = nil + +proc clear*[A](t: var SharedList[A]) = + withLock(t): + var it = t.head + while it != nil: + let nxt = it.next + deallocShared(it) + it = nxt + t.head = nil + t.tail = nil + +proc deinitSharedList*[A](t: var SharedList[A]) = + clear(t) + deinitLock t.lock + +{.pop.} diff --git a/lib/pure/collections/sharedstrings.nim b/lib/pure/collections/sharedstrings.nim deleted file mode 100644 index 10ab30767..000000000 --- a/lib/pure/collections/sharedstrings.nim +++ /dev/null @@ -1,154 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Shared string support for Nim. - -const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000 - -type - UncheckedCharArray {.unchecked.} = array[0..ArrayDummySize, char] - -type - Buffer = ptr object - refcount: int - capacity, realLen: int - data: UncheckedCharArray - - SharedString* = object ## A string that can be shared. Slicing is O(1). - buffer: Buffer - first, len: int - -proc decRef(b: Buffer) {.inline.} = - if atomicDec(b.refcount) <= 0: - deallocShared(b) - -proc incRef(b: Buffer) {.inline.} = - atomicInc(b.refcount) - -{.experimental.} - -proc `=destroy`*(s: SharedString) = - #echo "destroyed" - if not s.buffer.isNil: - decRef(s.buffer) - -when false: - proc `=`*(dest: var SharedString; src: SharedString) = - incRef(src.buffer) - if not dest.buffer.isNil: - decRef(dest.buffer) - dest.buffer = src.buffer - dest.first = src.first - dest.len = src.len - -proc len*(s: SharedString): int = s.len - -proc `[]`*(s: SharedString; i: Natural): char = - if i < s.len: result = s.buffer.data[i+s.first] - else: raise newException(IndexError, "index out of bounds") - -proc `[]=`*(s: var SharedString; i: Natural; value: char) = - if i < s.len: s.buffer.data[i+s.first] = value - else: raise newException(IndexError, "index out of bounds") - -proc `[]`*(s: SharedString; ab: Slice[int]): SharedString = - #incRef(src.buffer) - if ab.a < s.len: - result.buffer = s.buffer - result.first = ab.a - result.len = min(s.len, ab.b - ab.a + 1) - # else: produce empty string ;-) - -proc newBuffer(cap, len: int): Buffer = - assert cap >= len - result = cast[Buffer](allocShared0(sizeof(int)*3 + cap)) - result.refcount = 0 - result.capacity = cap - result.realLen = len - -proc newSharedString*(len: Natural): SharedString = - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - result.first = 0 - result.len = len - -proc newSharedString*(s: string): SharedString = - let len = s.len - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - copyMem(addr result.buffer.data[0], cstring(s), s.len) - result.first = 0 - result.len = len - -when declared(atomicLoadN): - template load(x): expr = atomicLoadN(addr x, ATOMIC_SEQ_CST) -else: - # XXX Fixme - template load(x): expr = x - -proc add*(s: var SharedString; t: cstring; len: Natural) = - if len == 0: return - let newLen = s.len + len - if s.buffer.isNil: - s.buffer = newBuffer(len, len) - copyMem(addr s.buffer.data[0], t, len) - s.len = len - elif newLen >= s.buffer.capacity or s.first != 0 or - s.len != s.buffer.realLen or load(s.buffer.refcount) > 1: - let oldBuf = s.buffer - s.buffer = newBuffer(max(s.buffer.capacity * 3 div 2, newLen), newLen) - copyMem(addr s.buffer.data[0], addr oldBuf.data[s.first], s.len) - copyMem(addr s.buffer.data[s.len], t, len) - decRef(oldBuf) - else: - copyMem(addr s.buffer.data[s.len], t, len) - s.buffer.realLen += len - s.len += len - -proc add*(s: var SharedString; t: string) = - s.add(t.cstring, t.len) - -proc rawData*(s: var SharedString): pointer = - if s.buffer.isNil: result = nil - else: result = addr s.buffer.data[s.first] - -proc add*(s: var SharedString; t: SharedString) = - if t.buffer.isNil: return - s.add(cast[cstring](addr s.buffer.data[s.first]), t.len) - -proc `$`*(s: SharedString): string = - result = newString(s.len) - if s.len > 0: - copyMem(addr result[0], addr s.buffer.data[s.first], s.len) - -proc `==`*(s: SharedString; t: string): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - cstring(t), t.len) - -proc `==`*(s, t: SharedString): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - addr t.buffer.data[t.first], t.len) - -iterator items*(s: SharedString): char = - let buf = s.buffer.data - let x = s.first - if buf != nil: - for i in 0..<s.len: - yield buf[i+x] - -import hashes - -proc hash*(s: SharedString): THash = - var h: THash = 0 - for x in s: h = h !& x.hash - result = !$h diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 20e1bb7a9..b474ecd31 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -11,22 +11,34 @@ ## you'll be in trouble. Uses a single lock to protect the table, lockfree ## implementations welcome but if lock contention is so high that you need a ## lockfree hash table, you're doing it wrong. +## +## Unstable API. + +{.deprecated.} import - hashes, math, locks + std/[hashes, math, locks] type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] - KeyValuePairSeq[A, B] = ptr array[10_000_000, KeyValuePair[A, B]] - SharedTable* [A, B] = object ## generic hash SharedTable + KeyValuePairSeq[A, B] = ptr UncheckedArray[KeyValuePair[A, B]] + SharedTable*[A, B] = object ## generic hash SharedTable data: KeyValuePairSeq[A, B] counter, dataLen: int lock: Lock -template maxHash(t): expr = t.dataLen-1 +template maxHash(t): untyped = t.dataLen-1 include tableimpl +template st_maybeRehashPutImpl(enlarge) {.dirty.} = + if mustRehash(t): + enlarge(t) + index = rawGetKnownHC(t, key, hc) + index = -1 - index # important to transform for mgetOrPutImpl + rawInsert(t, t.data, key, val, hc, index) + inc(t.counter) + proc enlarge[A, B](t: var SharedTable[A, B]) = let oldSize = t.dataLen let size = oldSize * growthFactor @@ -35,9 +47,12 @@ proc enlarge[A, B](t: var SharedTable[A, B]) = t.dataLen = size swap(t.data, n) for i in 0..<oldSize: - if isFilled(n[i].hcode): - var j = -1 - rawGetKnownHC(t, n[i].key, n[i].hcode) - rawInsert(t, t.data, n[i].key, n[i].val, n[i].hcode, j) + let eh = n[i].hcode + if isFilled(eh): + var j: Hash = eh and maxHash(t) + while isFilled(t.data[j].hcode): + j = nextTry(j, maxHash(t)) + rawInsert(t, t.data, n[i].key, n[i].val, eh, j) deallocShared(n) template withLock(t, x: untyped) = @@ -45,9 +60,85 @@ template withLock(t, x: untyped) = x release(t.lock) +template withValue*[A, B](t: var SharedTable[A, B], key: A, + value, body: untyped) = + ## Retrieves the value at `t[key]`. + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + table.withValue("a", value): + assert value[] == "x" + + table.withValue("b", value): + value[] = "modified" + + table.withValue("b", value): + assert value[] == "modified" + + table.withValue("nonexistent", value): + assert false # not called + acquire(t.lock) + try: + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body + finally: + release(t.lock) + +template withValue*[A, B](t: var SharedTable[A, B], key: A, + value, body1, body2: untyped) = + ## Retrieves the value at `t[key]`. + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + + table.withValue("a", value): + value[] = "m" + + var flag = false + table.withValue("d", value): + discard value + doAssert false + do: # if "d" notin table + flag = true + + if flag: + table["d"] = "n" + + assert table.mget("a") == "m" + assert table.mget("d") == "n" + + acquire(t.lock) + try: + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body1 + else: + body2 + finally: + release(t.lock) + proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. + ## Retrieves the value at `t[key]`. The value can be modified. + ## If `key` is not in `t`, the `KeyError` exception is raised. withLock t: var hc: Hash var index = rawGet(t, key, hc) @@ -60,45 +151,101 @@ proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = raise newException(KeyError, "key not found") proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. **Note**: This is inherently ## unsafe in the context of multi-threading since it returns a pointer - ## to ``B``. + ## to `B`. withLock t: mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = - ## returns true iff `key` is in the table, otherwise inserts `value`. + ## Returns true if `key` is in the table, otherwise inserts `value`. withLock t: hasKeyOrPutImpl(enlarge) +template tabMakeEmpty(i) = t.data[i].hcode = 0 +template tabCellEmpty(i) = isEmpty(t.data[i].hcode) +template tabCellHash(i) = t.data[i].hcode + +proc withKey*[A, B](t: var SharedTable[A, B], key: A, + mapper: proc(key: A, val: var B, pairExists: var bool)) = + ## Computes a new mapping for the `key` with the specified `mapper` + ## procedure. + ## + ## The `mapper` takes 3 arguments: + ## + ## 1. `key` - the current key, if it exists, or the key passed to + ## `withKey` otherwise; + ## 2. `val` - the current value, if the key exists, or default value + ## of the type otherwise; + ## 3. `pairExists` - `true` if the key exists, `false` otherwise. + ## + ## The `mapper` can can modify `val` and `pairExists` values to change + ## the mapping of the key or delete it from the table. + ## When adding a value, make sure to set `pairExists` to `true` along + ## with modifying the `val`. + ## + ## The operation is performed atomically and other operations on the table + ## will be blocked while the `mapper` is invoked, so it should be short and + ## simple. + ## + ## Example usage: + ## + ## ```nim + ## # If value exists, decrement it. + ## # If it becomes zero or less, delete the key + ## t.withKey(1'i64) do (k: int64, v: var int, pairExists: var bool): + ## if pairExists: + ## dec v + ## if v <= 0: + ## pairExists = false + ## ``` + withLock t: + var hc: Hash + var index = rawGet(t, key, hc) + + var pairExists = index >= 0 + if pairExists: + mapper(t.data[index].key, t.data[index].val, pairExists) + if not pairExists: + delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash) + else: + var val: B + mapper(key, val, pairExists) + if pairExists: + st_maybeRehashPutImpl(enlarge) + proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. + ## Puts a (key, value)-pair into `t`. withLock t: putImpl(enlarge) proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## Puts a new (key, value)-pair into `t` even if `t[key]` already exists. + ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) proc del*[A, B](t: var SharedTable[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. + withLock t: + delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) + +proc len*[A, B](t: var SharedTable[A, B]): int = + ## Number of elements in `t`. withLock t: - delImpl() + result = t.counter -proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] = - ## creates a new hash table that is empty. +proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = + ## Creates a new hash table that is empty. ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc from this module. - assert isPowerOfTwo(initialSize) - result.counter = 0 - result.dataLen = initialSize - result.data = cast[KeyValuePairSeq[A, B]](allocShared0( + ## This proc must be called before any other usage of `t`. + let initialSize = slotsNeeded(initialSize) + t.counter = 0 + t.dataLen = initialSize + t.data = cast[KeyValuePairSeq[A, B]](allocShared0( sizeof(KeyValuePair[A, B]) * initialSize)) - initLock result.lock + initLock t.lock proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) = deallocShared(t.data) diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index beafe1109..3542741fa 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -7,48 +7,15 @@ # distribution, for details about the copyright. # -## An ``include`` file for the different table implementations. +# An `include` file for the different table implementations. -# hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These -# two procs retain clarity of that encoding without the space cost of an enum. -proc isEmpty(hcode: Hash): bool {.inline.} = - result = hcode == 0 - -proc isFilled(hcode: Hash): bool {.inline.} = - result = hcode != 0 +include hashcommon const - growthFactor = 2 - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = (h + 1) and maxHash - -template rawGetKnownHCImpl() {.dirty.} = - var h: Hash = hc and maxHash(t) # start with real hash value - while isFilled(t.data[h].hcode): - # Compare hc THEN key with boolean short circuit. This makes the common case - # zero ==key's for missing (e.g.inserts) and exactly one ==key for present. - # It does slow down succeeding lookups by one extra Hash cmp&and..usually - # just a few clock cycles, generally worth it for any non-integer-like A. - if t.data[h].hcode == hc and t.data[h].key == key: - return h - h = nextTry(h, maxHash(t)) - result = -1 - h # < 0 => MISSING; insert idx = -1 - result - -template rawGetImpl() {.dirty.} = - hc = hash(key) - if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. - rawGetKnownHCImpl() + defaultInitialSize* = 32 template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add - hc = hash(key) - if hc == 0: - hc = 314159265 + genHashImpl(key, hc) var h: Hash = hc and maxHash(t) while isFilled(t.data[h].hcode): h = nextTry(h, maxHash(t)) @@ -59,74 +26,206 @@ template rawInsertImpl() {.dirty.} = data[h].val = val data[h].hcode = hc -proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} = - rawGetKnownHCImpl() - -proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetDeepImpl() -proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} = - rawGetImpl() - proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], - key: A, val: B, hc: Hash, h: Hash) = + key: A, val: sink B, hc: Hash, h: Hash) = rawInsertImpl() -template addImpl(enlarge) {.dirty, immediate.} = - if mustRehash(t.dataLen, t.counter): enlarge(t) +template checkIfInitialized() = + if t.dataLen == 0: + initImpl(t, defaultInitialSize) + +template addImpl(enlarge) {.dirty.} = + checkIfInitialized() + if mustRehash(t): enlarge(t) var hc: Hash var j = rawGetDeep(t, key, hc) rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty, immediate.} = - if mustRehash(t.dataLen, t.counter): +template maybeRehashPutImpl(enlarge, val) {.dirty.} = + checkIfInitialized() + if mustRehash(t): enlarge(t) index = rawGetKnownHC(t, key, hc) index = -1 - index # important to transform for mgetOrPutImpl rawInsert(t, t.data, key, val, hc, index) inc(t.counter) -template putImpl(enlarge) {.dirty, immediate.} = - var hc: Hash +template putImpl(enlarge) {.dirty.} = + checkIfInitialized() + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val - else: maybeRehashPutImpl(enlarge) + else: maybeRehashPutImpl(enlarge, val) -template mgetOrPutImpl(enlarge) {.dirty, immediate.} = - var hc: Hash +template mgetOrPutImpl(enlarge) {.dirty.} = + checkIfInitialized() + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: # not present: insert (flipping index) - maybeRehashPutImpl(enlarge) + when declared(val): + maybeRehashPutImpl(enlarge, val) + else: + maybeRehashPutImpl(enlarge, default(B)) # either way return modifiable val result = t.data[index].val -template hasKeyOrPutImpl(enlarge) {.dirty, immediate.} = - var hc: Hash +# template mgetOrPutDefaultImpl(enlarge) {.dirty.} = +# checkIfInitialized() +# var hc: Hash = default(Hash) +# var index = rawGet(t, key, hc) +# if index < 0: +# # not present: insert (flipping index) +# maybeRehashPutImpl(enlarge, default(B)) +# # either way return modifiable val +# result = t.data[index].val + +template hasKeyOrPutImpl(enlarge) {.dirty.} = + checkIfInitialized() + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: result = false - maybeRehashPutImpl(enlarge) + maybeRehashPutImpl(enlarge, val) else: result = true -template delImpl() {.dirty, immediate.} = - var hc: Hash - var i = rawGet(t, key, hc) +# delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to +# be called "back shift delete". It shifts elements in the collision cluster of +# a victim backward to make things as-if the victim were never inserted in the +# first place. This is desirable to keep things "ageless" after many deletes. +# It is trickier than you might guess since initial probe (aka "home") locations +# of keys in a cluster may collide and since table addresses wrap around. +# +# A before-after diagram might look like ('.' means empty): +# slot: 0 1 2 3 4 5 6 7 +# before(1) +# hash1: 6 7 . 3 . 5 5 6 ; Really hash() and msk +# data1: E F . A . B C D ; About to delete C @index 6 +# after(2) +# hash2: 7 . . 3 . 5 6 6 ; Really hash() and msk +# data2: F . . A . B D E ; After deletion of C +# +# This lowers total search depth over the whole table from 1+1+2+2+2+2=10 to 7. +# Had the victim been B@5, C would need back shifting to slot 5. Total depth is +# always lowered by at least 1, e.g. victim A@3. This is all quite fast when +# empty slots are frequent (also needed to keep insert/miss searches fast) and +# hash() is either fast or avoided (via `.hcode`). It need not compare keys. +# +# delImplIdx realizes the above transformation, but only works for dense Linear +# Probing, nextTry(h)=h+1. This is not an important limitation since that's the +# fastest sequence on any CPU made since the 1980s. { Performance analysis often +# overweights "key cmp" neglecting cache behavior, giving bad ideas how big/slow +# tables behave (when perf matters most!). Comparing hcode first means usually +# only 1 key cmp is needed for *any* seq. Timing only predictable activity, +# small tables, and/or integer keys often perpetuates such bad ideas. } + +template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) = let msk = maxHash(t) if i >= 0: - t.data[i].hcode = 0 dec(t.counter) block outer: while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 - var j = i # The correctness of this depends on (h+1) in nextTry, + var j = i # The correctness of this depends on (h+1) in nextTry var r = j # though may be adaptable to other simple sequences. - t.data[i].hcode = 0 # mark current EMPTY + makeEmpty(i) # mark current EMPTY + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} while true: i = (i + 1) and msk # increment mod table size - if isEmpty(t.data[i].hcode): # end of collision cluster; So all done + if cellEmpty(i): # end of collision cluster; So all done break outer - r = t.data[i].hcode and msk # "home" location of key@i + r = cellHash(i) and msk # initial probe index for key@slot i if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): break - shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop + when defined(js): + t.data[j] = t.data[i] + else: + t.data[j] = move(t.data[i]) # data[j] will be marked EMPTY next loop + +template delImpl(makeEmpty, cellEmpty, cellHash) {.dirty.} = + var hc: Hash + var i = rawGet(t, key, hc) + delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) + +template delImplNoHCode(makeEmpty, cellEmpty, cellHash) {.dirty.} = + if t.dataLen > 0: + var i: Hash = hash(key) and maxHash(t) + while not cellEmpty(i): + if t.data[i].key == key: + delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) + break + i = nextTry(i, maxHash(t)) + +template clearImpl() {.dirty.} = + for i in 0 ..< t.dataLen: + when compiles(t.data[i].hcode): # CountTable records don't contain a hcode + t.data[i].hcode = 0 + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} + t.counter = 0 + +template ctAnd(a, b): bool = + when a: + when b: true + else: false + else: false + +template initImpl(result: typed, size: int) = + let correctSize = slotsNeeded(size) + when ctAnd(declared(SharedTable), typeof(result) is SharedTable): + init(result, correctSize) + else: + result.counter = 0 + newSeq(result.data, correctSize) + when compiles(result.first): + result.first = -1 + result.last = -1 + +template insertImpl() = # for CountTable + if t.dataLen == 0: initImpl(t, defaultInitialSize) + if mustRehash(t): enlarge(t) + ctRawInsert(t, t.data, key, val) + inc(t.counter) + +template getOrDefaultImpl(t, key): untyped = + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + if index >= 0: result = t.data[index].val + +template getOrDefaultImpl(t, key, default: untyped): untyped = + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + result = if index >= 0: t.data[index].val else: default + +template dollarImpl(): untyped {.dirty.} = + if t.len == 0: + result = "{:}" + else: + result = "{" + for key, val in pairs(t): + if result.len > 1: result.add(", ") + result.addQuoted(key) + result.add(": ") + result.addQuoted(val) + result.add("}") + +template equalsImpl(s, t: typed) = + if s.counter == t.counter: + # different insertion orders mean different 'data' seqs, so we have + # to use the slow route here: + for key, val in s: + if not t.hasKey(key): return false + if t.getOrDefault(key) != val: return false + return true + else: + return false diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 329b2a1cb..d414caeed 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -7,260 +7,591 @@ # distribution, for details about the copyright. # -## The ``tables`` module implements variants of an efficient `hash table`:idx: +## The `tables` module implements variants of an efficient `hash table`:idx: ## (also often named `dictionary`:idx: in other programming languages) that is -## a mapping from keys to values. ``Table`` is the usual hash table, -## ``OrderedTable`` is like ``Table`` but remembers insertion order -## and ``CountTable`` is a mapping from a key to its number of occurrences. +## a mapping from keys to values. +## +## There are several different types of hash tables available: +## * `Table<#Table>`_ is the usual hash table, +## * `OrderedTable<#OrderedTable>`_ is like `Table` but remembers insertion order, +## * `CountTable<#CountTable>`_ is a mapping from a key to its number of occurrences +## ## For consistency with every other data type in Nim these have **value** -## semantics, this means that ``=`` performs a copy of the hash table. -## For **reference** semantics use the ``Ref`` variant: ``TableRef``, -## ``OrderedTableRef``, ``CountTableRef``. +## semantics, this means that `=` performs a copy of the hash table. +## +## For `ref semantics<manual.html#types-reference-and-pointer-types>`_ +## use their `Ref` variants: `TableRef<#TableRef>`_, +## `OrderedTableRef<#OrderedTableRef>`_, and `CountTableRef<#CountTableRef>`_. +## +## To give an example, when `a` is a `Table`, then `var b = a` gives `b` +## as a new independent table. `b` is initialised with the contents of `a`. +## Changing `b` does not affect `a` and vice versa: + +runnableExamples: + var + a = {1: "one", 2: "two"}.toTable # creates a Table + b = a + + assert a == b + + b[3] = "three" + assert 3 notin a + assert 3 in b + assert a != b + +## On the other hand, when `a` is a `TableRef` instead, then changes to `b` +## also affect `a`. Both `a` and `b` **ref** the same data structure: + +runnableExamples: + var + a = {1: "one", 2: "two"}.newTable # creates a TableRef + b = a + + assert a == b + + b[3] = "three" + + assert 3 in a + assert 3 in b + assert a == b + ## -## If you are using simple standard types like ``int`` or ``string`` for the +## ---- +## + +## # Basic usage + + +## ## Table +runnableExamples: + from std/sequtils import zip + + let + names = ["John", "Paul", "George", "Ringo"] + years = [1940, 1942, 1943, 1940] + + var beatles = initTable[string, int]() + + for pairs in zip(names, years): + let (name, birthYear) = pairs + beatles[name] = birthYear + + assert beatles == {"George": 1943, "Ringo": 1940, "Paul": 1942, "John": 1940}.toTable + + + var beatlesByYear = initTable[int, seq[string]]() + + for pairs in zip(years, names): + let (birthYear, name) = pairs + if not beatlesByYear.hasKey(birthYear): + # if a key doesn't exist, we create one with an empty sequence + # before we can add elements to it + beatlesByYear[birthYear] = @[] + beatlesByYear[birthYear].add(name) + + assert beatlesByYear == {1940: @["John", "Ringo"], 1942: @["Paul"], 1943: @["George"]}.toTable + +## ## OrderedTable +## `OrderedTable<#OrderedTable>`_ is used when it is important to preserve +## the insertion order of keys. + +runnableExamples: + let + a = [('z', 1), ('y', 2), ('x', 3)] + ot = a.toOrderedTable # ordered tables + + assert $ot == """{'z': 1, 'y': 2, 'x': 3}""" + +## ## CountTable +## `CountTable<#CountTable>`_ is useful for counting number of items of some +## container (e.g. string, sequence or array), as it is a mapping where the +## items are the keys, and their number of occurrences are the values. +## For that purpose `toCountTable proc<#toCountTable,openArray[A]>`_ +## comes handy: + +runnableExamples: + let myString = "abracadabra" + let letterFrequencies = toCountTable(myString) + assert $letterFrequencies == "{'a': 5, 'd': 1, 'b': 2, 'r': 2, 'c': 1}" + +## The same could have been achieved by manually iterating over a container +## and increasing each key's value with `inc proc +## <#inc,CountTable[A],A,int>`_: + +runnableExamples: + let myString = "abracadabra" + var letterFrequencies = initCountTable[char]() + for c in myString: + letterFrequencies.inc(c) + assert $letterFrequencies == "{'d': 1, 'r': 2, 'c': 1, 'a': 5, 'b': 2}" + +## +## ---- +## + +## ## Hashing +## +## If you are using simple standard types like `int` or `string` for the ## keys of the table you won't have any problems, but as soon as you try to use ## a more complex object as a key you will be greeted by a strange compiler -## error:: +## error: ## -## Error: type mismatch: got (Person) -## but expected one of: -## hashes.hash(x: openarray[A]): Hash -## hashes.hash(x: int): Hash -## hashes.hash(x: float): Hash -## … +## Error: type mismatch: got (Person) +## but expected one of: +## hashes.hash(x: openArray[A]): Hash +## hashes.hash(x: int): Hash +## hashes.hash(x: float): Hash ## ## What is happening here is that the types used for table keys require to have -## a ``hash()`` proc which will convert them to a `Hash <hashes.html#Hash>`_ +## a `hash()` proc which will convert them to a `Hash <hashes.html#Hash>`_ ## value, and the compiler is listing all the hash functions it knows. -## Additionally there has to be a ``==`` operator that provides the same -## semantics as its corresponding ``hash`` proc. +## Additionally there has to be a `==` operator that provides the same +## semantics as its corresponding `hash` proc. ## -## After you add ``hash`` and ``==`` for your custom type everything will work. -## Currently however ``hash`` for objects is not defined, whereas -## ``system.==`` for objects does exist and performs a "deep" comparison (every +## After you add `hash` and `==` for your custom type everything will work. +## Currently, however, `hash` for objects is not defined, whereas +## `system.==` for objects does exist and performs a "deep" comparison (every ## field is compared) which is usually what you want. So in the following -## example implementing only ``hash`` suffices: -## -## .. code-block:: -## type -## Person = object -## firstName, lastName: string -## -## proc hash(x: Person): Hash = -## ## Piggyback on the already available string hash proc. -## ## -## ## Without this proc nothing works! -## result = x.firstName.hash !& x.lastName.hash -## result = !$result +## example implementing only `hash` suffices: + +runnableExamples: + import std/hashes + + type + Person = object + firstName, lastName: string + + proc hash(x: Person): Hash = + ## Piggyback on the already available string hash proc. + ## + ## Without this proc nothing works! + result = x.firstName.hash !& x.lastName.hash + result = !$result + + var + salaries = initTable[Person, int]() + p1, p2: Person + + p1.firstName = "Jon" + p1.lastName = "Ross" + salaries[p1] = 30_000 + + p2.firstName = "소진" + p2.lastName = "박" + salaries[p2] = 45_000 + ## -## var -## salaries = initTable[Person, int]() -## p1, p2: Person +## ---- ## -## p1.firstName = "Jon" -## p1.lastName = "Ross" -## salaries[p1] = 30_000 + +## # See also ## -## p2.firstName = "소진" -## p2.lastName = "박" -## salaries[p2] = 45_000 +## * `json module<json.html>`_ for table-like structure which allows +## heterogeneous members +## * `strtabs module<strtabs.html>`_ for efficient hash tables +## mapping from strings to strings +## * `hashes module<hashes.html>`_ for helper functions for hashing + -import - hashes, math +import std/private/since +import std/[hashes, math, algorithm] -include "system/inclrtl" + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] KeyValuePairSeq[A, B] = seq[KeyValuePair[A, B]] - Table*[A, B] = object ## generic hash table + Table*[A, B] = object + ## Generic hash table, consisting of a key-value pair. + ## + ## `data` and `counter` are internal implementation details which + ## can't be accessed. + ## + ## For creating an empty Table, use `initTable proc<#initTable>`_. data: KeyValuePairSeq[A, B] counter: int - TableRef*[A,B] = ref Table[A, B] + TableRef*[A, B] = ref Table[A, B] ## Ref version of `Table<#Table>`_. + ## + ## For creating a new empty TableRef, use `newTable proc + ## <#newTable>`_. -{.deprecated: [TTable: Table, PTable: TableRef].} -template maxHash(t): expr {.immediate.} = high(t.data) -template dataLen(t): expr = len(t.data) +# ------------------------------ helpers --------------------------------- -include tableimpl +# Do NOT move these to tableimpl.nim, because sharedtables uses that +# file and has its own implementation. +template maxHash(t): untyped = high(t.data) +template dataLen(t): untyped = len(t.data) -proc rightSize*(count: Natural): int {.inline.} = - ## Return the value of `initialSize` to support `count` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - ## - ## Internally, we want mustRehash(rightSize(x), x) == false. - result = nextPowerOfTwo(count * 3 div 2 + 4) +include tableimpl -proc len*[A, B](t: Table[A, B]): int = - ## returns the number of keys in `t`. - result = t.counter +proc raiseKeyError[T](key: T) {.noinline, noreturn.} = + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") -template get(t, key): untyped {.immediate.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. +template get(t, key): untyped = + ## retrieves the value at `t[key]`. The value can be modified. + ## If `key` is not in `t`, the `KeyError` exception is raised. mixin rawGet var hc: Hash var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raiseKeyError(key) -template getOrDefaultImpl(t, key): untyped {.immediate.} = - mixin rawGet - var hc: Hash - var index = rawGet(t, key, hc) - if index >= 0: result = t.data[index].val +proc enlarge[A, B](t: var Table[A, B]) = + var n: KeyValuePairSeq[A, B] + newSeq(n, len(t.data) * growthFactor) + swap(t.data, n) + for i in countup(0, high(n)): + let eh = n[i].hcode + if isFilled(eh): + var j: Hash = eh and maxHash(t) + while isFilled(t.data[j].hcode): + j = nextTry(j, maxHash(t)) + when defined(js): + rawInsert(t, t.data, n[i].key, n[i].val, eh, j) + else: + rawInsert(t, t.data, move n[i].key, move n[i].val, eh, j) -proc `[]`*[A, B](t: Table[A, B], key: A): B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. - get(t, key) -proc `[]`*[A, B](t: var Table[A, B], key: A): var B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - get(t, key) -proc mget*[A, B](t: var Table[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. Use ```[]``` - ## instead. - get(t, key) -proc getOrDefault*[A, B](t: Table[A, B], key: A): B = getOrDefaultImpl(t, key) +# ------------------------------------------------------------------- +# ------------------------------ Table ------------------------------ +# ------------------------------------------------------------------- -iterator allValues*[A, B](t: Table[A, B]; key: A): B = - ## iterates over any value in the table `t` that belongs to the given `key`. - var h: Hash = hash(key) and high(t.data) - while isFilled(t.data[h].hcode): - if t.data[h].key == key: - yield t.data[h].val - h = nextTry(h, high(t.data)) +proc initTable*[A, B](initialSize = defaultInitialSize): Table[A, B] = + ## Creates a new hash table that is empty. + ## + ## Starting from Nim v0.20, tables are initialized by default and it is + ## not necessary to call this function explicitly. + ## + ## See also: + ## * `toTable proc<#toTable,openArray[]>`_ + ## * `newTable proc<#newTable>`_ for creating a `TableRef` + runnableExamples: + let + a = initTable[int, string]() + b = initTable[char, seq[int]]() + result = default(Table[A, B]) + initImpl(result, initialSize) + +proc `[]=`*[A, B](t: var Table[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,Table[A,B],A,B>`_ + ## * `del proc<#del,Table[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = initTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.toTable + + putImpl(enlarge) + +proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = + ## Creates a new hash table that contains the given `pairs`. + ## + ## `pairs` is a container consisting of `(key, value)` tuples. + ## + ## See also: + ## * `initTable proc<#initTable>`_ + ## * `newTable proc<#newTable,openArray[]>`_ for a `TableRef` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = toTable(a) + assert b == {'a': 5, 'b': 9}.toTable + + result = initTable[A, B](pairs.len) + for key, val in items(pairs): result[key] = val + +proc `[]`*[A, B](t: Table[A, B], key: A): lent B = + ## Retrieves the value at `t[key]`. + ## + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## One can check with `hasKey proc<#hasKey,Table[A,B],A>`_ whether + ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in + ## the table + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] + get(t, key) + +proc `[]`*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at `t[key]`. The value can be modified. + ## + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in + ## the table + get(t, key) proc hasKey*[A, B](t: Table[A, B], key: A): bool = - ## returns true iff `key` is in the table `t`. + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `contains proc<#contains,Table[A,B],A>`_ for use with the `in` operator + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + var hc: Hash result = rawGet(t, key, hc) >= 0 proc contains*[A, B](t: Table[A, B], key: A): bool = - ## alias of `hasKey` for use with the `in` operator. + ## Alias of `hasKey proc<#hasKey,Table[A,B],A>`_ for use with + ## the `in` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + return hasKey[A, B](t, key) -iterator pairs*[A, B](t: Table[A, B]): (A, B) = - ## iterates over any (key, value) pair in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) +proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = + ## Returns true if `key` is in the table, otherwise inserts `value`. + ## + ## See also: + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.toTable -iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = - ## iterates over any (key, value) pair in the table `t`. The values - ## can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) + hasKeyOrPutImpl(enlarge) -iterator keys*[A, B](t: Table[A, B]): A = - ## iterates over any key in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].key +proc getOrDefault*[A, B](t: Table[A, B], key: A): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,Table[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + result = default(B) + getOrDefaultImpl(t, key) -iterator values*[A, B](t: Table[A, B]): B = - ## iterates over any value in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val +proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,Table[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + result = default(B) + getOrDefaultImpl(t, key, default) -iterator mvalues*[A, B](t: var Table[A, B]): var B = - ## iterates over any value in the table `t`. The values can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val +proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = + ## Retrieves value at `t[key]` or puts `val` if not present, either way + ## returning a value which can be modified. + ## + ## + ## Note that while the value returned is of type `var B`, + ## it is easy to accidentally create a copy of the value at `t[key]`. + ## Remember that seqs and strings are value types, and therefore + ## cannot be copied into a separate variable for modification. + ## See the example below. + ## + ## See also: + ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,Table[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,Table[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.toTable + + # An example of accidentally creating a copy + var t = initTable[int, seq[int]]() + # In this example, we expect t[10] to be modified, + # but it is not. + var copiedSeq = t.mgetOrPut(10, @[10]) + copiedSeq.add(20) + doAssert t[10] == @[10] + # Correct + t.mgetOrPut(25, @[25]).add(35) + doAssert t[25] == @[25, 35] -proc del*[A, B](t: var Table[A, B], key: A) = - ## deletes `key` from hash table `t`. - delImpl() + mgetOrPutImpl(enlarge) -proc enlarge[A, B](t: var Table[A, B]) = - var n: KeyValuePairSeq[A, B] - newSeq(n, len(t.data) * growthFactor) - swap(t.data, n) - for i in countup(0, high(n)): - if isFilled(n[i].hcode): - var j = -1 - rawGetKnownHC(t, n[i].key, n[i].hcode) - rawInsert(t, t.data, n[i].key, n[i].val, n[i].hcode, j) +proc mgetOrPut*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable -proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way - ## returning a value which can be modified. mgetOrPutImpl(enlarge) -proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = - ## returns true iff `key` is in the table, otherwise inserts `value`. - hasKeyOrPutImpl(enlarge) +proc len*[A, B](t: Table[A, B]): int = + ## Returns the number of keys in `t`. + runnableExamples: + let a = {'a': 5, 'b': 9}.toTable + doAssert len(a) == 2 -proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. - putImpl(enlarge) + result = t.counter -proc add*[A, B](t: var Table[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. +proc add*[A, B](t: var Table[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. addImpl(enlarge) -proc len*[A, B](t: TableRef[A, B]): int = - ## returns the number of keys in `t`. - result = t.counter +template tabMakeEmpty(i) = t.data[i].hcode = 0 +template tabCellEmpty(i) = isEmpty(t.data[i].hcode) +template tabCellHash(i) = t.data[i].hcode -proc initTable*[A, B](initialSize=64): Table[A, B] = - ## creates a new hash table that is empty. +proc del*[A, B](t: var Table[A, B], key: A) = + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. + ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## + ## See also: + ## * `pop proc<#pop,Table[A,B],A,B>`_ + ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.toTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.toTable + + delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) + +proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool = + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is + ## unchanged. ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc from this module. - assert isPowerOfTwo(initialSize) - result.counter = 0 - newSeq(result.data, initialSize) + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## + ## See also: + ## * `del proc<#del,Table[A,B],A>`_ + ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table + runnableExamples: + var + a = {'a': 5, 'b': 9, 'c': 13}.toTable + i: int + doAssert a.pop('b', i) == true + doAssert a == {'a': 5, 'c': 13}.toTable + doAssert i == 9 + i = 0 + doAssert a.pop('z', i) == false + doAssert a == {'a': 5, 'c': 13}.toTable + doAssert i == 0 -proc toTable*[A, B](pairs: openArray[(A, - B)]): Table[A, B] = - ## creates a new hash table that contains the given `pairs`. - result = initTable[A, B](rightSize(pairs.len)) - for key, val in items(pairs): result[key] = val + var hc: Hash + var index = rawGet(t, key, hc) + result = index >= 0 + if result: + val = move(t.data[index].val) + delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash) + +proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool {.inline.} = + ## Alias for: + ## * `pop proc<#pop,Table[A,B],A,B>`_ + pop(t, key, val) + +proc clear*[A, B](t: var Table[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,Table[A,B],A>`_ + ## * `pop proc<#pop,Table[A,B],A,B>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 -template dollarImpl(): stmt {.dirty.} = - if t.len == 0: - result = "{:}" - else: - result = "{" - for key, val in pairs(t): - if result.len > 1: result.add(", ") - result.add($key) - result.add(": ") - result.add($val) - result.add("}") + clearImpl() proc `$`*[A, B](t: Table[A, B]): string = - ## The `$` operator for hash tables. + ## The `$` operator for hash tables. Used internally when calling `echo` + ## on a table. dollarImpl() -proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = - ## returns true iff `key` is in the table `t`. - result = t[].hasKey(key) - -template equalsImpl() = - if s.counter == t.counter: - # different insertion orders mean different 'data' seqs, so we have - # to use the slow route here: - for key, val in s: - # prefix notation leads to automatic dereference in case of PTable - if not t.hasKey(key): return false - if t[key] != val: return false - return true - proc `==`*[A, B](s, t: Table[A, B]): bool = - equalsImpl() + ## The `==` operator for hash tables. Returns `true` if the content of both + ## tables contains the same key-value pairs. Insert order does not matter. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.toTable + b = {'b': 9, 'c': 13, 'a': 5}.toTable + doAssert a == b + + equalsImpl(s, t) proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. @@ -269,147 +600,692 @@ proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = for item in collection: result[index(item)] = item -iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = - ## iterates over any (key, value) pair in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = - ## iterates over any (key, value) pair in the table `t`. The values - ## can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield (t.data[h].key, t.data[h].val) -iterator keys*[A, B](t: TableRef[A, B]): A = - ## iterates over any key in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].key +template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = + ## Retrieves the value at `t[key]`. + ## + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + type + User = object + name: string + uid: int -iterator values*[A, B](t: TableRef[A, B]): B = - ## iterates over any value in the table `t`. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val + var t = initTable[int, User]() + let u = User(name: "Hello", uid: 99) + t[1] = u -iterator mvalues*[A, B](t: TableRef[A, B]): var B = - ## iterates over any value in the table `t`. The values can be modified. - for h in 0..high(t.data): - if isFilled(t.data[h].hcode): yield t.data[h].val + t.withValue(1, value): + # block is executed only if `key` in `t` + value.name = "Nim" + value.uid = 1314 -proc `[]`*[A, B](t: TableRef[A, B], key: A): var B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether + t.withValue(2, value): + value.name = "No" + value.uid = 521 + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body + +template withValue*[A, B](t: var Table[A, B], key: A, + value, body1, body2: untyped) = + ## Retrieves the value at `t[key]`. + ## + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + type + User = object + name: string + uid: int + + var t = initTable[int, User]() + let u = User(name: "Hello", uid: 99) + t[1] = u + + t.withValue(1, value): + # block is executed only if `key` in `t` + value.name = "Nim" + value.uid = 1314 + + t.withValue(521, value): + doAssert false + do: + # block is executed when `key` not in `t` + t[1314] = User(name: "exist", uid: 521) + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + assert t[1314].name == "exist" + assert t[1314].uid == 521 + + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body1 + else: + body2 + + +iterator pairs*[A, B](t: Table[A, B]): (A, B) = + ## Iterates over any `(key, value)` pair in the table `t`. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,Table[A,B]>`_ + ## * `keys iterator<#keys.i,Table[A,B]>`_ + ## * `values iterator<#values.i,Table[A,B]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.toTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: e + ## # value: [2, 4, 6, 8] + ## # key: o + ## # value: [1, 5, 7, 9] + ## ``` + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = + ## Iterates over any `(key, value)` pair in the table `t` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,Table[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,Table[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'e': @[2, 4, 6, 8, 12], 'o': @[1, 5, 7, 9, 11]}.toTable + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator keys*[A, B](t: Table[A, B]): lent A = + ## Iterates over any key in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,Table[A,B]>`_ + ## * `values iterator<#values.i,Table[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for k in a.keys: + a[k].add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.toTable + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield t.data[h].key + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator values*[A, B](t: Table[A, B]): lent B = + ## Iterates over any value in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,Table[A,B]>`_ + ## * `keys iterator<#keys.i,Table[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,Table[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for v in a.values: + doAssert v.len == 4 + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator mvalues*[A, B](t: var Table[A, B]): var B = + ## Iterates over any value in the table `t` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,Table[A,B]>`_ + ## * `values iterator<#values.i,Table[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toTable + for v in a.mvalues: + v.add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.toTable + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator allValues*[A, B](t: Table[A, B]; key: A): B {.deprecated: + "Deprecated since v1.4; tables with duplicated keys are deprecated".} = + ## Iterates over any value in the table `t` that belongs to the given `key`. + ## + ## Used if you have a table with duplicate keys (as a result of using + ## `add proc<#add,Table[A,B],A,sinkB>`_). + ## + runnableExamples: + import std/[sequtils, algorithm] + + var a = {'a': 3, 'b': 5}.toTable + for i in 1..3: a.add('z', 10*i) + doAssert toSeq(a.pairs).sorted == @[('a', 3), ('b', 5), ('z', 10), ('z', 20), ('z', 30)] + doAssert sorted(toSeq(a.allValues('z'))) == @[10, 20, 30] + var h: Hash = genHash(key) and high(t.data) + let L = len(t) + while isFilled(t.data[h].hcode): + if t.data[h].key == key: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + h = nextTry(h, high(t.data)) + + + +# ------------------------------------------------------------------- +# ---------------------------- TableRef ----------------------------- +# ------------------------------------------------------------------- + + +proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = + ## Creates a new ref hash table that is empty. + ## + ## See also: + ## * `newTable proc<#newTable,openArray[]>`_ for creating a `TableRef` + ## from a collection of `(key, value)` pairs + ## * `initTable proc<#initTable>`_ for creating a `Table` + runnableExamples: + let + a = newTable[int, string]() + b = newTable[char, seq[int]]() + + new(result) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) + +proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = + ## Creates a new ref hash table that contains the given `pairs`. + ## + ## `pairs` is a container consisting of `(key, value)` tuples. + ## + ## See also: + ## * `newTable proc<#newTable>`_ + ## * `toTable proc<#toTable,openArray[]>`_ for a `Table` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = newTable(a) + assert b == {'a': 5, 'b': 9}.newTable + + new(result) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) + +proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = + ## Index the collection with the proc provided. + # TODO: As soon as supported, change collection: A to collection: A[B] + result = newTable[C, B]() + {.noSideEffect.}: + for item in collection: + result[index(item)] = item + +proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]`. + ## + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## One can check with `hasKey proc<#hasKey,TableRef[A,B],A>`_ whether ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,TableRef[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ for checking if a key is in + ## the table + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] + result = t[][key] -proc mget*[A, B](t: TableRef[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. - t[][key] +proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,TableRef[A,B],A,B>`_ + ## * `del proc<#del,TableRef[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = newTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.newTable + + t[][key] = val + +proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `contains proc<#contains,TableRef[A,B],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + + result = t[].hasKey(key) + +proc contains*[A, B](t: TableRef[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,TableRef[A,B],A>`_ for use with + ## the `in` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert 'b' in a == true + doAssert a.contains('z') == false -proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = getOrDefault(t[], key) + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: TableRef[A, B], key: A, val: B): bool = + ## Returns true if `key` is in the table, otherwise inserts `value`. + ## + ## See also: + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.newTable + + t[].hasKeyOrPut(key, val) + +proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,TableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + + getOrDefault(t[], key) + +proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,TableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + + getOrDefault(t[], key, default) proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. + ## + ## Note that while the value returned is of type `var B`, + ## it is easy to accidentally create an copy of the value at `t[key]`. + ## Remember that seqs and strings are value types, and therefore + ## cannot be copied into a separate variable for modification. + ## See the example below. + ## + ## See also: + ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,TableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.newTable + + # An example of accidentally creating a copy + var t = newTable[int, seq[int]]() + # In this example, we expect t[10] to be modified, + # but it is not. + var copiedSeq = t.mgetOrPut(10, @[10]) + copiedSeq.add(20) + doAssert t[10] == @[10] + # Correct + t.mgetOrPut(25, @[25]).add(35) + doAssert t[25] == @[25, 35] t[].mgetOrPut(key, val) -proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = - ## returns true iff `key` is in the table, otherwise inserts `value`. - t[].hasKeyOrPut(key, val) +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable -proc contains*[A, B](t: TableRef[A, B], key: A): bool = - ## alias of `hasKey` for use with the `in` operator. - return hasKey[A, B](t, key) + t[].mgetOrPut(key) -proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. - t[][key] = val +proc len*[A, B](t: TableRef[A, B]): int = + ## Returns the number of keys in `t`. + runnableExamples: + let a = {'a': 5, 'b': 9}.newTable + doAssert len(a) == 2 + + result = t.counter -proc add*[A, B](t: TableRef[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. +proc add*[A, B](t: TableRef[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,TableRef[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. t[].add(key, val) proc del*[A, B](t: TableRef[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. + ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## + ## See also: + ## * `pop proc<#pop,TableRef[A,B],A,B>`_ + ## * `clear proc<#clear,TableRef[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.newTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.newTable + t[].del(key) -proc newTable*[A, B](initialSize=64): TableRef[A, B] = - new(result) - result[] = initTable[A, B](initialSize) +proc pop*[A, B](t: TableRef[A, B], key: A, val: var B): bool = + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is + ## unchanged. + ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## + ## See also: + ## * `del proc<#del,TableRef[A,B],A>`_ + ## * `clear proc<#clear,TableRef[A,B]>`_ to empty the whole table + runnableExamples: + var + a = {'a': 5, 'b': 9, 'c': 13}.newTable + i: int + doAssert a.pop('b', i) == true + doAssert a == {'a': 5, 'c': 13}.newTable + doAssert i == 9 + i = 0 + doAssert a.pop('z', i) == false + doAssert a == {'a': 5, 'c': 13}.newTable + doAssert i == 0 + + result = t[].pop(key, val) + +proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool {.inline.} = + ## Alias for: + ## * `pop proc<#pop,TableRef[A,B],A,B>`_ + pop(t, key, val) + +proc clear*[A, B](t: TableRef[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,Table[A,B],A>`_ + ## * `pop proc<#pop,Table[A,B],A,B>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 -proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = - ## creates a new hash table that contains the given `pairs`. - new(result) - result[] = toTable[A, B](pairs) + clearImpl() proc `$`*[A, B](t: TableRef[A, B]): string = - ## The `$` operator for hash tables. + ## The `$` operator for hash tables. Used internally when calling `echo` + ## on a table. dollarImpl() proc `==`*[A, B](s, t: TableRef[A, B]): bool = + ## The `==` operator for hash tables. Returns `true` if either both tables + ## are `nil`, or neither is `nil` and the content of both tables contains the + ## same key-value pairs. Insert order does not matter. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.newTable + b = {'b': 9, 'c': 13, 'a': 5}.newTable + doAssert a == b + if isNil(s): result = isNil(t) elif isNil(t): result = false - else: equalsImpl() + else: equalsImpl(s[], t[]) -proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = - ## Index the collection with the proc provided. - # TODO: As soon as supported, change collection: A to collection: A[B] - result = newTable[C, B]() - for item in collection: - result[index(item)] = item -# ------------------------------ ordered table ------------------------------ -type - OrderedKeyValuePair[A, B] = tuple[ - hcode: Hash, next: int, key: A, val: B] - OrderedKeyValuePairSeq[A, B] = seq[OrderedKeyValuePair[A, B]] - OrderedTable* [A, B] = object ## table that remembers insertion order - data: OrderedKeyValuePairSeq[A, B] - counter, first, last: int - OrderedTableRef*[A, B] = ref OrderedTable[A, B] +iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = + ## Iterates over any `(key, value)` pair in the table `t`. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,TableRef[A,B]>`_ + ## * `keys iterator<#keys.i,TableRef[A,B]>`_ + ## * `values iterator<#values.i,TableRef[A,B]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.newTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: e + ## # value: [2, 4, 6, 8] + ## # key: o + ## # value: [1, 5, 7, 9] + ## ``` + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") -{.deprecated: [TOrderedTable: OrderedTable, POrderedTable: OrderedTableRef].} +iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = + ## Iterates over any `(key, value)` pair in the table `t`. The values + ## can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'e': @[2, 4, 6, 8, 12], 'o': @[1, 5, 7, 9, 11]}.newTable + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator keys*[A, B](t: TableRef[A, B]): lent A = + ## Iterates over any key in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ + ## * `values iterator<#values.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for k in a.keys: + a[k].add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.newTable + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield t.data[h].key + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator values*[A, B](t: TableRef[A, B]): lent B = + ## Iterates over any value in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ + ## * `keys iterator<#keys.i,TableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for v in a.values: + doAssert v.len == 4 + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") -proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = - ## returns the number of keys in `t`. - result = t.counter +iterator mvalues*[A, B](t: TableRef[A, B]): var B = + ## Iterates over any value in the table `t`. The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,TableRef[A,B]>`_ + ## * `values iterator<#values.i,TableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newTable + for v in a.mvalues: + v.add(99) + doAssert a == {'e': @[2, 4, 6, 8, 99], 'o': @[1, 5, 7, 9, 99]}.newTable + + let L = len(t) + for h in 0 .. high(t.data): + if isFilled(t.data[h].hcode): + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = - var h = t.first - while h >= 0: - var nxt = t.data[h].next - if isFilled(t.data[h].hcode): yieldStmt - h = nxt -iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = - ## iterates over any (key, value) pair in the table `t` in insertion - ## order. - forAllOrderedPairs: - yield (t.data[h].key, t.data[h].val) -iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = - ## iterates over any (key, value) pair in the table `t` in insertion - ## order. The values can be modified. - forAllOrderedPairs: - yield (t.data[h].key, t.data[h].val) -iterator keys*[A, B](t: OrderedTable[A, B]): A = - ## iterates over any key in the table `t` in insertion order. - forAllOrderedPairs: - yield t.data[h].key -iterator values*[A, B](t: OrderedTable[A, B]): B = - ## iterates over any value in the table `t` in insertion order. - forAllOrderedPairs: - yield t.data[h].val -iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = - ## iterates over any value in the table `t` in insertion order. The values - ## can be modified. - forAllOrderedPairs: - yield t.data[h].val + + +# --------------------------------------------------------------------------- +# ------------------------------ OrderedTable ------------------------------- +# --------------------------------------------------------------------------- + +type + OrderedKeyValuePair[A, B] = tuple[ + hcode: Hash, next: int, key: A, val: B] + OrderedKeyValuePairSeq[A, B] = seq[OrderedKeyValuePair[A, B]] + OrderedTable*[A, B] = object + ## Hash table that remembers insertion order. + ## + ## For creating an empty OrderedTable, use `initOrderedTable proc + ## <#initOrderedTable>`_. + data: OrderedKeyValuePairSeq[A, B] + counter, first, last: int + OrderedTableRef*[A, B] = ref OrderedTable[A, B] ## Ref version of + ## `OrderedTable<#OrderedTable>`_. + ## + ## For creating a new empty OrderedTableRef, use `newOrderedTable proc + ## <#newOrderedTable>`_. + + +# ------------------------------ helpers --------------------------------- proc rawGetKnownHC[A, B](t: OrderedTable[A, B], key: A, hc: Hash): int = rawGetKnownHCImpl() @@ -420,39 +1296,9 @@ proc rawGetDeep[A, B](t: OrderedTable[A, B], key: A, hc: var Hash): int {.inline proc rawGet[A, B](t: OrderedTable[A, B], key: A, hc: var Hash): int = rawGetImpl() -proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. - get(t, key) - -proc `[]`*[A, B](t: var OrderedTable[A, B], key: A): var B{.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - get(t, key) - -proc mget*[A, B](t: var OrderedTable[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. - get(t, key) - -proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = - getOrDefaultImpl(t, key) - - -proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = - ## returns true iff `key` is in the table `t`. - var hc: Hash - result = rawGet(t, key, hc) >= 0 - -proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = - ## alias of `hasKey` for use with the `in` operator. - return hasKey[A, B](t, key) - proc rawInsert[A, B](t: var OrderedTable[A, B], data: var OrderedKeyValuePairSeq[A, B], - key: A, val: B, hc: Hash, h: Hash) = + key: A, val: sink B, hc: Hash, h: Hash) = rawInsertImpl() data[h].next = -1 if t.first < 0: t.first = h @@ -468,56 +1314,349 @@ proc enlarge[A, B](t: var OrderedTable[A, B]) = swap(t.data, n) while h >= 0: var nxt = n[h].next - if isFilled(n[h].hcode): - var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) - rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) + let eh = n[h].hcode + if isFilled(eh): + var j: Hash = eh and maxHash(t) + while isFilled(t.data[j].hcode): + j = nextTry(j, maxHash(t)) + rawInsert(t, t.data, move n[h].key, move n[h].val, n[h].hcode, j) h = nxt -proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. +template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = + if t.counter > 0: + var h = t.first + while h >= 0: + var nxt = t.data[h].next + if isFilled(t.data[h].hcode): + yieldStmt + h = nxt + +# ---------------------------------------------------------------------- + +proc initOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTable[A, B] = + ## Creates a new ordered hash table that is empty. + ## + ## Starting from Nim v0.20, tables are initialized by default and it is + ## not necessary to call this function explicitly. + ## + ## See also: + ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_ + ## * `newOrderedTable proc<#newOrderedTable>`_ for creating an + ## `OrderedTableRef` + runnableExamples: + let + a = initOrderedTable[int, string]() + b = initOrderedTable[char, seq[int]]() + result = default(OrderedTable[A, B]) + initImpl(result, initialSize) + +proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTable[A,B],A,B>`_ + ## * `del proc<#del,OrderedTable[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = initOrderedTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.toOrderedTable + putImpl(enlarge) -proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. - addImpl(enlarge) +proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = + ## Creates a new ordered hash table that contains the given `pairs`. + ## + ## `pairs` is a container consisting of `(key, value)` tuples. + ## + ## See also: + ## * `initOrderedTable proc<#initOrderedTable>`_ + ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for an + ## `OrderedTableRef` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = toOrderedTable(a) + assert b == {'a': 5, 'b': 9}.toOrderedTable + + result = initOrderedTable[A, B](pairs.len) + for key, val in items(pairs): result[key] = val + +proc `[]`*[A, B](t: OrderedTable[A, B], key: A): lent B = + ## Retrieves the value at `t[key]`. + ## + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## One can check with `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ whether + ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,OrderedTable[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for checking if a + ## key is in the table + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] + + get(t, key) + +proc `[]`*[A, B](t: var OrderedTable[A, B], key: A): var B = + ## Retrieves the value at `t[key]`. The value can be modified. + ## + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,OrderedTable[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for checking if a + ## key is in the table + get(t, key) + +proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `contains proc<#contains,OrderedTable[A,B],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + + var hc: Hash = default(Hash) + result = rawGet(t, key, hc) >= 0 + +proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for use with + ## the `in` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = + ## Returns true if `key` is in the table, otherwise inserts `value`. + ## + ## See also: + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toOrderedTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.toOrderedTable + + hasKeyOrPutImpl(enlarge) + +proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTable[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + result = default(B) + getOrDefaultImpl(t, key) + +proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTable[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + result = default(B) + getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``value`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. + ## + ## See also: + ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTable[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.toOrderedTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.toOrderedTable + mgetOrPutImpl(enlarge) -proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = - ## returns true iff `key` is in the table, otherwise inserts `value`. - hasKeyOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable -proc initOrderedTable*[A, B](initialSize=64): OrderedTable[A, B] = - ## creates a new ordered hash table that is empty. - ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc from this module. - assert isPowerOfTwo(initialSize) - result.counter = 0 - result.first = -1 - result.last = -1 - newSeq(result.data, initialSize) - -proc toOrderedTable*[A, B](pairs: openArray[(A, - B)]): OrderedTable[A, B] = - ## creates a new ordered hash table that contains the given `pairs`. - result = initOrderedTable[A, B](rightSize(pairs.len)) - for key, val in items(pairs): result[key] = val + mgetOrPutImpl(enlarge) -proc `$`*[A, B](t: OrderedTable[A, B]): string = - ## The `$` operator for ordered hash tables. - dollarImpl() +proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = + ## Returns the number of keys in `t`. + runnableExamples: + let a = {'a': 5, 'b': 9}.toOrderedTable + doAssert len(a) == 2 + + result = t.counter -proc sort*[A, B](t: var OrderedTable[A, B], - cmp: proc (x,y: (A, B)): int) = - ## sorts `t` according to `cmp`. This modifies the internal list +proc add*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,OrderedTable[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. + addImpl(enlarge) + +proc del*[A, B](t: var OrderedTable[A, B], key: A) = + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. + ## + ## O(n) complexity. + ## + ## See also: + ## * `pop proc<#pop,OrderedTable[A,B],A,B>`_ + ## * `clear proc<#clear,OrderedTable[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toOrderedTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.toOrderedTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.toOrderedTable + + if t.counter == 0: return + var n: OrderedKeyValuePairSeq[A, B] + newSeq(n, len(t.data)) + var h = t.first + t.first = -1 + t.last = -1 + swap(t.data, n) + let hc = genHash(key) + while h >= 0: + var nxt = n[h].next + if isFilled(n[h].hcode): + if n[h].hcode == hc and n[h].key == key: + dec t.counter + else: + var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) + rawInsert(t, t.data, move n[h].key, move n[h].val, n[h].hcode, j) + h = nxt + +proc pop*[A, B](t: var OrderedTable[A, B], key: A, val: var B): bool {.since: (1, 1).} = + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is + ## unchanged. + ## + ## O(n) complexity. + ## + ## See also: + ## * `del proc<#del,OrderedTable[A,B],A>`_ + ## * `clear proc<#clear,OrderedTable[A,B]>`_ to empty the whole table + runnableExamples: + var + a = {'c': 5, 'b': 9, 'a': 13}.toOrderedTable + i: int + doAssert a.pop('b', i) == true + doAssert a == {'c': 5, 'a': 13}.toOrderedTable + doAssert i == 9 + i = 0 + doAssert a.pop('z', i) == false + doAssert a == {'c': 5, 'a': 13}.toOrderedTable + doAssert i == 0 + + var hc: Hash + var index = rawGet(t, key, hc) + result = index >= 0 + if result: + val = move(t.data[index].val) + del(t, key) + +proc clear*[A, B](t: var OrderedTable[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,OrderedTable[A,B],A>`_ + ## * `pop proc<#pop,OrderedTable[A,B],A,B>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.toOrderedTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 + + clearImpl() + t.first = -1 + t.last = -1 + +proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x, y: (A, B)): int, + order = SortOrder.Ascending) {.effectsOf: cmp.} = + ## Sorts `t` according to the function `cmp`. + ## + ## This modifies the internal list ## that kept the insertion order, so insertion order is lost after this ## call but key lookup and insertions remain possible after `sort` (in - ## contrast to the `sort` for count tables). + ## contrast to the `sort proc<#sort,CountTable[A]>`_ for count tables). + runnableExamples: + import std/[algorithm] + var a = initOrderedTable[char, int]() + for i, c in "cab": + a[c] = 10*i + doAssert a == {'c': 0, 'a': 10, 'b': 20}.toOrderedTable + a.sort(system.cmp) + doAssert a == {'a': 10, 'b': 20, 'c': 0}.toOrderedTable + a.sort(system.cmp, order = SortOrder.Descending) + doAssert a == {'c': 0, 'b': 20, 'a': 10}.toOrderedTable + var list = t.first var p, q, e, tail, oldhead: int @@ -544,7 +1683,7 @@ proc sort*[A, B](t: var OrderedTable[A, B], elif qsize == 0 or q < 0: e = p; p = t.data[p].next; dec(psize) elif cmp((t.data[p].key, t.data[p].val), - (t.data[q].key, t.data[q].val)) <= 0: + (t.data[q].key, t.data[q].val)) * order <= 0: e = p; p = t.data[p].next; dec(psize) else: e = q; q = t.data[q].next; dec(qsize) @@ -558,198 +1697,607 @@ proc sort*[A, B](t: var OrderedTable[A, B], t.first = list t.last = tail -proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = - ## returns the number of keys in `t`. - result = t.counter +proc `$`*[A, B](t: OrderedTable[A, B]): string = + ## The `$` operator for ordered hash tables. Used internally when calling + ## `echo` on a table. + dollarImpl() -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = - var h = t.first - while h >= 0: - var nxt = t.data[h].next - if isFilled(t.data[h].hcode): yieldStmt - h = nxt +proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = + ## The `==` operator for ordered hash tables. Returns `true` if both the + ## content and the order are equal. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.toOrderedTable + b = {'b': 9, 'c': 13, 'a': 5}.toOrderedTable + doAssert a != b + + if s.counter != t.counter: + return false + if s.counter == 0 and t.counter == 0: + return true + var ht = t.first + var hs = s.first + while ht >= 0 and hs >= 0: + var nxtt = t.data[ht].next + var nxts = s.data[hs].next + if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode): + if (s.data[hs].key != t.data[ht].key) or (s.data[hs].val != t.data[ht].val): + return false + ht = nxtt + hs = nxts + return true -iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = - ## iterates over any (key, value) pair in the table `t` in insertion + + +iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = + ## Iterates over any `(key, value)` pair in the table `t` in insertion ## order. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTable[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTable[A,B]>`_ + ## * `values iterator<#values.i,OrderedTable[A,B]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.toOrderedTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: o + ## # value: [1, 5, 7, 9] + ## # key: e + ## # value: [2, 4, 6, 8] + ## ``` + + let L = len(t) forAllOrderedPairs: yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = - ## iterates over any (key, value) pair in the table `t` in insertion - ## order. The values can be modified. +iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = + ## Iterates over any `(key, value)` pair in the table `t` (must be + ## declared as `var`) in insertion order. The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTable[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'o': @[1, 5, 7, 9, 11], + 'e': @[2, 4, 6, 8, 12]}.toOrderedTable + + let L = len(t) forAllOrderedPairs: yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTableRef[A, B]): A = - ## iterates over any key in the table `t` in insertion order. +iterator keys*[A, B](t: OrderedTable[A, B]): lent A = + ## Iterates over any key in the table `t` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ + ## * `values iterator<#values.i,OrderedTable[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for k in a.keys: + a[k].add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], + 'e': @[2, 4, 6, 8, 99]}.toOrderedTable + + let L = len(t) forAllOrderedPairs: yield t.data[h].key + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTableRef[A, B]): B = - ## iterates over any value in the table `t` in insertion order. +iterator values*[A, B](t: OrderedTable[A, B]): lent B = + ## Iterates over any value in the table `t` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTable[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTable[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for v in a.values: + doAssert v.len == 4 + + let L = len(t) forAllOrderedPairs: yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator mvalues*[A, B](t: OrderedTableRef[A, B]): var B = - ## iterates over any value in the table `t` in insertion order. The values +iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = + ## Iterates over any value in the table `t` (must be + ## declared as `var`) in insertion order. The values ## can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTable[A,B]>`_ + ## * `values iterator<#values.i,OrderedTable[A,B]>`_ + runnableExamples: + var a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.toOrderedTable + for v in a.mvalues: + v.add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], + 'e': @[2, 4, 6, 8, 99]}.toOrderedTable + + let L = len(t) forAllOrderedPairs: yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + +# --------------------------------------------------------------------------- +# --------------------------- OrderedTableRef ------------------------------- +# --------------------------------------------------------------------------- + +proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = + ## Creates a new ordered ref hash table that is empty. + ## + ## See also: + ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for creating + ## an `OrderedTableRef` from a collection of `(key, value)` pairs + ## * `initOrderedTable proc<#initOrderedTable>`_ for creating an + ## `OrderedTable` + runnableExamples: + let + a = newOrderedTable[int, string]() + b = newOrderedTable[char, seq[int]]() + new(result) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) + +proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = + ## Creates a new ordered ref hash table that contains the given `pairs`. + ## + ## `pairs` is a container consisting of `(key, value)` tuples. + ## + ## See also: + ## * `newOrderedTable proc<#newOrderedTable>`_ + ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_ for an + ## `OrderedTable` version + runnableExamples: + let a = [('a', 5), ('b', 9)] + let b = newOrderedTable(a) + assert b == {'a': 5, 'b': 9}.newOrderedTable + + result = newOrderedTable[A, B](pairs.len) + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val + proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = - ## retrieves the value at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether + ## Retrieves the value at `t[key]`. + ## + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## One can check with `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ whether ## the key exists. + ## + ## See also: + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]=,OrderedTableRef[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ for checking if + ## a key is in the table + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a['a'] == 5 + doAssertRaises(KeyError): + echo a['z'] result = t[][key] -proc mget*[A, B](t: OrderedTableRef[A, B], key: A): var B {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. - result = t[][key] +proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `del proc<#del,OrderedTableRef[A,B],A>`_ for removing a key from the table + runnableExamples: + var a = newOrderedTable[char, int]() + a['x'] = 7 + a['y'] = 33 + doAssert a == {'x': 7, 'y': 33}.newOrderedTable + + t[][key] = val + +proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `contains proc<#contains,OrderedTableRef[A,B],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.hasKey('a') == true + doAssert a.hasKey('z') == false + + result = t[].hasKey(key) + +proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = + ## Alias of `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ for use with + ## the `in` operator. + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert 'b' in a == true + doAssert a.contains('z') == false + + return hasKey[A, B](t, key) + +proc hasKeyOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): bool = + ## Returns true if `key` is in the table, otherwise inserts `value`. + ## + ## See also: + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newOrderedTable + if a.hasKeyOrPut('a', 50): + a['a'] = 99 + if a.hasKeyOrPut('z', 50): + a['z'] = 99 + doAssert a == {'a': 99, 'b': 9, 'z': 50}.newOrderedTable + + result = t[].hasKeyOrPut(key, val) proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any + ## integer type). + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.getOrDefault('a') == 5 + doAssert a.getOrDefault('z') == 0 + getOrDefault(t[], key) +proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `mgetOrPut proc<#mgetOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.getOrDefault('a', 99) == 5 + doAssert a.getOrDefault('z', 99) == 99 + + getOrDefault(t[], key, default) + proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. + ## + ## See also: + ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ + ## * `hasKeyOrPut proc<#hasKeyOrPut,OrderedTableRef[A,B],A,B>`_ + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A>`_ to return + ## a default value (e.g. zero for int) if the key doesn't exist + ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return + ## a custom value if the key doesn't exist + runnableExamples: + var a = {'a': 5, 'b': 9}.newOrderedTable + doAssert a.mgetOrPut('a', 99) == 5 + doAssert a.mgetOrPut('z', 99) == 99 + doAssert a == {'a': 5, 'b': 9, 'z': 99}.newOrderedTable + result = t[].mgetOrPut(key, val) -proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = - ## returns true iff `key` is in the table, otherwise inserts `val`. - result = t[].hasKeyOrPut(key, val) +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable -proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = - ## returns true iff `key` is in the table `t`. - result = t[].hasKey(key) + t[].mgetOrPut(key) -proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = - ## alias of `hasKey` for use with the `in` operator. - return hasKey[A, B](t, key) +proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = + ## Returns the number of keys in `t`. + runnableExamples: + let a = {'a': 5, 'b': 9}.newOrderedTable + doAssert len(a) == 2 -proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. - t[][key] = val + result = t.counter -proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. +proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. + ## + ## **This can introduce duplicate keys into the table!** + ## + ## Use `[]= proc<#[]=,OrderedTableRef[A,B],A,sinkB>`_ for inserting a new + ## (key, value) pair in the table without introducing duplicates. t[].add(key, val) -proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] = - ## creates a new ordered hash table that is empty. +proc del*[A, B](t: OrderedTableRef[A, B], key: A) = + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc from this module. - new(result) - result[] = initOrderedTable[A, B]() + ## See also: + ## * `clear proc<#clear,OrderedTableRef[A,B]>`_ to empty the whole table + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable + a.del('a') + doAssert a == {'b': 9, 'c': 13}.newOrderedTable + a.del('z') + doAssert a == {'b': 9, 'c': 13}.newOrderedTable -proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = - ## creates a new ordered hash table that contains the given `pairs`. - result = newOrderedTable[A, B](rightSize(pairs.len)) - for key, val in items(pairs): result[key] = val + t[].del(key) + +proc pop*[A, B](t: OrderedTableRef[A, B], key: A, val: var B): bool {.since: (1, 1).} = + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is + ## unchanged. + ## + ## See also: + ## * `del proc<#del,OrderedTableRef[A,B],A>`_ + ## * `clear proc<#clear,OrderedTableRef[A,B]>`_ to empty the whole table + runnableExamples: + var + a = {'c': 5, 'b': 9, 'a': 13}.newOrderedTable + i: int + doAssert a.pop('b', i) == true + doAssert a == {'c': 5, 'a': 13}.newOrderedTable + doAssert i == 9 + i = 0 + doAssert a.pop('z', i) == false + doAssert a == {'c': 5, 'a': 13}.newOrderedTable + doAssert i == 0 + + pop(t[], key, val) + +proc clear*[A, B](t: OrderedTableRef[A, B]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,OrderedTableRef[A,B],A>`_ + runnableExamples: + var a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable + doAssert len(a) == 3 + clear(a) + doAssert len(a) == 0 + + clear(t[]) + +proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x, y: (A, B)): int, + order = SortOrder.Ascending) {.effectsOf: cmp.} = + ## Sorts `t` according to the function `cmp`. + ## + ## This modifies the internal list + ## that kept the insertion order, so insertion order is lost after this + ## call but key lookup and insertions remain possible after `sort` (in + ## contrast to the `sort proc<#sort,CountTableRef[A]>`_ for count tables). + runnableExamples: + import std/[algorithm] + var a = newOrderedTable[char, int]() + for i, c in "cab": + a[c] = 10*i + doAssert a == {'c': 0, 'a': 10, 'b': 20}.newOrderedTable + a.sort(system.cmp) + doAssert a == {'a': 10, 'b': 20, 'c': 0}.newOrderedTable + a.sort(system.cmp, order = SortOrder.Descending) + doAssert a == {'c': 0, 'b': 20, 'a': 10}.newOrderedTable + + t[].sort(cmp, order = order) proc `$`*[A, B](t: OrderedTableRef[A, B]): string = - ## The `$` operator for ordered hash tables. + ## The `$` operator for hash tables. Used internally when calling `echo` + ## on a table. dollarImpl() -proc sort*[A, B](t: OrderedTableRef[A, B], - cmp: proc (x,y: (A, B)): int) = - ## sorts `t` according to `cmp`. This modifies the internal list - ## that kept the insertion order, so insertion order is lost after this - ## call but key lookup and insertions remain possible after `sort` (in - ## contrast to the `sort` for count tables). - t[].sort(cmp) +proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = + ## The `==` operator for ordered hash tables. Returns true if either both + ## tables are `nil`, or neither is `nil` and the content and the order of + ## both are equal. + runnableExamples: + let + a = {'a': 5, 'b': 9, 'c': 13}.newOrderedTable + b = {'b': 9, 'c': 13, 'a': 5}.newOrderedTable + doAssert a != b -# ------------------------------ count tables ------------------------------- + if isNil(s): result = isNil(t) + elif isNil(t): result = false + else: result = s[] == t[] -type - CountTable* [ - A] = object ## table that counts the number of each key - data: seq[tuple[key: A, val: int]] - counter: int - CountTableRef*[A] = ref CountTable[A] -{.deprecated: [TCountTable: CountTable, PCountTable: CountTableRef].} -proc len*[A](t: CountTable[A]): int = - ## returns the number of keys in `t`. - result = t.counter +iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = + ## Iterates over any `(key, value)` pair in the table `t` in insertion + ## order. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTableRef[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTableRef[A,B]>`_ + ## * `values iterator<#values.i,OrderedTableRef[A,B]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## let a = { + ## 'o': [1, 5, 7, 9], + ## 'e': [2, 4, 6, 8] + ## }.newOrderedTable + ## + ## for k, v in a.pairs: + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: o + ## # value: [1, 5, 7, 9] + ## # key: e + ## # value: [2, 4, 6, 8] + ## ``` -iterator pairs*[A](t: CountTable[A]): (A, int) = - ## iterates over any (key, value) pair in the table `t`. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + let L = len(t) + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator mpairs*[A](t: var CountTable[A]): (A, var int) = - ## iterates over any (key, value) pair in the table `t`. The values can - ## be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) +iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = + ## Iterates over any `(key, value)` pair in the table `t` in insertion + ## order. The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for k, v in a.mpairs: + v.add(v[0] + 10) + doAssert a == {'o': @[1, 5, 7, 9, 11], + 'e': @[2, 4, 6, 8, 12]}.newOrderedTable + + let L = len(t) + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A](t: CountTable[A]): A = - ## iterates over any key in the table `t`. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].key +iterator keys*[A, B](t: OrderedTableRef[A, B]): lent A = + ## Iterates over any key in the table `t` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ + ## * `values iterator<#values.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for k in a.keys: + a[k].add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], 'e': @[2, 4, 6, 8, + 99]}.newOrderedTable + + let L = len(t) + forAllOrderedPairs: + yield t.data[h].key + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator values*[A, B](t: OrderedTableRef[A, B]): lent B = + ## Iterates over any value in the table `t` in insertion order. + ## + ## See also: + ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ + ## * `keys iterator<#keys.i,OrderedTableRef[A,B]>`_ + ## * `mvalues iterator<#mvalues.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for v in a.values: + doAssert v.len == 4 + + let L = len(t) + forAllOrderedPairs: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator mvalues*[A, B](t: OrderedTableRef[A, B]): var B = + ## Iterates over any value in the table `t` in insertion order. The values + ## can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,OrderedTableRef[A,B]>`_ + ## * `values iterator<#values.i,OrderedTableRef[A,B]>`_ + runnableExamples: + let a = { + 'o': @[1, 5, 7, 9], + 'e': @[2, 4, 6, 8] + }.newOrderedTable + for v in a.mvalues: + v.add(99) + doAssert a == {'o': @[1, 5, 7, 9, 99], + 'e': @[2, 4, 6, 8, 99]}.newOrderedTable + + let L = len(t) + forAllOrderedPairs: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A](t: CountTable[A]): int = - ## iterates over any value in the table `t`. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val -iterator mvalues*[A](t: CountTable[A]): var int = - ## iterates over any value in the table `t`. The values can be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val -proc rawGet[A](t: CountTable[A], key: A): int = - var h: Hash = hash(key) and high(t.data) # start with real hash value - while t.data[h].val != 0: - if t.data[h].key == key: return h - h = nextTry(h, high(t.data)) - result = -1 - h # < 0 => MISSING; insert idx = -1 - result -template ctget(t, key: untyped): untyped {.immediate.} = - var index = rawGet(t, key) - if index >= 0: result = t.data[index].val - else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") - -proc `[]`*[A](t: CountTable[A], key: A): int {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. If `key` is not in `t`, - ## the ``KeyError`` exception is raised. One can check with ``hasKey`` - ## whether the key exists. - ctget(t, key) - -proc `[]`*[A](t: var CountTable[A], key: A): var int {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ctget(t, key) - -proc mget*[A](t: var CountTable[A], key: A): var int {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. - ctget(t, key) - -proc getOrDefault*[A](t: CountTable[A], key: A): int = - var index = rawGet(t, key) - if index >= 0: result = t.data[index].val -proc hasKey*[A](t: CountTable[A], key: A): bool = - ## returns true iff `key` is in the table `t`. - result = rawGet(t, key) >= 0 -proc contains*[A](t: CountTable[A], key: A): bool = - ## alias of `hasKey` for use with the `in` operator. - return hasKey[A](t, key) -proc rawInsert[A](t: CountTable[A], data: var seq[tuple[key: A, val: int]], +# ------------------------------------------------------------------------- +# ------------------------------ CountTable ------------------------------- +# ------------------------------------------------------------------------- + +type + CountTable*[A] = object + ## Hash table that counts the number of each key. + ## + ## For creating an empty CountTable, use `initCountTable proc + ## <#initCountTable>`_. + data: seq[tuple[key: A, val: int]] + counter: int + isSorted: bool + CountTableRef*[A] = ref CountTable[A] ## Ref version of + ## `CountTable<#CountTable>`_. + ## + ## For creating a new empty CountTableRef, use `newCountTable proc + ## <#newCountTable>`_. + + +# ------------------------------ helpers --------------------------------- + +proc ctRawInsert[A](t: CountTable[A], data: var seq[tuple[key: A, val: int]], key: A, val: int) = var h: Hash = hash(key) and high(data) while data[h].val != 0: h = nextTry(h, high(data)) @@ -760,271 +2308,665 @@ proc enlarge[A](t: var CountTable[A]) = var n: seq[tuple[key: A, val: int]] newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): - if t.data[i].val != 0: rawInsert(t, n, t.data[i].key, t.data[i].val) + if t.data[i].val != 0: ctRawInsert(t, n, move t.data[i].key, move t.data[i].val) swap(t.data, n) -proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = - ## puts a (key, value)-pair into `t`. - assert val > 0 - var h = rawGet(t, key) - if h >= 0: - t.data[h].val = val - else: - if mustRehash(len(t.data), t.counter): enlarge(t) - rawInsert(t, t.data, key, val) - inc(t.counter) - #h = -1 - h - #t.data[h].key = key - #t.data[h].val = val - -proc initCountTable*[A](initialSize=64): CountTable[A] = - ## creates a new count table that is empty. - ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc in this module. - assert isPowerOfTwo(initialSize) - result.counter = 0 - newSeq(result.data, initialSize) +proc rawGet[A](t: CountTable[A], key: A): int = + if t.data.len == 0: + return -1 + var h: Hash = hash(key) and high(t.data) # start with real hash value + while t.data[h].val != 0: + if t.data[h].key == key: return h + h = nextTry(h, high(t.data)) + result = -1 - h # < 0 => MISSING; insert idx = -1 - result + +template ctget(t, key, default: untyped): untyped = + var index = rawGet(t, key) + result = if index >= 0: t.data[index].val else: default + +proc inc*[A](t: var CountTable[A], key: A, val = 1) + +# ---------------------------------------------------------------------- + +proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = + ## Creates a new count table that is empty. + ## + ## Starting from Nim v0.20, tables are initialized by default and it is + ## not necessary to call this function explicitly. + ## + ## See also: + ## * `toCountTable proc<#toCountTable,openArray[A]>`_ + ## * `newCountTable proc<#newCountTable>`_ for creating a + ## `CountTableRef` + result = default(CountTable[A]) + initImpl(result, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = - ## creates a new count table with every key in `keys` having a count of 1. - result = initCountTable[A](rightSize(keys.len)) - for key in items(keys): result[key] = 1 + ## Creates a new count table with every member of a container `keys` + ## having a count of how many times it occurs in that container. + result = initCountTable[A](keys.len) + for key in items(keys): result.inc(key) + +proc `[]`*[A](t: CountTable[A], key: A): int = + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise `0` is returned. + ## + ## See also: + ## * `getOrDefault<#getOrDefault,CountTable[A],A,int>`_ to return + ## a custom value if the key doesn't exist + ## * `[]= proc<#[]%3D,CountTable[A],A,int>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,CountTable[A],A>`_ for checking if a key + ## is in the table + assert(not t.isSorted, "CountTable must not be used after sorting") + ctget(t, key, 0) + +template cntMakeEmpty(i) = t.data[i].val = 0 +template cntCellEmpty(i) = t.data[i].val == 0 +template cntCellHash(i) = hash(t.data[i].key) -proc `$`*[A](t: CountTable[A]): string = - ## The `$` operator for count tables. - dollarImpl() +proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key + ## * `inc proc<#inc,CountTable[A],A,int>`_ for incrementing a + ## value of a key + assert(not t.isSorted, "CountTable must not be used after sorting") + assert val >= 0 + if val == 0: + delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash) + else: + let h = rawGet(t, key) + if h >= 0: + t.data[h].val = val + else: + insertImpl() proc inc*[A](t: var CountTable[A], key: A, val = 1) = - ## increments `t[key]` by `val`. + ## Increments `t[key]` by `val` (default: 1). + runnableExamples: + var a = toCountTable("aab") + a.inc('a') + a.inc('b', 10) + doAssert a == toCountTable("aaabbbbbbbbbbb") + + assert(not t.isSorted, "CountTable must not be used after sorting") var index = rawGet(t, key) if index >= 0: inc(t.data[index].val, val) + if t.data[index].val == 0: + delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash) else: - if mustRehash(len(t.data), t.counter): enlarge(t) - rawInsert(t, t.data, key, val) - inc(t.counter) + if val != 0: + insertImpl() + +proc len*[A](t: CountTable[A]): int = + ## Returns the number of keys in `t`. + result = t.counter proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## returns the largest (key,val)-pair. Efficiency: O(n) - assert t.len > 0 - var minIdx = 0 - for h in 1..high(t.data): - if t.data[h].val > 0 and t.data[minIdx].val > t.data[h].val: minIdx = h + ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) + ## + ## See also: + ## * `largest proc<#largest,CountTable[A]>`_ + assert t.len > 0, "counttable is empty" + var minIdx = -1 + for h in 0 .. high(t.data): + if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val): + minIdx = h result.key = t.data[minIdx].key result.val = t.data[minIdx].val proc largest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## returns the (key,val)-pair with the largest `val`. Efficiency: O(n) - assert t.len > 0 + ## Returns the `(key, value)` pair with the largest `val`. Efficiency: O(n) + ## + ## See also: + ## * `smallest proc<#smallest,CountTable[A]>`_ + assert t.len > 0, "counttable is empty" var maxIdx = 0 - for h in 1..high(t.data): + for h in 1 .. high(t.data): if t.data[maxIdx].val < t.data[h].val: maxIdx = h result.key = t.data[maxIdx].key result.val = t.data[maxIdx].val -proc sort*[A](t: var CountTable[A]) = - ## sorts the count table so that the entry with the highest counter comes - ## first. This is destructive! You must not modify `t` afterwards! - ## You can use the iterators `pairs`, `keys`, and `values` to iterate over - ## `t` in the sorted order. +proc hasKey*[A](t: CountTable[A], key: A): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `contains proc<#contains,CountTable[A],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,CountTable[A],A,int>`_ to return + ## a custom value if the key doesn't exist + assert(not t.isSorted, "CountTable must not be used after sorting") + result = rawGet(t, key) >= 0 - # we use shellsort here; fast enough and simple - var h = 1 - while true: - h = 3 * h + 1 - if h >= high(t.data): break - while true: - h = h div 3 - for i in countup(h, high(t.data)): - var j = i - while t.data[j-h].val <= t.data[j].val: - swap(t.data[j], t.data[j-h]) - j = j-h - if j < h: break - if h == 1: break +proc contains*[A](t: CountTable[A], key: A): bool = + ## Alias of `hasKey proc<#hasKey,CountTable[A],A>`_ for use with + ## the `in` operator. + return hasKey[A](t, key) -proc len*[A](t: CountTableRef[A]): int = - ## returns the number of keys in `t`. - result = t.counter +proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. + ## + ## See also: + ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,CountTable[A],A>`_ for checking if a key + ## is in the table + ctget(t, key, default) + +proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} = + ## Deletes `key` from table `t`. Does nothing if the key does not exist. + ## + ## See also: + ## * `pop proc<#pop,CountTable[A],A,int>`_ + ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table + runnableExamples: + var a = toCountTable("aabbbccccc") + a.del('b') + assert a == toCountTable("aaccccc") + a.del('b') + assert a == toCountTable("aaccccc") + a.del('c') + assert a == toCountTable("aa") + + delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash) + +proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).} = + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is + ## unchanged. + ## + ## See also: + ## * `del proc<#del,CountTable[A],A>`_ + ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table + runnableExamples: + var a = toCountTable("aabbbccccc") + var i = 0 + assert a.pop('b', i) + assert i == 3 + i = 99 + assert not a.pop('b', i) + assert i == 99 -iterator pairs*[A](t: CountTableRef[A]): (A, int) = - ## iterates over any (key, value) pair in the table `t`. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) + var index = rawGet(t, key) + result = index >= 0 + if result: + val = move(t.data[index].val) + delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash) -iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = - ## iterates over any (key, value) pair in the table `t`. The values can - ## be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield (t.data[h].key, t.data[h].val) +proc clear*[A](t: var CountTable[A]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,CountTable[A],A>`_ + ## * `pop proc<#pop,CountTable[A],A,int>`_ + clearImpl() + t.isSorted = false + +func ctCmp[T](a, b: tuple[key: T, val: int]): int = + result = system.cmp(a.val, b.val) + +proc sort*[A](t: var CountTable[A], order = SortOrder.Descending) = + ## Sorts the count table so that, by default, the entry with the + ## highest counter comes first. + ## + ## .. warning:: This is destructive! Once sorted, you must not modify `t` afterwards! + ## + ## You can use the iterators `pairs<#pairs.i,CountTable[A]>`_, + ## `keys<#keys.i,CountTable[A]>`_, and `values<#values.i,CountTable[A]>`_ + ## to iterate over `t` in the sorted order. + runnableExamples: + import std/[algorithm, sequtils] + var a = toCountTable("abracadabra") + doAssert a == "aaaaabbrrcd".toCountTable + a.sort() + doAssert toSeq(a.values) == @[5, 2, 2, 1, 1] + a.sort(SortOrder.Ascending) + doAssert toSeq(a.values) == @[1, 1, 2, 2, 5] + + t.data.sort(cmp = ctCmp, order = order) + t.isSorted = true -iterator keys*[A](t: CountTableRef[A]): A = - ## iterates over any key in the table `t`. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].key +proc merge*[A](s: var CountTable[A], t: CountTable[A]) = + ## Merges the second table into the first one (must be declared as `var`). + runnableExamples: + var a = toCountTable("aaabbc") + let b = toCountTable("bcc") + a.merge(b) + doAssert a == toCountTable("aaabbbccc") + + assert(not s.isSorted, "CountTable must not be used after sorting") + for key, value in t: + s.inc(key, value) -iterator values*[A](t: CountTableRef[A]): int = - ## iterates over any value in the table `t`. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val +when (NimMajor, NimMinor) <= (1, 0): + proc merge*[A](s, t: CountTable[A]): CountTable[A] = + ## Merges the two tables into a new one. + runnableExamples: + let + a = toCountTable("aaabbc") + b = toCountTable("bcc") + doAssert merge(a, b) == toCountTable("aaabbbccc") -iterator mvalues*[A](t: CountTableRef[A]): var int = - ## iterates over any value in the table `t`. The values can be modified. - for h in 0..high(t.data): - if t.data[h].val != 0: yield t.data[h].val + result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len))) + for table in @[s, t]: + for key, value in table: + result.inc(key, value) -proc `[]`*[A](t: CountTableRef[A], key: A): var int {.deprecatedGet.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - result = t[][key] +proc `$`*[A](t: CountTable[A]): string = + ## The `$` operator for count tables. Used internally when calling `echo` + ## on a table. + dollarImpl() -proc mget*[A](t: CountTableRef[A], key: A): var int {.deprecated.} = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. - ## Use ```[]``` instead. - result = t[][key] +proc `==`*[A](s, t: CountTable[A]): bool = + ## The `==` operator for count tables. Returns `true` if both tables + ## contain the same keys with the same count. Insert order does not matter. + equalsImpl(s, t) -proc getOrDefault*[A](t: CountTableRef[A], key: A): int = - getOrDefaultImpl(t, key) -proc hasKey*[A](t: CountTableRef[A], key: A): bool = - ## returns true iff `key` is in the table `t`. - result = t[].hasKey(key) +iterator pairs*[A](t: CountTable[A]): (A, int) = + ## Iterates over any `(key, value)` pair in the table `t`. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTable[A]>`_ + ## * `keys iterator<#keys.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## let a = toCountTable("abracadabra") + ## + ## for k, v in pairs(a): + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: a + ## # value: 5 + ## # key: b + ## # value: 2 + ## # key: c + ## # value: 1 + ## # key: d + ## # value: 1 + ## # key: r + ## # value: 2 + ## ``` + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") -proc contains*[A](t: CountTableRef[A], key: A): bool = - ## alias of `hasKey` for use with the `in` operator. - return hasKey[A](t, key) +iterator mpairs*[A](t: var CountTable[A]): (A, var int) = + ## Iterates over any `(key, value)` pair in the table `t` (must be + ## declared as `var`). The values can be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTable[A]>`_ + runnableExamples: + var a = toCountTable("abracadabra") + for k, v in mpairs(a): + v = 2 + doAssert a == toCountTable("aabbccddrr") + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator keys*[A](t: CountTable[A]): lent A = + ## Iterates over any key in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + runnableExamples: + var a = toCountTable("abracadabra") + for k in keys(a): + a[k] = 2 + doAssert a == toCountTable("aabbccddrr") + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield t.data[h].key + assert(len(t) == L, "the length of the table changed while iterating over it") -proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = - ## puts a (key, value)-pair into `t`. `val` has to be positive. - assert val > 0 - t[][key] = val +iterator values*[A](t: CountTable[A]): int = + ## Iterates over any value in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `keys iterator<#keys.i,CountTable[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTable[A]>`_ + runnableExamples: + let a = toCountTable("abracadabra") + for v in values(a): + assert v < 10 + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") -proc newCountTable*[A](initialSize=64): CountTableRef[A] = - ## creates a new count table that is empty. +iterator mvalues*[A](t: var CountTable[A]): var int = + ## Iterates over any value in the table `t` (must be + ## declared as `var`). The values can be modified. ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` method in this module. + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + runnableExamples: + var a = toCountTable("abracadabra") + for v in mvalues(a): + v = 2 + doAssert a == toCountTable("aabbccddrr") + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + + + + + + + +# --------------------------------------------------------------------------- +# ---------------------------- CountTableRef -------------------------------- +# --------------------------------------------------------------------------- + +proc inc*[A](t: CountTableRef[A], key: A, val = 1) + +proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = + ## Creates a new ref count table that is empty. + ## + ## See also: + ## * `newCountTable proc<#newCountTable,openArray[A]>`_ for creating + ## a `CountTableRef` from a collection + ## * `initCountTable proc<#initCountTable>`_ for creating a + ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = - ## creates a new count table with every key in `keys` having a count of 1. - result = newCountTable[A](rightSize(keys.len)) - for key in items(keys): result[key] = 1 + ## Creates a new ref count table with every member of a container `keys` + ## having a count of how many times it occurs in that container. + result = newCountTable[A](keys.len) + {.noSideEffect.}: + for key in items(keys): result.inc(key) + +proc `[]`*[A](t: CountTableRef[A], key: A): int = + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise `0` is returned. + ## + ## See also: + ## * `getOrDefault<#getOrDefault,CountTableRef[A],A,int>`_ to return + ## a custom value if the key doesn't exist + ## * `inc proc<#inc,CountTableRef[A],A,int>`_ to inc even if missing + ## * `[]= proc<#[]%3D,CountTableRef[A],A,int>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key + ## is in the table + result = t[][key] -proc `$`*[A](t: CountTableRef[A]): string = - ## The `$` operator for count tables. - dollarImpl() +proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key + ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a + ## value of a key + assert val > 0 + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = - ## increments `t[key]` by `val`. - t[].inc(key, val) - -proc smallest*[A](t: CountTableRef[A]): (A, int) = - ## returns the largest (key,val)-pair. Efficiency: O(n) + ## Increments `t[key]` by `val` (default: 1). + runnableExamples: + var a = newCountTable("aab") + a.inc('a') + a.inc('b', 10) + doAssert a == newCountTable("aaabbbbbbbbbbb") + {.noSideEffect.}: + t[].inc(key, val) + +proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = + ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) + ## + ## See also: + ## * `largest proc<#largest,CountTableRef[A]>`_ t[].smallest -proc largest*[A](t: CountTableRef[A]): (A, int) = - ## returns the (key,val)-pair with the largest `val`. Efficiency: O(n) +proc largest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = + ## Returns the `(key, value)` pair with the largest `val`. Efficiency: O(n) + ## + ## See also: + ## * `smallest proc<#smallest,CountTable[A]>`_ t[].largest -proc sort*[A](t: CountTableRef[A]) = - ## sorts the count table so that the entry with the highest counter comes - ## first. This is destructive! You must not modify `t` afterwards! - ## You can use the iterators `pairs`, `keys`, and `values` to iterate over - ## `t` in the sorted order. - t[].sort +proc hasKey*[A](t: CountTableRef[A], key: A): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `contains proc<#contains,CountTableRef[A],A>`_ for use with the `in` + ## operator + ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key + ## * `getOrDefault proc<#getOrDefault,CountTableRef[A],A,int>`_ to return + ## a custom value if the key doesn't exist + result = t[].hasKey(key) -proc merge*[A](s: var CountTable[A], t: CountTable[A]) = - ## merges the second table into the first one - for key, value in t: - s.inc(key, value) +proc contains*[A](t: CountTableRef[A], key: A): bool = + ## Alias of `hasKey proc<#hasKey,CountTableRef[A],A>`_ for use with + ## the `in` operator. + return hasKey[A](t, key) -proc merge*[A](s, t: CountTable[A]): CountTable[A] = - ## merges the two tables into a new one - result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len))) - for table in @[s, t]: - for key, value in table: - result.inc(key, value) +proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. + ## + ## See also: + ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key + ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key + ## is in the table + result = t[].getOrDefault(key, default) + +proc len*[A](t: CountTableRef[A]): int = + ## Returns the number of keys in `t`. + result = t.counter + +proc del*[A](t: CountTableRef[A], key: A) {.since: (1, 1).} = + ## Deletes `key` from table `t`. Does nothing if the key does not exist. + ## + ## See also: + ## * `pop proc<#pop,CountTableRef[A],A,int>`_ + ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table + del(t[], key) + +proc pop*[A](t: CountTableRef[A], key: A, val: var int): bool {.since: (1, 1).} = + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is + ## unchanged. + ## + ## See also: + ## * `del proc<#del,CountTableRef[A],A>`_ + ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table + pop(t[], key, val) + +proc clear*[A](t: CountTableRef[A]) = + ## Resets the table so that it is empty. + ## + ## See also: + ## * `del proc<#del,CountTableRef[A],A>`_ + ## * `pop proc<#pop,CountTableRef[A],A,int>`_ + clear(t[]) + +proc sort*[A](t: CountTableRef[A], order = SortOrder.Descending) = + ## Sorts the count table so that, by default, the entry with the + ## highest counter comes first. + ## + ## **This is destructive! You must not modify `t` afterwards!** + ## + ## You can use the iterators `pairs<#pairs.i,CountTableRef[A]>`_, + ## `keys<#keys.i,CountTableRef[A]>`_, and `values<#values.i,CountTableRef[A]>`_ + ## to iterate over `t` in the sorted order. + t[].sort(order = order) proc merge*[A](s, t: CountTableRef[A]) = - ## merges the second table into the first one + ## Merges the second table into the first one. + runnableExamples: + let + a = newCountTable("aaabbc") + b = newCountTable("bcc") + a.merge(b) + doAssert a == newCountTable("aaabbbccc") + s[].merge(t[]) -when isMainModule: - type - Person = object - firstName, lastName: string +proc `$`*[A](t: CountTableRef[A]): string = + ## The `$` operator for count tables. Used internally when calling `echo` + ## on a table. + dollarImpl() - proc hash(x: Person): Hash = - ## Piggyback on the already available string hash proc. - ## - ## Without this proc nothing works! - result = x.firstName.hash !& x.lastName.hash - result = !$result +proc `==`*[A](s, t: CountTableRef[A]): bool = + ## The `==` operator for count tables. Returns `true` if either both tables + ## are `nil`, or neither is `nil` and both contain the same keys with the same + ## count. Insert order does not matter. + if isNil(s): result = isNil(t) + elif isNil(t): result = false + else: result = s[] == t[] - var - salaries = initTable[Person, int]() - p1, p2: Person - p1.firstName = "Jon" - p1.lastName = "Ross" - salaries[p1] = 30_000 - p2.firstName = "소진" - p2.lastName = "박" - salaries[p2] = 45_000 - var - s2 = initOrderedTable[Person, int]() - s3 = initCountTable[Person]() - s2[p1] = 30_000 - s2[p2] = 45_000 - s3[p1] = 30_000 - s3[p2] = 45_000 - var - t1 = initCountTable[string]() - t2 = initCountTable[string]() - t1.inc("foo") - t1.inc("bar", 2) - t1.inc("baz", 3) - t2.inc("foo", 4) - t2.inc("bar") - t2.inc("baz", 11) - merge(t1, t2) - assert(t1["foo"] == 5) - assert(t1["bar"] == 3) - assert(t1["baz"] == 14) +iterator pairs*[A](t: CountTableRef[A]): (A, int) = + ## Iterates over any `(key, value)` pair in the table `t`. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTableRef[A]>`_ + ## * `keys iterator<#keys.i,CountTableRef[A]>`_ + ## * `values iterator<#values.i,CountTableRef[A]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## let a = newCountTable("abracadabra") + ## + ## for k, v in pairs(a): + ## echo "key: ", k + ## echo "value: ", v + ## + ## # key: a + ## # value: 5 + ## # key: b + ## # value: 2 + ## # key: c + ## # value: 1 + ## # key: d + ## # value: 1 + ## # key: r + ## # value: 2 + ## ``` + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "the length of the table changed while iterating over it") - let - t1r = newCountTable[string]() - t2r = newCountTable[string]() - t1r.inc("foo") - t1r.inc("bar", 2) - t1r.inc("baz", 3) - t2r.inc("foo", 4) - t2r.inc("bar") - t2r.inc("baz", 11) - merge(t1r, t2r) - assert(t1r["foo"] == 5) - assert(t1r["bar"] == 3) - assert(t1r["baz"] == 14) +iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = + ## Iterates over any `(key, value)` pair in the table `t`. The values can + ## be modified. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTableRef[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTableRef[A]>`_ + runnableExamples: + let a = newCountTable("abracadabra") + for k, v in mpairs(a): + v = 2 + doAssert a == newCountTable("aabbccddrr") + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield (t.data[h].key, t.data[h].val) + assert(len(t) == L, "table modified while iterating over it") - var - t1l = initCountTable[string]() - t2l = initCountTable[string]() - t1l.inc("foo") - t1l.inc("bar", 2) - t1l.inc("baz", 3) - t2l.inc("foo", 4) - t2l.inc("bar") - t2l.inc("baz", 11) - let - t1merging = t1l - t2merging = t2l - let merged = merge(t1merging, t2merging) - assert(merged["foo"] == 5) - assert(merged["bar"] == 3) - assert(merged["baz"] == 14) +iterator keys*[A](t: CountTableRef[A]): A = + ## Iterates over any key in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTable[A]>`_ + ## * `values iterator<#values.i,CountTable[A]>`_ + runnableExamples: + let a = newCountTable("abracadabra") + for k in keys(a): + a[k] = 2 + doAssert a == newCountTable("aabbccddrr") + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield t.data[h].key + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator values*[A](t: CountTableRef[A]): int = + ## Iterates over any value in the table `t`. + ## + ## See also: + ## * `pairs iterator<#pairs.i,CountTableRef[A]>`_ + ## * `keys iterator<#keys.i,CountTableRef[A]>`_ + ## * `mvalues iterator<#mvalues.i,CountTableRef[A]>`_ + runnableExamples: + let a = newCountTable("abracadabra") + for v in values(a): + assert v < 10 + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + +iterator mvalues*[A](t: CountTableRef[A]): var int = + ## Iterates over any value in the table `t`. The values can be modified. + ## + ## See also: + ## * `mpairs iterator<#mpairs.i,CountTableRef[A]>`_ + ## * `values iterator<#values.i,CountTableRef[A]>`_ + runnableExamples: + var a = newCountTable("abracadabra") + for v in mvalues(a): + v = 2 + doAssert a == newCountTable("aabbccddrr") + + let L = len(t) + for h in 0 .. high(t.data): + if t.data[h].val != 0: + yield t.data[h].val + assert(len(t) == L, "the length of the table changed while iterating over it") + +proc hash*[K,V](s: Table[K,V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result + +proc hash*[K,V](s: OrderedTable[K,V]): Hash = + for p in pairs(s): + result = result !& hash(p) + result = !$result + +proc hash*[V](s: CountTable[V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 7328f7c24..d3e6dc063 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -6,31 +6,39 @@ # distribution, for details about the copyright. # -## This module implements color handling for Nimrod. It is used by -## the ``graphics`` module. +## This module implements color handling for Nim, +## namely color mixing and parsing the CSS color names. -import strutils +import std/strutils +from std/algorithm import binarySearch type - Color* = distinct int ## a color stored as RGB + Color* = distinct int ## A color stored as RGB, e.g. `0xff00cc`. -{.deprecated: [TColor: Color].} +proc `==`*(a, b: Color): bool {.borrow.} + ## Compares two colors. + ## + ## ```Nim + ## var + ## a = Color(0xff_00_ff) + ## b = colFuchsia + ## c = Color(0x00_ff_cc) + ## assert a == b + ## assert not (a == c) + ## ``` -proc `==` *(a, b: Color): bool {.borrow.} - ## compares two colors. - -template extract(a: Color, r, g, b: expr) {.immediate.}= +template extract(a: Color, r, g, b: untyped) = var r = a.int shr 16 and 0xff var g = a.int shr 8 and 0xff var b = a.int and 0xff -template rawRGB(r, g, b: int): expr = +template rawRGB(r, g, b: int): Color = Color(r shl 16 or g shl 8 or b) -template colorOp(op: expr) {.immediate.} = +template colorOp(op): Color = extract(a, ar, ag, ab) extract(b, br, bg, bb) - result = rawRGB(op(ar, br), op(ag, bg), op(ab, bb)) + rawRGB(op(ar, br), op(ag, bg), op(ab, bb)) proc satPlus(a, b: int): int {.inline.} = result = a +% b @@ -41,24 +49,64 @@ proc satMinus(a, b: int): int {.inline.} = if result < 0: result = 0 proc `+`*(a, b: Color): Color = - ## adds two colors: This uses saturated artithmetic, so that each color + ## Adds two colors. + ## + ## This uses saturated arithmetic, so that each color ## component cannot overflow (255 is used as a maximum). + ## + runnableExamples: + var + a = Color(0xaa_00_ff) + b = Color(0x11_cc_cc) + assert a + b == Color(0xbb_cc_ff) + colorOp(satPlus) proc `-`*(a, b: Color): Color = - ## subtracts two colors: This uses saturated artithmetic, so that each color - ## component cannot overflow (255 is used as a maximum). + ## Subtracts two colors. + ## + ## This uses saturated arithmetic, so that each color + ## component cannot underflow (0 is used as a minimum). + ## + runnableExamples: + var + a = Color(0xff_33_ff) + b = Color(0x11_ff_cc) + assert a - b == Color(0xee_00_33) + colorOp(satMinus) proc extractRGB*(a: Color): tuple[r, g, b: range[0..255]] = - ## extracts the red/green/blue components of the color `a`. + ## Extracts the red/green/blue components of the color `a`. + ## + runnableExamples: + var + a = Color(0xff_00_ff) + b = Color(0x00_ff_cc) + type + Col = range[0..255] + # assert extractRGB(a) == (r: 255.Col, g: 0.Col, b: 255.Col) + # assert extractRGB(b) == (r: 0.Col, g: 255.Col, b: 204.Col) + echo extractRGB(a) + echo typeof(extractRGB(a)) + echo extractRGB(b) + echo typeof(extractRGB(b)) + result.r = a.int shr 16 and 0xff result.g = a.int shr 8 and 0xff result.b = a.int and 0xff proc intensity*(a: Color, f: float): Color = - ## returns `a` with intensity `f`. `f` should be a float from 0.0 (completely + ## Returns `a` with intensity `f`. `f` should be a float from 0.0 (completely ## dark) to 1.0 (full color intensity). + ## + runnableExamples: + var + a = Color(0xff_00_ff) + b = Color(0x00_42_cc) + assert a.intensity(0.5) == Color(0x80_00_80) + assert b.intensity(0.5) == Color(0x00_21_66) + var r = toInt(toFloat(a.int shr 16 and 0xff) * f) var g = toInt(toFloat(a.int shr 8 and 0xff) * f) var b = toInt(toFloat(a.int and 0xff) * f) @@ -67,12 +115,24 @@ proc intensity*(a: Color, f: float): Color = if b >% 255: b = 255 result = rawRGB(r, g, b) -template mix*(a, b: Color, fn: expr): expr = - ## uses `fn` to mix the colors `a` and `b`. `fn` is invoked for each component - ## R, G, and B. This is a template because `fn` should be inlined and the - ## compiler cannot inline proc pointers yet. If `fn`'s result is not in the - ## range[0..255], it will be saturated to be so. - template `><` (x: expr): expr = +template mix*(a, b: Color, fn: untyped): untyped = + ## Uses `fn` to mix the colors `a` and `b`. + ## + ## `fn` is invoked for each component R, G, and B. + ## If `fn`'s result is not in the `range[0..255]`, + ## it will be saturated to be so. + ## + runnableExamples: + var + a = Color(0x0a2814) + b = Color(0x050a03) + + proc myMix(x, y: int): int = + 2 * x - 3 * y + + assert mix(a, b, myMix) == Color(0x05_32_1f) + + template `><` (x: untyped): untyped = # keep it in the range 0..255 block: var y = x # eval only once @@ -80,9 +140,9 @@ template mix*(a, b: Color, fn: expr): expr = y = if y < 0: 0 else: 255 y - (bind extract)(a, ar, ag, ab) - (bind extract)(b, br, bg, bb) - (bind rawRGB)(><fn(ar, br), ><fn(ag, bg), ><fn(ab, bb)) + extract(a, ar, ag, ab) + extract(b, br, bg, bb) + rawRGB(><fn(ar, br), ><fn(ag, bg), ><fn(ab, bb)) const @@ -112,6 +172,7 @@ const colDarkGoldenRod* = Color(0xB8860B) colDarkGray* = Color(0xA9A9A9) colDarkGreen* = Color(0x006400) + colDarkGrey* = Color(0xA9A9A9) colDarkKhaki* = Color(0xBDB76B) colDarkMagenta* = Color(0x8B008B) colDarkOliveGreen* = Color(0x556B2F) @@ -122,11 +183,13 @@ const colDarkSeaGreen* = Color(0x8FBC8F) colDarkSlateBlue* = Color(0x483D8B) colDarkSlateGray* = Color(0x2F4F4F) + colDarkSlateGrey* = Color(0x2F4F4F) colDarkTurquoise* = Color(0x00CED1) colDarkViolet* = Color(0x9400D3) colDeepPink* = Color(0xFF1493) colDeepSkyBlue* = Color(0x00BFFF) colDimGray* = Color(0x696969) + colDimGrey* = Color(0x696969) colDodgerBlue* = Color(0x1E90FF) colFireBrick* = Color(0xB22222) colFloralWhite* = Color(0xFFFAF0) @@ -139,6 +202,7 @@ const colGray* = Color(0x808080) colGreen* = Color(0x008000) colGreenYellow* = Color(0xADFF2F) + colGrey* = Color(0x808080) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) colIndianRed* = Color(0xCD5C5C) @@ -153,13 +217,15 @@ const colLightCoral* = Color(0xF08080) colLightCyan* = Color(0xE0FFFF) colLightGoldenRodYellow* = Color(0xFAFAD2) - colLightGrey* = Color(0xD3D3D3) + colLightGray* = Color(0xD3D3D3) colLightGreen* = Color(0x90EE90) + colLightGrey* = Color(0xD3D3D3) colLightPink* = Color(0xFFB6C1) colLightSalmon* = Color(0xFFA07A) colLightSeaGreen* = Color(0x20B2AA) colLightSkyBlue* = Color(0x87CEFA) colLightSlateGray* = Color(0x778899) + colLightSlateGrey* = Color(0x778899) colLightSteelBlue* = Color(0xB0C4DE) colLightYellow* = Color(0xFFFFE0) colLime* = Color(0x00FF00) @@ -170,7 +236,7 @@ const colMediumAquaMarine* = Color(0x66CDAA) colMediumBlue* = Color(0x0000CD) colMediumOrchid* = Color(0xBA55D3) - colMediumPurple* = Color(0x9370D8) + colMediumPurple* = Color(0x9370DB) colMediumSeaGreen* = Color(0x3CB371) colMediumSlateBlue* = Color(0x7B68EE) colMediumSpringGreen* = Color(0x00FA9A) @@ -191,7 +257,7 @@ const colPaleGoldenRod* = Color(0xEEE8AA) colPaleGreen* = Color(0x98FB98) colPaleTurquoise* = Color(0xAFEEEE) - colPaleVioletRed* = Color(0xD87093) + colPaleVioletRed* = Color(0xDB7093) colPapayaWhip* = Color(0xFFEFD5) colPeachPuff* = Color(0xFFDAB9) colPeru* = Color(0xCD853F) @@ -199,6 +265,7 @@ const colPlum* = Color(0xDDA0DD) colPowderBlue* = Color(0xB0E0E6) colPurple* = Color(0x800080) + colRebeccaPurple* = Color(0x663399) colRed* = Color(0xFF0000) colRosyBrown* = Color(0xBC8F8F) colRoyalBlue* = Color(0x4169E1) @@ -212,6 +279,7 @@ const colSkyBlue* = Color(0x87CEEB) colSlateBlue* = Color(0x6A5ACD) colSlateGray* = Color(0x708090) + colSlateGrey* = Color(0x708090) colSnow* = Color(0xFFFAFA) colSpringGreen* = Color(0x00FF7F) colSteelBlue* = Color(0x4682B4) @@ -227,185 +295,213 @@ const colYellow* = Color(0xFFFF00) colYellowGreen* = Color(0x9ACD32) - colorNames = [ - ("aliceblue", colAliceBlue), - ("antiquewhite", colAntiqueWhite), - ("aqua", colAqua), - ("aquamarine", colAquamarine), - ("azure", colAzure), - ("beige", colBeige), - ("bisque", colBisque), - ("black", colBlack), - ("blanchedalmond", colBlanchedAlmond), - ("blue", colBlue), - ("blueviolet", colBlueViolet), - ("brown", colBrown), - ("burlywood", colBurlyWood), - ("cadetblue", colCadetBlue), - ("chartreuse", colChartreuse), - ("chocolate", colChocolate), - ("coral", colCoral), - ("cornflowerblue", colCornflowerBlue), - ("cornsilk", colCornsilk), - ("crimson", colCrimson), - ("cyan", colCyan), - ("darkblue", colDarkBlue), - ("darkcyan", colDarkCyan), - ("darkgoldenrod", colDarkGoldenRod), - ("darkgray", colDarkGray), - ("darkgreen", colDarkGreen), - ("darkkhaki", colDarkKhaki), - ("darkmagenta", colDarkMagenta), - ("darkolivegreen", colDarkOliveGreen), - ("darkorange", colDarkorange), - ("darkorchid", colDarkOrchid), - ("darkred", colDarkRed), - ("darksalmon", colDarkSalmon), - ("darkseagreen", colDarkSeaGreen), - ("darkslateblue", colDarkSlateBlue), - ("darkslategray", colDarkSlateGray), - ("darkturquoise", colDarkTurquoise), - ("darkviolet", colDarkViolet), - ("deeppink", colDeepPink), - ("deepskyblue", colDeepSkyBlue), - ("dimgray", colDimGray), - ("dodgerblue", colDodgerBlue), - ("firebrick", colFireBrick), - ("floralwhite", colFloralWhite), - ("forestgreen", colForestGreen), - ("fuchsia", colFuchsia), - ("gainsboro", colGainsboro), - ("ghostwhite", colGhostWhite), - ("gold", colGold), - ("goldenrod", colGoldenRod), - ("gray", colGray), - ("green", colGreen), - ("greenyellow", colGreenYellow), - ("honeydew", colHoneyDew), - ("hotpink", colHotPink), - ("indianred", colIndianRed), - ("indigo", colIndigo), - ("ivory", colIvory), - ("khaki", colKhaki), - ("lavender", colLavender), - ("lavenderblush", colLavenderBlush), - ("lawngreen", colLawnGreen), - ("lemonchiffon", colLemonChiffon), - ("lightblue", colLightBlue), - ("lightcoral", colLightCoral), - ("lightcyan", colLightCyan), - ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgrey", colLightGrey), - ("lightgreen", colLightGreen), - ("lightpink", colLightPink), - ("lightsalmon", colLightSalmon), - ("lightseagreen", colLightSeaGreen), - ("lightskyblue", colLightSkyBlue), - ("lightslategray", colLightSlateGray), - ("lightsteelblue", colLightSteelBlue), - ("lightyellow", colLightYellow), - ("lime", colLime), - ("limegreen", colLimeGreen), - ("linen", colLinen), - ("magenta", colMagenta), - ("maroon", colMaroon), - ("mediumaquamarine", colMediumAquaMarine), - ("mediumblue", colMediumBlue), - ("mediumorchid", colMediumOrchid), - ("mediumpurple", colMediumPurple), - ("mediumseagreen", colMediumSeaGreen), - ("mediumslateblue", colMediumSlateBlue), - ("mediumspringgreen", colMediumSpringGreen), - ("mediumturquoise", colMediumTurquoise), - ("mediumvioletred", colMediumVioletRed), - ("midnightblue", colMidnightBlue), - ("mintcream", colMintCream), - ("mistyrose", colMistyRose), - ("moccasin", colMoccasin), - ("navajowhite", colNavajoWhite), - ("navy", colNavy), - ("oldlace", colOldLace), - ("olive", colOlive), - ("olivedrab", colOliveDrab), - ("orange", colOrange), - ("orangered", colOrangeRed), - ("orchid", colOrchid), - ("palegoldenrod", colPaleGoldenRod), - ("palegreen", colPaleGreen), - ("paleturquoise", colPaleTurquoise), - ("palevioletred", colPaleVioletRed), - ("papayawhip", colPapayaWhip), - ("peachpuff", colPeachPuff), - ("peru", colPeru), - ("pink", colPink), - ("plum", colPlum), - ("powderblue", colPowderBlue), - ("purple", colPurple), - ("red", colRed), - ("rosybrown", colRosyBrown), - ("royalblue", colRoyalBlue), - ("saddlebrown", colSaddleBrown), - ("salmon", colSalmon), - ("sandybrown", colSandyBrown), - ("seagreen", colSeaGreen), - ("seashell", colSeaShell), - ("sienna", colSienna), - ("silver", colSilver), - ("skyblue", colSkyBlue), - ("slateblue", colSlateBlue), - ("slategray", colSlateGray), - ("snow", colSnow), - ("springgreen", colSpringGreen), - ("steelblue", colSteelBlue), - ("tan", colTan), - ("teal", colTeal), - ("thistle", colThistle), - ("tomato", colTomato), - ("turquoise", colTurquoise), - ("violet", colViolet), - ("wheat", colWheat), - ("white", colWhite), - ("whitesmoke", colWhiteSmoke), - ("yellow", colYellow), - ("yellowgreen", colYellowGreen)] + colorNames = { + "aliceblue": colAliceBlue, + "antiquewhite": colAntiqueWhite, + "aqua": colAqua, + "aquamarine": colAquamarine, + "azure": colAzure, + "beige": colBeige, + "bisque": colBisque, + "black": colBlack, + "blanchedalmond": colBlanchedAlmond, + "blue": colBlue, + "blueviolet": colBlueViolet, + "brown": colBrown, + "burlywood": colBurlyWood, + "cadetblue": colCadetBlue, + "chartreuse": colChartreuse, + "chocolate": colChocolate, + "coral": colCoral, + "cornflowerblue": colCornflowerBlue, + "cornsilk": colCornsilk, + "crimson": colCrimson, + "cyan": colCyan, + "darkblue": colDarkBlue, + "darkcyan": colDarkCyan, + "darkgoldenrod": colDarkGoldenRod, + "darkgray": colDarkGray, + "darkgreen": colDarkGreen, + "darkgrey": colDarkGrey, + "darkkhaki": colDarkKhaki, + "darkmagenta": colDarkMagenta, + "darkolivegreen": colDarkOliveGreen, + "darkorange": colDarkorange, + "darkorchid": colDarkOrchid, + "darkred": colDarkRed, + "darksalmon": colDarkSalmon, + "darkseagreen": colDarkSeaGreen, + "darkslateblue": colDarkSlateBlue, + "darkslategray": colDarkSlateGray, + "darkslategrey": colDarkSlateGrey, + "darkturquoise": colDarkTurquoise, + "darkviolet": colDarkViolet, + "deeppink": colDeepPink, + "deepskyblue": colDeepSkyBlue, + "dimgray": colDimGray, + "dimgrey": colDimGrey, + "dodgerblue": colDodgerBlue, + "firebrick": colFireBrick, + "floralwhite": colFloralWhite, + "forestgreen": colForestGreen, + "fuchsia": colFuchsia, + "gainsboro": colGainsboro, + "ghostwhite": colGhostWhite, + "gold": colGold, + "goldenrod": colGoldenRod, + "gray": colGray, + "green": colGreen, + "greenyellow": colGreenYellow, + "grey": colGrey, + "honeydew": colHoneyDew, + "hotpink": colHotPink, + "indianred": colIndianRed, + "indigo": colIndigo, + "ivory": colIvory, + "khaki": colKhaki, + "lavender": colLavender, + "lavenderblush": colLavenderBlush, + "lawngreen": colLawnGreen, + "lemonchiffon": colLemonChiffon, + "lightblue": colLightBlue, + "lightcoral": colLightCoral, + "lightcyan": colLightCyan, + "lightgoldenrodyellow": colLightGoldenRodYellow, + "lightgray": colLightGray, + "lightgreen": colLightGreen, + "lightgrey": colLightGrey, + "lightpink": colLightPink, + "lightsalmon": colLightSalmon, + "lightseagreen": colLightSeaGreen, + "lightskyblue": colLightSkyBlue, + "lightslategray": colLightSlateGray, + "lightslategrey": colLightSlateGrey, + "lightsteelblue": colLightSteelBlue, + "lightyellow": colLightYellow, + "lime": colLime, + "limegreen": colLimeGreen, + "linen": colLinen, + "magenta": colMagenta, + "maroon": colMaroon, + "mediumaquamarine": colMediumAquaMarine, + "mediumblue": colMediumBlue, + "mediumorchid": colMediumOrchid, + "mediumpurple": colMediumPurple, + "mediumseagreen": colMediumSeaGreen, + "mediumslateblue": colMediumSlateBlue, + "mediumspringgreen": colMediumSpringGreen, + "mediumturquoise": colMediumTurquoise, + "mediumvioletred": colMediumVioletRed, + "midnightblue": colMidnightBlue, + "mintcream": colMintCream, + "mistyrose": colMistyRose, + "moccasin": colMoccasin, + "navajowhite": colNavajoWhite, + "navy": colNavy, + "oldlace": colOldLace, + "olive": colOlive, + "olivedrab": colOliveDrab, + "orange": colOrange, + "orangered": colOrangeRed, + "orchid": colOrchid, + "palegoldenrod": colPaleGoldenRod, + "palegreen": colPaleGreen, + "paleturquoise": colPaleTurquoise, + "palevioletred": colPaleVioletRed, + "papayawhip": colPapayaWhip, + "peachpuff": colPeachPuff, + "peru": colPeru, + "pink": colPink, + "plum": colPlum, + "powderblue": colPowderBlue, + "purple": colPurple, + "rebeccapurple": colRebeccaPurple, + "red": colRed, + "rosybrown": colRosyBrown, + "royalblue": colRoyalBlue, + "saddlebrown": colSaddleBrown, + "salmon": colSalmon, + "sandybrown": colSandyBrown, + "seagreen": colSeaGreen, + "seashell": colSeaShell, + "sienna": colSienna, + "silver": colSilver, + "skyblue": colSkyBlue, + "slateblue": colSlateBlue, + "slategray": colSlateGray, + "slategrey": colSlateGrey, + "snow": colSnow, + "springgreen": colSpringGreen, + "steelblue": colSteelBlue, + "tan": colTan, + "teal": colTeal, + "thistle": colThistle, + "tomato": colTomato, + "turquoise": colTurquoise, + "violet": colViolet, + "wheat": colWheat, + "white": colWhite, + "whitesmoke": colWhiteSmoke, + "yellow": colYellow, + "yellowgreen": colYellowGreen} proc `$`*(c: Color): string = - ## converts a color into its textual representation. Example: ``#00FF00``. + ## Converts a color into its textual representation. + ## + runnableExamples: + assert $colFuchsia == "#FF00FF" result = '#' & toHex(int(c), 6) -proc binaryStrSearch(x: openArray[tuple[name: string, col: Color]], - y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmp(x[mid].name, y) - if c < 0: a = mid + 1 - elif c > 0: b = mid - 1 - else: return mid - result = - 1 +proc colorNameCmp(x: tuple[name: string, col: Color], y: string): int = + result = cmpIgnoreCase(x.name, y) proc parseColor*(name: string): Color = - ## parses `name` to a color value. If no valid color could be - ## parsed ``EInvalidValue`` is raised. - if name[0] == '#': + ## Parses `name` to a color value. + ## + ## If no valid color could be parsed `ValueError` is raised. + ## Case insensitive. + ## + runnableExamples: + var + a = "silver" + b = "#0179fc" + c = "#zzmmtt" + assert parseColor(a) == Color(0xc0_c0_c0) + assert parseColor(b) == Color(0x01_79_fc) + doAssertRaises(ValueError): discard parseColor(c) + + if name.len > 0 and name[0] == '#': result = Color(parseHexInt(name)) else: - var idx = binaryStrSearch(colorNames, name) + var idx = binarySearch(colorNames, name, colorNameCmp) if idx < 0: raise newException(ValueError, "unknown color: " & name) result = colorNames[idx][1] proc isColor*(name: string): bool = - ## returns true if `name` is a known color name or a hexadecimal color - ## prefixed with ``#``. + ## Returns true if `name` is a known color name or a hexadecimal color + ## prefixed with `#`. Case insensitive. + ## + runnableExamples: + var + a = "silver" + b = "#0179fc" + c = "#zzmmtt" + assert a.isColor + assert b.isColor + assert not c.isColor + + if name.len == 0: return false if name[0] == '#': for i in 1 .. name.len-1: - if name[i] notin {'0'..'9', 'a'..'f', 'A'..'F'}: return false + if name[i] notin HexDigits: return false result = true else: - result = binaryStrSearch(colorNames, name) >= 0 + result = binarySearch(colorNames, name, colorNameCmp) >= 0 proc rgb*(r, g, b: range[0..255]): Color = - ## constructs a color from RGB values. - result = rawRGB(r, g, b) + ## Constructs a color from RGB values. + ## + runnableExamples: + assert rgb(0, 255, 128) == Color(0x00_ff_80) + result = rawRGB(r, g, b) diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index ccde5ee0a..b48811eae 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -7,184 +7,187 @@ # distribution, for details about the copyright. # +## This module implements complex numbers +## and basic mathematical operations on them. +## +## Complex numbers are currently generic over 64-bit or 32-bit floats. +runnableExamples: + from std/math import almostEqual, sqrt -## This module implements complex numbers. -{.push checks:off, line_dir:off, stack_trace:off, debugger:off.} -# the user does not want to trace a part -# of the standard library! + let + z1 = complex(1.0, 2.0) + z2 = complex(3.0, -4.0) + assert almostEqual(z1 + z2, complex(4.0, -2.0)) + assert almostEqual(z1 - z2, complex(-2.0, 6.0)) + assert almostEqual(z1 * z2, complex(11.0, 2.0)) + assert almostEqual(z1 / z2, complex(-0.2, 0.4)) -import - math + assert almostEqual(abs(z1), sqrt(5.0)) + assert almostEqual(conjugate(z1), complex(1.0, -2.0)) -const - EPS = 1.0e-7 ## Epsilon used for float comparisons. + let (r, phi) = z1.polar + assert almostEqual(rect(r, phi), z1) -type - Complex* = tuple[re, im: float] - ## a complex number, consisting of a real and an imaginary part +{.push checks: off, line_dir: off, stack_trace: off, debugger: off.} +# the user does not want to trace a part of the standard library! -{.deprecated: [TComplex: Complex].} +import std/[math, strformat] -proc toComplex*(x: SomeInteger): Complex = - ## Convert some integer ``x`` to a complex number. - result.re = x - result.im = 0 +type + Complex*[T: SomeFloat] = object + ## A complex number, consisting of a real and an imaginary part. + re*, im*: T + Complex64* = Complex[float64] + ## Alias for a complex number using 64-bit floats. + Complex32* = Complex[float32] + ## Alias for a complex number using 32-bit floats. + +func complex*[T: SomeFloat](re: T; im: T = 0.0): Complex[T] = + ## Returns a `Complex[T]` with real part `re` and imaginary part `im`. + result.re = re + result.im = im + +func complex32*(re: float32; im: float32 = 0.0): Complex32 = + ## Returns a `Complex32` with real part `re` and imaginary part `im`. + result.re = re + result.im = im + +func complex64*(re: float64; im: float64 = 0.0): Complex64 = + ## Returns a `Complex64` with real part `re` and imaginary part `im`. + result.re = re + result.im = im + +template im*(arg: typedesc[float32]): Complex32 = complex32(0, 1) + ## Returns the imaginary unit (`complex32(0, 1)`). +template im*(arg: typedesc[float64]): Complex64 = complex64(0, 1) + ## Returns the imaginary unit (`complex64(0, 1)`). +template im*(arg: float32): Complex32 = complex32(0, arg) + ## Returns `arg` as an imaginary number (`complex32(0, arg)`). +template im*(arg: float64): Complex64 = complex64(0, arg) + ## Returns `arg` as an imaginary number (`complex64(0, arg)`). + +func abs*[T](z: Complex[T]): T = + ## Returns the absolute value of `z`, + ## that is the distance from (0, 0) to `z`. + result = hypot(z.re, z.im) + +func abs2*[T](z: Complex[T]): T = + ## Returns the squared absolute value of `z`, + ## that is the squared distance from (0, 0) to `z`. + ## This is more efficient than `abs(z) ^ 2`. + result = z.re * z.re + z.im * z.im + +func sgn*[T](z: Complex[T]): Complex[T] = + ## Returns the phase of `z` as a unit complex number, + ## or 0 if `z` is 0. + let a = abs(z) + if a != 0: + result = z / a + +func conjugate*[T](z: Complex[T]): Complex[T] = + ## Returns the complex conjugate of `z` (`complex(z.re, -z.im)`). + result.re = z.re + result.im = -z.im -proc `==` *(x, y: Complex): bool = - ## Compare two complex numbers `x` and `y` for equality. - result = x.re == y.re and x.im == y.im +func inv*[T](z: Complex[T]): Complex[T] = + ## Returns the multiplicative inverse of `z` (`1/z`). + conjugate(z) / abs2(z) -proc `=~` *(x, y: Complex): bool = - ## Compare two complex numbers `x` and `y` approximately. - result = abs(x.re-y.re)<EPS and abs(x.im-y.im)<EPS +func `==`*[T](x, y: Complex[T]): bool = + ## Compares two complex numbers for equality. + result = x.re == y.re and x.im == y.im -proc `+` *(x, y: Complex): Complex = - ## Add two complex numbers. - result.re = x.re + y.re - result.im = x.im + y.im +func `+`*[T](x: T; y: Complex[T]): Complex[T] = + ## Adds a real number to a complex number. + result.re = x + y.re + result.im = y.im -proc `+` *(x: Complex, y: float): Complex = - ## Add complex `x` to float `y`. +func `+`*[T](x: Complex[T]; y: T): Complex[T] = + ## Adds a complex number to a real number. result.re = x.re + y result.im = x.im -proc `+` *(x: float, y: Complex): Complex = - ## Add float `x` to complex `y`. - result.re = x + y.re - result.im = y.im - +func `+`*[T](x, y: Complex[T]): Complex[T] = + ## Adds two complex numbers. + result.re = x.re + y.re + result.im = x.im + y.im -proc `-` *(z: Complex): Complex = +func `-`*[T](z: Complex[T]): Complex[T] = ## Unary minus for complex numbers. result.re = -z.re result.im = -z.im -proc `-` *(x, y: Complex): Complex = - ## Subtract two complex numbers. - result.re = x.re - y.re - result.im = x.im - y.im - -proc `-` *(x: Complex, y: float): Complex = - ## Subtracts float `y` from complex `x`. - result = x + (-y) +func `-`*[T](x: T; y: Complex[T]): Complex[T] = + ## Subtracts a complex number from a real number. + result.re = x - y.re + result.im = -y.im -proc `-` *(x: float, y: Complex): Complex = - ## Subtracts complex `y` from float `x`. - result = x + (-y) - - -proc `/` *(x, y: Complex): Complex = - ## Divide `x` by `y`. - var - r, den: float - if abs(y.re) < abs(y.im): - r = y.re / y.im - den = y.im + r * y.re - result.re = (x.re * r + x.im) / den - result.im = (x.im * r - x.re) / den - else: - r = y.im / y.re - den = y.re + r * y.im - result.re = (x.re + r * x.im) / den - result.im = (x.im - r * x.re) / den +func `-`*[T](x: Complex[T]; y: T): Complex[T] = + ## Subtracts a real number from a complex number. + result.re = x.re - y + result.im = x.im -proc `/` *(x : Complex, y: float ): Complex = - ## Divide complex `x` by float `y`. - result.re = x.re/y - result.im = x.im/y +func `-`*[T](x, y: Complex[T]): Complex[T] = + ## Subtracts two complex numbers. + result.re = x.re - y.re + result.im = x.im - y.im -proc `/` *(x : float, y: Complex ): Complex = - ## Divide float `x` by complex `y`. - var num : Complex = (x, 0.0) - result = num/y +func `*`*[T](x: T; y: Complex[T]): Complex[T] = + ## Multiplies a real number with a complex number. + result.re = x * y.re + result.im = x * y.im +func `*`*[T](x: Complex[T]; y: T): Complex[T] = + ## Multiplies a complex number with a real number. + result.re = x.re * y + result.im = x.im * y -proc `*` *(x, y: Complex): Complex = - ## Multiply `x` with `y`. +func `*`*[T](x, y: Complex[T]): Complex[T] = + ## Multiplies two complex numbers. result.re = x.re * y.re - x.im * y.im result.im = x.im * y.re + x.re * y.im -proc `*` *(x: float, y: Complex): Complex = - ## Multiply float `x` with complex `y`. - result.re = x * y.re - result.im = x * y.im +func `/`*[T](x: Complex[T]; y: T): Complex[T] = + ## Divides a complex number by a real number. + result.re = x.re / y + result.im = x.im / y -proc `*` *(x: Complex, y: float): Complex = - ## Multiply complex `x` with float `y`. - result.re = x.re * y - result.im = x.im * y +func `/`*[T](x: T; y: Complex[T]): Complex[T] = + ## Divides a real number by a complex number. + result = x * inv(y) +func `/`*[T](x, y: Complex[T]): Complex[T] = + ## Divides two complex numbers. + x * conjugate(y) / abs2(y) -proc `+=` *(x: var Complex, y: Complex) = - ## Add `y` to `x`. +func `+=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Adds `y` to `x`. x.re += y.re x.im += y.im -proc `+=` *(x: var Complex, y: float) = - ## Add `y` to the complex number `x`. - x.re += y - -proc `-=` *(x: var Complex, y: Complex) = - ## Subtract `y` from `x`. +func `-=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Subtracts `y` from `x`. x.re -= y.re x.im -= y.im -proc `-=` *(x: var Complex, y: float) = - ## Subtract `y` from the complex number `x`. - x.re -= y - -proc `*=` *(x: var Complex, y: Complex) = - ## Multiply `y` to `x`. +func `*=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Multiplies `x` by `y`. let im = x.im * y.re + x.re * y.im x.re = x.re * y.re - x.im * y.im x.im = im -proc `*=` *(x: var Complex, y: float) = - ## Multiply `y` to the complex number `x`. - x.re *= y - x.im *= y - -proc `/=` *(x: var Complex, y: Complex) = - ## Divide `x` by `y` in place. +func `/=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Divides `x` by `y` in place. x = x / y -proc `/=` *(x : var Complex, y: float) = - ## Divide complex `x` by float `y` in place. - x.re /= y - x.im /= y - -proc abs*(z: Complex): float = - ## Return the distance from (0,0) to `z`. - - # optimized by checking special cases (sqrt is expensive) - var x, y, temp: float - - x = abs(z.re) - y = abs(z.im) - if x == 0.0: - result = y - elif y == 0.0: - result = x - elif x > y: - temp = y / x - result = x * sqrt(1.0 + temp * temp) - else: - temp = x / y - result = y * sqrt(1.0 + temp * temp) - - -proc conjugate*(z: Complex): Complex = - ## Conjugate of complex number `z`. - result.re = z.re - result.im = -z.im - - -proc sqrt*(z: Complex): Complex = - ## Square root for a complex number `z`. - var x, y, w, r: float +func sqrt*[T](z: Complex[T]): Complex[T] = + ## Computes the + ## ([principal](https://en.wikipedia.org/wiki/Square_root#Principal_square_root_of_a_complex_number)) + ## square root of a complex number `z`. + var x, y, w, r: T if z.re == 0.0 and z.im == 0.0: result = z @@ -197,247 +200,274 @@ proc sqrt*(z: Complex): Complex = else: r = x / y w = sqrt(y) * sqrt(0.5 * (r + sqrt(1.0 + r * r))) + if z.re >= 0.0: result.re = w result.im = z.im / (w * 2.0) else: - if z.im >= 0.0: result.im = w - else: result.im = -w + result.im = if z.im >= 0.0: w else: -w result.re = z.im / (result.im + result.im) - -proc exp*(z: Complex): Complex = - ## e raised to the power `z`. - var rho = exp(z.re) - var theta = z.im - result.re = rho*cos(theta) - result.im = rho*sin(theta) - - -proc ln*(z: Complex): Complex = - ## Returns the natural log of `z`. +func exp*[T](z: Complex[T]): Complex[T] = + ## Computes the exponential function (`e^z`). + let + rho = exp(z.re) + theta = z.im + result.re = rho * cos(theta) + result.im = rho * sin(theta) + +func ln*[T](z: Complex[T]): Complex[T] = + ## Returns the + ## ([principal value](https://en.wikipedia.org/wiki/Complex_logarithm#Principal_value) + ## of the) natural logarithm of `z`. result.re = ln(abs(z)) - result.im = arctan2(z.im,z.re) - -proc log10*(z: Complex): Complex = - ## Returns the log base 10 of `z`. - result = ln(z)/ln(10.0) - -proc log2*(z: Complex): Complex = - ## Returns the log base 2 of `z`. - result = ln(z)/ln(2.0) - - -proc pow*(x, y: Complex): Complex = - ## `x` raised to the power `y`. - if x.re == 0.0 and x.im == 0.0: - if y.re == 0.0 and y.im == 0.0: + result.im = arctan2(z.im, z.re) + +func log10*[T](z: Complex[T]): Complex[T] = + ## Returns the logarithm base 10 of `z`. + ## + ## **See also:** + ## * `ln func<#ln,Complex[T]>`_ + result = ln(z) / ln(10.0) + +func log2*[T](z: Complex[T]): Complex[T] = + ## Returns the logarithm base 2 of `z`. + ## + ## **See also:** + ## * `ln func<#ln,Complex[T]>`_ + result = ln(z) / ln(2.0) + +func pow*[T](x, y: Complex[T]): Complex[T] = + ## `x` raised to the power of `y`. + if x.re == 0.0 and x.im == 0.0: + if y.re == 0.0 and y.im == 0.0: result.re = 1.0 result.im = 0.0 else: result.re = 0.0 result.im = 0.0 - elif y.re == 1.0 and y.im == 0.0: - result = x - elif y.re == -1.0 and y.im == 0.0: - result = 1.0/x + elif y.im == 0.0: + if y.re == 1.0: + result = x + elif y.re == -1.0: + result = T(1.0) / x + elif y.re == 2.0: + result = x * x + elif y.re == 0.5: + result = sqrt(x) + elif x.im == 0.0: + # Revert to real pow when both base and exponent are real + result.re = pow(x.re, y.re) + result.im = 0.0 + else: + # Special case when the exponent is real + let + rho = abs(x) + theta = arctan2(x.im, x.re) + s = pow(rho, y.re) + r = y.re * theta + result.re = s * cos(r) + result.im = s * sin(r) + elif x.im == 0.0 and x.re == E: + # Special case Euler's formula + result = exp(y) else: - var rho = sqrt(x.re*x.re + x.im*x.im) - var theta = arctan2(x.im,x.re) - var s = pow(rho,y.re) * exp(-y.im*theta) - var r = y.re*theta + y.im*ln(rho) - result.re = s*cos(r) - result.im = s*sin(r) + let + rho = abs(x) + theta = arctan2(x.im, x.re) + s = pow(rho, y.re) * exp(-y.im * theta) + r = y.re * theta + y.im * ln(rho) + result.re = s * cos(r) + result.im = s * sin(r) + +func pow*[T](x: Complex[T]; y: T): Complex[T] = + ## The complex number `x` raised to the power of the real number `y`. + pow(x, complex[T](y)) -proc sin*(z: Complex): Complex = +func sin*[T](z: Complex[T]): Complex[T] = ## Returns the sine of `z`. - result.re = sin(z.re)*cosh(z.im) - result.im = cos(z.re)*sinh(z.im) + result.re = sin(z.re) * cosh(z.im) + result.im = cos(z.re) * sinh(z.im) -proc arcsin*(z: Complex): Complex = +func arcsin*[T](z: Complex[T]): Complex[T] = ## Returns the inverse sine of `z`. - var i: Complex = (0.0,1.0) - result = -i*ln(i*z + sqrt(1.0-z*z)) + result = -im(T) * ln(im(T) * z + sqrt(T(1.0) - z*z)) -proc cos*(z: Complex): Complex = +func cos*[T](z: Complex[T]): Complex[T] = ## Returns the cosine of `z`. - result.re = cos(z.re)*cosh(z.im) - result.im = -sin(z.re)*sinh(z.im) + result.re = cos(z.re) * cosh(z.im) + result.im = -sin(z.re) * sinh(z.im) -proc arccos*(z: Complex): Complex = +func arccos*[T](z: Complex[T]): Complex[T] = ## Returns the inverse cosine of `z`. - var i: Complex = (0.0,1.0) - result = -i*ln(z + sqrt(z*z-1.0)) + result = -im(T) * ln(z + sqrt(z*z - T(1.0))) -proc tan*(z: Complex): Complex = +func tan*[T](z: Complex[T]): Complex[T] = ## Returns the tangent of `z`. - result = sin(z)/cos(z) + result = sin(z) / cos(z) -proc arctan*(z: Complex): Complex = +func arctan*[T](z: Complex[T]): Complex[T] = ## Returns the inverse tangent of `z`. - var i: Complex = (0.0,1.0) - result = 0.5*i*(ln(1-i*z)-ln(1+i*z)) + result = T(0.5)*im(T) * (ln(T(1.0) - im(T)*z) - ln(T(1.0) + im(T)*z)) -proc cot*(z: Complex): Complex = +func cot*[T](z: Complex[T]): Complex[T] = ## Returns the cotangent of `z`. result = cos(z)/sin(z) -proc arccot*(z: Complex): Complex = +func arccot*[T](z: Complex[T]): Complex[T] = ## Returns the inverse cotangent of `z`. - var i: Complex = (0.0,1.0) - result = 0.5*i*(ln(1-i/z)-ln(1+i/z)) + result = T(0.5)*im(T) * (ln(T(1.0) - im(T)/z) - ln(T(1.0) + im(T)/z)) -proc sec*(z: Complex): Complex = +func sec*[T](z: Complex[T]): Complex[T] = ## Returns the secant of `z`. - result = 1.0/cos(z) + result = T(1.0) / cos(z) -proc arcsec*(z: Complex): Complex = +func arcsec*[T](z: Complex[T]): Complex[T] = ## Returns the inverse secant of `z`. - var i: Complex = (0.0,1.0) - result = -i*ln(i*sqrt(1-1/(z*z))+1/z) + result = -im(T) * ln(im(T) * sqrt(1.0 - 1.0/(z*z)) + T(1.0)/z) -proc csc*(z: Complex): Complex = +func csc*[T](z: Complex[T]): Complex[T] = ## Returns the cosecant of `z`. - result = 1.0/sin(z) + result = T(1.0) / sin(z) -proc arccsc*(z: Complex): Complex = +func arccsc*[T](z: Complex[T]): Complex[T] = ## Returns the inverse cosecant of `z`. - var i: Complex = (0.0,1.0) - result = -i*ln(sqrt(1-1/(z*z))+i/z) - + result = -im(T) * ln(sqrt(T(1.0) - T(1.0)/(z*z)) + im(T)/z) -proc sinh*(z: Complex): Complex = +func sinh*[T](z: Complex[T]): Complex[T] = ## Returns the hyperbolic sine of `z`. - result = 0.5*(exp(z)-exp(-z)) + result = T(0.5) * (exp(z) - exp(-z)) -proc arcsinh*(z: Complex): Complex = +func arcsinh*[T](z: Complex[T]): Complex[T] = ## Returns the inverse hyperbolic sine of `z`. - result = ln(z+sqrt(z*z+1)) + result = ln(z + sqrt(z*z + 1.0)) -proc cosh*(z: Complex): Complex = +func cosh*[T](z: Complex[T]): Complex[T] = ## Returns the hyperbolic cosine of `z`. - result = 0.5*(exp(z)+exp(-z)) + result = T(0.5) * (exp(z) + exp(-z)) -proc arccosh*(z: Complex): Complex = +func arccosh*[T](z: Complex[T]): Complex[T] = ## Returns the inverse hyperbolic cosine of `z`. - result = ln(z+sqrt(z*z-1)) + result = ln(z + sqrt(z*z - T(1.0))) -proc tanh*(z: Complex): Complex = +func tanh*[T](z: Complex[T]): Complex[T] = ## Returns the hyperbolic tangent of `z`. - result = sinh(z)/cosh(z) + result = sinh(z) / cosh(z) -proc arctanh*(z: Complex): Complex = +func arctanh*[T](z: Complex[T]): Complex[T] = ## Returns the inverse hyperbolic tangent of `z`. - result = 0.5*(ln((1+z)/(1-z))) + result = T(0.5) * (ln((T(1.0)+z) / (T(1.0)-z))) + +func coth*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic cotangent of `z`. + result = cosh(z) / sinh(z) -proc sech*(z: Complex): Complex = +func arccoth*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic cotangent of `z`. + result = T(0.5) * (ln(T(1.0) + T(1.0)/z) - ln(T(1.0) - T(1.0)/z)) + +func sech*[T](z: Complex[T]): Complex[T] = ## Returns the hyperbolic secant of `z`. - result = 2/(exp(z)+exp(-z)) + result = T(2.0) / (exp(z) + exp(-z)) -proc arcsech*(z: Complex): Complex = +func arcsech*[T](z: Complex[T]): Complex[T] = ## Returns the inverse hyperbolic secant of `z`. - result = ln(1/z+sqrt(1/z+1)*sqrt(1/z-1)) + result = ln(1.0/z + sqrt(T(1.0)/z+T(1.0)) * sqrt(T(1.0)/z-T(1.0))) -proc csch*(z: Complex): Complex = +func csch*[T](z: Complex[T]): Complex[T] = ## Returns the hyperbolic cosecant of `z`. - result = 2/(exp(z)-exp(-z)) + result = T(2.0) / (exp(z) - exp(-z)) -proc arccsch*(z: Complex): Complex = +func arccsch*[T](z: Complex[T]): Complex[T] = ## Returns the inverse hyperbolic cosecant of `z`. - result = ln(1/z+sqrt(1/(z*z)+1)) - -proc coth*(z: Complex): Complex = - ## Returns the hyperbolic cotangent of `z`. - result = cosh(z)/sinh(z) + result = ln(T(1.0)/z + sqrt(T(1.0)/(z*z) + T(1.0))) -proc arccoth*(z: Complex): Complex = - ## Returns the inverse hyperbolic cotangent of `z`. - result = 0.5*(ln(1+1/z)-ln(1-1/z)) - -proc phase*(z: Complex): float = - ## Returns the phase of `z`. +func phase*[T](z: Complex[T]): T = + ## Returns the phase (or argument) of `z`, that is the angle in polar representation. + ## + ## | `result = arctan2(z.im, z.re)` arctan2(z.im, z.re) -proc polar*(z: Complex): tuple[r, phi: float] = +func polar*[T](z: Complex[T]): tuple[r, phi: T] = ## Returns `z` in polar coordinates. - result.r = abs(z) - result.phi = phase(z) - -proc rect*(r: float, phi: float): Complex = + ## + ## | `result.r = abs(z)` + ## | `result.phi = phase(z)` + ## + ## **See also:** + ## * `rect func<#rect,T,T>`_ for the inverse operation + (r: abs(z), phi: phase(z)) + +func rect*[T](r, phi: T): Complex[T] = ## Returns the complex number with polar coordinates `r` and `phi`. - result.re = r * cos(phi) - result.im = r * sin(phi) + ## + ## | `result.re = r * cos(phi)` + ## | `result.im = r * sin(phi)` + ## + ## **See also:** + ## * `polar func<#polar,Complex[T]>`_ for the inverse operation + complex(r * cos(phi), r * sin(phi)) + +func almostEqual*[T: SomeFloat](x, y: Complex[T]; unitsInLastPlace: Natural = 4): bool = + ## Checks if two complex values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## Two complex values are considered almost equal if their real and imaginary + ## components are almost equal. + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + almostEqual(x.re, y.re, unitsInLastPlace = unitsInLastPlace) and + almostEqual(x.im, y.im, unitsInLastPlace = unitsInLastPlace) + +func `$`*(z: Complex): string = + ## Returns `z`'s string representation as `"(re, im)"`. + runnableExamples: + doAssert $complex(1.0, 2.0) == "(1.0, 2.0)" - -proc `$`*(z: Complex): string = - ## Returns `z`'s string representation as ``"(re, im)"``. result = "(" & $z.re & ", " & $z.im & ")" -{.pop.} - +proc formatValueAsTuple(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (real, imaginary) tuple. + result.add "(" + formatValue(result, value.re, specifier) + result.add ", " + formatValue(result, value.im, specifier) + result.add ")" + +proc formatValueAsComplexNumber(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (RE+IMj) number + ## By default, the real and imaginary parts are formatted using the general ('g') format + let specifier = if specifier.contains({'e', 'E', 'f', 'F', 'g', 'G'}): + specifier.replace("j") + else: + specifier.replace('j', 'g') + result.add "(" + formatValue(result, value.re, specifier) + if value.im >= 0 and not specifier.contains({'+', '-'}): + result.add "+" + formatValue(result, value.im, specifier) + result.add "j)" + +proc formatValue*(result: var string; value: Complex; specifier: string) = + ## Standard format implementation for `Complex`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + ## For complex numbers, we add a specific 'j' specifier, which formats + ## the value as (A+Bj) like in mathematics. + if specifier.len == 0: + result.add $value + elif 'j' in specifier: + formatValueAsComplexNumber(result, value, specifier) + else: + formatValueAsTuple(result, value, specifier) -when isMainModule: - var z = (0.0, 0.0) - var oo = (1.0,1.0) - var a = (1.0, 2.0) - var b = (-1.0, -2.0) - var m1 = (-1.0, 0.0) - var i = (0.0,1.0) - var one = (1.0,0.0) - var tt = (10.0, 20.0) - var ipi = (0.0, -PI) - - assert( a == a ) - assert( (a-a) == z ) - assert( (a+b) == z ) - assert( (a/b) == m1 ) - assert( (1.0/a) == (0.2, -0.4) ) - assert( (a*b) == (3.0, -4.0) ) - assert( 10.0*a == tt ) - assert( a*10.0 == tt ) - assert( tt/10.0 == a ) - assert( oo+(-1.0) == i ) - assert( (-1.0)+oo == i ) - assert( abs(oo) == sqrt(2.0) ) - assert( conjugate(a) == (1.0, -2.0) ) - assert( sqrt(m1) == i ) - assert( exp(ipi) =~ m1 ) - - assert( pow(a,b) =~ (-3.72999124927876, -1.68815826725068) ) - assert( pow(z,a) =~ (0.0, 0.0) ) - assert( pow(z,z) =~ (1.0, 0.0) ) - assert( pow(a,one) =~ a ) - assert( pow(a,m1) =~ (0.2, -0.4) ) - - assert( ln(a) =~ (0.804718956217050, 1.107148717794090) ) - assert( log10(a) =~ (0.349485002168009, 0.480828578784234) ) - assert( log2(a) =~ (1.16096404744368, 1.59727796468811) ) - - assert( sin(a) =~ (3.16577851321617, 1.95960104142161) ) - assert( cos(a) =~ (2.03272300701967, -3.05189779915180) ) - assert( tan(a) =~ (0.0338128260798967, 1.0147936161466335) ) - assert( cot(a) =~ 1.0/tan(a) ) - assert( sec(a) =~ 1.0/cos(a) ) - assert( csc(a) =~ 1.0/sin(a) ) - assert( arcsin(a) =~ (0.427078586392476, 1.528570919480998) ) - assert( arccos(a) =~ (1.14371774040242, -1.52857091948100) ) - assert( arctan(a) =~ (1.338972522294494, 0.402359478108525) ) - - assert( cosh(a) =~ (-0.642148124715520, 1.068607421382778) ) - assert( sinh(a) =~ (-0.489056259041294, 1.403119250622040) ) - assert( tanh(a) =~ (1.1667362572409199,-0.243458201185725) ) - assert( sech(a) =~ 1/cosh(a) ) - assert( csch(a) =~ 1/sinh(a) ) - assert( coth(a) =~ 1/tanh(a) ) - assert( arccosh(a) =~ (1.528570919480998, 1.14371774040242) ) - assert( arcsinh(a) =~ (1.469351744368185, 1.06344002357775) ) - assert( arctanh(a) =~ (0.173286795139986, 1.17809724509617) ) - assert( arcsech(a) =~ arccosh(1/a) ) - assert( arccsch(a) =~ arcsinh(1/a) ) - assert( arccoth(a) =~ arctanh(1/a) ) - - assert( phase(a) == 1.1071487177940904 ) - var t = polar(a) - assert( rect(t.r, t.phi) =~ a ) - assert( rect(1.0, 2.0) =~ (-0.4161468365471424, 0.9092974268256817) ) +{.pop.} diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim new file mode 100644 index 000000000..818f1b37a --- /dev/null +++ b/lib/pure/concurrency/atomics.nim @@ -0,0 +1,433 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Jörg Wollenschläger +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Types and operations for atomic operations and lockless algorithms. +## +## Unstable API. +## +## By default, C++ uses C11 atomic primitives. To use C++ `std::atomic`, +## `-d:nimUseCppAtomics` can be defined. + +runnableExamples: + # Atomic + var loc: Atomic[int] + loc.store(4) + assert loc.load == 4 + loc.store(2) + assert loc.load(moRelaxed) == 2 + loc.store(9) + assert loc.load(moAcquire) == 9 + loc.store(0, moRelease) + assert loc.load == 0 + + assert loc.exchange(7) == 0 + assert loc.load == 7 + + var expected = 7 + assert loc.compareExchange(expected, 5, moRelaxed, moRelaxed) + assert expected == 7 + assert loc.load == 5 + + assert not loc.compareExchange(expected, 12, moRelaxed, moRelaxed) + assert expected == 5 + assert loc.load == 5 + + assert loc.fetchAdd(1) == 5 + assert loc.fetchAdd(2) == 6 + assert loc.fetchSub(3) == 8 + + loc.atomicInc(1) + assert loc.load == 6 + + # AtomicFlag + var flag: AtomicFlag + + assert not flag.testAndSet + assert flag.testAndSet + flag.clear(moRelaxed) + assert not flag.testAndSet + +when (defined(cpp) and defined(nimUseCppAtomics)) or defined(nimdoc): + # For the C++ backend, types and operations map directly to C++11 atomics. + + {.push, header: "<atomic>".} + + type + MemoryOrder* {.importcpp: "std::memory_order".} = enum + ## Specifies how non-atomic operations can be reordered around atomic + ## operations. + + moRelaxed + ## No ordering constraints. Only the atomicity and ordering against + ## other atomic operations is guaranteed. + + moConsume + ## This ordering is currently discouraged as it's semantics are + ## being revised. Acquire operations should be preferred. + + moAcquire + ## When applied to a load operation, no reads or writes in the + ## current thread can be reordered before this operation. + + moRelease + ## When applied to a store operation, no reads or writes in the + ## current thread can be reorderd after this operation. + + moAcquireRelease + ## When applied to a read-modify-write operation, this behaves like + ## both an acquire and a release operation. + + moSequentiallyConsistent + ## Behaves like Acquire when applied to load, like Release when + ## applied to a store and like AcquireRelease when applied to a + ## read-modify-write operation. + ## Also guarantees that all threads observe the same total ordering + ## with other moSequentiallyConsistent operations. + + type + Atomic*[T] {.importcpp: "std::atomic", completeStruct.} = object + ## An atomic object with underlying type `T`. + raw: T + + AtomicFlag* {.importcpp: "std::atomic_flag", size: 1.} = object + ## An atomic boolean state. + + # Access operations + + proc load*[T](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.load(@)".} + ## Atomically obtains the value of the atomic object. + + proc store*[T](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importcpp: "#.store(@)".} + ## Atomically replaces the value of the atomic object with the `desired` + ## value. + + proc exchange*[T](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.exchange(@)".} + ## Atomically replaces the value of the atomic object with the `desired` + ## value and returns the old value. + + proc compareExchange*[T](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.importcpp: "#.compare_exchange_strong(@)".} + ## Atomically compares the value of the atomic object with the `expected` + ## value and performs exchange with the `desired` one if equal or load if + ## not. Returns true if the exchange was successful. + + proc compareExchange*[T](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.importcpp: "#.compare_exchange_strong(@)".} + ## Same as above, but allows for different memory orders for success and + ## failure. + + proc compareExchangeWeak*[T](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.importcpp: "#.compare_exchange_weak(@)".} + ## Same as above, but is allowed to fail spuriously. + + proc compareExchangeWeak*[T](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.importcpp: "#.compare_exchange_weak(@)".} + ## Same as above, but allows for different memory orders for success and + ## failure. + + # Numerical operations + + proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_add(@)".} + ## Atomically adds a `value` to the atomic integer and returns the + ## original value. + + proc fetchSub*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_sub(@)".} + ## Atomically subtracts a `value` to the atomic integer and returns the + ## original value. + + proc fetchAnd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_and(@)".} + ## Atomically replaces the atomic integer with it's bitwise AND + ## with the specified `value` and returns the original value. + + proc fetchOr*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_or(@)".} + ## Atomically replaces the atomic integer with it's bitwise OR + ## with the specified `value` and returns the original value. + + proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importcpp: "#.fetch_xor(@)".} + ## Atomically replaces the atomic integer with it's bitwise XOR + ## with the specified `value` and returns the original value. + + # Flag operations + + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importcpp: "#.test_and_set(@)".} + ## Atomically sets the atomic flag to true and returns the original value. + + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importcpp: "#.clear(@)".} + ## Atomically sets the value of the atomic flag to false. + + proc fence*(order: MemoryOrder) {.importcpp: "std::atomic_thread_fence(@)".} + ## Ensures memory ordering without using atomic operations. + + proc signalFence*(order: MemoryOrder) {.importcpp: "std::atomic_signal_fence(@)".} + ## Prevents reordering of accesses by the compiler as would fence, but + ## inserts no CPU instructions for memory ordering. + + {.pop.} + +else: + # For the C backend, atomics map to C11 built-ins on GCC and Clang for + # trivial Nim types. Other types are implemented using spin locks. + # This could be overcome by supporting advanced importc-patterns. + + # Since MSVC does not implement C11, we fall back to MS intrinsics + # where available. + + type + Trivial = SomeNumber | bool | enum | ptr | pointer + # A type that is known to be atomic and whose size is known at + # compile time to be 8 bytes or less + + template nonAtomicType*(T: typedesc[Trivial]): untyped = + # Maps types to integers of the same size + when sizeof(T) == 1: int8 + elif sizeof(T) == 2: int16 + elif sizeof(T) == 4: int32 + elif sizeof(T) == 8: int64 + + when defined(vcc): + + # TODO: Trivial types should be volatile and use VC's special volatile + # semantics for store and loads. + + type + MemoryOrder* = enum + moRelaxed + moConsume + moAcquire + moRelease + moAcquireRelease + moSequentiallyConsistent + + Atomic*[T] = object + when T is Trivial: + value: T.nonAtomicType + else: + nonAtomicValue: T + guard: AtomicFlag + + AtomicFlag* = distinct int8 + + {.push header: "<intrin.h>".} + + # MSVC intrinsics + proc interlockedExchange(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8".} + proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange".} + proc interlockedExchange(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64".} + + proc interlockedCompareExchange(location: pointer; desired, expected: int8): int8 {.importc: "_InterlockedCompareExchange8".} + proc interlockedCompareExchange(location: pointer; desired, expected: int16): int16 {.importc: "_InterlockedCompareExchange16".} + proc interlockedCompareExchange(location: pointer; desired, expected: int32): int32 {.importc: "_InterlockedCompareExchange".} + proc interlockedCompareExchange(location: pointer; desired, expected: int64): int64 {.importc: "_InterlockedCompareExchange64".} + + proc interlockedAnd(location: pointer; value: int8): int8 {.importc: "_InterlockedAnd8".} + proc interlockedAnd(location: pointer; value: int16): int16 {.importc: "_InterlockedAnd16".} + proc interlockedAnd(location: pointer; value: int32): int32 {.importc: "_InterlockedAnd".} + proc interlockedAnd(location: pointer; value: int64): int64 {.importc: "_InterlockedAnd64".} + + proc interlockedOr(location: pointer; value: int8): int8 {.importc: "_InterlockedOr8".} + proc interlockedOr(location: pointer; value: int16): int16 {.importc: "_InterlockedOr16".} + proc interlockedOr(location: pointer; value: int32): int32 {.importc: "_InterlockedOr".} + proc interlockedOr(location: pointer; value: int64): int64 {.importc: "_InterlockedOr64".} + + proc interlockedXor(location: pointer; value: int8): int8 {.importc: "_InterlockedXor8".} + proc interlockedXor(location: pointer; value: int16): int16 {.importc: "_InterlockedXor16".} + proc interlockedXor(location: pointer; value: int32): int32 {.importc: "_InterlockedXor".} + proc interlockedXor(location: pointer; value: int64): int64 {.importc: "_InterlockedXor64".} + + proc fence(order: MemoryOrder): int64 {.importc: "_ReadWriteBarrier()".} + proc signalFence(order: MemoryOrder): int64 {.importc: "_ReadWriteBarrier()".} + + {.pop.} + + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool = + interlockedOr(addr(location), 1'i8) == 1'i8 + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) = + discard interlockedAnd(addr(location), 0'i8) + + proc load*[T: Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedOr(addr(location.value), (nonAtomicType(T))0)) + proc store*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + discard interlockedExchange(addr(location.value), cast[nonAtomicType(T)](desired)) + + proc exchange*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedExchange(addr(location.value), cast[int64](desired))) + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + cast[T](interlockedCompareExchange(addr(location.value), cast[nonAtomicType(T)](desired), cast[nonAtomicType(T)](expected))) == expected + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, order) + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + compareExchange(location, expected, desired, success, failure) + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, order) + + proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + var currentValue = location.load() + while not compareExchangeWeak(location, currentValue, currentValue + value): discard + proc fetchSub*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + fetchAdd(location, -value, order) + proc fetchAnd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedAnd(addr(location.value), cast[nonAtomicType(T)](value))) + proc fetchOr*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedOr(addr(location.value), cast[nonAtomicType(T)](value))) + proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](interlockedXor(addr(location.value), cast[nonAtomicType(T)](value))) + + else: + when defined(cpp): + {.push, header: "<atomic>".} + template maybeWrapStd(x: string): string = + "std::" & x + else: + {.push, header: "<stdatomic.h>".} + template maybeWrapStd(x: string): string = + x + + type + MemoryOrder* {.importc: "memory_order".maybeWrapStd.} = enum + moRelaxed + moConsume + moAcquire + moRelease + moAcquireRelease + moSequentiallyConsistent + + when defined(cpp): + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object + + AtomicInt8 {.importc: "std::atomic<NI8>".} = int8 + AtomicInt16 {.importc: "std::atomic<NI16>".} = int16 + AtomicInt32 {.importc: "std::atomic<NI32>".} = int32 + AtomicInt64 {.importc: "std::atomic<NI64>".} = int64 + else: + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object + + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 + + type + AtomicFlag* {.importc: "atomic_flag".maybeWrapStd, size: 1.} = object + + Atomic*[T] = object + when T is Trivial: + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: value: AtomicInt8 + elif sizeof(T) == 2: value: AtomicInt16 + elif sizeof(T) == 4: value: AtomicInt32 + elif sizeof(T) == 8: value: AtomicInt64 + else: + nonAtomicValue: T + guard: AtomicFlag + + #proc init*[T](location: var Atomic[T]; value: T): T {.importcpp: "atomic_init(@)".} + proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc: "atomic_load_explicit".maybeWrapStd.} + proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_store_explicit".maybeWrapStd.} + proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_exchange_explicit".maybeWrapStd.} + proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_strong_explicit".maybeWrapStd.} + proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_weak_explicit".maybeWrapStd.} + + # Numerical operations + proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_add_explicit".maybeWrapStd.} + proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_sub_explicit".maybeWrapStd.} + proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_and_explicit".maybeWrapStd.} + proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_or_explicit".maybeWrapStd.} + proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_xor_explicit".maybeWrapStd.} + + # Flag operations + # var ATOMIC_FLAG_INIT {.importc, nodecl.}: AtomicFlag + # proc init*(location: var AtomicFlag) {.inline.} = location = ATOMIC_FLAG_INIT + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".maybeWrapStd.} + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".maybeWrapStd.} + + proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".maybeWrapStd.} + proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".maybeWrapStd.} + + {.pop.} + + proc load*[T: Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_load_explicit[nonAtomicType(T), typeof(location.value)](addr(location.value), order)) + proc store*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + atomic_store_explicit(addr(location.value), cast[nonAtomicType(T)](desired), order) + proc exchange*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_exchange_explicit(addr(location.value), cast[nonAtomicType(T)](desired), order)) + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + atomic_compare_exchange_strong_explicit(addr(location.value), cast[ptr nonAtomicType(T)](addr(expected)), cast[nonAtomicType(T)](desired), success, failure) + proc compareExchange*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, order) + + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + atomic_compare_exchange_weak_explicit(addr(location.value), cast[ptr nonAtomicType(T)](addr(expected)), cast[nonAtomicType(T)](desired), success, failure) + proc compareExchangeWeak*[T: Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, order) + + # Numerical operations + proc fetchAdd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_add_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchSub*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_sub_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchAnd*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_and_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchOr*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_or_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + proc fetchXor*[T: SomeInteger](location: var Atomic[T]; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) + + template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = + while testAndSet(location.guard, moAcquire): discard + try: + body + finally: + clear(location.guard, moRelease) + + proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + withLock(location, order): + result = location.nonAtomicValue + + proc store*[T: not Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = + withLock(location, order): + location.nonAtomicValue = desired + + proc exchange*[T: not Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = + withLock(location, order): + result = location.nonAtomicValue + location.nonAtomicValue = desired + + proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + withLock(location, success): + if location.nonAtomicValue != expected: + expected = location.nonAtomicValue + return false + expected = desired + swap(location.nonAtomicValue, expected) + return true + + proc compareExchangeWeak*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = + compareExchange(location, expected, desired, success, failure) + + proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchange(location, expected, desired, order, order) + + proc compareExchangeWeak*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = + compareExchangeWeak(location, expected, desired, order, order) + +proc atomicInc*[T: SomeInteger](location: var Atomic[T]; value: T = 1) {.inline.} = + ## Atomically increments the atomic integer by some `value`. + discard location.fetchAdd(value) + +proc atomicDec*[T: SomeInteger](location: var Atomic[T]; value: T = 1) {.inline.} = + ## Atomically decrements the atomic integer by some `value`. + discard location.fetchSub(value) + +proc `+=`*[T: SomeInteger](location: var Atomic[T]; value: T) {.inline.} = + ## Atomically increments the atomic integer by some `value`. + discard location.fetchAdd(value) + +proc `-=`*[T: SomeInteger](location: var Atomic[T]; value: T) {.inline.} = + ## Atomically decrements the atomic integer by some `value`. + discard location.fetchSub(value) diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index 8c87c77df..9bc3fd579 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -7,61 +7,104 @@ # distribution, for details about the copyright. # -## This module implements procs to determine the number of CPUs / cores. +## This module implements a proc to determine the number of CPUs / cores. + +runnableExamples: + doAssert countProcessors() > 0 + include "system/inclrtl" -import strutils, os +when defined(js): + import std/jsffi + proc countProcessorsImpl(): int = + when defined(nodejs): + let jsOs = require("os") + let jsObj = jsOs.cpus().length + else: + # `navigator.hardwareConcurrency` + # works on browser as well as deno. + let navigator{.importcpp.}: JsObject + let jsObj = navigator.hardwareConcurrency + result = jsObj.to int +else: + when defined(posix) and not (defined(macosx) or defined(bsd)): + import std/posix + + when defined(windows): + import std/private/win_getsysteminfo + + when defined(freebsd) or defined(macosx): + {.emit: "#include <sys/types.h>".} + + when defined(openbsd) or defined(netbsd): + {.emit: "#include <sys/param.h>".} -when not defined(windows): - import posix + when defined(macosx) or defined(bsd): + # we HAVE to emit param.h before sysctl.h so we cannot use .header here + # either. The amount of archaic bullshit in Poonix based OSes is just insane. + {.emit: "#include <sys/sysctl.h>".} + {.push nodecl.} + when defined(macosx): + proc sysctlbyname(name: cstring, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + let + CTL_HW{.importc.}: cint + HW_NCPU{.importc.}: cint + proc sysctl[I: static[int]](name: var array[I, cint], namelen: cuint, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + {.pop.} -when defined(linux): - import linux + when defined(genode): + import genode/env -when defined(freebsd) or defined(macosx): - {.emit:"#include <sys/types.h>".} + proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. + importcpp: "@->cpu().affinity_space().total()".} + + when defined(haiku): + type + SystemInfo {.importc: "system_info", header: "<OS.h>".} = object + cpuCount {.importc: "cpu_count".}: uint32 + + proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", + header: "<OS.h>".} + + proc countProcessorsImpl(): int {.inline.} = + when defined(windows): + var + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) + elif defined(macosx) or defined(bsd): + let dest = addr result + var len = sizeof(result).csize_t + when defined(macosx): + # alias of "hw.activecpu" + if sysctlbyname("hw.logicalcpu", dest, len, nil, 0) == 0: + return + var mib = [CTL_HW, HW_NCPU] + if sysctl(mib, 2, dest, len, nil, 0) == 0: + return + elif defined(hpux): + result = mpctl(MPC_GETNUMSPUS, nil, nil) + elif defined(irix): + var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint + result = sysconf(SC_NPROC_ONLN) + elif defined(genode): + result = runtimeEnv.affinitySpaceTotal().int + elif defined(haiku): + var sysinfo: SystemInfo + if getSystemInfo(addr sysinfo) == 0: + result = sysinfo.cpuCount.int + else: + result = sysconf(SC_NPROCESSORS_ONLN) + if result < 0: result = 0 -when defined(openbsd) or defined(netbsd): - {.emit:"#include <sys/param.h>".} -when defined(macosx) or defined(bsd): - # we HAVE to emit param.h before sysctl.h so we cannot use .header here - # either. The amount of archaic bullshit in Poonix based OSes is just insane. - {.emit:"#include <sys/sysctl.h>".} - const - CTL_HW = 6 - HW_AVAILCPU = 25 - HW_NCPU = 3 - proc sysctl(x: ptr array[0..3, cint], y: cint, z: pointer, - a: var csize, b: pointer, c: int): cint {. - importc: "sysctl", nodecl.} proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = - ## returns the numer of the processors/cores the machine has. + ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. - when defined(windows): - var x = getEnv("NUMBER_OF_PROCESSORS") - if x.len > 0: result = parseInt(x.string) - elif defined(macosx) or defined(bsd): - var - mib: array[0..3, cint] - numCPU: int - len: csize - mib[0] = CTL_HW - mib[1] = HW_AVAILCPU - len = sizeof(numCPU) - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - if numCPU < 1: - mib[1] = HW_NCPU - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - result = numCPU - elif defined(hpux): - result = mpctl(MPC_GETNUMSPUS, nil, nil) - elif defined(irix): - var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint - result = sysconf(SC_NPROC_ONLN) - else: - result = sysconf(SC_NPROCESSORS_ONLN) - if result <= 0: result = 1 - + countProcessorsImpl() diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 22598b5c9..bfbf16721 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -9,13 +9,18 @@ ## This module implements a helper for a thread pool to determine whether ## creating a thread is a good idea. +## +## Unstable API. when defined(windows): - import winlean, os, strutils, math + import std/[winlean, os, strutils, math] proc `-`(a, b: FILETIME): int64 = a.rdFileTime - b.rdFileTime elif defined(linux): - from cpuinfo import countProcessors + from std/cpuinfo import countProcessors + +when defined(nimPreviewSlimSystem): + import std/syncio type ThreadPoolAdvice* = enum @@ -45,12 +50,12 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = procKernelDiff = procKernel - s.prevProcKernel procUserDiff = procUser - s.prevProcUser - sysTotal = int(sysKernelDiff + sysUserDiff) - procTotal = int(procKernelDiff + procUserDiff) + sysTotal = sysKernelDiff + sysUserDiff + procTotal = procKernelDiff + procUserDiff # total CPU usage < 85% --> create a new worker thread. # Measurements show that 100% and often even 90% is not reached even # if all my cores are busy. - if sysTotal == 0 or procTotal / sysTotal < 0.85: + if sysTotal == 0 or procTotal.float / sysTotal.float < 0.85: result = doCreateThread s.prevSysKernel = sysKernel s.prevSysUser = sysUser @@ -60,17 +65,20 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = proc fscanf(c: File, frmt: cstring) {.varargs, importc, header: "<stdio.h>".} - var f = open("/proc/loadavg") - var b: float - var busy, total: int - fscanf(f,"%lf %lf %lf %ld/%ld", - addr b, addr b, addr b, addr busy, addr total) - f.close() - let cpus = countProcessors() - if busy-1 < cpus: - result = doCreateThread - elif busy-1 >= cpus*2: - result = doShutdownThread + var f: File + if f.open("/proc/loadavg"): + var b: float + var busy, total: int + fscanf(f,"%lf %lf %lf %ld/%ld", + addr b, addr b, addr b, addr busy, addr total) + f.close() + let cpus = countProcessors() + if busy-1 < cpus: + result = doCreateThread + elif busy-1 >= cpus*2: + result = doShutdownThread + else: + result = doNothing else: result = doNothing else: @@ -78,10 +86,12 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = result = doNothing inc s.calls -when not defined(testing) and isMainModule: +when not defined(testing) and isMainModule and not defined(nimdoc): + import std/random + proc busyLoop() = while true: - discard random(80) + discard rand(80) os.sleep(100) spawn busyLoop() diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 2603835dd..06ed2fe54 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -7,12 +7,25 @@ # distribution, for details about the copyright. # -## Implements Nim's 'spawn'. +{.deprecated: "use the nimble packages `malebolgia`, `taskpools` or `weave` instead".} + +## Implements Nim's `parallel & spawn statements <manual_experimental.html#parallel-amp-spawn>`_. +## +## Unstable API. +## +## See also +## ======== +## * `threads module <typedthreads.html>`_ for basic thread support +## * `locks module <locks.html>`_ for locks and condition variables +## * `asyncdispatch module <asyncdispatch.html>`_ for asynchronous IO when not compileOption("threads"): {.error: "Threadpool requires --threads:on option.".} -import cpuinfo, cpuload, locks +import std/[cpuinfo, cpuload, locks, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, typedthreads, sysatomics] {.push stackTrace:off.} @@ -22,15 +35,15 @@ type L: Lock counter: int -proc createSemaphore(): Semaphore = - initCond(result.c) - initLock(result.L) +proc initSemaphore(cv: var Semaphore) = + initCond(cv.c) + initLock(cv.L) proc destroySemaphore(cv: var Semaphore) {.inline.} = deinitCond(cv.c) deinitLock(cv.L) -proc await(cv: var Semaphore) = +proc blockUntil(cv: var Semaphore) = acquire(cv.L) while cv.counter <= 0: wait(cv.c, cv.L) @@ -43,45 +56,42 @@ proc signal(cv: var Semaphore) = release(cv.L) signal(cv.c) -const CacheLineSize = 32 # true for most archs +const CacheLineSize = 64 # true for most archs type - Barrier {.compilerProc.} = object + Barrier {.compilerproc.} = object entered: int cv: Semaphore # Semaphore takes 3 words at least - when sizeof(int) < 8: - cacheAlign: array[CacheLineSize-4*sizeof(int), byte] - left: int - cacheAlign2: array[CacheLineSize-sizeof(int), byte] - interest: bool ## wether the master is interested in the "all done" event + left {.align(CacheLineSize).}: int + interest {.align(CacheLineSize).} : bool # whether the master is interested in the "all done" event -proc barrierEnter(b: ptr Barrier) {.compilerProc, inline.} = +proc barrierEnter(b: ptr Barrier) {.compilerproc, inline.} = # due to the signaling between threads, it is ensured we are the only # one with access to 'entered' so we don't need 'atomicInc' here: inc b.entered # also we need no 'fence' instructions here as soon 'nimArgsPassingDone' # will be called which already will perform a fence for us. -proc barrierLeave(b: ptr Barrier) {.compilerProc, inline.} = +proc barrierLeave(b: ptr Barrier) {.compilerproc, inline.} = atomicInc b.left when not defined(x86): fence() # We may not have seen the final value of b.entered yet, # so we need to check for >= instead of ==. if b.interest and b.left >= b.entered: signal(b.cv) -proc openBarrier(b: ptr Barrier) {.compilerProc, inline.} = +proc openBarrier(b: ptr Barrier) {.compilerproc, inline.} = b.entered = 0 b.left = 0 b.interest = false -proc closeBarrier(b: ptr Barrier) {.compilerProc.} = +proc closeBarrier(b: ptr Barrier) {.compilerproc.} = fence() if b.left != b.entered: - b.cv = createSemaphore() + b.cv.initSemaphore() fence() b.interest = true fence() - while b.left != b.entered: await(b.cv) + while b.left != b.entered: blockUntil(b.cv) destroySemaphore(b.cv) {.pop.} @@ -89,27 +99,24 @@ proc closeBarrier(b: ptr Barrier) {.compilerProc.} = # ---------------------------------------------------------------------------- type - foreign* = object ## a region that indicates the pointer comes from a - ## foreign thread heap. AwaitInfo = object cv: Semaphore idx: int - FlowVarBase* = ref FlowVarBaseObj ## untyped base class for 'FlowVar[T]' - FlowVarBaseObj = object of RootObj + FlowVarBase* = ref FlowVarBaseObj ## Untyped base class for `FlowVar[T] <#FlowVar>`_. + FlowVarBaseObj {.acyclic.} = object of RootObj ready, usesSemaphore, awaited: bool - cv: Semaphore #\ - # for 'awaitAny' support + cv: Semaphore # for 'blockUntilAny' support ai: ptr AwaitInfo idx: int data: pointer # we incRef and unref it to keep it alive; note this MUST NOT # be RootRef here otherwise the wrong GC keeps track of it! owner: pointer # ptr Worker - FlowVarObj[T] = object of FlowVarBaseObj + FlowVarObj[T] {.acyclic.} = object of FlowVarBaseObj blob: T - FlowVar*{.compilerProc.}[T] = ref FlowVarObj[T] ## a data flow variable + FlowVar*[T] {.compilerproc.} = ref FlowVarObj[T] ## A data flow variable. ToFreeQueue = object len: int @@ -130,12 +137,15 @@ type q: ToFreeQueue readyForTask: Semaphore -proc await*(fv: FlowVarBase) = - ## waits until the value for the flowVar arrives. Usually it is not necessary - ## to call this explicitly. +const threadpoolWaitMs {.intdefine.}: int = 100 + +proc blockUntil*(fv: var FlowVarBaseObj) = + ## Waits until the value for `fv` arrives. + ## + ## Usually it is not necessary to call this explicitly. if fv.usesSemaphore and not fv.awaited: fv.awaited = true - await(fv.cv) + blockUntil(fv.cv) destroySemaphore(fv.cv) proc selectWorker(w: ptr Worker; fn: WorkerProc; data: pointer): bool = @@ -143,13 +153,13 @@ proc selectWorker(w: ptr Worker; fn: WorkerProc; data: pointer): bool = w.data = data w.f = fn signal(w.taskArrived) - await(w.taskStarted) + blockUntil(w.taskStarted) result = true proc cleanFlowVars(w: ptr Worker) = let q = addr(w.q) acquire(q.lock) - for i in 0 .. <q.len: + for i in 0 ..< q.len: GC_unref(cast[RootRef](q.data[i])) #echo "GC_unref" q.len = 0 @@ -168,12 +178,21 @@ proc wakeupWorkerToProcessQueue(w: ptr Worker) = signal(w.q.empty) signal(w.taskArrived) -proc finished(fv: FlowVarBase) = - doAssert fv.ai.isNil, "flowVar is still attached to an 'awaitAny'" +proc attach(fv: FlowVarBase; i: int): bool = + acquire(fv.cv.L) + if fv.cv.counter <= 0: + fv.idx = i + result = true + else: + result = false + release(fv.cv.L) + +proc finished(fv: var FlowVarBaseObj) = + doAssert fv.ai.isNil, "flowVar is still attached to an 'blockUntilAny'" # we have to protect against the rare cases where the owner of the flowVar # simply disregards the flowVar and yet the "flowVar" has not yet written # anything to it: - await(fv) + blockUntil(fv) if fv.data.isNil: return let owner = cast[ptr Worker](fv.owner) let q = addr(owner.q) @@ -182,23 +201,27 @@ proc finished(fv: FlowVarBase) = #echo "EXHAUSTED!" release(q.lock) wakeupWorkerToProcessQueue(owner) - await(q.empty) + blockUntil(q.empty) acquire(q.lock) q.data[q.len] = cast[pointer](fv.data) inc q.len release(q.lock) fv.data = nil + # the worker thread waits for "data" to be set to nil before shutting down + owner.data = nil -proc fvFinalizer[T](fv: FlowVar[T]) = finished(fv) +proc `=destroy`[T](fv: var FlowVarObj[T]) = + finished(fv) + `=destroy`(fv.blob) -proc nimCreateFlowVar[T](): FlowVar[T] {.compilerProc.} = - new(result, fvFinalizer) +proc nimCreateFlowVar[T](): FlowVar[T] {.compilerproc.} = + new(result) -proc nimFlowVarCreateSemaphore(fv: FlowVarBase) {.compilerProc.} = - fv.cv = createSemaphore() +proc nimFlowVarCreateSemaphore(fv: FlowVarBase) {.compilerproc.} = + fv.cv.initSemaphore() fv.usesSemaphore = true -proc nimFlowVarSignal(fv: FlowVarBase) {.compilerProc.} = +proc nimFlowVarSignal(fv: FlowVarBase) {.compilerproc.} = if fv.ai != nil: acquire(fv.ai.cv.L) fv.ai.idx = fv.idx @@ -209,68 +232,74 @@ proc nimFlowVarSignal(fv: FlowVarBase) {.compilerProc.} = signal(fv.cv) proc awaitAndThen*[T](fv: FlowVar[T]; action: proc (x: T) {.closure.}) = - ## blocks until the ``fv`` is available and then passes its value - ## to ``action``. Note that due to Nim's parameter passing semantics this - ## means that ``T`` doesn't need to be copied and so ``awaitAndThen`` can - ## sometimes be more efficient than ``^``. - await(fv) - when T is string or T is seq: + ## Blocks until `fv` is available and then passes its value + ## to `action`. + ## + ## Note that due to Nim's parameter passing semantics, this + ## means that `T` doesn't need to be copied, so `awaitAndThen` can + ## sometimes be more efficient than the `^ proc <#^,FlowVar[T]>`_. + blockUntil(fv[]) + when defined(nimV2): + action(fv.blob) + elif T is string or T is seq: action(cast[T](fv.data)) elif T is ref: {.error: "'awaitAndThen' not available for FlowVar[ref]".} else: action(fv.blob) - finished(fv) - -proc unsafeRead*[T](fv: FlowVar[ref T]): foreign ptr T = - ## blocks until the value is available and then returns this value. - await(fv) - result = cast[foreign ptr T](fv.data) + finished(fv[]) -proc `^`*[T](fv: FlowVar[ref T]): ref T = - ## blocks until the value is available and then returns this value. - await(fv) - let src = cast[ref T](fv.data) - deepCopy result, src +proc unsafeRead*[T](fv: FlowVar[ref T]): ptr T = + ## Blocks until the value is available and then returns this value. + blockUntil(fv[]) + when defined(nimV2): + result = cast[ptr T](fv.blob) + else: + result = cast[ptr T](fv.data) + finished(fv[]) proc `^`*[T](fv: FlowVar[T]): T = - ## blocks until the value is available and then returns this value. - await(fv) - when T is string or T is seq: - # XXX closures? deepCopy? - result = cast[T](fv.data) + ## Blocks until the value is available and then returns this value. + blockUntil(fv[]) + when not defined(nimV2) and (T is string or T is seq or T is ref): + deepCopy result, cast[T](fv.data) else: result = fv.blob + finished(fv[]) -proc awaitAny*(flowVars: openArray[FlowVarBase]): int = - ## awaits any of the given flowVars. Returns the index of one flowVar for - ## which a value arrived. A flowVar only supports one call to 'awaitAny' at - ## the same time. That means if you await([a,b]) and await([b,c]) the second - ## call will only await 'c'. If there is no flowVar left to be able to wait - ## on, -1 is returned. - ## **Note**: This results in non-deterministic behaviour and so should be - ## avoided. +proc blockUntilAny*(flowVars: openArray[FlowVarBase]): int = + ## Awaits any of the given `flowVars`. Returns the index of one `flowVar` + ## for which a value arrived. + ## + ## A `flowVar` only supports one call to `blockUntilAny` at the same time. + ## That means if you `blockUntilAny([a,b])` and `blockUntilAny([b,c])` + ## the second call will only block until `c`. If there is no `flowVar` left + ## to be able to wait on, -1 is returned. + ## + ## **Note:** This results in non-deterministic behaviour and should be avoided. var ai: AwaitInfo - ai.cv = createSemaphore() + ai.cv.initSemaphore() var conflicts = 0 + result = -1 for i in 0 .. flowVars.high: if cas(addr flowVars[i].ai, nil, addr ai): - flowVars[i].idx = i + if not attach(flowVars[i], i): + result = i + break else: inc conflicts if conflicts < flowVars.len: - await(ai.cv) - result = ai.idx + if result < 0: + blockUntil(ai.cv) + result = ai.idx for i in 0 .. flowVars.high: discard cas(addr flowVars[i].ai, addr ai, nil) - else: - result = -1 destroySemaphore(ai.cv) proc isReady*(fv: FlowVarBase): bool = - ## Determines whether the specified ``FlowVarBase``'s value is available. + ## Determines whether the specified `FlowVarBase`'s value is available. ## - ## If ``true`` awaiting ``fv`` will not block. + ## If `true`, awaiting `fv` will not block. if fv.usesSemaphore and not fv.awaited: acquire(fv.cv.L) result = fv.cv.counter > 0 @@ -278,42 +307,74 @@ proc isReady*(fv: FlowVarBase): bool = else: result = true -proc nimArgsPassingDone(p: pointer) {.compilerProc.} = +proc nimArgsPassingDone(p: pointer) {.compilerproc.} = let w = cast[ptr Worker](p) signal(w.taskStarted) const - MaxThreadPoolSize* = 256 ## maximal size of the thread pool. 256 threads - ## should be good enough for anybody ;-) - MaxDistinguishedThread* = 32 ## maximal number of "distinguished" threads. + MaxThreadPoolSize* {.intdefine.} = 256 ## Maximum size of the thread pool. 256 threads + ## should be good enough for anybody ;-) + MaxDistinguishedThread* {.intdefine.} = 32 ## Maximum number of "distinguished" threads. type - ThreadId* = range[0..MaxDistinguishedThread-1] + ThreadId* = range[0..MaxDistinguishedThread-1] ## A thread identifier. var currentPoolSize: int maxPoolSize = MaxThreadPoolSize minPoolSize = 4 - gSomeReady = createSemaphore() + gSomeReady: Semaphore readyWorker: ptr Worker +# A workaround for recursion deadlock issue +# https://github.com/nim-lang/Nim/issues/4597 +var + numSlavesLock: Lock + numSlavesRunning {.guard: numSlavesLock.}: int + numSlavesWaiting {.guard: numSlavesLock.}: int + isSlave {.threadvar.}: bool + +numSlavesLock.initLock + +gSomeReady.initSemaphore() + proc slave(w: ptr Worker) {.thread.} = + isSlave = true while true: + if w.shutdown: + w.shutdown = false + atomicDec currentPoolSize + while true: + if w.data != nil: + sleep(threadpoolWaitMs) + else: + # The flowvar finalizer ("finished()") set w.data to nil, so we can + # safely terminate the thread. + # + # TODO: look for scenarios in which the flowvar is never finalized, so + # a shut down thread gets stuck in this loop until the main thread exits. + break + break when declared(atomicStoreN): atomicStoreN(addr(w.ready), true, ATOMIC_SEQ_CST) else: w.ready = true readyWorker = w signal(gSomeReady) - await(w.taskArrived) + blockUntil(w.taskArrived) # XXX Somebody needs to look into this (why does this assertion fail # in Visual Studio?) - when not defined(vcc): assert(not w.ready) + when not defined(vcc) and not defined(tcc): assert(not w.ready) + + withLock numSlavesLock: + inc numSlavesRunning + w.f(w, w.data) + + withLock numSlavesLock: + dec numSlavesRunning + if w.q.len != 0: w.cleanFlowVars - if w.shutdown: - w.shutdown = false - atomicDec currentPoolSize proc distinguishedSlave(w: ptr Worker) {.thread.} = while true: @@ -322,28 +383,28 @@ proc distinguishedSlave(w: ptr Worker) {.thread.} = else: w.ready = true signal(w.readyForTask) - await(w.taskArrived) + blockUntil(w.taskArrived) assert(not w.ready) w.f(w, w.data) if w.q.len != 0: w.cleanFlowVars var - workers: array[MaxThreadPoolSize, TThread[ptr Worker]] + workers: array[MaxThreadPoolSize, Thread[ptr Worker]] workersData: array[MaxThreadPoolSize, Worker] - distinguished: array[MaxDistinguishedThread, TThread[ptr Worker]] + distinguished: array[MaxDistinguishedThread, Thread[ptr Worker]] distinguishedData: array[MaxDistinguishedThread, Worker] when defined(nimPinToCpu): var gCpus: Natural proc setMinPoolSize*(size: range[1..MaxThreadPoolSize]) = - ## sets the minimal thread pool size. The default value of this is 4. + ## Sets the minimum thread pool size. The default value of this is 4. minPoolSize = size proc setMaxPoolSize*(size: range[1..MaxThreadPoolSize]) = - ## sets the maximal thread pool size. The default value of this - ## is ``MaxThreadPoolSize``. + ## Sets the maximum thread pool size. The default value of this + ## is `MaxThreadPoolSize <#MaxThreadPoolSize>`_. maxPoolSize = size if currentPoolSize > maxPoolSize: for i in maxPoolSize..currentPoolSize-1: @@ -354,10 +415,10 @@ when defined(nimRecursiveSpawn): var localThreadId {.threadvar.}: int proc activateWorkerThread(i: int) {.noinline.} = - workersData[i].taskArrived = createSemaphore() - workersData[i].taskStarted = createSemaphore() + workersData[i].taskArrived.initSemaphore() + workersData[i].taskStarted.initSemaphore() workersData[i].initialized = true - workersData[i].q.empty = createSemaphore() + workersData[i].q.empty.initSemaphore() initLock(workersData[i].q.lock) createThread(workers[i], slave, addr(workersData[i])) when defined(nimRecursiveSpawn): @@ -366,12 +427,12 @@ proc activateWorkerThread(i: int) {.noinline.} = if gCpus > 0: pinToCpu(workers[i], i mod gCpus) proc activateDistinguishedThread(i: int) {.noinline.} = - distinguishedData[i].taskArrived = createSemaphore() - distinguishedData[i].taskStarted = createSemaphore() + distinguishedData[i].taskArrived.initSemaphore() + distinguishedData[i].taskStarted.initSemaphore() distinguishedData[i].initialized = true - distinguishedData[i].q.empty = createSemaphore() + distinguishedData[i].q.empty.initSemaphore() initLock(distinguishedData[i].q.lock) - distinguishedData[i].readyForTask = createSemaphore() + distinguishedData[i].readyForTask.initSemaphore() createThread(distinguished[i], distinguishedSlave, addr(distinguishedData[i])) proc setup() = @@ -380,40 +441,51 @@ proc setup() = gCpus = p currentPoolSize = min(p, MaxThreadPoolSize) readyWorker = addr(workersData[0]) - for i in 0.. <currentPoolSize: activateWorkerThread(i) + for i in 0..<currentPoolSize: activateWorkerThread(i) proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a 'spawn' or a direct call is - ## preferable. If it returns 'true' a 'spawn' may make sense. In general - ## it is not necessary to call this directly; use 'spawnX' instead. + ## Use this proc to determine quickly if a `spawn` or a direct call is + ## preferable. + ## + ## If it returns `true`, a `spawn` may make sense. In general + ## it is not necessary to call this directly; use the `spawnX template + ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: expr): expr {.magic: "Spawn".} - ## always spawns a new task, so that the 'call' is never executed on - ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has a return type that is either 'void' or compatible - ## with ``FlowVar[T]``. - -proc pinnedSpawn*(id: ThreadId; call: expr): expr {.magic: "Spawn".} - ## always spawns a new task on the worker thread with ``id``, so that - ## the 'call' is **always** executed on - ## the this thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has a return type that is either 'void' or compatible - ## with ``FlowVar[T]``. - -template spawnX*(call: expr): expr = - ## spawns a new task if a CPU core is ready, otherwise executes the - ## call in the calling thread. Usually it is advised to - ## use 'spawn' in order to not block the producer for an unknown - ## amount of time. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has a return type that is either 'void' or compatible - ## with ``FlowVar[T]``. +proc spawn*(call: sink typed) {.magic: "Spawn".} = + ## Always spawns a new task, so that the `call` is never executed on + ## the calling thread. + ## + ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a + ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn3` internally" + +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} = + ## Always spawns a new task on the worker thread with `id`, so that + ## the `call` is **always** executed on the thread. + ## + ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a + ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn4` internally" + +template spawnX*(call) = + ## Spawns a new task if a CPU core is ready, otherwise executes the + ## call in the calling thread. + ## + ## Usually, it is advised to use the `spawn proc <#spawn,sinktyped>`_ + ## in order to not block the producer for an unknown amount of time. + ## + ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a + ## return type that is either 'void' or compatible with `FlowVar[T]`. (if preferSpawn(): spawn call else: call) -proc parallel*(body: stmt) {.magic: "Parallel".} - ## a parallel section can be used to execute a block in parallel. ``body`` - ## has to be in a DSL that is a particular subset of the language. Please - ## refer to the manual for further information. +proc parallel*(body: untyped) {.magic: "Parallel".} + ## A parallel section can be used to execute a block in parallel. + ## + ## `body` has to be in a DSL that is a particular subset of the language. + ## + ## Please refer to `the manual <manual_experimental.html#parallel-amp-spawn>`_ + ## for further information. var state: ThreadPoolState @@ -421,18 +493,28 @@ var initLock stateLock -proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = +proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerproc.} = # implementation of 'spawn' that is used by the code generator. while true: if selectWorker(readyWorker, fn, data): return - for i in 0.. <currentPoolSize: + for i in 0..<currentPoolSize: if selectWorker(addr(workersData[i]), fn, data): return + # determine what to do, but keep in mind this is expensive too: # state.calls < maxPoolSize: warmup phase # (state.calls and 127) == 0: periodic check if state.calls < maxPoolSize or (state.calls and 127) == 0: # ensure the call to 'advice' is atomic: if tryAcquire(stateLock): + if currentPoolSize < minPoolSize: + if not workersData[currentPoolSize].initialized: + activateWorkerThread(currentPoolSize) + let w = addr(workersData[currentPoolSize]) + atomicInc currentPoolSize + if selectWorker(w, fn, data): + release(stateLock) + return + case advice(state) of doNothing: discard of doCreateThread: @@ -460,37 +542,65 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = # on the current thread instead. var self = addr(workersData[localThreadId-1]) fn(self, data) - await(self.taskStarted) + blockUntil(self.taskStarted) return - else: - await(gSomeReady) - else: - await(gSomeReady) + + if isSlave: + # Run under lock until `numSlavesWaiting` increment to avoid a + # race (otherwise two last threads might start waiting together) + withLock numSlavesLock: + if numSlavesRunning <= numSlavesWaiting + 1: + # All the other slaves are waiting + # If we wait now, we-re deadlocked until + # an external spawn happens ! + if currentPoolSize < maxPoolSize: + if not workersData[currentPoolSize].initialized: + activateWorkerThread(currentPoolSize) + let w = addr(workersData[currentPoolSize]) + atomicInc currentPoolSize + if selectWorker(w, fn, data): + return + else: + # There is no place in the pool. We're deadlocked. + # echo "Deadlock!" + discard + + inc numSlavesWaiting + + blockUntil(gSomeReady) + + if isSlave: + withLock numSlavesLock: + dec numSlavesWaiting var - distinguishedLock: TLock + distinguishedLock: Lock initLock distinguishedLock -proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} = +proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerproc.} = acquire(distinguishedLock) if not distinguishedData[id].initialized: activateDistinguishedThread(id) release(distinguishedLock) while true: if selectWorker(addr(distinguishedData[id]), fn, data): break - await(distinguishedData[id].readyForTask) + blockUntil(distinguishedData[id].readyForTask) proc sync*() = - ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate - ## waiting, you have to use an explicit barrier. + ## A simple barrier to wait for all `spawn`ed tasks. + ## + ## If you need more elaborate waiting, you have to use an explicit barrier. while true: var allReady = true - for i in 0 .. <currentPoolSize: + for i in 0 ..< currentPoolSize: if not allReady: break allReady = allReady and workersData[i].ready if allReady: break - await(gSomeReady) + sleep(threadpoolWaitMs) + # We cannot "blockUntil(gSomeReady)" because workers may be shut down between + # the time we establish that some are not "ready" and the time we wait for a + # "signal(gSomeReady)" from inside "slave()" that can never come. setup() diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 8090cd49d..f628aaf6b 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -9,59 +9,73 @@ ## This module implements helper procs for parsing Cookies. -import strtabs, times +import std/[strtabs, times, options] + +when defined(nimPreviewSlimSystem): + import std/assertions + + +type + SameSite* {.pure.} = enum ## The SameSite cookie attribute. + ## `Default` means that `setCookie` + ## proc will not set `SameSite` attribute. + Default, None, Lax, Strict proc parseCookies*(s: string): StringTableRef = - ## parses cookies into a string table. + ## Parses cookies into a string table. + ## + ## The proc is meant to parse the Cookie header set by a client, not the + ## "Set-Cookie" header set by servers. + runnableExamples: + import std/strtabs + let cookieJar = parseCookies("a=1; foo=bar") + assert cookieJar["a"] == "1" + assert cookieJar["foo"] == "bar" + result = newStringTable(modeCaseInsensitive) var i = 0 while true: - while s[i] == ' ' or s[i] == '\t': inc(i) + while i < s.len and (s[i] == ' ' or s[i] == '\t'): inc(i) var keystart = i - while s[i] != '=' and s[i] != '\0': inc(i) + while i < s.len and s[i] != '=': inc(i) var keyend = i-1 - if s[i] == '\0': break + if i >= s.len: break inc(i) # skip '=' var valstart = i - while s[i] != ';' and s[i] != '\0': inc(i) + while i < s.len and s[i] != ';': inc(i) result[substr(s, keystart, keyend)] = substr(s, valstart, i-1) - if s[i] == '\0': break + if i >= s.len: break inc(i) # skip ';' proc setCookie*(key, value: string, domain = "", path = "", expires = "", noName = false, - secure = false, httpOnly = false): string = + secure = false, httpOnly = false, + maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of - ## ``Set-Cookie: key=value; Domain=...; ...`` + ## `Set-Cookie: key=value; Domain=...; ...` + ## + ## .. tip:: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value if domain != "": result.add("; Domain=" & domain) if path != "": result.add("; Path=" & path) if expires != "": result.add("; Expires=" & expires) - if secure: result.add("; secure") + if secure: result.add("; Secure") if httpOnly: result.add("; HttpOnly") + if maxAge.isSome: result.add("; Max-Age=" & $maxAge.unsafeGet) -proc setCookie*(key, value: string, expires: TimeInfo, + if sameSite != SameSite.Default: + if sameSite == SameSite.None: + doAssert secure, "Cookies with SameSite=None must specify the Secure attribute!" + result.add("; SameSite=" & $sameSite) + +proc setCookie*(key, value: string, expires: DateTime|Time, domain = "", path = "", noName = false, - secure = false, httpOnly = false): string = + secure = false, httpOnly = false, + maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of - ## ``Set-Cookie: key=value; Domain=...; ...`` - ## - ## **Note:** UTC is assumed as the timezone for ``expires``. - return setCookie(key, value, domain, path, - format(expires, "ddd',' dd MMM yyyy HH:mm:ss 'UTC'"), - noname, secure, httpOnly) - -when isMainModule: - var tim = Time(int(getTime()) + 76 * (60 * 60 * 24)) - - let cookie = setCookie("test", "value", tim.getGMTime()) - when not defined(testing): - echo cookie - let start = "Set-Cookie: test=value; Expires=" - assert cookie[0..start.high] == start - - let table = parseCookies("uid=1; kp=2") - assert table["uid"] == "1" - assert table["kp"] == "2" + ## `Set-Cookie: key=value; Domain=...; ...` + result = setCookie(key, value, domain, path, + format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), + noName, secure, httpOnly, maxAge, sameSite) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index c5724f26f..24836e316 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -7,137 +7,339 @@ # distribution, for details about the copyright. # -when not defined(nimCoroutines): - {.error: "Coroutines require -d:nimCoroutines".} +## Nim coroutines implementation, supports several context switching methods: +## ======== ============ +## ucontext available on unix and alike (default) +## setjmp available on unix and alike (x86/64 only) +## fibers available and required on windows. +## ======== ============ +## +## -d:nimCoroutines Required to build this module. +## -d:nimCoroutinesUcontext Use ucontext backend. +## -d:nimCoroutinesSetjmp Use setjmp backend. +## -d:nimCoroutinesSetjmpBundled Use bundled setjmp implementation. +## +## Unstable API. -import os, times -import macros -import arch -import lists +import system/coro_detection + +when not nimCoroutines and not defined(nimdoc): + when defined(noNimCoroutines): + {.error: "Coroutines can not be used with -d:noNimCoroutines".} + else: + {.error: "Coroutines require -d:nimCoroutines".} + +import std/[os, lists] +include system/timers + +when defined(nimPreviewSlimSystem): + import std/assertions const defaultStackSize = 512 * 1024 +const useOrcArc = defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc) -type Coroutine = ref object - # prev: ptr Coroutine - # next: ptr Coroutine - ctx: JmpBuf - fn: proc() - started: bool - lastRun: float - sleepTime: float - stack: pointer - stacksize: int - -var coroutines = initDoublyLinkedList[Coroutine]() -var current: Coroutine -var mainCtx: JmpBuf - - -proc GC_addStack(starts: pointer) {.cdecl, importc.} -proc GC_removeStack(starts: pointer) {.cdecl, importc.} -proc GC_setCurrentStack(starts, pos: pointer) {.cdecl, importc.} - -proc start*(c: proc(), stacksize: int=defaultStackSize) = - ## Adds coroutine to event loop. It does not run immediately. - var coro = Coroutine() - coro.fn = c - while coro.stack == nil: - coro.stack = alloc0(stacksize) - coro.stacksize = stacksize - coroutines.append(coro) - -{.push stackTrace: off.} -proc suspend*(sleepTime: float=0) = - ## Stops coroutine execution and resumes no sooner than after ``sleeptime`` seconds. +when useOrcArc: + proc nimGC_setStackBottom*(theStackBottom: pointer) = discard + +proc GC_addStack(bottom: pointer) {.cdecl, importc.} +proc GC_removeStack(bottom: pointer) {.cdecl, importc.} +proc GC_setActiveStack(bottom: pointer) {.cdecl, importc.} +proc GC_getActiveStack() : pointer {.cdecl, importc.} + +const + CORO_BACKEND_UCONTEXT = 0 + CORO_BACKEND_SETJMP = 1 + CORO_BACKEND_FIBERS = 2 + +when defined(windows): + const coroBackend = CORO_BACKEND_FIBERS + when defined(nimCoroutinesUcontext): + {.warning: "ucontext coroutine backend is not available on windows, defaulting to fibers.".} + when defined(nimCoroutinesSetjmp): + {.warning: "setjmp coroutine backend is not available on windows, defaulting to fibers.".} +elif defined(haiku) or defined(openbsd): + const coroBackend = CORO_BACKEND_SETJMP + when defined(nimCoroutinesUcontext): + {.warning: "ucontext coroutine backend is not available on haiku, defaulting to setjmp".} +elif defined(nimCoroutinesSetjmp) or defined(nimCoroutinesSetjmpBundled): + const coroBackend = CORO_BACKEND_SETJMP +else: + const coroBackend = CORO_BACKEND_UCONTEXT + +when coroBackend == CORO_BACKEND_FIBERS: + import std/winlean + type + Context = pointer + +elif coroBackend == CORO_BACKEND_UCONTEXT: + type + stack_t {.importc, header: "<ucontext.h>".} = object + ss_sp: pointer + ss_flags: int + ss_size: int + + ucontext_t {.importc, header: "<ucontext.h>".} = object + uc_link: ptr ucontext_t + uc_stack: stack_t + + Context = ucontext_t + + proc getcontext(context: var ucontext_t): int32 {.importc, + header: "<ucontext.h>".} + proc setcontext(context: var ucontext_t): int32 {.importc, + header: "<ucontext.h>".} + proc swapcontext(fromCtx, toCtx: var ucontext_t): int32 {.importc, + header: "<ucontext.h>".} + proc makecontext(context: var ucontext_t, fn: pointer, argc: int32) {.importc, + header: "<ucontext.h>", varargs.} + +elif coroBackend == CORO_BACKEND_SETJMP: + proc coroExecWithStack*(fn: pointer, stack: pointer) {.noreturn, + importc: "narch_$1", fastcall.} + when defined(amd64): + {.compile: "../arch/x86/amd64.S".} + elif defined(i386): + {.compile: "../arch/x86/i386.S".} + else: + # coroExecWithStack is defined in assembly. To support other platforms + # please provide implementation of this procedure. + {.error: "Unsupported architecture.".} + + when defined(nimCoroutinesSetjmpBundled): + # Use setjmp/longjmp implementation shipped with compiler. + when defined(amd64): + type + JmpBuf = array[0x50 + 0x10, uint8] + elif defined(i386): + type + JmpBuf = array[0x1C, uint8] + else: + # Bundled setjmp/longjmp are defined in assembly. To support other + # platforms please provide implementations of these procedures. + {.error: "Unsupported architecture.".} + + proc setjmp(ctx: var JmpBuf): int {.importc: "narch_$1".} + proc longjmp(ctx: JmpBuf, ret = 1) {.importc: "narch_$1".} + else: + # Use setjmp/longjmp implementation provided by the system. + type + JmpBuf {.importc: "jmp_buf", header: "<setjmp.h>".} = object + + proc setjmp(ctx: var JmpBuf): int {.importc, header: "<setjmp.h>".} + proc longjmp(ctx: JmpBuf, ret = 1) {.importc, header: "<setjmp.h>".} + + type + Context = JmpBuf + +when defined(unix): + # GLibc fails with "*** longjmp causes uninitialized stack frame ***" because + # our custom stacks are not initialized to a magic value. + when defined(osx): + # workaround: error: The deprecated ucontext routines require _XOPEN_SOURCE to be defined + const extra = " -D_XOPEN_SOURCE" + else: + const extra = "" + {.passc: "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0" & extra.} + +const + CORO_CREATED = 0 + CORO_EXECUTING = 1 + CORO_FINISHED = 2 + +type + Stack {.pure.} = object + top: pointer # Top of the stack. Pointer used for deallocating stack if we own it. + bottom: pointer # Very bottom of the stack, acts as unique stack identifier. + size: int + + Coroutine {.pure.} = object + execContext: Context + fn: proc() + state: int + lastRun: Ticks + sleepTime: float + stack: Stack + reference: CoroutineRef + + CoroutinePtr = ptr Coroutine + + CoroutineRef* = ref object + ## CoroutineRef holds a pointer to actual coroutine object. Public API always returns + ## CoroutineRef instead of CoroutinePtr in order to allow holding a reference to coroutine + ## object while it can be safely deallocated by coroutine scheduler loop. In this case + ## Coroutine.reference.coro is set to nil. Public API checks for it being nil and + ## gracefully fails if it is nil. + coro: CoroutinePtr + + CoroutineLoopContext = ref object + coroutines: DoublyLinkedList[CoroutinePtr] + current: DoublyLinkedNode[CoroutinePtr] + loop: Coroutine + ncbottom: pointer # non coroutine stack botttom + +var ctx {.threadvar.}: CoroutineLoopContext + +proc getCurrent(): CoroutinePtr = + ## Returns current executing coroutine object. + var node = ctx.current + if node != nil: + return node.value + return nil + +proc initialize() = + ## Initializes coroutine state of current thread. + if ctx == nil: + ctx = CoroutineLoopContext() + ctx.coroutines = initDoublyLinkedList[CoroutinePtr]() + ctx.loop = Coroutine() + ctx.loop.state = CORO_EXECUTING + when not useOrcArc: + ctx.ncbottom = GC_getActiveStack() + when coroBackend == CORO_BACKEND_FIBERS: + ctx.loop.execContext = ConvertThreadToFiberEx(nil, FIBER_FLAG_FLOAT_SWITCH) + +proc runCurrentTask() + +proc switchTo(current, to: CoroutinePtr) = + ## Switches execution from `current` into `to` context. + to.lastRun = getTicks() + # Update position of current stack so gc invoked from another stack knows how much to scan. + when not useOrcArc: + GC_setActiveStack(current.stack.bottom) + nimGC_setStackBottom(current.stack.bottom) + var frame = getFrameState() + block: + # Execution will switch to another fiber now. We do not need to update current stack + when coroBackend == CORO_BACKEND_FIBERS: + SwitchToFiber(to.execContext) + elif coroBackend == CORO_BACKEND_UCONTEXT: + discard swapcontext(current.execContext, to.execContext) + elif coroBackend == CORO_BACKEND_SETJMP: + var res = setjmp(current.execContext) + if res == 0: + if to.state == CORO_EXECUTING: + # Coroutine is resumed. + longjmp(to.execContext, 1) + elif to.state == CORO_CREATED: + # Coroutine is started. + coroExecWithStack(runCurrentTask, to.stack.bottom) + #raiseAssert "unreachable" + else: + {.error: "Invalid coroutine backend set.".} + # Execution was just resumed. Restore frame information and set active stack. + setFrameState(frame) + when not useOrcArc: + GC_setActiveStack(current.stack.bottom) + nimGC_setStackBottom(ctx.ncbottom) + +proc suspend*(sleepTime: float = 0) = + ## Stops coroutine execution and resumes no sooner than after `sleeptime` seconds. ## Until then other coroutines are executed. - ## - ## This is similar to a `yield`:idx:, or a `yieldFrom`:idx in Python. - var oldFrame = getFrame() - var sp {.volatile.}: pointer - GC_setCurrentStack(current.stack, cast[pointer](addr sp)) + var current = getCurrent() current.sleepTime = sleepTime - current.lastRun = epochTime() - if setjmp(current.ctx) == 0: - longjmp(mainCtx, 1) - setFrame(oldFrame) -{.pop.} + nimGC_setStackBottom(ctx.ncbottom) + switchTo(current, addr(ctx.loop)) + +proc runCurrentTask() = + ## Starts execution of current coroutine and updates it's state through coroutine's life. + var sp {.volatile.}: pointer + sp = addr(sp) + block: + var current = getCurrent() + current.stack.bottom = sp + nimGC_setStackBottom(current.stack.bottom) + # Execution of new fiber just started. Since it was entered not through `switchTo` we + # have to set active stack here as well. GC_removeStack() has to be called in main loop + # because we still need stack available in final suspend(0) call from which we will not + # return. + when not useOrcArc: + GC_addStack(sp) + # Activate current stack because we are executing in a new coroutine. + GC_setActiveStack(sp) + current.state = CORO_EXECUTING + try: + current.fn() # Start coroutine execution + except: + echo "Unhandled exception in coroutine." + writeStackTrace() + current.state = CORO_FINISHED + nimGC_setStackBottom(ctx.ncbottom) + suspend(0) # Exit coroutine without returning from coroExecWithStack() + raiseAssert "unreachable" + +proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discardable.} = + ## Schedule coroutine for execution. It does not run immediately. + if ctx == nil: + initialize() + + var coro: CoroutinePtr + when coroBackend == CORO_BACKEND_FIBERS: + coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine))) + coro.execContext = CreateFiberEx(stacksize, stacksize, + FIBER_FLAG_FLOAT_SWITCH, + (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) + else: + coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) + when coroBackend == CORO_BACKEND_UCONTEXT: + discard getcontext(coro.execContext) + coro.execContext.uc_stack.ss_sp = coro.stack.top + coro.execContext.uc_stack.ss_size = stacksize + coro.execContext.uc_link = addr(ctx.loop.execContext) + makecontext(coro.execContext, runCurrentTask, 0) + coro.fn = c + coro.stack.size = stacksize + coro.state = CORO_CREATED + coro.reference = CoroutineRef(coro: coro) + ctx.coroutines.append(coro) + return coro.reference proc run*() = - ## Starts main event loop which exits when all coroutines exit. Calling this proc - ## starts execution of first coroutine. - var node = coroutines.head + ## Starts main coroutine scheduler loop which exits when all coroutines exit. + ## Calling this proc starts execution of first coroutine. + initialize() + ctx.current = ctx.coroutines.head var minDelay: float = 0 - var frame: PFrame - while node != nil: - var coro = node.value - current = coro - os.sleep(int(minDelay * 1000)) + while ctx.current != nil: + var current = getCurrent() - var remaining = coro.sleepTime - (epochTime() - coro.lastRun); + var remaining = current.sleepTime - (float(getTicks() - current.lastRun) / 1_000_000_000) if remaining <= 0: - remaining = 0 - let res = setjmp(mainCtx) - if res == 0: - frame = getFrame() - if coro.started: # coroutine resumes - longjmp(coro.ctx, 1) - else: - coro.started = true # coroutine starts - var stackEnd = cast[pointer](cast[ByteAddress](coro.stack) + coro.stacksize) - GC_addStack(coro.stack) - coroSwitchStack(stackEnd) - coro.fn() - coroRestoreStack() - GC_removeStack(coro.stack) - var next = node.prev - coroutines.remove(node) - dealloc(coro.stack) - node = next - setFrame(frame) - else: - setFrame(frame) - - elif remaining > 0: + # Save main loop context. Suspending coroutine will resume after this statement with + switchTo(addr(ctx.loop), current) + else: if minDelay > 0 and remaining > 0: minDelay = min(remaining, minDelay) else: minDelay = remaining - if node == nil or node.next == nil: - node = coroutines.head + if current.state == CORO_FINISHED: + var next = ctx.current.prev + if next == nil: + # If first coroutine ends then `prev` is nil even if more coroutines + # are to be scheduled. + next = ctx.current.next + current.reference.coro = nil + ctx.coroutines.remove(ctx.current) + when not useOrcArc: + GC_removeStack(current.stack.bottom) + when coroBackend == CORO_BACKEND_FIBERS: + DeleteFiber(current.execContext) + else: + dealloc(current.stack.top) + dealloc(current) + ctx.current = next + elif ctx.current == nil or ctx.current.next == nil: + ctx.current = ctx.coroutines.head + os.sleep(int(minDelay * 1000)) else: - node = node.next + ctx.current = ctx.current.next -proc alive*(c: proc()): bool = - ## Returns ``true`` if coroutine has not returned, ``false`` otherwise. - for coro in items(coroutines): - if coro.fn == c: - return true +proc alive*(c: CoroutineRef): bool = c.coro != nil and c.coro.state != CORO_FINISHED + ## Returns `true` if coroutine has not returned, `false` otherwise. -proc wait*(c: proc(), interval=0.01) = - ## Returns only after coroutine ``c`` has returned. ``interval`` is time in seconds how often. +proc wait*(c: CoroutineRef, interval = 0.01) = + ## Returns only after coroutine `c` has returned. `interval` is time in seconds how often. while alive(c): - suspend interval - -when defined(nimCoroutines) and isMainModule: - var stackCheckValue = 1100220033 - proc c2() - - proc c1() = - for i in 0 .. 3: - echo "c1" - suspend 0.05 - echo "c1 exits" - - - proc c2() = - for i in 0 .. 3: - echo "c2" - suspend 0.025 - wait(c1) - echo "c2 exits" - - start(c1) - start(c2) - run() - echo "done ", stackCheckValue + suspend(interval) diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim new file mode 100644 index 000000000..c907e54d8 --- /dev/null +++ b/lib/pure/cstrutils.nim @@ -0,0 +1,122 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module supports helper routines for working with `cstring` +## without having to convert `cstring` to `string`, in order to +## save allocations. +## +## See also +## ======== +## * `strutils module <strutils.html>`_ for working with `string` + +include system/inclrtl +import std/private/strimpl + + +when defined(js): + func jsStartsWith(s, prefix: cstring): bool {.importjs: "#.startsWith(#)".} + func jsEndsWith(s, suffix: cstring): bool {.importjs: "#.endsWith(#)".} + + +func startsWith*(s, prefix: cstring): bool {.rtl, extern: "csuStartsWith".} = + ## Returns true if `s` starts with `prefix`. + ## + ## The JS backend uses the native `String.prototype.startsWith` function. + runnableExamples: + assert startsWith(cstring"Hello, Nimion", cstring"Hello") + assert not startsWith(cstring"Hello, Nimion", cstring"Nimion") + assert startsWith(cstring"Hello", cstring"") + + when nimvm: + startsWithImpl(s, prefix) + else: + when defined(js): + result = jsStartsWith(s, prefix) + else: + var i = 0 + while true: + if prefix[i] == '\0': return true + if s[i] != prefix[i]: return false + inc(i) + +func endsWith*(s, suffix: cstring): bool {.rtl, extern: "csuEndsWith".} = + ## Returns true if `s` ends with `suffix`. + ## + ## The JS backend uses the native `String.prototype.endsWith` function. + runnableExamples: + assert endsWith(cstring"Hello, Nimion", cstring"Nimion") + assert not endsWith(cstring"Hello, Nimion", cstring"Hello") + assert endsWith(cstring"Hello", cstring"") + + when nimvm: + endsWithImpl(s, suffix) + else: + when defined(js): + result = jsEndsWith(s, suffix) + else: + let slen = s.len + var i = 0 + var j = slen - len(suffix) + while i + j <% slen: + if s[i + j] != suffix[i]: return false + inc(i) + if suffix[i] == '\0': return true + +func cmpIgnoreStyle*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreStyle".} = + ## Semantically the same as `cmp(normalize($a), normalize($b))`. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names, use `macros.eqIdent` + ## for that. Returns: + ## * 0 if `a == b` + ## * < 0 if `a < b` + ## * \> 0 if `a > b` + runnableExamples: + assert cmpIgnoreStyle(cstring"hello", cstring"H_e_L_Lo") == 0 + + when nimvm: + cmpIgnoreStyleImpl(a, b) + else: + when defined(js): + cmpIgnoreStyleImpl(a, b) + else: + var i = 0 + var j = 0 + while true: + while a[i] == '_': inc(i) + while b[j] == '_': inc(j) # BUGFIX: typo + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) + inc(j) + +func cmpIgnoreCase*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreCase".} = + ## Compares two strings in a case insensitive manner. Returns: + ## * 0 if `a == b` + ## * < 0 if `a < b` + ## * \> 0 if `a > b` + runnableExamples: + assert cmpIgnoreCase(cstring"hello", cstring"HeLLo") == 0 + assert cmpIgnoreCase(cstring"echo", cstring"hello") < 0 + assert cmpIgnoreCase(cstring"yellow", cstring"hello") > 0 + + when nimvm: + cmpIgnoreCaseImpl(a, b) + else: + when defined(js): + cmpIgnoreCaseImpl(a, b) + else: + var i = 0 + while true: + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[i]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim deleted file mode 100644 index 9187a67ce..000000000 --- a/lib/pure/db_common.nim +++ /dev/null @@ -1,101 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Common datatypes and definitions for all db_*.nim modules. - -type - DbError* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - - DbEffect* = object of IOEffect ## effect that denotes a database operation - ReadDbEffect* = object of DbEffect ## effect that denotes a read operation - WriteDbEffect* = object of DbEffect ## effect that denotes a write operation - - DbTypeKind* = enum ## a superset of datatypes that might be supported. - dbUnknown, ## unknown datatype - dbSerial, ## datatype used for primary auto-increment keys - dbNull, ## datatype used for the NULL value - dbBit, ## bit datatype - dbBool, ## boolean datatype - dbBlob, ## blob datatype - dbFixedChar, ## string of fixed length - dbVarchar, ## string datatype - dbJson, ## JSON datatype - dbXml, ## XML datatype - dbInt, ## some integer type - dbUInt, ## some unsigned integer type - dbDecimal, ## decimal numbers (fixed-point number) - dbFloat, ## some floating point type - dbDate, ## a year-month-day description - dbTime, ## HH:MM:SS information - dbDatetime, ## year-month-day and HH:MM:SS information, - ## plus optional time or timezone information - dbTimestamp, ## Timestamp values are stored as the number of seconds - ## since the epoch ('1970-01-01 00:00:00' UTC). - dbTimeInterval, ## an interval [a,b] of times - dbEnum, ## some enum - dbSet, ## set of enum values - dbArray, ## an array of values - dbComposite, ## composite type (record, struct, etc) - dbUrl, ## a URL - dbUuid, ## a UUID - dbInet, ## an IP address - dbMacAddress, ## a MAC address - dbGeometry, ## some geometric type - dbPoint, ## Point on a plane (x,y) - dbLine, ## Infinite line ((x1,y1),(x2,y2)) - dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) - dbBox, ## Rectangular box ((x1,y1),(x2,y2)) - dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) - dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) - dbCircle, ## Circle <(x,y),r> (center point and radius) - dbUser1, ## user definable datatype 1 (for unknown extensions) - dbUser2, ## user definable datatype 2 (for unknown extensions) - dbUser3, ## user definable datatype 3 (for unknown extensions) - dbUser4, ## user definable datatype 4 (for unknown extensions) - dbUser5 ## user definable datatype 5 (for unknown extensions) - - DbType* = object ## describes a database type - kind*: DbTypeKind ## the kind of the described type - notNull*: bool ## does the type contain NULL? - name*: string ## the name of the type - size*: Natural ## the size of the datatype; 0 if of variable size - maxReprLen*: Natural ## maximal length required for the representation - precision*, scale*: Natural ## precision and scale of the number - min*, max*: BiggestInt ## the minimum and maximum of allowed values - validValues*: seq[string] ## valid values of an enum or a set - - DbColumn* = object ## information about a database column - name*: string ## name of the column - tableName*: string ## name of the table the column belongs to (optional) - typ*: DbType ## type of the column - primaryKey*: bool ## is this a primary key? - foreignKey*: bool ## is this a foreign key? - DbColumns* = seq[DbColumn] - -{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect, - FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].} - -template sql*(query: string): SqlQuery = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - SqlQuery(query) - -proc dbError*(msg: string) {.noreturn, noinline.} = - ## raises an DbError exception with message `msg`. - var e: ref DbError - new(e) - e.msg = msg - raise e diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim new file mode 100644 index 000000000..9e71d4ce0 --- /dev/null +++ b/lib/pure/distros.nim @@ -0,0 +1,279 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the basics for Linux distribution ("distro") +## detection and the OS's native package manager. Its primary purpose is to +## produce output for Nimble packages, like: +## +## To complete the installation, run: +## +## sudo apt-get install libblas-dev +## sudo apt-get install libvoodoo +## +## The above output could be the result of a code snippet like: +## +## ```nim +## if detectOs(Ubuntu): +## foreignDep "lbiblas-dev" +## foreignDep "libvoodoo" +## ``` +## +## See `packaging <packaging.html>`_ for hints on distributing Nim using OS packages. + +from std/strutils import contains, toLowerAscii + +when not defined(nimscript): + from std/osproc import execProcess + from std/envvars import existsEnv + +type + Distribution* {.pure.} = enum ## the list of known distributions + Windows ## some version of Windows + Posix ## some POSIX system + MacOSX ## some version of OSX + Linux ## some version of Linux + Ubuntu + Debian + Gentoo + Fedora + RedHat + + OpenSUSE + Manjaro + Elementary + Zorin + CentOS + Deepin + ArchLinux + Artix + Antergos + PCLinuxOS + Mageia + LXLE + Solus + Lite + Slackware + Androidx86 + Puppy + Peppermint + Tails + AntiX + Kali + SparkyLinux + Apricity + BlackLab + Bodhi + TrueOS + ArchBang + KaOS + WattOS + Korora + Simplicity + RemixOS + OpenMandriva + Netrunner + Alpine + BlackArch + Ultimate + Gecko + Parrot + KNOPPIX + GhostBSD + Sabayon + Salix + Q4OS + ClearOS + Container + ROSA + Zenwalk + Parabola + ChaletOS + BackBox + MXLinux + Vector + Maui + Qubes + RancherOS + Oracle + TinyCore + Robolinux + Trisquel + Voyager + Clonezilla + SteamOS + Absolute + NixOS ## NixOS or a Nix build environment + AUSTRUMI + Arya + Porteus + AVLinux + Elive + Bluestar + SliTaz + Solaris + Chakra + Wifislax + Scientific + ExTiX + Rockstor + GoboLinux + Void + + BSD + FreeBSD + NetBSD + OpenBSD + DragonFlyBSD + + Haiku + + +const + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, + Distribution.ArchLinux, Distribution.Artix, Distribution.Antergos, + Distribution.BlackArch, Distribution.ArchBang} + +# we cache the result of the 'cmdRelease' +# execution for faster platform detections. +var unameRes, osReleaseIDRes, releaseRes, hostnamectlRes: string + +template cmdRelease(cmd, cache): untyped = + if cache.len == 0: + cache = (when defined(nimscript): gorge(cmd) else: execProcess(cmd)) + cache + +template uname(): untyped = cmdRelease("uname -a", unameRes) +template osReleaseID(): untyped = cmdRelease("cat /etc/os-release | grep ^ID=", osReleaseIDRes) +template release(): untyped = cmdRelease("lsb_release -d", releaseRes) +template hostnamectl(): untyped = cmdRelease("hostnamectl", hostnamectlRes) + +proc detectOsWithAllCmd(d: Distribution): bool = + let dd = toLowerAscii($d) + result = dd in toLowerAscii(osReleaseID()) or dd in toLowerAscii(release()) or + dd in toLowerAscii(uname()) or ("operating system: " & dd) in + toLowerAscii(hostnamectl()) + +proc detectOsImpl(d: Distribution): bool = + case d + of Distribution.Windows: result = defined(windows) + of Distribution.Posix: result = defined(posix) + of Distribution.MacOSX: result = defined(macosx) + of Distribution.Linux: result = defined(linux) + of Distribution.BSD: result = defined(bsd) + else: + when defined(bsd): + case d + of Distribution.FreeBSD, Distribution.NetBSD, Distribution.OpenBSD: + result = $d in uname() + else: + result = false + elif defined(linux): + case d + of Distribution.Gentoo: + result = ("-" & $d & " ") in uname() + of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, + Distribution.Fedora, Distribution.OpenMandriva, Distribution.CentOS, + Distribution.Alpine, Distribution.Mageia, Distribution.Zorin, Distribution.Void: + result = toLowerAscii($d) in osReleaseID() + of Distribution.RedHat: + result = "rhel" in osReleaseID() + of Distribution.ArchLinux: + result = "arch" in osReleaseID() + of Distribution.Artix: + result = "artix" in osReleaseID() + of Distribution.NixOS: + # Check if this is a Nix build or NixOS environment + result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") + of Distribution.OpenSUSE: + result = "suse" in toLowerAscii(uname()) or "suse" in toLowerAscii(release()) + of Distribution.GoboLinux: + result = "-Gobo " in uname() + of Distribution.Solaris: + let uname = toLowerAscii(uname()) + result = ("sun" in uname) or ("solaris" in uname) + of Distribution.Haiku: + result = defined(haiku) + else: + result = detectOsWithAllCmd(d) + else: + result = false + +template detectOs*(d: untyped): bool = + ## Distro/OS detection. For convenience, the + ## required `Distribution.` qualifier is added to the + ## enum value. + detectOsImpl(Distribution.d) + +when not defined(nimble): + var foreignDeps*: seq[string] = @[] ## Registered foreign deps. + +proc foreignCmd*(cmd: string; requiresSudo = false) = + ## Registers a foreign command to the internal list of commands + ## that can be queried later. + let c = (if requiresSudo: "sudo " else: "") & cmd + when defined(nimble): + nimscriptapi.foreignDeps.add(c) + else: + foreignDeps.add(c) + +proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = + ## Returns the distro's native command to install `foreignPackageName` + ## and whether it requires root/admin rights. + let p = foreignPackageName + when defined(windows): + result = ("choco install " & p, false) + elif defined(bsd): + result = ("ports install " & p, true) + elif defined(linux): + if detectOs(Ubuntu) or detectOs(Elementary) or detectOs(Debian) or + detectOs(KNOPPIX) or detectOs(SteamOS): + result = ("apt-get install " & p, true) + elif detectOs(Gentoo): + result = ("emerge install " & p, true) + elif detectOs(Fedora): + result = ("yum install " & p, true) + elif detectOs(RedHat): + result = ("rpm install " & p, true) + elif detectOs(OpenSUSE): + result = ("yast -i " & p, true) + elif detectOs(Slackware): + result = ("installpkg " & p, true) + elif detectOs(OpenMandriva): + result = ("urpmi " & p, true) + elif detectOs(ZenWalk): + result = ("netpkg install " & p, true) + elif detectOs(NixOS): + result = ("nix-env -i " & p, false) + elif detectOs(Solaris) or detectOs(FreeBSD): + result = ("pkg install " & p, true) + elif detectOs(NetBSD) or detectOs(OpenBSD): + result = ("pkg_add " & p, true) + elif detectOs(PCLinuxOS): + result = ("rpm -ivh " & p, true) + elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): + result = ("pacman -S " & p, true) + elif detectOs(Void): + result = ("xbps-install " & p, true) + else: + result = ("<your package manager here> install " & p, true) + elif defined(haiku): + result = ("pkgman install " & p, true) + else: + result = ("brew install " & p, false) + +proc foreignDep*(foreignPackageName: string) = + ## Registers `foreignPackageName` to the internal list of foreign deps. + ## It is your job to ensure that the package name is correct. + let (installCmd, sudo) = foreignDepInstallCmd(foreignPackageName) + foreignCmd(installCmd, sudo) + +proc echoForeignDeps*() = + ## Writes the list of registered foreign deps to stdout. + for d in foreignDeps: + echo d diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 80fdc9dbc..a162fe37f 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -8,43 +8,95 @@ # ## This module implements the ability to access symbols from shared -## libraries. On POSIX this uses the ``dlsym`` mechanism, on -## Windows ``LoadLibrary``. +## libraries. On POSIX this uses the `dlsym` mechanism, on +## Windows `LoadLibrary`. +## +## Examples +## ======== +## +## Loading a simple C function +## --------------------------- +## +## The following example demonstrates loading a function called `greet` +## from a library that is determined at runtime based upon a language choice. +## If the library fails to load or the function `greet` is not found, +## it quits with a failure error code. +## +runnableExamples: + type + GreetFunction = proc (): cstring {.gcsafe, stdcall.} -type - LibHandle* = pointer ## a handle to a dynamically loaded library + proc loadGreet(lang: string) = + let lib = + case lang + of "french": + loadLib("french.dll") + else: + loadLib("english.dll") + assert lib != nil, "Error loading library" + + let greet = cast[GreetFunction](lib.symAddr("greet")) + assert greet != nil, "Error loading 'greet' function from library" + + echo greet() + + unloadLib(lib) + + +import std/strutils -{.deprecated: [TLibHandle: LibHandle].} +type + LibHandle* = pointer ## A handle to a dynamically loaded library. -proc loadLib*(path: string, global_symbols=false): LibHandle - ## loads a library from `path`. Returns nil if the library could not +proc loadLib*(path: string, globalSymbols = false): LibHandle {.gcsafe.} + ## Loads a library from `path`. Returns nil if the library could not ## be loaded. -proc loadLib*(): LibHandle - ## gets the handle from the current executable. Returns nil if the +proc loadLib*(): LibHandle {.gcsafe.} + ## Gets the handle from the current executable. Returns nil if the ## library could not be loaded. -proc unloadLib*(lib: LibHandle) - ## unloads the library `lib` +proc unloadLib*(lib: LibHandle) {.gcsafe.} + ## Unloads the library `lib`. proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = - ## raises an `EInvalidLibrary` exception. - var e: ref LibraryError - new(e) - e.msg = "could not find symbol: " & $name - raise e - -proc symAddr*(lib: LibHandle, name: cstring): pointer - ## retrieves the address of a procedure/variable from `lib`. Returns nil + ## Raises a `LibraryError` exception. + raise newException(LibraryError, "could not find symbol: " & $name) + +proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.} + ## Retrieves the address of a procedure/variable from `lib`. Returns nil ## if the symbol could not be found. proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer = - ## retrieves the address of a procedure/variable from `lib`. Raises - ## `EInvalidLibrary` if the symbol could not be found. + ## Retrieves the address of a procedure/variable from `lib`. Raises + ## `LibraryError` if the symbol could not be found. result = symAddr(lib, name) if result == nil: raiseInvalidLibrary(name) -when defined(posix): +proc libCandidates*(s: string, dest: var seq[string]) = + ## Given a library name pattern `s`, write possible library names to `dest`. + var le = strutils.find(s, '(') + var ri = strutils.find(s, ')', le+1) + if le >= 0 and ri > le: + var prefix = substr(s, 0, le - 1) + var suffix = substr(s, ri + 1) + for middle in split(substr(s, le + 1, ri - 1), '|'): + libCandidates(prefix & middle & suffix, dest) + else: + add(dest, s) + +proc loadLibPattern*(pattern: string, globalSymbols = false): LibHandle = + ## Loads a library with name matching `pattern`, similar to what the `dynlib` + ## pragma does. Returns nil if the library could not be loaded. + ## + ## .. warning:: this proc uses the GC and so cannot be used to load the GC. + var candidates = newSeq[string]() + libCandidates(pattern, candidates) + for c in candidates: + result = loadLib(c, globalSymbols) + if not result.isNil: break + +when defined(posix) and not defined(nintendoswitch): # # ========================================================================= # This is an implementation based on the dlfcn interface. @@ -53,24 +105,66 @@ when defined(posix): # as an emulation layer on top of native functions. # ========================================================================= # - var - RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int - RTLD_GLOBAL {.importc: "RTLD_GLOBAL", header: "<dlfcn.h>".}: int - - proc dlclose(lib: LibHandle) {.importc, header: "<dlfcn.h>".} - proc dlopen(path: cstring, mode: int): LibHandle {. - importc, header: "<dlfcn.h>".} - proc dlsym(lib: LibHandle, name: cstring): pointer {. - importc, header: "<dlfcn.h>".} - - proc loadLib(path: string, global_symbols=false): LibHandle = - var flags = RTLD_NOW - if global_symbols: flags = flags or RTLD_GLOBAL - return dlopen(path, flags) - proc loadLib(): LibHandle = return dlopen(nil, RTLD_NOW) - proc unloadLib(lib: LibHandle) = dlclose(lib) + import std/posix + + proc loadLib(path: string, globalSymbols = false): LibHandle = + let flags = + if globalSymbols: RTLD_NOW or RTLD_GLOBAL + else: RTLD_NOW + + dlopen(path, flags) + + proc loadLib(): LibHandle = dlopen(nil, RTLD_NOW) + proc unloadLib(lib: LibHandle) = discard dlclose(lib) + proc symAddr(lib: LibHandle, name: cstring): pointer = dlsym(lib, name) + +elif defined(nintendoswitch): + # + # ========================================================================= + # Nintendo switch DevkitPro sdk does not have these. Raise an error if called. + # ========================================================================= + # + + proc dlclose(lib: LibHandle) = + raise newException(OSError, "dlclose not implemented on Nintendo Switch!") + proc dlopen(path: cstring, mode: int): LibHandle = + raise newException(OSError, "dlopen not implemented on Nintendo Switch!") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raise newException(OSError, "dlsym not implemented on Nintendo Switch!") + proc loadLib(path: string, global_symbols = false): LibHandle = + raise newException(OSError, "loadLib not implemented on Nintendo Switch!") + proc loadLib(): LibHandle = + raise newException(OSError, "loadLib not implemented on Nintendo Switch!") + proc unloadLib(lib: LibHandle) = + raise newException(OSError, "unloadLib not implemented on Nintendo Switch!") + proc symAddr(lib: LibHandle, name: cstring): pointer = + raise newException(OSError, "symAddr not implemented on Nintendo Switch!") + +elif defined(genode): + # + # ========================================================================= + # Not implemented for Genode without POSIX. Raise an error if called. + # ========================================================================= + # + + template raiseErr(prc: string) = + raise newException(OSError, prc & " not implemented, compile with POSIX support") + + proc dlclose(lib: LibHandle) = + raiseErr(OSError, "dlclose") + proc dlopen(path: cstring, mode: int): LibHandle = + raiseErr(OSError, "dlopen") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "dlsym") + proc loadLib(path: string, global_symbols = false): LibHandle = + raiseErr(OSError, "loadLib") + proc loadLib(): LibHandle = + raiseErr(OSError, "loadLib") + proc unloadLib(lib: LibHandle) = + raiseErr(OSError, "unloadLib") proc symAddr(lib: LibHandle, name: cstring): pointer = - return dlsym(lib, name) + raiseErr(OSError, "symAddr") + elif defined(windows) or defined(dos): # @@ -79,22 +173,23 @@ elif defined(windows) or defined(dos): # ======================================================================= # type - THINSTANCE {.importc: "HINSTANCE".} = pointer + HMODULE {.importc: "HMODULE".} = pointer + FARPROC {.importc: "FARPROC".} = pointer - proc FreeLibrary(lib: THINSTANCE) {.importc, header: "<windows.h>", stdcall.} - proc winLoadLibrary(path: cstring): THINSTANCE {. + proc FreeLibrary(lib: HMODULE) {.importc, header: "<windows.h>", stdcall.} + proc winLoadLibrary(path: cstring): HMODULE {. importc: "LoadLibraryA", header: "<windows.h>", stdcall.} - proc getProcAddress(lib: THINSTANCE, name: cstring): pointer {. + proc getProcAddress(lib: HMODULE, name: cstring): FARPROC {. importc: "GetProcAddress", header: "<windows.h>", stdcall.} - proc loadLib(path: string, global_symbols=false): LibHandle = + proc loadLib(path: string, globalSymbols = false): LibHandle = result = cast[LibHandle](winLoadLibrary(path)) proc loadLib(): LibHandle = result = cast[LibHandle](winLoadLibrary(nil)) - proc unloadLib(lib: LibHandle) = FreeLibrary(cast[THINSTANCE](lib)) + proc unloadLib(lib: LibHandle) = FreeLibrary(cast[HMODULE](lib)) proc symAddr(lib: LibHandle, name: cstring): pointer = - result = getProcAddress(cast[THINSTANCE](lib), name) + result = cast[pointer](getProcAddress(cast[HMODULE](lib), name)) else: {.error: "no implementation for dynlib".} diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index f89786c1c..bbadca655 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -7,15 +7,46 @@ # distribution, for details about the copyright. # -## Converts between different character encodings. On UNIX, this uses +## Routines for converting between different character encodings. On UNIX, this uses ## the `iconv`:idx: library, on Windows the Windows API. +## +## The following example shows how to change character encodings. +runnableExamples: + when defined(windows): + let + orig = "öäüß" + # convert `orig` from "UTF-8" to "CP1252" + cp1252 = convert(orig, "CP1252", "UTF-8") + # convert `cp1252` from "CP1252" to "ibm850" + ibm850 = convert(cp1252, "ibm850", "CP1252") + current = getCurrentEncoding() + assert orig == "\195\182\195\164\195\188\195\159" + assert ibm850 == "\148\132\129\225" + assert convert(ibm850, current, "ibm850") == orig + +## The example below uses a reuseable `EncodingConverter` object which is +## created by `open` with `destEncoding` and `srcEncoding` specified. You can use +## `convert` on this object multiple times. +runnableExamples: + when defined(windows): + var fromGB2312 = open("utf-8", "gb2312") + let first = "\203\173\197\194\163\191\210\187" & + "\203\242\209\204\211\234\200\206\198\189\201\250" + assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生" + + let second = "\211\208\176\215\205\183\200\231" & + "\208\194\163\172\199\227\184\199\200\231\185\202" + assert fromGB2312.convert(second) == "有白头如新，倾盖如故" + -import os, parseutils, strutils +import std/os +when defined(nimPreviewSlimSystem): + import std/assertions when not defined(windows): type ConverterObj = object - EncodingConverter* = ptr ConverterObj ## can convert between two character sets + EncodingConverter* = ptr ConverterObj ## Can convert between two character sets. else: type @@ -24,78 +55,80 @@ else: dest, src: CodePage type - EncodingError* = object of ValueError ## exception that is raised - ## for encoding errors - -{.deprecated: [EInvalidEncoding: EncodingError, PConverter: EncodingConverter].} + EncodingError* = object of ValueError ## Exception that is raised + ## for encoding errors. when defined(windows): + import std/[parseutils, strutils] proc eqEncodingNames(a, b: string): bool = var i = 0 var j = 0 while i < a.len and j < b.len: if a[i] in {'-', '_'}: inc i if b[j] in {'-', '_'}: inc j - if a[i].toLower != b[j].toLower: return false + if i < a.len and j < b.len and + a[i].toLowerAscii != b[j].toLowerAscii: + return false inc i inc j result = i == a.len and j == b.len const winEncodings = [ - (1, "OEMCP"), # current OEM codepage - (037, "IBM037"), # IBM EBCDIC US-Canada - (437, "IBM437"), # OEM United States - (500, "IBM500"), # IBM EBCDIC International - (708, "ASMO-708"), # Arabic (ASMO 708) - (709, "ASMO_449"), # Arabic (ASMO-449+, BCON V4) - (710, ""), # Arabic - Transparent Arabic - (720, "DOS-720"), # Arabic (Transparent ASMO); Arabic (DOS) - (737, "ibm737"), # OEM Greek (formerly 437G); Greek (DOS) - (775, "ibm775"), # OEM Baltic; Baltic (DOS) - (850, "ibm850"), # OEM Multilingual Latin 1; Western European (DOS) - (852, "ibm852"), # OEM Latin 2; Central European (DOS) - (855, "IBM855"), # OEM Cyrillic (primarily Russian) - (857, "ibm857"), # OEM Turkish; Turkish (DOS) - (858, "IBM00858"), # OEM Multilingual Latin 1 + Euro symbol - (860, "IBM860"), # OEM Portuguese; Portuguese (DOS) - (861, "ibm861"), # OEM Icelandic; Icelandic (DOS) - (862, "DOS-862"), # OEM Hebrew; Hebrew (DOS) - (863, "IBM863"), # OEM French Canadian; French Canadian (DOS) - (864, "IBM864"), # OEM Arabic; Arabic (864) - (865, "IBM865"), # OEM Nordic; Nordic (DOS) - (866, "cp866"), # OEM Russian; Cyrillic (DOS) - (869, "ibm869"), # OEM Modern Greek; Greek, Modern (DOS) - (870, "IBM870"), # IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 - (874, "windows-874"), # ANSI/OEM Thai (same as 28605, ISO 8859-15); Thai (Windows) - (875, "cp875"), # IBM EBCDIC Greek Modern - (932, "shift_jis"), # ANSI/OEM Japanese; Japanese (Shift-JIS) - (936, "gb2312"), # ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) + (1, "OEMCP"), # current OEM codepage + (037, "IBM037"), # IBM EBCDIC US-Canada + (437, "IBM437"), # OEM United States + (500, "IBM500"), # IBM EBCDIC International + (708, "ASMO-708"), # Arabic (ASMO 708) + (709, "ASMO_449"), # Arabic (ASMO-449+, BCON V4) + (710, ""), # Arabic - Transparent Arabic + (720, "DOS-720"), # Arabic (Transparent ASMO); Arabic (DOS) + (737, "ibm737"), # OEM Greek (formerly 437G); Greek (DOS) + (775, "ibm775"), # OEM Baltic; Baltic (DOS) + (850, "ibm850"), # OEM Multilingual Latin 1; Western European (DOS) + (852, "ibm852"), # OEM Latin 2; Central European (DOS) + (855, "IBM855"), # OEM Cyrillic (primarily Russian) + (857, "ibm857"), # OEM Turkish; Turkish (DOS) + (858, "IBM00858"), # OEM Multilingual Latin 1 + Euro symbol + (860, "IBM860"), # OEM Portuguese; Portuguese (DOS) + (861, "ibm861"), # OEM Icelandic; Icelandic (DOS) + (862, "DOS-862"), # OEM Hebrew; Hebrew (DOS) + (863, "IBM863"), # OEM French Canadian; French Canadian (DOS) + (864, "IBM864"), # OEM Arabic; Arabic (864) + (865, "IBM865"), # OEM Nordic; Nordic (DOS) + (866, "cp866"), # OEM Russian; Cyrillic (DOS) + (869, "ibm869"), # OEM Modern Greek; Greek, Modern (DOS) + (870, "IBM870"), # IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 + (874, "windows-874"), # ANSI/OEM Thai (same as 28605, ISO 8859-15); Thai (Windows) + (875, "cp875"), # IBM EBCDIC Greek Modern + (932, "shift_jis"), # ANSI/OEM Japanese; Japanese (Shift-JIS) + (936, "gb2312"), # ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) + (936, "gbk"), # Alias for GB2312 encoding (949, "ks_c_5601-1987"), # ANSI/OEM Korean (Unified Hangul Code) - (950, "big5"), # ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) - (1026, "IBM1026"), # IBM EBCDIC Turkish (Latin 5) - (1047, "IBM01047"), # IBM EBCDIC Latin 1/Open System - (1140, "IBM01140"), # IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) - (1141, "IBM01141"), # IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) - (1142, "IBM01142"), # IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) - (1143, "IBM01143"), # IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) - (1144, "IBM01144"), # IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) - (1145, "IBM01145"), # IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) - (1146, "IBM01146"), # IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) - (1147, "IBM01147"), # IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) - (1148, "IBM01148"), # IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) - (1149, "IBM01149"), # IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) - (1200, "utf-16"), # Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications - (1201, "unicodeFFFE"), # Unicode UTF-16, big endian byte order; available only to managed applications - (1250, "windows-1250"), # ANSI Central European; Central European (Windows) - (1251, "windows-1251"), # ANSI Cyrillic; Cyrillic (Windows) - (1252, "windows-1252"), # ANSI Latin 1; Western European (Windows) - (1253, "windows-1253"), # ANSI Greek; Greek (Windows) - (1254, "windows-1254"), # ANSI Turkish; Turkish (Windows) - (1255, "windows-1255"), # ANSI Hebrew; Hebrew (Windows) - (1256, "windows-1256"), # ANSI Arabic; Arabic (Windows) - (1257, "windows-1257"), # ANSI Baltic; Baltic (Windows) - (1258, "windows-1258"), # ANSI/OEM Vietnamese; Vietnamese (Windows) + (950, "big5"), # ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) + (1026, "IBM1026"), # IBM EBCDIC Turkish (Latin 5) + (1047, "IBM01047"), # IBM EBCDIC Latin 1/Open System + (1140, "IBM01140"), # IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) + (1141, "IBM01141"), # IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) + (1142, "IBM01142"), # IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) + (1143, "IBM01143"), # IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) + (1144, "IBM01144"), # IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) + (1145, "IBM01145"), # IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) + (1146, "IBM01146"), # IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) + (1147, "IBM01147"), # IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) + (1148, "IBM01148"), # IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) + (1149, "IBM01149"), # IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) + (1200, "utf-16"), # Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications + (1201, "unicodeFFFE"), # Unicode UTF-16, big endian byte order; available only to managed applications + (1250, "windows-1250"), # ANSI Central European; Central European (Windows) + (1251, "windows-1251"), # ANSI Cyrillic; Cyrillic (Windows) + (1252, "windows-1252"), # ANSI Latin 1; Western European (Windows) + (1253, "windows-1253"), # ANSI Greek; Greek (Windows) + (1254, "windows-1254"), # ANSI Turkish; Turkish (Windows) + (1255, "windows-1255"), # ANSI Hebrew; Hebrew (Windows) + (1256, "windows-1256"), # ANSI Arabic; Arabic (Windows) + (1257, "windows-1257"), # ANSI Baltic; Baltic (Windows) + (1258, "windows-1258"), # ANSI/OEM Vietnamese; Vietnamese (Windows) (1250, "cp-1250"), # ANSI Central European; Central European (Windows) (1251, "cp-1251"), # ANSI Cyrillic; Cyrillic (Windows) @@ -107,106 +140,106 @@ when defined(windows): (1257, "cp-1257"), # ANSI Baltic; Baltic (Windows) (1258, "cp-1258"), # ANSI/OEM Vietnamese; Vietnamese (Windows) - (1361, "Johab"), # Korean (Johab) - (10000, "macintosh"), # MAC Roman; Western European (Mac) - (10001, "x-mac-japanese"), # Japanese (Mac) - (10002, "x-mac-chinesetrad"), # MAC Traditional Chinese (Big5); Chinese Traditional (Mac) - (10003, "x-mac-korean"), # Korean (Mac) - (10004, "x-mac-arabic"), # Arabic (Mac) - (10005, "x-mac-hebrew"), # Hebrew (Mac) - (10006, "x-mac-greek"), # Greek (Mac) - (10007, "x-mac-cyrillic"), # Cyrillic (Mac) - (10008, "x-mac-chinesesimp"), # MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) - (10010, "x-mac-romanian"), # Romanian (Mac) - (10017, "x-mac-ukrainian"), # Ukrainian (Mac) - (10021, "x-mac-thai"), # Thai (Mac) - (10029, "x-mac-ce"), # MAC Latin 2; Central European (Mac) - (10079, "x-mac-icelandic"), # Icelandic (Mac) - (10081, "x-mac-turkish"), # Turkish (Mac) - (10082, "x-mac-croatian"), # Croatian (Mac) - (12000, "utf-32"), # Unicode UTF-32, little endian byte order; available only to managed applications - (12001, "utf-32BE"), # Unicode UTF-32, big endian byte order; available only to managed applications - (20000, "x-Chinese_CNS"), # CNS Taiwan; Chinese Traditional (CNS) - (20001, "x-cp20001"), # TCA Taiwan - (20002, "x_Chinese-Eten"), # Eten Taiwan; Chinese Traditional (Eten) - (20003, "x-cp20003"), # IBM5550 Taiwan - (20004, "x-cp20004"), # TeleText Taiwan - (20005, "x-cp20005"), # Wang Taiwan - (20105, "x-IA5"), # IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) - (20106, "x-IA5-German"), # IA5 German (7-bit) - (20107, "x-IA5-Swedish"), # IA5 Swedish (7-bit) - (20108, "x-IA5-Norwegian"), # IA5 Norwegian (7-bit) - (20127, "us-ascii"), # US-ASCII (7-bit) - (20261, "x-cp20261"), # T.61 - (20269, "x-cp20269"), # ISO 6937 Non-Spacing Accent - (20273, "IBM273"), # IBM EBCDIC Germany - (20277, "IBM277"), # IBM EBCDIC Denmark-Norway - (20278, "IBM278"), # IBM EBCDIC Finland-Sweden - (20280, "IBM280"), # IBM EBCDIC Italy - (20284, "IBM284"), # IBM EBCDIC Latin America-Spain - (20285, "IBM285"), # IBM EBCDIC United Kingdom - (20290, "IBM290"), # IBM EBCDIC Japanese Katakana Extended - (20297, "IBM297"), # IBM EBCDIC France - (20420, "IBM420"), # IBM EBCDIC Arabic - (20423, "IBM423"), # IBM EBCDIC Greek - (20424, "IBM424"), # IBM EBCDIC Hebrew + (1361, "Johab"), # Korean (Johab) + (10000, "macintosh"), # MAC Roman; Western European (Mac) + (10001, "x-mac-japanese"), # Japanese (Mac) + (10002, "x-mac-chinesetrad"), # MAC Traditional Chinese (Big5); Chinese Traditional (Mac) + (10003, "x-mac-korean"), # Korean (Mac) + (10004, "x-mac-arabic"), # Arabic (Mac) + (10005, "x-mac-hebrew"), # Hebrew (Mac) + (10006, "x-mac-greek"), # Greek (Mac) + (10007, "x-mac-cyrillic"), # Cyrillic (Mac) + (10008, "x-mac-chinesesimp"), # MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) + (10010, "x-mac-romanian"), # Romanian (Mac) + (10017, "x-mac-ukrainian"), # Ukrainian (Mac) + (10021, "x-mac-thai"), # Thai (Mac) + (10029, "x-mac-ce"), # MAC Latin 2; Central European (Mac) + (10079, "x-mac-icelandic"), # Icelandic (Mac) + (10081, "x-mac-turkish"), # Turkish (Mac) + (10082, "x-mac-croatian"), # Croatian (Mac) + (12000, "utf-32"), # Unicode UTF-32, little endian byte order; available only to managed applications + (12001, "utf-32BE"), # Unicode UTF-32, big endian byte order; available only to managed applications + (20000, "x-Chinese_CNS"), # CNS Taiwan; Chinese Traditional (CNS) + (20001, "x-cp20001"), # TCA Taiwan + (20002, "x_Chinese-Eten"), # Eten Taiwan; Chinese Traditional (Eten) + (20003, "x-cp20003"), # IBM5550 Taiwan + (20004, "x-cp20004"), # TeleText Taiwan + (20005, "x-cp20005"), # Wang Taiwan + (20105, "x-IA5"), # IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) + (20106, "x-IA5-German"), # IA5 German (7-bit) + (20107, "x-IA5-Swedish"), # IA5 Swedish (7-bit) + (20108, "x-IA5-Norwegian"), # IA5 Norwegian (7-bit) + (20127, "us-ascii"), # US-ASCII (7-bit) + (20261, "x-cp20261"), # T.61 + (20269, "x-cp20269"), # ISO 6937 Non-Spacing Accent + (20273, "IBM273"), # IBM EBCDIC Germany + (20277, "IBM277"), # IBM EBCDIC Denmark-Norway + (20278, "IBM278"), # IBM EBCDIC Finland-Sweden + (20280, "IBM280"), # IBM EBCDIC Italy + (20284, "IBM284"), # IBM EBCDIC Latin America-Spain + (20285, "IBM285"), # IBM EBCDIC United Kingdom + (20290, "IBM290"), # IBM EBCDIC Japanese Katakana Extended + (20297, "IBM297"), # IBM EBCDIC France + (20420, "IBM420"), # IBM EBCDIC Arabic + (20423, "IBM423"), # IBM EBCDIC Greek + (20424, "IBM424"), # IBM EBCDIC Hebrew (20833, "x-EBCDIC-KoreanExtended"), # IBM EBCDIC Korean Extended - (20838, "IBM-Thai"), # IBM EBCDIC Thai - (20866, "koi8-r"), # Russian (KOI8-R); Cyrillic (KOI8-R) - (20871, "IBM871"), # IBM EBCDIC Icelandic - (20880, "IBM880"), # IBM EBCDIC Cyrillic Russian - (20905, "IBM905"), # IBM EBCDIC Turkish - (20924, "IBM00924"), # IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) - (20932, "EUC-JP"), # Japanese (JIS 0208-1990 and 0121-1990) - (20936, "x-cp20936"), # Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) - (20949, "x-cp20949"), # Korean Wansung - (21025, "cp1025"), # IBM EBCDIC Cyrillic Serbian-Bulgarian - (21027, ""), # (deprecated) - (21866, "koi8-u"), # Ukrainian (KOI8-U); Cyrillic (KOI8-U) - (28591, "iso-8859-1"), # ISO 8859-1 Latin 1; Western European (ISO) - (28592, "iso-8859-2"), # ISO 8859-2 Central European; Central European (ISO) - (28593, "iso-8859-3"), # ISO 8859-3 Latin 3 - (28594, "iso-8859-4"), # ISO 8859-4 Baltic - (28595, "iso-8859-5"), # ISO 8859-5 Cyrillic - (28596, "iso-8859-6"), # ISO 8859-6 Arabic - (28597, "iso-8859-7"), # ISO 8859-7 Greek - (28598, "iso-8859-8"), # ISO 8859-8 Hebrew; Hebrew (ISO-Visual) - (28599, "iso-8859-9"), # ISO 8859-9 Turkish - (28603, "iso-8859-13"), # ISO 8859-13 Estonian - (28605, "iso-8859-15"), # ISO 8859-15 Latin 9 - (29001, "x-Europa"), # Europa 3 - (38598, "iso-8859-8-i"), # ISO 8859-8 Hebrew; Hebrew (ISO-Logical) - (50220, "iso-2022-jp"), # ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) - (50221, "csISO2022JP"), # ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) - (50222, "iso-2022-jp"), # ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) - (50225, "iso-2022-kr"), # ISO 2022 Korean - (50227, "x-cp50227"), # ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) - (50229, ""), # ISO 2022 Traditional Chinese - (50930, ""), # EBCDIC Japanese (Katakana) Extended - (50931, ""), # EBCDIC US-Canada and Japanese - (50933, ""), # EBCDIC Korean Extended and Korean - (50935, ""), # EBCDIC Simplified Chinese Extended and Simplified Chinese - (50936, ""), # EBCDIC Simplified Chinese - (50937, ""), # EBCDIC US-Canada and Traditional Chinese - (50939, ""), # EBCDIC Japanese (Latin) Extended and Japanese - (51932, "euc-jp"), # EUC Japanese - (51936, "EUC-CN"), # EUC Simplified Chinese; Chinese Simplified (EUC) - (51949, "euc-kr"), # EUC Korean - (51950, ""), # EUC Traditional Chinese - (52936, "hz-gb-2312"), # HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) - (54936, "GB18030"), # Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) - (57002, "x-iscii-de"), # ISCII Devanagari - (57003, "x-iscii-be"), # ISCII Bengali - (57004, "x-iscii-ta"), # ISCII Tamil - (57005, "x-iscii-te"), # ISCII Telugu - (57006, "x-iscii-as"), # ISCII Assamese - (57007, "x-iscii-or"), # ISCII Oriya - (57008, "x-iscii-ka"), # ISCII Kannada - (57009, "x-iscii-ma"), # ISCII Malayalam - (57010, "x-iscii-gu"), # ISCII Gujarati - (57011, "x-iscii-pa"), # ISCII Punjabi - (65000, "utf-7"), # Unicode (UTF-7) - (65001, "utf-8")] # Unicode (UTF-8) + (20838, "IBM-Thai"), # IBM EBCDIC Thai + (20866, "koi8-r"), # Russian (KOI8-R); Cyrillic (KOI8-R) + (20871, "IBM871"), # IBM EBCDIC Icelandic + (20880, "IBM880"), # IBM EBCDIC Cyrillic Russian + (20905, "IBM905"), # IBM EBCDIC Turkish + (20924, "IBM00924"), # IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) + (20932, "EUC-JP"), # Japanese (JIS 0208-1990 and 0121-1990) + (20936, "x-cp20936"), # Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) + (20949, "x-cp20949"), # Korean Wansung + (21025, "cp1025"), # IBM EBCDIC Cyrillic Serbian-Bulgarian + (21027, ""), # (deprecated) + (21866, "koi8-u"), # Ukrainian (KOI8-U); Cyrillic (KOI8-U) + (28591, "iso-8859-1"), # ISO 8859-1 Latin 1; Western European (ISO) + (28592, "iso-8859-2"), # ISO 8859-2 Central European; Central European (ISO) + (28593, "iso-8859-3"), # ISO 8859-3 Latin 3 + (28594, "iso-8859-4"), # ISO 8859-4 Baltic + (28595, "iso-8859-5"), # ISO 8859-5 Cyrillic + (28596, "iso-8859-6"), # ISO 8859-6 Arabic + (28597, "iso-8859-7"), # ISO 8859-7 Greek + (28598, "iso-8859-8"), # ISO 8859-8 Hebrew; Hebrew (ISO-Visual) + (28599, "iso-8859-9"), # ISO 8859-9 Turkish + (28603, "iso-8859-13"), # ISO 8859-13 Estonian + (28605, "iso-8859-15"), # ISO 8859-15 Latin 9 + (29001, "x-Europa"), # Europa 3 + (38598, "iso-8859-8-i"), # ISO 8859-8 Hebrew; Hebrew (ISO-Logical) + (50220, "iso-2022-jp"), # ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) + (50221, "csISO2022JP"), # ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) + (50222, "iso-2022-jp"), # ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) + (50225, "iso-2022-kr"), # ISO 2022 Korean + (50227, "x-cp50227"), # ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) + (50229, ""), # ISO 2022 Traditional Chinese + (50930, ""), # EBCDIC Japanese (Katakana) Extended + (50931, ""), # EBCDIC US-Canada and Japanese + (50933, ""), # EBCDIC Korean Extended and Korean + (50935, ""), # EBCDIC Simplified Chinese Extended and Simplified Chinese + (50936, ""), # EBCDIC Simplified Chinese + (50937, ""), # EBCDIC US-Canada and Traditional Chinese + (50939, ""), # EBCDIC Japanese (Latin) Extended and Japanese + (51932, "euc-jp"), # EUC Japanese + (51936, "EUC-CN"), # EUC Simplified Chinese; Chinese Simplified (EUC) + (51949, "euc-kr"), # EUC Korean + (51950, ""), # EUC Traditional Chinese + (52936, "hz-gb-2312"), # HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) + (54936, "GB18030"), # Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) + (57002, "x-iscii-de"), # ISCII Devanagari + (57003, "x-iscii-be"), # ISCII Bengali + (57004, "x-iscii-ta"), # ISCII Tamil + (57005, "x-iscii-te"), # ISCII Telugu + (57006, "x-iscii-as"), # ISCII Assamese + (57007, "x-iscii-or"), # ISCII Oriya + (57008, "x-iscii-ka"), # ISCII Kannada + (57009, "x-iscii-ma"), # ISCII Malayalam + (57010, "x-iscii-gu"), # ISCII Gujarati + (57011, "x-iscii-pa"), # ISCII Punjabi + (65000, "utf-7"), # Unicode (UTF-7) + (65001, "utf-8")] # Unicode (UTF-8) when false: # not needed yet: @@ -215,25 +248,26 @@ when defined(windows): maxCharSize: int32 defaultChar: array[0..1, char] leadByte: array[0..12-1, char] - {.deprecated: [TCpInfo: CpInfo].} proc getCPInfo(codePage: CodePage, lpCPInfo: var CpInfo): int32 {. stdcall, importc: "GetCPInfo", dynlib: "kernel32".} - proc nameToCodePage(name: string): CodePage = + proc nameToCodePage*(name: string): CodePage = var nameAsInt: int if parseInt(name, nameAsInt) == 0: nameAsInt = -1 for no, na in items(winEncodings): if no == nameAsInt or eqEncodingNames(na, name): return CodePage(no) result = CodePage(-1) - proc codePageToName(c: CodePage): string = + proc codePageToName*(c: CodePage): string = for no, na in items(winEncodings): if no == int(c): return if na.len != 0: na else: $no result = "" proc getACP(): CodePage {.stdcall, importc: "GetACP", dynlib: "kernel32".} + proc getGetConsoleCP(): CodePage {.stdcall, importc: "GetConsoleCP", + dynlib: "kernel32".} proc multiByteToWideChar( codePage: CodePage, @@ -251,13 +285,13 @@ when defined(windows): cchWideChar: cint, lpMultiByteStr: cstring, cbMultiByte: cint, - lpDefaultChar: cstring=nil, - lpUsedDefaultChar: pointer=nil): cint {. + lpDefaultChar: cstring = nil, + lpUsedDefaultChar: pointer = nil): cint {. stdcall, importc: "WideCharToMultiByte", dynlib: "kernel32".} else: when defined(haiku): - const iconvDll = "(libc.so.6|libiconv.so|libtextencoding.so)" + const iconvDll = "libiconv.so" elif defined(macosx): const iconvDll = "libiconv.dylib" else: @@ -274,33 +308,41 @@ else: const EILSEQ = 86.cint elif defined(solaris): const EILSEQ = 88.cint + elif defined(haiku): + const EILSEQ = -2147454938.cint var errno {.importc, header: "<errno.h>".}: cint + when defined(bsd): + {.pragma: importIconv, cdecl, header: "<iconv.h>".} + when defined(openbsd): + {.passL: "-liconv".} + else: + {.pragma: importIconv, cdecl, dynlib: iconvDll.} + proc iconvOpen(tocode, fromcode: cstring): EncodingConverter {. - importc: "iconv_open", cdecl, dynlib: iconvDll.} + importc: "iconv_open", importIconv.} proc iconvClose(c: EncodingConverter) {. - importc: "iconv_close", cdecl, dynlib: iconvDll.} - proc iconv(c: EncodingConverter, inbuf: var cstring, inbytesLeft: var int, - outbuf: var cstring, outbytesLeft: var int): int {. - importc: "iconv", cdecl, dynlib: iconvDll.} - proc iconv(c: EncodingConverter, inbuf: pointer, inbytesLeft: pointer, - outbuf: var cstring, outbytesLeft: var int): int {. - importc: "iconv", cdecl, dynlib: iconvDll.} - -proc getCurrentEncoding*(): string = - ## retrieves the current encoding. On Unix, always "UTF-8" is returned. + importc: "iconv_close", importIconv.} + proc iconv(c: EncodingConverter, inbuf: ptr cstring, inbytesLeft: ptr csize_t, + outbuf: ptr cstring, outbytesLeft: ptr csize_t): csize_t {. + importc: "iconv", importIconv.} + +proc getCurrentEncoding*(uiApp = false): string = + ## Retrieves the current encoding. On Unix, "UTF-8" is always returned. + ## The `uiApp` parameter is Windows specific. If true, the UI's code-page + ## is returned, if false, the Console's code-page is returned. when defined(windows): - result = codePageToName(getACP()) + result = codePageToName(if uiApp: getACP() else: getGetConsoleCP()) else: result = "UTF-8" proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = - ## opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `EIO` if it cannot fulfill the request. + ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) - if result == nil: + if result == cast[EncodingConverter](-1): raise newException(EncodingError, "cannot create encoding converter from " & srcEncoding & " to " & destEncoding) @@ -315,37 +357,34 @@ proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = "cannot find encoding " & srcEncoding) proc close*(c: EncodingConverter) = - ## frees the resources the converter `c` holds. + ## Frees the resources the converter `c` holds. when not defined(windows): iconvClose(c) when defined(windows): - proc convert*(c: EncodingConverter, s: string): string = - ## converts `s` to `destEncoding` that was given to the converter `c`. It - ## assumed that `s` is in `srcEncoding`. - - # special case: empty string: needed because MultiByteToWideChar - # return 0 in case of error: - if s.len == 0: return "" + proc convertToWideString(codePage: CodePage, s: string): string = # educated guess of capacity: var cap = s.len + s.len shr 2 - result = newStringOfCap(cap*2) + result = newString(cap*2) # convert to utf-16 LE - var m = multiByteToWideChar(codePage = c.src, dwFlags = 0'i32, + var m = multiByteToWideChar(codePage, + dwFlags = 0'i32, lpMultiByteStr = cstring(s), cbMultiByte = cint(s.len), lpWideCharStr = cstring(result), cchWideChar = cint(cap)) if m == 0: # try again; ask for capacity: - cap = multiByteToWideChar(codePage = c.src, dwFlags = 0'i32, + cap = multiByteToWideChar(codePage, + dwFlags = 0'i32, lpMultiByteStr = cstring(s), cbMultiByte = cint(s.len), lpWideCharStr = nil, cchWideChar = cint(0)) # and do the conversion properly: - result = newStringOfCap(cap*2) - m = multiByteToWideChar(codePage = c.src, dwFlags = 0'i32, + result = newString(cap*2) + m = multiByteToWideChar(codePage, + dwFlags = 0'i32, lpMultiByteStr = cstring(s), cbMultiByte = cint(s.len), lpWideCharStr = cstring(result), @@ -357,56 +396,78 @@ when defined(windows): else: assert(false) # cannot happen - # if already utf-16 LE, no further need to do something: - if int(c.dest) == 1200: return - # otherwise the fun starts again: - cap = s.len + s.len shr 2 - var res = newStringOfCap(cap) - m = wideCharToMultiByte( - codePage = c.dest, - dwFlags = 0'i32, - lpWideCharStr = cstring(result), - cchWideChar = cint(result.len div 2), - lpMultiByteStr = cstring(res), - cbMultiByte = cap.cint) + proc convertFromWideString(codePage: CodePage, s: string): string = + let charCount = s.len div 2 + var cap = s.len + s.len shr 2 + result = newString(cap) + var m = wideCharToMultiByte(codePage, + dwFlags = 0'i32, + lpWideCharStr = cstring(s), + cchWideChar = cint(charCount), + lpMultiByteStr = cstring(result), + cbMultiByte = cap.cint) if m == 0: # try again; ask for capacity: - cap = wideCharToMultiByte( - codePage = c.dest, - dwFlags = 0'i32, - lpWideCharStr = cstring(result), - cchWideChar = cint(result.len div 2), - lpMultiByteStr = nil, - cbMultiByte = cint(0)) + cap = wideCharToMultiByte(codePage, + dwFlags = 0'i32, + lpWideCharStr = cstring(s), + cchWideChar = cint(charCount), + lpMultiByteStr = nil, + cbMultiByte = cint(0)) # and do the conversion properly: - res = newStringOfCap(cap) - m = wideCharToMultiByte( - codePage = c.dest, - dwFlags = 0'i32, - lpWideCharStr = cstring(result), - cchWideChar = cint(result.len div 2), - lpMultiByteStr = cstring(res), - cbMultiByte = cap.cint) + result = newString(cap) + m = wideCharToMultiByte(codePage, + dwFlags = 0'i32, + lpWideCharStr = cstring(s), + cchWideChar = cint(charCount), + lpMultiByteStr = cstring(result), + cbMultiByte = cap.cint) if m == 0: raiseOSError(osLastError()) - setLen(res, m) - result = res + setLen(result, m) elif m <= cap: - setLen(res, m) - result = res + setLen(result, m) else: assert(false) # cannot happen + proc convertWin(codePageFrom: CodePage, codePageTo: CodePage, + s: string): string = + # special case: empty string: needed because MultiByteToWideChar, WideCharToMultiByte + # return 0 in case of error + if s.len == 0: return "" + # multiByteToWideChar does not support encoding from code pages below + let unsupported = [1201, 12000, 12001] + + if int(codePageFrom) in unsupported: + let message = "encoding from " & codePageToName(codePageFrom) & " is not supported on windows" + raise newException(EncodingError, message) + + if int(codePageTo) in unsupported: + let message = "encoding to " & codePageToName(codePageTo) & " is not supported on windows" + raise newException(EncodingError, message) + + # in case it's already UTF-16 little endian - conversion can be simplified + let wideString = if int(codePageFrom) == 1200: s + else: convertToWideString(codePageFrom, s) + return if int(codePageTo) == 1200: wideString + else: convertFromWideString(codePageTo, wideString) + + proc convert*(c: EncodingConverter, s: string): string = + result = convertWin(c.src, c.dest, s) else: proc convert*(c: EncodingConverter, s: string): string = + ## Converts `s` to `destEncoding` that was given to the converter `c`. It + ## assumes that `s` is in `srcEncoding`. + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. result = newString(s.len) - var inLen = len(s) - var outLen = len(result) + var inLen = csize_t len(s) + var outLen = csize_t len(result) var src = cstring(s) var dst = cstring(result) - var iconvres: int + var iconvres: csize_t while inLen > 0: - iconvres = iconv(c, src, inLen, dst, outLen) - if iconvres == -1: + iconvres = iconv(c, addr src, addr inLen, addr dst, addr outLen) + if iconvres == high(csize_t): var lerr = errno if lerr == EILSEQ or lerr == EINVAL: # unknown char, skip @@ -417,43 +478,34 @@ else: dec(outLen) elif lerr == E2BIG: var offset = cast[int](dst) - cast[int](cstring(result)) - setLen(result, len(result)+inLen*2+5) + setLen(result, len(result) + inLen.int * 2 + 5) # 5 is minimally one utf-8 char dst = cast[cstring](cast[int](cstring(result)) + offset) - outLen = len(result) - offset + outLen = csize_t(len(result) - offset) else: raiseOSError(lerr.OSErrorCode) # iconv has a buffer that needs flushing, specially if the last char is # not '\0' - discard iconv(c, nil, nil, dst, outLen) - if iconvres == cint(-1) and errno == E2BIG: + discard iconv(c, nil, nil, addr dst, addr outLen) + if iconvres == high(csize_t) and errno == E2BIG: var offset = cast[int](dst) - cast[int](cstring(result)) - setLen(result, len(result)+inLen*2+5) + setLen(result, len(result) + inLen.int * 2 + 5) # 5 is minimally one utf-8 char dst = cast[cstring](cast[int](cstring(result)) + offset) - outLen = len(result) - offset - discard iconv(c, nil, nil, dst, outLen) + outLen = csize_t(len(result) - offset) + discard iconv(c, nil, nil, addr dst, addr outLen) # trim output buffer - setLen(result, len(result) - outLen) + setLen(result, len(result) - outLen.int) proc convert*(s: string, destEncoding = "UTF-8", srcEncoding = "CP1252"): string = - ## converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. + ## Converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. ## This opens a converter, uses it and closes it again and is thus more - ## convienent but also likely less efficient than re-using a converter. + ## convenient but also likely less efficient than re-using a converter. + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. var c = open(destEncoding, srcEncoding) try: result = convert(c, s) finally: close(c) - -when not defined(testing) and isMainModule: - let - orig = "öäüß" - cp1252 = convert(orig, "CP1252", "UTF-8") - ibm850 = convert(cp1252, "ibm850", "CP1252") - current = getCurrentEncoding() - echo "Original string from source code: ", orig - echo "Forced ibm850 encoding: ", ibm850 - echo "Current encoding: ", current - echo "From ibm850 to current: ", convert(ibm850, current, "ibm850") diff --git a/lib/pure/endians.nim b/lib/pure/endians.nim index 5a23169d4..4c1d45ae5 100644 --- a/lib/pure/endians.nim +++ b/lib/pure/endians.nim @@ -9,39 +9,116 @@ ## This module contains helpers that deal with different byte orders ## (`endian`:idx:). +## +## Endianness is the order of bytes of a value in memory. Big-endian means that +## the most significant byte is stored at the smallest memory address, +## while little endian means that the least-significant byte is stored +## at the smallest address. See also https://en.wikipedia.org/wiki/Endianness. +## +## Unstable API. -proc swapEndian64*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 8 bytes. - var i = cast[cstring](inp) - var o = cast[cstring](outp) - o[0] = i[7] - o[1] = i[6] - o[2] = i[5] - o[3] = i[4] - o[4] = i[3] - o[5] = i[2] - o[6] = i[1] - o[7] = i[0] - -proc swapEndian32*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 4 bytes. - var i = cast[cstring](inp) - var o = cast[cstring](outp) - o[0] = i[3] - o[1] = i[2] - o[2] = i[1] - o[3] = i[0] - -proc swapEndian16*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 2 bytes. - var - i = cast[cstring](inp) - o = cast[cstring](outp) - o[0] = i[1] - o[1] = i[0] +when defined(gcc) or defined(llvm_gcc) or defined(clang): + const useBuiltinSwap = true + proc builtin_bswap16(a: uint16): uint16 {. + importc: "__builtin_bswap16", nodecl, noSideEffect.} + + proc builtin_bswap32(a: uint32): uint32 {. + importc: "__builtin_bswap32", nodecl, noSideEffect.} + + proc builtin_bswap64(a: uint64): uint64 {. + importc: "__builtin_bswap64", nodecl, noSideEffect.} +elif defined(icc): + const useBuiltinSwap = true + proc builtin_bswap16(a: uint16): uint16 {. + importc: "_bswap16", nodecl, noSideEffect.} + + proc builtin_bswap32(a: uint32): uint32 {. + importc: "_bswap", nodecl, noSideEffect.} + + proc builtin_bswap64(a: uint64): uint64 {. + importc: "_bswap64", nodecl, noSideEffect.} +elif defined(vcc): + const useBuiltinSwap = true + proc builtin_bswap16(a: uint16): uint16 {. + importc: "_byteswap_ushort", nodecl, header: "<intrin.h>", noSideEffect.} + + proc builtin_bswap32(a: uint32): uint32 {. + importc: "_byteswap_ulong", nodecl, header: "<intrin.h>", noSideEffect.} + + proc builtin_bswap64(a: uint64): uint64 {. + importc: "_byteswap_uint64", nodecl, header: "<intrin.h>", noSideEffect.} +else: + const useBuiltinSwap = false + +when useBuiltinSwap: + template swapOpImpl(T: typedesc, op: untyped) = + ## We have to use `copyMem` here instead of a simple dereference because they + ## may point to a unaligned address. A sufficiently smart compiler _should_ + ## be able to elide them when they're not necessary. + var tmp: T + copyMem(addr tmp, inp, sizeof(T)) + tmp = op(tmp) + copyMem(outp, addr tmp, sizeof(T)) + + proc swapEndian64*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 8 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4, 5, 6, 7, 8] + var b: array[8, uint8] + swapEndian64(addr b, addr a) + assert b == [8'u8, 7, 6, 5, 4, 3, 2, 1] + + swapOpImpl(uint64, builtin_bswap64) + + proc swapEndian32*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 4 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4] + var b: array[4, uint8] + swapEndian32(addr b, addr a) + assert b == [4'u8, 3, 2, 1] + + swapOpImpl(uint32, builtin_bswap32) + + proc swapEndian16*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 2 bytes. + runnableExamples: + var a = [1'u8, 2] + var b: array[2, uint8] + swapEndian16(addr b, addr a) + assert b == [2'u8, 1] + + swapOpImpl(uint16, builtin_bswap16) + +else: + proc swapEndian64*(outp, inp: pointer) = + var i = cast[cstring](inp) + var o = cast[cstring](outp) + o[0] = i[7] + o[1] = i[6] + o[2] = i[5] + o[3] = i[4] + o[4] = i[3] + o[5] = i[2] + o[6] = i[1] + o[7] = i[0] + + proc swapEndian32*(outp, inp: pointer) = + var i = cast[cstring](inp) + var o = cast[cstring](outp) + o[0] = i[3] + o[1] = i[2] + o[2] = i[1] + o[3] = i[0] + + proc swapEndian16*(outp, inp: pointer) = + var i = cast[cstring](inp) + var o = cast[cstring](outp) + o[0] = i[1] + o[1] = i[0] when system.cpuEndian == bigEndian: proc littleEndian64*(outp, inp: pointer) {.inline.} = swapEndian64(outp, inp) @@ -52,8 +129,20 @@ when system.cpuEndian == bigEndian: proc bigEndian16*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 2) else: proc littleEndian64*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 8) + ## Copies `inp` to `outp`, storing it in 64-bit little-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc littleEndian32*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 4) + ## Copies `inp` to `outp`, storing it in 32-bit little-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc littleEndian16*(outp, inp: pointer){.inline.} = copyMem(outp, inp, 2) + ## Copies `inp` to `outp`, storing it in 16-bit little-endian order. + ## Both buffers are supposed to contain at least 2 bytes. proc bigEndian64*(outp, inp: pointer) {.inline.} = swapEndian64(outp, inp) + ## Copies `inp` to `outp`, storing it in 64-bit big-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc bigEndian32*(outp, inp: pointer) {.inline.} = swapEndian32(outp, inp) + ## Copies `inp` to `outp`, storing it in 32-bit big-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc bigEndian16*(outp, inp: pointer) {.inline.} = swapEndian16(outp, inp) + ## Copies `inp` to `outp`, storing it in 16-bit big-endian order. + ## Both buffers are supposed to contain at least 2 bytes. diff --git a/lib/pure/etcpriv.nim b/lib/pure/etcpriv.nim deleted file mode 100644 index 5b785b051..000000000 --- a/lib/pure/etcpriv.nim +++ /dev/null @@ -1,23 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Nim Authors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module contains utils that are less then easy to categorize and -## don't really warrant a specific module. They are private to compiler -## and stdlib usage, and should not be used outside of that - they may -## change or disappear at any time. - - -# Used by pure/hashes.nim, and the compiler parsing -const magicIdentSeparatorRuneByteWidth* = 3 - -# Used by pure/hashes.nim, and the compiler parsing -proc isMagicIdentSeparatorRune*(cs: cstring, i: int): bool {. inline } = - result = cs[i] == '\226' and - cs[i + 1] == '\128' and - cs[i + 2] == '\147' # en-dash # 145 = nb-hyphen diff --git a/lib/pure/events.nim b/lib/pure/events.nim deleted file mode 100644 index 62800c5c8..000000000 --- a/lib/pure/events.nim +++ /dev/null @@ -1,103 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2011 Alexander Mitchell-Robinson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## :Author: Alexander Mitchell-Robinson (Amrykid) -## -## This module implements an event system that is not dependent on external -## graphical toolkits. It was originally called ``NimEE`` because -## it was inspired by Python's PyEE module. There are two ways you can use -## events: one is a python-inspired way; the other is more of a C-style way. -## -## .. code-block:: Nim -## var ee = initEventEmitter() -## var genericargs: EventArgs -## proc handleevent(e: EventArgs) = -## echo("Handled!") -## -## # Python way -## ee.on("EventName", handleevent) -## ee.emit("EventName", genericargs) -## -## # C/Java way -## # Declare a type -## type -## SomeObject = object of RootObj -## SomeEvent: EventHandler -## var myobj: SomeObject -## myobj.SomeEvent = initEventHandler("SomeEvent") -## myobj.SomeEvent.addHandler(handleevent) -## ee.emit(myobj.SomeEvent, genericargs) - -type - EventArgs* = object of RootObj ## Base object for event arguments that are passed to callback functions. - EventHandler* = tuple[name: string, handlers: seq[proc(e: EventArgs) {.closure.}]] ## An eventhandler for an event. - -type - EventEmitter* = object ## An object that fires events and holds event handlers for an object. - s: seq[EventHandler] - EventError* = object of ValueError - -{.deprecated: [TEventArgs: EventArgs, TEventHandler: EventHandler, - TEventEmitter: EventEmitter, EInvalidEvent: EventError].} - -proc initEventHandler*(name: string): EventHandler = - ## Initializes an EventHandler with the specified name and returns it. - result.handlers = @[] - result.name = name - -proc addHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) = - ## Adds the callback to the specified event handler. - handler.handlers.add(fn) - -proc removeHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) = - ## Removes the callback from the specified event handler. - for i in countup(0, len(handler.handlers) -1): - if fn == handler.handlers[i]: - handler.handlers.del(i) - break - -proc containsHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}): bool = - ## Checks if a callback is registered to this event handler. - return handler.handlers.contains(fn) - - -proc clearHandlers*(handler: var EventHandler) = - ## Clears all of the callbacks from the event handler. - setLen(handler.handlers, 0) - -proc getEventHandler(emitter: var EventEmitter, event: string): int = - for k in 0..high(emitter.s): - if emitter.s[k].name == event: return k - return -1 - -proc on*(emitter: var EventEmitter, event: string, fn: proc(e: EventArgs) {.closure.}) = - ## Assigns a event handler with the specified callback. If the event - ## doesn't exist, it will be created. - var i = getEventHandler(emitter, event) - if i < 0: - var eh = initEventHandler(event) - addHandler(eh, fn) - emitter.s.add(eh) - else: - addHandler(emitter.s[i], fn) - -proc emit*(emitter: var EventEmitter, eventhandler: var EventHandler, - args: EventArgs) = - ## Fires an event handler with specified event arguments. - for fn in items(eventhandler.handlers): fn(args) - -proc emit*(emitter: var EventEmitter, event: string, args: EventArgs) = - ## Fires an event handler with specified event arguments. - var i = getEventHandler(emitter, event) - if i >= 0: - emit(emitter, emitter.s[i], args) - -proc initEventEmitter*(): EventEmitter = - ## Creates and returns a new EventEmitter. - result.s = @[] diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index f8f115ecc..1d96fd6be 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -9,10 +9,10 @@ ## Floating-point environment. Handling of floating-point rounding and ## exceptions (overflow, division by zero, etc.). +## The types, vars and procs are bindings for the C standard library +## [<fenv.h>](https://en.cppreference.com/w/c/numeric/fenv) header. -{.deadCodeElim:on.} - -when defined(Posix) and not defined(haiku): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} var @@ -37,8 +37,8 @@ var FE_UPWARD* {.importc, header: "<fenv.h>".}: cint ## round toward +Inf FE_DFL_ENV* {.importc, header: "<fenv.h>".}: cint - ## macro of type pointer to fenv_t to be used as the argument - ## to functions taking an argument of type fenv_t; in this + ## macro of type pointer to `fenv_t` to be used as the argument + ## to functions taking an argument of type `fenv_t`; in this ## case the default environment will be used type @@ -102,80 +102,81 @@ proc feupdateenv*(envp: ptr Tfenv): cint {.importc, header: "<fenv.h>".} ## represented by object pointed to by `envp` and raise exceptions ## according to saved exceptions. -var FP_RADIX_INTERNAL {. importc: "FLT_RADIX" header: "<float.h>" .} : int - -template fpRadix* : int = FP_RADIX_INTERNAL +const + FLT_RADIX = 2 ## the radix of the exponent representation + + FLT_MANT_DIG = 24 ## the number of base FLT_RADIX digits in the mantissa part of a float + FLT_DIG = 6 ## the number of digits of precision of a float + FLT_MIN_EXP = -125 ## the minimum value of base FLT_RADIX in the exponent part of a float + FLT_MAX_EXP = 128 ## the maximum value of base FLT_RADIX in the exponent part of a float + FLT_MIN_10_EXP = -37 ## the minimum value in base 10 of the exponent part of a float + FLT_MAX_10_EXP = 38 ## the maximum value in base 10 of the exponent part of a float + FLT_MIN = 1.17549435e-38'f32 ## the minimum value of a float + FLT_MAX = 3.40282347e+38'f32 ## the maximum value of a float + FLT_EPSILON = 1.19209290e-07'f32 ## the difference between 1 and the least value greater than 1 of a float + + DBL_MANT_DIG = 53 ## the number of base FLT_RADIX digits in the mantissa part of a double + DBL_DIG = 15 ## the number of digits of precision of a double + DBL_MIN_EXP = -1021 ## the minimum value of base FLT_RADIX in the exponent part of a double + DBL_MAX_EXP = 1024 ## the maximum value of base FLT_RADIX in the exponent part of a double + DBL_MIN_10_EXP = -307 ## the minimum value in base 10 of the exponent part of a double + DBL_MAX_10_EXP = 308 ## the maximum value in base 10 of the exponent part of a double + DBL_MIN = 2.2250738585072014E-308 ## the minimal value of a double + DBL_MAX = 1.7976931348623157E+308 ## the minimal value of a double + DBL_EPSILON = 2.2204460492503131E-16 ## the difference between 1 and the least value greater than 1 of a double + +template fpRadix*: int = FLT_RADIX ## The (integer) value of the radix used to represent any floating ## point type on the architecture used to build the program. -var FLT_MANT_DIG {. importc: "FLT_MANT_DIG" header: "<float.h>" .} : int -var FLT_DIG {. importc: "FLT_DIG" header: "<float.h>" .} : int -var FLT_MIN_EXP {. importc: "FLT_MIN_EXP" header: "<float.h>" .} : int -var FLT_MAX_EXP {. importc: "FLT_MAX_EXP" header: "<float.h>" .} : int -var FLT_MIN_10_EXP {. importc: "FLT_MIN_10_EXP" header: "<float.h>" .} : int -var FLT_MAX_10_EXP {. importc: "FLT_MAX_10_EXP" header: "<float.h>" .} : int -var FLT_MIN {. importc: "FLT_MIN" header: "<float.h>" .} : cfloat -var FLT_MAX {. importc: "FLT_MAX" header: "<float.h>" .} : cfloat -var FLT_EPSILON {. importc: "FLT_EPSILON" header: "<float.h>" .} : cfloat - -var DBL_MANT_DIG {. importc: "DBL_MANT_DIG" header: "<float.h>" .} : int -var DBL_DIG {. importc: "DBL_DIG" header: "<float.h>" .} : int -var DBL_MIN_EXP {. importc: "DBL_MIN_EXP" header: "<float.h>" .} : int -var DBL_MAX_EXP {. importc: "DBL_MAX_EXP" header: "<float.h>" .} : int -var DBL_MIN_10_EXP {. importc: "DBL_MIN_10_EXP" header: "<float.h>" .} : int -var DBL_MAX_10_EXP {. importc: "DBL_MAX_10_EXP" header: "<float.h>" .} : int -var DBL_MIN {. importc: "DBL_MIN" header: "<float.h>" .} : cdouble -var DBL_MAX {. importc: "DBL_MAX" header: "<float.h>" .} : cdouble -var DBL_EPSILON {. importc: "DBL_EPSILON" header: "<float.h>" .} : cdouble - -template mantissaDigits*(T : typedesc[float32]) : int = FLT_MANT_DIG - ## Number of digits (in base ``floatingPointRadix``) in the mantissa +template mantissaDigits*(T: typedesc[float32]): int = FLT_MANT_DIG + ## Number of digits (in base `floatingPointRadix`) in the mantissa ## of 32-bit floating-point numbers. -template digits*(T : typedesc[float32]) : int = FLT_DIG +template digits*(T: typedesc[float32]): int = FLT_DIG ## Number of decimal digits that can be represented in a ## 32-bit floating-point type without losing precision. -template minExponent*(T : typedesc[float32]) : int = FLT_MIN_EXP +template minExponent*(T: typedesc[float32]): int = FLT_MIN_EXP ## Minimum (negative) exponent for 32-bit floating-point numbers. -template maxExponent*(T : typedesc[float32]) : int = FLT_MAX_EXP +template maxExponent*(T: typedesc[float32]): int = FLT_MAX_EXP ## Maximum (positive) exponent for 32-bit floating-point numbers. -template min10Exponent*(T : typedesc[float32]) : int = FLT_MIN_10_EXP +template min10Exponent*(T: typedesc[float32]): int = FLT_MIN_10_EXP ## Minimum (negative) exponent in base 10 for 32-bit floating-point ## numbers. -template max10Exponent*(T : typedesc[float32]) : int = FLT_MAX_10_EXP +template max10Exponent*(T: typedesc[float32]): int = FLT_MAX_10_EXP ## Maximum (positive) exponent in base 10 for 32-bit floating-point ## numbers. -template minimumPositiveValue*(T : typedesc[float32]) : float32 = FLT_MIN +template minimumPositiveValue*(T: typedesc[float32]): float32 = FLT_MIN ## The smallest positive (nonzero) number that can be represented in a ## 32-bit floating-point type. -template maximumPositiveValue*(T : typedesc[float32]) : float32 = FLT_MAX +template maximumPositiveValue*(T: typedesc[float32]): float32 = FLT_MAX ## The largest positive number that can be represented in a 32-bit ## floating-point type. -template epsilon*(T : typedesc[float32]): float32 = FLT_EPSILON +template epsilon*(T: typedesc[float32]): float32 = FLT_EPSILON ## The difference between 1.0 and the smallest number greater than ## 1.0 that can be represented in a 32-bit floating-point type. -template mantissaDigits*(T : typedesc[float64]) : int = DBL_MANT_DIG - ## Number of digits (in base ``floatingPointRadix``) in the mantissa +template mantissaDigits*(T: typedesc[float64]): int = DBL_MANT_DIG + ## Number of digits (in base `floatingPointRadix`) in the mantissa ## of 64-bit floating-point numbers. -template digits*(T : typedesc[float64]) : int = DBL_DIG +template digits*(T: typedesc[float64]): int = DBL_DIG ## Number of decimal digits that can be represented in a ## 64-bit floating-point type without losing precision. -template minExponent*(T : typedesc[float64]) : int = DBL_MIN_EXP +template minExponent*(T: typedesc[float64]): int = DBL_MIN_EXP ## Minimum (negative) exponent for 64-bit floating-point numbers. -template maxExponent*(T : typedesc[float64]) : int = DBL_MAX_EXP +template maxExponent*(T: typedesc[float64]): int = DBL_MAX_EXP ## Maximum (positive) exponent for 64-bit floating-point numbers. -template min10Exponent*(T : typedesc[float64]) : int = DBL_MIN_10_EXP +template min10Exponent*(T: typedesc[float64]): int = DBL_MIN_10_EXP ## Minimum (negative) exponent in base 10 for 64-bit floating-point ## numbers. -template max10Exponent*(T : typedesc[float64]) : int = DBL_MAX_10_EXP +template max10Exponent*(T: typedesc[float64]): int = DBL_MAX_10_EXP ## Maximum (positive) exponent in base 10 for 64-bit floating-point ## numbers. -template minimumPositiveValue*(T : typedesc[float64]) : float64 = DBL_MIN +template minimumPositiveValue*(T: typedesc[float64]): float64 = DBL_MIN ## The smallest positive (nonzero) number that can be represented in a ## 64-bit floating-point type. -template maximumPositiveValue*(T : typedesc[float64]) : float64 = DBL_MAX +template maximumPositiveValue*(T: typedesc[float64]): float64 = DBL_MAX ## The largest positive number that can be represented in a 64-bit ## floating-point type. -template epsilon*(T : typedesc[float64]): float64 = DBL_EPSILON +template epsilon*(T: typedesc[float64]): float64 = DBL_EPSILON ## The difference between 1.0 and the smallest number greater than ## 1.0 that can be represented in a 64-bit floating-point type. diff --git a/lib/pure/fsmonitor.nim b/lib/pure/fsmonitor.nim deleted file mode 100644 index 115c4739e..000000000 --- a/lib/pure/fsmonitor.nim +++ /dev/null @@ -1,226 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module allows you to monitor files or directories for changes using -## asyncio. -## -## Windows support is not yet implemented. -## -## **Note:** This module uses ``inotify`` on Linux (Other Unixes are not yet -## supported). ``inotify`` was merged into the 2.6.13 Linux kernel, this -## module will therefore not work with any Linux kernel prior to that, unless -## it has been patched to support inotify. - -when defined(linux) or defined(nimdoc): - from posix import read -else: - {.error: "Your platform is not supported.".} - -import inotify, os, asyncio, tables - -type - FSMonitor* = ref FSMonitorObj - FSMonitorObj = object of RootObj - fd: cint - handleEvent: proc (m: FSMonitor, ev: MonitorEvent) {.closure.} - targets: Table[cint, string] - - MonitorEventType* = enum ## Monitor event type - MonitorAccess, ## File was accessed. - MonitorAttrib, ## Metadata changed. - MonitorCloseWrite, ## Writable file was closed. - MonitorCloseNoWrite, ## Non-writable file closed. - MonitorCreate, ## Subfile was created. - MonitorDelete, ## Subfile was deleted. - MonitorDeleteSelf, ## Watched file/directory was itself deleted. - MonitorModify, ## File was modified. - MonitorMoveSelf, ## Self was moved. - MonitorMoved, ## File was moved. - MonitorOpen, ## File was opened. - MonitorAll ## Filter for all event types. - - MonitorEvent* = object - case kind*: MonitorEventType ## Type of the event. - of MonitorMoveSelf, MonitorMoved: - oldPath*: string ## Old absolute location - newPath*: string ## New absolute location - else: - fullname*: string ## Absolute filename of the file/directory affected. - name*: string ## Non absolute filepath of the file/directory - ## affected relative to the directory watched. - ## "" if this event refers to the file/directory - ## watched. - wd*: cint ## Watch descriptor. - -{.deprecated: [PFSMonitor: FSMonitor, TFSMonitor: FSMonitorObj, - TMonitorEventType: MonitorEventType, TMonitorEvent: MonitorEvent].} - -const - MaxEvents = 100 - -proc newMonitor*(): FSMonitor = - ## Creates a new file system monitor. - new(result) - result.targets = initTable[cint, string]() - result.fd = inotifyInit() - if result.fd < 0: - raiseOSError(osLastError()) - -proc add*(monitor: FSMonitor, target: string, - filters = {MonitorAll}): cint {.discardable.} = - ## Adds ``target`` which may be a directory or a file to the list of - ## watched paths of ``monitor``. - ## You can specify the events to report using the ``filters`` parameter. - - var INFilter = 0 - for f in filters: - case f - of MonitorAccess: INFilter = INFilter or IN_ACCESS - of MonitorAttrib: INFilter = INFilter or IN_ATTRIB - of MonitorCloseWrite: INFilter = INFilter or IN_CLOSE_WRITE - of MonitorCloseNoWrite: INFilter = INFilter or IN_CLOSE_NO_WRITE - of MonitorCreate: INFilter = INFilter or IN_CREATE - of MonitorDelete: INFilter = INFilter or IN_DELETE - of MonitorDeleteSelf: INFilter = INFilter or IN_DELETE_SELF - of MonitorModify: INFilter = INFilter or IN_MODIFY - of MonitorMoveSelf: INFilter = INFilter or IN_MOVE_SELF - of MonitorMoved: INFilter = INFilter or IN_MOVED_FROM or IN_MOVED_TO - of MonitorOpen: INFilter = INFilter or IN_OPEN - of MonitorAll: INFilter = INFilter or IN_ALL_EVENTS - - result = inotifyAddWatch(monitor.fd, target, INFilter.uint32) - if result < 0: - raiseOSError(osLastError()) - monitor.targets.add(result, target) - -proc del*(monitor: FSMonitor, wd: cint) = - ## Removes watched directory or file as specified by ``wd`` from ``monitor``. - ## - ## If ``wd`` is not a part of ``monitor`` an EOS error is raised. - if inotifyRmWatch(monitor.fd, wd) < 0: - raiseOSError(osLastError()) - -proc getEvent(m: FSMonitor, fd: cint): seq[MonitorEvent] = - result = @[] - let size = (sizeof(INotifyEvent)+2000)*MaxEvents - var buffer = newString(size) - - let le = read(fd, addr(buffer[0]), size) - - var movedFrom = initTable[cint, tuple[wd: cint, old: string]]() - - var i = 0 - while i < le: - var event = cast[ptr INotifyEvent](addr(buffer[i])) - var mev: MonitorEvent - mev.wd = event.wd - if event.len.int != 0: - let cstr = event.name.addr.cstring - mev.name = $cstr - else: - mev.name = "" - - if (event.mask.int and IN_MOVED_FROM) != 0: - # Moved from event, add to m's collection - movedFrom.add(event.cookie.cint, (mev.wd, mev.name)) - inc(i, sizeof(INotifyEvent) + event.len.int) - continue - elif (event.mask.int and IN_MOVED_TO) != 0: - mev.kind = MonitorMoved - assert movedFrom.hasKey(event.cookie.cint) - # Find the MovedFrom event. - mev.oldPath = movedFrom[event.cookie.cint].old - mev.newPath = "" # Set later - # Delete it from the Table - movedFrom.del(event.cookie.cint) - elif (event.mask.int and IN_ACCESS) != 0: mev.kind = MonitorAccess - elif (event.mask.int and IN_ATTRIB) != 0: mev.kind = MonitorAttrib - elif (event.mask.int and IN_CLOSE_WRITE) != 0: - mev.kind = MonitorCloseWrite - elif (event.mask.int and IN_CLOSE_NOWRITE) != 0: - mev.kind = MonitorCloseNoWrite - elif (event.mask.int and IN_CREATE) != 0: mev.kind = MonitorCreate - elif (event.mask.int and IN_DELETE) != 0: - mev.kind = MonitorDelete - elif (event.mask.int and IN_DELETE_SELF) != 0: - mev.kind = MonitorDeleteSelf - elif (event.mask.int and IN_MODIFY) != 0: mev.kind = MonitorModify - elif (event.mask.int and IN_MOVE_SELF) != 0: - mev.kind = MonitorMoveSelf - elif (event.mask.int and IN_OPEN) != 0: mev.kind = MonitorOpen - - if mev.kind != MonitorMoved: - mev.fullname = "" - - result.add(mev) - inc(i, sizeof(INotifyEvent) + event.len.int) - - # If movedFrom events have not been matched with a moveTo. File has - # been moved to an unwatched location, emit a MonitorDelete. - for cookie, t in pairs(movedFrom): - var mev: MonitorEvent - mev.kind = MonitorDelete - mev.wd = t.wd - mev.name = t.old - result.add(mev) - -proc FSMonitorRead(h: RootRef) = - var events = FSMonitor(h).getEvent(FSMonitor(h).fd) - #var newEv: MonitorEvent - for ev in events: - var target = FSMonitor(h).targets[ev.wd] - var newEv = ev - if newEv.kind == MonitorMoved: - newEv.oldPath = target / newEv.oldPath - newEv.newPath = target / newEv.name - else: - newEv.fullName = target / newEv.name - FSMonitor(h).handleEvent(FSMonitor(h), newEv) - -proc toDelegate(m: FSMonitor): Delegate = - result = newDelegate() - result.deleVal = m - result.fd = (type(result.fd))(m.fd) - result.mode = fmRead - result.handleRead = FSMonitorRead - result.open = true - -proc register*(d: Dispatcher, monitor: FSMonitor, - handleEvent: proc (m: FSMonitor, ev: MonitorEvent) {.closure.}) = - ## Registers ``monitor`` with dispatcher ``d``. - monitor.handleEvent = handleEvent - var deleg = toDelegate(monitor) - d.register(deleg) - -when not defined(testing) and isMainModule: - proc main = - var - disp = newDispatcher() - monitor = newMonitor() - n = 0 - n = monitor.add("/tmp") - assert n == 1 - n = monitor.add("/tmp", {MonitorAll}) - assert n == 1 - n = monitor.add("/tmp", {MonitorCloseWrite, MonitorCloseNoWrite}) - assert n == 1 - n = monitor.add("/tmp", {MonitorMoved, MonitorOpen, MonitorAccess}) - assert n == 1 - disp.register(monitor, - proc (m: FSMonitor, ev: MonitorEvent) = - echo("Got event: ", ev.kind) - if ev.kind == MonitorMoved: - echo("From ", ev.oldPath, " to ", ev.newPath) - echo("Name is ", ev.name) - else: - echo("Name ", ev.name, " fullname ", ev.fullName)) - - while true: - if not disp.poll(): break - main() diff --git a/lib/pure/future.nim b/lib/pure/future.nim deleted file mode 100644 index 4767266e5..000000000 --- a/lib/pure/future.nim +++ /dev/null @@ -1,183 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements experimental features which may soon be moved to -## the system module (or other more appropriate modules). - -import macros - -proc createProcType(p, b: NimNode): NimNode {.compileTime.} = - #echo treeRepr(p) - #echo treeRepr(b) - result = newNimNode(nnkProcTy) - var formalParams = newNimNode(nnkFormalParams) - - formalParams.add b - - case p.kind - of nnkPar: - for i in 0 .. <p.len: - let ident = p[i] - var identDefs = newNimNode(nnkIdentDefs) - case ident.kind - of nnkExprColonExpr: - identDefs.add ident[0] - identDefs.add ident[1] - of nnkIdent: - identDefs.add newIdentNode("i" & $i) - identDefs.add(ident) - else: - error("Incorrect type list in proc type declaration.") - identDefs.add newEmptyNode() - formalParams.add identDefs - of nnkIdent: - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add newIdentNode("i0") - identDefs.add(p) - identDefs.add newEmptyNode() - formalParams.add identDefs - else: - error("Incorrect type list in proc type declaration.") - - result.add formalParams - result.add newEmptyNode() - #echo(treeRepr(result)) - #echo(result.toStrLit()) - -macro `=>`*(p, b: expr): expr {.immediate.} = - ## Syntax sugar for anonymous procedures. - ## - ## .. code-block:: nim - ## - ## proc passTwoAndTwo(f: (int, int) -> int): int = - ## f(2, 2) - ## - ## passTwoAndTwo((x, y) => x + y) # 4 - - #echo treeRepr(p) - #echo(treeRepr(b)) - var params: seq[NimNode] = @[newIdentNode("auto")] - - case p.kind - of nnkPar: - for c in children(p): - var identDefs = newNimNode(nnkIdentDefs) - case c.kind - of nnkExprColonExpr: - identDefs.add(c[0]) - identDefs.add(c[1]) - identDefs.add(newEmptyNode()) - of nnkIdent: - identDefs.add(c) - identDefs.add(newIdentNode("auto")) - identDefs.add(newEmptyNode()) - of nnkInfix: - if c[0].kind == nnkIdent and c[0].ident == !"->": - var procTy = createProcType(c[1], c[2]) - params[0] = procTy[0][0] - for i in 1 .. <procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $c[0].ident & ").") - break - else: - echo treeRepr c - error("Incorrect procedure parameter list.") - params.add(identDefs) - of nnkIdent: - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) - identDefs.add(newIdentNode("auto")) - identDefs.add(newEmptyNode()) - params.add(identDefs) - of nnkInfix: - if p[0].kind == nnkIdent and p[0].ident == !"->": - var procTy = createProcType(p[1], p[2]) - params[0] = procTy[0][0] - for i in 1 .. <procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $p[0].ident & ").") - else: - error("Incorrect procedure parameter list.") - result = newProc(params = params, body = b, procType = nnkLambda) - #echo(result.treeRepr) - #echo(result.toStrLit()) - #return result # TODO: Bug? - -macro `->`*(p, b: expr): expr {.immediate.} = - ## Syntax sugar for procedure types. - ## - ## .. code-block:: nim - ## - ## proc pass2(f: (float, float) -> float): float = - ## f(2, 2) - ## - ## # is the same as: - ## - ## proc pass2(f: proc (x, y: float): float): float = - ## f(2, 2) - - result = createProcType(p, b) - -type ListComprehension = object -var lc*: ListComprehension - -macro `[]`*(lc: ListComprehension, comp, typ: expr): expr = - ## List comprehension, returns a sequence. `comp` is the actual list - ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is - ## the type that will be stored inside the result seq. - ## - ## .. code-block:: nimrod - ## - ## echo lc[x | (x <- 1..10, x mod 2 == 0), int] - ## - ## const n = 20 - ## echo lc[(x,y,z) | (x <- 1..n, y <- x..n, z <- y..n, x*x + y*y == z*z), - ## tuple[a,b,c: int]] - - expectLen(comp, 3) - expectKind(comp, nnkInfix) - expectKind(comp[0], nnkIdent) - assert($comp[0].ident == "|") - - result = newCall( - newDotExpr( - newIdentNode("result"), - newIdentNode("add")), - comp[1]) - - for i in countdown(comp[2].len-1, 0): - let x = comp[2][i] - expectMinLen(x, 1) - if x[0].kind == nnkIdent and $x[0].ident == "<-": - expectLen(x, 3) - result = newNimNode(nnkForStmt).add(x[1], x[2], result) - else: - result = newIfStmt((x, result)) - - result = newNimNode(nnkCall).add( - newNimNode(nnkPar).add( - newNimNode(nnkLambda).add( - newEmptyNode(), - newEmptyNode(), - newEmptyNode(), - newNimNode(nnkFormalParams).add( - newNimNode(nnkBracketExpr).add( - newIdentNode("seq"), - typ)), - newEmptyNode(), - newEmptyNode(), - newStmtList( - newAssignment( - newIdentNode("result"), - newNimNode(nnkPrefix).add( - newIdentNode("@"), - newNimNode(nnkBracket))), - result)))) diff --git a/lib/pure/gentabs.nim b/lib/pure/gentabs.nim deleted file mode 100644 index e6a05ec63..000000000 --- a/lib/pure/gentabs.nim +++ /dev/null @@ -1,209 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## The ``gentabs`` module implements an efficient hash table that is a -## key-value mapping. The keys are required to be strings, but the values -## may be any Nim or user defined type. This module supports matching -## of keys in case-sensitive, case-insensitive and style-insensitive modes. - -{.deprecated.} - -import - os, hashes, strutils - -type - GenTableMode* = enum ## describes the table's key matching mode - modeCaseSensitive, ## case sensitive matching of keys - modeCaseInsensitive, ## case insensitive matching of keys - modeStyleInsensitive ## style sensitive matching of keys - - GenKeyValuePair[T] = tuple[key: string, val: T] - GenKeyValuePairSeq[T] = seq[GenKeyValuePair[T]] - GenTable*[T] = object of RootObj - counter: int - data: GenKeyValuePairSeq[T] - mode: GenTableMode - - PGenTable*[T] = ref GenTable[T] ## use this type to declare hash tables - -{.deprecated: [TGenTableMode: GenTableMode, TGenKeyValuePair: GenKeyValuePair, - TGenKeyValuePairSeq: GenKeyValuePairSeq, TGenTable: GenTable].} - -const - growthFactor = 2 - startSize = 64 - - -proc len*[T](tbl: PGenTable[T]): int {.inline.} = - ## returns the number of keys in `tbl`. - result = tbl.counter - -iterator pairs*[T](tbl: PGenTable[T]): tuple[key: string, value: T] = - ## iterates over any (key, value) pair in the table `tbl`. - for h in 0..high(tbl.data): - if not isNil(tbl.data[h].key): - yield (tbl.data[h].key, tbl.data[h].val) - -proc myhash[T](tbl: PGenTable[T], key: string): Hash = - case tbl.mode - of modeCaseSensitive: result = hashes.hash(key) - of modeCaseInsensitive: result = hashes.hashIgnoreCase(key) - of modeStyleInsensitive: result = hashes.hashIgnoreStyle(key) - -proc myCmp[T](tbl: PGenTable[T], a, b: string): bool = - case tbl.mode - of modeCaseSensitive: result = cmp(a, b) == 0 - of modeCaseInsensitive: result = cmpIgnoreCase(a, b) == 0 - of modeStyleInsensitive: result = cmpIgnoreStyle(a, b) == 0 - -proc mustRehash(length, counter: int): bool = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -proc newGenTable*[T](mode: GenTableMode): PGenTable[T] = - ## creates a new generic hash table that is empty. - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash - -proc rawGet[T](tbl: PGenTable[T], key: string): int = - var h: Hash - h = myhash(tbl, key) and high(tbl.data) # start with real hash value - while not isNil(tbl.data[h].key): - if myCmp(tbl, tbl.data[h].key, key): - return h - h = nextTry(h, high(tbl.data)) - result = - 1 - -proc rawInsert[T](tbl: PGenTable[T], data: var GenKeyValuePairSeq[T], - key: string, val: T) = - var h: Hash - h = myhash(tbl, key) and high(data) - while not isNil(data[h].key): - h = nextTry(h, high(data)) - data[h].key = key - data[h].val = val - -proc enlarge[T](tbl: PGenTable[T]) = - var n: GenKeyValuePairSeq[T] - newSeq(n, len(tbl.data) * growthFactor) - for i in countup(0, high(tbl.data)): - if not isNil(tbl.data[i].key): - rawInsert[T](tbl, n, tbl.data[i].key, tbl.data[i].val) - swap(tbl.data, n) - -proc hasKey*[T](tbl: PGenTable[T], key: string): bool = - ## returns true iff `key` is in the table `tbl`. - result = rawGet(tbl, key) >= 0 - -proc `[]`*[T](tbl: PGenTable[T], key: string): T = - ## retrieves the value at ``tbl[key]``. If `key` is not in `tbl`, - ## default(T) is returned and no exception is raised. One can check - ## with ``hasKey`` whether the key exists. - var index = rawGet(tbl, key) - if index >= 0: result = tbl.data[index].val - -proc `[]=`*[T](tbl: PGenTable[T], key: string, val: T) = - ## puts a (key, value)-pair into `tbl`. - var index = rawGet(tbl, key) - if index >= 0: - tbl.data[index].val = val - else: - if mustRehash(len(tbl.data), tbl.counter): enlarge(tbl) - rawInsert(tbl, tbl.data, key, val) - inc(tbl.counter) - - -when isMainModule: - # - # Verify tables of integer values (string keys) - # - var x = newGenTable[int](modeCaseInsensitive) - x["one"] = 1 - x["two"] = 2 - x["three"] = 3 - x["four"] = 4 - x["five"] = 5 - assert(len(x) == 5) # length procedure works - assert(x["one"] == 1) # case-sensitive lookup works - assert(x["ONE"] == 1) # case-insensitive should work for this table - assert(x["one"]+x["two"] == 3) # make sure we're getting back ints - assert(x.hasKey("one")) # hasKey should return 'true' for a key - # of "one"... - assert(not x.hasKey("NOPE")) # ...but key "NOPE" is not in the table. - for k,v in pairs(x): # make sure the 'pairs' iterator works - assert(x[k]==v) - - # - # Verify a table of user-defined types - # - type - MyType = tuple[first, second: string] # a pair of strings - {.deprecated: [TMyType: MyType].} - - var y = newGenTable[MyType](modeCaseInsensitive) # hash table where each - # value is MyType tuple - - #var junk: MyType = ("OK", "Here") - - #echo junk.first, " ", junk.second - - y["Hello"] = ("Hello", "World") - y["Goodbye"] = ("Goodbye", "Everyone") - #y["Hello"] = MyType( ("Hello", "World") ) - #y["Goodbye"] = MyType( ("Goodbye", "Everyone") ) - - assert( not isNil(y["Hello"].first) ) - assert( y["Hello"].first == "Hello" ) - assert( y["Hello"].second == "World" ) - - # - # Verify table of tables - # - var z: PGenTable[ PGenTable[int] ] # hash table where each value is - # a hash table of ints - - z = newGenTable[PGenTable[int]](modeCaseInsensitive) - z["first"] = newGenTable[int](modeCaseInsensitive) - z["first"]["one"] = 1 - z["first"]["two"] = 2 - z["first"]["three"] = 3 - - z["second"] = newGenTable[int](modeCaseInsensitive) - z["second"]["red"] = 10 - z["second"]["blue"] = 20 - - assert(len(z) == 2) # length of outer table - assert(len(z["first"]) == 3) # length of "first" table - assert(len(z["second"]) == 2) # length of "second" table - assert( z["first"]["one"] == 1) # retrieve from first inner table - assert( z["second"]["red"] == 10) # retrieve from second inner table - - when false: - # disabled: depends on hash order: - var output = "" - for k, v in pairs(z): - output.add( "$# ($#) ->\L" % [k,$len(v)] ) - for k2,v2 in pairs(v): - output.add( " $# <-> $#\L" % [k2,$v2] ) - - let expected = unindent """ - first (3) -> - two <-> 2 - three <-> 3 - one <-> 1 - second (2) -> - red <-> 10 - blue <-> 20 - """ - assert output == expected diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 11af81149..1038d55a1 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -9,64 +9,188 @@ ## This module implements efficient computations of hash values for diverse ## Nim types. All the procs are based on these two building blocks: -## - `!& proc <#!&>`_ used to start or mix a hash value, and -## - `!$ proc <#!$>`_ used to *finish* the hash value. -## If you want to implement hash procs for -## your custom types you will end up writing the following kind of skeleton of -## code: +## - `!& proc <#!&,Hash,int>`_ used to start or mix a hash value, and +## - `!$ proc <#!$,Hash>`_ used to finish the hash value. ## -## .. code-block:: Nim -## proc hash(x: Something): Hash = -## ## Computes a Hash from `x`. -## var h: Hash = 0 -## # Iterate over parts of `x`. -## for xAtom in x: -## # Mix the atom with the partial hash. -## h = h !& xAtom -## # Finish the hash. -## result = !$h +## If you want to implement hash procs for your custom types, +## you will end up writing the following kind of skeleton of code: + +runnableExamples: + type + Something = object + foo: int + bar: string + + iterator items(x: Something): Hash = + yield hash(x.foo) + yield hash(x.bar) + + proc hash(x: Something): Hash = + ## Computes a Hash from `x`. + var h: Hash = 0 + # Iterate over parts of `x`. + for xAtom in x: + # Mix the atom with the partial hash. + h = h !& xAtom + # Finish the hash. + result = !$h + +## If your custom types contain fields for which there already is a `hash` proc, +## you can simply hash together the hash values of the individual fields: + +runnableExamples: + type + Something = object + foo: int + bar: string + + proc hash(x: Something): Hash = + ## Computes a Hash from `x`. + var h: Hash = 0 + h = h !& hash(x.foo) + h = h !& hash(x.bar) + result = !$h + +## .. important:: Use `-d:nimPreviewHashRef` to +## enable hashing `ref`s. It is expected that this behavior +## becomes the new default in upcoming versions. ## -## If your custom types contain fields for which there already is a hash proc, -## like for example objects made up of ``strings``, you can simply hash -## together the hash value of the individual fields: +## .. note:: If the type has a `==` operator, the following must hold: +## If two values compare equal, their hashes must also be equal. ## -## .. code-block:: Nim -## proc hash(x: Something): Hash = -## ## Computes a Hash from `x`. -## var h: Hash = 0 -## h = h !& hash(x.foo) -## h = h !& hash(x.bar) -## result = !$h +## See also +## ======== +## * `md5 module <md5.html>`_ for the MD5 checksum algorithm +## * `base64 module <base64.html>`_ for a Base64 encoder and decoder +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm +## * `tables module <tables.html>`_ for hash tables + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions -import - strutils, etcpriv type - Hash* = int ## a hash value; hash tables using these values should - ## always have a size of a power of two and can use the ``and`` - ## operator instead of ``mod`` for truncation of the hash value. -{.deprecated: [THash: Hash].} + Hash* = int ## A hash value. Hash tables using these values should + ## always have a size of a power of two so they can use the `and` + ## operator instead of `mod` for truncation of the hash value. proc `!&`*(h: Hash, val: int): Hash {.inline.} = - ## mixes a hash value `h` with `val` to produce a new hash value. This is - ## only needed if you need to implement a hash proc for a new datatype. - result = h +% val - result = result +% result shl 10 - result = result xor (result shr 6) + ## Mixes a hash value `h` with `val` to produce a new hash value. + ## + ## This is only needed if you need to implement a `hash` proc for a new datatype. + let h = cast[uint](h) + let val = cast[uint](val) + var res = h + val + res = res + res shl 10 + res = res xor (res shr 6) + result = cast[Hash](res) proc `!$`*(h: Hash): Hash {.inline.} = - ## finishes the computation of the hash value. This is - ## only needed if you need to implement a hash proc for a new datatype. - result = h +% h shl 3 - result = result xor (result shr 11) - result = result +% result shl 15 + ## Finishes the computation of the hash value. + ## + ## This is only needed if you need to implement a `hash` proc for a new datatype. + let h = cast[uint](h) # Hash is practically unsigned. + var res = h + h shl 3 + res = res xor (res shr 11) + res = res + res shl 15 + result = cast[Hash](res) + +proc hiXorLoFallback64(a, b: uint64): uint64 {.inline.} = + let # Fall back in 64-bit arithmetic + aH = a shr 32 + aL = a and 0xFFFFFFFF'u64 + bH = b shr 32 + bL = b and 0xFFFFFFFF'u64 + rHH = aH * bH + rHL = aH * bL + rLH = aL * bH + rLL = aL * bL + t = rLL + (rHL shl 32) + var c = if t < rLL: 1'u64 else: 0'u64 + let lo = t + (rLH shl 32) + c += (if lo < t: 1'u64 else: 0'u64) + let hi = rHH + (rHL shr 32) + (rLH shr 32) + c + return hi xor lo + +proc hiXorLo(a, b: uint64): uint64 {.inline.} = + # XOR of the high & low 8 bytes of the full 16 byte product. + when nimvm: + result = hiXorLoFallback64(a, b) # `result =` is necessary here. + else: + when Hash.sizeof < 8: + result = hiXorLoFallback64(a, b) + elif defined(gcc) or defined(llvm_gcc) or defined(clang): + {.emit: """__uint128_t r = `a`; r *= `b`; `result` = (r >> 64) ^ r;""".} + elif defined(windows) and not defined(tcc): + proc umul128(a, b: uint64, c: ptr uint64): uint64 {.importc: "_umul128", header: "intrin.h".} + var b = b + let c = umul128(a, b, addr b) + result = c xor b + else: + result = hiXorLoFallback64(a, b) + +when defined(js): + import std/jsbigints + import std/private/jsutils + + proc hiXorLoJs(a, b: JsBigInt): JsBigInt = + let + prod = a * b + mask = big"0xffffffffffffffff" # (big"1" shl big"64") - big"1" + result = (prod shr big"64") xor (prod and mask) + + template hashWangYiJS(x: JsBigInt): Hash = + let + P0 = big"0xa0761d6478bd642f" + P1 = big"0xe7037ed1a0b428db" + P58 = big"0xeb44accab455d16d" # big"0xeb44accab455d165" xor big"8" + res = hiXorLoJs(hiXorLoJs(P0, x xor P1), P58) + cast[Hash](toNumber(wrapToInt(res, 32))) + + template toBits(num: float): JsBigInt = + let + x = newArrayBuffer(8) + y = newFloat64Array(x) + if hasBigUint64Array(): + let z = newBigUint64Array(x) + y[0] = num + z[0] + else: + let z = newUint32Array(x) + y[0] = num + big(z[0]) + big(z[1]) shl big(32) + +proc hashWangYi1*(x: int64|uint64|Hash): Hash {.inline.} = + ## Wang Yi's hash_v1 for 64-bit ints (see https://github.com/rurban/smhasher for + ## more details). This passed all scrambling tests in Spring 2019 and is simple. + ## + ## **Note:** It's ok to define `proc(x: int16): Hash = hashWangYi1(Hash(x))`. + const P0 = 0xa0761d6478bd642f'u64 + const P1 = 0xe7037ed1a0b428db'u64 + const P58 = 0xeb44accab455d165'u64 xor 8'u64 + template h(x): untyped = hiXorLo(hiXorLo(P0, uint64(x) xor P1), P58) + when nimvm: + when defined(js): # Nim int64<->JS Number & VM match => JS gets 32-bit hash + result = cast[Hash](h(x)) and cast[Hash](0xFFFFFFFF) + else: + result = cast[Hash](h(x)) + else: + when defined(js): + if hasJsBigInt(): + result = hashWangYiJS(big(x)) + else: + result = cast[Hash](x) and cast[Hash](0xFFFFFFFF) + else: + result = cast[Hash](h(x)) proc hashData*(data: pointer, size: int): Hash = - ## hashes an array of bytes of size `size` + ## Hashes an array of bytes of size `size`. var h: Hash = 0 when defined(js): var p: cstring - asm """`p` = `Data`;""" + {.emit: """`p` = `Data`;""".} else: var p = cast[cstring](data) var i = 0 @@ -77,13 +201,23 @@ proc hashData*(data: pointer, size: int): Hash = dec(s) result = !$h +proc hashIdentity*[T: Ordinal|enum](x: T): Hash {.inline, since: (1, 3).} = + ## The identity hash, i.e. `hashIdentity(x) = x`. + cast[Hash](ord(x)) + +when defined(nimIntHash1): + proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = + ## Efficient hashing of integers. + cast[Hash](ord(x)) +else: + proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = + ## Efficient hashing of integers. + hashWangYi1(uint64(ord(x))) + when defined(js): var objectID = 0 - -proc hash*(x: pointer): Hash {.inline.} = - ## efficient hashing of pointers - when defined(js): - asm """ + proc getObjectId(x: pointer): int = + {.emit: """ if (typeof `x` == "object") { if ("_NimID" in `x`) `result` = `x`["_NimID"]; @@ -92,53 +226,378 @@ proc hash*(x: pointer): Hash {.inline.} = `x`["_NimID"] = `result`; } } - """ + """.} + +proc hash*(x: pointer): Hash {.inline.} = + ## Efficient `hash` overload. + when defined(js): + let y = getObjectId(x) + else: + let y = cast[int](x) + hash(y) # consistent with code expecting scrambled hashes depending on `nimIntHash1`. + +proc hash*[T](x: ptr[T]): Hash {.inline.} = + ## Efficient `hash` overload. + runnableExamples: + var a: array[10, uint8] + assert a[0].addr.hash != a[1].addr.hash + assert cast[pointer](a[0].addr).hash == a[0].addr.hash + hash(cast[pointer](x)) + +when defined(nimPreviewHashRef) or defined(nimdoc): + proc hash*[T](x: ref[T]): Hash {.inline.} = + ## Efficient `hash` overload. + ## + ## .. important:: Use `-d:nimPreviewHashRef` to + ## enable hashing `ref`s. It is expected that this behavior + ## becomes the new default in upcoming versions. + runnableExamples("-d:nimPreviewHashRef"): + type A = ref object + x: int + let a = A(x: 3) + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + runnableExamples("-d:nimPreviewHashRef"): + # you can overload `hash` if you want to customize semantics + type A[T] = ref object + x, y: T + proc hash(a: A): Hash = hash(a.x) + assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash + # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` + # or `proc hash*[T: ref | ptr](x: T): Hash` + hash(cast[pointer](x)) + +proc hash*(x: float): Hash {.inline.} = + ## Efficient hashing of floats. + let y = x + 0.0 # for denormalization + when nimvm: + # workaround a JS VM bug: bug #16547 + result = hashWangYi1(cast[int64](float64(y))) else: - result = (cast[Hash](x)) shr 3 # skip the alignment + when not defined(js): + result = hashWangYi1(cast[Hash](y)) + else: + result = hashWangYiJS(toBits(y)) + +# Forward declarations before methods that hash containers. This allows +# containers to contain other containers +proc hash*[A](x: openArray[A]): Hash +proc hash*[A](x: set[A]): Hash + -when not defined(booting): - proc hash*[T: proc](x: T): Hash {.inline.} = - ## efficient hashing of proc vars; closures are supported too. - when T is "closure": - result = hash(rawProc(x)) !& hash(rawEnv(x)) +when defined(js): + proc imul(a, b: uint32): uint32 = + # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/imul + let mask = 0xffff'u32 + var + aHi = (a shr 16) and mask + aLo = a and mask + bHi = (b shr 16) and mask + bLo = b and mask + result = (aLo * bLo) + (aHi * bLo + aLo * bHi) shl 16 +else: + template imul(a, b: uint32): untyped = a * b + +proc rotl32(x: uint32, r: int): uint32 {.inline.} = + (x shl r) or (x shr (32 - r)) + +proc murmurHash(x: openArray[byte]): Hash = + # https://github.com/PeterScott/murmur3/blob/master/murmur3.c + const + c1 = 0xcc9e2d51'u32 + c2 = 0x1b873593'u32 + n1 = 0xe6546b64'u32 + m1 = 0x85ebca6b'u32 + m2 = 0xc2b2ae35'u32 + let + size = len(x) + stepSize = 4 # 32-bit + n = size div stepSize + var + h1: uint32 + i = 0 + + + template impl = + var j = stepSize + while j > 0: + dec j + k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + + # body + while i < n * stepSize: + var k1: uint32 + + when nimvm: + impl() else: - result = hash(pointer(x)) + when declared(copyMem): + copyMem(addr k1, addr x[i], 4) + else: + impl() + inc i, stepSize + + k1 = imul(k1, c1) + k1 = rotl32(k1, 15) + k1 = imul(k1, c2) -proc hash*(x: int): Hash {.inline.} = - ## efficient hashing of integers - result = x + h1 = h1 xor k1 + h1 = rotl32(h1, 13) + h1 = h1*5 + n1 -proc hash*(x: int64): Hash {.inline.} = - ## efficient hashing of int64 integers - result = toU32(x) + # tail + var k1: uint32 + var rem = size mod stepSize + while rem > 0: + dec rem + k1 = (k1 shl 8) or (ord(x[i+rem])).uint32 + k1 = imul(k1, c1) + k1 = rotl32(k1, 15) + k1 = imul(k1, c2) + h1 = h1 xor k1 -proc hash*(x: char): Hash {.inline.} = - ## efficient hashing of characters - result = ord(x) + # finalization + h1 = h1 xor size.uint32 + h1 = h1 xor (h1 shr 16) + h1 = imul(h1, m1) + h1 = h1 xor (h1 shr 13) + h1 = imul(h1, m2) + h1 = h1 xor (h1 shr 16) + return cast[Hash](h1) -proc hash*[T: Ordinal](x: T): Hash {.inline.} = - ## efficient hashing of other ordinal types (e.g., enums) - result = ord(x) +proc hashVmImpl(x: cstring, sPos, ePos: int): Hash = + raiseAssert "implementation override in compiler/vmops.nim" + +proc hashVmImpl(x: string, sPos, ePos: int): Hash = + raiseAssert "implementation override in compiler/vmops.nim" + +proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash = + raiseAssert "implementation override in compiler/vmops.nim" + +proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash = + raiseAssert "implementation override in compiler/vmops.nim" + +const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses +const k1 = 0xb492b66fbe98f273u64 +const k2 = 0x9ae16a3b2f90404fu64 + +proc load4e(s: openArray[byte], o=0): uint32 {.inline.} = + uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or + uint32(s[o + 1]) shl 8 or uint32(s[o + 0]) + +proc load8e(s: openArray[byte], o=0): uint64 {.inline.} = + uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or + uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or + uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or + uint64(s[o + 1]) shl 8 or uint64(s[o + 0]) + +proc load4(s: openArray[byte], o=0): uint32 {.inline.} = + when nimvm: result = load4e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load4e(s, o) + +proc load8(s: openArray[byte], o=0): uint64 {.inline.} = + when nimvm: result = load8e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load8e(s, o) + +proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64 + +proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47) + +proc rotR(v: uint64; bits: cint): uint64 {.inline.} = + (v shr bits) or (v shl (64 - bits)) + +proc len16(u: uint64; v: uint64; mul: uint64): uint64 {.inline.} = + var a = (u xor v)*mul + a = a xor (a shr 47) + var b = (v xor a)*mul + b = b xor (b shr 47) + b*mul + +proc len0_16(s: openArray[byte]): uint64 {.inline.} = + if s.len >= 8: + let mul = k2 + 2*s.lenU + let a = load8(s) + k2 + let b = load8(s, s.len - 8) + let c = rotR(b, 37)*mul + a + let d = (rotR(a, 25) + b)*mul + len16 c, d, mul + elif s.len >= 4: + let mul = k2 + 2*s.lenU + let a = load4(s).uint64 + len16 s.lenU + (a shl 3), load4(s, s.len - 4), mul + elif s.len > 0: + let a = uint32(s[0]) + let b = uint32(s[s.len shr 1]) + let c = uint32(s[s.len - 1]) + let y = a + (b shl 8) + let z = s.lenU + (c shl 2) + shiftMix(y*k2 xor z*k0)*k2 + else: k2 # s.len == 0 + +proc len17_32(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k1 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + len16 rotR(a + b, 43) + rotR(c, 30) + d, a + rotR(b + k2, 18) + c, mul + +proc len33_64(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k2 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + let y = rotR(a + b, 43) + rotR(c, 30) + d + let z = len16(y, a + rotR(b + k2, 18) + c, mul) + let e = load8(s, 16)*mul + let f = load8(s, 24) + let g = (y + load8(s, s.len - 32))*mul + let h = (z + load8(s, s.len - 24))*mul + len16 rotR(e + f, 43) + rotR(g, 30) + h, e + rotR(f + a, 18) + g, mul + +type Pair = tuple[first, second: uint64] + +proc weakLen32withSeeds2(w, x, y, z, a, b: uint64): Pair {.inline.} = + var a = a + w + var b = rotR(b + a + z, 21) + let c = a + a += x + a += y + b += rotR(a, 44) + result[0] = a + z + result[1] = b + c + +proc weakLen32withSeeds(s: openArray[byte]; o: int; a,b: uint64): Pair {.inline.} = + weakLen32withSeeds2 load8(s, o ), load8(s, o + 8), + load8(s, o + 16), load8(s, o + 24), a, b + +proc hashFarm(s: openArray[byte]): uint64 {.inline.} = + if s.len <= 16: return len0_16(s) + if s.len <= 32: return len17_32(s) + if s.len <= 64: return len33_64(s) + const seed = 81u64 # not const to use input `h` + var + o = 0 # s[] ptr arith -> variable origin variable `o` + x = seed + y = seed*k1 + 113 + z = shiftMix(y*k2 + 113)*k2 + v, w: Pair + x = x*k2 + load8(s) + let eos = ((s.len - 1) div 64)*64 + let last64 = eos + ((s.len - 1) and 63) - 63 + while true: + x = rotR(x + y + v[0] + load8(s, o+8), 37)*k1 + y = rotR(y + v[1] + load8(s, o+48), 42)*k1 + x = x xor w[1] + y += v[0] + load8(s, o+40) + z = rotR(z + w[0], 33)*k1 + v = weakLen32withSeeds(s, o+0 , v[1]*k1, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + inc o, 64 + if o == eos: break + let mul = k1 + ((z and 0xff) shl 1) + o = last64 + w[0] += (s.lenU - 1) and 63 + v[0] += w[0] + w[0] += v[0] + x = rotR(x + y + v[0] + load8(s, o+8), 37)*mul + y = rotR(y + v[1] + load8(s, o+48), 42)*mul + x = x xor w[1]*9 + y += v[0]*9 + load8(s, o+40) + z = rotR(z + w[0], 33)*mul + v = weakLen32withSeeds(s, o+0 , v[1]*mul, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + len16 len16(v[0],w[0],mul) + shiftMix(y)*k0 + z, len16(v[1],w[1],mul) + x, mul + +template jsNoInt64: untyped = + when defined js: + when compiles(compileOption("jsbigint64")): + when not compileOption("jsbigint64"): true + else: false + else: false + else: false +const sHash2 = (when defined(nimStringHash2) or jsNoInt64(): true else: false) + +template maybeFailJS_Number = + when jsNoInt64() and not defined(nimStringHash2): + {.error: "Must use `-d:nimStringHash2` when using `--jsbigint64:off`".} proc hash*(x: string): Hash = - ## efficient hashing of strings - var h: Hash = 0 - for i in 0..x.len-1: - h = h !& ord(x[i]) - result = !$h + ## Efficient hashing of strings. + ## + ## **See also:** + ## * `hashIgnoreStyle <#hashIgnoreStyle,string>`_ + ## * `hashIgnoreCase <#hashIgnoreCase,string>`_ + runnableExamples: + doAssert hash("abracadabra") != hash("AbracadabrA") + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) + else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: + result = murmurHash(toOpenArrayByte(x, 0, high(x))) + +proc hash*(x: cstring): Hash = + ## Efficient hashing of null-terminated strings. + runnableExamples: + doAssert hash(cstring"abracadabra") == hash("abracadabra") + doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") + doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") + + maybeFailJS_Number() + when not sHash2: + when defined js: + let xx = $x + result = cast[Hash](hashFarm(toOpenArrayByte(xx, 0, xx.high))) + else: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) + else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: + when not defined(js): + result = murmurHash(toOpenArrayByte(x, 0, x.high)) + else: + let xx = $x + result = murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = - ## efficient hashing of a string buffer, from starting - ## position `sPos` to ending position `ePos` + ## Efficient hashing of a string buffer, from starting + ## position `sPos` to ending position `ePos` (included). ## - ## ``hash(myStr, 0, myStr.high)`` is equivalent to ``hash(myStr)`` - var h: Hash = 0 - for i in sPos..ePos: - h = h !& ord(sBuf[i]) - result = !$h + ## `hash(myStr, 0, myStr.high)` is equivalent to `hash(myStr)`. + runnableExamples: + var a = "abracadabra" + doAssert hash(a, 0, 3) == hash(a, 7, 10) + + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(sBuf, sPos, ePos))) + else: + murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) proc hashIgnoreStyle*(x: string): Hash = - ## efficient hashing of strings; style is ignored + ## Efficient hashing of strings; style is ignored. + ## + ## **Note:** This uses a different hashing algorithm than `hash(string)`. + ## + ## **See also:** + ## * `hashIgnoreCase <#hashIgnoreCase,string>`_ + runnableExamples: + doAssert hashIgnoreStyle("aBr_aCa_dAB_ra") == hashIgnoreStyle("abracadabra") + doAssert hashIgnoreStyle("abcdefghi") != hash("abcdefghi") + var h: Hash = 0 var i = 0 let xLen = x.len @@ -146,30 +605,31 @@ proc hashIgnoreStyle*(x: string): Hash = var c = x[i] if c == '_': inc(i) - elif isMagicIdentSeparatorRune(cstring(x), i): - inc(i, magicIdentSeparatorRuneByteWidth) else: if c in {'A'..'Z'}: c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() h = h !& ord(c) inc(i) - result = !$h proc hashIgnoreStyle*(sBuf: string, sPos, ePos: int): Hash = - ## efficient hashing of a string buffer, from starting - ## position `sPos` to ending position `ePos`; style is ignored + ## Efficient hashing of a string buffer, from starting + ## position `sPos` to ending position `ePos` (included); style is ignored. ## - ## ``hashIgnoreStyle(myBuf, 0, myBuf.high)`` is equivalent - ## to ``hashIgnoreStyle(myBuf)`` + ## **Note:** This uses a different hashing algorithm than `hash(string)`. + ## + ## `hashIgnoreStyle(myBuf, 0, myBuf.high)` is equivalent + ## to `hashIgnoreStyle(myBuf)`. + runnableExamples: + var a = "ABracada_b_r_a" + doAssert hashIgnoreStyle(a, 0, 3) == hashIgnoreStyle(a, 7, a.high) + var h: Hash = 0 var i = sPos while i <= ePos: var c = sBuf[i] if c == '_': inc(i) - elif isMagicIdentSeparatorRune(cstring(sBuf), i): - inc(i, magicIdentSeparatorRuneByteWidth) else: if c in {'A'..'Z'}: c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() @@ -178,7 +638,16 @@ proc hashIgnoreStyle*(sBuf: string, sPos, ePos: int): Hash = result = !$h proc hashIgnoreCase*(x: string): Hash = - ## efficient hashing of strings; case is ignored + ## Efficient hashing of strings; case is ignored. + ## + ## **Note:** This uses a different hashing algorithm than `hash(string)`. + ## + ## **See also:** + ## * `hashIgnoreStyle <#hashIgnoreStyle,string>`_ + runnableExamples: + doAssert hashIgnoreCase("ABRAcaDABRA") == hashIgnoreCase("abRACAdabra") + doAssert hashIgnoreCase("abcdefghi") != hash("abcdefghi") + var h: Hash = 0 for i in 0..x.len-1: var c = x[i] @@ -188,11 +657,17 @@ proc hashIgnoreCase*(x: string): Hash = result = !$h proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = - ## efficient hashing of a string buffer, from starting - ## position `sPos` to ending position `ePos`; case is ignored + ## Efficient hashing of a string buffer, from starting + ## position `sPos` to ending position `ePos` (included); case is ignored. + ## + ## **Note:** This uses a different hashing algorithm than `hash(string)`. ## - ## ``hashIgnoreCase(myBuf, 0, myBuf.high)`` is equivalent - ## to ``hashIgnoreCase(myBuf)`` + ## `hashIgnoreCase(myBuf, 0, myBuf.high)` is equivalent + ## to `hashIgnoreCase(myBuf)`. + runnableExamples: + var a = "ABracadabRA" + doAssert hashIgnoreCase(a, 0, 3) == hashIgnoreCase(a, 7, 10) + var h: Hash = 0 for i in sPos..ePos: var c = sBuf[i] @@ -201,48 +676,104 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h -proc hash*(x: float): Hash {.inline.} = - ## efficient hashing of floats. - var y = x + 1.0 - result = cast[ptr Hash](addr(y))[] - - -# Forward declarations before methods that hash containers. This allows -# containers to contain other containers -proc hash*[A](x: openArray[A]): Hash -proc hash*[A](x: set[A]): Hash +proc hash*[T: tuple | object | proc | iterator {.closure.}](x: T): Hash = + ## Efficient `hash` overload. + runnableExamples: + # for `tuple|object`, `hash` must be defined for each component of `x`. + type Obj = object + x: int + y: string + type Obj2[T] = object + x: int + y: string + assert hash(Obj(x: 520, y: "Nim")) != hash(Obj(x: 520, y: "Nim2")) + # you can define custom hashes for objects (even if they're generic): + proc hash(a: Obj2): Hash = hash((a.x)) + assert hash(Obj2[float](x: 520, y: "Nim")) == hash(Obj2[float](x: 520, y: "Nim2")) + runnableExamples: + # proc + proc fn1() = discard + const fn1b = fn1 + assert hash(fn1b) == hash(fn1) + # closure + proc outer = + var a = 0 + proc fn2() = a.inc + assert fn2 is "closure" + let fn2b = fn2 + assert hash(fn2b) == hash(fn2) + assert hash(fn2) != hash(fn1) + outer() -proc hash*[T: tuple](x: T): Hash = - ## efficient hashing of tuples. - for f in fields(x): - result = result !& hash(f) - result = !$result + when T is "closure": + result = hash((rawProc(x), rawEnv(x))) + elif T is (proc): + result = hash(cast[pointer](x)) + else: + result = 0 + for f in fields(x): + result = result !& hash(f) + result = !$result proc hash*[A](x: openArray[A]): Hash = - ## efficient hashing of arrays and sequences. - for it in items(x): result = result !& hash(it) - result = !$result + ## Efficient hashing of arrays and sequences. + ## There must be a `hash` proc defined for the element type `A`. + when A is byte: + when not sHash2: + result = cast[Hash](hashFarm(x)) + else: + result = murmurHash(x) + elif A is char: + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) + else: + #when nimvm: + # result = hashVmImplChar(x, 0, x.high) + when true: + result = murmurHash(toOpenArrayByte(x, 0, x.high)) + else: + result = 0 + for a in x: + result = result !& hash(a) + result = !$result proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = - ## efficient hashing of portions of arrays and sequences. + ## Efficient hashing of portions of arrays and sequences, from starting + ## position `sPos` to ending position `ePos` (included). + ## There must be a `hash` proc defined for the element type `A`. ## - ## ``hash(myBuf, 0, myBuf.high)`` is equivalent to ``hash(myBuf)`` - for i in sPos..ePos: - result = result !& hash(aBuf[i]) - result = !$result + ## `hash(myBuf, 0, myBuf.high)` is equivalent to `hash(myBuf)`. + runnableExamples: + let a = [1, 2, 5, 1, 2, 6] + doAssert hash(a, 0, 1) == hash(a, 3, 4) + when A is byte: + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArray(aBuf, sPos, ePos))) + else: + #when nimvm: + # result = hashVmImplByte(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArray(aBuf, sPos, ePos)) + elif A is char: + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(aBuf, sPos, ePos))) + else: + #when nimvm: + # result = hashVmImplChar(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) + else: + for i in sPos .. ePos: + result = result !& hash(aBuf[i]) + result = !$result proc hash*[A](x: set[A]): Hash = - ## efficient hashing of sets. - for it in items(x): result = result !& hash(it) + ## Efficient hashing of sets. + ## There must be a `hash` proc defined for the element type `A`. + result = 0 + for it in items(x): + result = result !& hash(it) result = !$result - -when isMainModule: - doAssert( hash("aa bb aaaa1234") == hash("aa bb aaaa1234", 0, 13) ) - doAssert( hashIgnoreCase("aa bb aaaa1234") == hash("aa bb aaaa1234") ) - doAssert( hashIgnoreStyle("aa bb aaaa1234") == hashIgnoreCase("aa bb aaaa1234") ) - let xx = @['H','e','l','l','o'] - let ss = "Hello" - doAssert( hash(xx) == hash(ss) ) - doAssert( hash(xx) == hash(xx, 0, xx.high) ) - doAssert( hash(ss) == hash(ss, 0, ss.high) ) diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 225949350..fafa72463 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -7,43 +7,65 @@ # distribution, for details about the copyright. # -## **Warning**: This module uses ``immediate`` macros which are known to -## cause problems. Do yourself a favor and import the module -## as ``from htmlgen import nil`` and then fully qualify the macros. +## Do yourself a favor and import the module +## as `from std/htmlgen import nil` and then fully qualify the macros. +## +## *Note*: The Karax project (`nimble install karax`) has a better +## way to achieve the same, see https://github.com/pragmagic/karax/blob/master/tests/nativehtmlgen.nim +## for an example. ## ## ## This module implements a simple `XML`:idx: and `HTML`:idx: code ## generator. Each commonly used HTML tag has a corresponding macro ## that generates a string with its HTML representation. ## -## Example: +## MathML +## ====== +## +## `MathML <https://wikipedia.org/wiki/MathML>`_ is supported, MathML is part of HTML5. +## `MathML <https://wikipedia.org/wiki/MathML>`_ is an Standard ISO/IEC 40314 from year 2015. +## MathML allows you to `draw advanced math on the web <https://developer.mozilla.org/en-US/docs/Web/MathML/Element/math#Examples>`_, +## `visually similar to Latex math. <https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics#Example>`_ ## -## .. code-block:: Nim +## Examples +## ======== +## +## ```Nim ## var nim = "Nim" -## echo h1(a(href="http://nim-lang.org", nim)) +## echo h1(a(href="https://nim-lang.org", nim)) +## ``` ## -## Writes the string:: +## Writes the string: ## -## <h1><a href="http://nim-lang.org">Nim</a></h1> +## <h1><a href="https://nim-lang.org">Nim</a></h1> ## import - macros, strutils + std/[macros, strutils] const - coreAttr* = " id class title style " - eventAttr* = " onclick ondblclick onmousedown onmouseup " & - "onmouseover onmousemove onmouseout onkeypress onkeydown onkeyup onload " - commonAttr* = coreAttr & eventAttr - -proc getIdent(e: NimNode): string {.compileTime.} = + coreAttr* = " accesskey class contenteditable dir hidden id lang " & + "spellcheck style tabindex title translate " ## HTML DOM Core Attributes + eventAttr* = "onabort onblur oncancel oncanplay oncanplaythrough onchange " & + "onclick oncuechange ondblclick ondurationchange onemptied onended " & + "onerror onfocus oninput oninvalid onkeydown onkeypress onkeyup onload " & + "onloadeddata onloadedmetadata onloadstart onmousedown onmouseenter " & + "onmouseleave onmousemove onmouseout onmouseover onmouseup onmousewheel " & + "onpause onplay onplaying onprogress onratechange onreset onresize " & + "onscroll onseeked onseeking onselect onshow onstalled onsubmit " & + "onsuspend ontimeupdate ontoggle onvolumechange onwaiting " ## HTML DOM Event Attributes + ariaAttr* = " role " ## HTML DOM Aria Attributes + commonAttr* = coreAttr & eventAttr & ariaAttr ## HTML DOM Common Attributes + +proc getIdent(e: NimNode): string = case e.kind - of nnkIdent: result = normalize($e.ident) + of nnkIdent: + result = e.strVal.normalize of nnkAccQuoted: result = getIdent(e[0]) for i in 1 .. e.len-1: result.add getIdent(e[i]) - else: error("cannot extract identifier from node: " & toStrLit(e).strVal) + else: error("cannot extract identifier from node: " & toStrLit(e).strVal, e) proc delete[T](s: var seq[T], attr: T): bool = var idx = find(s, attr) @@ -53,438 +75,738 @@ proc delete[T](s: var seq[T], attr: T): bool = setLen(s, L-1) result = true -proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", - isLeaf = false): NimNode {.compileTime.} = +proc xmlCheckedTag*(argsList: NimNode, tag: string, optAttr = "", reqAttr = "", + isLeaf = false): NimNode = ## use this procedure to define a new XML tag # copy the attributes; when iterating over them these lists # will be modified, so that each attribute is only given one value - var req = split(reqAttr) - var opt = split(optAttr) - result = newNimNode(nnkBracket, e) + var req = splitWhitespace(reqAttr) + var opt = splitWhitespace(optAttr) + result = newNimNode(nnkBracket) result.add(newStrLitNode("<")) result.add(newStrLitNode(tag)) # first pass over attributes: - for i in 1..e.len-1: - if e[i].kind == nnkExprEqExpr: - var name = getIdent(e[i][0]) - if delete(req, name) or delete(opt, name): + for i in 0 ..< argsList.len: + if argsList[i].kind == nnkExprEqExpr: + var name = getIdent(argsList[i][0]) + if name.startsWith("data-") or delete(req, name) or delete(opt, name): result.add(newStrLitNode(" ")) result.add(newStrLitNode(name)) result.add(newStrLitNode("=\"")) - result.add(e[i][1]) + result.add(argsList[i][1]) result.add(newStrLitNode("\"")) else: - error("invalid attribute for '" & tag & "' element: " & name) + error("invalid attribute for '" & tag & "' element: " & name, argsList[i]) # check each required attribute exists: if req.len > 0: - error(req[0] & " attribute for '" & tag & "' element expected") + error(req[0] & " attribute for '" & tag & "' element expected", argsList) if isLeaf: - for i in 1..e.len-1: - if e[i].kind != nnkExprEqExpr: - error("element " & tag & " cannot be nested") + for i in 0 ..< argsList.len: + if argsList[i].kind != nnkExprEqExpr: + error("element " & tag & " cannot be nested", argsList[i]) result.add(newStrLitNode(" />")) else: result.add(newStrLitNode(">")) # second pass over elements: - for i in 1..e.len-1: - if e[i].kind != nnkExprEqExpr: result.add(e[i]) + for i in 0 ..< argsList.len: + if argsList[i].kind != nnkExprEqExpr: result.add(argsList[i]) result.add(newStrLitNode("</")) result.add(newStrLitNode(tag)) result.add(newStrLitNode(">")) - result = nestList(!"&", result) + result = nestList(ident"&", result) +macro a*(e: varargs[untyped]): untyped = + ## Generates the HTML `a` element. + result = xmlCheckedTag(e, "a", "href target download rel hreflang type " & + commonAttr) -macro a*(e: expr): expr {.immediate.} = - ## generates the HTML ``a`` element. - let e = callsite() - result = xmlCheckedTag(e, "a", "href charset type hreflang rel rev " & - "accesskey tabindex" & commonAttr) - -macro acronym*(e: expr): expr {.immediate.} = - ## generates the HTML ``acronym`` element. - let e = callsite() - result = xmlCheckedTag(e, "acronym", commonAttr) +macro abbr*(e: varargs[untyped]): untyped = + ## Generates the HTML `abbr` element. + result = xmlCheckedTag(e, "abbr", commonAttr) -macro address*(e: expr): expr {.immediate.} = - ## generates the HTML ``address`` element. - let e = callsite() +macro address*(e: varargs[untyped]): untyped = + ## Generates the HTML `address` element. result = xmlCheckedTag(e, "address", commonAttr) -macro area*(e: expr): expr {.immediate.} = - ## generates the HTML ``area`` element. - let e = callsite() - result = xmlCheckedTag(e, "area", "shape coords href nohref" & - " accesskey tabindex" & commonAttr, "alt", true) +macro area*(e: varargs[untyped]): untyped = + ## Generates the HTML `area` element. + result = xmlCheckedTag(e, "area", "coords download href hreflang rel " & + "shape target type" & commonAttr, "alt", true) + +macro article*(e: varargs[untyped]): untyped = + ## Generates the HTML `article` element. + result = xmlCheckedTag(e, "article", commonAttr) + +macro aside*(e: varargs[untyped]): untyped = + ## Generates the HTML `aside` element. + result = xmlCheckedTag(e, "aside", commonAttr) -macro b*(e: expr): expr {.immediate.} = - ## generates the HTML ``b`` element. - let e = callsite() +macro audio*(e: varargs[untyped]): untyped = + ## Generates the HTML `audio` element. + result = xmlCheckedTag(e, "audio", "src crossorigin preload " & + "autoplay mediagroup loop muted controls" & commonAttr) + +macro b*(e: varargs[untyped]): untyped = + ## Generates the HTML `b` element. result = xmlCheckedTag(e, "b", commonAttr) -macro base*(e: expr): expr {.immediate.} = - ## generates the HTML ``base`` element. - let e = callsite() - result = xmlCheckedTag(e, "base", "", "href", true) +macro base*(e: varargs[untyped]): untyped = + ## Generates the HTML `base` element. + result = xmlCheckedTag(e, "base", "href target" & commonAttr, "", true) + +macro bdi*(e: varargs[untyped]): untyped = + ## Generates the HTML `bdi` element. + result = xmlCheckedTag(e, "bdi", commonAttr) -macro big*(e: expr): expr {.immediate.} = - ## generates the HTML ``big`` element. - let e = callsite() +macro bdo*(e: varargs[untyped]): untyped = + ## Generates the HTML `bdo` element. + result = xmlCheckedTag(e, "bdo", commonAttr) + +macro big*(e: varargs[untyped]): untyped = + ## Generates the HTML `big` element. result = xmlCheckedTag(e, "big", commonAttr) -macro blockquote*(e: expr): expr {.immediate.} = - ## generates the HTML ``blockquote`` element. - let e = callsite() +macro blockquote*(e: varargs[untyped]): untyped = + ## Generates the HTML `blockquote` element. result = xmlCheckedTag(e, "blockquote", " cite" & commonAttr) -macro body*(e: expr): expr {.immediate.} = - ## generates the HTML ``body`` element. - let e = callsite() - result = xmlCheckedTag(e, "body", commonAttr) - -macro br*(e: expr): expr {.immediate.} = - ## generates the HTML ``br`` element. - let e = callsite() - result = xmlCheckedTag(e, "br", "", "", true) - -macro button*(e: expr): expr {.immediate.} = - ## generates the HTML ``button`` element. - let e = callsite() - result = xmlCheckedTag(e, "button", "accesskey tabindex " & - "disabled name type value" & commonAttr) - -macro caption*(e: expr): expr {.immediate.} = - ## generates the HTML ``caption`` element. - let e = callsite() +macro body*(e: varargs[untyped]): untyped = + ## Generates the HTML `body` element. + result = xmlCheckedTag(e, "body", "onafterprint onbeforeprint " & + "onbeforeunload onhashchange onmessage onoffline ononline onpagehide " & + "onpageshow onpopstate onstorage onunload" & commonAttr) + +macro br*(e: varargs[untyped]): untyped = + ## Generates the HTML `br` element. + result = xmlCheckedTag(e, "br", commonAttr, "", true) + +macro button*(e: varargs[untyped]): untyped = + ## Generates the HTML `button` element. + result = xmlCheckedTag(e, "button", "autofocus disabled form formaction " & + "formenctype formmethod formnovalidate formtarget menu name type value" & + commonAttr) + +macro canvas*(e: varargs[untyped]): untyped = + ## Generates the HTML `canvas` element. + result = xmlCheckedTag(e, "canvas", "width height" & commonAttr) + +macro caption*(e: varargs[untyped]): untyped = + ## Generates the HTML `caption` element. result = xmlCheckedTag(e, "caption", commonAttr) -macro cite*(e: expr): expr {.immediate.} = - ## generates the HTML ``cite`` element. - let e = callsite() +macro center*(e: varargs[untyped]): untyped = + ## Generates the HTML `center` element. + result = xmlCheckedTag(e, "center", commonAttr) + +macro cite*(e: varargs[untyped]): untyped = + ## Generates the HTML `cite` element. result = xmlCheckedTag(e, "cite", commonAttr) -macro code*(e: expr): expr {.immediate.} = - ## generates the HTML ``code`` element. - let e = callsite() +macro code*(e: varargs[untyped]): untyped = + ## Generates the HTML `code` element. result = xmlCheckedTag(e, "code", commonAttr) -macro col*(e: expr): expr {.immediate.} = - ## generates the HTML ``col`` element. - let e = callsite() - result = xmlCheckedTag(e, "col", "span align valign" & commonAttr, "", true) +macro col*(e: varargs[untyped]): untyped = + ## Generates the HTML `col` element. + result = xmlCheckedTag(e, "col", "span" & commonAttr, "", true) + +macro colgroup*(e: varargs[untyped]): untyped = + ## Generates the HTML `colgroup` element. + result = xmlCheckedTag(e, "colgroup", "span" & commonAttr) + +macro data*(e: varargs[untyped]): untyped = + ## Generates the HTML `data` element. + result = xmlCheckedTag(e, "data", "value" & commonAttr) -macro colgroup*(e: expr): expr {.immediate.} = - ## generates the HTML ``colgroup`` element. - let e = callsite() - result = xmlCheckedTag(e, "colgroup", "span align valign" & commonAttr) +macro datalist*(e: varargs[untyped]): untyped = + ## Generates the HTML `datalist` element. + result = xmlCheckedTag(e, "datalist", commonAttr) -macro dd*(e: expr): expr {.immediate.} = - ## generates the HTML ``dd`` element. - let e = callsite() +macro dd*(e: varargs[untyped]): untyped = + ## Generates the HTML `dd` element. result = xmlCheckedTag(e, "dd", commonAttr) -macro del*(e: expr): expr {.immediate.} = - ## generates the HTML ``del`` element. - let e = callsite() +macro del*(e: varargs[untyped]): untyped = + ## Generates the HTML `del` element. result = xmlCheckedTag(e, "del", "cite datetime" & commonAttr) -macro dfn*(e: expr): expr {.immediate.} = - ## generates the HTML ``dfn`` element. - let e = callsite() +macro details*(e: varargs[untyped]): untyped = + ## Generates the HTML `details` element. + result = xmlCheckedTag(e, "details", commonAttr & "open") + +macro dfn*(e: varargs[untyped]): untyped = + ## Generates the HTML `dfn` element. result = xmlCheckedTag(e, "dfn", commonAttr) -macro `div`*(e: expr): expr {.immediate.} = - ## generates the HTML ``div`` element. - let e = callsite() +macro dialog*(e: varargs[untyped]): untyped = + ## Generates the HTML `dialog` element. + result = xmlCheckedTag(e, "dialog", commonAttr & "open") + +macro `div`*(e: varargs[untyped]): untyped = + ## Generates the HTML `div` element. result = xmlCheckedTag(e, "div", commonAttr) -macro dl*(e: expr): expr {.immediate.} = - ## generates the HTML ``dl`` element. - let e = callsite() +macro dl*(e: varargs[untyped]): untyped = + ## Generates the HTML `dl` element. result = xmlCheckedTag(e, "dl", commonAttr) -macro dt*(e: expr): expr {.immediate.} = - ## generates the HTML ``dt`` element. - let e = callsite() +macro dt*(e: varargs[untyped]): untyped = + ## Generates the HTML `dt` element. result = xmlCheckedTag(e, "dt", commonAttr) -macro em*(e: expr): expr {.immediate.} = - ## generates the HTML ``em`` element. - let e = callsite() +macro em*(e: varargs[untyped]): untyped = + ## Generates the HTML `em` element. result = xmlCheckedTag(e, "em", commonAttr) -macro fieldset*(e: expr): expr {.immediate.} = - ## generates the HTML ``fieldset`` element. - let e = callsite() - result = xmlCheckedTag(e, "fieldset", commonAttr) +macro embed*(e: varargs[untyped]): untyped = + ## Generates the HTML `embed` element. + result = xmlCheckedTag(e, "embed", "src type height width" & + commonAttr, "", true) + +macro fieldset*(e: varargs[untyped]): untyped = + ## Generates the HTML `fieldset` element. + result = xmlCheckedTag(e, "fieldset", "disabled form name" & commonAttr) -macro form*(e: expr): expr {.immediate.} = - ## generates the HTML ``form`` element. - let e = callsite() - result = xmlCheckedTag(e, "form", "method encype accept accept-charset" & - commonAttr, "action") +macro figure*(e: varargs[untyped]): untyped = + ## Generates the HTML `figure` element. + result = xmlCheckedTag(e, "figure", commonAttr) -macro h1*(e: expr): expr {.immediate.} = - ## generates the HTML ``h1`` element. - let e = callsite() +macro figcaption*(e: varargs[untyped]): untyped = + ## Generates the HTML `figcaption` element. + result = xmlCheckedTag(e, "figcaption", commonAttr) + +macro footer*(e: varargs[untyped]): untyped = + ## Generates the HTML `footer` element. + result = xmlCheckedTag(e, "footer", commonAttr) + +macro form*(e: varargs[untyped]): untyped = + ## Generates the HTML `form` element. + result = xmlCheckedTag(e, "form", "accept-charset action autocomplete " & + "enctype method name novalidate target" & commonAttr) + +macro h1*(e: varargs[untyped]): untyped = + ## Generates the HTML `h1` element. result = xmlCheckedTag(e, "h1", commonAttr) -macro h2*(e: expr): expr {.immediate.} = - ## generates the HTML ``h2`` element. - let e = callsite() +macro h2*(e: varargs[untyped]): untyped = + ## Generates the HTML `h2` element. result = xmlCheckedTag(e, "h2", commonAttr) -macro h3*(e: expr): expr {.immediate.} = - ## generates the HTML ``h3`` element. - let e = callsite() +macro h3*(e: varargs[untyped]): untyped = + ## Generates the HTML `h3` element. result = xmlCheckedTag(e, "h3", commonAttr) -macro h4*(e: expr): expr {.immediate.} = - ## generates the HTML ``h4`` element. - let e = callsite() +macro h4*(e: varargs[untyped]): untyped = + ## Generates the HTML `h4` element. result = xmlCheckedTag(e, "h4", commonAttr) -macro h5*(e: expr): expr {.immediate.} = - ## generates the HTML ``h5`` element. - let e = callsite() +macro h5*(e: varargs[untyped]): untyped = + ## Generates the HTML `h5` element. result = xmlCheckedTag(e, "h5", commonAttr) -macro h6*(e: expr): expr {.immediate.} = - ## generates the HTML ``h6`` element. - let e = callsite() +macro h6*(e: varargs[untyped]): untyped = + ## Generates the HTML `h6` element. result = xmlCheckedTag(e, "h6", commonAttr) -macro head*(e: expr): expr {.immediate.} = - ## generates the HTML ``head`` element. - let e = callsite() - result = xmlCheckedTag(e, "head", "profile") +macro head*(e: varargs[untyped]): untyped = + ## Generates the HTML `head` element. + result = xmlCheckedTag(e, "head", commonAttr) -macro html*(e: expr): expr {.immediate.} = - ## generates the HTML ``html`` element. - let e = callsite() - result = xmlCheckedTag(e, "html", "xmlns", "") +macro header*(e: varargs[untyped]): untyped = + ## Generates the HTML `header` element. + result = xmlCheckedTag(e, "header", commonAttr) -macro hr*(): expr {.immediate.} = - ## generates the HTML ``hr`` element. - let e = callsite() - result = xmlCheckedTag(e, "hr", commonAttr, "", true) +macro html*(e: varargs[untyped]): untyped = + ## Generates the HTML `html` element. + result = xmlCheckedTag(e, "html", "xmlns" & commonAttr, "") -macro i*(e: expr): expr {.immediate.} = - ## generates the HTML ``i`` element. - let e = callsite() - result = xmlCheckedTag(e, "i", commonAttr) +macro hr*(): untyped = + ## Generates the HTML `hr` element. + result = xmlCheckedTag(newNimNode(nnkArgList), "hr", commonAttr, "", true) -macro img*(e: expr): expr {.immediate.} = - ## generates the HTML ``img`` element. - let e = callsite() - result = xmlCheckedTag(e, "img", "longdesc height width", "src alt", true) +macro i*(e: varargs[untyped]): untyped = + ## Generates the HTML `i` element. + result = xmlCheckedTag(e, "i", commonAttr) -macro input*(e: expr): expr {.immediate.} = - ## generates the HTML ``input`` element. - let e = callsite() - result = xmlCheckedTag(e, "input", "name type value checked maxlength src" & - " alt accept disabled readonly accesskey tabindex" & commonAttr, "", true) +macro iframe*(e: varargs[untyped]): untyped = + ## Generates the HTML `iframe` element. + result = xmlCheckedTag(e, "iframe", "src srcdoc name sandbox width height loading" & + commonAttr) -macro ins*(e: expr): expr {.immediate.} = - ## generates the HTML ``ins`` element. - let e = callsite() +macro img*(e: varargs[untyped]): untyped = + ## Generates the HTML `img` element. + result = xmlCheckedTag(e, "img", "crossorigin usemap ismap height width loading" & + commonAttr, "src alt", true) + +macro input*(e: varargs[untyped]): untyped = + ## Generates the HTML `input` element. + result = xmlCheckedTag(e, "input", "accept alt autocomplete autofocus " & + "checked dirname disabled form formaction formenctype formmethod " & + "formnovalidate formtarget height inputmode list max maxlength min " & + "minlength multiple name pattern placeholder readonly required size " & + "src step type value width" & commonAttr, "", true) + +macro ins*(e: varargs[untyped]): untyped = + ## Generates the HTML `ins` element. result = xmlCheckedTag(e, "ins", "cite datetime" & commonAttr) -macro kbd*(e: expr): expr {.immediate.} = - ## generates the HTML ``kbd`` element. - let e = callsite() +macro kbd*(e: varargs[untyped]): untyped = + ## Generates the HTML `kbd` element. result = xmlCheckedTag(e, "kbd", commonAttr) -macro label*(e: expr): expr {.immediate.} = - ## generates the HTML ``label`` element. - let e = callsite() - result = xmlCheckedTag(e, "label", "for accesskey" & commonAttr) - -macro legend*(e: expr): expr {.immediate.} = - ## generates the HTML ``legend`` element. - let e = callsite() - result = xmlCheckedTag(e, "legend", "accesskey" & commonAttr) - -macro li*(e: expr): expr {.immediate.} = - ## generates the HTML ``li`` element. - let e = callsite() - result = xmlCheckedTag(e, "li", commonAttr) - -macro link*(e: expr): expr {.immediate.} = - ## generates the HTML ``link`` element. - let e = callsite() - result = xmlCheckedTag(e, "link", "href charset hreflang type rel rev media" & +macro keygen*(e: varargs[untyped]): untyped = + ## Generates the HTML `keygen` element. + result = xmlCheckedTag(e, "keygen", "autofocus challenge disabled " & + "form keytype name" & commonAttr) + +macro label*(e: varargs[untyped]): untyped = + ## Generates the HTML `label` element. + result = xmlCheckedTag(e, "label", "form for" & commonAttr) + +macro legend*(e: varargs[untyped]): untyped = + ## Generates the HTML `legend` element. + result = xmlCheckedTag(e, "legend", commonAttr) + +macro li*(e: varargs[untyped]): untyped = + ## Generates the HTML `li` element. + result = xmlCheckedTag(e, "li", "value" & commonAttr) + +macro link*(e: varargs[untyped]): untyped = + ## Generates the HTML `link` element. + result = xmlCheckedTag(e, "link", "href crossorigin rel media hreflang " & + "type sizes" & commonAttr, "", true) + +macro main*(e: varargs[untyped]): untyped = + ## Generates the HTML `main` element. + result = xmlCheckedTag(e, "main", commonAttr) + +macro map*(e: varargs[untyped]): untyped = + ## Generates the HTML `map` element. + result = xmlCheckedTag(e, "map", "name" & commonAttr) + +macro mark*(e: varargs[untyped]): untyped = + ## Generates the HTML `mark` element. + result = xmlCheckedTag(e, "mark", commonAttr) + +macro marquee*(e: varargs[untyped]): untyped = + ## Generates the HTML `marquee` element. + result = xmlCheckedTag(e, "marquee", coreAttr & + "behavior bgcolor direction height hspace loop scrollamount " & + "scrolldelay truespeed vspace width onbounce onfinish onstart") + +macro meta*(e: varargs[untyped]): untyped = + ## Generates the HTML `meta` element. + result = xmlCheckedTag(e, "meta", "name http-equiv content charset" & commonAttr, "", true) -macro map*(e: expr): expr {.immediate.} = - ## generates the HTML ``map`` element. - let e = callsite() - result = xmlCheckedTag(e, "map", "class title" & eventAttr, "id", false) +macro meter*(e: varargs[untyped]): untyped = + ## Generates the HTML `meter` element. + result = xmlCheckedTag(e, "meter", "value min max low high optimum" & + commonAttr) -macro meta*(e: expr): expr {.immediate.} = - ## generates the HTML ``meta`` element. - let e = callsite() - result = xmlCheckedTag(e, "meta", "name http-equiv scheme", "content", true) +macro nav*(e: varargs[untyped]): untyped = + ## Generates the HTML `nav` element. + result = xmlCheckedTag(e, "nav", commonAttr) -macro noscript*(e: expr): expr {.immediate.} = - ## generates the HTML ``noscript`` element. - let e = callsite() +macro noscript*(e: varargs[untyped]): untyped = + ## Generates the HTML `noscript` element. result = xmlCheckedTag(e, "noscript", commonAttr) -macro `object`*(e: expr): expr {.immediate.} = - ## generates the HTML ``object`` element. - let e = callsite() - result = xmlCheckedTag(e, "object", "classid data codebase declare type " & - "codetype archive standby width height name tabindex" & commonAttr) +macro `object`*(e: varargs[untyped]): untyped = + ## Generates the HTML `object` element. + result = xmlCheckedTag(e, "object", "data type typemustmatch name usemap " & + "form width height" & commonAttr) -macro ol*(e: expr): expr {.immediate.} = - ## generates the HTML ``ol`` element. - let e = callsite() - result = xmlCheckedTag(e, "ol", commonAttr) +macro ol*(e: varargs[untyped]): untyped = + ## Generates the HTML `ol` element. + result = xmlCheckedTag(e, "ol", "reversed start type" & commonAttr) -macro optgroup*(e: expr): expr {.immediate.} = - ## generates the HTML ``optgroup`` element. - let e = callsite() +macro optgroup*(e: varargs[untyped]): untyped = + ## Generates the HTML `optgroup` element. result = xmlCheckedTag(e, "optgroup", "disabled" & commonAttr, "label", false) -macro option*(e: expr): expr {.immediate.} = - ## generates the HTML ``option`` element. - let e = callsite() - result = xmlCheckedTag(e, "option", "selected value" & commonAttr) +macro option*(e: varargs[untyped]): untyped = + ## Generates the HTML `option` element. + result = xmlCheckedTag(e, "option", "disabled label selected value" & + commonAttr) + +macro output*(e: varargs[untyped]): untyped = + ## Generates the HTML `output` element. + result = xmlCheckedTag(e, "output", "for form name" & commonAttr) -macro p*(e: expr): expr {.immediate.} = - ## generates the HTML ``p`` element. - let e = callsite() +macro p*(e: varargs[untyped]): untyped = + ## Generates the HTML `p` element. result = xmlCheckedTag(e, "p", commonAttr) -macro param*(e: expr): expr {.immediate.} = - ## generates the HTML ``param`` element. - let e = callsite() - result = xmlCheckedTag(e, "param", "value id type valuetype", "name", true) +macro param*(e: varargs[untyped]): untyped = + ## Generates the HTML `param` element. + result = xmlCheckedTag(e, "param", commonAttr, "name value", true) -macro pre*(e: expr): expr {.immediate.} = - ## generates the HTML ``pre`` element. - let e = callsite() +macro picture*(e: varargs[untyped]): untyped = + ## Generates the HTML `picture` element. + result = xmlCheckedTag(e, "picture", commonAttr) + +macro pre*(e: varargs[untyped]): untyped = + ## Generates the HTML `pre` element. result = xmlCheckedTag(e, "pre", commonAttr) -macro q*(e: expr): expr {.immediate.} = - ## generates the HTML ``q`` element. - let e = callsite() +macro progress*(e: varargs[untyped]): untyped = + ## Generates the HTML `progress` element. + result = xmlCheckedTag(e, "progress", "value max" & commonAttr) + +macro q*(e: varargs[untyped]): untyped = + ## Generates the HTML `q` element. result = xmlCheckedTag(e, "q", "cite" & commonAttr) -macro samp*(e: expr): expr {.immediate.} = - ## generates the HTML ``samp`` element. - let e = callsite() +macro rb*(e: varargs[untyped]): untyped = + ## Generates the HTML `rb` element. + result = xmlCheckedTag(e, "rb", commonAttr) + +macro rp*(e: varargs[untyped]): untyped = + ## Generates the HTML `rp` element. + result = xmlCheckedTag(e, "rp", commonAttr) + +macro rt*(e: varargs[untyped]): untyped = + ## Generates the HTML `rt` element. + result = xmlCheckedTag(e, "rt", commonAttr) + +macro rtc*(e: varargs[untyped]): untyped = + ## Generates the HTML `rtc` element. + result = xmlCheckedTag(e, "rtc", commonAttr) + +macro ruby*(e: varargs[untyped]): untyped = + ## Generates the HTML `ruby` element. + result = xmlCheckedTag(e, "ruby", commonAttr) + +macro s*(e: varargs[untyped]): untyped = + ## Generates the HTML `s` element. + result = xmlCheckedTag(e, "s", commonAttr) + +macro samp*(e: varargs[untyped]): untyped = + ## Generates the HTML `samp` element. result = xmlCheckedTag(e, "samp", commonAttr) -macro script*(e: expr): expr {.immediate.} = - ## generates the HTML ``script`` element. - let e = callsite() - result = xmlCheckedTag(e, "script", "src charset defer", "type", false) +macro script*(e: varargs[untyped]): untyped = + ## Generates the HTML `script` element. + result = xmlCheckedTag(e, "script", "src type charset async defer " & + "crossorigin" & commonAttr) -macro select*(e: expr): expr {.immediate.} = - ## generates the HTML ``select`` element. - let e = callsite() - result = xmlCheckedTag(e, "select", "name size multiple disabled tabindex" & - commonAttr) +macro section*(e: varargs[untyped]): untyped = + ## Generates the HTML `section` element. + result = xmlCheckedTag(e, "section", commonAttr) + +macro select*(e: varargs[untyped]): untyped = + ## Generates the HTML `select` element. + result = xmlCheckedTag(e, "select", "autofocus disabled form multiple " & + "name required size" & commonAttr) -macro small*(e: expr): expr {.immediate.} = - ## generates the HTML ``small`` element. - let e = callsite() +macro slot*(e: varargs[untyped]): untyped = + ## Generates the HTML `slot` element. + result = xmlCheckedTag(e, "slot", commonAttr) + +macro small*(e: varargs[untyped]): untyped = + ## Generates the HTML `small` element. result = xmlCheckedTag(e, "small", commonAttr) -macro span*(e: expr): expr {.immediate.} = - ## generates the HTML ``span`` element. - let e = callsite() +macro source*(e: varargs[untyped]): untyped = + ## Generates the HTML `source` element. + result = xmlCheckedTag(e, "source", "type" & commonAttr, "src", true) + +macro span*(e: varargs[untyped]): untyped = + ## Generates the HTML `span` element. result = xmlCheckedTag(e, "span", commonAttr) -macro strong*(e: expr): expr {.immediate.} = - ## generates the HTML ``strong`` element. - let e = callsite() +macro strong*(e: varargs[untyped]): untyped = + ## Generates the HTML `strong` element. result = xmlCheckedTag(e, "strong", commonAttr) -macro style*(e: expr): expr {.immediate.} = - ## generates the HTML ``style`` element. - let e = callsite() - result = xmlCheckedTag(e, "style", "media title", "type") +macro style*(e: varargs[untyped]): untyped = + ## Generates the HTML `style` element. + result = xmlCheckedTag(e, "style", "media type" & commonAttr) -macro sub*(e: expr): expr {.immediate.} = - ## generates the HTML ``sub`` element. - let e = callsite() +macro sub*(e: varargs[untyped]): untyped = + ## Generates the HTML `sub` element. result = xmlCheckedTag(e, "sub", commonAttr) -macro sup*(e: expr): expr {.immediate.} = - ## generates the HTML ``sup`` element. - let e = callsite() +macro summary*(e: varargs[untyped]): untyped = + ## Generates the HTML `summary` element. + result = xmlCheckedTag(e, "summary", commonAttr) + +macro sup*(e: varargs[untyped]): untyped = + ## Generates the HTML `sup` element. result = xmlCheckedTag(e, "sup", commonAttr) -macro table*(e: expr): expr {.immediate.} = - ## generates the HTML ``table`` element. - let e = callsite() - result = xmlCheckedTag(e, "table", "summary border cellpadding cellspacing" & - " frame rules width" & commonAttr) - -macro tbody*(e: expr): expr {.immediate.} = - ## generates the HTML ``tbody`` element. - let e = callsite() - result = xmlCheckedTag(e, "tbody", "align valign" & commonAttr) - -macro td*(e: expr): expr {.immediate.} = - ## generates the HTML ``td`` element. - let e = callsite() - result = xmlCheckedTag(e, "td", "colspan rowspan abbr axis headers scope" & - " align valign" & commonAttr) - -macro textarea*(e: expr): expr {.immediate.} = - ## generates the HTML ``textarea`` element. - let e = callsite() - result = xmlCheckedTag(e, "textarea", " name disabled readonly accesskey" & - " tabindex" & commonAttr, "rows cols", false) - -macro tfoot*(e: expr): expr {.immediate.} = - ## generates the HTML ``tfoot`` element. - let e = callsite() - result = xmlCheckedTag(e, "tfoot", "align valign" & commonAttr) - -macro th*(e: expr): expr {.immediate.} = - ## generates the HTML ``th`` element. - let e = callsite() - result = xmlCheckedTag(e, "th", "colspan rowspan abbr axis headers scope" & - " align valign" & commonAttr) - -macro thead*(e: expr): expr {.immediate.} = - ## generates the HTML ``thead`` element. - let e = callsite() - result = xmlCheckedTag(e, "thead", "align valign" & commonAttr) - -macro title*(e: expr): expr {.immediate.} = - ## generates the HTML ``title`` element. - let e = callsite() - result = xmlCheckedTag(e, "title") - -macro tr*(e: expr): expr {.immediate.} = - ## generates the HTML ``tr`` element. - let e = callsite() - result = xmlCheckedTag(e, "tr", "align valign" & commonAttr) - -macro tt*(e: expr): expr {.immediate.} = - ## generates the HTML ``tt`` element. - let e = callsite() +macro table*(e: varargs[untyped]): untyped = + ## Generates the HTML `table` element. + result = xmlCheckedTag(e, "table", "border sortable" & commonAttr) + +macro tbody*(e: varargs[untyped]): untyped = + ## Generates the HTML `tbody` element. + result = xmlCheckedTag(e, "tbody", commonAttr) + +macro td*(e: varargs[untyped]): untyped = + ## Generates the HTML `td` element. + result = xmlCheckedTag(e, "td", "colspan rowspan headers" & commonAttr) + +macro `template`*(e: varargs[untyped]): untyped = + ## Generates the HTML `template` element. + result = xmlCheckedTag(e, "template", commonAttr) + +macro textarea*(e: varargs[untyped]): untyped = + ## Generates the HTML `textarea` element. + result = xmlCheckedTag(e, "textarea", "autocomplete autofocus cols " & + "dirname disabled form inputmode maxlength minlength name placeholder " & + "readonly required rows wrap" & commonAttr) + +macro tfoot*(e: varargs[untyped]): untyped = + ## Generates the HTML `tfoot` element. + result = xmlCheckedTag(e, "tfoot", commonAttr) + +macro th*(e: varargs[untyped]): untyped = + ## Generates the HTML `th` element. + result = xmlCheckedTag(e, "th", "colspan rowspan headers abbr scope axis" & + " sorted" & commonAttr) + +macro thead*(e: varargs[untyped]): untyped = + ## Generates the HTML `thead` element. + result = xmlCheckedTag(e, "thead", commonAttr) + +macro time*(e: varargs[untyped]): untyped = + ## Generates the HTML `time` element. + result = xmlCheckedTag(e, "time", "datetime" & commonAttr) + +macro title*(e: varargs[untyped]): untyped = + ## Generates the HTML `title` element. + result = xmlCheckedTag(e, "title", commonAttr) + +macro tr*(e: varargs[untyped]): untyped = + ## Generates the HTML `tr` element. + result = xmlCheckedTag(e, "tr", commonAttr) + +macro track*(e: varargs[untyped]): untyped = + ## Generates the HTML `track` element. + result = xmlCheckedTag(e, "track", "kind srclang label default" & + commonAttr, "src", true) + +macro tt*(e: varargs[untyped]): untyped = + ## Generates the HTML `tt` element. result = xmlCheckedTag(e, "tt", commonAttr) -macro ul*(e: expr): expr {.immediate.} = - ## generates the HTML ``ul`` element. - let e = callsite() +macro u*(e: varargs[untyped]): untyped = + ## Generates the HTML `u` element. + result = xmlCheckedTag(e, "u", commonAttr) + +macro ul*(e: varargs[untyped]): untyped = + ## Generates the HTML `ul` element. result = xmlCheckedTag(e, "ul", commonAttr) -macro `var`*(e: expr): expr {.immediate.} = - ## generates the HTML ``var`` element. - let e = callsite() +macro `var`*(e: varargs[untyped]): untyped = + ## Generates the HTML `var` element. result = xmlCheckedTag(e, "var", commonAttr) -when isMainModule: +macro video*(e: varargs[untyped]): untyped = + ## Generates the HTML `video` element. + result = xmlCheckedTag(e, "video", "src crossorigin poster preload " & + "autoplay mediagroup loop muted controls width height" & commonAttr) + +macro wbr*(e: varargs[untyped]): untyped = + ## Generates the HTML `wbr` element. + result = xmlCheckedTag(e, "wbr", commonAttr, "", true) + +macro portal*(e: varargs[untyped]): untyped = + ## Generates the HTML `portal` element. + result = xmlCheckedTag(e, "portal", "width height type src disabled" & commonAttr, "", false) + + +macro math*(e: varargs[untyped]): untyped = + ## Generates the HTML `math` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/math#Examples + result = xmlCheckedTag(e, "math", "mathbackground mathcolor href overflow" & commonAttr) + +macro maction*(e: varargs[untyped]): untyped = + ## Generates the HTML `maction` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/maction + result = xmlCheckedTag(e, "maction", "mathbackground mathcolor href" & commonAttr) + +macro menclose*(e: varargs[untyped]): untyped = + ## Generates the HTML `menclose` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/menclose + result = xmlCheckedTag(e, "menclose", "mathbackground mathcolor href notation" & commonAttr) + +macro merror*(e: varargs[untyped]): untyped = + ## Generates the HTML `merror` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/merror + result = xmlCheckedTag(e, "merror", "mathbackground mathcolor href" & commonAttr) + +macro mfenced*(e: varargs[untyped]): untyped = + ## Generates the HTML `mfenced` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mfenced + result = xmlCheckedTag(e, "mfenced", "mathbackground mathcolor href open separators" & commonAttr) + +macro mfrac*(e: varargs[untyped]): untyped = + ## Generates the HTML `mfrac` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mfrac + result = xmlCheckedTag(e, "mfrac", "mathbackground mathcolor href linethickness numalign" & commonAttr) + +macro mglyph*(e: varargs[untyped]): untyped = + ## Generates the HTML `mglyph` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mglyph + result = xmlCheckedTag(e, "mglyph", "mathbackground mathcolor href src valign" & commonAttr) + +macro mi*(e: varargs[untyped]): untyped = + ## Generates the HTML `mi` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mi + result = xmlCheckedTag(e, "mi", "mathbackground mathcolor href mathsize mathvariant" & commonAttr) + +macro mlabeledtr*(e: varargs[untyped]): untyped = + ## Generates the HTML `mlabeledtr` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mlabeledtr + result = xmlCheckedTag(e, "mlabeledtr", "mathbackground mathcolor href columnalign groupalign rowalign" & commonAttr) + +macro mmultiscripts*(e: varargs[untyped]): untyped = + ## Generates the HTML `mmultiscripts` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mmultiscripts + result = xmlCheckedTag(e, "mmultiscripts", "mathbackground mathcolor href subscriptshift superscriptshift" & commonAttr) + +macro mn*(e: varargs[untyped]): untyped = + ## Generates the HTML `mn` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mn + result = xmlCheckedTag(e, "mn", "mathbackground mathcolor href mathsize mathvariant" & commonAttr) + +macro mo*(e: varargs[untyped]): untyped = + ## Generates the HTML `mo` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mo + result = xmlCheckedTag(e, "mo", + "mathbackground mathcolor fence form largeop lspace mathsize mathvariant movablelimits rspace separator stretchy symmetric" & commonAttr) + +macro mover*(e: varargs[untyped]): untyped = + ## Generates the HTML `mover` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mover + result = xmlCheckedTag(e, "mover", "mathbackground mathcolor accent href" & commonAttr) + +macro mpadded*(e: varargs[untyped]): untyped = + ## Generates the HTML `mpadded` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mpadded + result = xmlCheckedTag(e, "mpadded", "mathbackground mathcolor depth href lspace voffset" & commonAttr) + +macro mphantom*(e: varargs[untyped]): untyped = + ## Generates the HTML `mphantom` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mphantom + result = xmlCheckedTag(e, "mphantom", "mathbackground" & commonAttr) + +macro mroot*(e: varargs[untyped]): untyped = + ## Generates the HTML `mroot` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mroot + result = xmlCheckedTag(e, "mroot", "mathbackground mathcolor href" & commonAttr) + +macro mrow*(e: varargs[untyped]): untyped = + ## Generates the HTML `mrow` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mrow + result = xmlCheckedTag(e, "mrow", "mathbackground mathcolor href" & commonAttr) + +macro ms*(e: varargs[untyped]): untyped = + ## Generates the HTML `ms` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/ms + result = xmlCheckedTag(e, "ms", "mathbackground mathcolor href lquote mathsize mathvariant rquote" & commonAttr) + +macro mspace*(e: varargs[untyped]): untyped = + ## Generates the HTML `mspace` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mspace + result = xmlCheckedTag(e, "mspace", "mathbackground mathcolor href linebreak" & commonAttr) + +macro msqrt*(e: varargs[untyped]): untyped = + ## Generates the HTML `msqrt` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msqrt + result = xmlCheckedTag(e, "msqrt", "mathbackground mathcolor href" & commonAttr) + +macro mstyle*(e: varargs[untyped]): untyped = + ## Generates the HTML `mstyle` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mstyle + result = xmlCheckedTag(e, "mstyle", ("mathbackground mathcolor href decimalpoint displaystyle " & + "infixlinebreakstyle scriptlevel scriptminsize scriptsizemultiplier" & commonAttr)) + +macro msub*(e: varargs[untyped]): untyped = + ## Generates the HTML `msub` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msub + result = xmlCheckedTag(e, "msub", "mathbackground mathcolor href subscriptshift" & commonAttr) + +macro msubsup*(e: varargs[untyped]): untyped = + ## Generates the HTML `msubsup` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msubsup + result = xmlCheckedTag(e, "msubsup", "mathbackground mathcolor href subscriptshift superscriptshift" & commonAttr) + +macro msup*(e: varargs[untyped]): untyped = + ## Generates the HTML `msup` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msup + result = xmlCheckedTag(e, "msup", "mathbackground mathcolor href superscriptshift" & commonAttr) + +macro mtable*(e: varargs[untyped]): untyped = + ## Generates the HTML `mtable` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mtable + result = xmlCheckedTag(e, "mtable", ("mathbackground mathcolor href align " & + "alignmentscope columnalign columnlines columnspacing columnwidth " & + "displaystyle equalcolumns equalrows frame framespacing groupalign " & + "rowalign rowlines rowspacing side width" & commonAttr)) + +macro mtd*(e: varargs[untyped]): untyped = + ## Generates the HTML `mtd` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mtd + result = xmlCheckedTag(e, "mtd", + "mathbackground mathcolor href columnalign columnspan groupalign rowalign rowspan" & commonAttr) + +macro mtext*(e: varargs[untyped]): untyped = + ## Generates the HTML `mtext` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mtext + result = xmlCheckedTag(e, "mtext", "mathbackground mathcolor href mathsize mathvariant" & commonAttr) + +macro munder*(e: varargs[untyped]): untyped = + ## Generates the HTML `munder` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/munder + result = xmlCheckedTag(e, "munder", "mathbackground mathcolor href accentunder align" & commonAttr) + +macro munderover*(e: varargs[untyped]): untyped = + ## Generates the HTML `munderover` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/munderover + result = xmlCheckedTag(e, "munderover", "mathbackground mathcolor href accentunder accent align" & commonAttr) + +macro semantics*(e: varargs[untyped]): untyped = + ## Generates the HTML `semantics` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics + result = xmlCheckedTag(e, "semantics", "mathbackground mathcolor href definitionURL encoding cd src" & commonAttr) + +macro annotation*(e: varargs[untyped]): untyped = + ## Generates the HTML `annotation` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics + result = xmlCheckedTag(e, "annotation", "mathbackground mathcolor href definitionURL encoding cd src" & commonAttr) + +macro `annotation-xml`*(e: varargs[untyped]): untyped = + ## Generates the HTML `annotation-xml` element. MathML https://wikipedia.org/wiki/MathML + ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics + result = xmlCheckedTag(e, "annotation", "mathbackground mathcolor href definitionURL encoding cd src" & commonAttr) + + +runnableExamples: let nim = "Nim" - assert h1(a(href="http://nim-lang.org", nim)) == - """<h1><a href="http://nim-lang.org">Nim</a></h1>""" - assert form(action="test", `accept-charset` = "Content-Type") == + assert h1(a(href = "https://nim-lang.org", nim)) == + """<h1><a href="https://nim-lang.org">Nim</a></h1>""" + assert form(action = "test", `accept-charset` = "Content-Type") == """<form action="test" accept-charset="Content-Type"></form>""" + + + assert math( + semantics( + mrow( + msup( + mi("x"), + mn("42") + ) + ) + ) + ) == "<math><semantics><mrow><msup><mi>x</mi><mn>42</mn></msup></mrow></semantics></math>" + + assert math( + semantics( + annotation(encoding = "application/x-tex", title = "Latex on Web", r"x^{2} + y") + ) + ) == """<math><semantics><annotation encoding="application/x-tex" title="Latex on Web">x^{2} + y</annotation></semantics></math>""" diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index d620e816e..62919546f 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -7,19 +7,18 @@ # distribution, for details about the copyright. # -## This module parses an HTML document and creates its XML tree representation. -## It is supposed to handle the *wild* HTML the real world uses. +## **NOTE**: The behaviour might change in future versions as it is not +## clear what "*wild* HTML the real world uses" really implies. ## ## It can be used to parse a wild HTML document and output it as valid XHTML ## document (well, if you are lucky): -## -## .. code-block:: Nim -## +## ```Nim ## echo loadHtml("mydirty.html") +## ``` ## ## Every tag in the resulting tree is in lower case. ## -## **Note:** The resulting ``XmlNode`` already uses the ``clientData`` field, +## **Note:** The resulting `XmlNode` already uses the `clientData` field, ## so it cannot be used by clients of this library. ## ## Example: Transforming hyperlinks @@ -27,158 +26,161 @@ ## ## This code demonstrates how you can iterate over all the tags in an HTML file ## and write back the modified version. In this case we look for hyperlinks -## ending with the extension ``.rst`` and convert them to ``.html``. +## ending with the extension `.rst` and convert them to `.html`. ## -## .. code-block:: Nim -## -## import htmlparser -## import xmltree # To use '$' for XmlNode -## import strtabs # To access XmlAttributes -## import os # To use splitFile -## import strutils # To use cmpIgnoreCase +## ```Nim test +## import std/htmlparser +## import std/xmltree # To use '$' for XmlNode +## import std/strtabs # To access XmlAttributes +## import std/os # To use splitFile +## import std/strutils # To use cmpIgnoreCase ## ## proc transformHyperlinks() = -## let html = loadHTML("input.html") +## let html = loadHtml("input.html") ## ## for a in html.findAll("a"): -## let href = a.attrs["href"] -## if not href.isNil: -## let (dir, filename, ext) = splitFile(href) +## if a.attrs.hasKey "href": +## let (dir, filename, ext) = splitFile(a.attrs["href"]) ## if cmpIgnoreCase(ext, ".rst") == 0: ## a.attrs["href"] = dir / filename & ".html" ## ## writeFile("output.html", $html) +## ``` + +{.deprecated: "use `nimble install htmlparser` and import `pkg/htmlparser` instead".} -import strutils, streams, parsexml, xmltree, unicode, strtabs +import std/[strutils, streams, parsexml, xmltree, unicode, strtabs] + +when defined(nimPreviewSlimSystem): + import std/syncio type - HtmlTag* = enum ## list of all supported HTML tags; order will always be + HtmlTag* = enum ## list of all supported HTML tags; order will always be ## alphabetically tagUnknown, ## unknown HTML element - tagA, ## the HTML ``a`` element - tagAbbr, ## the deprecated HTML ``abbr`` element - tagAcronym, ## the HTML ``acronym`` element - tagAddress, ## the HTML ``address`` element - tagApplet, ## the deprecated HTML ``applet`` element - tagArea, ## the HTML ``area`` element - tagArticle, ## the HTML ``article`` element - tagAside, ## the HTML ``aside`` element - tagAudio, ## the HTML ``audio`` element - tagB, ## the HTML ``b`` element - tagBase, ## the HTML ``base`` element - tagBdi, ## the HTML ``bdi`` element - tagBdo, ## the deprecated HTML ``dbo`` element - tagBasefont, ## the deprecated HTML ``basefont`` element - tagBig, ## the HTML ``big`` element - tagBlockquote, ## the HTML ``blockquote`` element - tagBody, ## the HTML ``body`` element - tagBr, ## the HTML ``br`` element - tagButton, ## the HTML ``button`` element - tagCanvas, ## the HTML ``canvas`` element - tagCaption, ## the HTML ``caption`` element - tagCenter, ## the deprecated HTML ``center`` element - tagCite, ## the HTML ``cite`` element - tagCode, ## the HTML ``code`` element - tagCol, ## the HTML ``col`` element - tagColgroup, ## the HTML ``colgroup`` element - tagCommand, ## the HTML ``command`` element - tagDatalist, ## the HTML ``datalist`` element - tagDd, ## the HTML ``dd`` element - tagDel, ## the HTML ``del`` element - tagDetails, ## the HTML ``details`` element - tagDfn, ## the HTML ``dfn`` element - tagDialog, ## the HTML ``dialog`` element - tagDiv, ## the HTML ``div`` element - tagDir, ## the deprecated HTLM ``dir`` element - tagDl, ## the HTML ``dl`` element - tagDt, ## the HTML ``dt`` element - tagEm, ## the HTML ``em`` element - tagEmbed, ## the HTML ``embed`` element - tagFieldset, ## the HTML ``fieldset`` element - tagFigcaption, ## the HTML ``figcaption`` element - tagFigure, ## the HTML ``figure`` element - tagFont, ## the deprecated HTML ``font`` element - tagFooter, ## the HTML ``footer`` element - tagForm, ## the HTML ``form`` element - tagFrame, ## the HTML ``frame`` element - tagFrameset, ## the deprecated HTML ``frameset`` element - tagH1, ## the HTML ``h1`` element - tagH2, ## the HTML ``h2`` element - tagH3, ## the HTML ``h3`` element - tagH4, ## the HTML ``h4`` element - tagH5, ## the HTML ``h5`` element - tagH6, ## the HTML ``h6`` element - tagHead, ## the HTML ``head`` element - tagHeader, ## the HTML ``header`` element - tagHgroup, ## the HTML ``hgroup`` element - tagHtml, ## the HTML ``html`` element - tagHr, ## the HTML ``hr`` element - tagI, ## the HTML ``i`` element - tagIframe, ## the deprecated HTML ``iframe`` element - tagImg, ## the HTML ``img`` element - tagInput, ## the HTML ``input`` element - tagIns, ## the HTML ``ins`` element - tagIsindex, ## the deprecated HTML ``isindex`` element - tagKbd, ## the HTML ``kbd`` element - tagKeygen, ## the HTML ``keygen`` element - tagLabel, ## the HTML ``label`` element - tagLegend, ## the HTML ``legend`` element - tagLi, ## the HTML ``li`` element - tagLink, ## the HTML ``link`` element - tagMap, ## the HTML ``map`` element - tagMark, ## the HTML ``mark`` element - tagMenu, ## the deprecated HTML ``menu`` element - tagMeta, ## the HTML ``meta`` element - tagMeter, ## the HTML ``meter`` element - tagNav, ## the HTML ``nav`` element - tagNobr, ## the deprecated HTML ``nobr`` element - tagNoframes, ## the deprecated HTML ``noframes`` element - tagNoscript, ## the HTML ``noscript`` element - tagObject, ## the HTML ``object`` element - tagOl, ## the HTML ``ol`` element - tagOptgroup, ## the HTML ``optgroup`` element - tagOption, ## the HTML ``option`` element - tagOutput, ## the HTML ``output`` element - tagP, ## the HTML ``p`` element - tagParam, ## the HTML ``param`` element - tagPre, ## the HTML ``pre`` element - tagProgress, ## the HTML ``progress`` element - tagQ, ## the HTML ``q`` element - tagRp, ## the HTML ``rp`` element - tagRt, ## the HTML ``rt`` element - tagRuby, ## the HTML ``ruby`` element - tagS, ## the deprecated HTML ``s`` element - tagSamp, ## the HTML ``samp`` element - tagScript, ## the HTML ``script`` element - tagSection, ## the HTML ``section`` element - tagSelect, ## the HTML ``select`` element - tagSmall, ## the HTML ``small`` element - tagSource, ## the HTML ``source`` element - tagSpan, ## the HTML ``span`` element - tagStrike, ## the deprecated HTML ``strike`` element - tagStrong, ## the HTML ``strong`` element - tagStyle, ## the HTML ``style`` element - tagSub, ## the HTML ``sub`` element - tagSummary, ## the HTML ``summary`` element - tagSup, ## the HTML ``sup`` element - tagTable, ## the HTML ``table`` element - tagTbody, ## the HTML ``tbody`` element - tagTd, ## the HTML ``td`` element - tagTextarea, ## the HTML ``textarea`` element - tagTfoot, ## the HTML ``tfoot`` element - tagTh, ## the HTML ``th`` element - tagThead, ## the HTML ``thead`` element - tagTime, ## the HTML ``time`` element - tagTitle, ## the HTML ``title`` element - tagTr, ## the HTML ``tr`` element - tagTrack, ## the HTML ``track`` element - tagTt, ## the HTML ``tt`` element - tagU, ## the deprecated HTML ``u`` element - tagUl, ## the HTML ``ul`` element - tagVar, ## the HTML ``var`` element - tagVideo, ## the HTML ``video`` element - tagWbr ## the HTML ``wbr`` element -{.deprecated: [THtmlTag: HtmlTag].} + tagA, ## the HTML `a` element + tagAbbr, ## the deprecated HTML `abbr` element + tagAcronym, ## the HTML `acronym` element + tagAddress, ## the HTML `address` element + tagApplet, ## the deprecated HTML `applet` element + tagArea, ## the HTML `area` element + tagArticle, ## the HTML `article` element + tagAside, ## the HTML `aside` element + tagAudio, ## the HTML `audio` element + tagB, ## the HTML `b` element + tagBase, ## the HTML `base` element + tagBdi, ## the HTML `bdi` element + tagBdo, ## the deprecated HTML `dbo` element + tagBasefont, ## the deprecated HTML `basefont` element + tagBig, ## the HTML `big` element + tagBlockquote, ## the HTML `blockquote` element + tagBody, ## the HTML `body` element + tagBr, ## the HTML `br` element + tagButton, ## the HTML `button` element + tagCanvas, ## the HTML `canvas` element + tagCaption, ## the HTML `caption` element + tagCenter, ## the deprecated HTML `center` element + tagCite, ## the HTML `cite` element + tagCode, ## the HTML `code` element + tagCol, ## the HTML `col` element + tagColgroup, ## the HTML `colgroup` element + tagCommand, ## the HTML `command` element + tagDatalist, ## the HTML `datalist` element + tagDd, ## the HTML `dd` element + tagDel, ## the HTML `del` element + tagDetails, ## the HTML `details` element + tagDfn, ## the HTML `dfn` element + tagDialog, ## the HTML `dialog` element + tagDiv, ## the HTML `div` element + tagDir, ## the deprecated HTLM `dir` element + tagDl, ## the HTML `dl` element + tagDt, ## the HTML `dt` element + tagEm, ## the HTML `em` element + tagEmbed, ## the HTML `embed` element + tagFieldset, ## the HTML `fieldset` element + tagFigcaption, ## the HTML `figcaption` element + tagFigure, ## the HTML `figure` element + tagFont, ## the deprecated HTML `font` element + tagFooter, ## the HTML `footer` element + tagForm, ## the HTML `form` element + tagFrame, ## the HTML `frame` element + tagFrameset, ## the deprecated HTML `frameset` element + tagH1, ## the HTML `h1` element + tagH2, ## the HTML `h2` element + tagH3, ## the HTML `h3` element + tagH4, ## the HTML `h4` element + tagH5, ## the HTML `h5` element + tagH6, ## the HTML `h6` element + tagHead, ## the HTML `head` element + tagHeader, ## the HTML `header` element + tagHgroup, ## the HTML `hgroup` element + tagHtml, ## the HTML `html` element + tagHr, ## the HTML `hr` element + tagI, ## the HTML `i` element + tagIframe, ## the deprecated HTML `iframe` element + tagImg, ## the HTML `img` element + tagInput, ## the HTML `input` element + tagIns, ## the HTML `ins` element + tagIsindex, ## the deprecated HTML `isindex` element + tagKbd, ## the HTML `kbd` element + tagKeygen, ## the HTML `keygen` element + tagLabel, ## the HTML `label` element + tagLegend, ## the HTML `legend` element + tagLi, ## the HTML `li` element + tagLink, ## the HTML `link` element + tagMap, ## the HTML `map` element + tagMark, ## the HTML `mark` element + tagMenu, ## the deprecated HTML `menu` element + tagMeta, ## the HTML `meta` element + tagMeter, ## the HTML `meter` element + tagNav, ## the HTML `nav` element + tagNobr, ## the deprecated HTML `nobr` element + tagNoframes, ## the deprecated HTML `noframes` element + tagNoscript, ## the HTML `noscript` element + tagObject, ## the HTML `object` element + tagOl, ## the HTML `ol` element + tagOptgroup, ## the HTML `optgroup` element + tagOption, ## the HTML `option` element + tagOutput, ## the HTML `output` element + tagP, ## the HTML `p` element + tagParam, ## the HTML `param` element + tagPre, ## the HTML `pre` element + tagProgress, ## the HTML `progress` element + tagQ, ## the HTML `q` element + tagRp, ## the HTML `rp` element + tagRt, ## the HTML `rt` element + tagRuby, ## the HTML `ruby` element + tagS, ## the deprecated HTML `s` element + tagSamp, ## the HTML `samp` element + tagScript, ## the HTML `script` element + tagSection, ## the HTML `section` element + tagSelect, ## the HTML `select` element + tagSmall, ## the HTML `small` element + tagSource, ## the HTML `source` element + tagSpan, ## the HTML `span` element + tagStrike, ## the deprecated HTML `strike` element + tagStrong, ## the HTML `strong` element + tagStyle, ## the HTML `style` element + tagSub, ## the HTML `sub` element + tagSummary, ## the HTML `summary` element + tagSup, ## the HTML `sup` element + tagTable, ## the HTML `table` element + tagTbody, ## the HTML `tbody` element + tagTd, ## the HTML `td` element + tagTextarea, ## the HTML `textarea` element + tagTfoot, ## the HTML `tfoot` element + tagTh, ## the HTML `th` element + tagThead, ## the HTML `thead` element + tagTime, ## the HTML `time` element + tagTitle, ## the HTML `title` element + tagTr, ## the HTML `tr` element + tagTrack, ## the HTML `track` element + tagTt, ## the HTML `tt` element + tagU, ## the deprecated HTML `u` element + tagUl, ## the HTML `ul` element + tagVar, ## the HTML `var` element + tagVideo, ## the HTML `video` element + tagWbr ## the HTML `wbr` element const tagToStr* = [ @@ -216,80 +218,7 @@ const tagMenu, tagNoframes} SingleTags* = {tagArea, tagBase, tagBasefont, tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, - tagLink, tagMeta, tagParam, tagWbr} - - Entities = [ - ("nbsp", 0x00A0), ("iexcl", 0x00A1), ("cent", 0x00A2), ("pound", 0x00A3), - ("curren", 0x00A4), ("yen", 0x00A5), ("brvbar", 0x00A6), ("sect", 0x00A7), - ("uml", 0x00A8), ("copy", 0x00A9), ("ordf", 0x00AA), ("laquo", 0x00AB), - ("not", 0x00AC), ("shy", 0x00AD), ("reg", 0x00AE), ("macr", 0x00AF), - ("deg", 0x00B0), ("plusmn", 0x00B1), ("sup2", 0x00B2), ("sup3", 0x00B3), - ("acute", 0x00B4), ("micro", 0x00B5), ("para", 0x00B6), ("middot", 0x00B7), - ("cedil", 0x00B8), ("sup1", 0x00B9), ("ordm", 0x00BA), ("raquo", 0x00BB), - ("frac14", 0x00BC), ("frac12", 0x00BD), ("frac34", 0x00BE), - ("iquest", 0x00BF), ("Agrave", 0x00C0), ("Aacute", 0x00C1), - ("Acirc", 0x00C2), ("Atilde", 0x00C3), ("Auml", 0x00C4), ("Aring", 0x00C5), - ("AElig", 0x00C6), ("Ccedil", 0x00C7), ("Egrave", 0x00C8), - ("Eacute", 0x00C9), ("Ecirc", 0x00CA), ("Euml", 0x00CB), ("Igrave", 0x00CC), - ("Iacute", 0x00CD), ("Icirc", 0x00CE), ("Iuml", 0x00CF), ("ETH", 0x00D0), - ("Ntilde", 0x00D1), ("Ograve", 0x00D2), ("Oacute", 0x00D3), - ("Ocirc", 0x00D4), ("Otilde", 0x00D5), ("Ouml", 0x00D6), ("times", 0x00D7), - ("Oslash", 0x00D8), ("Ugrave", 0x00D9), ("Uacute", 0x00DA), - ("Ucirc", 0x00DB), ("Uuml", 0x00DC), ("Yacute", 0x00DD), ("THORN", 0x00DE), - ("szlig", 0x00DF), ("agrave", 0x00E0), ("aacute", 0x00E1), - ("acirc", 0x00E2), ("atilde", 0x00E3), ("auml", 0x00E4), ("aring", 0x00E5), - ("aelig", 0x00E6), ("ccedil", 0x00E7), ("egrave", 0x00E8), - ("eacute", 0x00E9), ("ecirc", 0x00EA), ("euml", 0x00EB), ("igrave", 0x00EC), - ("iacute", 0x00ED), ("icirc", 0x00EE), ("iuml", 0x00EF), ("eth", 0x00F0), - ("ntilde", 0x00F1), ("ograve", 0x00F2), ("oacute", 0x00F3), - ("ocirc", 0x00F4), ("otilde", 0x00F5), ("ouml", 0x00F6), ("divide", 0x00F7), - ("oslash", 0x00F8), ("ugrave", 0x00F9), ("uacute", 0x00FA), - ("ucirc", 0x00FB), ("uuml", 0x00FC), ("yacute", 0x00FD), ("thorn", 0x00FE), - ("yuml", 0x00FF), ("OElig", 0x0152), ("oelig", 0x0153), ("Scaron", 0x0160), - ("scaron", 0x0161), ("Yuml", 0x0178), ("fnof", 0x0192), ("circ", 0x02C6), - ("tilde", 0x02DC), ("Alpha", 0x0391), ("Beta", 0x0392), ("Gamma", 0x0393), - ("Delta", 0x0394), ("Epsilon", 0x0395), ("Zeta", 0x0396), ("Eta", 0x0397), - ("Theta", 0x0398), ("Iota", 0x0399), ("Kappa", 0x039A), ("Lambda", 0x039B), - ("Mu", 0x039C), ("Nu", 0x039D), ("Xi", 0x039E), ("Omicron", 0x039F), - ("Pi", 0x03A0), ("Rho", 0x03A1), ("Sigma", 0x03A3), ("Tau", 0x03A4), - ("Upsilon", 0x03A5), ("Phi", 0x03A6), ("Chi", 0x03A7), ("Psi", 0x03A8), - ("Omega", 0x03A9), ("alpha", 0x03B1), ("beta", 0x03B2), ("gamma", 0x03B3), - ("delta", 0x03B4), ("epsilon", 0x03B5), ("zeta", 0x03B6), ("eta", 0x03B7), - ("theta", 0x03B8), ("iota", 0x03B9), ("kappa", 0x03BA), ("lambda", 0x03BB), - ("mu", 0x03BC), ("nu", 0x03BD), ("xi", 0x03BE), ("omicron", 0x03BF), - ("pi", 0x03C0), ("rho", 0x03C1), ("sigmaf", 0x03C2), ("sigma", 0x03C3), - ("tau", 0x03C4), ("upsilon", 0x03C5), ("phi", 0x03C6), ("chi", 0x03C7), - ("psi", 0x03C8), ("omega", 0x03C9), ("thetasym", 0x03D1), ("upsih", 0x03D2), - ("piv", 0x03D6), ("ensp", 0x2002), ("emsp", 0x2003), ("thinsp", 0x2009), - ("zwnj", 0x200C), ("zwj", 0x200D), ("lrm", 0x200E), ("rlm", 0x200F), - ("ndash", 0x2013), ("mdash", 0x2014), ("lsquo", 0x2018), ("rsquo", 0x2019), - ("sbquo", 0x201A), ("ldquo", 0x201C), ("rdquo", 0x201D), ("bdquo", 0x201E), - ("dagger", 0x2020), ("Dagger", 0x2021), ("bull", 0x2022), - ("hellip", 0x2026), ("permil", 0x2030), ("prime", 0x2032), - ("Prime", 0x2033), ("lsaquo", 0x2039), ("rsaquo", 0x203A), - ("oline", 0x203E), ("frasl", 0x2044), ("euro", 0x20AC), - ("image", 0x2111), ("weierp", 0x2118), ("real", 0x211C), - ("trade", 0x2122), ("alefsym", 0x2135), ("larr", 0x2190), - ("uarr", 0x2191), ("rarr", 0x2192), ("darr", 0x2193), - ("harr", 0x2194), ("crarr", 0x21B5), ("lArr", 0x21D0), - ("uArr", 0x21D1), ("rArr", 0x21D2), ("dArr", 0x21D3), - ("hArr", 0x21D4), ("forall", 0x2200), ("part", 0x2202), - ("exist", 0x2203), ("empty", 0x2205), ("nabla", 0x2207), - ("isin", 0x2208), ("notin", 0x2209), ("ni", 0x220B), - ("prod", 0x220F), ("sum", 0x2211), ("minus", 0x2212), - ("lowast", 0x2217), ("radic", 0x221A), ("prop", 0x221D), - ("infin", 0x221E), ("ang", 0x2220), ("and", 0x2227), - ("or", 0x2228), ("cap", 0x2229), ("cup", 0x222A), - ("int", 0x222B), ("there4", 0x2234), ("sim", 0x223C), - ("cong", 0x2245), ("asymp", 0x2248), ("ne", 0x2260), - ("equiv", 0x2261), ("le", 0x2264), ("ge", 0x2265), - ("sub", 0x2282), ("sup", 0x2283), ("nsub", 0x2284), - ("sube", 0x2286), ("supe", 0x2287), ("oplus", 0x2295), - ("otimes", 0x2297), ("perp", 0x22A5), ("sdot", 0x22C5), - ("lceil", 0x2308), ("rceil", 0x2309), ("lfloor", 0x230A), - ("rfloor", 0x230B), ("lang", 0x2329), ("rang", 0x232A), - ("loz", 0x25CA), ("spades", 0x2660), ("clubs", 0x2663), - ("hearts", 0x2665), ("diams", 0x2666)] + tagLink, tagMeta, tagParam, tagWbr, tagSource} proc allLower(s: string): bool = for c in s: @@ -425,38 +354,1561 @@ proc toHtmlTag(s: string): HtmlTag = proc htmlTag*(n: XmlNode): HtmlTag = - ## gets `n`'s tag as a ``HtmlTag``. + ## Gets `n`'s tag as a `HtmlTag`. if n.clientData == 0: n.clientData = toHtmlTag(n.tag).ord result = HtmlTag(n.clientData) proc htmlTag*(s: string): HtmlTag = - ## converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is + ## Converts `s` to a `HtmlTag`. If `s` is no HTML tag, `tagUnknown` is ## returned. - let s = if allLower(s): s else: s.toLower + let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) +proc runeToEntity*(rune: Rune): string = + ## converts a Rune to its numeric HTML entity equivalent. + runnableExamples: + import std/unicode + doAssert runeToEntity(Rune(0)) == "" + doAssert runeToEntity(Rune(-1)) == "" + doAssert runeToEntity("Ü".runeAt(0)) == "#220" + doAssert runeToEntity("∈".runeAt(0)) == "#8712" + if rune.ord <= 0: result = "" + else: result = '#' & $rune.ord + +proc entityToRune*(entity: string): Rune = + ## Converts an HTML entity name like `Ü` or values like `Ü` + ## or `Ü` to its UTF-8 equivalent. + ## Rune(0) is returned if the entity name is unknown. + runnableExamples: + import std/unicode + doAssert entityToRune("") == Rune(0) + doAssert entityToRune("a") == Rune(0) + doAssert entityToRune("gt") == ">".runeAt(0) + doAssert entityToRune("Uuml") == "Ü".runeAt(0) + doAssert entityToRune("quest") == "?".runeAt(0) + doAssert entityToRune("#x0003F") == "?".runeAt(0) + if entity.len < 2: return # smallest entity has length 2 + if entity[0] == '#': + var runeValue = 0 + case entity[1] + of '0'..'9': + try: runeValue = parseInt(entity[1..^1]) + except ValueError: discard + of 'x', 'X': # not case sensitive here + try: runeValue = parseHexInt(entity[2..^1]) + except ValueError: discard + else: discard # other entities are not defined with prefix `#` + if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values + return Rune(runeValue) + case entity # entity names are case sensitive + of "Tab": Rune(0x00009) + of "NewLine": Rune(0x0000A) + of "excl": Rune(0x00021) + of "quot", "QUOT": Rune(0x00022) + of "num": Rune(0x00023) + of "dollar": Rune(0x00024) + of "percnt": Rune(0x00025) + of "amp", "AMP": Rune(0x00026) + of "apos": Rune(0x00027) + of "lpar": Rune(0x00028) + of "rpar": Rune(0x00029) + of "ast", "midast": Rune(0x0002A) + of "plus": Rune(0x0002B) + of "comma": Rune(0x0002C) + of "period": Rune(0x0002E) + of "sol": Rune(0x0002F) + of "colon": Rune(0x0003A) + of "semi": Rune(0x0003B) + of "lt", "LT": Rune(0x0003C) + of "equals": Rune(0x0003D) + of "gt", "GT": Rune(0x0003E) + of "quest": Rune(0x0003F) + of "commat": Rune(0x00040) + of "lsqb", "lbrack": Rune(0x0005B) + of "bsol": Rune(0x0005C) + of "rsqb", "rbrack": Rune(0x0005D) + of "Hat": Rune(0x0005E) + of "lowbar": Rune(0x0005F) + of "grave", "DiacriticalGrave": Rune(0x00060) + of "lcub", "lbrace": Rune(0x0007B) + of "verbar", "vert", "VerticalLine": Rune(0x0007C) + of "rcub", "rbrace": Rune(0x0007D) + of "nbsp", "NonBreakingSpace": Rune(0x000A0) + of "iexcl": Rune(0x000A1) + of "cent": Rune(0x000A2) + of "pound": Rune(0x000A3) + of "curren": Rune(0x000A4) + of "yen": Rune(0x000A5) + of "brvbar": Rune(0x000A6) + of "sect": Rune(0x000A7) + of "Dot", "die", "DoubleDot", "uml": Rune(0x000A8) + of "copy", "COPY": Rune(0x000A9) + of "ordf": Rune(0x000AA) + of "laquo": Rune(0x000AB) + of "not": Rune(0x000AC) + of "shy": Rune(0x000AD) + of "reg", "circledR", "REG": Rune(0x000AE) + of "macr", "OverBar", "strns": Rune(0x000AF) + of "deg": Rune(0x000B0) + of "plusmn", "pm", "PlusMinus": Rune(0x000B1) + of "sup2": Rune(0x000B2) + of "sup3": Rune(0x000B3) + of "acute", "DiacriticalAcute": Rune(0x000B4) + of "micro": Rune(0x000B5) + of "para": Rune(0x000B6) + of "middot", "centerdot", "CenterDot": Rune(0x000B7) + of "cedil", "Cedilla": Rune(0x000B8) + of "sup1": Rune(0x000B9) + of "ordm": Rune(0x000BA) + of "raquo": Rune(0x000BB) + of "frac14": Rune(0x000BC) + of "frac12", "half": Rune(0x000BD) + of "frac34": Rune(0x000BE) + of "iquest": Rune(0x000BF) + of "Agrave": Rune(0x000C0) + of "Aacute": Rune(0x000C1) + of "Acirc": Rune(0x000C2) + of "Atilde": Rune(0x000C3) + of "Auml": Rune(0x000C4) + of "Aring": Rune(0x000C5) + of "AElig": Rune(0x000C6) + of "Ccedil": Rune(0x000C7) + of "Egrave": Rune(0x000C8) + of "Eacute": Rune(0x000C9) + of "Ecirc": Rune(0x000CA) + of "Euml": Rune(0x000CB) + of "Igrave": Rune(0x000CC) + of "Iacute": Rune(0x000CD) + of "Icirc": Rune(0x000CE) + of "Iuml": Rune(0x000CF) + of "ETH": Rune(0x000D0) + of "Ntilde": Rune(0x000D1) + of "Ograve": Rune(0x000D2) + of "Oacute": Rune(0x000D3) + of "Ocirc": Rune(0x000D4) + of "Otilde": Rune(0x000D5) + of "Ouml": Rune(0x000D6) + of "times": Rune(0x000D7) + of "Oslash": Rune(0x000D8) + of "Ugrave": Rune(0x000D9) + of "Uacute": Rune(0x000DA) + of "Ucirc": Rune(0x000DB) + of "Uuml": Rune(0x000DC) + of "Yacute": Rune(0x000DD) + of "THORN": Rune(0x000DE) + of "szlig": Rune(0x000DF) + of "agrave": Rune(0x000E0) + of "aacute": Rune(0x000E1) + of "acirc": Rune(0x000E2) + of "atilde": Rune(0x000E3) + of "auml": Rune(0x000E4) + of "aring": Rune(0x000E5) + of "aelig": Rune(0x000E6) + of "ccedil": Rune(0x000E7) + of "egrave": Rune(0x000E8) + of "eacute": Rune(0x000E9) + of "ecirc": Rune(0x000EA) + of "euml": Rune(0x000EB) + of "igrave": Rune(0x000EC) + of "iacute": Rune(0x000ED) + of "icirc": Rune(0x000EE) + of "iuml": Rune(0x000EF) + of "eth": Rune(0x000F0) + of "ntilde": Rune(0x000F1) + of "ograve": Rune(0x000F2) + of "oacute": Rune(0x000F3) + of "ocirc": Rune(0x000F4) + of "otilde": Rune(0x000F5) + of "ouml": Rune(0x000F6) + of "divide", "div": Rune(0x000F7) + of "oslash": Rune(0x000F8) + of "ugrave": Rune(0x000F9) + of "uacute": Rune(0x000FA) + of "ucirc": Rune(0x000FB) + of "uuml": Rune(0x000FC) + of "yacute": Rune(0x000FD) + of "thorn": Rune(0x000FE) + of "yuml": Rune(0x000FF) + of "Amacr": Rune(0x00100) + of "amacr": Rune(0x00101) + of "Abreve": Rune(0x00102) + of "abreve": Rune(0x00103) + of "Aogon": Rune(0x00104) + of "aogon": Rune(0x00105) + of "Cacute": Rune(0x00106) + of "cacute": Rune(0x00107) + of "Ccirc": Rune(0x00108) + of "ccirc": Rune(0x00109) + of "Cdot": Rune(0x0010A) + of "cdot": Rune(0x0010B) + of "Ccaron": Rune(0x0010C) + of "ccaron": Rune(0x0010D) + of "Dcaron": Rune(0x0010E) + of "dcaron": Rune(0x0010F) + of "Dstrok": Rune(0x00110) + of "dstrok": Rune(0x00111) + of "Emacr": Rune(0x00112) + of "emacr": Rune(0x00113) + of "Edot": Rune(0x00116) + of "edot": Rune(0x00117) + of "Eogon": Rune(0x00118) + of "eogon": Rune(0x00119) + of "Ecaron": Rune(0x0011A) + of "ecaron": Rune(0x0011B) + of "Gcirc": Rune(0x0011C) + of "gcirc": Rune(0x0011D) + of "Gbreve": Rune(0x0011E) + of "gbreve": Rune(0x0011F) + of "Gdot": Rune(0x00120) + of "gdot": Rune(0x00121) + of "Gcedil": Rune(0x00122) + of "Hcirc": Rune(0x00124) + of "hcirc": Rune(0x00125) + of "Hstrok": Rune(0x00126) + of "hstrok": Rune(0x00127) + of "Itilde": Rune(0x00128) + of "itilde": Rune(0x00129) + of "Imacr": Rune(0x0012A) + of "imacr": Rune(0x0012B) + of "Iogon": Rune(0x0012E) + of "iogon": Rune(0x0012F) + of "Idot": Rune(0x00130) + of "imath", "inodot": Rune(0x00131) + of "IJlig": Rune(0x00132) + of "ijlig": Rune(0x00133) + of "Jcirc": Rune(0x00134) + of "jcirc": Rune(0x00135) + of "Kcedil": Rune(0x00136) + of "kcedil": Rune(0x00137) + of "kgreen": Rune(0x00138) + of "Lacute": Rune(0x00139) + of "lacute": Rune(0x0013A) + of "Lcedil": Rune(0x0013B) + of "lcedil": Rune(0x0013C) + of "Lcaron": Rune(0x0013D) + of "lcaron": Rune(0x0013E) + of "Lmidot": Rune(0x0013F) + of "lmidot": Rune(0x00140) + of "Lstrok": Rune(0x00141) + of "lstrok": Rune(0x00142) + of "Nacute": Rune(0x00143) + of "nacute": Rune(0x00144) + of "Ncedil": Rune(0x00145) + of "ncedil": Rune(0x00146) + of "Ncaron": Rune(0x00147) + of "ncaron": Rune(0x00148) + of "napos": Rune(0x00149) + of "ENG": Rune(0x0014A) + of "eng": Rune(0x0014B) + of "Omacr": Rune(0x0014C) + of "omacr": Rune(0x0014D) + of "Odblac": Rune(0x00150) + of "odblac": Rune(0x00151) + of "OElig": Rune(0x00152) + of "oelig": Rune(0x00153) + of "Racute": Rune(0x00154) + of "racute": Rune(0x00155) + of "Rcedil": Rune(0x00156) + of "rcedil": Rune(0x00157) + of "Rcaron": Rune(0x00158) + of "rcaron": Rune(0x00159) + of "Sacute": Rune(0x0015A) + of "sacute": Rune(0x0015B) + of "Scirc": Rune(0x0015C) + of "scirc": Rune(0x0015D) + of "Scedil": Rune(0x0015E) + of "scedil": Rune(0x0015F) + of "Scaron": Rune(0x00160) + of "scaron": Rune(0x00161) + of "Tcedil": Rune(0x00162) + of "tcedil": Rune(0x00163) + of "Tcaron": Rune(0x00164) + of "tcaron": Rune(0x00165) + of "Tstrok": Rune(0x00166) + of "tstrok": Rune(0x00167) + of "Utilde": Rune(0x00168) + of "utilde": Rune(0x00169) + of "Umacr": Rune(0x0016A) + of "umacr": Rune(0x0016B) + of "Ubreve": Rune(0x0016C) + of "ubreve": Rune(0x0016D) + of "Uring": Rune(0x0016E) + of "uring": Rune(0x0016F) + of "Udblac": Rune(0x00170) + of "udblac": Rune(0x00171) + of "Uogon": Rune(0x00172) + of "uogon": Rune(0x00173) + of "Wcirc": Rune(0x00174) + of "wcirc": Rune(0x00175) + of "Ycirc": Rune(0x00176) + of "ycirc": Rune(0x00177) + of "Yuml": Rune(0x00178) + of "Zacute": Rune(0x00179) + of "zacute": Rune(0x0017A) + of "Zdot": Rune(0x0017B) + of "zdot": Rune(0x0017C) + of "Zcaron": Rune(0x0017D) + of "zcaron": Rune(0x0017E) + of "fnof": Rune(0x00192) + of "imped": Rune(0x001B5) + of "gacute": Rune(0x001F5) + of "jmath": Rune(0x00237) + of "circ": Rune(0x002C6) + of "caron", "Hacek": Rune(0x002C7) + of "breve", "Breve": Rune(0x002D8) + of "dot", "DiacriticalDot": Rune(0x002D9) + of "ring": Rune(0x002DA) + of "ogon": Rune(0x002DB) + of "tilde", "DiacriticalTilde": Rune(0x002DC) + of "dblac", "DiacriticalDoubleAcute": Rune(0x002DD) + of "DownBreve": Rune(0x00311) + of "UnderBar": Rune(0x00332) + of "Alpha": Rune(0x00391) + of "Beta": Rune(0x00392) + of "Gamma": Rune(0x00393) + of "Delta": Rune(0x00394) + of "Epsilon": Rune(0x00395) + of "Zeta": Rune(0x00396) + of "Eta": Rune(0x00397) + of "Theta": Rune(0x00398) + of "Iota": Rune(0x00399) + of "Kappa": Rune(0x0039A) + of "Lambda": Rune(0x0039B) + of "Mu": Rune(0x0039C) + of "Nu": Rune(0x0039D) + of "Xi": Rune(0x0039E) + of "Omicron": Rune(0x0039F) + of "Pi": Rune(0x003A0) + of "Rho": Rune(0x003A1) + of "Sigma": Rune(0x003A3) + of "Tau": Rune(0x003A4) + of "Upsilon": Rune(0x003A5) + of "Phi": Rune(0x003A6) + of "Chi": Rune(0x003A7) + of "Psi": Rune(0x003A8) + of "Omega": Rune(0x003A9) + of "alpha": Rune(0x003B1) + of "beta": Rune(0x003B2) + of "gamma": Rune(0x003B3) + of "delta": Rune(0x003B4) + of "epsiv", "varepsilon", "epsilon": Rune(0x003B5) + of "zeta": Rune(0x003B6) + of "eta": Rune(0x003B7) + of "theta": Rune(0x003B8) + of "iota": Rune(0x003B9) + of "kappa": Rune(0x003BA) + of "lambda": Rune(0x003BB) + of "mu": Rune(0x003BC) + of "nu": Rune(0x003BD) + of "xi": Rune(0x003BE) + of "omicron": Rune(0x003BF) + of "pi": Rune(0x003C0) + of "rho": Rune(0x003C1) + of "sigmav", "varsigma", "sigmaf": Rune(0x003C2) + of "sigma": Rune(0x003C3) + of "tau": Rune(0x003C4) + of "upsi", "upsilon": Rune(0x003C5) + of "phi", "phiv", "varphi": Rune(0x003C6) + of "chi": Rune(0x003C7) + of "psi": Rune(0x003C8) + of "omega": Rune(0x003C9) + of "thetav", "vartheta", "thetasym": Rune(0x003D1) + of "Upsi", "upsih": Rune(0x003D2) + of "straightphi": Rune(0x003D5) + of "piv", "varpi": Rune(0x003D6) + of "Gammad": Rune(0x003DC) + of "gammad", "digamma": Rune(0x003DD) + of "kappav", "varkappa": Rune(0x003F0) + of "rhov", "varrho": Rune(0x003F1) + of "epsi", "straightepsilon": Rune(0x003F5) + of "bepsi", "backepsilon": Rune(0x003F6) + of "IOcy": Rune(0x00401) + of "DJcy": Rune(0x00402) + of "GJcy": Rune(0x00403) + of "Jukcy": Rune(0x00404) + of "DScy": Rune(0x00405) + of "Iukcy": Rune(0x00406) + of "YIcy": Rune(0x00407) + of "Jsercy": Rune(0x00408) + of "LJcy": Rune(0x00409) + of "NJcy": Rune(0x0040A) + of "TSHcy": Rune(0x0040B) + of "KJcy": Rune(0x0040C) + of "Ubrcy": Rune(0x0040E) + of "DZcy": Rune(0x0040F) + of "Acy": Rune(0x00410) + of "Bcy": Rune(0x00411) + of "Vcy": Rune(0x00412) + of "Gcy": Rune(0x00413) + of "Dcy": Rune(0x00414) + of "IEcy": Rune(0x00415) + of "ZHcy": Rune(0x00416) + of "Zcy": Rune(0x00417) + of "Icy": Rune(0x00418) + of "Jcy": Rune(0x00419) + of "Kcy": Rune(0x0041A) + of "Lcy": Rune(0x0041B) + of "Mcy": Rune(0x0041C) + of "Ncy": Rune(0x0041D) + of "Ocy": Rune(0x0041E) + of "Pcy": Rune(0x0041F) + of "Rcy": Rune(0x00420) + of "Scy": Rune(0x00421) + of "Tcy": Rune(0x00422) + of "Ucy": Rune(0x00423) + of "Fcy": Rune(0x00424) + of "KHcy": Rune(0x00425) + of "TScy": Rune(0x00426) + of "CHcy": Rune(0x00427) + of "SHcy": Rune(0x00428) + of "SHCHcy": Rune(0x00429) + of "HARDcy": Rune(0x0042A) + of "Ycy": Rune(0x0042B) + of "SOFTcy": Rune(0x0042C) + of "Ecy": Rune(0x0042D) + of "YUcy": Rune(0x0042E) + of "YAcy": Rune(0x0042F) + of "acy": Rune(0x00430) + of "bcy": Rune(0x00431) + of "vcy": Rune(0x00432) + of "gcy": Rune(0x00433) + of "dcy": Rune(0x00434) + of "iecy": Rune(0x00435) + of "zhcy": Rune(0x00436) + of "zcy": Rune(0x00437) + of "icy": Rune(0x00438) + of "jcy": Rune(0x00439) + of "kcy": Rune(0x0043A) + of "lcy": Rune(0x0043B) + of "mcy": Rune(0x0043C) + of "ncy": Rune(0x0043D) + of "ocy": Rune(0x0043E) + of "pcy": Rune(0x0043F) + of "rcy": Rune(0x00440) + of "scy": Rune(0x00441) + of "tcy": Rune(0x00442) + of "ucy": Rune(0x00443) + of "fcy": Rune(0x00444) + of "khcy": Rune(0x00445) + of "tscy": Rune(0x00446) + of "chcy": Rune(0x00447) + of "shcy": Rune(0x00448) + of "shchcy": Rune(0x00449) + of "hardcy": Rune(0x0044A) + of "ycy": Rune(0x0044B) + of "softcy": Rune(0x0044C) + of "ecy": Rune(0x0044D) + of "yucy": Rune(0x0044E) + of "yacy": Rune(0x0044F) + of "iocy": Rune(0x00451) + of "djcy": Rune(0x00452) + of "gjcy": Rune(0x00453) + of "jukcy": Rune(0x00454) + of "dscy": Rune(0x00455) + of "iukcy": Rune(0x00456) + of "yicy": Rune(0x00457) + of "jsercy": Rune(0x00458) + of "ljcy": Rune(0x00459) + of "njcy": Rune(0x0045A) + of "tshcy": Rune(0x0045B) + of "kjcy": Rune(0x0045C) + of "ubrcy": Rune(0x0045E) + of "dzcy": Rune(0x0045F) + of "ensp": Rune(0x02002) + of "emsp": Rune(0x02003) + of "emsp13": Rune(0x02004) + of "emsp14": Rune(0x02005) + of "numsp": Rune(0x02007) + of "puncsp": Rune(0x02008) + of "thinsp", "ThinSpace": Rune(0x02009) + of "hairsp", "VeryThinSpace": Rune(0x0200A) + of "ZeroWidthSpace", "NegativeVeryThinSpace", "NegativeThinSpace", + "NegativeMediumSpace", "NegativeThickSpace": Rune(0x0200B) + of "zwnj": Rune(0x0200C) + of "zwj": Rune(0x0200D) + of "lrm": Rune(0x0200E) + of "rlm": Rune(0x0200F) + of "hyphen", "dash": Rune(0x02010) + of "ndash": Rune(0x02013) + of "mdash": Rune(0x02014) + of "horbar": Rune(0x02015) + of "Verbar", "Vert": Rune(0x02016) + of "lsquo", "OpenCurlyQuote": Rune(0x02018) + of "rsquo", "rsquor", "CloseCurlyQuote": Rune(0x02019) + of "lsquor", "sbquo": Rune(0x0201A) + of "ldquo", "OpenCurlyDoubleQuote": Rune(0x0201C) + of "rdquo", "rdquor", "CloseCurlyDoubleQuote": Rune(0x0201D) + of "ldquor", "bdquo": Rune(0x0201E) + of "dagger": Rune(0x02020) + of "Dagger", "ddagger": Rune(0x02021) + of "bull", "bullet": Rune(0x02022) + of "nldr": Rune(0x02025) + of "hellip", "mldr": Rune(0x02026) + of "permil": Rune(0x02030) + of "pertenk": Rune(0x02031) + of "prime": Rune(0x02032) + of "Prime": Rune(0x02033) + of "tprime": Rune(0x02034) + of "bprime", "backprime": Rune(0x02035) + of "lsaquo": Rune(0x02039) + of "rsaquo": Rune(0x0203A) + of "oline": Rune(0x0203E) + of "caret": Rune(0x02041) + of "hybull": Rune(0x02043) + of "frasl": Rune(0x02044) + of "bsemi": Rune(0x0204F) + of "qprime": Rune(0x02057) + of "MediumSpace": Rune(0x0205F) + of "NoBreak": Rune(0x02060) + of "ApplyFunction", "af": Rune(0x02061) + of "InvisibleTimes", "it": Rune(0x02062) + of "InvisibleComma", "ic": Rune(0x02063) + of "euro": Rune(0x020AC) + of "tdot", "TripleDot": Rune(0x020DB) + of "DotDot": Rune(0x020DC) + of "Copf", "complexes": Rune(0x02102) + of "incare": Rune(0x02105) + of "gscr": Rune(0x0210A) + of "hamilt", "HilbertSpace", "Hscr": Rune(0x0210B) + of "Hfr", "Poincareplane": Rune(0x0210C) + of "quaternions", "Hopf": Rune(0x0210D) + of "planckh": Rune(0x0210E) + of "planck", "hbar", "plankv", "hslash": Rune(0x0210F) + of "Iscr", "imagline": Rune(0x02110) + of "image", "Im", "imagpart", "Ifr": Rune(0x02111) + of "Lscr", "lagran", "Laplacetrf": Rune(0x02112) + of "ell": Rune(0x02113) + of "Nopf", "naturals": Rune(0x02115) + of "numero": Rune(0x02116) + of "copysr": Rune(0x02117) + of "weierp", "wp": Rune(0x02118) + of "Popf", "primes": Rune(0x02119) + of "rationals", "Qopf": Rune(0x0211A) + of "Rscr", "realine": Rune(0x0211B) + of "real", "Re", "realpart", "Rfr": Rune(0x0211C) + of "reals", "Ropf": Rune(0x0211D) + of "rx": Rune(0x0211E) + of "trade", "TRADE": Rune(0x02122) + of "integers", "Zopf": Rune(0x02124) + of "ohm": Rune(0x02126) + of "mho": Rune(0x02127) + of "Zfr", "zeetrf": Rune(0x02128) + of "iiota": Rune(0x02129) + of "angst": Rune(0x0212B) + of "bernou", "Bernoullis", "Bscr": Rune(0x0212C) + of "Cfr", "Cayleys": Rune(0x0212D) + of "escr": Rune(0x0212F) + of "Escr", "expectation": Rune(0x02130) + of "Fscr", "Fouriertrf": Rune(0x02131) + of "phmmat", "Mellintrf", "Mscr": Rune(0x02133) + of "order", "orderof", "oscr": Rune(0x02134) + of "alefsym", "aleph": Rune(0x02135) + of "beth": Rune(0x02136) + of "gimel": Rune(0x02137) + of "daleth": Rune(0x02138) + of "CapitalDifferentialD", "DD": Rune(0x02145) + of "DifferentialD", "dd": Rune(0x02146) + of "ExponentialE", "exponentiale", "ee": Rune(0x02147) + of "ImaginaryI", "ii": Rune(0x02148) + of "frac13": Rune(0x02153) + of "frac23": Rune(0x02154) + of "frac15": Rune(0x02155) + of "frac25": Rune(0x02156) + of "frac35": Rune(0x02157) + of "frac45": Rune(0x02158) + of "frac16": Rune(0x02159) + of "frac56": Rune(0x0215A) + of "frac18": Rune(0x0215B) + of "frac38": Rune(0x0215C) + of "frac58": Rune(0x0215D) + of "frac78": Rune(0x0215E) + of "larr", "leftarrow", "LeftArrow", "slarr", + "ShortLeftArrow": Rune(0x02190) + of "uarr", "uparrow", "UpArrow", "ShortUpArrow": Rune(0x02191) + of "rarr", "rightarrow", "RightArrow", "srarr", + "ShortRightArrow": Rune(0x02192) + of "darr", "downarrow", "DownArrow", + "ShortDownArrow": Rune(0x02193) + of "harr", "leftrightarrow", "LeftRightArrow": Rune(0x02194) + of "varr", "updownarrow", "UpDownArrow": Rune(0x02195) + of "nwarr", "UpperLeftArrow", "nwarrow": Rune(0x02196) + of "nearr", "UpperRightArrow", "nearrow": Rune(0x02197) + of "searr", "searrow", "LowerRightArrow": Rune(0x02198) + of "swarr", "swarrow", "LowerLeftArrow": Rune(0x02199) + of "nlarr", "nleftarrow": Rune(0x0219A) + of "nrarr", "nrightarrow": Rune(0x0219B) + of "rarrw", "rightsquigarrow": Rune(0x0219D) + of "Larr", "twoheadleftarrow": Rune(0x0219E) + of "Uarr": Rune(0x0219F) + of "Rarr", "twoheadrightarrow": Rune(0x021A0) + of "Darr": Rune(0x021A1) + of "larrtl", "leftarrowtail": Rune(0x021A2) + of "rarrtl", "rightarrowtail": Rune(0x021A3) + of "LeftTeeArrow", "mapstoleft": Rune(0x021A4) + of "UpTeeArrow", "mapstoup": Rune(0x021A5) + of "map", "RightTeeArrow", "mapsto": Rune(0x021A6) + of "DownTeeArrow", "mapstodown": Rune(0x021A7) + of "larrhk", "hookleftarrow": Rune(0x021A9) + of "rarrhk", "hookrightarrow": Rune(0x021AA) + of "larrlp", "looparrowleft": Rune(0x021AB) + of "rarrlp", "looparrowright": Rune(0x021AC) + of "harrw", "leftrightsquigarrow": Rune(0x021AD) + of "nharr", "nleftrightarrow": Rune(0x021AE) + of "lsh", "Lsh": Rune(0x021B0) + of "rsh", "Rsh": Rune(0x021B1) + of "ldsh": Rune(0x021B2) + of "rdsh": Rune(0x021B3) + of "crarr": Rune(0x021B5) + of "cularr", "curvearrowleft": Rune(0x021B6) + of "curarr", "curvearrowright": Rune(0x021B7) + of "olarr", "circlearrowleft": Rune(0x021BA) + of "orarr", "circlearrowright": Rune(0x021BB) + of "lharu", "LeftVector", "leftharpoonup": Rune(0x021BC) + of "lhard", "leftharpoondown", "DownLeftVector": Rune(0x021BD) + of "uharr", "upharpoonright", "RightUpVector": Rune(0x021BE) + of "uharl", "upharpoonleft", "LeftUpVector": Rune(0x021BF) + of "rharu", "RightVector", "rightharpoonup": Rune(0x021C0) + of "rhard", "rightharpoondown", "DownRightVector": Rune(0x021C1) + of "dharr", "RightDownVector", "downharpoonright": Rune(0x021C2) + of "dharl", "LeftDownVector", "downharpoonleft": Rune(0x021C3) + of "rlarr", "rightleftarrows", "RightArrowLeftArrow": Rune(0x021C4) + of "udarr", "UpArrowDownArrow": Rune(0x021C5) + of "lrarr", "leftrightarrows", "LeftArrowRightArrow": Rune(0x021C6) + of "llarr", "leftleftarrows": Rune(0x021C7) + of "uuarr", "upuparrows": Rune(0x021C8) + of "rrarr", "rightrightarrows": Rune(0x021C9) + of "ddarr", "downdownarrows": Rune(0x021CA) + of "lrhar", "ReverseEquilibrium", + "leftrightharpoons": Rune(0x021CB) + of "rlhar", "rightleftharpoons", "Equilibrium": Rune(0x021CC) + of "nlArr", "nLeftarrow": Rune(0x021CD) + of "nhArr", "nLeftrightarrow": Rune(0x021CE) + of "nrArr", "nRightarrow": Rune(0x021CF) + of "lArr", "Leftarrow", "DoubleLeftArrow": Rune(0x021D0) + of "uArr", "Uparrow", "DoubleUpArrow": Rune(0x021D1) + of "rArr", "Rightarrow", "Implies", + "DoubleRightArrow": Rune(0x021D2) + of "dArr", "Downarrow", "DoubleDownArrow": Rune(0x021D3) + of "hArr", "Leftrightarrow", "DoubleLeftRightArrow", + "iff": Rune(0x021D4) + of "vArr", "Updownarrow", "DoubleUpDownArrow": Rune(0x021D5) + of "nwArr": Rune(0x021D6) + of "neArr": Rune(0x021D7) + of "seArr": Rune(0x021D8) + of "swArr": Rune(0x021D9) + of "lAarr", "Lleftarrow": Rune(0x021DA) + of "rAarr", "Rrightarrow": Rune(0x021DB) + of "zigrarr": Rune(0x021DD) + of "larrb", "LeftArrowBar": Rune(0x021E4) + of "rarrb", "RightArrowBar": Rune(0x021E5) + of "duarr", "DownArrowUpArrow": Rune(0x021F5) + of "loarr": Rune(0x021FD) + of "roarr": Rune(0x021FE) + of "hoarr": Rune(0x021FF) + of "forall", "ForAll": Rune(0x02200) + of "comp", "complement": Rune(0x02201) + of "part", "PartialD": Rune(0x02202) + of "exist", "Exists": Rune(0x02203) + of "nexist", "NotExists", "nexists": Rune(0x02204) + of "empty", "emptyset", "emptyv", "varnothing": Rune(0x02205) + of "nabla", "Del": Rune(0x02207) + of "isin", "isinv", "Element", "in": Rune(0x02208) + of "notin", "NotElement", "notinva": Rune(0x02209) + of "niv", "ReverseElement", "ni", "SuchThat": Rune(0x0220B) + of "notni", "notniva", "NotReverseElement": Rune(0x0220C) + of "prod", "Product": Rune(0x0220F) + of "coprod", "Coproduct": Rune(0x02210) + of "sum", "Sum": Rune(0x02211) + of "minus": Rune(0x02212) + of "mnplus", "mp", "MinusPlus": Rune(0x02213) + of "plusdo", "dotplus": Rune(0x02214) + of "setmn", "setminus", "Backslash", "ssetmn", + "smallsetminus": Rune(0x02216) + of "lowast": Rune(0x02217) + of "compfn", "SmallCircle": Rune(0x02218) + of "radic", "Sqrt": Rune(0x0221A) + of "prop", "propto", "Proportional", "vprop", + "varpropto": Rune(0x0221D) + of "infin": Rune(0x0221E) + of "angrt": Rune(0x0221F) + of "ang", "angle": Rune(0x02220) + of "angmsd", "measuredangle": Rune(0x02221) + of "angsph": Rune(0x02222) + of "mid", "VerticalBar", "smid", "shortmid": Rune(0x02223) + of "nmid", "NotVerticalBar", "nsmid", "nshortmid": Rune(0x02224) + of "par", "parallel", "DoubleVerticalBar", "spar", + "shortparallel": Rune(0x02225) + of "npar", "nparallel", "NotDoubleVerticalBar", "nspar", + "nshortparallel": Rune(0x02226) + of "and", "wedge": Rune(0x02227) + of "or", "vee": Rune(0x02228) + of "cap": Rune(0x02229) + of "cup": Rune(0x0222A) + of "int", "Integral": Rune(0x0222B) + of "Int": Rune(0x0222C) + of "tint", "iiint": Rune(0x0222D) + of "conint", "oint", "ContourIntegral": Rune(0x0222E) + of "Conint", "DoubleContourIntegral": Rune(0x0222F) + of "Cconint": Rune(0x02230) + of "cwint": Rune(0x02231) + of "cwconint", "ClockwiseContourIntegral": Rune(0x02232) + of "awconint", "CounterClockwiseContourIntegral": Rune(0x02233) + of "there4", "therefore", "Therefore": Rune(0x02234) + of "becaus", "because", "Because": Rune(0x02235) + of "ratio": Rune(0x02236) + of "Colon", "Proportion": Rune(0x02237) + of "minusd", "dotminus": Rune(0x02238) + of "mDDot": Rune(0x0223A) + of "homtht": Rune(0x0223B) + of "sim", "Tilde", "thksim", "thicksim": Rune(0x0223C) + of "bsim", "backsim": Rune(0x0223D) + of "ac", "mstpos": Rune(0x0223E) + of "acd": Rune(0x0223F) + of "wreath", "VerticalTilde", "wr": Rune(0x02240) + of "nsim", "NotTilde": Rune(0x02241) + of "esim", "EqualTilde", "eqsim": Rune(0x02242) + of "sime", "TildeEqual", "simeq": Rune(0x02243) + of "nsime", "nsimeq", "NotTildeEqual": Rune(0x02244) + of "cong", "TildeFullEqual": Rune(0x02245) + of "simne": Rune(0x02246) + of "ncong", "NotTildeFullEqual": Rune(0x02247) + of "asymp", "ap", "TildeTilde", "approx", "thkap", + "thickapprox": Rune(0x02248) + of "nap", "NotTildeTilde", "napprox": Rune(0x02249) + of "ape", "approxeq": Rune(0x0224A) + of "apid": Rune(0x0224B) + of "bcong", "backcong": Rune(0x0224C) + of "asympeq", "CupCap": Rune(0x0224D) + of "bump", "HumpDownHump", "Bumpeq": Rune(0x0224E) + of "bumpe", "HumpEqual", "bumpeq": Rune(0x0224F) + of "esdot", "DotEqual", "doteq": Rune(0x02250) + of "eDot", "doteqdot": Rune(0x02251) + of "efDot", "fallingdotseq": Rune(0x02252) + of "erDot", "risingdotseq": Rune(0x02253) + of "colone", "coloneq", "Assign": Rune(0x02254) + of "ecolon", "eqcolon": Rune(0x02255) + of "ecir", "eqcirc": Rune(0x02256) + of "cire", "circeq": Rune(0x02257) + of "wedgeq": Rune(0x02259) + of "veeeq": Rune(0x0225A) + of "trie", "triangleq": Rune(0x0225C) + of "equest", "questeq": Rune(0x0225F) + of "ne", "NotEqual": Rune(0x02260) + of "equiv", "Congruent": Rune(0x02261) + of "nequiv", "NotCongruent": Rune(0x02262) + of "le", "leq": Rune(0x02264) + of "ge", "GreaterEqual", "geq": Rune(0x02265) + of "lE", "LessFullEqual", "leqq": Rune(0x02266) + of "gE", "GreaterFullEqual", "geqq": Rune(0x02267) + of "lnE", "lneqq": Rune(0x02268) + of "gnE", "gneqq": Rune(0x02269) + of "Lt", "NestedLessLess", "ll": Rune(0x0226A) + of "Gt", "NestedGreaterGreater", "gg": Rune(0x0226B) + of "twixt", "between": Rune(0x0226C) + of "NotCupCap": Rune(0x0226D) + of "nlt", "NotLess", "nless": Rune(0x0226E) + of "ngt", "NotGreater", "ngtr": Rune(0x0226F) + of "nle", "NotLessEqual", "nleq": Rune(0x02270) + of "nge", "NotGreaterEqual", "ngeq": Rune(0x02271) + of "lsim", "LessTilde", "lesssim": Rune(0x02272) + of "gsim", "gtrsim", "GreaterTilde": Rune(0x02273) + of "nlsim", "NotLessTilde": Rune(0x02274) + of "ngsim", "NotGreaterTilde": Rune(0x02275) + of "lg", "lessgtr", "LessGreater": Rune(0x02276) + of "gl", "gtrless", "GreaterLess": Rune(0x02277) + of "ntlg", "NotLessGreater": Rune(0x02278) + of "ntgl", "NotGreaterLess": Rune(0x02279) + of "pr", "Precedes", "prec": Rune(0x0227A) + of "sc", "Succeeds", "succ": Rune(0x0227B) + of "prcue", "PrecedesSlantEqual", "preccurlyeq": Rune(0x0227C) + of "sccue", "SucceedsSlantEqual", "succcurlyeq": Rune(0x0227D) + of "prsim", "precsim", "PrecedesTilde": Rune(0x0227E) + of "scsim", "succsim", "SucceedsTilde": Rune(0x0227F) + of "npr", "nprec", "NotPrecedes": Rune(0x02280) + of "nsc", "nsucc", "NotSucceeds": Rune(0x02281) + of "sub", "subset": Rune(0x02282) + of "sup", "supset", "Superset": Rune(0x02283) + of "nsub": Rune(0x02284) + of "nsup": Rune(0x02285) + of "sube", "SubsetEqual", "subseteq": Rune(0x02286) + of "supe", "supseteq", "SupersetEqual": Rune(0x02287) + of "nsube", "nsubseteq", "NotSubsetEqual": Rune(0x02288) + of "nsupe", "nsupseteq", "NotSupersetEqual": Rune(0x02289) + of "subne", "subsetneq": Rune(0x0228A) + of "supne", "supsetneq": Rune(0x0228B) + of "cupdot": Rune(0x0228D) + of "uplus", "UnionPlus": Rune(0x0228E) + of "sqsub", "SquareSubset", "sqsubset": Rune(0x0228F) + of "sqsup", "SquareSuperset", "sqsupset": Rune(0x02290) + of "sqsube", "SquareSubsetEqual", "sqsubseteq": Rune(0x02291) + of "sqsupe", "SquareSupersetEqual", "sqsupseteq": Rune(0x02292) + of "sqcap", "SquareIntersection": Rune(0x02293) + of "sqcup", "SquareUnion": Rune(0x02294) + of "oplus", "CirclePlus": Rune(0x02295) + of "ominus", "CircleMinus": Rune(0x02296) + of "otimes", "CircleTimes": Rune(0x02297) + of "osol": Rune(0x02298) + of "odot", "CircleDot": Rune(0x02299) + of "ocir", "circledcirc": Rune(0x0229A) + of "oast", "circledast": Rune(0x0229B) + of "odash", "circleddash": Rune(0x0229D) + of "plusb", "boxplus": Rune(0x0229E) + of "minusb", "boxminus": Rune(0x0229F) + of "timesb", "boxtimes": Rune(0x022A0) + of "sdotb", "dotsquare": Rune(0x022A1) + of "vdash", "RightTee": Rune(0x022A2) + of "dashv", "LeftTee": Rune(0x022A3) + of "top", "DownTee": Rune(0x022A4) + of "bottom", "bot", "perp", "UpTee": Rune(0x022A5) + of "models": Rune(0x022A7) + of "vDash", "DoubleRightTee": Rune(0x022A8) + of "Vdash": Rune(0x022A9) + of "Vvdash": Rune(0x022AA) + of "VDash": Rune(0x022AB) + of "nvdash": Rune(0x022AC) + of "nvDash": Rune(0x022AD) + of "nVdash": Rune(0x022AE) + of "nVDash": Rune(0x022AF) + of "prurel": Rune(0x022B0) + of "vltri", "vartriangleleft", "LeftTriangle": Rune(0x022B2) + of "vrtri", "vartriangleright", "RightTriangle": Rune(0x022B3) + of "ltrie", "trianglelefteq", "LeftTriangleEqual": Rune(0x022B4) + of "rtrie", "trianglerighteq", "RightTriangleEqual": Rune(0x022B5) + of "origof": Rune(0x022B6) + of "imof": Rune(0x022B7) + of "mumap", "multimap": Rune(0x022B8) + of "hercon": Rune(0x022B9) + of "intcal", "intercal": Rune(0x022BA) + of "veebar": Rune(0x022BB) + of "barvee": Rune(0x022BD) + of "angrtvb": Rune(0x022BE) + of "lrtri": Rune(0x022BF) + of "xwedge", "Wedge", "bigwedge": Rune(0x022C0) + of "xvee", "Vee", "bigvee": Rune(0x022C1) + of "xcap", "Intersection", "bigcap": Rune(0x022C2) + of "xcup", "Union", "bigcup": Rune(0x022C3) + of "diam", "diamond", "Diamond": Rune(0x022C4) + of "sdot": Rune(0x022C5) + of "sstarf", "Star": Rune(0x022C6) + of "divonx", "divideontimes": Rune(0x022C7) + of "bowtie": Rune(0x022C8) + of "ltimes": Rune(0x022C9) + of "rtimes": Rune(0x022CA) + of "lthree", "leftthreetimes": Rune(0x022CB) + of "rthree", "rightthreetimes": Rune(0x022CC) + of "bsime", "backsimeq": Rune(0x022CD) + of "cuvee", "curlyvee": Rune(0x022CE) + of "cuwed", "curlywedge": Rune(0x022CF) + of "Sub", "Subset": Rune(0x022D0) + of "Sup", "Supset": Rune(0x022D1) + of "Cap": Rune(0x022D2) + of "Cup": Rune(0x022D3) + of "fork", "pitchfork": Rune(0x022D4) + of "epar": Rune(0x022D5) + of "ltdot", "lessdot": Rune(0x022D6) + of "gtdot", "gtrdot": Rune(0x022D7) + of "Ll": Rune(0x022D8) + of "Gg", "ggg": Rune(0x022D9) + of "leg", "LessEqualGreater", "lesseqgtr": Rune(0x022DA) + of "gel", "gtreqless", "GreaterEqualLess": Rune(0x022DB) + of "cuepr", "curlyeqprec": Rune(0x022DE) + of "cuesc", "curlyeqsucc": Rune(0x022DF) + of "nprcue", "NotPrecedesSlantEqual": Rune(0x022E0) + of "nsccue", "NotSucceedsSlantEqual": Rune(0x022E1) + of "nsqsube", "NotSquareSubsetEqual": Rune(0x022E2) + of "nsqsupe", "NotSquareSupersetEqual": Rune(0x022E3) + of "lnsim": Rune(0x022E6) + of "gnsim": Rune(0x022E7) + of "prnsim", "precnsim": Rune(0x022E8) + of "scnsim", "succnsim": Rune(0x022E9) + of "nltri", "ntriangleleft", "NotLeftTriangle": Rune(0x022EA) + of "nrtri", "ntriangleright", "NotRightTriangle": Rune(0x022EB) + of "nltrie", "ntrianglelefteq", + "NotLeftTriangleEqual": Rune(0x022EC) + of "nrtrie", "ntrianglerighteq", + "NotRightTriangleEqual": Rune(0x022ED) + of "vellip": Rune(0x022EE) + of "ctdot": Rune(0x022EF) + of "utdot": Rune(0x022F0) + of "dtdot": Rune(0x022F1) + of "disin": Rune(0x022F2) + of "isinsv": Rune(0x022F3) + of "isins": Rune(0x022F4) + of "isindot": Rune(0x022F5) + of "notinvc": Rune(0x022F6) + of "notinvb": Rune(0x022F7) + of "isinE": Rune(0x022F9) + of "nisd": Rune(0x022FA) + of "xnis": Rune(0x022FB) + of "nis": Rune(0x022FC) + of "notnivc": Rune(0x022FD) + of "notnivb": Rune(0x022FE) + of "barwed", "barwedge": Rune(0x02305) + of "Barwed", "doublebarwedge": Rune(0x02306) + of "lceil", "LeftCeiling": Rune(0x02308) + of "rceil", "RightCeiling": Rune(0x02309) + of "lfloor", "LeftFloor": Rune(0x0230A) + of "rfloor", "RightFloor": Rune(0x0230B) + of "drcrop": Rune(0x0230C) + of "dlcrop": Rune(0x0230D) + of "urcrop": Rune(0x0230E) + of "ulcrop": Rune(0x0230F) + of "bnot": Rune(0x02310) + of "profline": Rune(0x02312) + of "profsurf": Rune(0x02313) + of "telrec": Rune(0x02315) + of "target": Rune(0x02316) + of "ulcorn", "ulcorner": Rune(0x0231C) + of "urcorn", "urcorner": Rune(0x0231D) + of "dlcorn", "llcorner": Rune(0x0231E) + of "drcorn", "lrcorner": Rune(0x0231F) + of "frown", "sfrown": Rune(0x02322) + of "smile", "ssmile": Rune(0x02323) + of "cylcty": Rune(0x0232D) + of "profalar": Rune(0x0232E) + of "topbot": Rune(0x02336) + of "ovbar": Rune(0x0233D) + of "solbar": Rune(0x0233F) + of "angzarr": Rune(0x0237C) + of "lmoust", "lmoustache": Rune(0x023B0) + of "rmoust", "rmoustache": Rune(0x023B1) + of "tbrk", "OverBracket": Rune(0x023B4) + of "bbrk", "UnderBracket": Rune(0x023B5) + of "bbrktbrk": Rune(0x023B6) + of "OverParenthesis": Rune(0x023DC) + of "UnderParenthesis": Rune(0x023DD) + of "OverBrace": Rune(0x023DE) + of "UnderBrace": Rune(0x023DF) + of "trpezium": Rune(0x023E2) + of "elinters": Rune(0x023E7) + of "blank": Rune(0x02423) + of "oS", "circledS": Rune(0x024C8) + of "boxh", "HorizontalLine": Rune(0x02500) + of "boxv": Rune(0x02502) + of "boxdr": Rune(0x0250C) + of "boxdl": Rune(0x02510) + of "boxur": Rune(0x02514) + of "boxul": Rune(0x02518) + of "boxvr": Rune(0x0251C) + of "boxvl": Rune(0x02524) + of "boxhd": Rune(0x0252C) + of "boxhu": Rune(0x02534) + of "boxvh": Rune(0x0253C) + of "boxH": Rune(0x02550) + of "boxV": Rune(0x02551) + of "boxdR": Rune(0x02552) + of "boxDr": Rune(0x02553) + of "boxDR": Rune(0x02554) + of "boxdL": Rune(0x02555) + of "boxDl": Rune(0x02556) + of "boxDL": Rune(0x02557) + of "boxuR": Rune(0x02558) + of "boxUr": Rune(0x02559) + of "boxUR": Rune(0x0255A) + of "boxuL": Rune(0x0255B) + of "boxUl": Rune(0x0255C) + of "boxUL": Rune(0x0255D) + of "boxvR": Rune(0x0255E) + of "boxVr": Rune(0x0255F) + of "boxVR": Rune(0x02560) + of "boxvL": Rune(0x02561) + of "boxVl": Rune(0x02562) + of "boxVL": Rune(0x02563) + of "boxHd": Rune(0x02564) + of "boxhD": Rune(0x02565) + of "boxHD": Rune(0x02566) + of "boxHu": Rune(0x02567) + of "boxhU": Rune(0x02568) + of "boxHU": Rune(0x02569) + of "boxvH": Rune(0x0256A) + of "boxVh": Rune(0x0256B) + of "boxVH": Rune(0x0256C) + of "uhblk": Rune(0x02580) + of "lhblk": Rune(0x02584) + of "block": Rune(0x02588) + of "blk14": Rune(0x02591) + of "blk12": Rune(0x02592) + of "blk34": Rune(0x02593) + of "squ", "square", "Square": Rune(0x025A1) + of "squf", "squarf", "blacksquare", + "FilledVerySmallSquare": Rune(0x025AA) + of "EmptyVerySmallSquare": Rune(0x025AB) + of "rect": Rune(0x025AD) + of "marker": Rune(0x025AE) + of "fltns": Rune(0x025B1) + of "xutri", "bigtriangleup": Rune(0x025B3) + of "utrif", "blacktriangle": Rune(0x025B4) + of "utri", "triangle": Rune(0x025B5) + of "rtrif", "blacktriangleright": Rune(0x025B8) + of "rtri", "triangleright": Rune(0x025B9) + of "xdtri", "bigtriangledown": Rune(0x025BD) + of "dtrif", "blacktriangledown": Rune(0x025BE) + of "dtri", "triangledown": Rune(0x025BF) + of "ltrif", "blacktriangleleft": Rune(0x025C2) + of "ltri", "triangleleft": Rune(0x025C3) + of "loz", "lozenge": Rune(0x025CA) + of "cir": Rune(0x025CB) + of "tridot": Rune(0x025EC) + of "xcirc", "bigcirc": Rune(0x025EF) + of "ultri": Rune(0x025F8) + of "urtri": Rune(0x025F9) + of "lltri": Rune(0x025FA) + of "EmptySmallSquare": Rune(0x025FB) + of "FilledSmallSquare": Rune(0x025FC) + of "starf", "bigstar": Rune(0x02605) + of "star": Rune(0x02606) + of "phone": Rune(0x0260E) + of "female": Rune(0x02640) + of "male": Rune(0x02642) + of "spades", "spadesuit": Rune(0x02660) + of "clubs", "clubsuit": Rune(0x02663) + of "hearts", "heartsuit": Rune(0x02665) + of "diams", "diamondsuit": Rune(0x02666) + of "sung": Rune(0x0266A) + of "flat": Rune(0x0266D) + of "natur", "natural": Rune(0x0266E) + of "sharp": Rune(0x0266F) + of "check", "checkmark": Rune(0x02713) + of "cross": Rune(0x02717) + of "malt", "maltese": Rune(0x02720) + of "sext": Rune(0x02736) + of "VerticalSeparator": Rune(0x02758) + of "lbbrk": Rune(0x02772) + of "rbbrk": Rune(0x02773) + of "lobrk", "LeftDoubleBracket": Rune(0x027E6) + of "robrk", "RightDoubleBracket": Rune(0x027E7) + of "lang", "LeftAngleBracket", "langle": Rune(0x027E8) + of "rang", "RightAngleBracket", "rangle": Rune(0x027E9) + of "Lang": Rune(0x027EA) + of "Rang": Rune(0x027EB) + of "loang": Rune(0x027EC) + of "roang": Rune(0x027ED) + of "xlarr", "longleftarrow", "LongLeftArrow": Rune(0x027F5) + of "xrarr", "longrightarrow", "LongRightArrow": Rune(0x027F6) + of "xharr", "longleftrightarrow", + "LongLeftRightArrow": Rune(0x027F7) + of "xlArr", "Longleftarrow", "DoubleLongLeftArrow": Rune(0x027F8) + of "xrArr", "Longrightarrow", "DoubleLongRightArrow": Rune(0x027F9) + of "xhArr", "Longleftrightarrow", + "DoubleLongLeftRightArrow": Rune(0x027FA) + of "xmap", "longmapsto": Rune(0x027FC) + of "dzigrarr": Rune(0x027FF) + of "nvlArr": Rune(0x02902) + of "nvrArr": Rune(0x02903) + of "nvHarr": Rune(0x02904) + of "Map": Rune(0x02905) + of "lbarr": Rune(0x0290C) + of "rbarr", "bkarow": Rune(0x0290D) + of "lBarr": Rune(0x0290E) + of "rBarr", "dbkarow": Rune(0x0290F) + of "RBarr", "drbkarow": Rune(0x02910) + of "DDotrahd": Rune(0x02911) + of "UpArrowBar": Rune(0x02912) + of "DownArrowBar": Rune(0x02913) + of "Rarrtl": Rune(0x02916) + of "latail": Rune(0x02919) + of "ratail": Rune(0x0291A) + of "lAtail": Rune(0x0291B) + of "rAtail": Rune(0x0291C) + of "larrfs": Rune(0x0291D) + of "rarrfs": Rune(0x0291E) + of "larrbfs": Rune(0x0291F) + of "rarrbfs": Rune(0x02920) + of "nwarhk": Rune(0x02923) + of "nearhk": Rune(0x02924) + of "searhk", "hksearow": Rune(0x02925) + of "swarhk", "hkswarow": Rune(0x02926) + of "nwnear": Rune(0x02927) + of "nesear", "toea": Rune(0x02928) + of "seswar", "tosa": Rune(0x02929) + of "swnwar": Rune(0x0292A) + of "rarrc": Rune(0x02933) + of "cudarrr": Rune(0x02935) + of "ldca": Rune(0x02936) + of "rdca": Rune(0x02937) + of "cudarrl": Rune(0x02938) + of "larrpl": Rune(0x02939) + of "curarrm": Rune(0x0293C) + of "cularrp": Rune(0x0293D) + of "rarrpl": Rune(0x02945) + of "harrcir": Rune(0x02948) + of "Uarrocir": Rune(0x02949) + of "lurdshar": Rune(0x0294A) + of "ldrushar": Rune(0x0294B) + of "LeftRightVector": Rune(0x0294E) + of "RightUpDownVector": Rune(0x0294F) + of "DownLeftRightVector": Rune(0x02950) + of "LeftUpDownVector": Rune(0x02951) + of "LeftVectorBar": Rune(0x02952) + of "RightVectorBar": Rune(0x02953) + of "RightUpVectorBar": Rune(0x02954) + of "RightDownVectorBar": Rune(0x02955) + of "DownLeftVectorBar": Rune(0x02956) + of "DownRightVectorBar": Rune(0x02957) + of "LeftUpVectorBar": Rune(0x02958) + of "LeftDownVectorBar": Rune(0x02959) + of "LeftTeeVector": Rune(0x0295A) + of "RightTeeVector": Rune(0x0295B) + of "RightUpTeeVector": Rune(0x0295C) + of "RightDownTeeVector": Rune(0x0295D) + of "DownLeftTeeVector": Rune(0x0295E) + of "DownRightTeeVector": Rune(0x0295F) + of "LeftUpTeeVector": Rune(0x02960) + of "LeftDownTeeVector": Rune(0x02961) + of "lHar": Rune(0x02962) + of "uHar": Rune(0x02963) + of "rHar": Rune(0x02964) + of "dHar": Rune(0x02965) + of "luruhar": Rune(0x02966) + of "ldrdhar": Rune(0x02967) + of "ruluhar": Rune(0x02968) + of "rdldhar": Rune(0x02969) + of "lharul": Rune(0x0296A) + of "llhard": Rune(0x0296B) + of "rharul": Rune(0x0296C) + of "lrhard": Rune(0x0296D) + of "udhar", "UpEquilibrium": Rune(0x0296E) + of "duhar", "ReverseUpEquilibrium": Rune(0x0296F) + of "RoundImplies": Rune(0x02970) + of "erarr": Rune(0x02971) + of "simrarr": Rune(0x02972) + of "larrsim": Rune(0x02973) + of "rarrsim": Rune(0x02974) + of "rarrap": Rune(0x02975) + of "ltlarr": Rune(0x02976) + of "gtrarr": Rune(0x02978) + of "subrarr": Rune(0x02979) + of "suplarr": Rune(0x0297B) + of "lfisht": Rune(0x0297C) + of "rfisht": Rune(0x0297D) + of "ufisht": Rune(0x0297E) + of "dfisht": Rune(0x0297F) + of "lopar": Rune(0x02985) + of "ropar": Rune(0x02986) + of "lbrke": Rune(0x0298B) + of "rbrke": Rune(0x0298C) + of "lbrkslu": Rune(0x0298D) + of "rbrksld": Rune(0x0298E) + of "lbrksld": Rune(0x0298F) + of "rbrkslu": Rune(0x02990) + of "langd": Rune(0x02991) + of "rangd": Rune(0x02992) + of "lparlt": Rune(0x02993) + of "rpargt": Rune(0x02994) + of "gtlPar": Rune(0x02995) + of "ltrPar": Rune(0x02996) + of "vzigzag": Rune(0x0299A) + of "vangrt": Rune(0x0299C) + of "angrtvbd": Rune(0x0299D) + of "ange": Rune(0x029A4) + of "range": Rune(0x029A5) + of "dwangle": Rune(0x029A6) + of "uwangle": Rune(0x029A7) + of "angmsdaa": Rune(0x029A8) + of "angmsdab": Rune(0x029A9) + of "angmsdac": Rune(0x029AA) + of "angmsdad": Rune(0x029AB) + of "angmsdae": Rune(0x029AC) + of "angmsdaf": Rune(0x029AD) + of "angmsdag": Rune(0x029AE) + of "angmsdah": Rune(0x029AF) + of "bemptyv": Rune(0x029B0) + of "demptyv": Rune(0x029B1) + of "cemptyv": Rune(0x029B2) + of "raemptyv": Rune(0x029B3) + of "laemptyv": Rune(0x029B4) + of "ohbar": Rune(0x029B5) + of "omid": Rune(0x029B6) + of "opar": Rune(0x029B7) + of "operp": Rune(0x029B9) + of "olcross": Rune(0x029BB) + of "odsold": Rune(0x029BC) + of "olcir": Rune(0x029BE) + of "ofcir": Rune(0x029BF) + of "olt": Rune(0x029C0) + of "ogt": Rune(0x029C1) + of "cirscir": Rune(0x029C2) + of "cirE": Rune(0x029C3) + of "solb": Rune(0x029C4) + of "bsolb": Rune(0x029C5) + of "boxbox": Rune(0x029C9) + of "trisb": Rune(0x029CD) + of "rtriltri": Rune(0x029CE) + of "LeftTriangleBar": Rune(0x029CF) + of "RightTriangleBar": Rune(0x029D0) + of "race": Rune(0x029DA) + of "iinfin": Rune(0x029DC) + of "infintie": Rune(0x029DD) + of "nvinfin": Rune(0x029DE) + of "eparsl": Rune(0x029E3) + of "smeparsl": Rune(0x029E4) + of "eqvparsl": Rune(0x029E5) + of "lozf", "blacklozenge": Rune(0x029EB) + of "RuleDelayed": Rune(0x029F4) + of "dsol": Rune(0x029F6) + of "xodot", "bigodot": Rune(0x02A00) + of "xoplus", "bigoplus": Rune(0x02A01) + of "xotime", "bigotimes": Rune(0x02A02) + of "xuplus", "biguplus": Rune(0x02A04) + of "xsqcup", "bigsqcup": Rune(0x02A06) + of "qint", "iiiint": Rune(0x02A0C) + of "fpartint": Rune(0x02A0D) + of "cirfnint": Rune(0x02A10) + of "awint": Rune(0x02A11) + of "rppolint": Rune(0x02A12) + of "scpolint": Rune(0x02A13) + of "npolint": Rune(0x02A14) + of "pointint": Rune(0x02A15) + of "quatint": Rune(0x02A16) + of "intlarhk": Rune(0x02A17) + of "pluscir": Rune(0x02A22) + of "plusacir": Rune(0x02A23) + of "simplus": Rune(0x02A24) + of "plusdu": Rune(0x02A25) + of "plussim": Rune(0x02A26) + of "plustwo": Rune(0x02A27) + of "mcomma": Rune(0x02A29) + of "minusdu": Rune(0x02A2A) + of "loplus": Rune(0x02A2D) + of "roplus": Rune(0x02A2E) + of "Cross": Rune(0x02A2F) + of "timesd": Rune(0x02A30) + of "timesbar": Rune(0x02A31) + of "smashp": Rune(0x02A33) + of "lotimes": Rune(0x02A34) + of "rotimes": Rune(0x02A35) + of "otimesas": Rune(0x02A36) + of "Otimes": Rune(0x02A37) + of "odiv": Rune(0x02A38) + of "triplus": Rune(0x02A39) + of "triminus": Rune(0x02A3A) + of "tritime": Rune(0x02A3B) + of "iprod", "intprod": Rune(0x02A3C) + of "amalg": Rune(0x02A3F) + of "capdot": Rune(0x02A40) + of "ncup": Rune(0x02A42) + of "ncap": Rune(0x02A43) + of "capand": Rune(0x02A44) + of "cupor": Rune(0x02A45) + of "cupcap": Rune(0x02A46) + of "capcup": Rune(0x02A47) + of "cupbrcap": Rune(0x02A48) + of "capbrcup": Rune(0x02A49) + of "cupcup": Rune(0x02A4A) + of "capcap": Rune(0x02A4B) + of "ccups": Rune(0x02A4C) + of "ccaps": Rune(0x02A4D) + of "ccupssm": Rune(0x02A50) + of "And": Rune(0x02A53) + of "Or": Rune(0x02A54) + of "andand": Rune(0x02A55) + of "oror": Rune(0x02A56) + of "orslope": Rune(0x02A57) + of "andslope": Rune(0x02A58) + of "andv": Rune(0x02A5A) + of "orv": Rune(0x02A5B) + of "andd": Rune(0x02A5C) + of "ord": Rune(0x02A5D) + of "wedbar": Rune(0x02A5F) + of "sdote": Rune(0x02A66) + of "simdot": Rune(0x02A6A) + of "congdot": Rune(0x02A6D) + of "easter": Rune(0x02A6E) + of "apacir": Rune(0x02A6F) + of "apE": Rune(0x02A70) + of "eplus": Rune(0x02A71) + of "pluse": Rune(0x02A72) + of "Esim": Rune(0x02A73) + of "Colone": Rune(0x02A74) + of "Equal": Rune(0x02A75) + of "eDDot", "ddotseq": Rune(0x02A77) + of "equivDD": Rune(0x02A78) + of "ltcir": Rune(0x02A79) + of "gtcir": Rune(0x02A7A) + of "ltquest": Rune(0x02A7B) + of "gtquest": Rune(0x02A7C) + of "les", "LessSlantEqual", "leqslant": Rune(0x02A7D) + of "ges", "GreaterSlantEqual", "geqslant": Rune(0x02A7E) + of "lesdot": Rune(0x02A7F) + of "gesdot": Rune(0x02A80) + of "lesdoto": Rune(0x02A81) + of "gesdoto": Rune(0x02A82) + of "lesdotor": Rune(0x02A83) + of "gesdotol": Rune(0x02A84) + of "lap", "lessapprox": Rune(0x02A85) + of "gap", "gtrapprox": Rune(0x02A86) + of "lne", "lneq": Rune(0x02A87) + of "gne", "gneq": Rune(0x02A88) + of "lnap", "lnapprox": Rune(0x02A89) + of "gnap", "gnapprox": Rune(0x02A8A) + of "lEg", "lesseqqgtr": Rune(0x02A8B) + of "gEl", "gtreqqless": Rune(0x02A8C) + of "lsime": Rune(0x02A8D) + of "gsime": Rune(0x02A8E) + of "lsimg": Rune(0x02A8F) + of "gsiml": Rune(0x02A90) + of "lgE": Rune(0x02A91) + of "glE": Rune(0x02A92) + of "lesges": Rune(0x02A93) + of "gesles": Rune(0x02A94) + of "els", "eqslantless": Rune(0x02A95) + of "egs", "eqslantgtr": Rune(0x02A96) + of "elsdot": Rune(0x02A97) + of "egsdot": Rune(0x02A98) + of "el": Rune(0x02A99) + of "eg": Rune(0x02A9A) + of "siml": Rune(0x02A9D) + of "simg": Rune(0x02A9E) + of "simlE": Rune(0x02A9F) + of "simgE": Rune(0x02AA0) + of "LessLess": Rune(0x02AA1) + of "GreaterGreater": Rune(0x02AA2) + of "glj": Rune(0x02AA4) + of "gla": Rune(0x02AA5) + of "ltcc": Rune(0x02AA6) + of "gtcc": Rune(0x02AA7) + of "lescc": Rune(0x02AA8) + of "gescc": Rune(0x02AA9) + of "smt": Rune(0x02AAA) + of "lat": Rune(0x02AAB) + of "smte": Rune(0x02AAC) + of "late": Rune(0x02AAD) + of "bumpE": Rune(0x02AAE) + of "pre", "preceq", "PrecedesEqual": Rune(0x02AAF) + of "sce", "succeq", "SucceedsEqual": Rune(0x02AB0) + of "prE": Rune(0x02AB3) + of "scE": Rune(0x02AB4) + of "prnE", "precneqq": Rune(0x02AB5) + of "scnE", "succneqq": Rune(0x02AB6) + of "prap", "precapprox": Rune(0x02AB7) + of "scap", "succapprox": Rune(0x02AB8) + of "prnap", "precnapprox": Rune(0x02AB9) + of "scnap", "succnapprox": Rune(0x02ABA) + of "Pr": Rune(0x02ABB) + of "Sc": Rune(0x02ABC) + of "subdot": Rune(0x02ABD) + of "supdot": Rune(0x02ABE) + of "subplus": Rune(0x02ABF) + of "supplus": Rune(0x02AC0) + of "submult": Rune(0x02AC1) + of "supmult": Rune(0x02AC2) + of "subedot": Rune(0x02AC3) + of "supedot": Rune(0x02AC4) + of "subE", "subseteqq": Rune(0x02AC5) + of "supE", "supseteqq": Rune(0x02AC6) + of "subsim": Rune(0x02AC7) + of "supsim": Rune(0x02AC8) + of "subnE", "subsetneqq": Rune(0x02ACB) + of "supnE", "supsetneqq": Rune(0x02ACC) + of "csub": Rune(0x02ACF) + of "csup": Rune(0x02AD0) + of "csube": Rune(0x02AD1) + of "csupe": Rune(0x02AD2) + of "subsup": Rune(0x02AD3) + of "supsub": Rune(0x02AD4) + of "subsub": Rune(0x02AD5) + of "supsup": Rune(0x02AD6) + of "suphsub": Rune(0x02AD7) + of "supdsub": Rune(0x02AD8) + of "forkv": Rune(0x02AD9) + of "topfork": Rune(0x02ADA) + of "mlcp": Rune(0x02ADB) + of "Dashv", "DoubleLeftTee": Rune(0x02AE4) + of "Vdashl": Rune(0x02AE6) + of "Barv": Rune(0x02AE7) + of "vBar": Rune(0x02AE8) + of "vBarv": Rune(0x02AE9) + of "Vbar": Rune(0x02AEB) + of "Not": Rune(0x02AEC) + of "bNot": Rune(0x02AED) + of "rnmid": Rune(0x02AEE) + of "cirmid": Rune(0x02AEF) + of "midcir": Rune(0x02AF0) + of "topcir": Rune(0x02AF1) + of "nhpar": Rune(0x02AF2) + of "parsim": Rune(0x02AF3) + of "parsl": Rune(0x02AFD) + of "fflig": Rune(0x0FB00) + of "filig": Rune(0x0FB01) + of "fllig": Rune(0x0FB02) + of "ffilig": Rune(0x0FB03) + of "ffllig": Rune(0x0FB04) + of "Ascr": Rune(0x1D49C) + of "Cscr": Rune(0x1D49E) + of "Dscr": Rune(0x1D49F) + of "Gscr": Rune(0x1D4A2) + of "Jscr": Rune(0x1D4A5) + of "Kscr": Rune(0x1D4A6) + of "Nscr": Rune(0x1D4A9) + of "Oscr": Rune(0x1D4AA) + of "Pscr": Rune(0x1D4AB) + of "Qscr": Rune(0x1D4AC) + of "Sscr": Rune(0x1D4AE) + of "Tscr": Rune(0x1D4AF) + of "Uscr": Rune(0x1D4B0) + of "Vscr": Rune(0x1D4B1) + of "Wscr": Rune(0x1D4B2) + of "Xscr": Rune(0x1D4B3) + of "Yscr": Rune(0x1D4B4) + of "Zscr": Rune(0x1D4B5) + of "ascr": Rune(0x1D4B6) + of "bscr": Rune(0x1D4B7) + of "cscr": Rune(0x1D4B8) + of "dscr": Rune(0x1D4B9) + of "fscr": Rune(0x1D4BB) + of "hscr": Rune(0x1D4BD) + of "iscr": Rune(0x1D4BE) + of "jscr": Rune(0x1D4BF) + of "kscr": Rune(0x1D4C0) + of "lscr": Rune(0x1D4C1) + of "mscr": Rune(0x1D4C2) + of "nscr": Rune(0x1D4C3) + of "pscr": Rune(0x1D4C5) + of "qscr": Rune(0x1D4C6) + of "rscr": Rune(0x1D4C7) + of "sscr": Rune(0x1D4C8) + of "tscr": Rune(0x1D4C9) + of "uscr": Rune(0x1D4CA) + of "vscr": Rune(0x1D4CB) + of "wscr": Rune(0x1D4CC) + of "xscr": Rune(0x1D4CD) + of "yscr": Rune(0x1D4CE) + of "zscr": Rune(0x1D4CF) + of "Afr": Rune(0x1D504) + of "Bfr": Rune(0x1D505) + of "Dfr": Rune(0x1D507) + of "Efr": Rune(0x1D508) + of "Ffr": Rune(0x1D509) + of "Gfr": Rune(0x1D50A) + of "Jfr": Rune(0x1D50D) + of "Kfr": Rune(0x1D50E) + of "Lfr": Rune(0x1D50F) + of "Mfr": Rune(0x1D510) + of "Nfr": Rune(0x1D511) + of "Ofr": Rune(0x1D512) + of "Pfr": Rune(0x1D513) + of "Qfr": Rune(0x1D514) + of "Sfr": Rune(0x1D516) + of "Tfr": Rune(0x1D517) + of "Ufr": Rune(0x1D518) + of "Vfr": Rune(0x1D519) + of "Wfr": Rune(0x1D51A) + of "Xfr": Rune(0x1D51B) + of "Yfr": Rune(0x1D51C) + of "afr": Rune(0x1D51E) + of "bfr": Rune(0x1D51F) + of "cfr": Rune(0x1D520) + of "dfr": Rune(0x1D521) + of "efr": Rune(0x1D522) + of "ffr": Rune(0x1D523) + of "gfr": Rune(0x1D524) + of "hfr": Rune(0x1D525) + of "ifr": Rune(0x1D526) + of "jfr": Rune(0x1D527) + of "kfr": Rune(0x1D528) + of "lfr": Rune(0x1D529) + of "mfr": Rune(0x1D52A) + of "nfr": Rune(0x1D52B) + of "ofr": Rune(0x1D52C) + of "pfr": Rune(0x1D52D) + of "qfr": Rune(0x1D52E) + of "rfr": Rune(0x1D52F) + of "sfr": Rune(0x1D530) + of "tfr": Rune(0x1D531) + of "ufr": Rune(0x1D532) + of "vfr": Rune(0x1D533) + of "wfr": Rune(0x1D534) + of "xfr": Rune(0x1D535) + of "yfr": Rune(0x1D536) + of "zfr": Rune(0x1D537) + of "Aopf": Rune(0x1D538) + of "Bopf": Rune(0x1D539) + of "Dopf": Rune(0x1D53B) + of "Eopf": Rune(0x1D53C) + of "Fopf": Rune(0x1D53D) + of "Gopf": Rune(0x1D53E) + of "Iopf": Rune(0x1D540) + of "Jopf": Rune(0x1D541) + of "Kopf": Rune(0x1D542) + of "Lopf": Rune(0x1D543) + of "Mopf": Rune(0x1D544) + of "Oopf": Rune(0x1D546) + of "Sopf": Rune(0x1D54A) + of "Topf": Rune(0x1D54B) + of "Uopf": Rune(0x1D54C) + of "Vopf": Rune(0x1D54D) + of "Wopf": Rune(0x1D54E) + of "Xopf": Rune(0x1D54F) + of "Yopf": Rune(0x1D550) + of "aopf": Rune(0x1D552) + of "bopf": Rune(0x1D553) + of "copf": Rune(0x1D554) + of "dopf": Rune(0x1D555) + of "eopf": Rune(0x1D556) + of "fopf": Rune(0x1D557) + of "gopf": Rune(0x1D558) + of "hopf": Rune(0x1D559) + of "iopf": Rune(0x1D55A) + of "jopf": Rune(0x1D55B) + of "kopf": Rune(0x1D55C) + of "lopf": Rune(0x1D55D) + of "mopf": Rune(0x1D55E) + of "nopf": Rune(0x1D55F) + of "oopf": Rune(0x1D560) + of "popf": Rune(0x1D561) + of "qopf": Rune(0x1D562) + of "ropf": Rune(0x1D563) + of "sopf": Rune(0x1D564) + of "topf": Rune(0x1D565) + of "uopf": Rune(0x1D566) + of "vopf": Rune(0x1D567) + of "wopf": Rune(0x1D568) + of "xopf": Rune(0x1D569) + of "yopf": Rune(0x1D56A) + of "zopf": Rune(0x1D56B) + else: Rune(0) + proc entityToUtf8*(entity: string): string = - ## converts an HTML entity name like ``Ü`` to its UTF-8 equivalent. + ## Converts an HTML entity name like `Ü` or values like `Ü` + ## or `Ü` to its UTF-8 equivalent. ## "" is returned if the entity name is unknown. The HTML parser ## already converts entities to UTF-8. - for name, val in items(Entities): - if name == entity: return toUTF8(Rune(val)) - result = "" + runnableExamples: + const sigma = "Σ" + doAssert entityToUtf8("") == "" + doAssert entityToUtf8("a") == "" + doAssert entityToUtf8("gt") == ">" + doAssert entityToUtf8("Uuml") == "Ü" + doAssert entityToUtf8("quest") == "?" + doAssert entityToUtf8("#63") == "?" + doAssert entityToUtf8("Sigma") == sigma + doAssert entityToUtf8("#931") == sigma + doAssert entityToUtf8("#0931") == sigma + doAssert entityToUtf8("#x3A3") == sigma + doAssert entityToUtf8("#x03A3") == sigma + doAssert entityToUtf8("#x3a3") == sigma + doAssert entityToUtf8("#X3a3") == sigma + let rune = entityToRune(entity) + if rune.ord <= 0: result = "" + else: result = toUTF8(rune) proc addNode(father, son: XmlNode) = if son != nil: add(father, son) -proc parse(x: var XmlParser, errors: var seq[string]): XmlNode +proc parse(x: var XmlParser, errors: var seq[string]): XmlNode {.gcsafe.} proc expected(x: var XmlParser, n: XmlNode): string = result = errorMsg(x, "</" & n.tag & "> expected") -template elemName(x: expr): expr = rawData(x) +template elemName(x: untyped): untyped = rawData(x) + +template adderr(x: untyped) = + errors.add(x) proc untilElementEnd(x: var XmlParser, result: XmlNode, errors: var seq[string]) = - # we parsed e.g. ``<br>`` and don't really expect a ``</br>``: + # we parsed e.g. `<br>` and don't really expect a `</br>`: if result.htmlTag in SingleTags: if x.kind != xmlElementEnd or cmpIgnoreCase(x.elemName, result.tag) != 0: return @@ -464,36 +1916,46 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, case x.kind of xmlElementStart, xmlElementOpen: case result.htmlTag - of tagLi, tagP, tagDt, tagDd, tagInput, tagOption: - # some tags are common to have no ``</end>``, like ``<li>``: + of tagP, tagInput, tagOption: + # some tags are common to have no `</end>`, like `<li>` but + # allow `<p>` in `<dd>`, `<dt>` and `<li>` in next case if htmlTag(x.elemName) in {tagLi, tagP, tagDt, tagDd, tagInput, tagOption}: - errors.add(expected(x, result)) + adderr(expected(x, result)) + break + of tagDd, tagDt, tagLi: + if htmlTag(x.elemName) in {tagLi, tagDt, tagDd, tagInput, + tagOption}: + adderr(expected(x, result)) break of tagTd, tagTh: if htmlTag(x.elemName) in {tagTr, tagTd, tagTh, tagTfoot, tagThead}: - errors.add(expected(x, result)) + adderr(expected(x, result)) break of tagTr: if htmlTag(x.elemName) == tagTr: - errors.add(expected(x, result)) + adderr(expected(x, result)) break of tagOptgroup: if htmlTag(x.elemName) in {tagOption, tagOptgroup}: - errors.add(expected(x, result)) + adderr(expected(x, result)) break else: discard result.addNode(parse(x, errors)) of xmlElementEnd: - if cmpIgnoreCase(x.elemName, result.tag) == 0: - next(x) - else: + if cmpIgnoreCase(x.elemName, result.tag) != 0: #echo "5; expected: ", result.htmltag, " ", x.elemName - errors.add(expected(x, result)) - # do not skip it here! + adderr(expected(x, result)) + # this seems to do better match error corrections in browsers: + while x.kind in {xmlElementEnd, xmlWhitespace}: + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elemName, + result.tag) == 0: + break + next(x) + next(x) break of xmlEof: - errors.add(expected(x, result)) + adderr(expected(x, result)) break else: result.addNode(parse(x, errors)) @@ -510,16 +1972,16 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = # we just ignore processing instructions for now next(x) of xmlError: - errors.add(errorMsg(x)) + adderr(errorMsg(x)) next(x) of xmlElementStart: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) untilElementEnd(x, result, errors) of xmlElementEnd: - errors.add(errorMsg(x, "unexpected ending tag: " & x.elemName)) + adderr(errorMsg(x, "unexpected ending tag: " & x.elemName)) of xmlElementOpen: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) result.attrs = newStringTable() while true: @@ -531,16 +1993,16 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = next(x) break of xmlError: - errors.add(errorMsg(x)) + adderr(errorMsg(x)) next(x) break else: - errors.add(errorMsg(x, "'>' expected")) + adderr(errorMsg(x, "'>' expected")) next(x) break untilElementEnd(x, result, errors) of xmlAttribute, xmlElementClose: - errors.add(errorMsg(x, "<some_tag> expected")) + adderr(errorMsg(x, "<some_tag> expected")) next(x) of xmlCData: result = newCData(x.rawData) @@ -553,10 +2015,11 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseHtml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = - ## parses the XML from stream `s` and returns a ``PXmlNode``. Every + ## Parses the XML from stream `s` and returns a `XmlNode`. Every ## occurred parsing error is added to the `errors` sequence. var x: XmlParser - open(x, s, filename, {reportComments, reportWhitespace}) + open(x, s, filename, {reportComments, reportWhitespace, allowUnquotedAttribs, + allowEmptyAttribs}) next(x) # skip the DOCTYPE: if x.kind == xmlSpecial: next(x) @@ -564,9 +2027,9 @@ proc parseHtml*(s: Stream, filename: string, result = newElement("document") result.addNode(parse(x, errors)) #if x.kind != xmlEof: - # errors.add(errorMsg(x, "EOF expected")) + # adderr(errorMsg(x, "EOF expected")) while x.kind != xmlEof: - var oldPos = x.bufpos # little hack to see if we made any progess + var oldPos = x.bufpos # little hack to see if we made any progress result.addNode(parse(x, errors)) if x.bufpos == oldPos: # force progress! @@ -576,27 +2039,32 @@ proc parseHtml*(s: Stream, filename: string, result = result[0] proc parseHtml*(s: Stream): XmlNode = - ## parses the XTML from stream `s` and returns a ``PXmlNode``. All parsing + ## Parses the HTML from stream `s` and returns a `XmlNode`. All parsing ## errors are ignored. var errors: seq[string] = @[] result = parseHtml(s, "unknown_html_doc", errors) +proc parseHtml*(html: string): XmlNode = + ## Parses the HTML from string `html` and returns a `XmlNode`. All parsing + ## errors are ignored. + parseHtml(newStringStream(html)) + proc loadHtml*(path: string, errors: var seq[string]): XmlNode = - ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occurred parsing error is added to + ## Loads and parses HTML from file specified by `path`, and returns + ## a `XmlNode`. Every occurred parsing error is added to ## the `errors` sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) result = parseHtml(s, path, errors) proc loadHtml*(path: string): XmlNode = - ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``PXmlNode``. All parsing errors are ignored. + ## Loads and parses HTML from file specified by `path`, and returns + ## a `XmlNode`. All parsing errors are ignored. var errors: seq[string] = @[] result = loadHtml(path, errors) when not defined(testing) and isMainModule: - import os + import std/os var errors: seq[string] = @[] var x = loadHtml(paramStr(1), errors) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 8e182e274..08ea99627 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -1,116 +1,365 @@ # # # Nim's Runtime Library -# (c) Copyright 2010 Dominik Picheta, Andreas Rumpf +# (c) Copyright 2019 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements a simple HTTP client that can be used to retrieve -## webpages/other data. +## webpages and other data. ## -## -## **Note**: This module is not ideal, connection is not kept alive so sites with -## many redirects are expensive. As such in the future this module may change, -## and the current procedures will be deprecated. +## .. warning:: Validate untrusted inputs: URI parsers and getters are not detecting malicious URIs. ## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve -## ``http://google.com`` +## `http://google.com`: +## +## ```Nim +## import std/httpclient +## var client = newHttpClient() +## try: +## echo client.getContent("http://google.com") +## finally: +## client.close() +## ``` +## +## The same action can also be performed asynchronously, simply use the +## `AsyncHttpClient`: +## +## ```Nim +## import std/[asyncdispatch, httpclient] +## +## proc asyncProc(): Future[string] {.async.} = +## var client = newAsyncHttpClient() +## try: +## return await client.getContent("http://google.com") +## finally: +## client.close() ## -## .. code-block:: Nim -## echo(getContent("http://google.com")) +## echo waitFor asyncProc() +## ``` +## +## The functionality implemented by `HttpClient` and `AsyncHttpClient` +## is the same, so you can use whichever one suits you best in the examples +## shown here. +## +## **Note:** You need to run asynchronous examples in an async proc +## otherwise you will get an `Undeclared identifier: 'await'` error. +## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. ## ## Using HTTP POST ## =============== ## ## This example demonstrates the usage of the W3 HTML Validator, it -## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to -## the server. +## uses `multipart/form-data` as the `Content-Type` to send the HTML to be +## validated to the server. ## -## .. code-block:: Nim +## ```Nim +## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` +## +## To stream files from disk when performing the request, use `addFiles`. +## +## **Note:** This will allocate a new `Mimetypes` database every time you call +## it, you can pass your own via the `mimeDb` parameter to avoid this. +## +## ```Nim +## let mimes = newMimetypes() +## var client = newHttpClient() +## var data = newMultipartData() +## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` +## +## You can also make post requests with custom headers. +## This example sets `Content-Type` to `application/json` +## and uses a json object for the body +## +## ```Nim +## import std/[httpclient, json] +## +## let client = newHttpClient() +## client.headers = newHttpHeaders({ "Content-Type": "application/json" }) +## let body = %*{ +## "data": "some text" +## } +## try: +## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) +## echo response.status +## finally: +## client.close() +## ``` +## +## Progress reporting +## ================== +## +## You may specify a callback procedure to be called during an HTTP request. +## This callback will be executed every second with information about the +## progress of the HTTP request. ## -## echo postContent("http://validator.w3.org/check", multipart=data) +## ```Nim +## import std/[asyncdispatch, httpclient] ## -## Asynchronous HTTP requests -## ========================== +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") ## -## You simply have to create a new instance of the ``AsyncHttpClient`` object. -## You may then use ``await`` on the functions defined for that object. -## Keep in mind that the following code needs to be inside an asynchronous -## procedure. +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## try: +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## finally: +## client.close() ## -## .. code-block::nim +## waitFor asyncProc() +## ``` +## +## If you would like to remove the callback simply set it to `nil`. +## +## ```Nim +## client.onProgressChanged = nil +## ``` +## +## .. warning:: The `total` reported by httpclient may be 0 in some cases. ## -## var client = newAsyncHttpClient() -## var resp = await client.request("http://google.com") ## ## SSL/TLS support ## =============== -## This requires the OpenSSL library, fortunately it's widely used and installed +## This requires the OpenSSL library. Fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give -## any of the functions a url with the ``https`` schema, for example: -## ``https://github.com/``, you also have to compile with ``ssl`` defined like so: -## ``nim c -d:ssl ...``. +## any of the functions a url with the `https` schema, for example: +## `https://github.com/`. +## +## You will also have to compile with `ssl` defined like so: +## `nim c -d:ssl ...`. +## +## Certificate validation is performed by default. +## +## A set of directories and files from the `ssl_certs <ssl_certs.html>`_ +## module are scanned to locate CA certificates. +## +## Example of setting SSL verification parameters in a new client: +## +## ```Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ``` +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## +## See `newContext <net.html#newContext.string,string,string,string>`_ to tweak or disable certificate validation. ## ## Timeouts ## ======== -## Currently all functions support an optional timeout, by default the timeout is set to -## `-1` which means that the function will never time out. The timeout is +## +## Currently only the synchronous functions support a timeout. +## The timeout is ## measured in milliseconds, once it is set any call on a socket which may -## block will be susceptible to this timeout, however please remember that the +## block will be susceptible to this timeout. +## +## It may be surprising but the ## function as a whole can take longer than the specified timeout, only ## individual internal calls on the socket are affected. In practice this means ## that as long as the server is sending data an exception will not be raised, -## if however data does not reach client within the specified timeout an ETimeout -## exception will then be raised. +## if however data does not reach the client within the specified timeout a +## `TimeoutError` exception will be raised. +## +## Here is how to set a timeout when creating an `HttpClient` instance: +## +## ```Nim +## import std/httpclient +## +## let client = newHttpClient(timeout = 42) +## ``` ## ## Proxy ## ===== ## -## A proxy can be specified as a param to any of these procedures, the ``newProxy`` -## constructor should be used for this purpose. However, -## currently only basic authentication is supported. +## A proxy can be specified as a param to any of the procedures defined in +## this module. To do this, use the `newProxy` constructor. Unfortunately, +## only basic authentication is supported at the moment. +## +## Some examples on how to configure a Proxy for `HttpClient`: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Use proxies with basic authentication: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Get Proxy URL from environment variables: +## +## ```Nim +## import std/httpclient +## +## var url = "" +## try: +## if existsEnv("http_proxy"): +## url = getEnv("http_proxy") +## elif existsEnv("https_proxy"): +## url = getEnv("https_proxy") +## except ValueError: +## echo "Unable to parse proxy from environment variables." +## +## let myProxy = newProxy(url = url) +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Redirects +## ========= +## +## The maximum redirects can be set with the `maxRedirects` of `int` type, +## it specifies the maximum amount of redirects to follow, +## it defaults to `5`, you can set it to `0` to disable redirects. +## +## Here you can see an example about how to set the `maxRedirects` of `HttpClient`: +## +## ```Nim +## import std/httpclient +## +## let client = newHttpClient(maxRedirects = 0) +## ``` +## + +import std/private/since + +import std/[ + net, strutils, uri, parseutils, base64, os, mimetypes, + math, random, httpcore, times, tables, streams, monotimes, + asyncnet, asyncdispatch, asyncfile, nativesockets, +] -import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, math -import asyncnet, asyncdispatch -import nativesockets +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +export httpcore except parseHeader # TODO: The `except` doesn't work type - Response* = tuple[ - version: string, - status: string, - headers: StringTableRef, - body: string] + Response* = ref object + version*: string + status*: string + headers*: HttpHeaders + body: string + bodyStream*: Stream + + AsyncResponse* = ref object + version*: string + status*: string + headers*: HttpHeaders + body: string + bodyStream*: FutureStream[string] + +proc code*(response: Response | AsyncResponse): HttpCode + {.raises: [ValueError, OverflowDefect].} = + ## Retrieves the specified response's `HttpCode`. + ## + ## Raises a `ValueError` if the response's `status` does not have a + ## corresponding `HttpCode`. + return response.status[0 .. 2].parseInt.HttpCode + +proc contentType*(response: Response | AsyncResponse): string {.inline.} = + ## Retrieves the specified response's content type. + ## + ## This is effectively the value of the "Content-Type" header. + response.headers.getOrDefault("content-type") + +proc contentLength*(response: Response | AsyncResponse): int = + ## Retrieves the specified response's content length. + ## + ## This is effectively the value of the "Content-Length" header. + ## + ## A `ValueError` exception will be raised if the value is not an integer. + ## If the Content-Length header is not set in the response, ContentLength is set to the value -1. + var contentLengthHeader = response.headers.getOrDefault("Content-Length", HttpHeaderValues(@["-1"])) + result = contentLengthHeader.parseInt() + +proc lastModified*(response: Response | AsyncResponse): DateTime = + ## Retrieves the specified response's last modified time. + ## + ## This is effectively the value of the "Last-Modified" header. + ## + ## Raises a `ValueError` if the parsing fails or the value is not a correctly + ## formatted time. + var lastModifiedHeader = response.headers.getOrDefault("last-modified") + result = parse(lastModifiedHeader, "ddd, dd MMM yyyy HH:mm:ss 'GMT'", utc()) + +proc body*(response: Response): string = + ## Retrieves the specified response's body. + ## + ## The response's body stream is read synchronously. + if response.body.len == 0: + response.body = response.bodyStream.readAll() + return response.body + +proc body*(response: AsyncResponse): Future[string] {.async.} = + ## Reads the response's body and caches it. The read is performed only + ## once. + if response.body.len == 0: + response.body = await readAll(response.bodyStream) + return response.body +type Proxy* = ref object url*: Uri auth*: string - MultipartEntries* = openarray[tuple[name, content: string]] + MultipartEntry = object + name, content: string + case isFile: bool + of true: + filename, contentType: string + fileSize: int64 + isStream: bool + else: discard + + MultipartEntries* = openArray[tuple[name, content: string]] MultipartData* = ref object - content: seq[string] + content: seq[MultipartEntry] - ProtocolError* = object of IOError ## exception that is raised when server - ## does not conform to the implemented - ## protocol + ProtocolError* = object of IOError ## exception that is raised when server + ## does not conform to the implemented + ## protocol - HttpRequestError* = object of IOError ## Thrown in the ``getContent`` proc - ## and ``postContent`` proc, + HttpRequestError* = object of IOError ## Thrown in the `getContent` proc + ## and `postContent` proc, ## when the server returns an error -{.deprecated: [TResponse: Response, PProxy: Proxy, - EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError -].} - -const defUserAgent* = "Nim httpclient/0.1" +const defUserAgent* = "Nim-httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -124,192 +373,78 @@ proc fileError(msg: string) = e.msg = msg raise e -proc parseChunks(s: Socket, timeout: int): string = - result = "" - var ri = 0 - while true: - var chunkSizeStr = "" - var chunkSize = 0 - s.readLine(chunkSizeStr, timeout) - var i = 0 - if chunkSizeStr == "": - httpError("Server terminated connection prematurely") - while true: - case chunkSizeStr[i] - of '0'..'9': - chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) - of 'a'..'f': - chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) - of 'A'..'F': - chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) - of '\0': - break - of ';': - # http://tools.ietf.org/html/rfc2616#section-3.6.1 - # We don't care about chunk-extensions. - break - else: - httpError("Invalid chunk size: " & chunkSizeStr) - inc(i) - if chunkSize <= 0: - s.skip(2, timeout) # Skip \c\L - break - result.setLen(ri+chunkSize) - var bytesRead = 0 - while bytesRead != chunkSize: - let ret = recv(s, addr(result[ri]), chunkSize-bytesRead, timeout) - ri += ret - bytesRead += ret - s.skip(2, timeout) # Skip \c\L - # Trailer headers will only be sent if the request specifies that we want - # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 - -proc parseBody(s: Socket, headers: StringTableRef, timeout: int): string = - result = "" - if headers.getOrDefault"Transfer-Encoding" == "chunked": - result = parseChunks(s, timeout) - else: - # -REGION- Content-Length - # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3 - var contentLengthHeader = headers.getOrDefault"Content-Length" - if contentLengthHeader != "": - var length = contentLengthHeader.parseint() - if length > 0: - result = newString(length) - var received = 0 - while true: - if received >= length: break - let r = s.recv(addr(result[received]), length-received, timeout) - if r == 0: break - received += r - if received != length: - httpError("Got invalid content length. Expected: " & $length & - " got: " & $received) - else: - # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO - - # -REGION- Connection: Close - # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": - var buf = "" - while true: - buf = newString(4000) - let r = s.recv(addr(buf[0]), 4000, timeout) - if r == 0: break - buf.setLen(r) - result.add(buf) - -proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = - var parsedStatus = false - var linei = 0 - var fullyRead = false - var line = "" - result.headers = newStringTable(modeCaseInsensitive) - while true: - line = "" - linei = 0 - s.readLine(line, timeout) - if line == "": break # We've been disconnected. - if line == "\c\L": - fullyRead = true - break - if not parsedStatus: - # Parse HTTP version info and status code. - var le = skipIgnoreCase(line, "HTTP/", linei) - if le <= 0: httpError("invalid http version") - inc(linei, le) - le = skipIgnoreCase(line, "1.1", linei) - if le > 0: result.version = "1.1" - else: - le = skipIgnoreCase(line, "1.0", linei) - if le <= 0: httpError("unsupported http version") - result.version = "1.0" - inc(linei, le) - # Status code - linei.inc skipWhitespace(line, linei) - result.status = line[linei .. ^1] - parsedStatus = true - else: - # Parse headers - var name = "" - var le = parseUntil(line, name, ':', linei) - if le <= 0: httpError("invalid headers") - inc(linei, le) - if line[linei] != ':': httpError("invalid headers") - inc(linei) # Skip : - - result.headers[name] = line[linei.. ^1].strip() - if not fullyRead: - httpError("Connection was closed before full request has been made") - if getBody: - result.body = parseBody(s, result.headers, timeout) - else: - result.body = "" - -type - HttpMethod* = enum ## the requested HttpMethod - httpHEAD, ## Asks for the response identical to the one that would - ## correspond to a GET request, but without the response - ## body. - httpGET, ## Retrieves the specified resource. - httpPOST, ## Submits data to be processed to the identified - ## resource. The data is included in the body of the - ## request. - httpPUT, ## Uploads a representation of the specified resource. - httpDELETE, ## Deletes the specified resource. - httpTRACE, ## Echoes back the received request, so that a client - ## can see what intermediate servers are adding or - ## changing in the request. - httpOPTIONS, ## Returns the HTTP methods that the server supports - ## for specified address. - httpCONNECT ## Converts the request connection to a transparent - ## TCP/IP tunnel, usually used for proxies. - -{.deprecated: [THttpMethod: HttpMethod].} - when not defined(ssl): - type SSLContext = ref object - let defaultSSLContext: SSLContext = nil -else: - let defaultSSLContext = newContext(verifyMode = CVerifyNone) + type SslContext = ref object +var defaultSslContext {.threadvar.}: SslContext -proc newProxy*(url: string, auth = ""): Proxy = - ## Constructs a new ``TProxy`` object. - result = Proxy(url: parseUri(url), auth: auth) +proc getDefaultSSL(): SslContext = + result = defaultSslContext + when defined(ssl): + if result == nil: + defaultSslContext = newContext(verifyMode = CVerifyPeer) + result = defaultSslContext + doAssert result != nil, "failure to initialize the SSL context" -proc newMultipartData*: MultipartData = - ## Constructs a new ``MultipartData`` object. - MultipartData(content: @[]) +proc newProxy*(url: string; auth = ""): Proxy = + ## Constructs a new `TProxy` object. + result = Proxy(url: parseUri(url), auth: auth) -proc add*(p: var MultipartData, name, content: string, filename: string = nil, - contentType: string = nil) = - ## Add a value to the multipart data. Raises a `ValueError` exception if +proc newProxy*(url: Uri; auth = ""): Proxy = + ## Constructs a new `TProxy` object. + result = Proxy(url: url, auth: auth) + +proc newMultipartData*: MultipartData {.inline.} = + ## Constructs a new `MultipartData` object. + MultipartData() + +proc `$`*(data: MultipartData): string {.since: (1, 1).} = + ## convert MultipartData to string so it's human readable when echo + ## see https://github.com/nim-lang/Nim/issues/11863 + const sep = "-".repeat(30) + for pos, entry in data.content: + result.add(sep & center($pos, 3) & sep) + result.add("\nname=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"\n") + result.add("Content-Type: " & entry.contentType) + result.add("\n\n" & entry.content & "\n") + +proc add*(p: MultipartData, name, content: string, filename: string = "", + contentType: string = "", useStream = true) = + ## Add a value to the multipart data. + ## + ## When `useStream` is `false`, the file will be read into memory. + ## + ## Raises a `ValueError` exception if ## `name`, `filename` or `contentType` contain newline characters. - - if {'\c','\L'} in name: + if {'\c', '\L'} in name: raise newException(ValueError, "name contains a newline character") - if filename != nil and {'\c','\L'} in filename: + if {'\c', '\L'} in filename: raise newException(ValueError, "filename contains a newline character") - if contentType != nil and {'\c','\L'} in contentType: + if {'\c', '\L'} in contentType: raise newException(ValueError, "contentType contains a newline character") - var str = "Content-Disposition: form-data; name=\"" & name & "\"" - if filename != nil: - str.add("; filename=\"" & filename & "\"") - str.add("\c\L") - if contentType != nil: - str.add("Content-Type: " & contentType & "\c\L") - str.add("\c\L" & content & "\c\L") + var entry = MultipartEntry( + name: name, + content: content, + isFile: filename.len > 0 + ) - p.content.add(str) + if entry.isFile: + entry.isStream = useStream + entry.filename = filename + entry.contentType = contentType -proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData + p.content.add(entry) + +proc add*(p: MultipartData, xs: MultipartEntries): MultipartData {.discardable.} = ## Add a list of multipart entries to the multipart data `p`. All values are ## added without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data.add({"action": "login", "format": "json"}) + ## ``` for name, content in xs.items: p.add(name, content) result = p @@ -318,340 +453,320 @@ proc newMultipartData*(xs: MultipartEntries): MultipartData = ## Create a new multipart data object and fill it with the entries `xs` ## directly. ## - ## .. code-block:: Nim + ## ```Nim ## var data = newMultipartData({"action": "login", "format": "json"}) - result = MultipartData(content: @[]) - result.add(xs) - -proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): - MultipartData {.discardable.} = - ## Add files to a multipart data object. The file will be opened from your - ## disk, read and sent with the automatically determined MIME type. Raises an - ## `IOError` if the file cannot be opened or reading fails. To manually - ## specify file content, filename and MIME type, use `[]=` instead. + ## ``` + result = MultipartData() + for entry in xs: + result.add(entry.name, entry.content) + +proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], + mimeDb = newMimetypes(), useStream = true): + MultipartData {.discardable.} = + ## Add files to a multipart data object. The files will be streamed from disk + ## when the request is being made. When `stream` is `false`, the files are + ## instead read into memory, but beware this is very memory ineffecient even + ## for small files. The MIME types will automatically be determined. + ## Raises an `IOError` if the file cannot be opened or reading fails. To + ## manually specify file content, filename and MIME type, use `[]=` instead. ## - ## .. code-block:: Nim + ## ```Nim ## data.addFiles({"uploaded_file": "public/test.html"}) - var m = newMimetypes() + ## ``` for name, file in xs.items: var contentType: string - let (dir, fName, ext) = splitFile(file) + let (_, fName, ext) = splitFile(file) if ext.len > 0: - contentType = m.getMimetype(ext[1..ext.high], nil) - p.add(name, readFile(file), fName & ext, contentType) + contentType = mimeDb.getMimetype(ext[1..ext.high], "") + let content = if useStream: file else: readFile(file) + p.add(name, content, fName & ext, contentType, useStream = useStream) result = p -proc `[]=`*(p: var MultipartData, name, content: string) = +proc `[]=`*(p: MultipartData, name, content: string) {.inline.} = ## Add a multipart entry to the multipart data `p`. The value is added ## without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data["username"] = "NimUser" + ## ``` p.add(name, content) -proc `[]=`*(p: var MultipartData, name: string, - file: tuple[name, contentType, content: string]) = - ## Add a file to the multipart data `p`, specifying filename, contentType and - ## content manually. +proc `[]=`*(p: MultipartData, name: string, + file: tuple[name, contentType, content: string]) {.inline.} = + ## Add a file to the multipart data `p`, specifying filename, contentType + ## and content manually. ## - ## .. code-block:: Nim + ## ```Nim ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") - p.add(name, file.content, file.name, file.contentType) + ## ``` + p.add(name, file.content, file.name, file.contentType, useStream = false) -proc format(p: MultipartData): tuple[header, body: string] = - if p == nil or p.content == nil or p.content.len == 0: - return ("", "") - - # Create boundary that is not in the data to be formatted - var bound: string +proc getBoundary(p: MultipartData): string = + if p == nil or p.content.len == 0: return while true: - bound = $random(int.high) - var found = false - for s in p.content: - if bound in s: - found = true - if not found: - break - - result.header = "Content-Type: multipart/form-data; boundary=" & bound & "\c\L" - result.body = "" - for s in p.content: - result.body.add("--" & bound & "\c\L" & s) - result.body.add("--" & bound & "--\c\L") - -proc request*(url: string, httpMethod: string, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = - ## | Requests ``url`` with the custom method string specified by the - ## | ``httpMethod`` parameter. - ## | Extra headers can be specified and must be separated by ``\c\L`` - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - var r = if proxy == nil: parseUri(url) else: proxy.url - var headers = substr(httpMethod, len("http")) - # TODO: Use generateHeaders further down once it supports proxies. - if proxy == nil: - headers.add ' ' - if r.path[0] != '/': headers.add '/' - headers.add(r.path) - if r.query.len > 0: - headers.add("?" & r.query) + result = $rand(int.high) + for i, entry in p.content: + if result in entry.content: break + elif i == p.content.high: return + +proc sendFile(socket: Socket | AsyncSocket, + entry: MultipartEntry) {.multisync.} = + const chunkSize = 2^18 + let file = + when socket is AsyncSocket: openAsync(entry.content) + else: newFileStream(entry.content, fmRead) + + var buffer: string + while true: + buffer = + when socket is AsyncSocket: (await read(file, chunkSize)) + else: readStr(file, chunkSize) + if buffer.len == 0: break + await socket.send(buffer) + file.close() + +proc getNewLocation(lastURL: Uri, headers: HttpHeaders): Uri = + let newLocation = headers.getOrDefault"Location" + if newLocation == "": httpError("location header expected") + # Relative URLs. (Not part of the spec, but soon will be.) + let parsedLocation = parseUri(newLocation) + if parsedLocation.hostname == "" and parsedLocation.path != "": + result = lastURL + result.path = parsedLocation.path + result.query = parsedLocation.query + result.anchor = parsedLocation.anchor else: - headers.add(" " & url) + result = parsedLocation - headers.add(" HTTP/1.1\c\L") +proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeaders, + proxy: Proxy): string = + # GET + result = $httpMethod + result.add ' ' - if r.port == "": - add(headers, "Host: " & r.hostname & "\c\L") + if proxy.isNil or requestUrl.scheme == "https": + # /path?query + if not requestUrl.path.startsWith("/"): result.add '/' + result.add(requestUrl.path) + if requestUrl.query.len > 0: + result.add("?" & requestUrl.query) else: - add(headers, "Host: " & r.hostname & ":" & r.port & "\c\L") - - if userAgent != "": - add(headers, "User-Agent: " & userAgent & "\c\L") - if proxy != nil and proxy.auth != "": - let auth = base64.encode(proxy.auth, newline = "") - add(headers, "Proxy-Authorization: basic " & auth & "\c\L") - add(headers, extraHeaders) - add(headers, "\c\L") - - var s = newSocket() - if s == nil: raiseOSError(osLastError()) - var port = net.Port(80) - if r.scheme == "https": - when defined(ssl): - sslContext.wrapSocket(s) - port = net.Port(443) + # Remove the 'http://' from the URL for CONNECT requests for TLS connections. + var modifiedUrl = requestUrl + if requestUrl.scheme == "https": modifiedUrl.scheme = "" + result.add($modifiedUrl) + + # HTTP/1.1\c\l + result.add(" HTTP/1.1" & httpNewLine) + + # Host header. + if not headers.hasKey("Host"): + if requestUrl.port == "": + add(result, "Host: " & requestUrl.hostname & httpNewLine) else: - raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") - if r.port != "": - port = net.Port(r.port.parseInt) + add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine) - if timeout == -1: - s.connect(r.hostname, port) - else: - s.connect(r.hostname, port, timeout) - s.send(headers) - if body != "": - s.send(body) - - result = parseResponse(s, httpMethod != "httpHEAD", timeout) - s.close() - -proc request*(url: string, httpMethod = httpGET, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = - ## | Requests ``url`` with the specified ``httpMethod``. - ## | Extra headers can be specified and must be separated by ``\c\L`` - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout, - userAgent, proxy) - -proc redirection(status: string): bool = - const redirectionNRs = ["301", "302", "303", "307"] - for i in items(redirectionNRs): - if status.startsWith(i): - return true - -proc getNewLocation(lastUrl: string, headers: StringTableRef): string = - result = headers.getOrDefault"Location" - if result == "": httpError("location header expected") - # Relative URLs. (Not part of the spec, but soon will be.) - let r = parseUri(result) - if r.hostname == "" and r.path != "": - let origParsed = parseUri(lastUrl) - result = origParsed.hostname & "/" & r.path - -proc get*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response = - ## | GETs the ``url`` and returns a ``Response`` object - ## | This proc also handles redirection - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - result = request(url, httpGET, extraHeaders, "", sslContext, timeout, - userAgent, proxy) - var lastURL = url - for i in 1..maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - result = request(redirectTo, httpGET, extraHeaders, "", sslContext, - timeout, userAgent, proxy) - lastUrl = redirectTo - -proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): string = - ## | GETs the body and returns it as a string. - ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - var r = get(url, extraHeaders, maxRedirects, sslContext, timeout, userAgent, - proxy) - if r.status[0] in {'4','5'}: - raise newException(HttpRequestError, r.status) - else: - return r.body - -proc post*(url: string, extraHeaders = "", body = "", - maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil, - multipart: MultipartData = nil): Response = - ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object. - ## | This proc adds the necessary Content-Length header. - ## | This proc also handles redirection. - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## | The optional ``multipart`` parameter can be used to create - ## ``multipart/form-data`` POSTs comfortably. - let (mpHeaders, mpBody) = format(multipart) - - template withNewLine(x): expr = - if x.len > 0 and not x.endsWith("\c\L"): - x & "\c\L" - else: - x - - var xb = mpBody.withNewLine() & body - - var xh = extraHeaders.withNewLine() & mpHeaders.withNewLine() & - withNewLine("Content-Length: " & $len(xb)) - - result = request(url, httpPOST, xh, xb, sslContext, timeout, userAgent, - proxy) - var lastUrl = "" - for i in 1..maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - var meth = if result.status != "307": httpGet else: httpPost - result = request(redirectTo, meth, xh, xb, sslContext, timeout, - userAgent, proxy) - lastUrl = redirectTo - -proc postContent*(url: string, extraHeaders = "", body = "", - maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil, - multipart: MultipartData = nil): string = - ## | POSTs ``body`` to ``url`` and returns the response's body as a string - ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - ## | Extra headers can be specified and must be separated by ``\c\L``. - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - ## | The optional ``multipart`` parameter can be used to create - ## ``multipart/form-data`` POSTs comfortably. - var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout, - userAgent, proxy, multipart) - if r.status[0] in {'4','5'}: - raise newException(HttpRequestError, r.status) - else: - return r.body - -proc downloadFile*(url: string, outputFilename: string, - sslContext: SSLContext = defaultSSLContext, - timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil) = - ## | Downloads ``url`` and saves it to ``outputFilename`` - ## | An optional timeout can be specified in milliseconds, if reading from the - ## server takes longer than specified an ETimeout exception will be raised. - var f: File - if open(f, outputFilename, fmWrite): - f.write(getContent(url, sslContext = sslContext, timeout = timeout, - userAgent = userAgent, proxy = proxy)) - f.close() - else: - fileError("Unable to open file") + # Connection header. + if not headers.hasKey("Connection"): + add(result, "Connection: Keep-Alive" & httpNewLine) -proc generateHeaders(r: Uri, httpMethod: string, - headers: StringTableRef, body: string): string = - # TODO: Use this in the blocking HttpClient once it supports proxies. - result = substr(httpMethod, len("http")) - # TODO: Proxies - result.add ' ' - if r.path[0] != '/': result.add '/' - result.add(r.path) - if r.query.len > 0: - result.add("?" & r.query) - result.add(" HTTP/1.1\c\L") - - if r.port == "": - add(result, "Host: " & r.hostname & "\c\L") - else: - add(result, "Host: " & r.hostname & ":" & r.port & "\c\L") + # Proxy auth header. + if not proxy.isNil and proxy.auth != "": + let auth = base64.encode(proxy.auth) + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) - add(result, "Connection: Keep-Alive\c\L") - if body.len > 0 and not headers.hasKey("Content-Length"): - add(result, "Content-Length: " & $body.len & "\c\L") for key, val in headers: - add(result, key & ": " & val & "\c\L") + add(result, key & ": " & val & httpNewLine) - add(result, "\c\L") + add(result, httpNewLine) type - AsyncHttpClient* = ref object - socket: AsyncSocket + ProgressChangedProc*[ReturnType] = + proc (total, progress, speed: BiggestInt): + ReturnType {.closure, gcsafe.} + + HttpClientBase*[SocketType] = ref object + socket: SocketType connected: bool - currentURL: Uri ## Where we are currently connected. - headers*: StringTableRef - maxRedirects: int + currentURL: Uri ## Where we are currently connected. + headers*: HttpHeaders ## Headers to send in requests. + maxRedirects: Natural ## Maximum redirects, set to `0` to disable. userAgent: string + timeout*: int ## Only used for blocking HttpClient for now. + proxy: Proxy + ## `nil` or the callback to call when request progress changes. + when SocketType is Socket: + onProgressChanged*: ProgressChangedProc[void] + else: + onProgressChanged*: ProgressChangedProc[Future[void]] when defined(ssl): sslContext: net.SslContext + contentTotal: BiggestInt + contentProgress: BiggestInt + oneSecondProgress: BiggestInt + lastProgressReport: MonoTime + when SocketType is AsyncSocket: + bodyStream: FutureStream[string] + parseBodyFut: Future[void] + else: + bodyStream: Stream + getBody: bool ## When `false`, the body is never read in requestAux. -{.deprecated: [PAsyncHttpClient: AsyncHttpClient].} +type + HttpClient* = HttpClientBase[Socket] + +proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + timeout = -1, headers = newHttpHeaders()): HttpClient = + ## Creates a new HttpClient instance. + ## + ## `userAgent` specifies the user agent that will be used when making + ## requests. + ## + ## `maxRedirects` specifies the maximum amount of redirects to follow, + ## default is 5. + ## + ## `sslContext` specifies the SSL context to use for HTTPS requests. + ## See `SSL/TLS support <#sslslashtls-support>`_ + ## + ## `proxy` specifies an HTTP proxy to use for this HTTP client's + ## connections. + ## + ## `timeout` specifies the number of milliseconds to allow before a + ## `TimeoutError` is raised. + ## + ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/strutils + + let exampleHtml = newHttpClient().getContent("http://example.com") + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + + new result + result.headers = headers + result.userAgent = userAgent + result.maxRedirects = maxRedirects + result.proxy = proxy + result.timeout = timeout + result.onProgressChanged = nil + result.bodyStream = newStringStream() + result.getBody = true + when defined(ssl): + result.sslContext = sslContext + +type + AsyncHttpClient* = HttpClientBase[AsyncSocket] -proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext): AsyncHttpClient = +proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + headers = newHttpHeaders()): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## - ## ``userAgent`` specifies the user agent that will be used when making + ## `userAgent` specifies the user agent that will be used when making ## requests. ## - ## ``maxRedirects`` specifies the maximum amount of redirects to follow, + ## `maxRedirects` specifies the maximum amount of redirects to follow, ## default is 5. ## - ## ``sslContext`` specifies the SSL context to use for HTTPS requests. + ## `sslContext` specifies the SSL context to use for HTTPS requests. + ## + ## `proxy` specifies an HTTP proxy to use for this HTTP client's + ## connections. + ## + ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/[asyncdispatch, strutils] + + proc asyncProc(): Future[string] {.async.} = + let client = newAsyncHttpClient() + result = await client.getContent("http://example.com") + + let exampleHtml = waitFor asyncProc() + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + new result - result.headers = newStringTable(modeCaseInsensitive) - result.userAgent = defUserAgent + result.headers = headers + result.userAgent = userAgent result.maxRedirects = maxRedirects + result.proxy = proxy + result.timeout = -1 # TODO + result.onProgressChanged = nil + result.bodyStream = newFutureStream[string]("newAsyncHttpClient") + result.getBody = true when defined(ssl): - result.sslContext = net.SslContext(sslContext) + result.sslContext = sslContext -proc close*(client: AsyncHttpClient) = +proc close*(client: HttpClient | AsyncHttpClient) = ## Closes any connections held by the HTTP client. if client.connected: client.socket.close() client.connected = false -proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} = +proc getSocket*(client: HttpClient): Socket {.inline.} = + ## Get network socket, useful if you want to find out more details about the connection. + ## + ## This example shows info about local and remote endpoints: + ## + ## ```Nim + ## if client.connected: + ## echo client.getSocket.getLocalAddr + ## echo client.getSocket.getPeerAddr + ## ``` + return client.socket + +proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} = + return client.socket + +proc reportProgress(client: HttpClient | AsyncHttpClient, + progress: BiggestInt) {.multisync.} = + client.contentProgress += progress + client.oneSecondProgress += progress + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: + if not client.onProgressChanged.isNil: + await client.onProgressChanged(client.contentTotal, + client.contentProgress, + client.oneSecondProgress) + client.oneSecondProgress = 0 + client.lastProgressReport = getMonoTime() + +proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int, + keep: bool): Future[int] {.multisync.} = ## Ensures that all the data requested is read and returned. - result = "" + var readLen = 0 while true: - if size == result.len: break - let data = await socket.recv(size - result.len) - if data == "": break # We've been disconnected. - result.add data + if size == readLen: break + + let remainingSize = size - readLen + let sizeToRecv = min(remainingSize, net.BufferSize) + + when client.socket is Socket: + let data = client.socket.recv(sizeToRecv, timeout) + else: + let data = await client.socket.recv(sizeToRecv) + if data == "": + client.close() + break # We've been disconnected. + + readLen.inc(data.len) + if keep: + await client.bodyStream.write(data) -proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = - result = "" + await reportProgress(client, data.len) + + return readLen + +proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] + {.multisync.} = while true: var chunkSize = 0 var chunkSizeStr = await client.socket.recvLine() var i = 0 if chunkSizeStr == "": httpError("Server terminated connection prematurely") - while true: + while i < chunkSizeStr.len: case chunkSizeStr[i] of '0'..'9': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) @@ -659,8 +774,6 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) of 'A'..'F': chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) - of '\0': - break of ';': # http://tools.ietf.org/html/rfc2616#section-3.6.1 # We don't care about chunk-extensions. @@ -669,62 +782,101 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = httpError("Invalid chunk size: " & chunkSizeStr) inc(i) if chunkSize <= 0: - discard await recvFull(client.socket, 2) # Skip \c\L + discard await recvFull(client, 2, client.timeout, false) # Skip \c\L break - result.add await recvFull(client.socket, chunkSize) - discard await recvFull(client.socket, 2) # Skip \c\L + var bytesRead = await recvFull(client, chunkSize, client.timeout, true) + if bytesRead != chunkSize: + httpError("Server terminated connection prematurely") + + bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L + if bytesRead != 2: + httpError("Server terminated connection prematurely") + # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(client: AsyncHttpClient, - headers: StringTableRef): Future[string] {.async.} = - result = "" +proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, + httpVersion: string): Future[void] {.multisync.} = + # Reset progress from previous requests. + client.contentTotal = 0 + client.contentProgress = 0 + client.oneSecondProgress = 0 + client.lastProgressReport = MonoTime() + + when client is AsyncHttpClient: + assert(not client.bodyStream.finished) + if headers.getOrDefault"Transfer-Encoding" == "chunked": - result = await parseChunks(client) + await parseChunks(client) else: # -REGION- Content-Length # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3 var contentLengthHeader = headers.getOrDefault"Content-Length" if contentLengthHeader != "": - var length = contentLengthHeader.parseint() + var length = contentLengthHeader.parseInt() + client.contentTotal = length if length > 0: - result = await client.socket.recvFull(length) - if result == "": + let recvLen = await client.recvFull(length, client.timeout, true) + if recvLen == 0: + client.close() httpError("Got disconnected while trying to read body.") - if result.len != length: + if recvLen != length: httpError("Received length doesn't match expected length. Wanted " & - $length & " got " & $result.len) + $length & " got: " & $recvLen) else: # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": - var buf = "" + let implicitConnectionClose = + httpVersion == "1.0" or + # This doesn't match the HTTP spec, but it fixes issues for non-conforming servers. + (httpVersion == "1.1" and headers.getOrDefault"Connection" == "") + if headers.getOrDefault"Connection" == "close" or implicitConnectionClose: while true: - buf = await client.socket.recvFull(4000) - if buf == "": break - result.add(buf) + let recvLen = await client.recvFull(4000, client.timeout, true) + if recvLen != 4000: + client.close() + break + + when client is AsyncHttpClient: + client.bodyStream.complete() + else: + client.bodyStream.setPosition(0) -proc parseResponse(client: AsyncHttpClient, - getBody: bool): Future[Response] {.async.} = + # If the server will close our connection, then no matter the method of + # reading the body, we need to close our socket. + if headers.getOrDefault"Connection" == "close": + client.close() + +proc parseResponse(client: HttpClient | AsyncHttpClient, + getBody: bool): Future[Response | AsyncResponse] + {.multisync.} = + new result var parsedStatus = false var linei = 0 var fullyRead = false + var lastHeaderName = "" var line = "" - result.headers = newStringTable(modeCaseInsensitive) + result.headers = newHttpHeaders() while true: linei = 0 - line = await client.socket.recvLine() - if line == "": break # We've been disconnected. - if line == "\c\L": + when client is HttpClient: + line = await client.socket.recvLine(client.timeout) + else: + line = await client.socket.recvLine() + if line == "": + # We've been disconnected. + client.close() + break + if line == httpNewLine: fullyRead = true break if not parsedStatus: # Parse HTTP version info and status code. var le = skipIgnoreCase(line, "HTTP/", linei) if le <= 0: - httpError("invalid http version, " & line.repr) + httpError("invalid http version, `" & line & "`") inc(linei, le) le = skipIgnoreCase(line, "1.1", linei) if le > 0: result.version = "1.1" @@ -739,152 +891,470 @@ proc parseResponse(client: AsyncHttpClient, parsedStatus = true else: # Parse headers - var name = "" - var le = parseUntil(line, name, ':', linei) - if le <= 0: httpError("invalid headers") - inc(linei, le) - if line[linei] != ':': httpError("invalid headers") - inc(linei) # Skip : + # There's at least one char because empty lines are handled above (with client.close) + if line[0] in {' ', '\t'}: + # Check if it's a multiline header value, if so, append to the header we're currently parsing + # This works because a line with a header must start with the header name without any leading space + # See https://datatracker.ietf.org/doc/html/rfc7230, section 3.2 and 3.2.4 + # Multiline headers are deprecated in the spec, but it's better to parse them than crash + if lastHeaderName == "": + # Some extra unparsable lines in the HTTP output - we ignore them + discard + else: + result.headers.table[result.headers.toCaseInsensitive(lastHeaderName)][^1].add "\n" & line + else: + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("Invalid headers - received empty header name") + if line.len == le: httpError("Invalid headers - no colon after header name") + inc(linei, le) # Skip the parsed header name + inc(linei) # Skip : + # If we want to be HTTP spec compliant later, error on linei == line.len (for empty header value) + lastHeaderName = name # Remember the header name for the possible multi-line header + result.headers.add(name, line[linei .. ^1].strip()) + if result.headers.len > headerLimit: + httpError("too many headers") - result.headers[name] = line[linei.. ^1].strip() if not fullyRead: httpError("Connection was closed before full request has been made") - if getBody: - result.body = await parseBody(client, result.headers) + + when client is HttpClient: + result.bodyStream = newStringStream() + else: + result.bodyStream = newFutureStream[string]("parseResponse") + + if getBody and result.code != Http204: + client.bodyStream = result.bodyStream + when client is HttpClient: + parseBody(client, result.headers, result.version) + else: + assert(client.parseBodyFut.isNil or client.parseBodyFut.finished) + # do not wait here for the body request to complete + client.parseBodyFut = parseBody(client, result.headers, result.version) + client.parseBodyFut.addCallback do(): + if client.parseBodyFut.failed: + client.bodyStream.fail(client.parseBodyFut.error) else: - result.body = "" + when client is AsyncHttpClient: + result.bodyStream.complete() -proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} = +proc newConnection(client: HttpClient | AsyncHttpClient, + url: Uri) {.multisync.} = if client.currentURL.hostname != url.hostname or - client.currentURL.scheme != url.scheme: - if client.connected: client.close() - client.socket = newAsyncSocket() + client.currentURL.scheme != url.scheme or + client.currentURL.port != url.port or + (not client.connected): + # Connect to proxy if specified + let connectionUrl = + if client.proxy.isNil: url else: client.proxy.url + + let isSsl = connectionUrl.scheme.toLowerAscii() == "https" + + if isSsl and not defined(ssl): + raise newException(HttpRequestError, + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") + + if client.connected: + client.close() + client.connected = false # TODO: I should be able to write 'net.Port' here... let port = - if url.port == "": - if url.scheme.toLower() == "https": + if connectionUrl.port == "": + if isSsl: nativesockets.Port(443) else: nativesockets.Port(80) - else: nativesockets.Port(url.port.parseInt) + else: nativesockets.Port(connectionUrl.port.parseInt) + + when client is HttpClient: + client.socket = await net.dial(connectionUrl.hostname, port) + elif client is AsyncHttpClient: + client.socket = await asyncnet.dial(connectionUrl.hostname, port) + else: {.fatal: "Unsupported client type".} - if url.scheme.toLower() == "https": + when defined(ssl): + if isSsl: + try: + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, connectionUrl.hostname) + except: + client.socket.close() + raise getCurrentException() + + # If need to CONNECT through proxy + if url.scheme == "https" and not client.proxy.isNil: when defined(ssl): - client.sslContext.wrapSocket(client.socket) + # Pass only host:port for CONNECT + var connectUrl = initUri() + connectUrl.hostname = url.hostname + connectUrl.port = if url.port != "": url.port else: "443" + + let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, + newHttpHeaders(), client.proxy) + await client.socket.send(proxyHeaderString) + let proxyResp = await parseResponse(client, false) + + if not proxyResp.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, url.hostname) else: raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") - await client.socket.connect(url.hostname, port) + # May be connected through proxy but remember actual URL being accessed client.currentURL = url client.connected = true -proc request*(client: AsyncHttpClient, url: string, httpMethod: string, - body = ""): Future[Response] {.async.} = - ## Connects to the hostname specified by the URL and performs a request - ## using the custom method string specified by ``httpMethod``. - ## - ## Connection will kept alive. Further requests on the same ``client`` to - ## the same hostname will not require a new connection to be made. The - ## connection can be closed by using the ``close`` procedure. - ## - ## The returned future will complete once the request is completed. - let r = parseUri(url) - await newConnection(client, r) +proc readFileSizes(client: HttpClient | AsyncHttpClient, + multipart: MultipartData) {.multisync.} = + for entry in multipart.content.mitems(): + if not entry.isFile: continue + if not entry.isStream: + entry.fileSize = entry.content.len + continue + + # TODO: look into making getFileSize work with async + let fileSize = getFileSize(entry.content) + entry.fileSize = fileSize + +proc format(entry: MultipartEntry, boundary: string): string = + result = "--" & boundary & httpNewLine + result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"" & httpNewLine) + result.add("Content-Type: " & entry.contentType & httpNewLine) + else: + result.add(httpNewLine & httpNewLine & entry.content) - if not client.headers.hasKey("user-agent") and client.userAgent != "": - client.headers["User-Agent"] = client.userAgent +proc format(client: HttpClient | AsyncHttpClient, + multipart: MultipartData): Future[seq[string]] {.multisync.} = + let bound = getBoundary(multipart) + client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound - var headers = generateHeaders(r, $httpMethod, client.headers, body) + await client.readFileSizes(multipart) - await client.socket.send(headers) - if body != "": - await client.socket.send(body) + var length: int64 + for entry in multipart.content: + result.add(format(entry, bound) & httpNewLine) + if entry.isFile: + length += entry.fileSize + httpNewLine.len - result = await parseResponse(client, httpMethod != "httpHEAD") + result.add "--" & bound & "--" & httpNewLine -proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET, - body = ""): Future[Response] = - ## Connects to the hostname specified by the URL and performs a request - ## using the method specified. - ## - ## Connection will kept alive. Further requests on the same ``client`` to - ## the same hostname will not require a new connection to be made. The - ## connection can be closed by using the ``close`` procedure. - ## - ## The returned future will complete once the request is completed. - result = request(client, url, $httpMethod, body) + for s in result: length += s.len + client.headers["Content-Length"] = $length -proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} = - ## Connects to the hostname specified by the URL and performs a GET request. - ## - ## This procedure will follow redirects up to a maximum number of redirects - ## specified in ``newAsyncHttpClient``. - result = await client.request(url, httpGET) - var lastURL = url - for i in 1..client.maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - result = await client.request(redirectTo, httpGET) - lastUrl = redirectTo +proc override(fallback, override: HttpHeaders): HttpHeaders = + # Right-biased map union for `HttpHeaders` -proc post*(client: AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response] {.async.} = - ## Connects to the hostname specified by the URL and performs a POST request. - ## - ## This procedure will follow redirects up to a maximum number of redirects - ## specified in ``newAsyncHttpClient``. - let (mpHeader, mpBody) = format(multipart) + result = newHttpHeaders() + # Copy by value + result.table[] = fallback.table[] - template withNewLine(x): expr = - if x.len > 0 and not x.endsWith("\c\L"): - x & "\c\L" - else: - x - var xb = mpBody.withNewLine() & body - if multipart != nil: - client.headers["Content-Type"] = mpHeader.split(": ")[1] - client.headers["Content-Length"] = $len(xb) + if override.isNil: + # Return the copy of fallback so it does not get modified + return result - result = await client.request(url, httpPOST, xb) + for k, vs in override.table: + result[k] = vs -when not defined(testing) and isMainModule: - when true: - # Async - proc main() {.async.} = - var client = newAsyncHttpClient() - var resp = await client.request("http://picheta.me") +proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, + httpMethod: HttpMethod, body = "", headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + # Helper that actually makes the request. Does not handle redirects. + if url.scheme == "": + raise newException(ValueError, "No uri scheme supplied.") - echo("Got response: ", resp.status) - echo("Body:\n") - echo(resp.body) + when client is AsyncHttpClient: + if not client.parseBodyFut.isNil: + # let the current operation finish before making another request + await client.parseBodyFut + client.parseBodyFut = nil - resp = await client.request("http://picheta.me/asfas.html") - echo("Got response: ", resp.status) + await newConnection(client, url) - resp = await client.request("http://picheta.me/aboutme.html") - echo("Got response: ", resp.status) + var newHeaders: HttpHeaders - resp = await client.request("http://nim-lang.org/") - echo("Got response: ", resp.status) + var data: seq[string] + if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 + data = await client.format(multipart) + newHeaders = client.headers.override(headers) + else: + newHeaders = client.headers.override(headers) + # Only change headers if they have not been specified already + if not newHeaders.hasKey("Content-Length"): + if body.len != 0: + newHeaders["Content-Length"] = $body.len + elif httpMethod notin {HttpGet, HttpHead}: + newHeaders["Content-Length"] = "0" + + if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0: + newHeaders["User-Agent"] = client.userAgent + + let headerString = generateHeaders(url, httpMethod, newHeaders, + client.proxy) + await client.socket.send(headerString) + + if data.len > 0: + var buffer: string + for i, entry in multipart.content: + buffer.add data[i] + if not entry.isFile: continue + if buffer.len > 0: + await client.socket.send(buffer) + buffer.setLen(0) + if entry.isStream: + await client.socket.sendFile(entry) + else: + await client.socket.send(entry.content) + buffer.add httpNewLine + # send the rest and the last boundary + await client.socket.send(buffer & data[^1]) + elif body.len > 0: + await client.socket.send(body) - resp = await client.request("http://nim-lang.org/download.html") - echo("Got response: ", resp.status) + let getBody = httpMethod notin {HttpHead, HttpConnect} and + client.getBody + result = await parseResponse(client, getBody) - waitFor main() +proc request*(client: HttpClient | AsyncHttpClient, url: Uri | string, + httpMethod: HttpMethod | string = HttpGet, body = "", + headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a request + ## using the custom method string specified by `httpMethod`. + ## + ## Connection will be kept alive. Further requests on the same `client` to + ## the same hostname will not require a new connection to be made. The + ## connection can be closed by using the `close` procedure. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in `client.maxRedirects`. + ## + ## You need to make sure that the `url` doesn't contain any newline + ## characters. Failing to do so will raise `AssertionDefect`. + ## + ## `headers` are HTTP headers that override the `client.headers` for + ## this specific request only and will not be persisted. + ## + ## **Deprecated since v1.5**: use HttpMethod enum instead; string parameter httpMethod is deprecated + when url is string: + doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters") + let url = parseUri(url) + + when httpMethod is string: + {.warning: + "Deprecated since v1.5; use HttpMethod enum instead; string parameter httpMethod is deprecated".} + let httpMethod = case httpMethod + of "HEAD": + HttpHead + of "GET": + HttpGet + of "POST": + HttpPost + of "PUT": + HttpPut + of "DELETE": + HttpDelete + of "TRACE": + HttpTrace + of "OPTIONS": + HttpOptions + of "CONNECT": + HttpConnect + of "PATCH": + HttpPatch + else: + raise newException(ValueError, "Invalid HTTP method name: " & httpMethod) - else: - #downloadFile("http://force7.de/nim/index.html", "nimindex.html") - #downloadFile("http://www.httpwatch.com/", "ChunkTest.html") - #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com", - # "validator.html") + result = await client.requestAux(url, httpMethod, body, headers, multipart) + + var lastURL = url + for i in 1..client.maxRedirects: + let statusCode = result.code + + if statusCode notin {Http301, Http302, Http303, Http307, Http308}: + break - #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& - # charset=%28detect+automatically%29&doctype=Inline&group=0") + let redirectTo = getNewLocation(lastURL, result.headers) + var redirectMethod: HttpMethod + var redirectBody: string + # For more informations about the redirect methods see: + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections + case statusCode + of Http301, Http302, Http303: + # The method is changed to GET unless it is GET or HEAD (RFC2616) + if httpMethod notin {HttpGet, HttpHead}: + redirectMethod = HttpGet + else: + redirectMethod = httpMethod + # The body is stripped away + redirectBody = "" + # Delete any header value associated with the body + if not headers.isNil(): + headers.del("Content-Length") + headers.del("Content-Type") + headers.del("Transfer-Encoding") + of Http307, Http308: + # The method and the body are unchanged + redirectMethod = httpMethod + redirectBody = body + else: + # Unreachable + doAssert(false) + + # Check if the redirection is to the same domain or a sub-domain (foo.com + # -> sub.foo.com) + if redirectTo.hostname != lastURL.hostname and + not redirectTo.hostname.endsWith("." & lastURL.hostname): + # Perform some cleanup of the header values + if headers != nil: + # Delete the Host header + headers.del("Host") + # Do not send any sensitive info to a unknown host + headers.del("Authorization") + + result = await client.requestAux(redirectTo, redirectMethod, redirectBody, + headers, multipart) + lastURL = redirectTo + +proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} = + ## Returns the content of a response as a string. + ## + ## A `HttpRequestError` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status.move) + else: + return await resp.bodyStream.readAll() - var data = newMultipartData() - data["output"] = "soap12" - data["uploaded_file"] = ("test.html", "text/html", - "<html><head></head><body><p>test</p></body></html>") +proc head*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a HEAD request. + ## + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpHead) - echo postContent("http://validator.w3.org/check", multipart=data) +proc get*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a GET request. + ## + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpGet) + +proc getContent*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a GET request. + let resp = await get(client, url) + return await responseContent(resp) + +proc delete*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a DELETE request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpDelete) + +proc deleteContent*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a DELETE request. + let resp = await delete(client, url) + return await responseContent(resp) + +proc post*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a POST request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPost, body, multipart=multipart) + +proc postContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a POST request. + let resp = await post(client, url, body, multipart) + return await responseContent(resp) + +proc put*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a PUT request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPut, body, multipart=multipart) + +proc putContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL andreturns the content of a PUT request. + let resp = await put(client, url, body, multipart) + return await responseContent(resp) + +proc patch*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a PATCH request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPatch, body, multipart=multipart) + +proc patchContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a PATCH request. + let resp = await patch(client, url, body, multipart) + return await responseContent(resp) + +proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) = + ## Downloads `url` and saves it to `filename`. + client.getBody = false + defer: + client.getBody = true + let resp = client.get(url) + + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + + client.bodyStream = newFileStream(filename, fmWrite) + if client.bodyStream.isNil: + fileError("Unable to open file") + parseBody(client, resp.headers, resp.version) + client.bodyStream.close() + +proc downloadFileEx(client: AsyncHttpClient, + url: Uri | string, filename: string): Future[void] {.async.} = + ## Downloads `url` and saves it to `filename`. + client.getBody = false + let resp = await client.get(url) + + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + + client.bodyStream = newFutureStream[string]("downloadFile") + var file = openAsync(filename, fmWrite) + defer: file.close() + # Let `parseBody` write response data into client.bodyStream in the + # background. + let parseBodyFut = parseBody(client, resp.headers, resp.version) + parseBodyFut.addCallback do(): + if parseBodyFut.failed: + client.bodyStream.fail(parseBodyFut.error) + # The `writeFromStream` proc will complete once all the data in the + # `bodyStream` has been written to the file. + await file.writeFromStream(client.bodyStream) + +proc downloadFile*(client: AsyncHttpClient, url: Uri | string, + filename: string): Future[void] = + result = newFuture[void]("downloadFile") + try: + result = downloadFileEx(client, url, filename) + except Exception as exc: + result.fail(exc) + finally: + result.addCallback( + proc () = client.getBody = true + ) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim new file mode 100644 index 000000000..5ccab379c --- /dev/null +++ b/lib/pure/httpcore.nim @@ -0,0 +1,368 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Contains functionality shared between the `httpclient` and +## `asynchttpserver` modules. +## +## Unstable API. +import std/private/since +import std/[tables, strutils, parseutils] + +type + HttpHeaders* = ref object + table*: TableRef[string, seq[string]] + isTitleCase: bool + + HttpHeaderValues* = distinct seq[string] + + # The range starts at '0' so that we don't have to explicitly initialise + # it. See: http://irclogs.nim-lang.org/19-09-2016.html#19:48:27 for context. + HttpCode* = distinct range[0 .. 599] + + HttpVersion* = enum + HttpVer11, + HttpVer10 + + HttpMethod* = enum ## the requested HttpMethod + HttpHead = "HEAD" ## Asks for the response identical to the one that + ## would correspond to a GET request, but without + ## the response body. + HttpGet = "GET" ## Retrieves the specified resource. + HttpPost = "POST" ## Submits data to be processed to the identified + ## resource. The data is included in the body of + ## the request. + HttpPut = "PUT" ## Uploads a representation of the specified + ## resource. + HttpDelete = "DELETE" ## Deletes the specified resource. + HttpTrace = "TRACE" ## Echoes back the received request, so that a + ## client + ## can see what intermediate servers are adding or + ## changing in the request. + HttpOptions = "OPTIONS" ## Returns the HTTP methods that the server + ## supports for specified address. + HttpConnect = "CONNECT" ## Converts the request connection to a transparent + ## TCP/IP tunnel, usually used for proxies. + HttpPatch = "PATCH" ## Applies partial modifications to a resource. + + +const + Http100* = HttpCode(100) + Http101* = HttpCode(101) + Http102* = HttpCode(102) ## https://tools.ietf.org/html/rfc2518.html WebDAV + Http103* = HttpCode(103) ## https://tools.ietf.org/html/rfc8297.html Early hints + Http200* = HttpCode(200) + Http201* = HttpCode(201) + Http202* = HttpCode(202) + Http203* = HttpCode(203) + Http204* = HttpCode(204) + Http205* = HttpCode(205) + Http206* = HttpCode(206) + Http207* = HttpCode(207) ## https://tools.ietf.org/html/rfc4918.html WebDAV + Http208* = HttpCode(208) ## https://tools.ietf.org/html/rfc5842.html WebDAV, Section 7.1 + Http226* = HttpCode(226) ## https://tools.ietf.org/html/rfc3229.html Delta encoding, Section 10.4.1 + Http300* = HttpCode(300) + Http301* = HttpCode(301) + Http302* = HttpCode(302) + Http303* = HttpCode(303) + Http304* = HttpCode(304) + Http305* = HttpCode(305) + Http307* = HttpCode(307) + Http308* = HttpCode(308) + Http400* = HttpCode(400) + Http401* = HttpCode(401) + Http402* = HttpCode(402) ## https://tools.ietf.org/html/rfc7231.html Payment required, Section 6.5.2 + Http403* = HttpCode(403) + Http404* = HttpCode(404) + Http405* = HttpCode(405) + Http406* = HttpCode(406) + Http407* = HttpCode(407) + Http408* = HttpCode(408) + Http409* = HttpCode(409) + Http410* = HttpCode(410) + Http411* = HttpCode(411) + Http412* = HttpCode(412) + Http413* = HttpCode(413) + Http414* = HttpCode(414) + Http415* = HttpCode(415) + Http416* = HttpCode(416) + Http417* = HttpCode(417) + Http418* = HttpCode(418) + Http421* = HttpCode(421) + Http422* = HttpCode(422) + Http423* = HttpCode(423) ## https://tools.ietf.org/html/rfc4918.html WebDAV, Section 11.3 + Http424* = HttpCode(424) ## https://tools.ietf.org/html/rfc4918.html WebDAV, Section 11.3 + Http425* = HttpCode(425) ## https://tools.ietf.org/html/rfc8470.html Early data + Http426* = HttpCode(426) + Http428* = HttpCode(428) + Http429* = HttpCode(429) + Http431* = HttpCode(431) + Http451* = HttpCode(451) + Http500* = HttpCode(500) + Http501* = HttpCode(501) + Http502* = HttpCode(502) + Http503* = HttpCode(503) + Http504* = HttpCode(504) + Http505* = HttpCode(505) + Http506* = HttpCode(506) ## https://tools.ietf.org/html/rfc2295.html Content negotiation, Section 8.1 + Http507* = HttpCode(507) ## https://tools.ietf.org/html/rfc4918.html WebDAV, Section 11.5 + Http508* = HttpCode(508) ## https://tools.ietf.org/html/rfc5842.html WebDAV, Section 7.2 + Http510* = HttpCode(510) ## https://tools.ietf.org/html/rfc2774.html Extension framework, Section 7 + Http511* = HttpCode(511) ## https://tools.ietf.org/html/rfc6585.html Additional status code, Section 6 + + +const httpNewLine* = "\c\L" +const headerLimit* = 10_000 + +func toTitleCase(s: string): string = + result = newString(len(s)) + var upper = true + for i in 0..len(s) - 1: + result[i] = if upper: toUpperAscii(s[i]) else: toLowerAscii(s[i]) + upper = s[i] == '-' + +func toCaseInsensitive*(headers: HttpHeaders, s: string): string {.inline.} = + ## For internal usage only. Do not use. + return if headers.isTitleCase: toTitleCase(s) else: toLowerAscii(s) + +func newHttpHeaders*(titleCase=false): HttpHeaders = + ## Returns a new `HttpHeaders` object. if `titleCase` is set to true, + ## headers are passed to the server in title case (e.g. "Content-Length") + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) + +func newHttpHeaders*(keyValuePairs: + openArray[tuple[key: string, val: string]], titleCase=false): HttpHeaders = + ## Returns a new `HttpHeaders` object from an array. if `titleCase` is set to true, + ## headers are passed to the server in title case (e.g. "Content-Length") + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) + + for pair in keyValuePairs: + let key = result.toCaseInsensitive(pair.key) + {.cast(noSideEffect).}: + if key in result.table: + result.table[key].add(pair.val) + else: + result.table[key] = @[pair.val] + +func `$`*(headers: HttpHeaders): string {.inline.} = + $headers.table + +proc clear*(headers: HttpHeaders) {.inline.} = + headers.table.clear() + +func `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = + ## Returns the values associated with the given `key`. If the returned + ## values are passed to a procedure expecting a `string`, the first + ## value is automatically picked. If there are + ## no values associated with the key, an exception is raised. + ## + ## To access multiple values of a key, use the overloaded `[]` below or + ## to get all of them access the `table` field directly. + {.cast(noSideEffect).}: + let tmp = headers.table[headers.toCaseInsensitive(key)] + return HttpHeaderValues(tmp) + +converter toString*(values: HttpHeaderValues): string = + return seq[string](values)[0] + +func `[]`*(headers: HttpHeaders, key: string, i: int): string = + ## Returns the `i`'th value associated with the given key. If there are + ## no values associated with the key or the `i`'th value doesn't exist, + ## an exception is raised. + {.cast(noSideEffect).}: + return headers.table[headers.toCaseInsensitive(key)][i] + +proc `[]=`*(headers: HttpHeaders, key, value: string) = + ## Sets the header entries associated with `key` to the specified value. + ## Replaces any existing values. + headers.table[headers.toCaseInsensitive(key)] = @[value] + +proc `[]=`*(headers: HttpHeaders, key: string, value: seq[string]) = + ## Sets the header entries associated with `key` to the specified list of + ## values. Replaces any existing values. If `value` is empty, + ## deletes the header entries associated with `key`. + if value.len > 0: + headers.table[headers.toCaseInsensitive(key)] = value + else: + headers.table.del(headers.toCaseInsensitive(key)) + +proc add*(headers: HttpHeaders, key, value: string) = + ## Adds the specified value to the specified key. Appends to any existing + ## values associated with the key. + if not headers.table.hasKey(headers.toCaseInsensitive(key)): + headers.table[headers.toCaseInsensitive(key)] = @[value] + else: + headers.table[headers.toCaseInsensitive(key)].add(value) + +proc del*(headers: HttpHeaders, key: string) = + ## Deletes the header entries associated with `key` + headers.table.del(headers.toCaseInsensitive(key)) + +iterator pairs*(headers: HttpHeaders): tuple[key, value: string] = + ## Yields each key, value pair. + for k, v in headers.table: + for value in v: + yield (k, value) + +func contains*(values: HttpHeaderValues, value: string): bool = + ## Determines if `value` is one of the values inside `values`. Comparison + ## is performed without case sensitivity. + for val in seq[string](values): + if val.toLowerAscii == value.toLowerAscii: return true + +func hasKey*(headers: HttpHeaders, key: string): bool = + return headers.table.hasKey(headers.toCaseInsensitive(key)) + +func getOrDefault*(headers: HttpHeaders, key: string, + default = @[""].HttpHeaderValues): HttpHeaderValues = + ## Returns the values associated with the given `key`. If there are no + ## values associated with the key, then `default` is returned. + if headers.hasKey(key): + return headers[key] + else: + return default + +func len*(headers: HttpHeaders): int {.inline.} = headers.table.len + +func parseList(line: string, list: var seq[string], start: int): int = + var i = 0 + var current = "" + while start+i < line.len and line[start + i] notin {'\c', '\l'}: + i += line.skipWhitespace(start + i) + i += line.parseUntil(current, {'\c', '\l', ','}, start + i) + list.add(move current) # implicit current.setLen(0) + if start+i < line.len and line[start + i] == ',': + i.inc # Skip , + +func parseHeader*(line: string): tuple[key: string, value: seq[string]] = + ## Parses a single raw header HTTP line into key value pairs. + ## + ## Used by `asynchttpserver` and `httpclient` internally and should not + ## be used by you. + result.value = @[] + var i = 0 + i = line.parseUntil(result.key, ':') + inc(i) # skip : + if i < len(line): + if cmpIgnoreCase(result.key, "cookie") == 0: + i += line.skipWhitespace(i) + result.value.add line.substr(i) + else: + i += parseList(line, result.value, i) + elif result.key.len > 0: + result.value = @[""] + else: + result.value = @[] + +func `==`*(protocol: tuple[orig: string, major, minor: int], + ver: HttpVersion): bool = + let major = + case ver + of HttpVer11, HttpVer10: 1 + let minor = + case ver + of HttpVer11: 1 + of HttpVer10: 0 + result = protocol.major == major and protocol.minor == minor + +func contains*(methods: set[HttpMethod], x: string): bool = + return parseEnum[HttpMethod](x) in methods + +func `$`*(code: HttpCode): string = + ## Converts the specified `HttpCode` into a HTTP status. + runnableExamples: + doAssert($Http404 == "404 Not Found") + case code.int + of 100: "100 Continue" + of 101: "101 Switching Protocols" + of 102: "102 Processing" + of 103: "103 Early Hints" + of 200: "200 OK" + of 201: "201 Created" + of 202: "202 Accepted" + of 203: "203 Non-Authoritative Information" + of 204: "204 No Content" + of 205: "205 Reset Content" + of 206: "206 Partial Content" + of 207: "207 Multi-Status" + of 208: "208 Already Reported" + of 226: "226 IM Used" + of 300: "300 Multiple Choices" + of 301: "301 Moved Permanently" + of 302: "302 Found" + of 303: "303 See Other" + of 304: "304 Not Modified" + of 305: "305 Use Proxy" + of 307: "307 Temporary Redirect" + of 308: "308 Permanent Redirect" + of 400: "400 Bad Request" + of 401: "401 Unauthorized" + of 402: "402 Payment Required" + of 403: "403 Forbidden" + of 404: "404 Not Found" + of 405: "405 Method Not Allowed" + of 406: "406 Not Acceptable" + of 407: "407 Proxy Authentication Required" + of 408: "408 Request Timeout" + of 409: "409 Conflict" + of 410: "410 Gone" + of 411: "411 Length Required" + of 412: "412 Precondition Failed" + of 413: "413 Request Entity Too Large" + of 414: "414 Request-URI Too Long" + of 415: "415 Unsupported Media Type" + of 416: "416 Requested Range Not Satisfiable" + of 417: "417 Expectation Failed" + of 418: "418 I'm a teapot" + of 421: "421 Misdirected Request" + of 422: "422 Unprocessable Entity" + of 423: "423 Locked" + of 424: "424 Failed Dependency" + of 425: "425 Too Early" + of 426: "426 Upgrade Required" + of 428: "428 Precondition Required" + of 429: "429 Too Many Requests" + of 431: "431 Request Header Fields Too Large" + of 451: "451 Unavailable For Legal Reasons" + of 500: "500 Internal Server Error" + of 501: "501 Not Implemented" + of 502: "502 Bad Gateway" + of 503: "503 Service Unavailable" + of 504: "504 Gateway Timeout" + of 505: "505 HTTP Version Not Supported" + of 506: "506 Variant Also Negotiates" + of 507: "507 Insufficient Storage" + of 508: "508 Loop Detected" + of 510: "510 Not Extended" + of 511: "511 Network Authentication Required" + else: $(int(code)) + +func `==`*(a, b: HttpCode): bool {.borrow.} + +func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = + ## Determines whether `code` is a 1xx HTTP status code. + runnableExamples: + doAssert is1xx(HttpCode(103)) + + code.int in 100 .. 199 + +func is2xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 2xx HTTP status code. + code.int in 200 .. 299 + +func is3xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 3xx HTTP status code. + code.int in 300 .. 399 + +func is4xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 4xx HTTP status code. + code.int in 400 .. 499 + +func is5xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 5xx HTTP status code. + code.int in 500 .. 599 diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim deleted file mode 100644 index 71ba04991..000000000 --- a/lib/pure/httpserver.nim +++ /dev/null @@ -1,534 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf, Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a simple HTTP-Server. -## -## Example: -## -## .. code-block:: nim -## import strutils, sockets, httpserver -## -## var counter = 0 -## proc handleRequest(client: Socket, path, query: string): bool {.procvar.} = -## inc(counter) -## client.send("Hello for the $#th time." % $counter & wwwNL) -## return false # do not stop processing -## -## run(handleRequest, Port(80)) -## - -import parseutils, strutils, os, osproc, strtabs, streams, sockets, asyncio - -const - wwwNL* = "\r\L" - ServerSig = "Server: httpserver.nim/1.0.0" & wwwNL - -# --------------- output messages -------------------------------------------- - -proc sendTextContentType(client: Socket) = - send(client, "Content-type: text/html" & wwwNL) - send(client, wwwNL) - -proc sendStatus(client: Socket, status: string) = - send(client, "HTTP/1.1 " & status & wwwNL) - -proc badRequest(client: Socket) = - # Inform the client that a request it has made has a problem. - send(client, "HTTP/1.1 400 Bad Request" & wwwNL) - sendTextContentType(client) - send(client, "<p>Your browser sent a bad request, " & - "such as a POST without a Content-Length.</p>" & wwwNL) - -when false: - proc cannotExec(client: Socket) = - send(client, "HTTP/1.1 500 Internal Server Error" & wwwNL) - sendTextContentType(client) - send(client, "<P>Error prohibited CGI execution." & wwwNL) - -proc headers(client: Socket, filename: string) = - # XXX could use filename to determine file type - send(client, "HTTP/1.1 200 OK" & wwwNL) - send(client, ServerSig) - sendTextContentType(client) - -proc notFound(client: Socket) = - send(client, "HTTP/1.1 404 NOT FOUND" & wwwNL) - send(client, ServerSig) - sendTextContentType(client) - send(client, "<html><title>Not Found</title>" & wwwNL) - send(client, "<body><p>The server could not fulfill" & wwwNL) - send(client, "your request because the resource specified" & wwwNL) - send(client, "is unavailable or nonexistent.</p>" & wwwNL) - send(client, "</body></html>" & wwwNL) - -proc unimplemented(client: Socket) = - send(client, "HTTP/1.1 501 Method Not Implemented" & wwwNL) - send(client, ServerSig) - sendTextContentType(client) - send(client, "<html><head><title>Method Not Implemented" & - "</title></head>" & - "<body><p>HTTP request method not supported.</p>" & - "</body></HTML>" & wwwNL) - -# ----------------- file serving --------------------------------------------- - -when false: - proc discardHeaders(client: Socket) = skip(client) - -proc serveFile*(client: Socket, filename: string) = - ## serves a file to the client. - var f: File - if open(f, filename): - headers(client, filename) - const bufSize = 8000 # != 8K might be good for memory manager - var buf = alloc(bufsize) - while true: - var bytesread = readBuffer(f, buf, bufsize) - if bytesread > 0: - var byteswritten = send(client, buf, bytesread) - if bytesread != bytesWritten: - dealloc(buf) - close(f) - raiseOSError(osLastError()) - if bytesread != bufSize: break - dealloc(buf) - close(f) - else: - notFound(client) - -# ------------------ CGI execution ------------------------------------------- -when false: - # TODO: Fix this, or get rid of it. - type - RequestMethod = enum reqGet, reqPost - {.deprecated: [TRequestMethod: RequestMethod].} - - proc executeCgi(client: Socket, path, query: string, meth: RequestMethod) = - var env = newStringTable(modeCaseInsensitive) - var contentLength = -1 - case meth - of reqGet: - discardHeaders(client) - - env["REQUEST_METHOD"] = "GET" - env["QUERY_STRING"] = query - of reqPost: - var buf = TaintedString"" - var dataAvail = false - while dataAvail: - dataAvail = recvLine(client, buf) # TODO: This is incorrect. - var L = toLower(buf.string) - if L.startsWith("content-length:"): - var i = len("content-length:") - while L[i] in Whitespace: inc(i) - contentLength = parseInt(substr(L, i)) - - if contentLength < 0: - badRequest(client) - return - - env["REQUEST_METHOD"] = "POST" - env["CONTENT_LENGTH"] = $contentLength - - send(client, "HTTP/1.0 200 OK" & wwwNL) - - var process = startProcess(command=path, env=env) - if meth == reqPost: - # get from client and post to CGI program: - var buf = alloc(contentLength) - if recv(client, buf, contentLength) != contentLength: - dealloc(buf) - raiseOSError() - var inp = process.inputStream - inp.writeData(buf, contentLength) - dealloc(buf) - - var outp = process.outputStream - var line = newStringOfCap(120).TaintedString - while true: - if outp.readLine(line): - send(client, line.string) - send(client, wwwNL) - elif not running(process): break - - # --------------- Server Setup ----------------------------------------------- - - proc acceptRequest(client: Socket) = - var cgi = false - var query = "" - var buf = TaintedString"" - discard recvLine(client, buf) - var path = "" - var data = buf.string.split() - var meth = reqGet - - var q = find(data[1], '?') - - # extract path - if q >= 0: - # strip "?..." from path, this may be found in both POST and GET - path = "." & data[1].substr(0, q-1) - else: - path = "." & data[1] - # path starts with "/", by adding "." in front of it we serve files from cwd - - if cmpIgnoreCase(data[0], "GET") == 0: - if q >= 0: - cgi = true - query = data[1].substr(q+1) - elif cmpIgnoreCase(data[0], "POST") == 0: - cgi = true - meth = reqPost - else: - unimplemented(client) - - if path[path.len-1] == '/' or existsDir(path): - path = path / "index.html" - - if not existsFile(path): - discardHeaders(client) - notFound(client) - else: - when defined(Windows): - var ext = splitFile(path).ext.toLower - if ext == ".exe" or ext == ".cgi": - # XXX: extract interpreter information here? - cgi = true - else: - if {fpUserExec, fpGroupExec, fpOthersExec} * path.getFilePermissions != {}: - cgi = true - if not cgi: - serveFile(client, path) - else: - executeCgi(client, path, query, meth) - -type - Server* = object of RootObj ## contains the current server state - socket: Socket - port: Port - client*: Socket ## the socket to write the file data to - reqMethod*: string ## Request method. GET or POST. - path*, query*: string ## path and query the client requested - headers*: StringTableRef ## headers with which the client made the request - body*: string ## only set with POST requests - ip*: string ## ip address of the requesting client - - PAsyncHTTPServer* = ref AsyncHTTPServer - AsyncHTTPServer = object of Server - asyncSocket: AsyncSocket -{.deprecated: [TAsyncHTTPServer: AsyncHTTPServer, TServer: Server].} - -proc open*(s: var Server, port = Port(80), reuseAddr = false) = - ## creates a new server at port `port`. If ``port == 0`` a free port is - ## acquired that can be accessed later by the ``port`` proc. - s.socket = socket(AF_INET) - if s.socket == invalidSocket: raiseOSError(osLastError()) - if reuseAddr: - s.socket.setSockOpt(OptReuseAddr, true) - bindAddr(s.socket, port) - listen(s.socket) - - if port == Port(0): - s.port = getSockName(s.socket) - else: - s.port = port - s.client = invalidSocket - s.reqMethod = "" - s.body = "" - s.path = "" - s.query = "" - s.headers = {:}.newStringTable() - -proc port*(s: var Server): Port = - ## get the port number the server has acquired. - result = s.port - -proc next*(s: var Server) = - ## proceed to the first/next request. - var client: Socket - new(client) - var ip: string - acceptAddr(s.socket, client, ip) - s.client = client - s.ip = ip - s.headers = newStringTable(modeCaseInsensitive) - #headers(s.client, "") - var data = "" - s.client.readLine(data) - if data == "": - # Socket disconnected - s.client.close() - next(s) - return - var header = "" - while true: - s.client.readLine(header) - if header == "\c\L": break - if header != "": - var i = 0 - var key = "" - var value = "" - i = header.parseUntil(key, ':') - inc(i) # skip : - i += header.skipWhiteSpace(i) - i += header.parseUntil(value, {'\c', '\L'}, i) - s.headers[key] = value - else: - s.client.close() - next(s) - return - - var i = skipWhitespace(data) - if skipIgnoreCase(data, "GET") > 0: - s.reqMethod = "GET" - inc(i, 3) - elif skipIgnoreCase(data, "POST") > 0: - s.reqMethod = "POST" - inc(i, 4) - else: - unimplemented(s.client) - s.client.close() - next(s) - return - - if s.reqMethod == "POST": - # Check for Expect header - if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLower == "100-continue": - s.client.sendStatus("100 Continue") - else: - s.client.sendStatus("417 Expectation Failed") - - # Read the body - # - Check for Content-length header - if s.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(s.headers["Content-Length"], contentLength) == 0: - badRequest(s.client) - s.client.close() - next(s) - return - else: - var totalRead = 0 - var totalBody = "" - while totalRead < contentLength: - var chunkSize = 8000 - if (contentLength - totalRead) < 8000: - chunkSize = (contentLength - totalRead) - var bodyData = newString(chunkSize) - var octetsRead = s.client.recv(cstring(bodyData), chunkSize) - if octetsRead <= 0: - s.client.close() - next(s) - return - totalRead += octetsRead - totalBody.add(bodyData) - if totalBody.len != contentLength: - s.client.close() - next(s) - return - - s.body = totalBody - else: - badRequest(s.client) - s.client.close() - next(s) - return - - var L = skipWhitespace(data, i) - inc(i, L) - # XXX we ignore "HTTP/1.1" etc. for now here - var query = 0 - var last = i - while last < data.len and data[last] notin Whitespace: - if data[last] == '?' and query == 0: query = last - inc(last) - if query > 0: - s.query = data.substr(query+1, last-1) - s.path = data.substr(i, query-1) - else: - s.query = "" - s.path = data.substr(i, last-1) - -proc close*(s: Server) = - ## closes the server (and the socket the server uses). - close(s.socket) - -proc run*(handleRequest: proc (client: Socket, - path, query: string): bool {.closure.}, - port = Port(80)) = - ## encapsulates the server object and main loop - var s: Server - open(s, port, reuseAddr = true) - #echo("httpserver running on port ", s.port) - while true: - next(s) - if handleRequest(s.client, s.path, s.query): break - close(s.client) - close(s) - -# -- AsyncIO begin - -proc nextAsync(s: PAsyncHTTPServer) = - ## proceed to the first/next request. - var client: Socket - new(client) - var ip: string - acceptAddr(getSocket(s.asyncSocket), client, ip) - s.client = client - s.ip = ip - s.headers = newStringTable(modeCaseInsensitive) - #headers(s.client, "") - var data = "" - s.client.readLine(data) - if data == "": - # Socket disconnected - s.client.close() - return - var header = "" - while true: - s.client.readLine(header) # TODO: Very inefficient here. Prone to DOS. - if header == "\c\L": break - if header != "": - var i = 0 - var key = "" - var value = "" - i = header.parseUntil(key, ':') - inc(i) # skip : - if i < header.len: - i += header.skipWhiteSpace(i) - i += header.parseUntil(value, {'\c', '\L'}, i) - s.headers[key] = value - else: - s.client.close() - return - - var i = skipWhitespace(data) - if skipIgnoreCase(data, "GET") > 0: - s.reqMethod = "GET" - inc(i, 3) - elif skipIgnoreCase(data, "POST") > 0: - s.reqMethod = "POST" - inc(i, 4) - else: - unimplemented(s.client) - s.client.close() - return - - if s.reqMethod == "POST": - # Check for Expect header - if s.headers.hasKey("Expect"): - if s.headers["Expect"].toLower == "100-continue": - s.client.sendStatus("100 Continue") - else: - s.client.sendStatus("417 Expectation Failed") - - # Read the body - # - Check for Content-length header - if s.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(s.headers["Content-Length"], contentLength) == 0: - badRequest(s.client) - s.client.close() - return - else: - var totalRead = 0 - var totalBody = "" - while totalRead < contentLength: - var chunkSize = 8000 - if (contentLength - totalRead) < 8000: - chunkSize = (contentLength - totalRead) - var bodyData = newString(chunkSize) - var octetsRead = s.client.recv(cstring(bodyData), chunkSize) - if octetsRead <= 0: - s.client.close() - return - totalRead += octetsRead - totalBody.add(bodyData) - if totalBody.len != contentLength: - s.client.close() - return - - s.body = totalBody - else: - badRequest(s.client) - s.client.close() - return - - var L = skipWhitespace(data, i) - inc(i, L) - # XXX we ignore "HTTP/1.1" etc. for now here - var query = 0 - var last = i - while last < data.len and data[last] notin Whitespace: - if data[last] == '?' and query == 0: query = last - inc(last) - if query > 0: - s.query = data.substr(query+1, last-1) - s.path = data.substr(i, query-1) - else: - s.query = "" - s.path = data.substr(i, last-1) - -proc asyncHTTPServer*(handleRequest: proc (server: PAsyncHTTPServer, client: Socket, - path, query: string): bool {.closure, gcsafe.}, - port = Port(80), address = "", - reuseAddr = false): PAsyncHTTPServer = - ## Creates an Asynchronous HTTP server at ``port``. - var capturedRet: PAsyncHTTPServer - new(capturedRet) - capturedRet.asyncSocket = asyncSocket() - capturedRet.asyncSocket.handleAccept = - proc (s: AsyncSocket) = - nextAsync(capturedRet) - let quit = handleRequest(capturedRet, capturedRet.client, capturedRet.path, - capturedRet.query) - if quit: capturedRet.asyncSocket.close() - if reuseAddr: - capturedRet.asyncSocket.setSockOpt(OptReuseAddr, true) - - capturedRet.asyncSocket.bindAddr(port, address) - capturedRet.asyncSocket.listen() - if port == Port(0): - capturedRet.port = getSockName(capturedRet.asyncSocket) - else: - capturedRet.port = port - - capturedRet.client = invalidSocket - capturedRet.reqMethod = "" - capturedRet.body = "" - capturedRet.path = "" - capturedRet.query = "" - capturedRet.headers = {:}.newStringTable() - result = capturedRet - -proc register*(d: Dispatcher, s: PAsyncHTTPServer) = - ## Registers a ``PAsyncHTTPServer`` with a ``Dispatcher``. - d.register(s.asyncSocket) - -proc close*(h: PAsyncHTTPServer) = - ## Closes the ``PAsyncHTTPServer``. - h.asyncSocket.close() - -when not defined(testing) and isMainModule: - var counter = 0 - - var s: Server - open(s, Port(0)) - echo("httpserver running on port ", s.port) - while true: - next(s) - - inc(counter) - s.client.send("Hello, Andreas, for the $#th time. $# ? $#" % [ - $counter, s.path, s.query] & wwwNL) - - close(s.client) - close(s) - diff --git a/lib/pure/includes/unicode_ranges.nim b/lib/pure/includes/unicode_ranges.nim new file mode 100644 index 000000000..04ccfb747 --- /dev/null +++ b/lib/pure/includes/unicode_ranges.nim @@ -0,0 +1,2007 @@ +# This file was created from a script. + +const + toLowerRanges = [ + 0x00041'i32, 0x0005A'i32, 532, + 0x000C0'i32, 0x000D6'i32, 532, + 0x000D8'i32, 0x000DE'i32, 532, + 0x00189'i32, 0x0018A'i32, 705, + 0x001B1'i32, 0x001B2'i32, 717, + 0x00388'i32, 0x0038A'i32, 537, + 0x0038E'i32, 0x0038F'i32, 563, + 0x00391'i32, 0x003A1'i32, 532, + 0x003A3'i32, 0x003AB'i32, 532, + 0x003FD'i32, 0x003FF'i32, 370, + 0x00400'i32, 0x0040F'i32, 580, + 0x00410'i32, 0x0042F'i32, 532, + 0x00531'i32, 0x00556'i32, 548, + 0x010A0'i32, 0x010C5'i32, 7764, + 0x013A0'i32, 0x013EF'i32, 39364, + 0x013F0'i32, 0x013F5'i32, 508, + 0x01C90'i32, 0x01CBA'i32, -2508, + 0x01CBD'i32, 0x01CBF'i32, -2508, + 0x01F08'i32, 0x01F0F'i32, 492, + 0x01F18'i32, 0x01F1D'i32, 492, + 0x01F28'i32, 0x01F2F'i32, 492, + 0x01F38'i32, 0x01F3F'i32, 492, + 0x01F48'i32, 0x01F4D'i32, 492, + 0x01F68'i32, 0x01F6F'i32, 492, + 0x01F88'i32, 0x01F8F'i32, 492, + 0x01F98'i32, 0x01F9F'i32, 492, + 0x01FA8'i32, 0x01FAF'i32, 492, + 0x01FB8'i32, 0x01FB9'i32, 492, + 0x01FBA'i32, 0x01FBB'i32, 426, + 0x01FC8'i32, 0x01FCB'i32, 414, + 0x01FD8'i32, 0x01FD9'i32, 492, + 0x01FDA'i32, 0x01FDB'i32, 400, + 0x01FE8'i32, 0x01FE9'i32, 492, + 0x01FEA'i32, 0x01FEB'i32, 388, + 0x01FF8'i32, 0x01FF9'i32, 372, + 0x01FFA'i32, 0x01FFB'i32, 374, + 0x02C00'i32, 0x02C2E'i32, 548, + 0x02C7E'i32, 0x02C7F'i32, -10315, + 0x0FF21'i32, 0x0FF3A'i32, 532, + 0x10400'i32, 0x10427'i32, 540, + 0x104B0'i32, 0x104D3'i32, 540, + 0x10C80'i32, 0x10CB2'i32, 564, + 0x118A0'i32, 0x118BF'i32, 532, + 0x16E40'i32, 0x16E5F'i32, 532, + 0x1E900'i32, 0x1E921'i32, 534, + ] + + toLowerSinglets = [ + 0x00100'i32, 501, + 0x00102'i32, 501, + 0x00104'i32, 501, + 0x00106'i32, 501, + 0x00108'i32, 501, + 0x0010A'i32, 501, + 0x0010C'i32, 501, + 0x0010E'i32, 501, + 0x00110'i32, 501, + 0x00112'i32, 501, + 0x00114'i32, 501, + 0x00116'i32, 501, + 0x00118'i32, 501, + 0x0011A'i32, 501, + 0x0011C'i32, 501, + 0x0011E'i32, 501, + 0x00120'i32, 501, + 0x00122'i32, 501, + 0x00124'i32, 501, + 0x00126'i32, 501, + 0x00128'i32, 501, + 0x0012A'i32, 501, + 0x0012C'i32, 501, + 0x0012E'i32, 501, + 0x00130'i32, 301, + 0x00132'i32, 501, + 0x00134'i32, 501, + 0x00136'i32, 501, + 0x00139'i32, 501, + 0x0013B'i32, 501, + 0x0013D'i32, 501, + 0x0013F'i32, 501, + 0x00141'i32, 501, + 0x00143'i32, 501, + 0x00145'i32, 501, + 0x00147'i32, 501, + 0x0014A'i32, 501, + 0x0014C'i32, 501, + 0x0014E'i32, 501, + 0x00150'i32, 501, + 0x00152'i32, 501, + 0x00154'i32, 501, + 0x00156'i32, 501, + 0x00158'i32, 501, + 0x0015A'i32, 501, + 0x0015C'i32, 501, + 0x0015E'i32, 501, + 0x00160'i32, 501, + 0x00162'i32, 501, + 0x00164'i32, 501, + 0x00166'i32, 501, + 0x00168'i32, 501, + 0x0016A'i32, 501, + 0x0016C'i32, 501, + 0x0016E'i32, 501, + 0x00170'i32, 501, + 0x00172'i32, 501, + 0x00174'i32, 501, + 0x00176'i32, 501, + 0x00178'i32, 379, + 0x00179'i32, 501, + 0x0017B'i32, 501, + 0x0017D'i32, 501, + 0x00181'i32, 710, + 0x00182'i32, 501, + 0x00184'i32, 501, + 0x00186'i32, 706, + 0x00187'i32, 501, + 0x0018B'i32, 501, + 0x0018E'i32, 579, + 0x0018F'i32, 702, + 0x00190'i32, 703, + 0x00191'i32, 501, + 0x00193'i32, 705, + 0x00194'i32, 707, + 0x00196'i32, 711, + 0x00197'i32, 709, + 0x00198'i32, 501, + 0x0019C'i32, 711, + 0x0019D'i32, 713, + 0x0019F'i32, 714, + 0x001A0'i32, 501, + 0x001A2'i32, 501, + 0x001A4'i32, 501, + 0x001A6'i32, 718, + 0x001A7'i32, 501, + 0x001A9'i32, 718, + 0x001AC'i32, 501, + 0x001AE'i32, 718, + 0x001AF'i32, 501, + 0x001B3'i32, 501, + 0x001B5'i32, 501, + 0x001B7'i32, 719, + 0x001B8'i32, 501, + 0x001BC'i32, 501, + 0x001C4'i32, 502, + 0x001C5'i32, 501, + 0x001C7'i32, 502, + 0x001C8'i32, 501, + 0x001CA'i32, 502, + 0x001CB'i32, 501, + 0x001CD'i32, 501, + 0x001CF'i32, 501, + 0x001D1'i32, 501, + 0x001D3'i32, 501, + 0x001D5'i32, 501, + 0x001D7'i32, 501, + 0x001D9'i32, 501, + 0x001DB'i32, 501, + 0x001DE'i32, 501, + 0x001E0'i32, 501, + 0x001E2'i32, 501, + 0x001E4'i32, 501, + 0x001E6'i32, 501, + 0x001E8'i32, 501, + 0x001EA'i32, 501, + 0x001EC'i32, 501, + 0x001EE'i32, 501, + 0x001F1'i32, 502, + 0x001F2'i32, 501, + 0x001F4'i32, 501, + 0x001F6'i32, 403, + 0x001F7'i32, 444, + 0x001F8'i32, 501, + 0x001FA'i32, 501, + 0x001FC'i32, 501, + 0x001FE'i32, 501, + 0x00200'i32, 501, + 0x00202'i32, 501, + 0x00204'i32, 501, + 0x00206'i32, 501, + 0x00208'i32, 501, + 0x0020A'i32, 501, + 0x0020C'i32, 501, + 0x0020E'i32, 501, + 0x00210'i32, 501, + 0x00212'i32, 501, + 0x00214'i32, 501, + 0x00216'i32, 501, + 0x00218'i32, 501, + 0x0021A'i32, 501, + 0x0021C'i32, 501, + 0x0021E'i32, 501, + 0x00220'i32, 370, + 0x00222'i32, 501, + 0x00224'i32, 501, + 0x00226'i32, 501, + 0x00228'i32, 501, + 0x0022A'i32, 501, + 0x0022C'i32, 501, + 0x0022E'i32, 501, + 0x00230'i32, 501, + 0x00232'i32, 501, + 0x0023A'i32, 11295, + 0x0023B'i32, 501, + 0x0023D'i32, 337, + 0x0023E'i32, 11292, + 0x00241'i32, 501, + 0x00243'i32, 305, + 0x00244'i32, 569, + 0x00245'i32, 571, + 0x00246'i32, 501, + 0x00248'i32, 501, + 0x0024A'i32, 501, + 0x0024C'i32, 501, + 0x0024E'i32, 501, + 0x00370'i32, 501, + 0x00372'i32, 501, + 0x00376'i32, 501, + 0x0037F'i32, 616, + 0x00386'i32, 538, + 0x0038C'i32, 564, + 0x003CF'i32, 508, + 0x003D8'i32, 501, + 0x003DA'i32, 501, + 0x003DC'i32, 501, + 0x003DE'i32, 501, + 0x003E0'i32, 501, + 0x003E2'i32, 501, + 0x003E4'i32, 501, + 0x003E6'i32, 501, + 0x003E8'i32, 501, + 0x003EA'i32, 501, + 0x003EC'i32, 501, + 0x003EE'i32, 501, + 0x003F4'i32, 440, + 0x003F7'i32, 501, + 0x003F9'i32, 493, + 0x003FA'i32, 501, + 0x00460'i32, 501, + 0x00462'i32, 501, + 0x00464'i32, 501, + 0x00466'i32, 501, + 0x00468'i32, 501, + 0x0046A'i32, 501, + 0x0046C'i32, 501, + 0x0046E'i32, 501, + 0x00470'i32, 501, + 0x00472'i32, 501, + 0x00474'i32, 501, + 0x00476'i32, 501, + 0x00478'i32, 501, + 0x0047A'i32, 501, + 0x0047C'i32, 501, + 0x0047E'i32, 501, + 0x00480'i32, 501, + 0x0048A'i32, 501, + 0x0048C'i32, 501, + 0x0048E'i32, 501, + 0x00490'i32, 501, + 0x00492'i32, 501, + 0x00494'i32, 501, + 0x00496'i32, 501, + 0x00498'i32, 501, + 0x0049A'i32, 501, + 0x0049C'i32, 501, + 0x0049E'i32, 501, + 0x004A0'i32, 501, + 0x004A2'i32, 501, + 0x004A4'i32, 501, + 0x004A6'i32, 501, + 0x004A8'i32, 501, + 0x004AA'i32, 501, + 0x004AC'i32, 501, + 0x004AE'i32, 501, + 0x004B0'i32, 501, + 0x004B2'i32, 501, + 0x004B4'i32, 501, + 0x004B6'i32, 501, + 0x004B8'i32, 501, + 0x004BA'i32, 501, + 0x004BC'i32, 501, + 0x004BE'i32, 501, + 0x004C0'i32, 515, + 0x004C1'i32, 501, + 0x004C3'i32, 501, + 0x004C5'i32, 501, + 0x004C7'i32, 501, + 0x004C9'i32, 501, + 0x004CB'i32, 501, + 0x004CD'i32, 501, + 0x004D0'i32, 501, + 0x004D2'i32, 501, + 0x004D4'i32, 501, + 0x004D6'i32, 501, + 0x004D8'i32, 501, + 0x004DA'i32, 501, + 0x004DC'i32, 501, + 0x004DE'i32, 501, + 0x004E0'i32, 501, + 0x004E2'i32, 501, + 0x004E4'i32, 501, + 0x004E6'i32, 501, + 0x004E8'i32, 501, + 0x004EA'i32, 501, + 0x004EC'i32, 501, + 0x004EE'i32, 501, + 0x004F0'i32, 501, + 0x004F2'i32, 501, + 0x004F4'i32, 501, + 0x004F6'i32, 501, + 0x004F8'i32, 501, + 0x004FA'i32, 501, + 0x004FC'i32, 501, + 0x004FE'i32, 501, + 0x00500'i32, 501, + 0x00502'i32, 501, + 0x00504'i32, 501, + 0x00506'i32, 501, + 0x00508'i32, 501, + 0x0050A'i32, 501, + 0x0050C'i32, 501, + 0x0050E'i32, 501, + 0x00510'i32, 501, + 0x00512'i32, 501, + 0x00514'i32, 501, + 0x00516'i32, 501, + 0x00518'i32, 501, + 0x0051A'i32, 501, + 0x0051C'i32, 501, + 0x0051E'i32, 501, + 0x00520'i32, 501, + 0x00522'i32, 501, + 0x00524'i32, 501, + 0x00526'i32, 501, + 0x00528'i32, 501, + 0x0052A'i32, 501, + 0x0052C'i32, 501, + 0x0052E'i32, 501, + 0x010C7'i32, 7764, + 0x010CD'i32, 7764, + 0x01E00'i32, 501, + 0x01E02'i32, 501, + 0x01E04'i32, 501, + 0x01E06'i32, 501, + 0x01E08'i32, 501, + 0x01E0A'i32, 501, + 0x01E0C'i32, 501, + 0x01E0E'i32, 501, + 0x01E10'i32, 501, + 0x01E12'i32, 501, + 0x01E14'i32, 501, + 0x01E16'i32, 501, + 0x01E18'i32, 501, + 0x01E1A'i32, 501, + 0x01E1C'i32, 501, + 0x01E1E'i32, 501, + 0x01E20'i32, 501, + 0x01E22'i32, 501, + 0x01E24'i32, 501, + 0x01E26'i32, 501, + 0x01E28'i32, 501, + 0x01E2A'i32, 501, + 0x01E2C'i32, 501, + 0x01E2E'i32, 501, + 0x01E30'i32, 501, + 0x01E32'i32, 501, + 0x01E34'i32, 501, + 0x01E36'i32, 501, + 0x01E38'i32, 501, + 0x01E3A'i32, 501, + 0x01E3C'i32, 501, + 0x01E3E'i32, 501, + 0x01E40'i32, 501, + 0x01E42'i32, 501, + 0x01E44'i32, 501, + 0x01E46'i32, 501, + 0x01E48'i32, 501, + 0x01E4A'i32, 501, + 0x01E4C'i32, 501, + 0x01E4E'i32, 501, + 0x01E50'i32, 501, + 0x01E52'i32, 501, + 0x01E54'i32, 501, + 0x01E56'i32, 501, + 0x01E58'i32, 501, + 0x01E5A'i32, 501, + 0x01E5C'i32, 501, + 0x01E5E'i32, 501, + 0x01E60'i32, 501, + 0x01E62'i32, 501, + 0x01E64'i32, 501, + 0x01E66'i32, 501, + 0x01E68'i32, 501, + 0x01E6A'i32, 501, + 0x01E6C'i32, 501, + 0x01E6E'i32, 501, + 0x01E70'i32, 501, + 0x01E72'i32, 501, + 0x01E74'i32, 501, + 0x01E76'i32, 501, + 0x01E78'i32, 501, + 0x01E7A'i32, 501, + 0x01E7C'i32, 501, + 0x01E7E'i32, 501, + 0x01E80'i32, 501, + 0x01E82'i32, 501, + 0x01E84'i32, 501, + 0x01E86'i32, 501, + 0x01E88'i32, 501, + 0x01E8A'i32, 501, + 0x01E8C'i32, 501, + 0x01E8E'i32, 501, + 0x01E90'i32, 501, + 0x01E92'i32, 501, + 0x01E94'i32, 501, + 0x01E9E'i32, -7115, + 0x01EA0'i32, 501, + 0x01EA2'i32, 501, + 0x01EA4'i32, 501, + 0x01EA6'i32, 501, + 0x01EA8'i32, 501, + 0x01EAA'i32, 501, + 0x01EAC'i32, 501, + 0x01EAE'i32, 501, + 0x01EB0'i32, 501, + 0x01EB2'i32, 501, + 0x01EB4'i32, 501, + 0x01EB6'i32, 501, + 0x01EB8'i32, 501, + 0x01EBA'i32, 501, + 0x01EBC'i32, 501, + 0x01EBE'i32, 501, + 0x01EC0'i32, 501, + 0x01EC2'i32, 501, + 0x01EC4'i32, 501, + 0x01EC6'i32, 501, + 0x01EC8'i32, 501, + 0x01ECA'i32, 501, + 0x01ECC'i32, 501, + 0x01ECE'i32, 501, + 0x01ED0'i32, 501, + 0x01ED2'i32, 501, + 0x01ED4'i32, 501, + 0x01ED6'i32, 501, + 0x01ED8'i32, 501, + 0x01EDA'i32, 501, + 0x01EDC'i32, 501, + 0x01EDE'i32, 501, + 0x01EE0'i32, 501, + 0x01EE2'i32, 501, + 0x01EE4'i32, 501, + 0x01EE6'i32, 501, + 0x01EE8'i32, 501, + 0x01EEA'i32, 501, + 0x01EEC'i32, 501, + 0x01EEE'i32, 501, + 0x01EF0'i32, 501, + 0x01EF2'i32, 501, + 0x01EF4'i32, 501, + 0x01EF6'i32, 501, + 0x01EF8'i32, 501, + 0x01EFA'i32, 501, + 0x01EFC'i32, 501, + 0x01EFE'i32, 501, + 0x01F59'i32, 492, + 0x01F5B'i32, 492, + 0x01F5D'i32, 492, + 0x01F5F'i32, 492, + 0x01FBC'i32, 491, + 0x01FCC'i32, 491, + 0x01FEC'i32, 493, + 0x01FFC'i32, 491, + 0x02126'i32, -7017, + 0x0212A'i32, -7883, + 0x0212B'i32, -7762, + 0x02132'i32, 528, + 0x02183'i32, 501, + 0x02C60'i32, 501, + 0x02C62'i32, -10243, + 0x02C63'i32, -3314, + 0x02C64'i32, -10227, + 0x02C67'i32, 501, + 0x02C69'i32, 501, + 0x02C6B'i32, 501, + 0x02C6D'i32, -10280, + 0x02C6E'i32, -10249, + 0x02C6F'i32, -10283, + 0x02C70'i32, -10282, + 0x02C72'i32, 501, + 0x02C75'i32, 501, + 0x02C80'i32, 501, + 0x02C82'i32, 501, + 0x02C84'i32, 501, + 0x02C86'i32, 501, + 0x02C88'i32, 501, + 0x02C8A'i32, 501, + 0x02C8C'i32, 501, + 0x02C8E'i32, 501, + 0x02C90'i32, 501, + 0x02C92'i32, 501, + 0x02C94'i32, 501, + 0x02C96'i32, 501, + 0x02C98'i32, 501, + 0x02C9A'i32, 501, + 0x02C9C'i32, 501, + 0x02C9E'i32, 501, + 0x02CA0'i32, 501, + 0x02CA2'i32, 501, + 0x02CA4'i32, 501, + 0x02CA6'i32, 501, + 0x02CA8'i32, 501, + 0x02CAA'i32, 501, + 0x02CAC'i32, 501, + 0x02CAE'i32, 501, + 0x02CB0'i32, 501, + 0x02CB2'i32, 501, + 0x02CB4'i32, 501, + 0x02CB6'i32, 501, + 0x02CB8'i32, 501, + 0x02CBA'i32, 501, + 0x02CBC'i32, 501, + 0x02CBE'i32, 501, + 0x02CC0'i32, 501, + 0x02CC2'i32, 501, + 0x02CC4'i32, 501, + 0x02CC6'i32, 501, + 0x02CC8'i32, 501, + 0x02CCA'i32, 501, + 0x02CCC'i32, 501, + 0x02CCE'i32, 501, + 0x02CD0'i32, 501, + 0x02CD2'i32, 501, + 0x02CD4'i32, 501, + 0x02CD6'i32, 501, + 0x02CD8'i32, 501, + 0x02CDA'i32, 501, + 0x02CDC'i32, 501, + 0x02CDE'i32, 501, + 0x02CE0'i32, 501, + 0x02CE2'i32, 501, + 0x02CEB'i32, 501, + 0x02CED'i32, 501, + 0x02CF2'i32, 501, + 0x0A640'i32, 501, + 0x0A642'i32, 501, + 0x0A644'i32, 501, + 0x0A646'i32, 501, + 0x0A648'i32, 501, + 0x0A64A'i32, 501, + 0x0A64C'i32, 501, + 0x0A64E'i32, 501, + 0x0A650'i32, 501, + 0x0A652'i32, 501, + 0x0A654'i32, 501, + 0x0A656'i32, 501, + 0x0A658'i32, 501, + 0x0A65A'i32, 501, + 0x0A65C'i32, 501, + 0x0A65E'i32, 501, + 0x0A660'i32, 501, + 0x0A662'i32, 501, + 0x0A664'i32, 501, + 0x0A666'i32, 501, + 0x0A668'i32, 501, + 0x0A66A'i32, 501, + 0x0A66C'i32, 501, + 0x0A680'i32, 501, + 0x0A682'i32, 501, + 0x0A684'i32, 501, + 0x0A686'i32, 501, + 0x0A688'i32, 501, + 0x0A68A'i32, 501, + 0x0A68C'i32, 501, + 0x0A68E'i32, 501, + 0x0A690'i32, 501, + 0x0A692'i32, 501, + 0x0A694'i32, 501, + 0x0A696'i32, 501, + 0x0A698'i32, 501, + 0x0A69A'i32, 501, + 0x0A722'i32, 501, + 0x0A724'i32, 501, + 0x0A726'i32, 501, + 0x0A728'i32, 501, + 0x0A72A'i32, 501, + 0x0A72C'i32, 501, + 0x0A72E'i32, 501, + 0x0A732'i32, 501, + 0x0A734'i32, 501, + 0x0A736'i32, 501, + 0x0A738'i32, 501, + 0x0A73A'i32, 501, + 0x0A73C'i32, 501, + 0x0A73E'i32, 501, + 0x0A740'i32, 501, + 0x0A742'i32, 501, + 0x0A744'i32, 501, + 0x0A746'i32, 501, + 0x0A748'i32, 501, + 0x0A74A'i32, 501, + 0x0A74C'i32, 501, + 0x0A74E'i32, 501, + 0x0A750'i32, 501, + 0x0A752'i32, 501, + 0x0A754'i32, 501, + 0x0A756'i32, 501, + 0x0A758'i32, 501, + 0x0A75A'i32, 501, + 0x0A75C'i32, 501, + 0x0A75E'i32, 501, + 0x0A760'i32, 501, + 0x0A762'i32, 501, + 0x0A764'i32, 501, + 0x0A766'i32, 501, + 0x0A768'i32, 501, + 0x0A76A'i32, 501, + 0x0A76C'i32, 501, + 0x0A76E'i32, 501, + 0x0A779'i32, 501, + 0x0A77B'i32, 501, + 0x0A77D'i32, -34832, + 0x0A77E'i32, 501, + 0x0A780'i32, 501, + 0x0A782'i32, 501, + 0x0A784'i32, 501, + 0x0A786'i32, 501, + 0x0A78B'i32, 501, + 0x0A78D'i32, -41780, + 0x0A790'i32, 501, + 0x0A792'i32, 501, + 0x0A796'i32, 501, + 0x0A798'i32, 501, + 0x0A79A'i32, 501, + 0x0A79C'i32, 501, + 0x0A79E'i32, 501, + 0x0A7A0'i32, 501, + 0x0A7A2'i32, 501, + 0x0A7A4'i32, 501, + 0x0A7A6'i32, 501, + 0x0A7A8'i32, 501, + 0x0A7AA'i32, -41808, + 0x0A7AB'i32, -41819, + 0x0A7AC'i32, -41815, + 0x0A7AD'i32, -41805, + 0x0A7AE'i32, -41808, + 0x0A7B0'i32, -41758, + 0x0A7B1'i32, -41782, + 0x0A7B2'i32, -41761, + 0x0A7B3'i32, 1428, + 0x0A7B4'i32, 501, + 0x0A7B6'i32, 501, + 0x0A7B8'i32, 501, + 0x0A7BA'i32, 501, + 0x0A7BC'i32, 501, + 0x0A7BE'i32, 501, + 0x0A7C2'i32, 501, + 0x0A7C4'i32, 452, + 0x0A7C5'i32, -41807, + 0x0A7C6'i32, -34884, + ] + + toUpperRanges = [ + 0x00061'i32, 0x0007A'i32, 468, + 0x000E0'i32, 0x000F6'i32, 468, + 0x000F8'i32, 0x000FE'i32, 468, + 0x0023F'i32, 0x00240'i32, 11315, + 0x00256'i32, 0x00257'i32, 295, + 0x0028A'i32, 0x0028B'i32, 283, + 0x0037B'i32, 0x0037D'i32, 630, + 0x003AD'i32, 0x003AF'i32, 463, + 0x003B1'i32, 0x003C1'i32, 468, + 0x003C3'i32, 0x003CB'i32, 468, + 0x003CD'i32, 0x003CE'i32, 437, + 0x00430'i32, 0x0044F'i32, 468, + 0x00450'i32, 0x0045F'i32, 420, + 0x00561'i32, 0x00586'i32, 452, + 0x010D0'i32, 0x010FA'i32, 3508, + 0x010FD'i32, 0x010FF'i32, 3508, + 0x013F8'i32, 0x013FD'i32, 492, + 0x01C83'i32, 0x01C84'i32, -5742, + 0x01F00'i32, 0x01F07'i32, 508, + 0x01F10'i32, 0x01F15'i32, 508, + 0x01F20'i32, 0x01F27'i32, 508, + 0x01F30'i32, 0x01F37'i32, 508, + 0x01F40'i32, 0x01F45'i32, 508, + 0x01F60'i32, 0x01F67'i32, 508, + 0x01F70'i32, 0x01F71'i32, 574, + 0x01F72'i32, 0x01F75'i32, 586, + 0x01F76'i32, 0x01F77'i32, 600, + 0x01F78'i32, 0x01F79'i32, 628, + 0x01F7A'i32, 0x01F7B'i32, 612, + 0x01F7C'i32, 0x01F7D'i32, 626, + 0x01F80'i32, 0x01F87'i32, 508, + 0x01F90'i32, 0x01F97'i32, 508, + 0x01FA0'i32, 0x01FA7'i32, 508, + 0x01FB0'i32, 0x01FB1'i32, 508, + 0x01FD0'i32, 0x01FD1'i32, 508, + 0x01FE0'i32, 0x01FE1'i32, 508, + 0x02C30'i32, 0x02C5E'i32, 452, + 0x02D00'i32, 0x02D25'i32, -6764, + 0x0AB70'i32, 0x0ABBF'i32, -38364, + 0x0FF41'i32, 0x0FF5A'i32, 468, + 0x10428'i32, 0x1044F'i32, 460, + 0x104D8'i32, 0x104FB'i32, 460, + 0x10CC0'i32, 0x10CF2'i32, 436, + 0x118C0'i32, 0x118DF'i32, 468, + 0x16E60'i32, 0x16E7F'i32, 468, + 0x1E922'i32, 0x1E943'i32, 466, + ] + + toUpperSinglets = [ + 0x000B5'i32, 1243, + 0x000FF'i32, 621, + 0x00101'i32, 499, + 0x00103'i32, 499, + 0x00105'i32, 499, + 0x00107'i32, 499, + 0x00109'i32, 499, + 0x0010B'i32, 499, + 0x0010D'i32, 499, + 0x0010F'i32, 499, + 0x00111'i32, 499, + 0x00113'i32, 499, + 0x00115'i32, 499, + 0x00117'i32, 499, + 0x00119'i32, 499, + 0x0011B'i32, 499, + 0x0011D'i32, 499, + 0x0011F'i32, 499, + 0x00121'i32, 499, + 0x00123'i32, 499, + 0x00125'i32, 499, + 0x00127'i32, 499, + 0x00129'i32, 499, + 0x0012B'i32, 499, + 0x0012D'i32, 499, + 0x0012F'i32, 499, + 0x00131'i32, 268, + 0x00133'i32, 499, + 0x00135'i32, 499, + 0x00137'i32, 499, + 0x0013A'i32, 499, + 0x0013C'i32, 499, + 0x0013E'i32, 499, + 0x00140'i32, 499, + 0x00142'i32, 499, + 0x00144'i32, 499, + 0x00146'i32, 499, + 0x00148'i32, 499, + 0x0014B'i32, 499, + 0x0014D'i32, 499, + 0x0014F'i32, 499, + 0x00151'i32, 499, + 0x00153'i32, 499, + 0x00155'i32, 499, + 0x00157'i32, 499, + 0x00159'i32, 499, + 0x0015B'i32, 499, + 0x0015D'i32, 499, + 0x0015F'i32, 499, + 0x00161'i32, 499, + 0x00163'i32, 499, + 0x00165'i32, 499, + 0x00167'i32, 499, + 0x00169'i32, 499, + 0x0016B'i32, 499, + 0x0016D'i32, 499, + 0x0016F'i32, 499, + 0x00171'i32, 499, + 0x00173'i32, 499, + 0x00175'i32, 499, + 0x00177'i32, 499, + 0x0017A'i32, 499, + 0x0017C'i32, 499, + 0x0017E'i32, 499, + 0x0017F'i32, 200, + 0x00180'i32, 695, + 0x00183'i32, 499, + 0x00185'i32, 499, + 0x00188'i32, 499, + 0x0018C'i32, 499, + 0x00192'i32, 499, + 0x00195'i32, 597, + 0x00199'i32, 499, + 0x0019A'i32, 663, + 0x0019E'i32, 630, + 0x001A1'i32, 499, + 0x001A3'i32, 499, + 0x001A5'i32, 499, + 0x001A8'i32, 499, + 0x001AD'i32, 499, + 0x001B0'i32, 499, + 0x001B4'i32, 499, + 0x001B6'i32, 499, + 0x001B9'i32, 499, + 0x001BD'i32, 499, + 0x001BF'i32, 556, + 0x001C5'i32, 499, + 0x001C6'i32, 498, + 0x001C8'i32, 499, + 0x001C9'i32, 498, + 0x001CB'i32, 499, + 0x001CC'i32, 498, + 0x001CE'i32, 499, + 0x001D0'i32, 499, + 0x001D2'i32, 499, + 0x001D4'i32, 499, + 0x001D6'i32, 499, + 0x001D8'i32, 499, + 0x001DA'i32, 499, + 0x001DC'i32, 499, + 0x001DD'i32, 421, + 0x001DF'i32, 499, + 0x001E1'i32, 499, + 0x001E3'i32, 499, + 0x001E5'i32, 499, + 0x001E7'i32, 499, + 0x001E9'i32, 499, + 0x001EB'i32, 499, + 0x001ED'i32, 499, + 0x001EF'i32, 499, + 0x001F2'i32, 499, + 0x001F3'i32, 498, + 0x001F5'i32, 499, + 0x001F9'i32, 499, + 0x001FB'i32, 499, + 0x001FD'i32, 499, + 0x001FF'i32, 499, + 0x00201'i32, 499, + 0x00203'i32, 499, + 0x00205'i32, 499, + 0x00207'i32, 499, + 0x00209'i32, 499, + 0x0020B'i32, 499, + 0x0020D'i32, 499, + 0x0020F'i32, 499, + 0x00211'i32, 499, + 0x00213'i32, 499, + 0x00215'i32, 499, + 0x00217'i32, 499, + 0x00219'i32, 499, + 0x0021B'i32, 499, + 0x0021D'i32, 499, + 0x0021F'i32, 499, + 0x00223'i32, 499, + 0x00225'i32, 499, + 0x00227'i32, 499, + 0x00229'i32, 499, + 0x0022B'i32, 499, + 0x0022D'i32, 499, + 0x0022F'i32, 499, + 0x00231'i32, 499, + 0x00233'i32, 499, + 0x0023C'i32, 499, + 0x00242'i32, 499, + 0x00247'i32, 499, + 0x00249'i32, 499, + 0x0024B'i32, 499, + 0x0024D'i32, 499, + 0x0024F'i32, 499, + 0x00250'i32, 11283, + 0x00251'i32, 11280, + 0x00252'i32, 11282, + 0x00253'i32, 290, + 0x00254'i32, 294, + 0x00259'i32, 298, + 0x0025B'i32, 297, + 0x0025C'i32, 42819, + 0x00260'i32, 295, + 0x00261'i32, 42815, + 0x00263'i32, 293, + 0x00265'i32, 42780, + 0x00266'i32, 42808, + 0x00268'i32, 291, + 0x00269'i32, 289, + 0x0026A'i32, 42808, + 0x0026B'i32, 11243, + 0x0026C'i32, 42805, + 0x0026F'i32, 289, + 0x00271'i32, 11249, + 0x00272'i32, 287, + 0x00275'i32, 286, + 0x0027D'i32, 11227, + 0x00280'i32, 282, + 0x00282'i32, 42807, + 0x00283'i32, 282, + 0x00287'i32, 42782, + 0x00288'i32, 282, + 0x00289'i32, 431, + 0x0028C'i32, 429, + 0x00292'i32, 281, + 0x0029D'i32, 42761, + 0x0029E'i32, 42758, + 0x00371'i32, 499, + 0x00373'i32, 499, + 0x00377'i32, 499, + 0x003AC'i32, 462, + 0x003C2'i32, 469, + 0x003CC'i32, 436, + 0x003D0'i32, 438, + 0x003D1'i32, 443, + 0x003D5'i32, 453, + 0x003D6'i32, 446, + 0x003D7'i32, 492, + 0x003D9'i32, 499, + 0x003DB'i32, 499, + 0x003DD'i32, 499, + 0x003DF'i32, 499, + 0x003E1'i32, 499, + 0x003E3'i32, 499, + 0x003E5'i32, 499, + 0x003E7'i32, 499, + 0x003E9'i32, 499, + 0x003EB'i32, 499, + 0x003ED'i32, 499, + 0x003EF'i32, 499, + 0x003F0'i32, 414, + 0x003F1'i32, 420, + 0x003F2'i32, 507, + 0x003F3'i32, 384, + 0x003F5'i32, 404, + 0x003F8'i32, 499, + 0x003FB'i32, 499, + 0x00461'i32, 499, + 0x00463'i32, 499, + 0x00465'i32, 499, + 0x00467'i32, 499, + 0x00469'i32, 499, + 0x0046B'i32, 499, + 0x0046D'i32, 499, + 0x0046F'i32, 499, + 0x00471'i32, 499, + 0x00473'i32, 499, + 0x00475'i32, 499, + 0x00477'i32, 499, + 0x00479'i32, 499, + 0x0047B'i32, 499, + 0x0047D'i32, 499, + 0x0047F'i32, 499, + 0x00481'i32, 499, + 0x0048B'i32, 499, + 0x0048D'i32, 499, + 0x0048F'i32, 499, + 0x00491'i32, 499, + 0x00493'i32, 499, + 0x00495'i32, 499, + 0x00497'i32, 499, + 0x00499'i32, 499, + 0x0049B'i32, 499, + 0x0049D'i32, 499, + 0x0049F'i32, 499, + 0x004A1'i32, 499, + 0x004A3'i32, 499, + 0x004A5'i32, 499, + 0x004A7'i32, 499, + 0x004A9'i32, 499, + 0x004AB'i32, 499, + 0x004AD'i32, 499, + 0x004AF'i32, 499, + 0x004B1'i32, 499, + 0x004B3'i32, 499, + 0x004B5'i32, 499, + 0x004B7'i32, 499, + 0x004B9'i32, 499, + 0x004BB'i32, 499, + 0x004BD'i32, 499, + 0x004BF'i32, 499, + 0x004C2'i32, 499, + 0x004C4'i32, 499, + 0x004C6'i32, 499, + 0x004C8'i32, 499, + 0x004CA'i32, 499, + 0x004CC'i32, 499, + 0x004CE'i32, 499, + 0x004CF'i32, 485, + 0x004D1'i32, 499, + 0x004D3'i32, 499, + 0x004D5'i32, 499, + 0x004D7'i32, 499, + 0x004D9'i32, 499, + 0x004DB'i32, 499, + 0x004DD'i32, 499, + 0x004DF'i32, 499, + 0x004E1'i32, 499, + 0x004E3'i32, 499, + 0x004E5'i32, 499, + 0x004E7'i32, 499, + 0x004E9'i32, 499, + 0x004EB'i32, 499, + 0x004ED'i32, 499, + 0x004EF'i32, 499, + 0x004F1'i32, 499, + 0x004F3'i32, 499, + 0x004F5'i32, 499, + 0x004F7'i32, 499, + 0x004F9'i32, 499, + 0x004FB'i32, 499, + 0x004FD'i32, 499, + 0x004FF'i32, 499, + 0x00501'i32, 499, + 0x00503'i32, 499, + 0x00505'i32, 499, + 0x00507'i32, 499, + 0x00509'i32, 499, + 0x0050B'i32, 499, + 0x0050D'i32, 499, + 0x0050F'i32, 499, + 0x00511'i32, 499, + 0x00513'i32, 499, + 0x00515'i32, 499, + 0x00517'i32, 499, + 0x00519'i32, 499, + 0x0051B'i32, 499, + 0x0051D'i32, 499, + 0x0051F'i32, 499, + 0x00521'i32, 499, + 0x00523'i32, 499, + 0x00525'i32, 499, + 0x00527'i32, 499, + 0x00529'i32, 499, + 0x0052B'i32, 499, + 0x0052D'i32, 499, + 0x0052F'i32, 499, + 0x01C80'i32, -5754, + 0x01C81'i32, -5753, + 0x01C82'i32, -5744, + 0x01C85'i32, -5743, + 0x01C86'i32, -5736, + 0x01C87'i32, -5681, + 0x01C88'i32, 35766, + 0x01D79'i32, 35832, + 0x01D7D'i32, 4314, + 0x01D8E'i32, 35884, + 0x01E01'i32, 499, + 0x01E03'i32, 499, + 0x01E05'i32, 499, + 0x01E07'i32, 499, + 0x01E09'i32, 499, + 0x01E0B'i32, 499, + 0x01E0D'i32, 499, + 0x01E0F'i32, 499, + 0x01E11'i32, 499, + 0x01E13'i32, 499, + 0x01E15'i32, 499, + 0x01E17'i32, 499, + 0x01E19'i32, 499, + 0x01E1B'i32, 499, + 0x01E1D'i32, 499, + 0x01E1F'i32, 499, + 0x01E21'i32, 499, + 0x01E23'i32, 499, + 0x01E25'i32, 499, + 0x01E27'i32, 499, + 0x01E29'i32, 499, + 0x01E2B'i32, 499, + 0x01E2D'i32, 499, + 0x01E2F'i32, 499, + 0x01E31'i32, 499, + 0x01E33'i32, 499, + 0x01E35'i32, 499, + 0x01E37'i32, 499, + 0x01E39'i32, 499, + 0x01E3B'i32, 499, + 0x01E3D'i32, 499, + 0x01E3F'i32, 499, + 0x01E41'i32, 499, + 0x01E43'i32, 499, + 0x01E45'i32, 499, + 0x01E47'i32, 499, + 0x01E49'i32, 499, + 0x01E4B'i32, 499, + 0x01E4D'i32, 499, + 0x01E4F'i32, 499, + 0x01E51'i32, 499, + 0x01E53'i32, 499, + 0x01E55'i32, 499, + 0x01E57'i32, 499, + 0x01E59'i32, 499, + 0x01E5B'i32, 499, + 0x01E5D'i32, 499, + 0x01E5F'i32, 499, + 0x01E61'i32, 499, + 0x01E63'i32, 499, + 0x01E65'i32, 499, + 0x01E67'i32, 499, + 0x01E69'i32, 499, + 0x01E6B'i32, 499, + 0x01E6D'i32, 499, + 0x01E6F'i32, 499, + 0x01E71'i32, 499, + 0x01E73'i32, 499, + 0x01E75'i32, 499, + 0x01E77'i32, 499, + 0x01E79'i32, 499, + 0x01E7B'i32, 499, + 0x01E7D'i32, 499, + 0x01E7F'i32, 499, + 0x01E81'i32, 499, + 0x01E83'i32, 499, + 0x01E85'i32, 499, + 0x01E87'i32, 499, + 0x01E89'i32, 499, + 0x01E8B'i32, 499, + 0x01E8D'i32, 499, + 0x01E8F'i32, 499, + 0x01E91'i32, 499, + 0x01E93'i32, 499, + 0x01E95'i32, 499, + 0x01E9B'i32, 441, + 0x01EA1'i32, 499, + 0x01EA3'i32, 499, + 0x01EA5'i32, 499, + 0x01EA7'i32, 499, + 0x01EA9'i32, 499, + 0x01EAB'i32, 499, + 0x01EAD'i32, 499, + 0x01EAF'i32, 499, + 0x01EB1'i32, 499, + 0x01EB3'i32, 499, + 0x01EB5'i32, 499, + 0x01EB7'i32, 499, + 0x01EB9'i32, 499, + 0x01EBB'i32, 499, + 0x01EBD'i32, 499, + 0x01EBF'i32, 499, + 0x01EC1'i32, 499, + 0x01EC3'i32, 499, + 0x01EC5'i32, 499, + 0x01EC7'i32, 499, + 0x01EC9'i32, 499, + 0x01ECB'i32, 499, + 0x01ECD'i32, 499, + 0x01ECF'i32, 499, + 0x01ED1'i32, 499, + 0x01ED3'i32, 499, + 0x01ED5'i32, 499, + 0x01ED7'i32, 499, + 0x01ED9'i32, 499, + 0x01EDB'i32, 499, + 0x01EDD'i32, 499, + 0x01EDF'i32, 499, + 0x01EE1'i32, 499, + 0x01EE3'i32, 499, + 0x01EE5'i32, 499, + 0x01EE7'i32, 499, + 0x01EE9'i32, 499, + 0x01EEB'i32, 499, + 0x01EED'i32, 499, + 0x01EEF'i32, 499, + 0x01EF1'i32, 499, + 0x01EF3'i32, 499, + 0x01EF5'i32, 499, + 0x01EF7'i32, 499, + 0x01EF9'i32, 499, + 0x01EFB'i32, 499, + 0x01EFD'i32, 499, + 0x01EFF'i32, 499, + 0x01F51'i32, 508, + 0x01F53'i32, 508, + 0x01F55'i32, 508, + 0x01F57'i32, 508, + 0x01FB3'i32, 509, + 0x01FBE'i32, -6705, + 0x01FC3'i32, 509, + 0x01FE5'i32, 507, + 0x01FF3'i32, 509, + 0x0214E'i32, 472, + 0x02184'i32, 499, + 0x02C61'i32, 499, + 0x02C65'i32, -10295, + 0x02C66'i32, -10292, + 0x02C68'i32, 499, + 0x02C6A'i32, 499, + 0x02C6C'i32, 499, + 0x02C73'i32, 499, + 0x02C76'i32, 499, + 0x02C81'i32, 499, + 0x02C83'i32, 499, + 0x02C85'i32, 499, + 0x02C87'i32, 499, + 0x02C89'i32, 499, + 0x02C8B'i32, 499, + 0x02C8D'i32, 499, + 0x02C8F'i32, 499, + 0x02C91'i32, 499, + 0x02C93'i32, 499, + 0x02C95'i32, 499, + 0x02C97'i32, 499, + 0x02C99'i32, 499, + 0x02C9B'i32, 499, + 0x02C9D'i32, 499, + 0x02C9F'i32, 499, + 0x02CA1'i32, 499, + 0x02CA3'i32, 499, + 0x02CA5'i32, 499, + 0x02CA7'i32, 499, + 0x02CA9'i32, 499, + 0x02CAB'i32, 499, + 0x02CAD'i32, 499, + 0x02CAF'i32, 499, + 0x02CB1'i32, 499, + 0x02CB3'i32, 499, + 0x02CB5'i32, 499, + 0x02CB7'i32, 499, + 0x02CB9'i32, 499, + 0x02CBB'i32, 499, + 0x02CBD'i32, 499, + 0x02CBF'i32, 499, + 0x02CC1'i32, 499, + 0x02CC3'i32, 499, + 0x02CC5'i32, 499, + 0x02CC7'i32, 499, + 0x02CC9'i32, 499, + 0x02CCB'i32, 499, + 0x02CCD'i32, 499, + 0x02CCF'i32, 499, + 0x02CD1'i32, 499, + 0x02CD3'i32, 499, + 0x02CD5'i32, 499, + 0x02CD7'i32, 499, + 0x02CD9'i32, 499, + 0x02CDB'i32, 499, + 0x02CDD'i32, 499, + 0x02CDF'i32, 499, + 0x02CE1'i32, 499, + 0x02CE3'i32, 499, + 0x02CEC'i32, 499, + 0x02CEE'i32, 499, + 0x02CF3'i32, 499, + 0x02D27'i32, -6764, + 0x02D2D'i32, -6764, + 0x0A641'i32, 499, + 0x0A643'i32, 499, + 0x0A645'i32, 499, + 0x0A647'i32, 499, + 0x0A649'i32, 499, + 0x0A64B'i32, 499, + 0x0A64D'i32, 499, + 0x0A64F'i32, 499, + 0x0A651'i32, 499, + 0x0A653'i32, 499, + 0x0A655'i32, 499, + 0x0A657'i32, 499, + 0x0A659'i32, 499, + 0x0A65B'i32, 499, + 0x0A65D'i32, 499, + 0x0A65F'i32, 499, + 0x0A661'i32, 499, + 0x0A663'i32, 499, + 0x0A665'i32, 499, + 0x0A667'i32, 499, + 0x0A669'i32, 499, + 0x0A66B'i32, 499, + 0x0A66D'i32, 499, + 0x0A681'i32, 499, + 0x0A683'i32, 499, + 0x0A685'i32, 499, + 0x0A687'i32, 499, + 0x0A689'i32, 499, + 0x0A68B'i32, 499, + 0x0A68D'i32, 499, + 0x0A68F'i32, 499, + 0x0A691'i32, 499, + 0x0A693'i32, 499, + 0x0A695'i32, 499, + 0x0A697'i32, 499, + 0x0A699'i32, 499, + 0x0A69B'i32, 499, + 0x0A723'i32, 499, + 0x0A725'i32, 499, + 0x0A727'i32, 499, + 0x0A729'i32, 499, + 0x0A72B'i32, 499, + 0x0A72D'i32, 499, + 0x0A72F'i32, 499, + 0x0A733'i32, 499, + 0x0A735'i32, 499, + 0x0A737'i32, 499, + 0x0A739'i32, 499, + 0x0A73B'i32, 499, + 0x0A73D'i32, 499, + 0x0A73F'i32, 499, + 0x0A741'i32, 499, + 0x0A743'i32, 499, + 0x0A745'i32, 499, + 0x0A747'i32, 499, + 0x0A749'i32, 499, + 0x0A74B'i32, 499, + 0x0A74D'i32, 499, + 0x0A74F'i32, 499, + 0x0A751'i32, 499, + 0x0A753'i32, 499, + 0x0A755'i32, 499, + 0x0A757'i32, 499, + 0x0A759'i32, 499, + 0x0A75B'i32, 499, + 0x0A75D'i32, 499, + 0x0A75F'i32, 499, + 0x0A761'i32, 499, + 0x0A763'i32, 499, + 0x0A765'i32, 499, + 0x0A767'i32, 499, + 0x0A769'i32, 499, + 0x0A76B'i32, 499, + 0x0A76D'i32, 499, + 0x0A76F'i32, 499, + 0x0A77A'i32, 499, + 0x0A77C'i32, 499, + 0x0A77F'i32, 499, + 0x0A781'i32, 499, + 0x0A783'i32, 499, + 0x0A785'i32, 499, + 0x0A787'i32, 499, + 0x0A78C'i32, 499, + 0x0A791'i32, 499, + 0x0A793'i32, 499, + 0x0A794'i32, 548, + 0x0A797'i32, 499, + 0x0A799'i32, 499, + 0x0A79B'i32, 499, + 0x0A79D'i32, 499, + 0x0A79F'i32, 499, + 0x0A7A1'i32, 499, + 0x0A7A3'i32, 499, + 0x0A7A5'i32, 499, + 0x0A7A7'i32, 499, + 0x0A7A9'i32, 499, + 0x0A7B5'i32, 499, + 0x0A7B7'i32, 499, + 0x0A7B9'i32, 499, + 0x0A7BB'i32, 499, + 0x0A7BD'i32, 499, + 0x0A7BF'i32, 499, + 0x0A7C3'i32, 499, + 0x0AB53'i32, -428, + ] + + toTitleSinglets = [ + 0x001C4'i32, 501, + 0x001C6'i32, 499, + 0x001C7'i32, 501, + 0x001C9'i32, 499, + 0x001CA'i32, 501, + 0x001CC'i32, 499, + 0x001F1'i32, 501, + 0x001F3'i32, 499, + ] + + alphaRanges = [ + 0x00041'i32, 0x0005A'i32, + 0x00061'i32, 0x0007A'i32, + 0x000C0'i32, 0x000D6'i32, + 0x000D8'i32, 0x000F6'i32, + 0x000F8'i32, 0x002C1'i32, + 0x002C6'i32, 0x002D1'i32, + 0x002E0'i32, 0x002E4'i32, + 0x00370'i32, 0x00374'i32, + 0x00376'i32, 0x00377'i32, + 0x0037A'i32, 0x0037D'i32, + 0x00388'i32, 0x0038A'i32, + 0x0038E'i32, 0x003A1'i32, + 0x003A3'i32, 0x003F5'i32, + 0x003F7'i32, 0x00481'i32, + 0x0048A'i32, 0x0052F'i32, + 0x00531'i32, 0x00556'i32, + 0x00560'i32, 0x00588'i32, + 0x005D0'i32, 0x005EA'i32, + 0x005EF'i32, 0x005F2'i32, + 0x00620'i32, 0x0064A'i32, + 0x0066E'i32, 0x0066F'i32, + 0x00671'i32, 0x006D3'i32, + 0x006E5'i32, 0x006E6'i32, + 0x006EE'i32, 0x006EF'i32, + 0x006FA'i32, 0x006FC'i32, + 0x00712'i32, 0x0072F'i32, + 0x0074D'i32, 0x007A5'i32, + 0x007CA'i32, 0x007EA'i32, + 0x007F4'i32, 0x007F5'i32, + 0x00800'i32, 0x00815'i32, + 0x00840'i32, 0x00858'i32, + 0x00860'i32, 0x0086A'i32, + 0x008A0'i32, 0x008B4'i32, + 0x008B6'i32, 0x008BD'i32, + 0x00904'i32, 0x00939'i32, + 0x00958'i32, 0x00961'i32, + 0x00971'i32, 0x00980'i32, + 0x00985'i32, 0x0098C'i32, + 0x0098F'i32, 0x00990'i32, + 0x00993'i32, 0x009A8'i32, + 0x009AA'i32, 0x009B0'i32, + 0x009B6'i32, 0x009B9'i32, + 0x009DC'i32, 0x009DD'i32, + 0x009DF'i32, 0x009E1'i32, + 0x009F0'i32, 0x009F1'i32, + 0x00A05'i32, 0x00A0A'i32, + 0x00A0F'i32, 0x00A10'i32, + 0x00A13'i32, 0x00A28'i32, + 0x00A2A'i32, 0x00A30'i32, + 0x00A32'i32, 0x00A33'i32, + 0x00A35'i32, 0x00A36'i32, + 0x00A38'i32, 0x00A39'i32, + 0x00A59'i32, 0x00A5C'i32, + 0x00A72'i32, 0x00A74'i32, + 0x00A85'i32, 0x00A8D'i32, + 0x00A8F'i32, 0x00A91'i32, + 0x00A93'i32, 0x00AA8'i32, + 0x00AAA'i32, 0x00AB0'i32, + 0x00AB2'i32, 0x00AB3'i32, + 0x00AB5'i32, 0x00AB9'i32, + 0x00AE0'i32, 0x00AE1'i32, + 0x00B05'i32, 0x00B0C'i32, + 0x00B0F'i32, 0x00B10'i32, + 0x00B13'i32, 0x00B28'i32, + 0x00B2A'i32, 0x00B30'i32, + 0x00B32'i32, 0x00B33'i32, + 0x00B35'i32, 0x00B39'i32, + 0x00B5C'i32, 0x00B5D'i32, + 0x00B5F'i32, 0x00B61'i32, + 0x00B85'i32, 0x00B8A'i32, + 0x00B8E'i32, 0x00B90'i32, + 0x00B92'i32, 0x00B95'i32, + 0x00B99'i32, 0x00B9A'i32, + 0x00B9E'i32, 0x00B9F'i32, + 0x00BA3'i32, 0x00BA4'i32, + 0x00BA8'i32, 0x00BAA'i32, + 0x00BAE'i32, 0x00BB9'i32, + 0x00C05'i32, 0x00C0C'i32, + 0x00C0E'i32, 0x00C10'i32, + 0x00C12'i32, 0x00C28'i32, + 0x00C2A'i32, 0x00C39'i32, + 0x00C58'i32, 0x00C5A'i32, + 0x00C60'i32, 0x00C61'i32, + 0x00C85'i32, 0x00C8C'i32, + 0x00C8E'i32, 0x00C90'i32, + 0x00C92'i32, 0x00CA8'i32, + 0x00CAA'i32, 0x00CB3'i32, + 0x00CB5'i32, 0x00CB9'i32, + 0x00CE0'i32, 0x00CE1'i32, + 0x00CF1'i32, 0x00CF2'i32, + 0x00D05'i32, 0x00D0C'i32, + 0x00D0E'i32, 0x00D10'i32, + 0x00D12'i32, 0x00D3A'i32, + 0x00D54'i32, 0x00D56'i32, + 0x00D5F'i32, 0x00D61'i32, + 0x00D7A'i32, 0x00D7F'i32, + 0x00D85'i32, 0x00D96'i32, + 0x00D9A'i32, 0x00DB1'i32, + 0x00DB3'i32, 0x00DBB'i32, + 0x00DC0'i32, 0x00DC6'i32, + 0x00E01'i32, 0x00E30'i32, + 0x00E32'i32, 0x00E33'i32, + 0x00E40'i32, 0x00E46'i32, + 0x00E81'i32, 0x00E82'i32, + 0x00E86'i32, 0x00E8A'i32, + 0x00E8C'i32, 0x00EA3'i32, + 0x00EA7'i32, 0x00EB0'i32, + 0x00EB2'i32, 0x00EB3'i32, + 0x00EC0'i32, 0x00EC4'i32, + 0x00EDC'i32, 0x00EDF'i32, + 0x00F40'i32, 0x00F47'i32, + 0x00F49'i32, 0x00F6C'i32, + 0x00F88'i32, 0x00F8C'i32, + 0x01000'i32, 0x0102A'i32, + 0x01050'i32, 0x01055'i32, + 0x0105A'i32, 0x0105D'i32, + 0x01065'i32, 0x01066'i32, + 0x0106E'i32, 0x01070'i32, + 0x01075'i32, 0x01081'i32, + 0x010A0'i32, 0x010C5'i32, + 0x010D0'i32, 0x010FA'i32, + 0x010FC'i32, 0x01248'i32, + 0x0124A'i32, 0x0124D'i32, + 0x01250'i32, 0x01256'i32, + 0x0125A'i32, 0x0125D'i32, + 0x01260'i32, 0x01288'i32, + 0x0128A'i32, 0x0128D'i32, + 0x01290'i32, 0x012B0'i32, + 0x012B2'i32, 0x012B5'i32, + 0x012B8'i32, 0x012BE'i32, + 0x012C2'i32, 0x012C5'i32, + 0x012C8'i32, 0x012D6'i32, + 0x012D8'i32, 0x01310'i32, + 0x01312'i32, 0x01315'i32, + 0x01318'i32, 0x0135A'i32, + 0x01380'i32, 0x0138F'i32, + 0x013A0'i32, 0x013F5'i32, + 0x013F8'i32, 0x013FD'i32, + 0x01401'i32, 0x0166C'i32, + 0x0166F'i32, 0x0167F'i32, + 0x01681'i32, 0x0169A'i32, + 0x016A0'i32, 0x016EA'i32, + 0x016F1'i32, 0x016F8'i32, + 0x01700'i32, 0x0170C'i32, + 0x0170E'i32, 0x01711'i32, + 0x01720'i32, 0x01731'i32, + 0x01740'i32, 0x01751'i32, + 0x01760'i32, 0x0176C'i32, + 0x0176E'i32, 0x01770'i32, + 0x01780'i32, 0x017B3'i32, + 0x01820'i32, 0x01878'i32, + 0x01880'i32, 0x01884'i32, + 0x01887'i32, 0x018A8'i32, + 0x018B0'i32, 0x018F5'i32, + 0x01900'i32, 0x0191E'i32, + 0x01950'i32, 0x0196D'i32, + 0x01970'i32, 0x01974'i32, + 0x01980'i32, 0x019AB'i32, + 0x019B0'i32, 0x019C9'i32, + 0x01A00'i32, 0x01A16'i32, + 0x01A20'i32, 0x01A54'i32, + 0x01B05'i32, 0x01B33'i32, + 0x01B45'i32, 0x01B4B'i32, + 0x01B83'i32, 0x01BA0'i32, + 0x01BAE'i32, 0x01BAF'i32, + 0x01BBA'i32, 0x01BE5'i32, + 0x01C00'i32, 0x01C23'i32, + 0x01C4D'i32, 0x01C4F'i32, + 0x01C5A'i32, 0x01C7D'i32, + 0x01C80'i32, 0x01C88'i32, + 0x01C90'i32, 0x01CBA'i32, + 0x01CBD'i32, 0x01CBF'i32, + 0x01CE9'i32, 0x01CEC'i32, + 0x01CEE'i32, 0x01CF3'i32, + 0x01CF5'i32, 0x01CF6'i32, + 0x01D00'i32, 0x01DBF'i32, + 0x01E00'i32, 0x01F15'i32, + 0x01F18'i32, 0x01F1D'i32, + 0x01F20'i32, 0x01F45'i32, + 0x01F48'i32, 0x01F4D'i32, + 0x01F50'i32, 0x01F57'i32, + 0x01F5F'i32, 0x01F7D'i32, + 0x01F80'i32, 0x01FB4'i32, + 0x01FB6'i32, 0x01FBC'i32, + 0x01FC2'i32, 0x01FC4'i32, + 0x01FC6'i32, 0x01FCC'i32, + 0x01FD0'i32, 0x01FD3'i32, + 0x01FD6'i32, 0x01FDB'i32, + 0x01FE0'i32, 0x01FEC'i32, + 0x01FF2'i32, 0x01FF4'i32, + 0x01FF6'i32, 0x01FFC'i32, + 0x02090'i32, 0x0209C'i32, + 0x0210A'i32, 0x02113'i32, + 0x02119'i32, 0x0211D'i32, + 0x0212A'i32, 0x0212D'i32, + 0x0212F'i32, 0x02139'i32, + 0x0213C'i32, 0x0213F'i32, + 0x02145'i32, 0x02149'i32, + 0x02183'i32, 0x02184'i32, + 0x02C00'i32, 0x02C2E'i32, + 0x02C30'i32, 0x02C5E'i32, + 0x02C60'i32, 0x02CE4'i32, + 0x02CEB'i32, 0x02CEE'i32, + 0x02CF2'i32, 0x02CF3'i32, + 0x02D00'i32, 0x02D25'i32, + 0x02D30'i32, 0x02D67'i32, + 0x02D80'i32, 0x02D96'i32, + 0x02DA0'i32, 0x02DA6'i32, + 0x02DA8'i32, 0x02DAE'i32, + 0x02DB0'i32, 0x02DB6'i32, + 0x02DB8'i32, 0x02DBE'i32, + 0x02DC0'i32, 0x02DC6'i32, + 0x02DC8'i32, 0x02DCE'i32, + 0x02DD0'i32, 0x02DD6'i32, + 0x02DD8'i32, 0x02DDE'i32, + 0x03005'i32, 0x03006'i32, + 0x03031'i32, 0x03035'i32, + 0x0303B'i32, 0x0303C'i32, + 0x03041'i32, 0x03096'i32, + 0x0309D'i32, 0x0309F'i32, + 0x030A1'i32, 0x030FA'i32, + 0x030FC'i32, 0x030FF'i32, + 0x03105'i32, 0x0312F'i32, + 0x03131'i32, 0x0318E'i32, + 0x031A0'i32, 0x031BA'i32, + 0x031F0'i32, 0x031FF'i32, + 0x03400'i32, 0x04DB5'i32, + 0x04E00'i32, 0x09FEF'i32, + 0x0A000'i32, 0x0A48C'i32, + 0x0A4D0'i32, 0x0A4FD'i32, + 0x0A500'i32, 0x0A60C'i32, + 0x0A610'i32, 0x0A61F'i32, + 0x0A62A'i32, 0x0A62B'i32, + 0x0A640'i32, 0x0A66E'i32, + 0x0A67F'i32, 0x0A69D'i32, + 0x0A6A0'i32, 0x0A6E5'i32, + 0x0A717'i32, 0x0A71F'i32, + 0x0A722'i32, 0x0A788'i32, + 0x0A78B'i32, 0x0A7BF'i32, + 0x0A7C2'i32, 0x0A7C6'i32, + 0x0A7F7'i32, 0x0A801'i32, + 0x0A803'i32, 0x0A805'i32, + 0x0A807'i32, 0x0A80A'i32, + 0x0A80C'i32, 0x0A822'i32, + 0x0A840'i32, 0x0A873'i32, + 0x0A882'i32, 0x0A8B3'i32, + 0x0A8F2'i32, 0x0A8F7'i32, + 0x0A8FD'i32, 0x0A8FE'i32, + 0x0A90A'i32, 0x0A925'i32, + 0x0A930'i32, 0x0A946'i32, + 0x0A960'i32, 0x0A97C'i32, + 0x0A984'i32, 0x0A9B2'i32, + 0x0A9E0'i32, 0x0A9E4'i32, + 0x0A9E6'i32, 0x0A9EF'i32, + 0x0A9FA'i32, 0x0A9FE'i32, + 0x0AA00'i32, 0x0AA28'i32, + 0x0AA40'i32, 0x0AA42'i32, + 0x0AA44'i32, 0x0AA4B'i32, + 0x0AA60'i32, 0x0AA76'i32, + 0x0AA7E'i32, 0x0AAAF'i32, + 0x0AAB5'i32, 0x0AAB6'i32, + 0x0AAB9'i32, 0x0AABD'i32, + 0x0AADB'i32, 0x0AADD'i32, + 0x0AAE0'i32, 0x0AAEA'i32, + 0x0AAF2'i32, 0x0AAF4'i32, + 0x0AB01'i32, 0x0AB06'i32, + 0x0AB09'i32, 0x0AB0E'i32, + 0x0AB11'i32, 0x0AB16'i32, + 0x0AB20'i32, 0x0AB26'i32, + 0x0AB28'i32, 0x0AB2E'i32, + 0x0AB30'i32, 0x0AB5A'i32, + 0x0AB5C'i32, 0x0AB67'i32, + 0x0AB70'i32, 0x0ABE2'i32, + 0x0AC00'i32, 0x0D7A3'i32, + 0x0D7B0'i32, 0x0D7C6'i32, + 0x0D7CB'i32, 0x0D7FB'i32, + 0x0F900'i32, 0x0FA6D'i32, + 0x0FA70'i32, 0x0FAD9'i32, + 0x0FB00'i32, 0x0FB06'i32, + 0x0FB13'i32, 0x0FB17'i32, + 0x0FB1F'i32, 0x0FB28'i32, + 0x0FB2A'i32, 0x0FB36'i32, + 0x0FB38'i32, 0x0FB3C'i32, + 0x0FB40'i32, 0x0FB41'i32, + 0x0FB43'i32, 0x0FB44'i32, + 0x0FB46'i32, 0x0FBB1'i32, + 0x0FBD3'i32, 0x0FD3D'i32, + 0x0FD50'i32, 0x0FD8F'i32, + 0x0FD92'i32, 0x0FDC7'i32, + 0x0FDF0'i32, 0x0FDFB'i32, + 0x0FE70'i32, 0x0FE74'i32, + 0x0FE76'i32, 0x0FEFC'i32, + 0x0FF21'i32, 0x0FF3A'i32, + 0x0FF41'i32, 0x0FF5A'i32, + 0x0FF66'i32, 0x0FFBE'i32, + 0x0FFC2'i32, 0x0FFC7'i32, + 0x0FFCA'i32, 0x0FFCF'i32, + 0x0FFD2'i32, 0x0FFD7'i32, + 0x0FFDA'i32, 0x0FFDC'i32, + 0x10000'i32, 0x1000B'i32, + 0x1000D'i32, 0x10026'i32, + 0x10028'i32, 0x1003A'i32, + 0x1003C'i32, 0x1003D'i32, + 0x1003F'i32, 0x1004D'i32, + 0x10050'i32, 0x1005D'i32, + 0x10080'i32, 0x100FA'i32, + 0x10280'i32, 0x1029C'i32, + 0x102A0'i32, 0x102D0'i32, + 0x10300'i32, 0x1031F'i32, + 0x1032D'i32, 0x10340'i32, + 0x10342'i32, 0x10349'i32, + 0x10350'i32, 0x10375'i32, + 0x10380'i32, 0x1039D'i32, + 0x103A0'i32, 0x103C3'i32, + 0x103C8'i32, 0x103CF'i32, + 0x10400'i32, 0x1049D'i32, + 0x104B0'i32, 0x104D3'i32, + 0x104D8'i32, 0x104FB'i32, + 0x10500'i32, 0x10527'i32, + 0x10530'i32, 0x10563'i32, + 0x10600'i32, 0x10736'i32, + 0x10740'i32, 0x10755'i32, + 0x10760'i32, 0x10767'i32, + 0x10800'i32, 0x10805'i32, + 0x1080A'i32, 0x10835'i32, + 0x10837'i32, 0x10838'i32, + 0x1083F'i32, 0x10855'i32, + 0x10860'i32, 0x10876'i32, + 0x10880'i32, 0x1089E'i32, + 0x108E0'i32, 0x108F2'i32, + 0x108F4'i32, 0x108F5'i32, + 0x10900'i32, 0x10915'i32, + 0x10920'i32, 0x10939'i32, + 0x10980'i32, 0x109B7'i32, + 0x109BE'i32, 0x109BF'i32, + 0x10A10'i32, 0x10A13'i32, + 0x10A15'i32, 0x10A17'i32, + 0x10A19'i32, 0x10A35'i32, + 0x10A60'i32, 0x10A7C'i32, + 0x10A80'i32, 0x10A9C'i32, + 0x10AC0'i32, 0x10AC7'i32, + 0x10AC9'i32, 0x10AE4'i32, + 0x10B00'i32, 0x10B35'i32, + 0x10B40'i32, 0x10B55'i32, + 0x10B60'i32, 0x10B72'i32, + 0x10B80'i32, 0x10B91'i32, + 0x10C00'i32, 0x10C48'i32, + 0x10C80'i32, 0x10CB2'i32, + 0x10CC0'i32, 0x10CF2'i32, + 0x10D00'i32, 0x10D23'i32, + 0x10F00'i32, 0x10F1C'i32, + 0x10F30'i32, 0x10F45'i32, + 0x10FE0'i32, 0x10FF6'i32, + 0x11003'i32, 0x11037'i32, + 0x11083'i32, 0x110AF'i32, + 0x110D0'i32, 0x110E8'i32, + 0x11103'i32, 0x11126'i32, + 0x11150'i32, 0x11172'i32, + 0x11183'i32, 0x111B2'i32, + 0x111C1'i32, 0x111C4'i32, + 0x11200'i32, 0x11211'i32, + 0x11213'i32, 0x1122B'i32, + 0x11280'i32, 0x11286'i32, + 0x1128A'i32, 0x1128D'i32, + 0x1128F'i32, 0x1129D'i32, + 0x1129F'i32, 0x112A8'i32, + 0x112B0'i32, 0x112DE'i32, + 0x11305'i32, 0x1130C'i32, + 0x1130F'i32, 0x11310'i32, + 0x11313'i32, 0x11328'i32, + 0x1132A'i32, 0x11330'i32, + 0x11332'i32, 0x11333'i32, + 0x11335'i32, 0x11339'i32, + 0x1135D'i32, 0x11361'i32, + 0x11400'i32, 0x11434'i32, + 0x11447'i32, 0x1144A'i32, + 0x11480'i32, 0x114AF'i32, + 0x114C4'i32, 0x114C5'i32, + 0x11580'i32, 0x115AE'i32, + 0x115D8'i32, 0x115DB'i32, + 0x11600'i32, 0x1162F'i32, + 0x11680'i32, 0x116AA'i32, + 0x11700'i32, 0x1171A'i32, + 0x11800'i32, 0x1182B'i32, + 0x118A0'i32, 0x118DF'i32, + 0x119A0'i32, 0x119A7'i32, + 0x119AA'i32, 0x119D0'i32, + 0x11A0B'i32, 0x11A32'i32, + 0x11A5C'i32, 0x11A89'i32, + 0x11AC0'i32, 0x11AF8'i32, + 0x11C00'i32, 0x11C08'i32, + 0x11C0A'i32, 0x11C2E'i32, + 0x11C72'i32, 0x11C8F'i32, + 0x11D00'i32, 0x11D06'i32, + 0x11D08'i32, 0x11D09'i32, + 0x11D0B'i32, 0x11D30'i32, + 0x11D60'i32, 0x11D65'i32, + 0x11D67'i32, 0x11D68'i32, + 0x11D6A'i32, 0x11D89'i32, + 0x11EE0'i32, 0x11EF2'i32, + 0x12000'i32, 0x12399'i32, + 0x12480'i32, 0x12543'i32, + 0x13000'i32, 0x1342E'i32, + 0x14400'i32, 0x14646'i32, + 0x16800'i32, 0x16A38'i32, + 0x16A40'i32, 0x16A5E'i32, + 0x16AD0'i32, 0x16AED'i32, + 0x16B00'i32, 0x16B2F'i32, + 0x16B40'i32, 0x16B43'i32, + 0x16B63'i32, 0x16B77'i32, + 0x16B7D'i32, 0x16B8F'i32, + 0x16E40'i32, 0x16E7F'i32, + 0x16F00'i32, 0x16F4A'i32, + 0x16F93'i32, 0x16F9F'i32, + 0x16FE0'i32, 0x16FE1'i32, + 0x17000'i32, 0x187F7'i32, + 0x18800'i32, 0x18AF2'i32, + 0x1B000'i32, 0x1B11E'i32, + 0x1B150'i32, 0x1B152'i32, + 0x1B164'i32, 0x1B167'i32, + 0x1B170'i32, 0x1B2FB'i32, + 0x1BC00'i32, 0x1BC6A'i32, + 0x1BC70'i32, 0x1BC7C'i32, + 0x1BC80'i32, 0x1BC88'i32, + 0x1BC90'i32, 0x1BC99'i32, + 0x1D400'i32, 0x1D454'i32, + 0x1D456'i32, 0x1D49C'i32, + 0x1D49E'i32, 0x1D49F'i32, + 0x1D4A5'i32, 0x1D4A6'i32, + 0x1D4A9'i32, 0x1D4AC'i32, + 0x1D4AE'i32, 0x1D4B9'i32, + 0x1D4BD'i32, 0x1D4C3'i32, + 0x1D4C5'i32, 0x1D505'i32, + 0x1D507'i32, 0x1D50A'i32, + 0x1D50D'i32, 0x1D514'i32, + 0x1D516'i32, 0x1D51C'i32, + 0x1D51E'i32, 0x1D539'i32, + 0x1D53B'i32, 0x1D53E'i32, + 0x1D540'i32, 0x1D544'i32, + 0x1D54A'i32, 0x1D550'i32, + 0x1D552'i32, 0x1D6A5'i32, + 0x1D6A8'i32, 0x1D6C0'i32, + 0x1D6C2'i32, 0x1D6DA'i32, + 0x1D6DC'i32, 0x1D6FA'i32, + 0x1D6FC'i32, 0x1D714'i32, + 0x1D716'i32, 0x1D734'i32, + 0x1D736'i32, 0x1D74E'i32, + 0x1D750'i32, 0x1D76E'i32, + 0x1D770'i32, 0x1D788'i32, + 0x1D78A'i32, 0x1D7A8'i32, + 0x1D7AA'i32, 0x1D7C2'i32, + 0x1D7C4'i32, 0x1D7CB'i32, + 0x1E100'i32, 0x1E12C'i32, + 0x1E137'i32, 0x1E13D'i32, + 0x1E2C0'i32, 0x1E2EB'i32, + 0x1E800'i32, 0x1E8C4'i32, + 0x1E900'i32, 0x1E943'i32, + 0x1EE00'i32, 0x1EE03'i32, + 0x1EE05'i32, 0x1EE1F'i32, + 0x1EE21'i32, 0x1EE22'i32, + 0x1EE29'i32, 0x1EE32'i32, + 0x1EE34'i32, 0x1EE37'i32, + 0x1EE4D'i32, 0x1EE4F'i32, + 0x1EE51'i32, 0x1EE52'i32, + 0x1EE61'i32, 0x1EE62'i32, + 0x1EE67'i32, 0x1EE6A'i32, + 0x1EE6C'i32, 0x1EE72'i32, + 0x1EE74'i32, 0x1EE77'i32, + 0x1EE79'i32, 0x1EE7C'i32, + 0x1EE80'i32, 0x1EE89'i32, + 0x1EE8B'i32, 0x1EE9B'i32, + 0x1EEA1'i32, 0x1EEA3'i32, + 0x1EEA5'i32, 0x1EEA9'i32, + 0x1EEAB'i32, 0x1EEBB'i32, + 0x20000'i32, 0x2A6D6'i32, + 0x2A700'i32, 0x2B734'i32, + 0x2B740'i32, 0x2B81D'i32, + 0x2B820'i32, 0x2CEA1'i32, + 0x2CEB0'i32, 0x2EBE0'i32, + 0x2F800'i32, 0x2FA1D'i32, + ] + + alphaSinglets = [ + 0x000AA'i32, + 0x000B5'i32, + 0x000BA'i32, + 0x002EC'i32, + 0x002EE'i32, + 0x0037F'i32, + 0x00386'i32, + 0x0038C'i32, + 0x00559'i32, + 0x006D5'i32, + 0x006FF'i32, + 0x00710'i32, + 0x007B1'i32, + 0x007FA'i32, + 0x0081A'i32, + 0x00824'i32, + 0x00828'i32, + 0x0093D'i32, + 0x00950'i32, + 0x009B2'i32, + 0x009BD'i32, + 0x009CE'i32, + 0x009FC'i32, + 0x00A5E'i32, + 0x00ABD'i32, + 0x00AD0'i32, + 0x00AF9'i32, + 0x00B3D'i32, + 0x00B71'i32, + 0x00B83'i32, + 0x00B9C'i32, + 0x00BD0'i32, + 0x00C3D'i32, + 0x00C80'i32, + 0x00CBD'i32, + 0x00CDE'i32, + 0x00D3D'i32, + 0x00D4E'i32, + 0x00DBD'i32, + 0x00E84'i32, + 0x00EA5'i32, + 0x00EBD'i32, + 0x00EC6'i32, + 0x00F00'i32, + 0x0103F'i32, + 0x01061'i32, + 0x0108E'i32, + 0x010C7'i32, + 0x010CD'i32, + 0x01258'i32, + 0x012C0'i32, + 0x017D7'i32, + 0x017DC'i32, + 0x018AA'i32, + 0x01AA7'i32, + 0x01CFA'i32, + 0x01F59'i32, + 0x01F5B'i32, + 0x01F5D'i32, + 0x01FBE'i32, + 0x02071'i32, + 0x0207F'i32, + 0x02102'i32, + 0x02107'i32, + 0x02115'i32, + 0x02124'i32, + 0x02126'i32, + 0x02128'i32, + 0x0214E'i32, + 0x02D27'i32, + 0x02D2D'i32, + 0x02D6F'i32, + 0x02E2F'i32, + 0x0A8FB'i32, + 0x0A9CF'i32, + 0x0AA7A'i32, + 0x0AAB1'i32, + 0x0AAC0'i32, + 0x0AAC2'i32, + 0x0FB1D'i32, + 0x0FB3E'i32, + 0x10808'i32, + 0x1083C'i32, + 0x10A00'i32, + 0x10F27'i32, + 0x11144'i32, + 0x11176'i32, + 0x111DA'i32, + 0x111DC'i32, + 0x11288'i32, + 0x1133D'i32, + 0x11350'i32, + 0x1145F'i32, + 0x114C7'i32, + 0x11644'i32, + 0x116B8'i32, + 0x118FF'i32, + 0x119E1'i32, + 0x119E3'i32, + 0x11A00'i32, + 0x11A3A'i32, + 0x11A50'i32, + 0x11A9D'i32, + 0x11C40'i32, + 0x11D46'i32, + 0x11D98'i32, + 0x16F50'i32, + 0x16FE3'i32, + 0x1D4A2'i32, + 0x1D4BB'i32, + 0x1D546'i32, + 0x1E14E'i32, + 0x1E94B'i32, + 0x1EE24'i32, + 0x1EE27'i32, + 0x1EE39'i32, + 0x1EE3B'i32, + 0x1EE42'i32, + 0x1EE47'i32, + 0x1EE49'i32, + 0x1EE4B'i32, + 0x1EE54'i32, + 0x1EE57'i32, + 0x1EE59'i32, + 0x1EE5B'i32, + 0x1EE5D'i32, + 0x1EE5F'i32, + 0x1EE64'i32, + 0x1EE7E'i32, + ] + + spaceRanges = [ + 0x00009'i32, 0x0000D'i32, + 0x00020'i32, 0x00020'i32, + 0x00085'i32, 0x00085'i32, + 0x000A0'i32, 0x000A0'i32, + 0x01680'i32, 0x01680'i32, + 0x02000'i32, 0x0200A'i32, + 0x02028'i32, 0x02029'i32, + 0x0202F'i32, 0x0202F'i32, + 0x0205F'i32, 0x0205F'i32, + 0x03000'i32, 0x03000'i32, + ] + + unicodeSpaces = [ + Rune 0x00009, + Rune 0x0000A, + Rune 0x0000B, + Rune 0x0000C, + Rune 0x0000D, + Rune 0x00020, + Rune 0x00085, + Rune 0x000A0, + Rune 0x01680, + Rune 0x02000, + Rune 0x02001, + Rune 0x02002, + Rune 0x02003, + Rune 0x02004, + Rune 0x02005, + Rune 0x02006, + Rune 0x02007, + Rune 0x02008, + Rune 0x02009, + Rune 0x0200A, + Rune 0x02028, + Rune 0x02029, + Rune 0x0202F, + Rune 0x0205F, + Rune 0x03000, + ] + diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim new file mode 100644 index 000000000..10658b78e --- /dev/null +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -0,0 +1,534 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Linux epoll(). + +import std/[posix, times, epoll] + +# Maximum number of events that can be returned +const MAX_EPOLL_EVENTS = 64 + +when not defined(android): + type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "<sys/signalfd.h>", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] + +proc timerfd_create(clock_id: ClockId, flags: cint): cint + {.cdecl, importc: "timerfd_create", header: "<sys/timerfd.h>".} +proc timerfd_settime(ufd: cint, flags: cint, + utmr: var Itimerspec, otmr: var Itimerspec): cint + {.cdecl, importc: "timerfd_settime", header: "<sys/timerfd.h>".} +proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} + +when not defined(android): + proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} + +when hasThreadSupport: + type + SelectorImpl[T] = object + epollFD: cint + maxFD: int + numFD: int + fds: ptr SharedArray[SelectorKey[T]] + count*: int + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + epollFD: cint + maxFD: int + numFD: int + fds: seq[SelectorKey[T]] + count*: int + Selector*[T] = ref SelectorImpl[T] +type + SelectEventImpl = object + efd: cint + SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 + # Retrieve the maximum fd count (for current OS) via getrlimit() + var maxFD = maxDescriptors() + doAssert(maxFD > 0) + # Start with a reasonable size, checkFd() will grow this on demand + let numFD = initialNumFD() + + var epollFD = epoll_create1(O_CLOEXEC) + if epollFD < 0: + raiseOSError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.epollFD = epollFD + result.maxFD = maxFD + result.numFD = numFD + result.fds = allocSharedArray[SelectorKey[T]](numFD) + else: + result = Selector[T]() + result.epollFD = epollFD + result.maxFD = maxFD + result.numFD = numFD + result.fds = newSeq[SelectorKey[T]](numFD) + + for i in 0 ..< numFD: + result.fds[i].ident = InvalidIdent + +proc close*[T](s: Selector[T]) = + let res = posix.close(s.epollFD) + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + if res != 0: + raiseIOSelectorsError(osLastError()) + +proc newSelectEvent*(): SelectEvent = + let fdci = eventfd(0, O_CLOEXEC or O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.efd = fdci + +proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: + raiseIOSelectorsError(osLastError()) + +proc close*(ev: SelectEvent) = + let res = posix.close(ev.efd) + deallocShared(cast[pointer](ev)) + if res != 0: + raiseIOSelectorsError(osLastError()) + +template checkFd(s, f) = + # TODO: I don't see how this can ever happen. You won't be able to create an + # FD if there is too many. -- DP + if f >= s.maxFD: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + if f >= s.numFD: + var numFD = s.numFD + while numFD <= f: numFD *= 2 + when hasThreadSupport: + s.fds = reallocSharedArray(s.fds, s.numFD, numFD) + else: + s.fds.setLen(numFD) + for i in s.numFD ..< numFD: + s.fds[i].ident = InvalidIdent + s.numFD = numFD + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent, "Descriptor $# already registered" % $fdi) + s.setKey(fdi, events, 0, data) + if events != {}: + var epv = EpollEvent(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor $# is not registered in the selector!" % $fdi) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + var epv = EpollEvent(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + + if pkey.events == {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + inc(s.count) + else: + if events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + else: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor $# is not registered in the selector!" % $fdi) + if pkey.events != {}: + when not defined(android): + if Event.Read in pkey.events or Event.Write in pkey.events or Event.User in pkey.events: + var epv = EpollEvent() + # TODO: Refactor all these EPOLL_CTL_DEL + dec(s.count) into a proc. + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + if Event.Finished notin pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Signal in pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Process in pkey.events: + if Event.Finished notin pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + else: + if Event.Read in pkey.events or Event.Write in pkey.events or Event.User in pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + if Event.Finished notin pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + clearKey(pkey) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.efd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") + doAssert(Event.User in pkey.events) + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + clearKey(pkey) + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var + newTs: Itimerspec + oldTs: Itimerspec + let fdi = timerfd_create(CLOCK_MONOTONIC, O_CLOEXEC or O_NONBLOCK).int + if fdi == -1: + raiseIOSelectorsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var events = {Event.Timer} + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if oneshot: + newTs.it_interval.tv_sec = posix.Time(0) + newTs.it_interval.tv_nsec = 0 + newTs.it_value.tv_sec = posix.Time(timeout div 1_000) + newTs.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 + incl(events, Event.Oneshot) + epv.events = epv.events or EPOLLONESHOT + else: + newTs.it_interval.tv_sec = posix.Time(timeout div 1000) + newTs.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 + newTs.it_value.tv_sec = newTs.it_interval.tv_sec + newTs.it_value.tv_nsec = newTs.it_interval.tv_nsec + + if timerfd_settime(fdi.cint, cint(0), newTs, oldTs) != 0: + raiseIOSelectorsError(osLastError()) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + s.setKey(fdi, events, 0, data) + inc(s.count) + result = fdi + +when not defined(android): + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, O_CLOEXEC or O_NONBLOCK).int + if fdi == -1: + raiseIOSelectorsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + s.setKey(fdi, {Event.Signal}, signal, data) + inc(s.count) + result = fdi + + proc registerProcess*[T](s: Selector, pid: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, O_CLOEXEC or O_NONBLOCK).int + if fdi == -1: + raiseIOSelectorsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + s.setKey(fdi, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = int(ev.efd) + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") + s.setKey(fdi, {Event.User}, 0, data) + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = ev.efd.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + inc(s.count) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int = + var + resTable: array[MAX_EPOLL_EVENTS, EpollEvent] + maxres = MAX_EPOLL_EVENTS + i, k: int + + if maxres > len(results): + maxres = len(results) + + verifySelectParams(timeout) + + let count = epoll_wait(s.epollFD, addr(resTable[0]), maxres.cint, + timeout.cint) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + i = 0 + k = 0 + while i < count: + let fdi = int(resTable[i].data.u64) + let pevents = resTable[i].events + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent) + var rkey = ReadyKey(fd: fdi, events: {}) + + if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + if (pevents and EPOLLHUP) != 0: + rkey.errorCode = OSErrorCode ECONNRESET + else: + # Try reading SO_ERROR from fd. + var error: cint + var size = SockLen sizeof(error) + if getsockopt(SocketHandle fdi, SOL_SOCKET, SO_ERROR, addr(error), + addr(size)) == 0'i32: + rkey.errorCode = OSErrorCode error + + rkey.events.incl(Event.Error) + if (pevents and EPOLLOUT) != 0: + rkey.events.incl(Event.Write) + when not defined(android): + if (pevents and EPOLLIN) != 0: + if Event.Read in pkey.events: + rkey.events.incl(Event.Read) + elif Event.Timer in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + rkey.events.incl(Event.Timer) + elif Event.Signal in pkey.events: + var data = SignalFdInfo() + if posix.read(cint(fdi), addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseIOSelectorsError(osLastError()) + rkey.events.incl(Event.Signal) + elif Event.Process in pkey.events: + var data = SignalFdInfo() + if posix.read(cint(fdi), addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseIOSelectorsError(osLastError()) + if cast[int](data.ssi_pid) == pkey.param: + rkey.events.incl(Event.Process) + else: + inc(i) + continue + elif Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseIOSelectorsError(err) + rkey.events.incl(Event.User) + else: + if (pevents and EPOLLIN) != 0: + if Event.Read in pkey.events: + rkey.events.incl(Event.Read) + elif Event.Timer in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + rkey.events.incl(Event.Timer) + elif Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseIOSelectorsError(err) + rkey.events.incl(Event.User) + + if Event.Oneshot in pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, cint(fdi), addr epv) != 0: + raiseIOSelectorsError(osLastError()) + # we will not clear key until it will be unregistered, so + # application can obtain data, but we will decrease counter, + # because epoll is empty. + dec(s.count) + # we are marking key with `Finished` event, to avoid double decrease. + pkey.events.incl(Event.Finished) + + results[k] = rkey + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](MAX_EPOLL_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd.int].ident != InvalidIdent + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + result = s.fds[fdi].data + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + s.fds[fdi].data = data + result = true + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.fds[fdi].data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.fds[fdi].data) + body1 + else: + body2 + +proc getFd*[T](s: Selector[T]): int = + return s.epollFD.int diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim new file mode 100644 index 000000000..513578eda --- /dev/null +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -0,0 +1,639 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements BSD kqueue(). + +import std/[posix, times, kqueue, nativesockets] + +const + # Maximum number of events that can be returned. + MAX_KQUEUE_EVENTS = 64 + # SIG_IGN and SIG_DFL declared in posix.nim as variables, but we need them + # to be constants and GC-safe. + SIG_DFL = cast[proc(x: cint) {.noconv,gcsafe.}](0) + SIG_IGN = cast[proc(x: cint) {.noconv,gcsafe.}](1) + +when defined(kqcache): + const CACHE_EVENTS = true + +when defined(macosx) or defined(freebsd) or defined(dragonfly): + when defined(macosx): + const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) + else: + const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, + newp: pointer, newplen: csize_t): cint + {.importc: "sysctl",header: """#include <sys/types.h> + #include <sys/sysctl.h>""".} +elif defined(netbsd) or defined(openbsd): + # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using + # KERN_MAXFILES, because KERN_MAXFILES is always bigger, + # than KERN_MAXFILESPERPROC. + const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, + newp: pointer, newplen: csize_t): cint + {.importc: "sysctl",header: """#include <sys/param.h> + #include <sys/sysctl.h>""".} + +when hasThreadSupport: + type + SelectorImpl[T] = object + kqFD: cint + maxFD: int + changes: ptr SharedArray[KEvent] + fds: ptr SharedArray[SelectorKey[T]] + count*: int + changesLock: Lock + changesSize: int + changesLength: int + sock: cint + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + kqFD: cint + maxFD: int + changes: seq[KEvent] + fds: seq[SelectorKey[T]] + count*: int + sock: cint + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + + SelectEvent* = ptr SelectEventImpl + # SelectEvent is declared as `ptr` to be placed in `shared memory`, + # so you can share one SelectEvent handle between threads. + +proc getUnique[T](s: Selector[T]): int {.inline.} = + # we create duplicated handles to get unique indexes for our `fds` array. + result = posix.fcntl(s.sock, F_DUPFD_CLOEXEC, s.sock) + if result == -1: + raiseIOSelectorsError(osLastError()) + +proc newSelector*[T](): owned(Selector[T]) = + var maxFD = 0.cint + var size = csize_t(sizeof(cint)) + var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] + # Obtain maximum number of opened file descriptors for process + if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, + nil, 0) != 0: + raiseIOSelectorsError(osLastError()) + + var kqFD = kqueue() + if kqFD < 0: + raiseIOSelectorsError(osLastError()) + + # we allocating empty socket to duplicate it handle in future, to get unique + # indexes for `fds` array. This is needed to properly identify + # {Event.Timer, Event.Signal, Event.Process} events. + let usock = createNativeSocket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).cint + if usock == -1: + let err = osLastError() + discard posix.close(kqFD) + raiseIOSelectorsError(err) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.changes = allocSharedArray[KEvent](MAX_KQUEUE_EVENTS) + result.changesSize = MAX_KQUEUE_EVENTS + initLock(result.changesLock) + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](maxFD) + result.changes = newSeqOfCap[KEvent](MAX_KQUEUE_EVENTS) + + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + + result.sock = usock + result.kqFD = kqFD + result.maxFD = maxFD.int + +proc close*[T](s: Selector[T]) = + let res1 = posix.close(s.kqFD) + let res2 = posix.close(s.sock) + when hasThreadSupport: + deinitLock(s.changesLock) + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + +proc close*(ev: SelectEvent) = + let res1 = posix.close(ev.rfd) + let res2 = posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +template checkFd(s, f) = + if f >= s.maxFD: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + +when hasThreadSupport: + template withChangeLock[T](s: Selector[T], body: untyped) = + acquire(s.changesLock) + {.locks: [s.changesLock].}: + try: + body + finally: + release(s.changesLock) +else: + template withChangeLock(s, body: untyped) = + body + +when hasThreadSupport: + template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + mixin withChangeLock + s.withChangeLock(): + if s.changesLength == s.changesSize: + # if cache array is full, we allocating new with size * 2 + let newSize = s.changesSize shl 1 + let rdata = allocSharedArray[KEvent](newSize) + copyMem(rdata, s.changes, s.changesSize * sizeof(KEvent)) + s.changesSize = newSize + s.changes[s.changesLength] = KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata) + inc(s.changesLength) + + when not declared(CACHE_EVENTS): + template flushKQueue[T](s: Selector[T]) = + mixin withChangeLock + s.withChangeLock(): + if s.changesLength > 0: + if kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), + nil, 0, nil) == -1: + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) + s.changesLength = 0 +else: + template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + s.changes.add(KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata)) + + when not declared(CACHE_EVENTS): + template flushKQueue[T](s: Selector[T]) = + let length = cint(len(s.changes)) + if length > 0: + if kevent(s.kqFD, addr(s.changes[0]), length, + nil, 0, nil) == -1: + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) + s.changes.setLen(0) + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + s.setKey(fdi, events, 0, data) + + if events != {}: + if Event.Read in events: + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if Event.Write in events: + modifyKQueue(s, uint(fdi), EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor $# is not registered in the queue!" % $fdi) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + pkey.events = events + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + let fdi = getUnique(s) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} + let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD + + s.setKey(fdi, events, 0, data) + + # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, + # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds + # too + modifyKQueue(s, fdi.uint, EVFILT_TIMER, flags, 0, cint(timeout), nil) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + let fdi = getUnique(s) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + s.setKey(fdi, {Event.Signal}, signal, data) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + # to be compatible with linux semantic we need to "eat" signals + posix.signal(cint(signal), SIG_IGN) + + modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, + cast[pointer](fdi)) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + let fdi = getUnique(s) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var kflags: cushort = EV_ONESHOT or EV_ADD + setKey(s, fdi, {Event.Process, Event.Oneshot}, pid, data) + + modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, + cast[pointer](fdi)) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = ev.rfd.int + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") + setKey(s, fdi, {Event.User}, 0, data) + + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + +template processVnodeEvents(events: set[Event]): cuint = + var rfflags = 0.cuint + if events == {Event.VnodeWrite, Event.VnodeDelete, Event.VnodeExtend, + Event.VnodeAttrib, Event.VnodeLink, Event.VnodeRename, + Event.VnodeRevoke}: + rfflags = NOTE_DELETE or NOTE_WRITE or NOTE_EXTEND or NOTE_ATTRIB or + NOTE_LINK or NOTE_RENAME or NOTE_REVOKE + else: + if Event.VnodeDelete in events: rfflags = rfflags or NOTE_DELETE + if Event.VnodeWrite in events: rfflags = rfflags or NOTE_WRITE + if Event.VnodeExtend in events: rfflags = rfflags or NOTE_EXTEND + if Event.VnodeAttrib in events: rfflags = rfflags or NOTE_ATTRIB + if Event.VnodeLink in events: rfflags = rfflags or NOTE_LINK + if Event.VnodeRename in events: rfflags = rfflags or NOTE_RENAME + if Event.VnodeRevoke in events: rfflags = rfflags or NOTE_REVOKE + rfflags + +proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], data: T) = + let fdi = fd.int + setKey(s, fdi, {Event.Vnode} + events, 0, data) + var fflags = processVnodeEvents(events) + + modifyKQueue(s, fdi.uint, EVFILT_VNODE, EV_ADD or EV_CLEAR, fflags, 0, nil) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor [" & $fdi & "] is not registered in the queue!") + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + if Event.Read in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if Event.Write in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + when not declared(CACHE_EVENTS): + flushKQueue(s) + elif Event.Timer in pkey.events: + if Event.Finished notin pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_TIMER, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + if posix.close(cint(pkey.ident)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Signal in pkey.events: + var nmask, omask: Sigset + let signal = cint(pkey.param) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, signal) + unblockSignals(nmask, omask) + posix.signal(signal, SIG_DFL) + modifyKQueue(s, uint(pkey.param), EVFILT_SIGNAL, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + if posix.close(cint(pkey.ident)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Process in pkey.events: + if Event.Finished notin pkey.events: + modifyKQueue(s, uint(pkey.param), EVFILT_PROC, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + if posix.close(cint(pkey.ident)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Vnode in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_VNODE, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + elif Event.User in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + + clearKey(pkey) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") + doAssert(Event.User in pkey.events) + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + clearKey(pkey) + dec(s.count) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int = + var + tv: Timespec + resTable: array[MAX_KQUEUE_EVENTS, KEvent] + ptv = addr tv + maxres = MAX_KQUEUE_EVENTS + + verifySelectParams(timeout) + + if timeout != -1: + if timeout >= 1000: + tv.tv_sec = posix.Time(timeout div 1_000) + tv.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tv.tv_sec = posix.Time(0) + tv.tv_nsec = timeout * 1_000_000 + else: + ptv = nil + + if maxres > len(results): + maxres = len(results) + + var count = 0 + when not declared(CACHE_EVENTS): + count = kevent(s.kqFD, nil, cint(0), addr(resTable[0]), cint(maxres), ptv) + else: + when hasThreadSupport: + s.withChangeLock(): + if s.changesLength > 0: + count = kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), + addr(resTable[0]), cint(maxres), ptv) + s.changesLength = 0 + else: + count = kevent(s.kqFD, nil, cint(0), addr(resTable[0]), cint(maxres), + ptv) + else: + let length = cint(len(s.changes)) + if length > 0: + count = kevent(s.kqFD, addr(s.changes[0]), length, + addr(resTable[0]), cint(maxres), ptv) + s.changes.setLen(0) + else: + count = kevent(s.kqFD, nil, cint(0), addr(resTable[0]), cint(maxres), + ptv) + + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 # do not delete this, because `continue` used in cycle. + var pkey: ptr SelectorKey[T] + while i < count: + let kevent = addr(resTable[i]) + var rkey = ReadyKey(fd: int(kevent.ident), events: {}) + + if (kevent.flags and EV_ERROR) != 0: + rkey.events = {Event.Error} + rkey.errorCode = OSErrorCode(kevent.data) + + case kevent.filter: + of EVFILT_READ: + pkey = addr(s.fds[int(kevent.ident)]) + rkey.events.incl(Event.Read) + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(cint(kevent.ident), addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + # someone already consumed event data + inc(i) + continue + else: + raiseIOSelectorsError(err) + rkey.events = {Event.User} + of EVFILT_WRITE: + pkey = addr(s.fds[int(kevent.ident)]) + rkey.events.incl(Event.Write) + rkey.events = {Event.Write} + of EVFILT_TIMER: + pkey = addr(s.fds[int(kevent.ident)]) + if Event.Oneshot in pkey.events: + # we will not clear key until it will be unregistered, so + # application can obtain data, but we will decrease counter, + # because kqueue is empty. + dec(s.count) + # we are marking key with `Finished` event, to avoid double decrease. + pkey.events.incl(Event.Finished) + rkey.events.incl(Event.Timer) + of EVFILT_VNODE: + pkey = addr(s.fds[int(kevent.ident)]) + rkey.events.incl(Event.Vnode) + if (kevent.fflags and NOTE_DELETE) != 0: + rkey.events.incl(Event.VnodeDelete) + if (kevent.fflags and NOTE_WRITE) != 0: + rkey.events.incl(Event.VnodeWrite) + if (kevent.fflags and NOTE_EXTEND) != 0: + rkey.events.incl(Event.VnodeExtend) + if (kevent.fflags and NOTE_ATTRIB) != 0: + rkey.events.incl(Event.VnodeAttrib) + if (kevent.fflags and NOTE_LINK) != 0: + rkey.events.incl(Event.VnodeLink) + if (kevent.fflags and NOTE_RENAME) != 0: + rkey.events.incl(Event.VnodeRename) + if (kevent.fflags and NOTE_REVOKE) != 0: + rkey.events.incl(Event.VnodeRevoke) + of EVFILT_SIGNAL: + pkey = addr(s.fds[cast[int](kevent.udata)]) + rkey.fd = cast[int](kevent.udata) + rkey.events.incl(Event.Signal) + of EVFILT_PROC: + rkey.fd = cast[int](kevent.udata) + pkey = addr(s.fds[cast[int](kevent.udata)]) + # we will not clear key, until it will be unregistered, so + # application can obtain data, but we will decrease counter, + # because kqueue is empty. + dec(s.count) + # we are marking key with `Finished` event, to avoid double decrease. + pkey.events.incl(Event.Finished) + rkey.events.incl(Event.Process) + else: + doAssert(true, "Unsupported kqueue filter in the queue!") + + if (kevent.flags and EV_EOF) != 0: + # TODO this error handling needs to be rethought. + # `fflags` can sometimes be `0x80000000` and thus we use 'cast' + # here: + if kevent.fflags != 0: + rkey.errorCode = cast[OSErrorCode](kevent.fflags) + else: + # This assumes we are dealing with sockets. + # TODO: For future-proofing it might be a good idea to give the + # user access to the raw `kevent`. + rkey.errorCode = OSErrorCode(ECONNRESET) + rkey.events.incl(Event.Error) + + results[k] = rkey + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](MAX_KQUEUE_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd.int].ident != InvalidIdent + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + result = s.fds[fdi].data + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + s.fds[fdi].data = data + result = true + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.fds[fdi].data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.fds[fdi].data) + body1 + else: + body2 + + +proc getFd*[T](s: Selector[T]): int = + return s.kqFD.int diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim new file mode 100644 index 000000000..7c5347156 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -0,0 +1,324 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix poll(). + +import std/[posix, times] + +# Maximum number of events that can be returned +const MAX_POLL_EVENTS = 64 + +const hasEventFds = defined(zephyr) or defined(nimPollHasEventFds) + +when hasEventFds: + proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} + +when hasThreadSupport: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: ptr SharedArray[SelectorKey[T]] + pollfds: ptr SharedArray[TPollFd] + count*: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: seq[SelectorKey[T]] + pollfds: seq[TPollFd] + count*: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withPollLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withPollLock(s, body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + var maxFD = maxDescriptors() + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.pollfds = allocSharedArray[TPollFd](maxFD) + initLock(result.lock) + else: + result = Selector[T]() + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + result.pollfds = newSeq[TPollFd](maxFD) + + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deinitLock(s.lock) + deallocSharedArray(s.fds) + deallocSharedArray(s.pollfds) + deallocShared(cast[pointer](s)) + +template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + s.pollfds[s.pollcnt].fd = cint(sock) + s.pollfds[s.pollcnt].events = pollev + inc(s.count) + inc(s.pollcnt) + +template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var i = 0 + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + s.pollfds[i].events = pollev + break + inc(i) + doAssert(i < s.pollcnt, + "Descriptor [" & $sock & "] is not registered in the queue!") + +template pollRemove[T](s: Selector[T], sock: cint) = + withPollLock(s): + var i = 0 + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + if i == s.pollcnt - 1: + s.pollfds[i].fd = 0 + s.pollfds[i].events = 0 + s.pollfds[i].revents = 0 + else: + while i < (s.pollcnt - 1): + s.pollfds[i].fd = s.pollfds[i + 1].fd + s.pollfds[i].events = s.pollfds[i + 1].events + inc(i) + break + inc(i) + dec(s.pollcnt) + dec(s.count) + +template checkFd(s, f) = + if f >= s.maxFD: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + var fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + setKey(s, fdi, events, 0, data) + if events != {}: s.pollAdd(fdi.cint, events) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor [" & $fdi & "] is not registered in the queue!") + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if pkey.events == {}: + s.pollAdd(fd.cint, events) + else: + if events != {}: + s.pollUpdate(fd.cint, events) + else: + s.pollRemove(fd.cint) + pkey.events = events + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + var fdi = int(ev.rfd) + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") + var events = {Event.User} + setKey(s, fdi, events, 0, data) + events.incl(Event.Read) + s.pollAdd(fdi.cint, events) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor [" & $fdi & "] is not registered in the queue!") + pkey.ident = InvalidIdent + if pkey.events != {}: + pkey.events = {} + s.pollRemove(fdi.cint) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") + doAssert(Event.User in pkey.events) + pkey.ident = InvalidIdent + pkey.events = {} + s.pollRemove(fdi.cint) + +proc newSelectEvent*(): SelectEvent = + when not hasEventFds: + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + else: + let fdci = eventfd(0, posix.O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fdci + result.wfd = fdci + +proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + +proc close*(ev: SelectEvent) = + let res1 = posix.close(ev.rfd) + let res2 = + when hasEventFds: 0 + else: posix.close(ev.wfd) + + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int = + var maxres = MAX_POLL_EVENTS + if maxres > len(results): + maxres = len(results) + + verifySelectParams(timeout) + + s.withPollLock(): + let count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var rindex = 0 + while (i < s.pollcnt) and (k < count) and (rindex < maxres): + let revents = s.pollfds[i].revents + if revents != 0: + let fd = s.pollfds[i].fd + var pkey = addr(s.fds[fd]) + var rkey = ReadyKey(fd: int(fd), events: {}) + + if (revents and POLLIN) != 0: + rkey.events.incl(Event.Read) + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(fd, addr data, sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err != OSErrorCode(EAGAIN): + raiseIOSelectorsError(err) + else: + # someone already consumed event data + inc(i) + continue + rkey.events = {Event.User} + if (revents and POLLOUT) != 0: + rkey.events.incl(Event.Write) + if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or + (revents and POLLNVAL) != 0: + rkey.events.incl(Event.Error) + results[rindex] = rkey + s.pollfds[i].revents = 0 + inc(rindex) + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](MAX_POLL_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd.int].ident != InvalidIdent + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + result = s.fds[fdi].data + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + s.fds[fdi].data = data + result = true + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.getData(fdi)) + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.getData(fdi)) + body1 + else: + body2 + + +proc getFd*[T](s: Selector[T]): int = + return -1 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim new file mode 100644 index 000000000..6c516395b --- /dev/null +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -0,0 +1,454 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix and Windows select(). + +import std/[times, nativesockets] + +when defined(windows): + import std/winlean + when defined(gcc): + {.passl: "-lws2_32".} + elif defined(vcc): + {.passl: "ws2_32.lib".} + const platformHeaders = """#include <winsock2.h> + #include <windows.h>""" + const EAGAIN = WSAEWOULDBLOCK +else: + const platformHeaders = """#include <sys/select.h> + #include <sys/time.h> + #include <sys/types.h> + #include <unistd.h>""" +type + FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object +var + FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint + +proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet) + {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} +proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet) + {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} +proc IOFD_ZERO(fdset: ptr FdSet) + {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} + +when defined(windows): + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint + {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, + timeout: ptr Timeval): cint + {.stdcall, importc: "select", header: platformHeaders.} +else: + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint + {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, + timeout: ptr Timeval): cint + {.cdecl, importc: "select", header: platformHeaders.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: ptr SharedArray[SelectorKey[T]] + count*: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: seq[SelectorKey[T]] + count*: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rsock: SocketHandle + wsock: SocketHandle + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](FD_SETSIZE) + initLock result.lock + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + + for i in 0 ..< FD_SETSIZE: + result.fds[i].ident = InvalidIdent + + IOFD_ZERO(addr result.rSet) + IOFD_ZERO(addr result.wSet) + IOFD_ZERO(addr result.eSet) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + deinitLock(s.lock) + +when defined(windows): + proc newSelectEvent*(): SelectEvent = + var ssock = createNativeSocket() + var wsock = createNativeSocket() + var rsock: SocketHandle = INVALID_SOCKET + var saddr = Sockaddr_in() + + saddr.sin_family = winlean.AF_INET + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseIOSelectorsError(osLastError()) + + if winlean.listen(ssock, 1) != 0: + raiseIOSelectorsError(osLastError()) + + var namelen = sizeof(saddr).SockLen + if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), + addr(namelen)) != 0'i32: + raiseIOSelectorsError(osLastError()) + + saddr.sin_addr.s_addr = 0x0100007F + if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) != 0: + raiseIOSelectorsError(osLastError()) + namelen = sizeof(saddr).SockLen + rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), + cast[ptr SockLen](addr(namelen))) + if rsock == SocketHandle(-1): + raiseIOSelectorsError(osLastError()) + + if winlean.closesocket(ssock) != 0: + raiseIOSelectorsError(osLastError()) + + var mode = clong(1) + if ioctlsocket(rsock, FIONBIO, addr(mode)) != 0: + raiseIOSelectorsError(osLastError()) + mode = clong(1) + if ioctlsocket(wsock, FIONBIO, addr(mode)) != 0: + raiseIOSelectorsError(osLastError()) + + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = rsock + result.wsock = wsock + + proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if winlean.send(ev.wsock, cast[pointer](addr data), + cint(sizeof(uint64)), 0) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + + proc close*(ev: SelectEvent) = + let res1 = winlean.closesocket(ev.rsock) + let res2 = winlean.closesocket(ev.wsock) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +else: + proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = SocketHandle(fds[0]) + result.wsock = SocketHandle(fds[1]) + + proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + + proc close*(ev: SelectEvent) = + let res1 = posix.close(cint(ev.rsock)) + let res2 = posix.close(cint(ev.wsock)) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +proc setSelectKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == InvalidIdent: + var pkey = addr(s.fds[i]) + pkey.ident = fdi + pkey.events = events + pkey.data = data + break + inc(i) + if i >= FD_SETSIZE: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + +proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + result = addr(s.fds[i]) + break + inc(i) + doAssert(i < FD_SETSIZE, + "Descriptor [" & $int(fd) & "] is not registered in the queue!") + +proc delKey[T](s: Selector[T], fd: SocketHandle) = + var empty: T + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fd.int: + s.fds[i].ident = InvalidIdent + s.fds[i].events = {} + s.fds[i].data = empty + break + inc(i) + doAssert(i < FD_SETSIZE, + "Descriptor [" & $int(fd) & "] is not registered in the queue!") + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + when not defined(windows): + let fdi = int(fd) + s.withSelectLock(): + s.setSelectKey(fd, events, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + if Event.Read in events: + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if Event.Write in events: + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + when not defined(windows): + let fdi = int(ev.rsock) + s.withSelectLock(): + s.setSelectKey(ev.rsock, {Event.User}, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + IOFD_SET(ev.rsock, addr s.rSet) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + s.withSelectLock(): + var pkey = s.getKey(fd) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: SocketHandle|int) = + s.withSelectLock(): + let fd = fd.SocketHandle + var pkey = s.getKey(fd) + if Event.Read in pkey.events or Event.User in pkey.events: + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if Event.Write in pkey.events: + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + s.delKey(fd) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fd = ev.rsock + s.withSelectLock(): + var pkey = s.getKey(fd) + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + s.delKey(fd) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int = + var tv = Timeval() + var ptv = addr tv + var rset, wset, eset: FdSet + + verifySelectParams(timeout) + + if timeout != -1: + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): + tv.tv_sec = posix.Time(timeout div 1_000) + else: + tv.tv_sec = timeout.int32 div 1_000 + tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 + else: + ptv = nil + + s.withSelectLock(): + rset = s.rSet + wset = s.wSet + eset = s.eSet + + var count = ioselect(cint(s.maxFD) + 1, addr(rset), addr(wset), + addr(eset), ptv) + if count < 0: + result = 0 + when defined(windows): + raiseIOSelectorsError(osLastError()) + else: + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + var rindex = 0 + var i = 0 + var k = 0 + + while (i < FD_SETSIZE) and (k < count): + if s.fds[i].ident != InvalidIdent: + var flag = false + var pkey = addr(s.fds[i]) + var rkey = ReadyKey(fd: int(pkey.ident), events: {}) + let fd = SocketHandle(pkey.ident) + if IOFD_ISSET(fd, addr rset) != 0: + if Event.User in pkey.events: + var data: uint64 = 0 + if recv(fd, cast[pointer](addr(data)), + sizeof(uint64).cint, 0) != sizeof(uint64): + let err = osLastError() + if cint(err) != EAGAIN: + raiseIOSelectorsError(err) + else: + inc(i) + inc(k) + continue + else: + flag = true + rkey.events = {Event.User} + else: + flag = true + rkey.events = {Event.Read} + if IOFD_ISSET(fd, addr wset) != 0: + rkey.events.incl(Event.Write) + if IOFD_ISSET(fd, addr eset) != 0: + rkey.events.incl(Event.Error) + flag = true + if flag: + results[rindex] = rkey + inc(rindex) + inc(k) + inc(i) + result = rindex + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](FD_SETSIZE) + var count = selectInto(s, timeout, result) + result.setLen(count) + +proc flush*[T](s: Selector[T]) = discard + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + s.withSelectLock(): + result = false + + let fdi = int(fd) + for i in 0..<FD_SETSIZE: + if s.fds[i].ident == fdi: + return true + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + s.withSelectLock(): + let fdi = int(fd) + for i in 0..<FD_SETSIZE: + if s.fds[i].ident == fdi: + return s.fds[i].data + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + s.withSelectLock(): + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + var pkey = addr(s.fds[i]) + pkey.data = data + result = true + break + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].data) + break + inc(i) + if i != FD_SETSIZE: + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body1, body2: untyped) = + mixin withSelectLock + s.withSelectLock(): + block: + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 + + +proc getFd*[T](s: Selector[T]): int = + return -1 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index ab7d18bd8..53fa7553a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -14,534 +14,172 @@ ## JSON is based on a subset of the JavaScript Programming Language, ## Standard ECMA-262 3rd Edition - December 1999. ## -## Usage example: +## See also +## ======== +## * `std/parsejson <parsejson.html>`_ +## * `std/jsonutils <jsonutils.html>`_ +## * `std/marshal <marshal.html>`_ +## * `std/jscore <jscore.html>`_ ## -## .. code-block:: nim -## let -## small_json = """{"test": 1.3, "key2": true}""" -## jobj = parseJson(small_json) -## assert (jobj.kind == JObject) -## echo($jobj["test"].fnum) -## echo($jobj["key2"].bval) ## -## Results in: +## Overview +## ======== ## -## .. code-block:: nim +## Parsing JSON +## ------------ ## -## 1.3000000000000000e+00 -## true +## JSON often arrives into your program (via an API or a file) as a `string`. +## The first step is to change it from its serialized form into a nested object +## structure called a `JsonNode`. +## +## The `parseJson` procedure takes a string containing JSON and returns a +## `JsonNode` object. This is an object variant and it is either a +## `JObject`, `JArray`, `JString`, `JInt`, `JFloat`, `JBool` or +## `JNull`. You check the kind of this object variant by using the `kind` +## accessor. +## +## For a `JsonNode` who's kind is `JObject`, you can access its fields using +## the `[]` operator. The following example shows how to do this: +## +## ```Nim +## import std/json +## +## let jsonNode = parseJson("""{"key": 3.14}""") +## +## doAssert jsonNode.kind == JObject +## doAssert jsonNode["key"].kind == JFloat +## ``` +## +## Reading values +## -------------- +## +## Once you have a `JsonNode`, retrieving the values can then be achieved +## by using one of the helper procedures, which include: +## +## * `getInt` +## * `getFloat` +## * `getStr` +## * `getBool` +## +## To retrieve the value of `"key"` you can do the following: +## +## ```Nim +## import std/json +## +## let jsonNode = parseJson("""{"key": 3.14}""") +## +## doAssert jsonNode["key"].getFloat() == 3.14 +## ``` +## +## **Important:** The `[]` operator will raise an exception when the +## specified field does not exist. +## +## Handling optional keys +## ---------------------- +## +## By using the `{}` operator instead of `[]`, it will return `nil` +## when the field is not found. The `get`-family of procedures will return a +## type's default value when called on `nil`. +## +## ```Nim +## import std/json +## +## let jsonNode = parseJson("{}") +## +## doAssert jsonNode{"nope"}.getInt() == 0 +## doAssert jsonNode{"nope"}.getFloat() == 0 +## doAssert jsonNode{"nope"}.getStr() == "" +## doAssert jsonNode{"nope"}.getBool() == false +## ``` +## +## Using default values +## -------------------- +## +## The `get`-family helpers also accept an additional parameter which allow +## you to fallback to a default value should the key's values be `null`: +## +## ```Nim +## import std/json +## +## let jsonNode = parseJson("""{"key": 3.14, "key2": null}""") +## +## doAssert jsonNode["key"].getFloat(6.28) == 3.14 +## doAssert jsonNode["key2"].getFloat(3.14) == 3.14 +## doAssert jsonNode{"nope"}.getFloat(3.14) == 3.14 # note the {} +## ``` +## +## Unmarshalling +## ------------- +## +## In addition to reading dynamic data, Nim can also unmarshal JSON directly +## into a type with the `to` macro. +## +## Note: Use `Option <options.html#Option>`_ for keys sometimes missing in json +## responses, and backticks around keys with a reserved keyword as name. +## +## ```Nim +## import std/json +## import std/options +## +## type +## User = object +## name: string +## age: int +## `type`: Option[string] +## +## let userJson = parseJson("""{ "name": "Nim", "age": 12 }""") +## let user = to(userJson, User) +## if user.`type`.isSome(): +## assert user.`type`.get() != "robot" +## ``` +## +## Creating JSON +## ============= ## ## This module can also be used to comfortably create JSON using the `%*` ## operator: ## -## .. code-block:: nim +## ```nim +## import std/json ## ## var hisName = "John" ## let herAge = 31 ## var j = %* ## [ -## { -## "name": hisName, -## "age": 30 -## }, -## { -## "name": "Susan", -## "age": herAge -## } +## { "name": hisName, "age": 30 }, +## { "name": "Susan", "age": herAge } ## ] +## +## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} +## j2["details"] = %* {"age":35, "pi":3.1415} +## echo j2 +## ``` +## +## See also: std/jsonutils for hookable json serialization/deserialization +## of arbitrary types. -import - hashes, strutils, lexbase, streams, unicode, macros +runnableExamples: + ## Note: for JObject, key ordering is preserved, unlike in some languages, + ## this is convenient for some use cases. Example: + type Foo = object + a1, a2, a0, a3, a4: int + doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" -type - JsonEventKind* = enum ## enumeration of all events that may occur when parsing - jsonError, ## an error occurred during parsing - jsonEof, ## end of file reached - jsonString, ## a string literal - jsonInt, ## an integer literal - jsonFloat, ## a float literal - jsonTrue, ## the value ``true`` - jsonFalse, ## the value ``false`` - jsonNull, ## the value ``null`` - jsonObjectStart, ## start of an object: the ``{`` token - jsonObjectEnd, ## end of an object: the ``}`` token - jsonArrayStart, ## start of an array: the ``[`` token - jsonArrayEnd ## start of an array: the ``]`` token - - TokKind = enum # must be synchronized with TJsonEventKind! - tkError, - tkEof, - tkString, - tkInt, - tkFloat, - tkTrue, - tkFalse, - tkNull, - tkCurlyLe, - tkCurlyRi, - tkBracketLe, - tkBracketRi, - tkColon, - tkComma - - JsonError* = enum ## enumeration that lists all errors that can occur - errNone, ## no error - errInvalidToken, ## invalid token - errStringExpected, ## string expected - errColonExpected, ## ``:`` expected - errCommaExpected, ## ``,`` expected - errBracketRiExpected, ## ``]`` expected - errCurlyRiExpected, ## ``}`` expected - errQuoteExpected, ## ``"`` or ``'`` expected - errEOC_Expected, ## ``*/`` expected - errEofExpected, ## EOF expected - errExprExpected ## expr expected - - ParserState = enum - stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, - stateExpectObjectComma, stateExpectColon, stateExpectValue - - JsonParser* = object of BaseLexer ## the parser object. - a: string - tok: TokKind - kind: JsonEventKind - err: JsonError - state: seq[ParserState] - filename: string - -{.deprecated: [TJsonEventKind: JsonEventKind, TJsonError: JsonError, - TJsonParser: JsonParser, TTokKind: TokKind].} - -const - errorMessages: array [JsonError, string] = [ - "no error", - "invalid token", - "string expected", - "':' expected", - "',' expected", - "']' expected", - "'}' expected", - "'\"' or \"'\" expected", - "'*/' expected", - "EOF expected", - "expression expected" - ] - tokToStr: array [TokKind, string] = [ - "invalid token", - "EOF", - "string literal", - "int literal", - "float literal", - "true", - "false", - "null", - "{", "}", "[", "]", ":", "," - ] - -proc open*(my: var JsonParser, input: Stream, filename: string) = - ## initializes the parser with an input stream. `Filename` is only used - ## for nice error messages. - lexbase.open(my, input) - my.filename = filename - my.state = @[stateStart] - my.kind = jsonError - my.a = "" - -proc close*(my: var JsonParser) {.inline.} = - ## closes the parser `my` and its associated input stream. - lexbase.close(my) - -proc str*(my: JsonParser): string {.inline.} = - ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, - ## ``jsonString`` - assert(my.kind in {jsonInt, jsonFloat, jsonString}) - return my.a - -proc getInt*(my: JsonParser): BiggestInt {.inline.} = - ## returns the number for the event: ``jsonInt`` - assert(my.kind == jsonInt) - return parseBiggestInt(my.a) - -proc getFloat*(my: JsonParser): float {.inline.} = - ## returns the number for the event: ``jsonFloat`` - assert(my.kind == jsonFloat) - return parseFloat(my.a) - -proc kind*(my: JsonParser): JsonEventKind {.inline.} = - ## returns the current event type for the JSON parser - return my.kind - -proc getColumn*(my: JsonParser): int {.inline.} = - ## get the current column the parser has arrived at. - result = getColNumber(my, my.bufpos) - -proc getLine*(my: JsonParser): int {.inline.} = - ## get the current line the parser has arrived at. - result = my.lineNumber - -proc getFilename*(my: JsonParser): string {.inline.} = - ## get the filename of the file that the parser processes. - result = my.filename - -proc errorMsg*(my: JsonParser): string = - ## returns a helpful error message for the event ``jsonError`` - assert(my.kind == jsonError) - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] - -proc errorMsgExpected*(my: JsonParser, e: string): string = - ## returns an error message "`e` expected" in the same format as the - ## other error messages - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), e & " expected"] - -proc handleHexChar(c: char, x: var int): bool = - result = true # Success - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: result = false # error - -proc parseEscapedUTF16(buf: cstring, pos: var int): int = - result = 0 - #UTF-16 escape is always 4 bytes. - for _ in 0..3: - if handleHexChar(buf[pos], result): - inc(pos) - else: - return -1 +import std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] -proc parseString(my: var JsonParser): TokKind = - result = tkString - var pos = my.bufpos + 1 - var buf = my.buf - while true: - case buf[pos] - of '\0': - my.err = errQuoteExpected - result = tkError - break - of '"': - inc(pos) - break - of '\\': - case buf[pos+1] - of '\\', '"', '\'', '/': - add(my.a, buf[pos+1]) - inc(pos, 2) - of 'b': - add(my.a, '\b') - inc(pos, 2) - of 'f': - add(my.a, '\f') - inc(pos, 2) - of 'n': - add(my.a, '\L') - inc(pos, 2) - of 'r': - add(my.a, '\C') - inc(pos, 2) - of 't': - add(my.a, '\t') - inc(pos, 2) - of 'u': - inc(pos, 2) - var r = parseEscapedUTF16(buf, pos) - if r < 0: - my.err = errInvalidToken - break - # Deal with surrogates - if (r and 0xfc00) == 0xd800: - if buf[pos] & buf[pos+1] != "\\u": - my.err = errInvalidToken - break - inc(pos, 2) - var s = parseEscapedUTF16(buf, pos) - if (s and 0xfc00) == 0xdc00 and s > 0: - r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) - else: - my.err = errInvalidToken - break - add(my.a, toUTF8(Rune(r))) - else: - # don't bother with the error - add(my.a, buf[pos]) - inc(pos) - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - add(my.a, '\c') - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - add(my.a, '\L') - else: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos # store back +import std/options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 +import std/private/since -proc skip(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - while true: - case buf[pos] - of '/': - if buf[pos+1] == '/': - # skip line comment: - inc(pos, 2) - while true: - case buf[pos] - of '\0': - break - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - break - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - break - else: - inc(pos) - elif buf[pos+1] == '*': - # skip long comment: - inc(pos, 2) - while true: - case buf[pos] - of '\0': - my.err = errEOC_Expected - break - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - of '*': - inc(pos) - if buf[pos] == '/': - inc(pos) - break - else: - inc(pos) - else: - break - of ' ', '\t': - inc(pos) - of '\c': - pos = lexbase.handleCR(my, pos) - buf = my.buf - of '\L': - pos = lexbase.handleLF(my, pos) - buf = my.buf - else: - break - my.bufpos = pos - -proc parseNumber(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - if buf[pos] == '-': - add(my.a, '-') - inc(pos) - if buf[pos] == '.': - add(my.a, "0.") - inc(pos) - else: - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] == '.': - add(my.a, '.') - inc(pos) - # digits after the dot: - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] in {'E', 'e'}: - add(my.a, buf[pos]) - inc(pos) - if buf[pos] in {'+', '-'}: - add(my.a, buf[pos]) - inc(pos) - while buf[pos] in Digits: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos - -proc parseName(my: var JsonParser) = - var pos = my.bufpos - var buf = my.buf - if buf[pos] in IdentStartChars: - while buf[pos] in IdentChars: - add(my.a, buf[pos]) - inc(pos) - my.bufpos = pos - -proc getTok(my: var JsonParser): TokKind = - setLen(my.a, 0) - skip(my) # skip whitespace, comments - case my.buf[my.bufpos] - of '-', '.', '0'..'9': - parseNumber(my) - if {'.', 'e', 'E'} in my.a: - result = tkFloat - else: - result = tkInt - of '"': - result = parseString(my) - of '[': - inc(my.bufpos) - result = tkBracketLe - of '{': - inc(my.bufpos) - result = tkCurlyLe - of ']': - inc(my.bufpos) - result = tkBracketRi - of '}': - inc(my.bufpos) - result = tkCurlyRi - of ',': - inc(my.bufpos) - result = tkComma - of ':': - inc(my.bufpos) - result = tkColon - of '\0': - result = tkEof - of 'a'..'z', 'A'..'Z', '_': - parseName(my) - case my.a - of "null": result = tkNull - of "true": result = tkTrue - of "false": result = tkFalse - else: result = tkError - else: - inc(my.bufpos) - result = tkError - my.tok = result - -proc next*(my: var JsonParser) = - ## retrieves the first/next event. This controls the parser. - var tk = getTok(my) - var i = my.state.len-1 - # the following code is a state machine. If we had proper coroutines, - # the code could be much simpler. - case my.state[i] - of stateEof: - if tk == tkEof: - my.kind = jsonEof - else: - my.kind = jsonError - my.err = errEofExpected - of stateStart: - # tokens allowed? - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state[i] = stateEof # expect EOF next! - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateArray) # we expect any - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkEof: - my.kind = jsonEof - else: - my.kind = jsonError - my.err = errEofExpected - of stateObject: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state.add(stateExpectColon) - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateExpectColon) - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateExpectColon) - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkCurlyRi: - my.kind = jsonObjectEnd - discard my.state.pop() - else: - my.kind = jsonError - my.err = errCurlyRiExpected - of stateArray: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state.add(stateExpectArrayComma) # expect value next! - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state.add(stateExpectArrayComma) - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state.add(stateExpectArrayComma) - my.state.add(stateObject) - my.kind = jsonObjectStart - of tkBracketRi: - my.kind = jsonArrayEnd - discard my.state.pop() - else: - my.kind = jsonError - my.err = errBracketRiExpected - of stateExpectArrayComma: - case tk - of tkComma: - discard my.state.pop() - next(my) - of tkBracketRi: - my.kind = jsonArrayEnd - discard my.state.pop() # pop stateExpectArrayComma - discard my.state.pop() # pop stateArray - else: - my.kind = jsonError - my.err = errBracketRiExpected - of stateExpectObjectComma: - case tk - of tkComma: - discard my.state.pop() - next(my) - of tkCurlyRi: - my.kind = jsonObjectEnd - discard my.state.pop() # pop stateExpectObjectComma - discard my.state.pop() # pop stateObject - else: - my.kind = jsonError - my.err = errCurlyRiExpected - of stateExpectColon: - case tk - of tkColon: - my.state[i] = stateExpectValue - next(my) - else: - my.kind = jsonError - my.err = errColonExpected - of stateExpectValue: - case tk - of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: - my.state[i] = stateExpectObjectComma - my.kind = JsonEventKind(ord(tk)) - of tkBracketLe: - my.state[i] = stateExpectObjectComma - my.state.add(stateArray) - my.kind = jsonArrayStart - of tkCurlyLe: - my.state[i] = stateExpectObjectComma - my.state.add(stateObject) - my.kind = jsonObjectStart - else: - my.kind = jsonError - my.err = errExprExpected +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, formatfloat] +export + tables.`$` -# ------------- higher level interface --------------------------------------- +export + parsejson.JsonEventKind, parsejson.JsonError, JsonParser, JsonKindError, + open, close, str, getInt, getFloat, kind, getColumn, getLine, getFilename, + errorMsg, errorMsgExpected, next, JsonParsingError, raiseParseErr, nimIdentNormalize type JsonNodeKind* = enum ## possible JSON node types @@ -555,6 +193,8 @@ type JsonNode* = ref JsonNodeObj ## JSON node JsonNodeObj* {.acyclic.} = object + isUnquoted: bool # the JString was a number-like token and + # so shouldn't be quoted case kind*: JsonNodeKind of JString: str*: string @@ -567,173 +207,244 @@ type of JNull: nil of JObject: - fields*: seq[tuple[key: string, val: JsonNode]] + fields*: OrderedTable[string, JsonNode] of JArray: elems*: seq[JsonNode] - JsonParsingError* = object of ValueError ## is raised for a JSON error - -{.deprecated: [EJsonParsingError: JsonParsingError, TJsonNode: JsonNodeObj, - PJsonNode: JsonNode, TJsonNodeKind: JsonNodeKind].} - -proc raiseParseErr*(p: JsonParser, msg: string) {.noinline, noreturn.} = - ## raises an `EJsonParsingError` exception. - raise newException(JsonParsingError, errorMsgExpected(p, msg)) +const DepthLimit = 1000 proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. - new(result) - result.kind = JString - result.str = s + result = JsonNode(kind: JString, str: s) -proc newJStringMove(s: string): JsonNode = - new(result) - result.kind = JString - shallowCopy(result.str, s) +proc newJRawNumber(s: string): JsonNode = + ## Creates a "raw JS number", that is a number that does not + ## fit into Nim's `BiggestInt` field. This is really a `JString` + ## with the additional information that it should be converted back + ## to the string representation without the quotes. + result = JsonNode(kind: JString, str: s, isUnquoted: true) proc newJInt*(n: BiggestInt): JsonNode = ## Creates a new `JInt JsonNode`. - new(result) - result.kind = JInt - result.num = n + result = JsonNode(kind: JInt, num: n) proc newJFloat*(n: float): JsonNode = ## Creates a new `JFloat JsonNode`. - new(result) - result.kind = JFloat - result.fnum = n + result = JsonNode(kind: JFloat, fnum: n) proc newJBool*(b: bool): JsonNode = ## Creates a new `JBool JsonNode`. - new(result) - result.kind = JBool - result.bval = b + result = JsonNode(kind: JBool, bval: b) proc newJNull*(): JsonNode = ## Creates a new `JNull JsonNode`. - new(result) + result = JsonNode(kind: JNull) proc newJObject*(): JsonNode = ## Creates a new `JObject JsonNode` - new(result) - result.kind = JObject - result.fields = @[] + result = JsonNode(kind: JObject, fields: initOrderedTable[string, JsonNode](2)) proc newJArray*(): JsonNode = ## Creates a new `JArray JsonNode` - new(result) - result.kind = JArray - result.elems = @[] + result = JsonNode(kind: JArray, elems: @[]) proc getStr*(n: JsonNode, default: string = ""): string = ## Retrieves the string value of a `JString JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JString``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JString`, or if `n` is nil. if n.isNil or n.kind != JString: return default else: return n.str -proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt = +proc getInt*(n: JsonNode, default: int = 0): int = ## Retrieves the int value of a `JInt JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JInt`, or if `n` is nil. + if n.isNil or n.kind != JInt: return default + else: return int(n.num) + +proc getBiggestInt*(n: JsonNode, default: BiggestInt = 0): BiggestInt = + ## Retrieves the BiggestInt value of a `JInt JsonNode`. + ## + ## Returns `default` if `n` is not a `JInt`, or if `n` is nil. if n.isNil or n.kind != JInt: return default else: return n.num -proc getFNum*(n: JsonNode, default: float = 0.0): float = +proc getFloat*(n: JsonNode, default: float = 0.0): float = ## Retrieves the float value of a `JFloat JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JFloat`` or ``JInt``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JFloat` or `JInt`, or if `n` is nil. if n.isNil: return default case n.kind of JFloat: return n.fnum of JInt: return float(n.num) else: return default -proc getBVal*(n: JsonNode, default: bool = false): bool = +proc getBool*(n: JsonNode, default: bool = false): bool = ## Retrieves the bool value of a `JBool JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JBool``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JBool`, or if `n` is nil. if n.isNil or n.kind != JBool: return default else: return n.bval proc getFields*(n: JsonNode, - default: seq[tuple[key: string, val: JsonNode]] = @[]): - seq[tuple[key: string, val: JsonNode]] = + default = initOrderedTable[string, JsonNode](2)): + OrderedTable[string, JsonNode] = ## Retrieves the key, value pairs of a `JObject JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JObject``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JObject`, or if `n` is nil. if n.isNil or n.kind != JObject: return default else: return n.fields proc getElems*(n: JsonNode, default: seq[JsonNode] = @[]): seq[JsonNode] = - ## Retrieves the int value of a `JArray JsonNode`. + ## Retrieves the array of a `JArray JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JArray``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JArray`, or if `n` is nil. if n.isNil or n.kind != JArray: return default else: return n.elems +proc add*(father, child: JsonNode) = + ## Adds `child` to a JArray node `father`. + assert father.kind == JArray + father.elems.add(child) + +proc add*(obj: JsonNode, key: string, val: JsonNode) = + ## Sets a field from a `JObject`. + assert obj.kind == JObject + obj.fields[key] = val + proc `%`*(s: string): JsonNode = ## Generic constructor for JSON data. Creates a new `JString JsonNode`. - new(result) - result.kind = JString - result.str = s + result = JsonNode(kind: JString, str: s) + +proc `%`*(n: uint): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + if n > cast[uint](int.high): + result = newJRawNumber($n) + else: + result = JsonNode(kind: JInt, num: BiggestInt(n)) + +proc `%`*(n: int): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + result = JsonNode(kind: JInt, num: n) + +proc `%`*(n: BiggestUInt): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + if n > cast[BiggestUInt](BiggestInt.high): + result = newJRawNumber($n) + else: + result = JsonNode(kind: JInt, num: BiggestInt(n)) proc `%`*(n: BiggestInt): JsonNode = ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. - new(result) - result.kind = JInt - result.num = n + result = JsonNode(kind: JInt, num: n) proc `%`*(n: float): JsonNode = ## Generic constructor for JSON data. Creates a new `JFloat JsonNode`. - new(result) - result.kind = JFloat - result.fnum = n + runnableExamples: + assert $(%[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2]) == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert (%NaN).kind == JString + assert (%0.0).kind == JFloat + # for those special cases, we could also have used `newJRawNumber` but then + # it would've been inconsisten with the case of `parseJson` vs `%` for representing them. + if n != n: newJString("nan") + elif n == Inf: newJString("inf") + elif n == -Inf: newJString("-inf") + else: JsonNode(kind: JFloat, fnum: n) proc `%`*(b: bool): JsonNode = ## Generic constructor for JSON data. Creates a new `JBool JsonNode`. - new(result) - result.kind = JBool - result.bval = b + result = JsonNode(kind: JBool, bval: b) proc `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode` - new(result) - result.kind = JObject - newSeq(result.fields, keyVals.len) - for i, p in pairs(keyVals): result.fields[i] = p + if keyVals.len == 0: return newJArray() + result = newJObject() + for key, val in items(keyVals): result.fields[key] = val + +template `%`*(j: JsonNode): JsonNode = j -proc `%`*(elements: openArray[JsonNode]): JsonNode = +proc `%`*[T](elements: openArray[T]): JsonNode = ## Generic constructor for JSON data. Creates a new `JArray JsonNode` - new(result) - result.kind = JArray - newSeq(result.elems, elements.len) - for i, p in pairs(elements): result.elems[i] = p + result = newJArray() + for elem in elements: result.add(%elem) -proc toJson(x: NimNode): NimNode {.compiletime.} = - case x.kind - of nnkBracket: - result = newNimNode(nnkBracket) - for i in 0 .. <x.len: - result.add(toJson(x[i])) +proc `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode`. + result = newJObject() + for k, v in table: result[k] = %v - of nnkTableConstr: - result = newNimNode(nnkTableConstr) - for i in 0 .. <x.len: - assert x[i].kind == nnkExprColonExpr - result.add(newNimNode(nnkExprColonExpr).add(x[i][0]).add(toJson(x[i][1]))) +proc `%`*[T](opt: Option[T]): JsonNode = + ## Generic constructor for JSON data. Creates a new `JNull JsonNode` + ## if `opt` is empty, otherwise it delegates to the underlying value. + if opt.isSome: %opt.get else: newJNull() +when false: + # For 'consistency' we could do this, but that only pushes people further + # into that evil comfort zone where they can use Nim without understanding it + # causing problems later on. + proc `%`*(elements: set[bool]): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode`. + ## This can only be used with the empty set `{}` and is supported + ## to prevent the gotcha `%*{}` which used to produce an empty + ## JSON array. + result = newJObject() + assert false notin elements, "usage error: only empty sets allowed" + assert true notin elements, "usage error: only empty sets allowed" + +proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = + ## Sets a field from a `JObject`. + assert(obj.kind == JObject) + obj.fields[key] = val + +proc `%`*[T: object](o: T): JsonNode = + ## Construct JsonNode from tuples and objects. + result = newJObject() + for k, v in o.fieldPairs: result[k] = %v + +proc `%`*(o: ref object): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + if o.isNil: + result = newJNull() else: - result = x + result = %(o[]) - result = prefix(result, "%") +proc `%`*(o: enum): JsonNode = + ## Construct a JsonNode that represents the specified enum value as a + ## string. Creates a new `JString JsonNode`. + result = %($o) -macro `%*`*(x: expr): expr = +proc toJsonImpl(x: NimNode): NimNode = + case x.kind + of nnkBracket: # array + if x.len == 0: return newCall(bindSym"newJArray") + result = newNimNode(nnkBracket) + for i in 0 ..< x.len: + result.add(toJsonImpl(x[i])) + result = newCall(bindSym("%", brOpen), result) + of nnkTableConstr: # object + if x.len == 0: return newCall(bindSym"newJObject") + result = newNimNode(nnkTableConstr) + for i in 0 ..< x.len: + x[i].expectKind nnkExprColonExpr + result.add newTree(nnkExprColonExpr, x[i][0], toJsonImpl(x[i][1])) + result = newCall(bindSym("%", brOpen), result) + of nnkCurly: # empty object + x.expectLen(0) + result = newCall(bindSym"newJObject") + of nnkNilLit: + result = newCall(bindSym"newJNull") + of nnkPar: + if x.len == 1: result = toJsonImpl(x[0]) + else: result = newCall(bindSym("%", brOpen), x) + else: + result = newCall(bindSym("%", brOpen), x) + +macro `%*`*(x: untyped): untyped = ## Convert an expression to a JsonNode directly, without having to specify ## `%` for every element. - result = toJson(x) + result = toJsonImpl(x) -proc `==`* (a,b: JsonNode): bool = +proc `==`*(a, b: JsonNode): bool {.noSideEffect, raises: [].} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -741,23 +452,37 @@ proc `==`* (a,b: JsonNode): bool = elif b.isNil or a.kind != b.kind: return false else: - return case a.kind + case a.kind of JString: - a.str == b.str + result = a.str == b.str of JInt: - a.num == b.num + result = a.num == b.num of JFloat: - a.fnum == b.fnum + result = a.fnum == b.fnum of JBool: - a.bval == b.bval + result = a.bval == b.bval of JNull: - true + result = true of JArray: - a.elems == b.elems + {.cast(raises: []).}: # bug #19303 + result = a.elems == b.elems of JObject: - a.fields == b.fields + # we cannot use OrderedTable's equality here as + # the order does not matter for equality here. + if a.fields.len != b.fields.len: return false + for key, val in a.fields: + if not b.fields.hasKey(key): return false + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: + if b.fields[key] != val: return false + result = true -proc hash* (n:JsonNode): Hash = +proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} + +proc hash*(n: JsonNode): Hash {.noSideEffect.} = ## Compute the hash for a JSON node case n.kind of JArray: @@ -773,7 +498,12 @@ proc hash* (n:JsonNode): Hash = of JString: result = hash(n.str) of JNull: - result = hash(0) + result = Hash(0) + +proc hash*(n: OrderedTable[string, JsonNode]): Hash = + for key, val in n: + result = result xor (hash(key) !& hash(val)) + result = !$result proc len*(n: JsonNode): int = ## If `n` is a `JArray`, it returns the number of elements. @@ -786,85 +516,129 @@ proc len*(n: JsonNode): int = proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## If the value at `name` does not exist, raises KeyError. assert(not isNil(node)) assert(node.kind == JObject) - for key, item in items(node.fields): - if key == name: - return item - return nil + when defined(nimJsonGet): + if not node.fields.hasKey(name): return nil + result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## is out of bounds, but as long as array bound checks are enabled it will + ## result in an exception. assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] +proc `[]`*(node: JsonNode, index: BackwardsIndex): JsonNode {.inline, since: (1, 5, 1).} = + ## Gets the node at `array.len-i` in an array through the `^` operator. + ## + ## i.e. `j[^i]` is a shortcut for `j[j.len-i]`. + runnableExamples: + let + j = parseJson("[1,2,3,4,5]") + + doAssert j[^1].getInt == 5 + doAssert j[^2].getInt == 4 + + `[]`(node, node.len - int(index)) + +proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode = + ## Slice operation for JArray. + ## + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + runnableExamples: + import std/json + let arr = %[0,1,2,3,4,5] + doAssert arr[2..4] == %[2,3,4] + doAssert arr[2..^2] == %[2,3,4] + doAssert arr[^4..^2] == %[2,3,4] + + assert(a.kind == JArray) + result = newJArray() + let xa = (when x.a is BackwardsIndex: a.len - int(x.a) else: int(x.a)) + let L = (when x.b is BackwardsIndex: a.len - int(x.b) else: int(x.b)) - xa + 1 + for i in 0..<L: + result.add(a[i + xa]) + proc hasKey*(node: JsonNode, key: string): bool = ## Checks if `key` exists in `node`. assert(node.kind == JObject) - for k, item in items(node.fields): - if k == key: return true - -proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) - ## Deprecated for `hasKey` + result = node.fields.hasKey(key) -proc add*(father, child: JsonNode) = - ## Adds `child` to a JArray node `father`. - assert father.kind == JArray - father.elems.add(child) - -proc add*(obj: JsonNode, key: string, val: JsonNode) = - ## Adds ``(key, val)`` pair to the JObject node `obj`. For speed - ## reasons no check for duplicate keys is performed! - ## But ``[]=`` performs the check. - assert obj.kind == JObject - obj.fields.add((key, val)) +proc contains*(node: JsonNode, key: string): bool = + ## Checks if `key` exists in `node`. + assert(node.kind == JObject) + node.fields.hasKey(key) -proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = - ## Sets a field from a `JObject`. Performs a check for duplicate keys. - assert(obj.kind == JObject) - for i in 0..obj.fields.len-1: - if obj.fields[i].key == key: - obj.fields[i].val = val - return - obj.fields.add((key, val)) +proc contains*(node: JsonNode, val: JsonNode): bool = + ## Checks if `val` exists in array `node`. + assert(node.kind == JArray) + find(node.elems, val) >= 0 proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns nil. Also returns nil if one of the - ## intermediate data structures is not an object + ## keys do not exist, returns `nil`. Also returns `nil` if one of the + ## intermediate data structures is not an object. + ## + ## This proc can be used to create tree structures on the + ## fly (sometimes called `autovivification`:idx:): + ## + runnableExamples: + var myjson = %* {"parent": {"child": {"grandchild": 1}}} + doAssert myjson{"parent", "child", "grandchild"} == newJInt(1) + result = node for key in keys: - if isNil(result) or result.kind!=JObject: + if isNil(result) or result.kind != JObject: + return nil + result = result.fields.getOrDefault(key) + +proc `{}`*(node: JsonNode, index: varargs[int]): JsonNode = + ## Traverses the node and gets the given value. If any of the + ## indexes do not exist, returns `nil`. Also returns `nil` if one of the + ## intermediate data structures is not an array. + result = node + for i in index: + if isNil(result) or result.kind != JArray or i >= node.len: return nil - result=result[key] + result = result.elems[i] + +proc getOrDefault*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + if not isNil(node) and node.kind == JObject: + result = node.fields.getOrDefault(key) + +proc `{}`*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to `value` If any of the keys are missing, they are added + ## to `value`. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): - if isNil(node[keys[i]]): + if not node.hasKey(keys[i]): node[keys[i]] = newJObject() node = node[keys[i]] node[keys[keys.len-1]] = value proc delete*(obj: JsonNode, key: string) = - ## Deletes ``obj[key]`` preserving the order of the other (key, value)-pairs. + ## Deletes `obj[key]`. assert(obj.kind == JObject) - for i in 0..obj.fields.len-1: - if obj.fields[i].key == key: - obj.fields.delete(i) - return - raise newException(IndexError, "key not in object") + if not obj.fields.hasKey(key): + raise newException(KeyError, "key not in object") + obj.fields.del(key) proc copy*(p: JsonNode): JsonNode = - ## Performs a deep copy of `a`. + ## Performs a deep copy of `p`. case p.kind of JString: result = newJString(p.str) + result.isUnquoted = p.isUnquoted of JInt: result = newJInt(p.num) of JFloat: @@ -875,8 +649,8 @@ proc copy*(p: JsonNode): JsonNode = result = newJNull() of JObject: result = newJObject() - for key, field in items(p.fields): - result.fields.add((key, copy(field))) + for key, val in pairs(p.fields): + result.fields[key] = copy(val) of JArray: result = newJArray() for i in items(p.elems): @@ -892,43 +666,102 @@ proc newIndent(curr, indent: int, ml: bool): int = else: return indent proc nl(s: var string, ml: bool) = - if ml: s.add("\n") + s.add(if ml: "\n" else: " ") + +proc escapeJsonUnquoted*(s: string; result: var string) = + ## Converts a string `s` to its JSON representation without quotes. + ## Appends to `result`. + for c in s: + case c + of '\L': result.add("\\n") + of '\b': result.add("\\b") + of '\f': result.add("\\f") + of '\t': result.add("\\t") + of '\v': result.add("\\u000b") + of '\r': result.add("\\r") + of '"': result.add("\\\"") + of '\0'..'\7': result.add("\\u000" & $ord(c)) + of '\14'..'\31': result.add("\\u00" & toHex(ord(c), 2)) + of '\\': result.add("\\\\") + else: result.add(c) + +proc escapeJsonUnquoted*(s: string): string = + ## Converts a string `s` to its JSON representation without quotes. + result = newStringOfCap(s.len + s.len shr 3) + escapeJsonUnquoted(s, result) + +proc escapeJson*(s: string; result: var string) = + ## Converts a string `s` to its JSON representation with quotes. + ## Appends to `result`. + result.add("\"") + escapeJsonUnquoted(s, result) + result.add("\"") proc escapeJson*(s: string): string = - ## Converts a string `s` to its JSON representation. + ## Converts a string `s` to its JSON representation with quotes. result = newStringOfCap(s.len + s.len shr 3) - result.add("\"") - for x in runes(s): - var r = int(x) - if r >= 32 and r <= 127: - var c = chr(r) - case c - of '"': result.add("\\\"") - of '\\': result.add("\\\\") - else: result.add(c) + escapeJson(s, result) + +proc toUgly*(result: var string, node: JsonNode) = + ## Converts `node` to its JSON Representation, without + ## regard for human readability. Meant to improve `$` string + ## conversion performance. + ## + ## JSON representation is stored in the passed `result` + ## + ## This provides higher efficiency than the `pretty` procedure as it + ## does **not** attempt to format the resulting JSON to make it human readable. + var comma = false + case node.kind: + of JArray: + result.add "[" + for child in node.elems: + if comma: result.add "," + else: comma = true + result.toUgly child + result.add "]" + of JObject: + result.add "{" + for key, value in pairs(node.fields): + if comma: result.add "," + else: comma = true + key.escapeJson(result) + result.add ":" + result.toUgly value + result.add "}" + of JString: + if node.isUnquoted: + result.add node.str else: - result.add("\\u") - result.add(toHex(r, 4)) - result.add("\"") + escapeJson(node.str, result) + of JInt: + result.addInt(node.num) + of JFloat: + result.addFloat(node.fnum) + of JBool: + result.add(if node.bval: "true" else: "false") + of JNull: + result.add "null" proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, lstArr = false, currIndent = 0) = case node.kind of JObject: - if currIndent != 0 and not lstArr: result.nl(ml) - result.indent(currIndent) # Indentation + if lstArr: result.indent(currIndent) # Indentation if node.fields.len > 0: result.add("{") result.nl(ml) # New line - for i in 0..len(node.fields)-1: + var i = 0 + for key, val in pairs(node.fields): if i > 0: - result.add(", ") + result.add(",") result.nl(ml) # New Line + inc i # Need to indent more than { result.indent(newIndent(currIndent, indent, ml)) - result.add(escapeJson(node.fields[i].key)) + escapeJson(key, result) result.add(": ") - toPretty(result, node.fields[i].val, indent, ml, false, + toPretty(result, val, indent, ml, false, newIndent(currIndent, indent, ml)) result.nl(ml) result.indent(currIndent) # indent the same as { @@ -937,16 +770,16 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("{}") of JString: if lstArr: result.indent(currIndent) - result.add(escapeJson(node.str)) + toUgly(result, node) of JInt: if lstArr: result.indent(currIndent) - result.add($node.num) + result.addInt(node.num) of JFloat: if lstArr: result.indent(currIndent) - result.add($node.fnum) + result.addFloat(node.fnum) of JBool: if lstArr: result.indent(currIndent) - result.add($node.bval) + result.add(if node.bval: "true" else: "false") of JArray: if lstArr: result.indent(currIndent) if len(node.elems) != 0: @@ -954,7 +787,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.nl(ml) for i in 0..len(node.elems)-1: if i > 0: - result.add(", ") + result.add(",") result.nl(ml) # New Line toPretty(result, node.elems[i], indent, ml, true, newIndent(currIndent, indent, ml)) @@ -967,47 +800,27 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("null") proc pretty*(node: JsonNode, indent = 2): string = - ## Converts `node` to its JSON Representation, with indentation and + ## Returns a JSON Representation of `node`, with indentation and ## on multiple lines. + ## + ## Similar to prettyprint in Python. + runnableExamples: + let j = %* {"name": "Isaac", "books": ["Robot Dreams"], + "details": {"age": 35, "pi": 3.1415}} + doAssert pretty(j) == """ +{ + "name": "Isaac", + "books": [ + "Robot Dreams" + ], + "details": { + "age": 35, + "pi": 3.1415 + } +}""" result = "" toPretty(result, node, indent) -proc toUgly*(result: var string, node: JsonNode) = - ## Converts `node` to its JSON Representation, without - ## regard for human readability. Meant to improve ``$`` string - ## conversion performance. - ## - ## This provides higher efficiency than the ``toPretty`` procedure as it - ## does **not** attempt to format the resulting JSON to make it human readable. - var comma = false - case node.kind: - of JArray: - result.add "[" - for child in node.elems: - if comma: result.add "," - else: comma = true - result.toUgly child - result.add "]" - of JObject: - result.add "{" - for key, value in items(node.fields): - if comma: result.add "," - else: comma = true - result.add key.escapeJson() - result.add ":" - result.toUgly value - result.add "}" - of JString: - result.add node.str.escapeJson() - of JInt: - result.add($node.num) - of JFloat: - result.add($node.fnum) - of JBool: - result.add(if node.bval: "true" else: "false") - of JNull: - result.add "null" - proc `$`*(node: JsonNode): string = ## Converts `node` to its JSON Representation on one line. result = newStringOfCap(node.len shl 1) @@ -1015,47 +828,65 @@ proc `$`*(node: JsonNode): string = iterator items*(node: JsonNode): JsonNode = ## Iterator for the items of `node`. `node` has to be a JArray. - assert node.kind == JArray + assert node.kind == JArray, ": items() can not iterate a JsonNode of kind " & $node.kind for i in items(node.elems): yield i iterator mitems*(node: var JsonNode): var JsonNode = ## Iterator for the items of `node`. `node` has to be a JArray. Items can be ## modified. - assert node.kind == JArray + assert node.kind == JArray, ": mitems() can not iterate a JsonNode of kind " & $node.kind for i in mitems(node.elems): yield i iterator pairs*(node: JsonNode): tuple[key: string, val: JsonNode] = ## Iterator for the child elements of `node`. `node` has to be a JObject. - assert node.kind == JObject - for key, val in items(node.fields): + assert node.kind == JObject, ": pairs() can not iterate a JsonNode of kind " & $node.kind + for key, val in pairs(node.fields): yield (key, val) -iterator mpairs*(node: var JsonNode): var tuple[key: string, val: JsonNode] = - ## Iterator for the child elements of `node`. `node` has to be a JObject. - ## Items can be modified - assert node.kind == JObject - for keyVal in mitems(node.fields): - yield keyVal +iterator keys*(node: JsonNode): string = + ## Iterator for the keys in `node`. `node` has to be a JObject. + assert node.kind == JObject, ": keys() can not iterate a JsonNode of kind " & $node.kind + for key in node.fields.keys: + yield key -proc eat(p: var JsonParser, tok: TokKind) = - if p.tok == tok: discard getTok(p) - else: raiseParseErr(p, tokToStr[tok]) +iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = + ## Iterator for the child elements of `node`. `node` has to be a JObject. + ## Values can be modified + assert node.kind == JObject, ": mpairs() can not iterate a JsonNode of kind " & $node.kind + for key, val in mpairs(node.fields): + yield (key, val) -proc parseJson(p: var JsonParser): JsonNode = +proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool, depth = 0): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: - result = newJStringMove(p.a) - p.a = "" + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + result = JsonNode(kind: JString, str: move p.a) + else: + result = JsonNode(kind: JString) + shallowCopy(result.str, p.a) + p.a = "" discard getTok(p) of tkInt: - result = newJInt(parseBiggestInt(p.a)) + if rawIntegers: + result = newJRawNumber(p.a) + else: + try: + result = newJInt(parseBiggestInt(p.a)) + except ValueError: + result = newJRawNumber(p.a) discard getTok(p) of tkFloat: - result = newJFloat(parseFloat(p.a)) + if rawFloats: + result = newJRawNumber(p.a) + else: + try: + result = newJFloat(parseFloat(p.a)) + except ValueError: + result = newJRawNumber(p.a) discard getTok(p) of tkTrue: result = newJBool(true) @@ -1067,68 +898,91 @@ proc parseJson(p: var JsonParser): JsonNode = result = newJNull() discard getTok(p) of tkCurlyLe: + if depth > DepthLimit: + raiseParseErr(p, "}") result = newJObject() discard getTok(p) while p.tok != tkCurlyRi: if p.tok != tkString: - raiseParseErr(p, "string literal as key expected") + raiseParseErr(p, "string literal as key") var key = p.a discard getTok(p) eat(p, tkColon) - var val = parseJson(p) + var val = parseJson(p, rawIntegers, rawFloats, depth+1) result[key] = val if p.tok != tkComma: break discard getTok(p) eat(p, tkCurlyRi) of tkBracketLe: + if depth > DepthLimit: + raiseParseErr(p, "]") result = newJArray() discard getTok(p) while p.tok != tkBracketRi: - result.add(parseJson(p)) + result.add(parseJson(p, rawIntegers, rawFloats, depth+1)) if p.tok != tkComma: break discard getTok(p) eat(p, tkBracketRi) of tkError, tkCurlyRi, tkBracketRi, tkColon, tkComma, tkEof: raiseParseErr(p, "{") -when not defined(js): - proc parseJson*(s: Stream, filename: string): JsonNode = - ## Parses from a stream `s` into a `JsonNode`. `filename` is only needed - ## for nice error messages. - var p: JsonParser - p.open(s, filename) - defer: p.close() +iterator parseJsonFragments*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats = false): JsonNode = + ## Parses from a stream `s` into `JsonNodes`. `filename` is only needed + ## for nice error messages. + ## The JSON fragments are separated by whitespace. This can be substantially + ## faster than the comparable loop + ## `for x in splitWhitespace(s): yield parseJson(x)`. + ## This closes the stream `s` after it's done. + ## If `rawIntegers` is true, integer literals will not be converted to a `JInt` + ## field but kept as raw numbers via `JString`. + ## If `rawFloats` is true, floating point literals will not be converted to a `JFloat` + ## field but kept as raw numbers via `JString`. + var p: JsonParser + p.open(s, filename) + try: discard getTok(p) # read first token - result = p.parseJson() + while p.tok != tkEof: + yield p.parseJson(rawIntegers, rawFloats) + finally: + p.close() - proc parseJson*(buffer: string): JsonNode = - ## Parses JSON from `buffer`. - result = parseJson(newStringStream(buffer), "input") +proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats = false): JsonNode = + ## Parses from a stream `s` into a `JsonNode`. `filename` is only needed + ## for nice error messages. + ## If `s` contains extra data, it will raise `JsonParsingError`. + ## This closes the stream `s` after it's done. + ## If `rawIntegers` is true, integer literals will not be converted to a `JInt` + ## field but kept as raw numbers via `JString`. + ## If `rawFloats` is true, floating point literals will not be converted to a `JFloat` + ## field but kept as raw numbers via `JString`. + var p: JsonParser + p.open(s, filename) + try: + discard getTok(p) # read first token + result = p.parseJson(rawIntegers, rawFloats) + eat(p, tkEof) # check if there is no extra data + finally: + p.close() - proc parseFile*(filename: string): JsonNode = - ## Parses `file` into a `JsonNode`. - var stream = newFileStream(filename, fmRead) - if stream == nil: - raise newException(IOError, "cannot read from file: " & filename) - result = parseJson(stream, filename) -else: - from math import `mod` - type - JSObject = object - {.deprecated: [TJSObject: JSObject].} +when defined(js): + from std/math import `mod` + from std/jsffi import JsObject, `[]`, to + from std/private/jsutils import getProtoName, isInteger, isSafeInteger - proc parseNativeJson(x: cstring): JSObject {.importc: "JSON.parse".} + proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".} - proc getVarType(x): JsonNodeKind = + proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind = result = JNull - proc getProtoName(y): cstring - {.importc: "Object.prototype.toString.call".} case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray of "[object Object]": return JObject of "[object Number]": - if cast[float](x) mod 1.0 == 0: - return JInt + if isInteger(x) and 1.0 / cast[float](x) != -Inf: # preserve -0.0 as float + if isSafeInteger(x): + return JInt + else: + isRawNumber = true + return JString else: return JFloat of "[object Boolean]": return JBool @@ -1136,56 +990,384 @@ else: of "[object String]": return JString else: assert false - proc len(x: JSObject): int = - assert x.getVarType == JArray - asm """ - return `x`.length; - """ - - proc `[]`(x: JSObject, y: string): JSObject = - assert x.getVarType == JObject - asm """ - return `x`[`y`]; - """ - - proc `[]`(x: JSObject, y: int): JSObject = - assert x.getVarType == JArray - asm """ - return `x`[`y`]; - """ - - proc convertObject(x: JSObject): JsonNode = - case getVarType(x) + proc len(x: JsObject): int = + {.emit: """ + `result` = `x`.length; + """.} + + proc convertObject(x: JsObject): JsonNode = + var isRawNumber = false + case getVarType(x, isRawNumber) of JArray: result = newJArray() - for i in 0 .. <x.len: + for i in 0 ..< x.len: result.add(x[i].convertObject()) of JObject: result = newJObject() - asm """for (property in `x`) { + {.emit: """for (var property in `x`) { if (`x`.hasOwnProperty(property)) { - """ + """.} + var nimProperty: cstring - var nimValue: JSObject - asm "`nimProperty` = property; `nimValue` = `x`[property];" + var nimValue: JsObject + {.emit: "`nimProperty` = property; `nimValue` = `x`[property];".} result[$nimProperty] = nimValue.convertObject() - asm "}}" + {.emit: "}}".} of JInt: - result = newJInt(cast[int](x)) + result = newJInt(x.to(int)) of JFloat: - result = newJFloat(cast[float](x)) + result = newJFloat(x.to(float)) of JString: - result = newJString($cast[cstring](x)) + # Dunno what to do with isUnquoted here + if isRawNumber: + var value: cstring + {.emit: "`value` = `x`.toString();".} + result = newJRawNumber($value) + else: + result = newJString($x.to(cstring)) of JBool: - result = newJBool(cast[bool](x)) + result = newJBool(x.to(bool)) of JNull: result = newJNull() proc parseJson*(buffer: string): JsonNode = - return parseNativeJson(buffer).convertObject() + when nimvm: + return parseJson(newStringStream(buffer), "input") + else: + return parseNativeJson(buffer).convertObject() + +else: + proc parseJson*(buffer: string; rawIntegers = false, rawFloats = false): JsonNode = + ## Parses JSON from `buffer`. + ## If `buffer` contains extra data, it will raise `JsonParsingError`. + ## If `rawIntegers` is true, integer literals will not be converted to a `JInt` + ## field but kept as raw numbers via `JString`. + ## If `rawFloats` is true, floating point literals will not be converted to a `JFloat` + ## field but kept as raw numbers via `JString`. + result = parseJson(newStringStream(buffer), "input", rawIntegers, rawFloats) + + proc parseFile*(filename: string): JsonNode = + ## Parses `file` into a `JsonNode`. + ## If `file` contains extra data, it will raise `JsonParsingError`. + var stream = newFileStream(filename, fmRead) + if stream == nil: + raise newException(IOError, "cannot read from file: " & filename) + result = parseJson(stream, filename, rawIntegers=false, rawFloats=false) + +# -- Json deserialiser. -- + +template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], + ast: string) = + if node == nil: + raise newException(KeyError, "key not found: " & ast) + elif node.kind notin kinds: + let msg = "Incorrect JSON kind. Wanted '$1' in '$2' but got '$3'." % [ + $kinds, + ast, + $node.kind + ] + raise newException(JsonKindError, msg) + +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) + +# The following forward declarations don't work in older versions of Nim + +# forward declare all initFromJson + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) + +# initFromJson definitions + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str + +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval + +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy + +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or int.sizeof == 4: + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) + else: + dst = T(jsonNode.num) + else: + verifyJsonKind(jsonNode, {JInt}, jsonPath) + dst = cast[T](jsonNode.num) + +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JInt, JFloat, JString}, jsonPath) + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: + if jsonNode.kind == JFloat: + dst = T(jsonNode.fnum) + else: + dst = T(jsonNode.num) + +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) + +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen + +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) + +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) + else: + dst = some(default(T)) + initFromJson(dst.get, jsonNode, jsonPath) + +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] + + result = quote do: + initFromJson(`baseTyp`(`dst`), `jsonNode`, `jsonPath`) + +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) + +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) + +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` + when nimvm: + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: + `tmpSym`.`kindSym` = kindTmp + else: + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch + + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + else: + error("unhandled kind: " & $typeNode.kind, typeNode) + +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) + +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) + +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) + + type + Person = object + name: string + age: int + + Data = object + person: Person + list: seq[int] + + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] + + var jsonPath = "" + result = default(T) + initFromJson(result, node, jsonPath) when false: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: JsonParser @@ -1210,93 +1392,3 @@ when false: # { "json": 5 } # To get that we shall use, obj["json"] - -when isMainModule: - var parsed = parseFile("tests/testdata/jsontest.json") - var parsed2 = parseFile("tests/testdata/jsontest2.json") - - try: - discard parsed["key2"][12123] - assert(false) - except IndexError: assert(true) - - let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" - # nil passthrough - assert(testJson{"doesnt_exist"}{"anything"}.isNil) - testJson{["e", "f"]} = %true - assert(testJson["e"]["f"].bval) - - # make sure UTF-16 decoding works. - assert(testJson["c"].str == "🎃") - assert(testJson["d"].str == "æ") - - # make sure no memory leek when parsing invalid string - let startMemory = getOccupiedMem() - for i in 0 .. 10000: - try: - discard parseJson"""{ invalid""" - except: - discard - # memory diff should less than 2M - assert(abs(getOccupiedMem() - startMemory) < 2 * 1024 * 1024) - - - # test `$` - let stringified = $testJson - let parsedAgain = parseJson(stringified) - assert(parsedAgain["b"].str == "asd") - - # Bounds checking - try: - let a = testJson["a"][9] - assert(false, "EInvalidIndex not thrown") - except IndexError: - discard - try: - let a = testJson["a"][-1] - assert(false, "EInvalidIndex not thrown") - except IndexError: - discard - try: - assert(testJson["a"][0].num == 1, "Index doesn't correspond to its value") - except: - assert(false, "EInvalidIndex thrown for valid index") - - assert(testJson{"b"}.str=="asd", "Couldn't fetch a singly nested key with {}") - assert(isNil(testJson{"nonexistent"}), "Non-existent keys should return nil") - assert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") - assert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") - assert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") - assert(testJson{"a"}==parseJson"[1, 2, 3, 4]", "Didn't return a non-JObject when there was one to be found") - assert(isNil(parseJson("[1, 2, 3]"){"foo"}), "Indexing directly into a list should return nil") - - # Generator: - var j = %* [{"name": "John", "age": 30}, {"name": "Susan", "age": 31}] - assert j == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] - - var j2 = %* - [ - { - "name": "John", - "age": 30 - }, - { - "name": "Susan", - "age": 31 - } - ] - assert j2 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] - - var name = "John" - let herAge = 30 - const hisAge = 31 - - var j3 = %* - [ { "name": "John" - , "age": herAge - } - , { "name": "Susan" - , "age": hisAge - } - ] - assert j3 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim new file mode 100644 index 000000000..a8fc78e39 --- /dev/null +++ b/lib/pure/lenientops.nim @@ -0,0 +1,58 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module offers implementations of common binary operations +## like `+`, `-`, `*`, `/` and comparison operations, +## which work for mixed float/int operands. +## All operations convert the integer operand into the +## type of the float operand. For numerical expressions, the return +## type is always the type of the float involved in the expression, +## i.e., there is no auto conversion from float32 to float64. +## +## **Note:** In general, auto-converting from int to float loses +## information, which is why these operators live in a separate +## module. Use with care. +## +## Regarding binary comparison, this module only provides unequal operators. +## The equality operator `==` is omitted, because depending on the use case +## either casting to float or rounding to int might be preferred, and users +## should make an explicit choice. + +func `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = + F(i) + f +func `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = + f + F(i) + +func `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = + F(i) - f +func `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = + f - F(i) + +func `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = + F(i) * f +func `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = + f * F(i) + +func `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = + F(i) / f +func `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = + f / F(i) + +func `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.inline.} = + F(i) < f +func `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.inline.} = + f < F(i) +func `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.inline.} = + F(i) <= f +func `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.inline.} = + f <= F(i) + +# Note that we must not defined `>=` and `>`, because system.nim already has a +# template with signature (x, y: untyped): untyped, which would lead to +# ambiguous calls. diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index bfecf6a58..1efd97b24 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2009 Andreas Rumpf # # See the file "copying.txt", included in this @@ -12,10 +12,13 @@ ## needs refilling. import - strutils, streams + std/[strutils, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions const - EndOfFile* = '\0' ## end of file marker + EndOfFile* = '\0' ## end of file marker NewLines* = {'\c', '\L'} # Buffer handling: @@ -27,23 +30,17 @@ const type BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from ## this object. - bufpos*: int ## the current position within the buffer - buf*: cstring ## the buffer itself - bufLen*: int ## length of buffer in characters - input: Stream ## the input stream - lineNumber*: int ## the current line number + bufpos*: int ## the current position within the buffer + buf*: string ## the buffer itself + input: Stream ## the input stream + lineNumber*: int ## the current line number sentinel: int - lineStart: int # index of last line start in buffer + lineStart: int # index of last line start in buffer + offsetBase*: int # use `offsetBase + bufpos` to get the offset refillChars: set[char] -{.deprecated: [TBaseLexer: BaseLexer].} - -const - chrSize = sizeof(char) - proc close*(L: var BaseLexer) = ## closes the base lexer. This closes `L`'s associated stream too. - dealloc(L.buf) close(L.input) proc fillBuffer(L: var BaseLexer) = @@ -54,23 +51,31 @@ proc fillBuffer(L: var BaseLexer) = oldBufLen: int # we know here that pos == L.sentinel, but not if this proc # is called the first time by initBaseLexer() - assert(L.sentinel < L.bufLen) - toCopy = L.bufLen - L.sentinel - 1 + assert(L.sentinel + 1 <= L.buf.len) + toCopy = L.buf.len - (L.sentinel + 1) assert(toCopy >= 0) if toCopy > 0: - moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) - # "moveMem" handles overlapping regions - charsRead = readData(L.input, addr(L.buf[toCopy]), - (L.sentinel + 1) * chrSize) div chrSize + when defined(js) or defined(nimscript): + # nimscript has to be here to avoid compiling other branch (moveMem) + for i in 0 ..< toCopy: + L.buf[i] = L.buf[L.sentinel + 1 + i] + else: + when nimvm: + for i in 0 ..< toCopy: + L.buf[i] = L.buf[L.sentinel + 1 + i] + else: + # "moveMem" handles overlapping regions + moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy) + charsRead = L.input.readDataStr(L.buf, toCopy ..< toCopy + L.sentinel + 1) s = toCopy + charsRead if charsRead < L.sentinel + 1: - L.buf[s] = EndOfFile # set end marker + L.buf[s] = EndOfFile # set end marker L.sentinel = s else: # compute sentinel: - dec(s) # BUGFIX (valgrind) + dec(s) # BUGFIX (valgrind) while true: - assert(s < L.bufLen) + assert(s < L.buf.len) while s >= 0 and L.buf[s] notin L.refillChars: dec(s) if s >= 0: # we found an appropriate character for a sentinel: @@ -79,31 +84,29 @@ proc fillBuffer(L: var BaseLexer) = else: # rather than to give up here because the line is too long, # double the buffer's size and try again: - oldBufLen = L.bufLen - L.bufLen = L.bufLen * 2 - L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize)) - assert(L.bufLen - oldBufLen == oldBufLen) - charsRead = readData(L.input, addr(L.buf[oldBufLen]), - oldBufLen * chrSize) div chrSize + oldBufLen = L.buf.len + L.buf.setLen(L.buf.len * 2) + charsRead = readDataStr(L.input, L.buf, oldBufLen ..< L.buf.len) if charsRead < oldBufLen: L.buf[oldBufLen + charsRead] = EndOfFile L.sentinel = oldBufLen + charsRead break - s = L.bufLen - 1 + s = L.buf.len - 1 proc fillBaseLexer(L: var BaseLexer, pos: int): int = assert(pos <= L.sentinel) if pos < L.sentinel: - result = pos + 1 # nothing to do + result = pos + 1 # nothing to do else: fillBuffer(L) - L.bufpos = 0 # XXX: is this really correct? + L.offsetBase += pos + L.bufpos = 0 result = 0 proc handleCR*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\c' in the buffer; it returns the the + ## Call this if you scanned over `'\c'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\c'. + ## of the `'\c'`. assert(L.buf[pos] == '\c') inc(L.lineNumber) result = fillBaseLexer(L, pos) @@ -112,16 +115,17 @@ proc handleCR*(L: var BaseLexer, pos: int): int = L.lineStart = result proc handleLF*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\L' in the buffer; it returns the the + ## Call this if you scanned over `'\L'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\L'. + ## of the `'\L'`. assert(L.buf[pos] == '\L') inc(L.lineNumber) result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result; L.lineStart = result proc handleRefillChar*(L: var BaseLexer, pos: int): int = - ## To be documented. + ## Call this if a terminator character other than a new line is scanned + ## at `pos`; it returns the position to continue the scanning from. assert(L.buf[pos] in L.refillChars) result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result; @@ -137,12 +141,12 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192; assert(input != nil) L.input = input L.bufpos = 0 - L.bufLen = bufLen + L.offsetBase = 0 L.refillChars = refillChars - L.buf = cast[cstring](alloc(bufLen * chrSize)) + L.buf = newString(bufLen) L.sentinel = bufLen - 1 L.lineStart = 0 - L.lineNumber = 1 # lines start at 1 + L.lineNumber = 1 # lines start at 1 fillBuffer(L) skipUtf8Bom(L) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index aa55b5ade..c30f68af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -7,94 +7,307 @@ # distribution, for details about the copyright. # -## This module implements a simple logger. It has been designed to be as simple -## as possible to avoid bloat, if this library does not fulfill your needs, -## write your own. +## This module implements a simple logger. ## -## Format strings support the following variables which must be prefixed with -## the dollar operator (``$``): +## It has been designed to be as simple as possible to avoid bloat. +## If this library does not fulfill your needs, write your own. +## +## Basic usage +## =========== +## +## To get started, first create a logger: +## +## ```Nim +## import std/logging +## +## var logger = newConsoleLogger() +## ``` +## +## The logger that was created above logs to the console, but this module +## also provides loggers that log to files, such as the +## `FileLogger<#FileLogger>`_. Creating custom loggers is also possible by +## inheriting from the `Logger<#Logger>`_ type. +## +## Once a logger has been created, call its `log proc +## <#log.e,ConsoleLogger,Level,varargs[string,]>`_ to log a message: +## +## ```Nim +## logger.log(lvlInfo, "a log message") +## # Output: INFO a log message +## ``` +## +## The ``INFO`` within the output is the result of a format string being +## prepended to the message, and it will differ depending on the message's +## level. Format strings are `explained in more detail +## here<#basic-usage-format-strings>`_. +## +## There are six logging levels: debug, info, notice, warn, error, and fatal. +## They are described in more detail within the `Level enum's documentation +## <#Level>`_. A message is logged if its level is at or above both the logger's +## ``levelThreshold`` field and the global log filter. The latter can be changed +## with the `setLogFilter proc<#setLogFilter,Level>`_. +## +## .. warning:: +## For loggers that log to a console or to files, only error and fatal +## messages will cause their output buffers to be flushed immediately by default. +## set ``flushThreshold`` when creating the logger to change this. +## +## Handlers +## -------- +## +## When using multiple loggers, calling the log proc for each logger can +## become repetitive. Instead of doing that, register each logger that will be +## used with the `addHandler proc<#addHandler,Logger>`_, which is demonstrated +## in the following example: +## +## ```Nim +## import std/logging +## +## var consoleLog = newConsoleLogger() +## var fileLog = newFileLogger("errors.log", levelThreshold=lvlError) +## var rollingLog = newRollingFileLogger("rolling.log") +## +## addHandler(consoleLog) +## addHandler(fileLog) +## addHandler(rollingLog) +## ``` +## +## After doing this, use either the `log template +## <#log.t,Level,varargs[string,]>`_ or one of the level-specific templates, +## such as the `error template<#error.t,varargs[string,]>`_, to log messages +## to all registered handlers at once. +## +## ```Nim +## # This example uses the loggers created above +## log(lvlError, "an error occurred") +## error("an error occurred") # Equivalent to the above line +## info("something normal happened") # Will not be written to errors.log +## ``` +## +## Note that a message's level is still checked against each handler's +## ``levelThreshold`` and the global log filter. +## +## Format strings +## -------------- +## +## Log messages are prefixed with format strings. These strings contain +## placeholders for variables, such as ``$time``, that are replaced with their +## corresponding values, such as the current time, before they are prepended to +## a log message. Characters that are not part of variables are unaffected. +## +## The format string used by a logger can be specified by providing the `fmtStr` +## argument when creating the logger or by setting its `fmtStr` field afterward. +## If not specified, the `default format string<#defaultFmtStr>`_ is used. +## +## The following variables, which must be prefixed with a dollar sign (``$``), +## are available: ## ## ============ ======================= -## Operator Output +## Variable Output ## ============ ======================= ## $date Current date ## $time Current time ## $datetime $dateT$time -## $app ``os.getAppFilename()`` -## $appname base name of $app -## $appdir directory name of $app -## $levelid first letter of log level -## $levelname log level name +## $app `os.getAppFilename()<os.html#getAppFilename>`_ +## $appname Base name of ``$app`` +## $appdir Directory name of ``$app`` +## $levelid First letter of log level +## $levelname Log level name ## ============ ======================= ## +## Note that ``$app``, ``$appname``, and ``$appdir`` are not supported when +## using the JavaScript backend. +## +## The following example illustrates how to use format strings: +## +## ```Nim +## import std/logging ## -## The following example demonstrates logging to three different handlers -## simultaneously: +## var logger = newConsoleLogger(fmtStr="[$time] - $levelname: ") +## logger.log(lvlInfo, "this is a message") +## # Output: [19:50:13] - INFO: this is a message +## ``` ## -## .. code-block:: nim +## Notes when using multiple threads +## --------------------------------- ## -## var L = newConsoleLogger() -## var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) -## var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) -## addHandler(L) -## addHandler(fL) -## addHandler(rL) -## info("920410:52 accepted") -## warn("4 8 15 16 23 4-- Error") -## error("922044:16 SYSTEM FAILURE") -## fatal("SYSTEM FAILURE SYSTEM FAILURE") +## There are a few details to keep in mind when using this module within +## multiple threads: +## * The global log filter is actually a thread-local variable, so it needs to +## be set in each thread that uses this module. +## * The list of registered handlers is also a thread-local variable. If a +## handler will be used in multiple threads, it needs to be registered in +## each of those threads. ## -## **Warning:** The global list of handlers is a thread var, this means that -## the handlers must be re-added in each thread. +## See also +## ======== +## * `strutils module<strutils.html>`_ for common string functions +## * `strformat module<strformat.html>`_ for string interpolation and formatting +## * `strscans module<strscans.html>`_ for ``scanf`` and ``scanp`` macros, which +## offer easier substring extraction than regular expressions -import strutils, os, times +import std/[strutils, times] +when not defined(js): + import std/os + +when defined(nimPreviewSlimSystem): + import std/syncio type - Level* = enum ## logging level - lvlAll, ## all levels active - lvlDebug, ## debug level (and any above) active - lvlInfo, ## info level (and any above) active - lvlWarn, ## warn level (and any above) active - lvlError, ## error level (and any above) active - lvlFatal, ## fatal level (and any above) active - lvlNone ## no levels active + Level* = enum ## \ + ## Enumeration of logging levels. + ## + ## Debug messages represent the lowest logging level, and fatal error + ## messages represent the highest logging level. ``lvlAll`` can be used + ## to enable all messages, while ``lvlNone`` can be used to disable all + ## messages. + ## + ## Typical usage for each logging level, from lowest to highest, is + ## described below: + ## + ## * **Debug** - debugging information helpful only to developers + ## * **Info** - anything associated with normal operation and without + ## any particular importance + ## * **Notice** - more important information that users should be + ## notified about + ## * **Warn** - impending problems that require some attention + ## * **Error** - error conditions that the application can recover from + ## * **Fatal** - fatal errors that prevent the application from continuing + ## + ## It is completely up to the application how to utilize each level. + ## + ## Individual loggers have a ``levelThreshold`` field that filters out + ## any messages with a level lower than the threshold. There is also + ## a global filter that applies to all log messages, and it can be changed + ## using the `setLogFilter proc<#setLogFilter,Level>`_. + lvlAll, ## All levels active + lvlDebug, ## Debug level and above are active + lvlInfo, ## Info level and above are active + lvlNotice, ## Notice level and above are active + lvlWarn, ## Warn level and above are active + lvlError, ## Error level and above are active + lvlFatal, ## Fatal level and above are active + lvlNone ## No levels active; nothing is logged const - LevelNames*: array [Level, string] = [ - "DEBUG", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE" - ] - - defaultFmtStr* = "$levelname " ## default format string - verboseFmtStr* = "$levelid, [$datetime] -- $appname: " + LevelNames*: array[Level, string] = [ + "DEBUG", "DEBUG", "INFO", "NOTICE", "WARN", "ERROR", "FATAL", "NONE" + ] ## Array of strings representing each logging level. + + defaultFmtStr* = "$levelname " ## The default format string. + verboseFmtStr* = "$levelid, [$datetime] -- $appname: " ## \ + ## A more verbose format string. + ## + ## This string can be passed as the ``frmStr`` argument to procs that create + ## new loggers, such as the `newConsoleLogger proc<#newConsoleLogger>`_. + ## + ## If a different format string is preferred, refer to the + ## `documentation about format strings<#basic-usage-format-strings>`_ + ## for more information, including a list of available variables. + defaultFlushThreshold = when NimMajor >= 2: + when defined(nimV1LogFlushBehavior): lvlError else: lvlAll + else: + when defined(nimFlushAllLogs): lvlAll else: lvlError + ## The threshold above which log messages to file-like loggers + ## are automatically flushed. + ## + ## By default, only error and fatal messages are logged, + ## but defining ``-d:nimFlushAllLogs`` will make all levels be flushed type - Logger* = ref object of RootObj ## abstract logger; the base type of all loggers - levelThreshold*: Level ## only messages of level >= levelThreshold - ## should be processed - fmtStr*: string ## = defaultFmtStr by default, see substituteLog for $date etc. - - ConsoleLogger* = ref object of Logger ## logger that writes the messages to the - ## console - - FileLogger* = ref object of Logger ## logger that writes the messages to a file - file*: File ## the wrapped file. - - RollingFileLogger* = ref object of FileLogger ## logger that writes the - ## messages to a file and - ## performs log rotation - maxLines: int # maximum number of lines - curLine : int - baseName: string # initial filename - baseMode: FileMode # initial file mode - logFiles: int # how many log files already created, e.g. basename.1, basename.2... - bufSize: int # size of output buffer (-1: use system defaults, 0: unbuffered, >0: fixed buffer size) - -{.deprecated: [TLevel: Level, PLogger: Logger, PConsoleLogger: ConsoleLogger, - PFileLogger: FileLogger, PRollingFileLogger: RollingFileLogger].} - -proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): string = - ## Format a log message using the ``frmt`` format string, ``level`` and varargs. - ## See the module documentation for the format string syntax. + Logger* = ref object of RootObj + ## The abstract base type of all loggers. + ## + ## Custom loggers should inherit from this type. They should also provide + ## their own implementation of the + ## `log method<#log.e,Logger,Level,varargs[string,]>`_. + ## + ## See also: + ## * `ConsoleLogger<#ConsoleLogger>`_ + ## * `FileLogger<#FileLogger>`_ + ## * `RollingFileLogger<#RollingFileLogger>`_ + levelThreshold*: Level ## Only messages that are at or above this + ## threshold will be logged + fmtStr*: string ## Format string to prepend to each log message; + ## defaultFmtStr is the default + + ConsoleLogger* = ref object of Logger + ## A logger that writes log messages to the console. + ## + ## Create a new ``ConsoleLogger`` with the `newConsoleLogger proc + ## <#newConsoleLogger>`_. + ## + ## See also: + ## * `FileLogger<#FileLogger>`_ + ## * `RollingFileLogger<#RollingFileLogger>`_ + useStderr*: bool ## If true, writes to stderr; otherwise, writes to stdout + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately + +when not defined(js): + type + FileLogger* = ref object of Logger + ## A logger that writes log messages to a file. + ## + ## Create a new ``FileLogger`` with the `newFileLogger proc + ## <#newFileLogger,File>`_. + ## + ## **Note:** This logger is not available for the JavaScript backend. + ## + ## See also: + ## * `ConsoleLogger<#ConsoleLogger>`_ + ## * `RollingFileLogger<#RollingFileLogger>`_ + file*: File ## The wrapped file + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately + + RollingFileLogger* = ref object of FileLogger + ## A logger that writes log messages to a file while performing log + ## rotation. + ## + ## Create a new ``RollingFileLogger`` with the `newRollingFileLogger proc + ## <#newRollingFileLogger,FileMode,Positive,int>`_. + ## + ## **Note:** This logger is not available for the JavaScript backend. + ## + ## See also: + ## * `ConsoleLogger<#ConsoleLogger>`_ + ## * `FileLogger<#FileLogger>`_ + maxLines: int # maximum number of lines + curLine: int + baseName: string # initial filename + baseMode: FileMode # initial file mode + logFiles: int # how many log files already created, e.g. basename.1, basename.2... + bufSize: int # size of output buffer (-1: use system defaults, 0: unbuffered, >0: fixed buffer size) + +var + level {.threadvar.}: Level ## global log filter + handlers {.threadvar.}: seq[Logger] ## handlers with their own log levels + +proc substituteLog*(frmt: string, level: Level, + args: varargs[string, `$`]): string = + ## Formats a log message at the specified level with the given format string. + ## + ## The `format variables<#basic-usage-format-strings>`_ present within + ## ``frmt`` will be replaced with the corresponding values before being + ## prepended to ``args`` and returned. + ## + ## Unless you are implementing a custom logger, there is little need to call + ## this directly. Use either a logger's log method or one of the logging + ## templates. + ## + ## See also: + ## * `log method<#log.e,ConsoleLogger,Level,varargs[string,]>`_ + ## for the ConsoleLogger + ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ + ## for the FileLogger + ## * `log method<#log.e,RollingFileLogger,Level,varargs[string,]>`_ + ## for the RollingFileLogger + ## * `log template<#log.t,Level,varargs[string,]>`_ + runnableExamples: + doAssert substituteLog(defaultFmtStr, lvlInfo, "a message") == "INFO a message" + doAssert substituteLog("$levelid - ", lvlError, "an error") == "E - an error" + doAssert substituteLog("$levelid", lvlDebug, "error") == "Derror" var msgLen = 0 for arg in args: msgLen += arg.len @@ -107,17 +320,19 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str else: inc(i) var v = "" - var app = getAppFilename() - while frmt[i] in IdentChars: - v.add(toLower(frmt[i])) + let app = when defined(js): "" else: getAppFilename() + while i < frmt.len and frmt[i] in IdentChars: + v.add(toLowerAscii(frmt[i])) inc(i) case v of "date": result.add(getDateStr()) of "time": result.add(getClockStr()) of "datetime": result.add(getDateStr() & "T" & getClockStr()) - of "app": result.add(app) - of "appdir": result.add(app.splitFile.dir) - of "appname": result.add(app.splitFile.name) + of "app": result.add(app) + of "appdir": + when not defined(js): result.add(app.splitFile.dir) + of "appname": + when not defined(js): result.add(app.splitFile.name) of "levelid": result.add(LevelNames[level][0]) of "levelname": result.add(LevelNames[level]) else: discard @@ -125,129 +340,358 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str result.add(arg) method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {. - raises: [Exception], - tags: [TimeEffect, WriteIOEffect, ReadIOEffect], base.} = - ## Override this method in custom loggers. Default implementation does + raises: [Exception], gcsafe, + tags: [RootEffect], base.} = + ## Override this method in custom loggers. The default implementation does ## nothing. + ## + ## See also: + ## * `log method<#log.e,ConsoleLogger,Level,varargs[string,]>`_ + ## for the ConsoleLogger + ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ + ## for the FileLogger + ## * `log method<#log.e,RollingFileLogger,Level,varargs[string,]>`_ + ## for the RollingFileLogger + ## * `log template<#log.t,Level,varargs[string,]>`_ discard method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = - ## Logs to the console using ``logger`` only. - if level >= logger.levelThreshold: - writeLine(stdout, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(stdout) - -method log*(logger: FileLogger, level: Level, args: varargs[string, `$`]) = - ## Logs to a file using ``logger`` only. - if level >= logger.levelThreshold: - writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) - -proc defaultFilename*(): string = - ## Returns the default filename for a logger. - var (path, name, _) = splitFile(getAppFilename()) - result = changeFileExt(path / name, "log") - -proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr): ConsoleLogger = - ## Creates a new console logger. This logger logs to the console. + ## Logs to the console with the given `ConsoleLogger<#ConsoleLogger>`_ only. + ## + ## This method ignores the list of registered handlers. + ## + ## Whether the message is logged depends on both the ConsoleLogger's + ## ``levelThreshold`` field and the global log filter set using the + ## `setLogFilter proc<#setLogFilter,Level>`_. + ## + ## **Note:** Only error and fatal messages will cause the output buffer + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. + ## + ## See also: + ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ + ## for the FileLogger + ## * `log method<#log.e,RollingFileLogger,Level,varargs[string,]>`_ + ## for the RollingFileLogger + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## var consoleLog = newConsoleLogger() + ## consoleLog.log(lvlInfo, "this is a message") + ## consoleLog.log(lvlError, "error code is: ", 404) + ## ``` + if level >= logging.level and level >= logger.levelThreshold: + let ln = substituteLog(logger.fmtStr, level, args) + when defined(js): + let cln = ln.cstring + case level + of lvlDebug: {.emit: "console.debug(`cln`);".} + of lvlInfo: {.emit: "console.info(`cln`);".} + of lvlWarn: {.emit: "console.warn(`cln`);".} + of lvlError: {.emit: "console.error(`cln`);".} + else: {.emit: "console.log(`cln`);".} + else: + try: + var handle = stdout + if logger.useStderr: + handle = stderr + writeLine(handle, ln) + if level >= logger.flushThreshold: flushFile(handle) + except IOError: + discard + +proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, + useStderr = false, flushThreshold = defaultFlushThreshold): ConsoleLogger = + ## Creates a new `ConsoleLogger<#ConsoleLogger>`_. + ## + ## By default, log messages are written to ``stdout``. If ``useStderr`` is + ## true, they are written to ``stderr`` instead. + ## + ## For the JavaScript backend, log messages are written to the console, + ## and ``useStderr`` is ignored. + ## + ## See also: + ## * `newFileLogger proc<#newFileLogger,File>`_ that uses a file handle + ## * `newFileLogger proc<#newFileLogger,FileMode,int>`_ + ## that accepts a filename + ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,Positive,int>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## var normalLog = newConsoleLogger() + ## var formatLog = newConsoleLogger(fmtStr=verboseFmtStr) + ## var errorLog = newConsoleLogger(levelThreshold=lvlError, useStderr=true) + ## ``` new result result.fmtStr = fmtStr result.levelThreshold = levelThreshold - -proc newFileLogger*(filename = defaultFilename(), - mode: FileMode = fmAppend, - levelThreshold = lvlAll, - fmtStr = defaultFmtStr, - bufSize: int = -1): FileLogger = - ## Creates a new file logger. This logger logs to a file. - ## Use ``bufSize`` as size of the output buffer when writing the file - ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). - new(result) - result.levelThreshold = levelThreshold - result.file = open(filename, mode, bufSize = bufSize) - result.fmtStr = fmtStr - -# ------ - -proc countLogLines(logger: RollingFileLogger): int = - result = 0 - for line in logger.file.lines(): - result.inc() - -proc countFiles(filename: string): int = - # Example: file.log.1 - result = 0 - let (dir, name, ext) = splitFile(filename) - for kind, path in walkDir(dir): - if kind == pcFile: - let llfn = name & ext & ExtSep - if path.extractFilename.startsWith(llfn): - let numS = path.extractFilename[llfn.len .. ^1] - try: - let num = parseInt(numS) - if num > result: - result = num - except ValueError: discard - -proc newRollingFileLogger*(filename = defaultFilename(), - mode: FileMode = fmReadWrite, - levelThreshold = lvlAll, - fmtStr = defaultFmtStr, - maxLines = 1000, - bufSize: int = -1): RollingFileLogger = - ## Creates a new rolling file logger. Once a file reaches ``maxLines`` lines - ## a new log file will be started and the old will be renamed. - ## Use ``bufSize`` as size of the output buffer when writing the file - ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). - new(result) - result.levelThreshold = levelThreshold - result.fmtStr = fmtStr - result.maxLines = maxLines - result.bufSize = bufSize - result.file = open(filename, mode, bufSize=result.bufSize) - result.curLine = 0 - result.baseName = filename - result.baseMode = mode - - result.logFiles = countFiles(filename) - - if mode == fmAppend: - # We need to get a line count because we will be appending to the file. - result.curLine = countLogLines(result) - -proc rotate(logger: RollingFileLogger) = - let (dir, name, ext) = splitFile(logger.baseName) - for i in countdown(logger.logFiles, 0): - let srcSuff = if i != 0: ExtSep & $i else: "" - moveFile(dir / (name & ext & srcSuff), - dir / (name & ext & ExtSep & $(i+1))) - -method log*(logger: RollingFileLogger, level: Level, args: varargs[string, `$`]) = - ## Logs to a file using rolling ``logger`` only. - if level >= logger.levelThreshold: - if logger.curLine >= logger.maxLines: - logger.file.close() - rotate(logger) - logger.logFiles.inc - logger.curLine = 0 - logger.file = open(logger.baseName, logger.baseMode, bufSize = logger.bufSize) - - writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) - logger.curLine.inc + result.flushThreshold = flushThreshold + result.useStderr = useStderr + +when not defined(js): + method log*(logger: FileLogger, level: Level, args: varargs[string, `$`]) = + ## Logs a message at the specified level using the given + ## `FileLogger<#FileLogger>`_ only. + ## + ## This method ignores the list of registered handlers. + ## + ## Whether the message is logged depends on both the FileLogger's + ## ``levelThreshold`` field and the global log filter set using the + ## `setLogFilter proc<#setLogFilter,Level>`_. + ## + ## **Notes:** + ## * Only error and fatal messages will cause the output buffer + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. + ## * This method is not available for the JavaScript backend. + ## + ## See also: + ## * `log method<#log.e,ConsoleLogger,Level,varargs[string,]>`_ + ## for the ConsoleLogger + ## * `log method<#log.e,RollingFileLogger,Level,varargs[string,]>`_ + ## for the RollingFileLogger + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## var fileLog = newFileLogger("messages.log") + ## fileLog.log(lvlInfo, "this is a message") + ## fileLog.log(lvlError, "error code is: ", 404) + ## ``` + if level >= logging.level and level >= logger.levelThreshold: + writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) + if level >= logger.flushThreshold: flushFile(logger.file) + + proc defaultFilename*(): string = + ## Returns the filename that is used by default when naming log files. + ## + ## **Note:** This proc is not available for the JavaScript backend. + var (path, name, _) = splitFile(getAppFilename()) + result = changeFileExt(path / name, "log") + + proc newFileLogger*(file: File, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr, + flushThreshold = defaultFlushThreshold): FileLogger = + ## Creates a new `FileLogger<#FileLogger>`_ that uses the given file handle. + ## + ## **Note:** This proc is not available for the JavaScript backend. + ## + ## See also: + ## * `newConsoleLogger proc<#newConsoleLogger>`_ + ## * `newFileLogger proc<#newFileLogger,FileMode,int>`_ + ## that accepts a filename + ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,Positive,int>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## var messages = open("messages.log", fmWrite) + ## var formatted = open("formatted.log", fmWrite) + ## var errors = open("errors.log", fmWrite) + ## + ## var normalLog = newFileLogger(messages) + ## var formatLog = newFileLogger(formatted, fmtStr=verboseFmtStr) + ## var errorLog = newFileLogger(errors, levelThreshold=lvlError) + ## ``` + new(result) + result.file = file + result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold + result.fmtStr = fmtStr + + proc newFileLogger*(filename = defaultFilename(), + mode: FileMode = fmAppend, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr, + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): FileLogger = + ## Creates a new `FileLogger<#FileLogger>`_ that logs to a file with the + ## given filename. + ## + ## ``bufSize`` controls the size of the output buffer that is used when + ## writing to the log file. The following values can be provided: + ## * ``-1`` - use system defaults + ## * ``0`` - unbuffered + ## * ``> 0`` - fixed buffer size + ## + ## **Note:** This proc is not available for the JavaScript backend. + ## + ## See also: + ## * `newConsoleLogger proc<#newConsoleLogger>`_ + ## * `newFileLogger proc<#newFileLogger,File>`_ that uses a file handle + ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,Positive,int>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## var normalLog = newFileLogger("messages.log") + ## var formatLog = newFileLogger("formatted.log", fmtStr=verboseFmtStr) + ## var errorLog = newFileLogger("errors.log", levelThreshold=lvlError) + ## ``` + let file = open(filename, mode, bufSize = bufSize) + newFileLogger(file, levelThreshold, fmtStr, flushThreshold) + + # ------ + + proc countLogLines(logger: RollingFileLogger): int = + let fp = open(logger.baseName, fmRead) + for line in fp.lines(): + result.inc() + fp.close() + + proc countFiles(filename: string): int = + # Example: file.log.1 + result = 0 + var (dir, name, ext) = splitFile(filename) + if dir == "": + dir = "." + for kind, path in walkDir(dir): + if kind == pcFile: + let llfn = name & ext & ExtSep + if path.extractFilename.startsWith(llfn): + let numS = path.extractFilename[llfn.len .. ^1] + try: + let num = parseInt(numS) + if num > result: + result = num + except ValueError: discard + + proc newRollingFileLogger*(filename = defaultFilename(), + mode: FileMode = fmReadWrite, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr, + maxLines: Positive = 1000, + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): RollingFileLogger = + ## Creates a new `RollingFileLogger<#RollingFileLogger>`_. + ## + ## Once the current log file being written to contains ``maxLines`` lines, + ## a new log file will be created, and the old log file will be renamed. + ## + ## ``bufSize`` controls the size of the output buffer that is used when + ## writing to the log file. The following values can be provided: + ## * ``-1`` - use system defaults + ## * ``0`` - unbuffered + ## * ``> 0`` - fixed buffer size + ## + ## **Note:** This proc is not available in the JavaScript backend. + ## + ## See also: + ## * `newConsoleLogger proc<#newConsoleLogger>`_ + ## * `newFileLogger proc<#newFileLogger,File>`_ that uses a file handle + ## * `newFileLogger proc<#newFileLogger,FileMode,int>`_ + ## that accepts a filename + ## + ## **Examples:** + ## + ## ```Nim + ## var normalLog = newRollingFileLogger("messages.log") + ## var formatLog = newRollingFileLogger("formatted.log", fmtStr=verboseFmtStr) + ## var shortLog = newRollingFileLogger("short.log", maxLines=200) + ## var errorLog = newRollingFileLogger("errors.log", levelThreshold=lvlError) + ## ``` + new(result) + result.levelThreshold = levelThreshold + result.fmtStr = fmtStr + result.maxLines = maxLines + result.bufSize = bufSize + result.file = open(filename, mode, bufSize = result.bufSize) + result.curLine = 0 + result.baseName = filename + result.baseMode = mode + result.flushThreshold = flushThreshold + + result.logFiles = countFiles(filename) + + if mode == fmAppend: + # We need to get a line count because we will be appending to the file. + result.curLine = countLogLines(result) + + proc rotate(logger: RollingFileLogger) = + let (dir, name, ext) = splitFile(logger.baseName) + for i in countdown(logger.logFiles, 0): + let srcSuff = if i != 0: ExtSep & $i else: "" + moveFile(dir / (name & ext & srcSuff), + dir / (name & ext & ExtSep & $(i+1))) + + method log*(logger: RollingFileLogger, level: Level, args: varargs[string, `$`]) = + ## Logs a message at the specified level using the given + ## `RollingFileLogger<#RollingFileLogger>`_ only. + ## + ## This method ignores the list of registered handlers. + ## + ## Whether the message is logged depends on both the RollingFileLogger's + ## ``levelThreshold`` field and the global log filter set using the + ## `setLogFilter proc<#setLogFilter,Level>`_. + ## + ## **Notes:** + ## * Only error and fatal messages will cause the output buffer + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. + ## * This method is not available for the JavaScript backend. + ## + ## See also: + ## * `log method<#log.e,ConsoleLogger,Level,varargs[string,]>`_ + ## for the ConsoleLogger + ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ + ## for the FileLogger + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## var rollingLog = newRollingFileLogger("messages.log") + ## rollingLog.log(lvlInfo, "this is a message") + ## rollingLog.log(lvlError, "error code is: ", 404) + ## ``` + if level >= logging.level and level >= logger.levelThreshold: + if logger.curLine >= logger.maxLines: + logger.file.close() + rotate(logger) + logger.logFiles.inc + logger.curLine = 0 + logger.file = open(logger.baseName, logger.baseMode, + bufSize = logger.bufSize) + + writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) + if level >= logger.flushThreshold: flushFile(logger.file) + logger.curLine.inc # -------- -var level {.threadvar.}: Level ## global log filter -var handlers {.threadvar.}: seq[Logger] ## handlers with their own log levels - proc logLoop(level: Level, args: varargs[string, `$`]) = for logger in items(handlers): if level >= logger.levelThreshold: log(logger, level, args) template log*(level: Level, args: varargs[string, `$`]) = - ## Logs a message to all registered handlers at the given level. + ## Logs a message at the specified level to all registered handlers. + ## + ## Whether the message is logged depends on both the FileLogger's + ## `levelThreshold` field and the global log filter set using the + ## `setLogFilter proc<#setLogFilter,Level>`_. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## log(lvlInfo, "This is an example.") + ## ``` + ## + ## See also: + ## * `debug template<#debug.t,varargs[string,]>`_ + ## * `info template<#info.t,varargs[string,]>`_ + ## * `notice template<#notice.t,varargs[string,]>`_ + ## * `warn template<#warn.t,varargs[string,]>`_ + ## * `error template<#error.t,varargs[string,]>`_ + ## * `fatal template<#fatal.t,varargs[string,]>`_ bind logLoop bind `%` bind logging.level @@ -257,49 +701,207 @@ template log*(level: Level, args: varargs[string, `$`]) = template debug*(args: varargs[string, `$`]) = ## Logs a debug message to all registered handlers. + ## + ## Debug messages are typically useful to the application developer only, + ## and they are usually disabled in release builds, although this template + ## does not make that distinction. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## debug("myProc called with arguments: foo, 5") + ## ``` + ## + ## See also: + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## * `info template<#info.t,varargs[string,]>`_ + ## * `notice template<#notice.t,varargs[string,]>`_ log(lvlDebug, args) template info*(args: varargs[string, `$`]) = ## Logs an info message to all registered handlers. + ## + ## Info messages are typically generated during the normal operation + ## of an application and are of no particular importance. It can be useful to + ## aggregate these messages for later analysis. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## info("Application started successfully.") + ## ``` + ## + ## See also: + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## * `debug template<#debug.t,varargs[string,]>`_ + ## * `notice template<#notice.t,varargs[string,]>`_ log(lvlInfo, args) +template notice*(args: varargs[string, `$`]) = + ## Logs an notice to all registered handlers. + ## + ## Notices are semantically very similar to info messages, but they are meant + ## to be messages that the user should be actively notified about, depending + ## on the application. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## notice("An important operation has completed.") + ## ``` + ## + ## See also: + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## * `debug template<#debug.t,varargs[string,]>`_ + ## * `info template<#info.t,varargs[string,]>`_ + log(lvlNotice, args) + template warn*(args: varargs[string, `$`]) = ## Logs a warning message to all registered handlers. + ## + ## A warning is a non-error message that may indicate impending problems or + ## degraded performance. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## warn("The previous operation took too long to process.") + ## ``` + ## + ## See also: + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## * `error template<#error.t,varargs[string,]>`_ + ## * `fatal template<#fatal.t,varargs[string,]>`_ log(lvlWarn, args) template error*(args: varargs[string, `$`]) = ## Logs an error message to all registered handlers. + ## + ## Error messages are for application-level error conditions, such as when + ## some user input generated an exception. Typically, the application will + ## continue to run, but with degraded functionality or loss of data, and + ## these effects might be visible to users. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## error("An exception occurred while processing the form.") + ## ``` + ## + ## See also: + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## * `warn template<#warn.t,varargs[string,]>`_ + ## * `fatal template<#fatal.t,varargs[string,]>`_ log(lvlError, args) template fatal*(args: varargs[string, `$`]) = ## Logs a fatal error message to all registered handlers. + ## + ## Fatal error messages usually indicate that the application cannot continue + ## to run and will exit due to a fatal condition. This template only logs the + ## message, and it is the application's responsibility to exit properly. + ## + ## **Examples:** + ## + ## ```Nim + ## var logger = newConsoleLogger() + ## addHandler(logger) + ## + ## fatal("Can't open database -- exiting.") + ## ``` + ## + ## See also: + ## * `log template<#log.t,Level,varargs[string,]>`_ + ## * `warn template<#warn.t,varargs[string,]>`_ + ## * `error template<#error.t,varargs[string,]>`_ log(lvlFatal, args) proc addHandler*(handler: Logger) = - ## Adds ``handler`` to the list of handlers. - if handlers.isNil: handlers = @[] + ## Adds a logger to the list of registered handlers. + ## + ## .. warning:: The list of handlers is a thread-local variable. If the given + ## handler will be used in multiple threads, this proc should be called in + ## each of those threads. + ## + ## See also: + ## * `removeHandler proc`_ + ## * `getHandlers proc<#getHandlers>`_ + runnableExamples: + var logger = newConsoleLogger() + addHandler(logger) + doAssert logger in getHandlers() handlers.add(handler) +proc removeHandler*(handler: Logger) = + ## Removes a logger from the list of registered handlers. + ## + ## Note that for n times a logger is registered, n calls to this proc + ## are required to remove that logger. + for i, hnd in handlers: + if hnd == handler: + handlers.delete(i) + return + proc getHandlers*(): seq[Logger] = ## Returns a list of all the registered handlers. + ## + ## See also: + ## * `addHandler proc<#addHandler,Logger>`_ return handlers proc setLogFilter*(lvl: Level) = ## Sets the global log filter. + ## + ## Messages below the provided level will not be logged regardless of an + ## individual logger's ``levelThreshold``. By default, all messages are + ## logged. + ## + ## .. warning:: The global log filter is a thread-local variable. If logging + ## is being performed in multiple threads, this proc should be called in each + ## thread unless it is intended that different threads should log at different + ## logging levels. + ## + ## See also: + ## * `getLogFilter proc<#getLogFilter>`_ + runnableExamples: + setLogFilter(lvlError) + doAssert getLogFilter() == lvlError level = lvl proc getLogFilter*(): Level = ## Gets the global log filter. + ## + ## See also: + ## * `setLogFilter proc<#setLogFilter,Level>`_ return level # -------------- when not defined(testing) and isMainModule: var L = newConsoleLogger() - var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) - var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) + when not defined(js): + var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) + var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) + addHandler(fL) + addHandler(rL) addHandler(L) - addHandler(fL) - addHandler(rL) for i in 0 .. 25: info("hello", i) + + var nilString: string + info "hello ", nilString diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 134581a06..f9b3d3e4c 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -7,31 +7,57 @@ # distribution, for details about the copyright. # -## This module contains procs for `serialization`:idx: and `deseralization`:idx: +## This module contains procs for `serialization`:idx: and `deserialization`:idx: ## of arbitrary Nim data structures. The serialization format uses `JSON`:idx:. ## -## **Restriction**: For objects their type is **not** serialized. This means +## **Restriction:** For objects, their type is **not** serialized. This means ## essentially that it does not work if the object has some other runtime -## type than its compiletime type: +## type than its compiletime type. ## -## .. code-block:: nim ## -## type -## A = object of RootObj -## B = object of A -## f: int +## Basic usage +## =========== ## -## var -## a: ref A -## b: ref B +runnableExamples: + type + A = object of RootObj + B = object of A + f: int + + let a: ref A = new(B) + assert $$a[] == "{}" # not "{f: 0}" + + # unmarshal + let c = to[B]("""{"f": 2}""") + assert typeof(c) is B + assert c.f == 2 + + # marshal + assert $$c == """{"f": 2}""" + +## **Note:** The `to` and `$$` operations are available at compile-time! ## -## new(b) -## a = b -## echo($$a[]) # produces "{}", not "{f: 0}" ## -## **Note**: The ``to`` and ``$$`` operations are available at compile-time! +## See also +## ======== +## * `streams module <streams.html>`_ +## * `json module <json.html>`_ + +const unsupportedPlatform = + when defined(js): "javascript" + elif defined(nimscript): "nimscript" + else: "" -import streams, typeinfo, json, intsets, tables +when unsupportedPlatform != "": + {.error: "marshal module is not supported in " & unsupportedPlatform & """. +Please use alternative packages for serialization. +It is possible to reimplement this module using generics and type traits. +Please contribute a new implementation.""".} + +import std/[streams, typeinfo, json, intsets, tables, unicode] + +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] proc ptrToInt(x: pointer): int {.inline.} = result = cast[int](x) # don't skip alignment @@ -47,13 +73,11 @@ proc storeAny(s: Stream, a: Any, stored: var IntSet) = else: s.write($int(ch)) of akArray, akSequence: - if a.kind == akSequence and isNil(a): s.write("null") - else: - s.write("[") - for i in 0 .. a.len-1: - if i > 0: s.write(", ") - storeAny(s, a[i], stored) - s.write("]") + s.write("[") + for i in 0 .. a.len-1: + if i > 0: s.write(", ") + storeAny(s, a[i], stored) + s.write("]") of akObject, akTuple: s.write("{") var i = 0 @@ -91,8 +115,15 @@ proc storeAny(s: Stream, a: Any, stored: var IntSet) = of akProc, akPointer, akCString: s.write($a.getPointer.ptrToInt) of akString: var x = getString(a) - if isNil(x): s.write("null") - else: s.write(escapeJson(x)) + if x.validateUtf8() == -1: s.write(escapeJson(x)) + else: + s.write("[") + var i = 0 + for c in x: + if i > 0: s.write(", ") + s.write($ord(c)) + inc(i) + s.write("]") of akInt..akInt64, akUInt..akUInt64: s.write($getBiggestInt(a)) of akFloat..akFloat128: s.write($getBiggestFloat(a)) @@ -135,7 +166,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akSequence: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + invokeNewSeq(a, 0) + else: + setPointer(a, nil) next(p) of jsonArrayStart: next(p) @@ -176,7 +210,8 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = setPointer(a, nil) next(p) of jsonInt: - setPointer(a, t.getOrDefault(p.getInt)) + var raw = t.getOrDefault(p.getInt) + setPointer(a, addr raw) next(p) of jsonArrayStart: next(p) @@ -202,11 +237,26 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akString: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + setString(a, "") + else: + setPointer(a, nil) next(p) of jsonString: setString(a, p.str) next(p) + of jsonArrayStart: + next(p) + var str = "" + while p.kind == jsonInt: + let code = p.getInt() + if code < 0 or code > 255: + raiseParseErr(p, "invalid charcode: " & $code) + str.add(chr(code)) + next(p) + if p.kind == jsonArrayEnd: next(p) + else: raiseParseErr(p, "an array of charcodes expected for string") + setString(a, str) else: raiseParseErr(p, "string expected") of akInt..akInt64, akUInt..akUInt64: if p.kind == jsonInt: @@ -230,103 +280,88 @@ proc loadAny(s: Stream, a: Any, t: var Table[BiggestInt, pointer]) = close(p) proc load*[T](s: Stream, data: var T) = - ## loads `data` from the stream `s`. Raises `EIO` in case of an error. + ## Loads `data` from the stream `s`. Raises `IOError` in case of an error. + runnableExamples: + import std/streams + + var s = newStringStream("[1, 3, 5]") + var a: array[3, int] + load(s, a) + assert a == [1, 3, 5] + var tab = initTable[BiggestInt, pointer]() loadAny(s, toAny(data), tab) -proc store*[T](s: Stream, data: T) = - ## stores `data` into the stream `s`. Raises `EIO` in case of an error. - var stored = initIntSet() - var d: T - shallowCopy(d, data) - storeAny(s, toAny(d), stored) +proc store*[T](s: Stream, data: sink T) = + ## Stores `data` into the stream `s`. Raises `IOError` in case of an error. + runnableExamples: + import std/streams + + var s = newStringStream("") + var a = [1, 3, 5] + store(s, a) + s.setPosition(0) + assert s.readAll() == "[1, 3, 5]" -proc `$$`*[T](x: T): string = - ## returns a string representation of `x`. var stored = initIntSet() var d: T - shallowCopy(d, x) - var s = newStringStream() + when defined(gcArc) or defined(gcOrc)or defined(gcAtomicArc): + d = data + else: + shallowCopy(d, data) storeAny(s, toAny(d), stored) - result = s.data - -proc to*[T](data: string): T = - ## reads data and transforms it to a ``T``. - var tab = initTable[BiggestInt, pointer]() - loadAny(newStringStream(data), toAny(result), tab) - -when not defined(testing) and isMainModule: - template testit(x: expr) = echo($$to[type(x)]($$x)) - - var x: array[0..4, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"]] - testit(x) - var test2: tuple[name: string, s: uint] = ("tuple test", 56u) - testit(test2) - type - TE = enum - blah, blah2 - - TestObj = object - test, asd: int - case test2: TE - of blah: - help: string - else: - nil - - PNode = ref Node - Node = object - next, prev: PNode - data: string - {.deprecated: [TNode: Node].} - - proc buildList(): PNode = - new(result) - new(result.next) - new(result.prev) - result.data = "middle" - result.next.data = "next" - result.prev.data = "prev" - result.next.next = result.prev - result.next.prev = result - result.prev.next = result - result.prev.prev = result.next - - var test3: TestObj - test3.test = 42 - test3.test2 = blah - testit(test3) - - var test4: ref tuple[a, b: string] - new(test4) - test4.a = "ref string test: A" - test4.b = "ref string test: B" - testit(test4) +proc loadVM[T](typ: typedesc[T], x: T): string = + discard "the implementation is in the compiler/vmops" - var test5 = @[(0,1),(2,3),(4,5)] - testit(test5) +proc `$$`*[T](x: sink T): string = + ## Returns a string representation of `x` (serialization, marshalling). + ## + ## **Note:** to serialize `x` to JSON use `%x` from the `json` module + ## or `jsonutils.toJson(x)`. + runnableExamples: + type + Foo = object + id: int + bar: string + let x = Foo(id: 1, bar: "baz") + ## serialize: + let y = $$x + assert y == """{"id": 1, "bar": "baz"}""" - var test6: set[char] = {'A'..'Z', '_'} - testit(test6) - - var test7 = buildList() - echo($$test7) - testit(test7) - - type - A {.inheritable.} = object - B = object of A - f: int + when nimvm: + result = loadVM(T, x) + else: + var stored = initIntSet() + var d: T + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + d = x + else: + shallowCopy(d, x) + var s = newStringStream() + storeAny(s, toAny(d), stored) + result = s.data - var - a: ref A - b: ref B - new(b) - a = b - echo($$a[]) # produces "{}", not "{f: 0}" +proc toVM[T](typ: typedesc[T], data: string): T = + discard "the implementation is in the compiler/vmops" +proc to*[T](data: string): T = + ## Reads data and transforms it to a type `T` (deserialization, unmarshalling). + runnableExamples: + type + Foo = object + id: int + bar: string + let y = """{"id": 1, "bar": "baz"}""" + assert typeof(y) is string + ## deserialize to type 'Foo': + let z = y.to[:Foo] + assert typeof(z) is Foo + assert z.id == 1 + assert z.bar == "baz" + when nimvm: + result = toVM(T, data) + else: + var tab = initTable[BiggestInt, pointer]() + loadAny(newStringStream(data), toAny(result), tab) diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim deleted file mode 100644 index 5c28f65a0..000000000 --- a/lib/pure/matchers.nim +++ /dev/null @@ -1,64 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module contains various string matchers for email addresses, etc. -{.deadCodeElim: on.} - -{.push debugger:off .} # the user does not want to trace a part - # of the standard library! - -include "system/inclrtl" - -import parseutils, strutils - -proc validEmailAddress*(s: string): bool {.noSideEffect, - rtl, extern: "nsuValidEmailAddress".} = - ## returns true if `s` seems to be a valid e-mail address. - ## The checking also uses a domain list. - const - chars = Letters + Digits + {'!','#','$','%','&', - '\'','*','+','/','=','?','^','_','`','{','}','|','~','-','.'} - var i = 0 - if s[i] notin chars or s[i] == '.': return false - while s[i] in chars: - if s[i] == '.' and s[i+1] == '.': return false - inc(i) - if s[i] != '@': return false - var j = len(s)-1 - if s[j] notin Letters: return false - while j >= i and s[j] in Letters: dec(j) - inc(i) # skip '@' - while s[i] in {'0'..'9', 'a'..'z', '-', '.'}: inc(i) - if s[i] != '\0': return false - - var x = substr(s, j+1) - if len(x) == 2 and x[0] in Letters and x[1] in Letters: return true - case toLower(x) - of "com", "org", "net", "gov", "mil", "biz", "info", "mobi", "name", - "aero", "jobs", "museum": return true - else: return false - -proc parseInt*(s: string, value: var int, validRange: Slice[int]) {. - noSideEffect, rtl, extern: "nmatchParseInt".} = - ## parses `s` into an integer in the range `validRange`. If successful, - ## `value` is modified to contain the result. Otherwise no exception is - ## raised and `value` is not touched; this way a reasonable default value - ## won't be overwritten. - var x = value - try: - discard parseutils.parseInt(s, x, 0) - except OverflowError: - discard - if x in validRange: value = x - -when isMainModule: - doAssert "wuseldusel@codehome.com".validEmailAddress - -{.pop.} - diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 391a880ae..ed7d2382f 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -7,93 +7,348 @@ # distribution, for details about the copyright. # -## Constructive mathematics is naturally typed. -- Simon Thompson +## *Constructive mathematics is naturally typed.* -- Simon Thompson ## ## Basic math routines for Nim. -## This module is available for the `JavaScript target -## <backends.html#the-javascript-target>`_. ## ## Note that the trigonometric functions naturally operate on radians. -## The helper functions `degToRad` and `radToDeg` provide conversion -## between radians and degrees. +## The helper functions `degToRad <#degToRad,T>`_ and `radToDeg <#radToDeg,T>`_ +## provide conversion between radians and degrees. + +runnableExamples: + from std/fenv import epsilon + from std/random import rand + + proc generateGaussianNoise(mu: float = 0.0, sigma: float = 1.0): (float, float) = + # Generates values from a normal distribution. + # Translated from https://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform#Implementation. + var u1: float + var u2: float + while true: + u1 = rand(1.0) + u2 = rand(1.0) + if u1 > epsilon(float): break + let mag = sigma * sqrt(-2 * ln(u1)) + let z0 = mag * cos(2 * PI * u2) + mu + let z1 = mag * sin(2 * PI * u2) + mu + (z0, z1) + + echo generateGaussianNoise() -include "system/inclrtl" -{.push debugger:off .} # the user does not want to trace a part +## This module is available for the `JavaScript target +## <backends.html#backends-the-javascript-target>`_. +## +## See also +## ======== +## * `complex module <complex.html>`_ for complex numbers and their +## mathematical operations +## * `rationals module <rationals.html>`_ for rational numbers and their +## mathematical operations +## * `fenv module <fenv.html>`_ for handling of floating-point rounding +## and exceptions (overflow, zero-divide, etc.) +## * `random module <random.html>`_ for a fast and tiny random number generator +## * `stats module <stats.html>`_ for statistical analysis +## * `strformat module <strformat.html>`_ for formatting floats for printing +## * `system module <system.html>`_ for some very basic and trivial math operators +## (`shr`, `shl`, `xor`, `clamp`, etc.) + + +import std/private/since +{.push debugger: off.} # the user does not want to trace a part # of the standard library! -proc binom*(n, k: int): int {.noSideEffect.} = - ## Computes the binomial coefficient +import std/[bitops, fenv] +import system/countbits_impl + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when not defined(js) and not defined(nimscript): # C + proc c_isnan(x: float): bool {.importc: "isnan", header: "<math.h>".} + # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. + + proc c_copysign(x, y: cfloat): cfloat {.importc: "copysignf", header: "<math.h>".} + proc c_copysign(x, y: cdouble): cdouble {.importc: "copysign", header: "<math.h>".} + + proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "<math.h>".} + + # don't export `c_frexp` in the future and remove `c_frexp2`. + func c_frexp2(x: cfloat, exponent: var cint): cfloat {. + importc: "frexpf", header: "<math.h>".} + func c_frexp2(x: cdouble, exponent: var cint): cdouble {. + importc: "frexp", header: "<math.h>".} + + type + div_t {.importc, header: "<stdlib.h>".} = object + quot: cint + rem: cint + ldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clong + rem: clong + lldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clonglong + rem: clonglong + + when cint isnot clong: + func divmod_c(x, y: cint): div_t {.importc: "div", header: "<stdlib.h>".} + when clong isnot clonglong: + func divmod_c(x, y: clonglong): lldiv_t {.importc: "lldiv", header: "<stdlib.h>".} + func divmod_c(x, y: clong): ldiv_t {.importc: "ldiv", header: "<stdlib.h>".} + func divmod*[T: SomeInteger](x, y: T): (T, T) {.inline.} = + ## Specialized instructions for computing both division and modulus. + ## Return structure is: (quotient, remainder) + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + when T is cint | clong | clonglong: + when compileOption("overflowChecks"): + if y == 0: + raise new(DivByZeroDefect) + elif (x == T.low and y == -1.T): + raise new(OverflowDefect) + let res = divmod_c(x, y) + result[0] = res.quot + result[1] = res.rem + else: + result[0] = x div y + result[1] = x mod y + +func binom*(n, k: int): int = + ## Computes the [binomial coefficient](https://en.wikipedia.org/wiki/Binomial_coefficient). + runnableExamples: + doAssert binom(6, 2) == 15 + doAssert binom(-6, 2) == 1 + doAssert binom(6, 0) == 1 + if k <= 0: return 1 - if 2*k > n: return binom(n, n-k) + if 2 * k > n: return binom(n, n - k) result = n for i in countup(2, k): result = (result * (n + 1 - i)) div i -proc fac*(n: int): int {.noSideEffect.} = - ## Computes the faculty/factorial function. - result = 1 - for i in countup(2, n): - result = result * i +func createFactTable[N: static[int]]: array[N, int] = + result[0] = 1 + for i in 1 ..< N: + result[i] = result[i - 1] * i + +func fac*(n: int): int = + ## Computes the [factorial](https://en.wikipedia.org/wiki/Factorial) of + ## a non-negative integer `n`. + ## + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ + runnableExamples: + doAssert fac(0) == 1 + doAssert fac(4) == 24 + doAssert fac(10) == 3628800 + + const factTable = + when sizeof(int) == 2: + createFactTable[5]() + elif sizeof(int) == 4: + createFactTable[13]() + else: + createFactTable[21]() + assert(n >= 0, $n & " must not be negative.") + assert(n < factTable.len, $n & " is too large to look up in the table") + factTable[n] -{.push checks:off, line_dir:off, stack_trace:off.} +{.push checks: off, line_dir: off, stack_trace: off.} -when defined(Posix) and not defined(haiku): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} -when not defined(js) and not defined(nimscript): - import times const - PI* = 3.1415926535897932384626433 ## the circle constant PI (Ludolph's number) - E* = 2.71828182845904523536028747 ## Euler's number - - MaxFloat64Precision* = 16 ## maximum number of meaningful digits - ## after the decimal point for Nim's - ## ``float64`` type. - MaxFloat32Precision* = 8 ## maximum number of meaningful digits - ## after the decimal point for Nim's - ## ``float32`` type. - MaxFloatPrecision* = MaxFloat64Precision ## maximum number of - ## meaningful digits - ## after the decimal point - ## for Nim's ``float`` type. - RadPerDeg = PI / 180.0 ## number of radians per degree + PI* = 3.1415926535897932384626433 ## The circle constant PI (Ludolph's number). + TAU* = 2.0 * PI ## The circle constant TAU (= 2 * PI). + E* = 2.71828182845904523536028747 ## Euler's number. + + MaxFloat64Precision* = 16 ## Maximum number of meaningful digits + ## after the decimal point for Nim's + ## `float64` type. + MaxFloat32Precision* = 8 ## Maximum number of meaningful digits + ## after the decimal point for Nim's + ## `float32` type. + MaxFloatPrecision* = MaxFloat64Precision ## Maximum number of + ## meaningful digits + ## after the decimal point + ## for Nim's `float` type. + MinFloatNormal* = 2.225073858507201e-308 ## Smallest normal number for Nim's + ## `float` type (= 2^-1022). + RadPerDeg = PI / 180.0 ## Number of radians per degree. type - FloatClass* = enum ## describes the class a floating point value belongs to. - ## This is the type that is returned by `classify`. - fcNormal, ## value is an ordinary nonzero floating point value - fcSubnormal, ## value is a subnormal (a very small) floating point value - fcZero, ## value is zero - fcNegZero, ## value is the negative zero - fcNan, ## value is Not-A-Number (NAN) - fcInf, ## value is positive infinity - fcNegInf ## value is negative infinity - -proc classify*(x: float): FloatClass = - ## Classifies a floating point value. Returns `x`'s class as specified by - ## `FloatClass`. + FloatClass* = enum ## Describes the class a floating point value belongs to. + ## This is the type that is returned by the + ## `classify func <#classify,float>`_. + fcNormal, ## value is an ordinary nonzero floating point value + fcSubnormal, ## value is a subnormal (a very small) floating point value + fcZero, ## value is zero + fcNegZero, ## value is the negative zero + fcNan, ## value is Not a Number (NaN) + fcInf, ## value is positive infinity + fcNegInf ## value is negative infinity + +func isNaN*(x: SomeFloat): bool {.inline, since: (1,5,1).} = + ## Returns whether `x` is a `NaN`, more efficiently than via `classify(x) == fcNan`. + ## Works even with `--passc:-ffast-math`. + runnableExamples: + doAssert NaN.isNaN + doAssert not Inf.isNaN + doAssert not isNaN(3.1415926) + + template fn: untyped = result = x != x + when nimvm: fn() + else: + when defined(js) or defined(nimscript): fn() + else: result = c_isnan(x) + +when defined(js): + import std/private/jsutils + + proc toBitsImpl(x: float): array[2, uint32] = + let buffer = newArrayBuffer(8) + let a = newFloat64Array(buffer) + let b = newUint32Array(buffer) + a[0] = x + {.emit: "`result` = `b`;".} + # result = cast[array[2, uint32]](b) + + proc jsSetSign(x: float, sgn: bool): float = + let buffer = newArrayBuffer(8) + let a = newFloat64Array(buffer) + let b = newUint32Array(buffer) + a[0] = x + {.emit: """ + function updateBit(num, bitPos, bitVal) { + return (num & ~(1 << bitPos)) | (bitVal << bitPos); + } + `b`[1] = updateBit(`b`[1], 31, `sgn`); + `result` = `a`[0]; + """.} + +proc signbit*(x: SomeFloat): bool {.inline, since: (1, 5, 1).} = + ## Returns true if `x` is negative, false otherwise. + runnableExamples: + doAssert not signbit(0.0) + doAssert signbit(-0.0) + doAssert signbit(-0.1) + doAssert not signbit(0.1) + + when defined(js): + let uintBuffer = toBitsImpl(x) + result = (uintBuffer[1] shr 31) != 0 + else: + result = c_signbit(x) != 0 + +func copySign*[T: SomeFloat](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Returns a value with the magnitude of `x` and the sign of `y`; + ## this works even if x or y are NaN, infinity or zero, all of which can carry a sign. + runnableExamples: + doAssert copySign(10.0, 1.0) == 10.0 + doAssert copySign(10.0, -1.0) == -10.0 + doAssert copySign(-Inf, -0.0) == -Inf + doAssert copySign(NaN, 1.0).isNaN + doAssert copySign(1.0, copySign(NaN, -1.0)) == -1.0 + + # TODO: use signbit for examples + when defined(js): + let uintBuffer = toBitsImpl(y) + let sgn = (uintBuffer[1] shr 31) != 0 + result = jsSetSign(x, sgn) + else: + when nimvm: # not exact but we have a vmops for recent enough nim + if y > 0.0 or (y == 0.0 and 1.0 / y > 0.0): + result = abs(x) + elif y <= 0.0: + result = -abs(x) + else: # must be NaN + result = abs(x) + else: result = c_copysign(x, y) + +func classify*(x: float): FloatClass = + ## Classifies a floating point value. + ## + ## Returns `x`'s class as specified by the `FloatClass enum<#FloatClass>`_. + runnableExamples: + doAssert classify(0.3) == fcNormal + doAssert classify(0.0) == fcZero + doAssert classify(0.3 / 0.0) == fcInf + doAssert classify(-0.3 / 0.0) == fcNegInf + doAssert classify(5.0e-324) == fcSubnormal # JavaScript and most C compilers have no classify: + if isNan(x): return fcNan if x == 0.0: - if 1.0/x == Inf: + if 1.0 / x == Inf: return fcZero else: return fcNegZero - if x*0.5 == x: + if x * 0.5 == x: if x > 0.0: return fcInf else: return fcNegInf - if x != x: return fcNan + if abs(x) < MinFloatNormal: + return fcSubnormal return fcNormal - # XXX: fcSubnormal is not detected! -proc isPowerOfTwo*(x: int): bool {.noSideEffect.} = - ## Returns true, if `x` is a power of two, false otherwise. +func almostEqual*[T: SomeFloat](x, y: T; unitsInLastPlace: Natural = 4): bool {. + since: (1, 5), inline.} = + ## Checks if two float values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + ## + # taken from: https://en.cppreference.com/w/cpp/types/numeric_limits/epsilon + runnableExamples: + doAssert almostEqual(PI, 3.14159265358979) + doAssert almostEqual(Inf, Inf) + doAssert not almostEqual(NaN, NaN) + + if x == y: + # short circuit exact equality -- needed to catch two infinities of + # the same sign. And perhaps speeds things up a bit sometimes. + return true + let diff = abs(x - y) + result = diff <= epsilon(T) * abs(x + y) * T(unitsInLastPlace) or + diff < minimumPositiveValue(T) + +func isPowerOfTwo*(x: int): bool = + ## Returns `true`, if `x` is a power of two, `false` otherwise. + ## ## Zero and negative numbers are not a power of two. + ## + ## **See also:** + ## * `nextPowerOfTwo func <#nextPowerOfTwo,int>`_ + runnableExamples: + doAssert isPowerOfTwo(16) + doAssert not isPowerOfTwo(5) + doAssert not isPowerOfTwo(0) + doAssert not isPowerOfTwo(-16) + return (x > 0) and ((x and (x - 1)) == 0) -proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = +func nextPowerOfTwo*(x: int): int = ## Returns `x` rounded up to the nearest power of two. + ## ## Zero and negative numbers get rounded up to 1. + ## + ## **See also:** + ## * `isPowerOfTwo func <#isPowerOfTwo,int>`_ + runnableExamples: + doAssert nextPowerOfTwo(16) == 16 + doAssert nextPowerOfTwo(5) == 8 + doAssert nextPowerOfTwo(0) == 1 + doAssert nextPowerOfTwo(-16) == 1 + result = x - 1 when defined(cpu64): result = result or (result shr 32) @@ -104,378 +359,956 @@ proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = result = result or (result shr 4) result = result or (result shr 2) result = result or (result shr 1) - result += 1 + ord(x<=0) + result += 1 + ord(x <= 0) + -proc countBits32*(n: int32): int {.noSideEffect.} = - ## Counts the set bits in `n`. - var v = n - v = v -% ((v shr 1'i32) and 0x55555555'i32) - v = (v and 0x33333333'i32) +% ((v shr 2'i32) and 0x33333333'i32) - result = ((v +% (v shr 4'i32) and 0xF0F0F0F'i32) *% 0x1010101'i32) shr 24'i32 -proc sum*[T](x: openArray[T]): T {.noSideEffect.} = - ## Computes the sum of the elements in `x`. - ## If `x` is empty, 0 is returned. - for i in items(x): result = result + i -template toFloat(f: float): float = f - -proc mean*[T](x: openArray[T]): float {.noSideEffect.} = - ## Computes the mean of the elements in `x`, which are first converted to floats. - ## If `x` is empty, NaN is returned. - ## ``toFloat(x: T): float`` must be defined. - for i in items(x): result = result + toFloat(i) - result = result / toFloat(len(x)) - -proc variance*[T](x: openArray[T]): float {.noSideEffect.} = - ## Computes the variance of the elements in `x`. - ## If `x` is empty, NaN is returned. - ## ``toFloat(x: T): float`` must be defined. - result = 0.0 - var m = mean(x) - for i in items(x): - var diff = toFloat(i) - m - result = result + diff*diff - result = result / toFloat(len(x)) - -proc random*(max: int): int {.benign.} - ## Returns a random number in the range 0..max-1. The sequence of - ## random number is always the same, unless `randomize` is called - ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. - -proc random*(max: float): float {.benign.} - ## Returns a random number in the range 0..<max. The sequence of - ## random number is always the same, unless `randomize` is called - ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. This has a 16-bit resolution on windows - ## and a 48-bit resolution on other platforms. - -when not defined(nimscript): - proc randomize*() {.benign.} - ## Initializes the random number generator with a "random" - ## number, i.e. a tickcount. Note: Does nothing for the JavaScript target, - ## as JavaScript does not support this. Nor does it work for NimScript. - -proc randomize*(seed: int) {.benign.} - ## Initializes the random number generator with a specific seed. - ## Note: Does nothing for the JavaScript target, - ## as JavaScript does not support this. - -{.push noSideEffect.} -when not defined(JS): - proc sqrt*(x: float): float {.importc: "sqrt", header: "<math.h>".} +when not defined(js): # C + func sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} + func sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} = ## Computes the square root of `x`. - proc cbrt*(x: float): float {.importc: "cbrt", header: "<math.h>".} - ## Computes the cubic root of `x` - - proc ln*(x: float): float {.importc: "log", header: "<math.h>".} - ## Computes the natural log of `x` - proc log10*(x: float): float {.importc: "log10", header: "<math.h>".} - ## Computes the common logarithm (base 10) of `x` - proc log2*(x: float): float = return ln(x) / ln(2.0) - ## Computes the binary logarithm (base 2) of `x` - proc exp*(x: float): float {.importc: "exp", header: "<math.h>".} - ## Computes the exponential function of `x` (pow(E, x)) - - proc frexp*(x: float, exponent: var int): float {. - importc: "frexp", header: "<math.h>".} - ## Split a number into mantissa and exponent. - ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 - ## and less than 1) and the integer value n such that `x` (the original - ## float value) equals m * 2**n. frexp stores n in `exponent` and returns - ## m. - - proc round*(x: float): int {.importc: "lrint", header: "<math.h>".} - ## Converts a float to an int by rounding. - - proc arccos*(x: float): float {.importc: "acos", header: "<math.h>".} - ## Computes the arc cosine of `x` - proc arcsin*(x: float): float {.importc: "asin", header: "<math.h>".} - ## Computes the arc sine of `x` - proc arctan*(x: float): float {.importc: "atan", header: "<math.h>".} - ## Calculate the arc tangent of `y` / `x` - proc arctan2*(y, x: float): float {.importc: "atan2", header: "<math.h>".} - ## Calculate the arc tangent of `y` / `x`. - ## `atan2` returns the arc tangent of `y` / `x`; it produces correct - ## results even when the resulting angle is near pi/2 or -pi/2 - ## (`x` near 0). - - proc cos*(x: float): float {.importc: "cos", header: "<math.h>".} - ## Computes the cosine of `x` - proc cosh*(x: float): float {.importc: "cosh", header: "<math.h>".} - ## Computes the hyperbolic cosine of `x` - proc hypot*(x, y: float): float {.importc: "hypot", header: "<math.h>".} - ## Computes the hypotenuse of a right-angle triangle with `x` and - ## `y` as its base and height. Equivalent to ``sqrt(x*x + y*y)``. - - proc sinh*(x: float): float {.importc: "sinh", header: "<math.h>".} - ## Computes the hyperbolic sine of `x` - proc sin*(x: float): float {.importc: "sin", header: "<math.h>".} - ## Computes the sine of `x` - proc tan*(x: float): float {.importc: "tan", header: "<math.h>".} - ## Computes the tangent of `x` - proc tanh*(x: float): float {.importc: "tanh", header: "<math.h>".} - ## Computes the hyperbolic tangent of `x` - proc pow*(x, y: float): float {.importc: "pow", header: "<math.h>".} - ## Computes `x` to power of `y`. - - proc erf*(x: float): float {.importc: "erf", header: "<math.h>".} - ## The error function - proc erfc*(x: float): float {.importc: "erfc", header: "<math.h>".} - ## The complementary error function - - proc lgamma*(x: float): float {.importc: "lgamma", header: "<math.h>".} - ## Natural log of the gamma function - proc tgamma*(x: float): float {.importc: "tgamma", header: "<math.h>".} - ## The gamma function - - # C procs: - when defined(vcc) and false: - # The "secure" random, available from Windows XP - # https://msdn.microsoft.com/en-us/library/sxtz2fa8.aspx - # Present in some variants of MinGW but not enough to justify - # `when defined(windows)` yet - proc rand_s(val: var cuint) {.importc: "rand_s", header: "<stdlib.h>".} - # To behave like the normal version - proc rand(): cuint = rand_s(result) - else: - proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>".} - proc rand(): cint {.importc: "rand", header: "<stdlib.h>".} - - when not defined(windows): - proc srand48(seed: clong) {.importc: "srand48", header: "<stdlib.h>".} - proc drand48(): float {.importc: "drand48", header: "<stdlib.h>".} - proc random(max: float): float = - result = drand48() * max - else: - when defined(vcc): # Windows with Visual C - proc random(max: float): float = - # we are hardcoding this because - # importc-ing macros is extremely problematic - # and because the value is publicly documented - # on MSDN and very unlikely to change - # See https://msdn.microsoft.com/en-us/library/296az74e.aspx - const rand_max = 4294967295 # UINT_MAX - result = (float(rand()) / float(rand_max)) * max - proc randomize() = discard - proc randomize(seed: int) = discard - else: # Windows with another compiler - proc random(max: float): float = - # we are hardcoding this because - # importc-ing macros is extremely problematic - # and because the value is publicly documented - # on MSDN and very unlikely to change - const rand_max = 32767 - result = (float(rand()) / float(rand_max)) * max - - when not defined(vcc): # the above code for vcc uses `discard` instead - # this is either not Windows or is Windows without vcc - when not defined(nimscript): - proc randomize() = - randomize(cast[int](epochTime())) - proc randomize(seed: int) = - srand(cint(seed)) # rand_s doesn't use srand - when declared(srand48): srand48(seed) - - proc random(max: int): int = - result = int(rand()) mod max - - proc trunc*(x: float): float {.importc: "trunc", header: "<math.h>".} - ## Truncates `x` to the decimal point ## - ## .. code-block:: nim - ## echo trunc(PI) # 3.0 - proc floor*(x: float): float {.importc: "floor", header: "<math.h>".} - ## Computes the floor function (i.e., the largest integer not greater than `x`) + ## **See also:** + ## * `cbrt func <#cbrt,float64>`_ for the cube root + runnableExamples: + doAssert almostEqual(sqrt(4.0), 2.0) + doAssert almostEqual(sqrt(1.44), 1.2) + func cbrt*(x: float32): float32 {.importc: "cbrtf", header: "<math.h>".} + func cbrt*(x: float64): float64 {.importc: "cbrt", header: "<math.h>".} = + ## Computes the cube root of `x`. ## - ## .. code-block:: nim - ## echo floor(-3.5) ## -4.0 - proc ceil*(x: float): float {.importc: "ceil", header: "<math.h>".} - ## Computes the ceiling function (i.e., the smallest integer not less than `x`) + ## **See also:** + ## * `sqrt func <#sqrt,float64>`_ for the square root + runnableExamples: + doAssert almostEqual(cbrt(8.0), 2.0) + doAssert almostEqual(cbrt(2.197), 1.3) + doAssert almostEqual(cbrt(-27.0), -3.0) + func ln*(x: float32): float32 {.importc: "logf", header: "<math.h>".} + func ln*(x: float64): float64 {.importc: "log", header: "<math.h>".} = + ## Computes the [natural logarithm](https://en.wikipedia.org/wiki/Natural_logarithm) + ## of `x`. ## - ## .. code-block:: nim - ## echo ceil(-2.1) ## -2.0 - - proc fmod*(x, y: float): float {.importc: "fmod", header: "<math.h>".} - ## Computes the remainder of `x` divided by `y` + ## **See also:** + ## * `log func <#log,T,T>`_ + ## * `log10 func <#log10,float64>`_ + ## * `log2 func <#log2,float64>`_ + ## * `exp func <#exp,float64>`_ + runnableExamples: + doAssert almostEqual(ln(exp(4.0)), 4.0) + doAssert almostEqual(ln(1.0), 0.0) + doAssert almostEqual(ln(0.0), -Inf) + doAssert ln(-7.0).isNaN +else: # JS + func sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} + func sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} + + func cbrt*(x: float32): float32 {.importc: "Math.cbrt", nodecl.} + func cbrt*(x: float64): float64 {.importc: "Math.cbrt", nodecl.} + + func ln*(x: float32): float32 {.importc: "Math.log", nodecl.} + func ln*(x: float64): float64 {.importc: "Math.log", nodecl.} + +func log*[T: SomeFloat](x, base: T): T = + ## Computes the logarithm of `x` to base `base`. + ## + ## **See also:** + ## * `ln func <#ln,float64>`_ + ## * `log10 func <#log10,float64>`_ + ## * `log2 func <#log2,float64>`_ + runnableExamples: + doAssert almostEqual(log(9.0, 3.0), 2.0) + doAssert almostEqual(log(0.0, 2.0), -Inf) + doAssert log(-7.0, 4.0).isNaN + doAssert log(8.0, -2.0).isNaN + + ln(x) / ln(base) + +when not defined(js): # C + func log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} + func log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} = + ## Computes the common logarithm (base 10) of `x`. + ## + ## **See also:** + ## * `ln func <#ln,float64>`_ + ## * `log func <#log,T,T>`_ + ## * `log2 func <#log2,float64>`_ + runnableExamples: + doAssert almostEqual(log10(100.0) , 2.0) + doAssert almostEqual(log10(0.0), -Inf) + doAssert log10(-100.0).isNaN + func exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} + func exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} = + ## Computes the exponential function of `x` (`e^x`). + ## + ## **See also:** + ## * `ln func <#ln,float64>`_ + runnableExamples: + doAssert almostEqual(exp(1.0), E) + doAssert almostEqual(ln(exp(4.0)), 4.0) + doAssert almostEqual(exp(0.0), 1.0) + func sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} + func sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} = + ## Computes the sine of `x`. + ## + ## **See also:** + ## * `arcsin func <#arcsin,float64>`_ + runnableExamples: + doAssert almostEqual(sin(PI / 6), 0.5) + doAssert almostEqual(sin(degToRad(90.0)), 1.0) + func cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} + func cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} = + ## Computes the cosine of `x`. + ## + ## **See also:** + ## * `arccos func <#arccos,float64>`_ + runnableExamples: + doAssert almostEqual(cos(2 * PI), 1.0) + doAssert almostEqual(cos(degToRad(60.0)), 0.5) + func tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} + func tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} = + ## Computes the tangent of `x`. + ## + ## **See also:** + ## * `arctan func <#arctan,float64>`_ + runnableExamples: + doAssert almostEqual(tan(degToRad(45.0)), 1.0) + doAssert almostEqual(tan(PI / 4), 1.0) + func sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} + func sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} = + ## Computes the [hyperbolic sine](https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions) of `x`. + ## + ## **See also:** + ## * `arcsinh func <#arcsinh,float64>`_ + runnableExamples: + doAssert almostEqual(sinh(0.0), 0.0) + doAssert almostEqual(sinh(1.0), 1.175201193643801) + func cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} + func cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} = + ## Computes the [hyperbolic cosine](https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions) of `x`. + ## + ## **See also:** + ## * `arccosh func <#arccosh,float64>`_ + runnableExamples: + doAssert almostEqual(cosh(0.0), 1.0) + doAssert almostEqual(cosh(1.0), 1.543080634815244) + func tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} + func tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} = + ## Computes the [hyperbolic tangent](https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions) of `x`. + ## + ## **See also:** + ## * `arctanh func <#arctanh,float64>`_ + runnableExamples: + doAssert almostEqual(tanh(0.0), 0.0) + doAssert almostEqual(tanh(1.0), 0.7615941559557649) + func arcsin*(x: float32): float32 {.importc: "asinf", header: "<math.h>".} + func arcsin*(x: float64): float64 {.importc: "asin", header: "<math.h>".} = + ## Computes the arc sine of `x`. + ## + ## **See also:** + ## * `sin func <#sin,float64>`_ + runnableExamples: + doAssert almostEqual(radToDeg(arcsin(0.0)), 0.0) + doAssert almostEqual(radToDeg(arcsin(1.0)), 90.0) + func arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} + func arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} = + ## Computes the arc cosine of `x`. + ## + ## **See also:** + ## * `cos func <#cos,float64>`_ + runnableExamples: + doAssert almostEqual(radToDeg(arccos(0.0)), 90.0) + doAssert almostEqual(radToDeg(arccos(1.0)), 0.0) + func arctan*(x: float32): float32 {.importc: "atanf", header: "<math.h>".} + func arctan*(x: float64): float64 {.importc: "atan", header: "<math.h>".} = + ## Calculate the arc tangent of `x`. + ## + ## **See also:** + ## * `arctan2 func <#arctan2,float64,float64>`_ + ## * `tan func <#tan,float64>`_ + runnableExamples: + doAssert almostEqual(arctan(1.0), 0.7853981633974483) + doAssert almostEqual(radToDeg(arctan(1.0)), 45.0) + func arctan2*(y, x: float32): float32 {.importc: "atan2f", header: "<math.h>".} + func arctan2*(y, x: float64): float64 {.importc: "atan2", header: "<math.h>".} = + ## Calculate the arc tangent of `y/x`. ## - ## .. code-block:: nim - ## echo fmod(-2.5, 0.3) ## -0.1 - -else: - proc mathrandom(): float {.importc: "Math.random", nodecl.} - proc floor*(x: float): float {.importc: "Math.floor", nodecl.} - proc ceil*(x: float): float {.importc: "Math.ceil", nodecl.} - proc random(max: int): int = - result = int(floor(mathrandom() * float(max))) - proc random(max: float): float = - result = float(mathrandom() * float(max)) - proc randomize() = discard - proc randomize(seed: int) = discard - - proc sqrt*(x: float): float {.importc: "Math.sqrt", nodecl.} - proc ln*(x: float): float {.importc: "Math.log", nodecl.} - proc log10*(x: float): float = return ln(x) / ln(10.0) - proc log2*(x: float): float = return ln(x) / ln(2.0) - - proc exp*(x: float): float {.importc: "Math.exp", nodecl.} - proc round*(x: float): int {.importc: "Math.round", nodecl.} - proc pow*(x, y: float): float {.importc: "Math.pow", nodecl.} - - proc frexp*(x: float, exponent: var int): float = + ## It produces correct results even when the resulting angle is near + ## `PI/2` or `-PI/2` (`x` near 0). + ## + ## **See also:** + ## * `arctan func <#arctan,float64>`_ + runnableExamples: + doAssert almostEqual(arctan2(1.0, 0.0), PI / 2.0) + doAssert almostEqual(radToDeg(arctan2(1.0, 0.0)), 90.0) + func arcsinh*(x: float32): float32 {.importc: "asinhf", header: "<math.h>".} + func arcsinh*(x: float64): float64 {.importc: "asinh", header: "<math.h>".} + ## Computes the inverse hyperbolic sine of `x`. + ## + ## **See also:** + ## * `sinh func <#sinh,float64>`_ + func arccosh*(x: float32): float32 {.importc: "acoshf", header: "<math.h>".} + func arccosh*(x: float64): float64 {.importc: "acosh", header: "<math.h>".} + ## Computes the inverse hyperbolic cosine of `x`. + ## + ## **See also:** + ## * `cosh func <#cosh,float64>`_ + func arctanh*(x: float32): float32 {.importc: "atanhf", header: "<math.h>".} + func arctanh*(x: float64): float64 {.importc: "atanh", header: "<math.h>".} + ## Computes the inverse hyperbolic tangent of `x`. + ## + ## **See also:** + ## * `tanh func <#tanh,float64>`_ + +else: # JS + func log10*(x: float32): float32 {.importc: "Math.log10", nodecl.} + func log10*(x: float64): float64 {.importc: "Math.log10", nodecl.} + func log2*(x: float32): float32 {.importc: "Math.log2", nodecl.} + func log2*(x: float64): float64 {.importc: "Math.log2", nodecl.} + func exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} + func exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} + + func sin*[T: float32|float64](x: T): T {.importc: "Math.sin", nodecl.} + func cos*[T: float32|float64](x: T): T {.importc: "Math.cos", nodecl.} + func tan*[T: float32|float64](x: T): T {.importc: "Math.tan", nodecl.} + + func sinh*[T: float32|float64](x: T): T {.importc: "Math.sinh", nodecl.} + func cosh*[T: float32|float64](x: T): T {.importc: "Math.cosh", nodecl.} + func tanh*[T: float32|float64](x: T): T {.importc: "Math.tanh", nodecl.} + + func arcsin*[T: float32|float64](x: T): T {.importc: "Math.asin", nodecl.} + # keep this as generic or update test in `tvmops.nim` to make sure we + # keep testing that generic importc procs work + func arccos*[T: float32|float64](x: T): T {.importc: "Math.acos", nodecl.} + func arctan*[T: float32|float64](x: T): T {.importc: "Math.atan", nodecl.} + func arctan2*[T: float32|float64](y, x: T): T {.importc: "Math.atan2", nodecl.} + + func arcsinh*[T: float32|float64](x: T): T {.importc: "Math.asinh", nodecl.} + func arccosh*[T: float32|float64](x: T): T {.importc: "Math.acosh", nodecl.} + func arctanh*[T: float32|float64](x: T): T {.importc: "Math.atanh", nodecl.} + +func cot*[T: float32|float64](x: T): T = 1.0 / tan(x) + ## Computes the cotangent of `x` (`1/tan(x)`). +func sec*[T: float32|float64](x: T): T = 1.0 / cos(x) + ## Computes the secant of `x` (`1/cos(x)`). +func csc*[T: float32|float64](x: T): T = 1.0 / sin(x) + ## Computes the cosecant of `x` (`1/sin(x)`). + +func coth*[T: float32|float64](x: T): T = 1.0 / tanh(x) + ## Computes the hyperbolic cotangent of `x` (`1/tanh(x)`). +func sech*[T: float32|float64](x: T): T = 1.0 / cosh(x) + ## Computes the hyperbolic secant of `x` (`1/cosh(x)`). +func csch*[T: float32|float64](x: T): T = 1.0 / sinh(x) + ## Computes the hyperbolic cosecant of `x` (`1/sinh(x)`). + +func arccot*[T: float32|float64](x: T): T = arctan(1.0 / x) + ## Computes the inverse cotangent of `x` (`arctan(1/x)`). +func arcsec*[T: float32|float64](x: T): T = arccos(1.0 / x) + ## Computes the inverse secant of `x` (`arccos(1/x)`). +func arccsc*[T: float32|float64](x: T): T = arcsin(1.0 / x) + ## Computes the inverse cosecant of `x` (`arcsin(1/x)`). + +func arccoth*[T: float32|float64](x: T): T = arctanh(1.0 / x) + ## Computes the inverse hyperbolic cotangent of `x` (`arctanh(1/x)`). +func arcsech*[T: float32|float64](x: T): T = arccosh(1.0 / x) + ## Computes the inverse hyperbolic secant of `x` (`arccosh(1/x)`). +func arccsch*[T: float32|float64](x: T): T = arcsinh(1.0 / x) + ## Computes the inverse hyperbolic cosecant of `x` (`arcsinh(1/x)`). + +const windowsCC89 = defined(windows) and defined(bcc) + +when not defined(js): # C + func hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} + func hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} = + ## Computes the length of the hypotenuse of a right-angle triangle with + ## `x` as its base and `y` as its height. Equivalent to `sqrt(x*x + y*y)`. + runnableExamples: + doAssert almostEqual(hypot(3.0, 4.0), 5.0) + func pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} + func pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} = + ## Computes `x` raised to the power of `y`. + ## + ## To compute the power between integers (e.g. 2^6), + ## use the `^ func <#^,T,Natural>`_. + ## + ## **See also:** + ## * `^ func <#^,T,Natural>`_ + ## * `sqrt func <#sqrt,float64>`_ + ## * `cbrt func <#cbrt,float64>`_ + runnableExamples: + doAssert almostEqual(pow(100, 1.5), 1000.0) + doAssert almostEqual(pow(16.0, 0.5), 4.0) + + # TODO: add C89 version on windows + when not windowsCC89: + func erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} + func erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} + ## Computes the [error function](https://en.wikipedia.org/wiki/Error_function) for `x`. + ## + ## **Note:** Not available for the JS backend. + func erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} + func erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} + ## Computes the [complementary error function](https://en.wikipedia.org/wiki/Error_function#Complementary_error_function) for `x`. + ## + ## **Note:** Not available for the JS backend. + func gamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} + func gamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} = + ## Computes the [gamma function](https://en.wikipedia.org/wiki/Gamma_function) for `x`. + ## + ## **Note:** Not available for the JS backend. + ## + ## **See also:** + ## * `lgamma func <#lgamma,float64>`_ for the natural logarithm of the gamma function + runnableExamples: + doAssert almostEqual(gamma(1.0), 1.0) + doAssert almostEqual(gamma(4.0), 6.0) + doAssert almostEqual(gamma(11.0), 3628800.0) + func lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} + func lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} = + ## Computes the natural logarithm of the gamma function for `x`. + ## + ## **Note:** Not available for the JS backend. + ## + ## **See also:** + ## * `gamma func <#gamma,float64>`_ for gamma function + + func floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} + func floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} = + ## Computes the floor function (i.e. the largest integer not greater than `x`). + ## + ## **See also:** + ## * `ceil func <#ceil,float64>`_ + ## * `round func <#round,float64>`_ + ## * `trunc func <#trunc,float64>`_ + runnableExamples: + doAssert floor(2.1) == 2.0 + doAssert floor(2.9) == 2.0 + doAssert floor(-3.5) == -4.0 + + func ceil*(x: float32): float32 {.importc: "ceilf", header: "<math.h>".} + func ceil*(x: float64): float64 {.importc: "ceil", header: "<math.h>".} = + ## Computes the ceiling function (i.e. the smallest integer not smaller + ## than `x`). + ## + ## **See also:** + ## * `floor func <#floor,float64>`_ + ## * `round func <#round,float64>`_ + ## * `trunc func <#trunc,float64>`_ + runnableExamples: + doAssert ceil(2.1) == 3.0 + doAssert ceil(2.9) == 3.0 + doAssert ceil(-2.1) == -2.0 + + when windowsCC89: + # MSVC 2010 don't have trunc/truncf + # this implementation was inspired by Go-lang Math.Trunc + func truncImpl(f: float64): float64 = + const + mask: uint64 = 0x7FF + shift: uint64 = 64 - 12 + bias: uint64 = 0x3FF + + if f < 1: + if f < 0: return -truncImpl(-f) + elif f == 0: return f # Return -0 when f == -0 + else: return 0 + + var x = cast[uint64](f) + let e = (x shr shift) and mask - bias + + # Keep the top 12+e bits, the integer part; clear the rest. + if e < 64 - 12: + x = x and (not (1'u64 shl (64'u64 - 12'u64 - e) - 1'u64)) + + result = cast[float64](x) + + func truncImpl(f: float32): float32 = + const + mask: uint32 = 0xFF + shift: uint32 = 32 - 9 + bias: uint32 = 0x7F + + if f < 1: + if f < 0: return -truncImpl(-f) + elif f == 0: return f # Return -0 when f == -0 + else: return 0 + + var x = cast[uint32](f) + let e = (x shr shift) and mask - bias + + # Keep the top 9+e bits, the integer part; clear the rest. + if e < 32 - 9: + x = x and (not (1'u32 shl (32'u32 - 9'u32 - e) - 1'u32)) + + result = cast[float32](x) + + func trunc*(x: float64): float64 = + if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x + result = truncImpl(x) + + func trunc*(x: float32): float32 = + if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x + result = truncImpl(x) + + func round*[T: float32|float64](x: T): T = + ## Windows compilers prior to MSVC 2012 do not implement 'round', + ## 'roundl' or 'roundf'. + result = if x < 0.0: ceil(x - T(0.5)) else: floor(x + T(0.5)) + else: + func round*(x: float32): float32 {.importc: "roundf", header: "<math.h>".} + func round*(x: float64): float64 {.importc: "round", header: "<math.h>".} = + ## Rounds a float to zero decimal places. + ## + ## Used internally by the `round func <#round,T,int>`_ + ## when the specified number of places is 0. + ## + ## **See also:** + ## * `round func <#round,T,int>`_ for rounding to the specific + ## number of decimal places + ## * `floor func <#floor,float64>`_ + ## * `ceil func <#ceil,float64>`_ + ## * `trunc func <#trunc,float64>`_ + runnableExamples: + doAssert round(3.4) == 3.0 + doAssert round(3.5) == 4.0 + doAssert round(4.5) == 5.0 + + func trunc*(x: float32): float32 {.importc: "truncf", header: "<math.h>".} + func trunc*(x: float64): float64 {.importc: "trunc", header: "<math.h>".} = + ## Truncates `x` to the decimal point. + ## + ## **See also:** + ## * `floor func <#floor,float64>`_ + ## * `ceil func <#ceil,float64>`_ + ## * `round func <#round,float64>`_ + runnableExamples: + doAssert trunc(PI) == 3.0 + doAssert trunc(-1.85) == -1.0 + + func `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} + func `mod`*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} = + ## Computes the modulo operation for float values (the remainder of `x` divided by `y`). + ## + ## **See also:** + ## * `floorMod func <#floorMod,T,T>`_ for Python-like (`%` operator) behavior + runnableExamples: + doAssert 6.5 mod 2.5 == 1.5 + doAssert -6.5 mod 2.5 == -1.5 + doAssert 6.5 mod -2.5 == 1.5 + doAssert -6.5 mod -2.5 == -1.5 + +else: # JS + func hypot*(x, y: float32): float32 {.importc: "Math.hypot", varargs, nodecl.} + func hypot*(x, y: float64): float64 {.importc: "Math.hypot", varargs, nodecl.} + func pow*(x, y: float32): float32 {.importc: "Math.pow", nodecl.} + func pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} + func floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} + func floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} + func ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} + func ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} + + when (NimMajor, NimMinor) < (1, 5) or defined(nimLegacyJsRound): + func round*(x: float): float {.importc: "Math.round", nodecl.} + else: + func jsRound(x: float): float {.importc: "Math.round", nodecl.} + func round*[T: float64 | float32](x: T): T = + if x >= 0: result = jsRound(x) + else: + result = ceil(x) + if result - x >= T(0.5): + result -= T(1.0) + func trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} + func trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} + + func `mod`*(x, y: float32): float32 {.importjs: "(# % #)".} + func `mod`*(x, y: float64): float64 {.importjs: "(# % #)".} = + ## Computes the modulo operation for float values (the remainder of `x` divided by `y`). + runnableExamples: + doAssert 6.5 mod 2.5 == 1.5 + doAssert -6.5 mod 2.5 == -1.5 + doAssert 6.5 mod -2.5 == 1.5 + doAssert -6.5 mod -2.5 == -1.5 + + func divmod*[T:SomeInteger](num, denom: T): (T, T) = + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + result[0] = num div denom + result[1] = num mod denom + + +func round*[T: float32|float64](x: T, places: int): T = + ## Decimal rounding on a binary floating point number. + ## + ## This function is NOT reliable. Floating point numbers cannot hold + ## non integer decimals precisely. If `places` is 0 (or omitted), + ## round to the nearest integral value following normal mathematical + ## rounding rules (e.g. `round(54.5) -> 55.0`). If `places` is + ## greater than 0, round to the given number of decimal places, + ## e.g. `round(54.346, 2) -> 54.350000000000001421…`. If `places` is negative, round + ## to the left of the decimal place, e.g. `round(537.345, -1) -> 540.0`. + runnableExamples: + doAssert round(PI, 2) == 3.14 + doAssert round(PI, 4) == 3.1416 + + if places == 0: + result = round(x) + else: + var mult = pow(10.0, T(places)) + result = round(x * mult) / mult + +func floorDiv*[T: SomeInteger](x, y: T): T = + ## Floor division is conceptually defined as `floor(x / y)`. + ## + ## This is different from the `system.div <system.html#div,int,int>`_ + ## operator, which is defined as `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `floorDiv` rounds down. + ## + ## **See also:** + ## * `system.div proc <system.html#div,int,int>`_ for integer division + ## * `floorMod func <#floorMod,T,T>`_ for Python-like (`%` operator) behavior + runnableExamples: + doAssert floorDiv( 13, 3) == 4 + doAssert floorDiv(-13, 3) == -5 + doAssert floorDiv( 13, -3) == -5 + doAssert floorDiv(-13, -3) == 4 + + result = x div y + let r = x mod y + if (r > 0 and y < 0) or (r < 0 and y > 0): result.dec 1 + +func floorMod*[T: SomeNumber](x, y: T): T = + ## Floor modulo is conceptually defined as `x - (floorDiv(x, y) * y)`. + ## + ## This func behaves the same as the `%` operator in Python. + ## + ## **See also:** + ## * `mod func <#mod,float64,float64>`_ + ## * `floorDiv func <#floorDiv,T,T>`_ + runnableExamples: + doAssert floorMod( 13, 3) == 1 + doAssert floorMod(-13, 3) == 2 + doAssert floorMod( 13, -3) == -2 + doAssert floorMod(-13, -3) == -1 + + result = x mod y + if (result > 0 and y < 0) or (result < 0 and y > 0): result += y + +func euclDiv*[T: SomeInteger](x, y: T): T {.since: (1, 5, 1).} = + ## Returns euclidean division of `x` by `y`. + runnableExamples: + doAssert euclDiv(13, 3) == 4 + doAssert euclDiv(-13, 3) == -5 + doAssert euclDiv(13, -3) == -4 + doAssert euclDiv(-13, -3) == 5 + + result = x div y + if x mod y < 0: + if y > 0: + dec result + else: + inc result + +func euclMod*[T: SomeNumber](x, y: T): T {.since: (1, 5, 1).} = + ## Returns euclidean modulo of `x` by `y`. + ## `euclMod(x, y)` is non-negative. + runnableExamples: + doAssert euclMod(13, 3) == 1 + doAssert euclMod(-13, 3) == 2 + doAssert euclMod(13, -3) == 1 + doAssert euclMod(-13, -3) == 2 + + result = x mod y + if result < 0: + result += abs(y) + +func ceilDiv*[T: SomeInteger](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Ceil division is conceptually defined as `ceil(x / y)`. + ## + ## Assumes `x >= 0` and `y > 0` (and `x + y - 1 <= high(T)` if T is SomeUnsignedInt). + ## + ## This is different from the `system.div <system.html#div,int,int>`_ + ## operator, which works like `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `ceilDiv` rounds up. + ## + ## This function has the above input limitation, because that allows the + ## compiler to generate faster code and it is rarely used with + ## negative values or unsigned integers close to `high(T)/2`. + ## If you need a `ceilDiv` that works with any input, see: + ## https://github.com/demotomohiro/divmath. + ## + ## **See also:** + ## * `system.div proc <system.html#div,int,int>`_ for integer division + ## * `floorDiv func <#floorDiv,T,T>`_ for integer division which rounds down. + runnableExamples: + assert ceilDiv(12, 3) == 4 + assert ceilDiv(13, 3) == 5 + + when sizeof(T) == 8: + type UT = uint64 + elif sizeof(T) == 4: + type UT = uint32 + elif sizeof(T) == 2: + type UT = uint16 + elif sizeof(T) == 1: + type UT = uint8 + else: + {.fatal: "Unsupported int type".} + + assert x >= 0 and y > 0 + when T is SomeUnsignedInt: + assert x + y - 1 >= x + + # If the divisor is const, the backend C/C++ compiler generates code without a `div` + # instruction, as it is slow on most CPUs. + # If the divisor is a power of 2 and a const unsigned integer type, the + # compiler generates faster code. + # If the divisor is const and a signed integer, generated code becomes slower + # than the code with unsigned integers, because division with signed integers + # need to works for both positive and negative value without `idiv`/`sdiv`. + # That is why this code convert parameters to unsigned. + # This post contains a comparison of the performance of signed/unsigned integers: + # https://github.com/nim-lang/Nim/pull/18596#issuecomment-894420984. + # If signed integer arguments were not converted to unsigned integers, + # `ceilDiv` wouldn't work for any positive signed integer value, because + # `x + (y - 1)` can overflow. + ((x.UT + (y.UT - 1.UT)) div y.UT).T + +func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = + ## Splits `x` into a normalized fraction `frac` and an integral power of 2 `exp`, + ## such that `abs(frac) in 0.5..<1` and `x == frac * 2 ^ exp`, except for special + ## cases shown below. + runnableExamples: + doAssert frexp(8.0) == (0.5, 4) + doAssert frexp(-8.0) == (-0.5, 4) + doAssert frexp(0.0) == (0.0, 0) + + # special cases: + when sizeof(int) == 8: + doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0 + doAssert frexp(Inf).frac == Inf # +- Inf preserved + doAssert frexp(NaN).frac.isNaN + + when not defined(js): + var exp: cint + result.frac = c_frexp2(x, exp) + result.exp = exp + else: if x == 0.0: - exponent = 0 - result = 0.0 + # reuse signbit implementation + let uintBuffer = toBitsImpl(x) + if (uintBuffer[1] shr 31) != 0: + # x is -0.0 + result = (-0.0, 0) + else: + result = (0.0, 0) elif x < 0.0: - result = -frexp(-x, exponent) + result = frexp(-x) + result.frac = -result.frac else: - var ex = floor(log2(x)) - exponent = round(ex) - result = x / pow(2.0, ex) - - proc arccos*(x: float): float {.importc: "Math.acos", nodecl.} - proc arcsin*(x: float): float {.importc: "Math.asin", nodecl.} - proc arctan*(x: float): float {.importc: "Math.atan", nodecl.} - proc arctan2*(y, x: float): float {.importc: "Math.atan2", nodecl.} - - proc cos*(x: float): float {.importc: "Math.cos", nodecl.} - proc cosh*(x: float): float = return (exp(x)+exp(-x))*0.5 - proc hypot*(x, y: float): float = return sqrt(x*x + y*y) - proc sinh*(x: float): float = return (exp(x)-exp(-x))*0.5 - proc sin*(x: float): float {.importc: "Math.sin", nodecl.} - proc tan*(x: float): float {.importc: "Math.tan", nodecl.} - proc tanh*(x: float): float = - var y = exp(2.0*x) - return (y-1.0)/(y+1.0) - -{.pop.} + var ex = trunc(log2(x)) + result.exp = int(ex) + result.frac = x / pow(2.0, ex) + if abs(result.frac) >= 1: + inc(result.exp) + result.frac = result.frac / 2 + if result.exp == 1024 and result.frac == 0.0: + result.frac = 0.99999999999999988898 + +func frexp*[T: float32|float64](x: T, exponent: var int): T {.inline.} = + ## Overload of `frexp` that calls `(result, exponent) = frexp(x)`. + runnableExamples: + var x: int + doAssert frexp(5.0, x) == 0.625 + doAssert x == 3 + + (result, exponent) = frexp(x) + + +when not defined(js): + when windowsCC89: + # taken from Go-lang Math.Log2 + const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 + template log2Impl[T](x: T): T = + var exp: int + var frac = frexp(x, exp) + # Make sure exact powers of two give an exact answer. + # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. + if frac == 0.5: return T(exp - 1) + log10(frac) * (1 / ln2) + T(exp) + + func log2*(x: float32): float32 = log2Impl(x) + func log2*(x: float64): float64 = log2Impl(x) + ## Log2 returns the binary logarithm of x. + ## The special cases are the same as for Log. -proc degToRad*[T: float32|float64](d: T): T {.inline.} = - ## Convert from degrees to radians - result = T(d) * RadPerDeg - -proc radToDeg*[T: float32|float64](d: T): T {.inline.} = - ## Convert from radians to degrees - result = T(d) / RadPerDeg - -proc `mod`*(x, y: float): float = - ## Computes the modulo operation for float operators. Equivalent - ## to ``x - y * floor(x/y)``. Note that the remainder will always - ## have the same sign as the divisor. + else: + func log2*(x: float32): float32 {.importc: "log2f", header: "<math.h>".} + func log2*(x: float64): float64 {.importc: "log2", header: "<math.h>".} = + ## Computes the binary logarithm (base 2) of `x`. + ## + ## **See also:** + ## * `log func <#log,T,T>`_ + ## * `log10 func <#log10,float64>`_ + ## * `ln func <#ln,float64>`_ + runnableExamples: + doAssert almostEqual(log2(8.0), 3.0) + doAssert almostEqual(log2(1.0), 0.0) + doAssert almostEqual(log2(0.0), -Inf) + doAssert log2(-2.0).isNaN + +func splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = + ## Breaks `x` into an integer and a fractional part. ## - ## .. code-block:: nim - ## echo (4.0 mod -3.1) # -2.2 - result = if y == 0.0: x else: x - y * (x/y).floor - -proc random*[T](x: Slice[T]): T = - ## For a slice `a .. b` returns a value in the range `a .. b-1`. - result = random(x.b - x.a) + x.a + ## Returns a tuple containing `intpart` and `floatpart`, representing + ## the integer part and the fractional part, respectively. + ## + ## Both parts have the same sign as `x`. Analogous to the `modf` + ## function in C. + runnableExamples: + doAssert splitDecimal(5.25) == (intpart: 5.0, floatpart: 0.25) + doAssert splitDecimal(-2.73) == (intpart: -2.0, floatpart: -0.73) + + var + absolute: T + absolute = abs(x) + result.intpart = floor(absolute) + result.floatpart = absolute - result.intpart + if x < 0: + result.intpart = -result.intpart + result.floatpart = -result.floatpart + + +func degToRad*[T: float32|float64](d: T): T {.inline.} = + ## Converts from degrees to radians. + ## + ## **See also:** + ## * `radToDeg func <#radToDeg,T>`_ + runnableExamples: + doAssert almostEqual(degToRad(180.0), PI) -proc random*[T](a: openArray[T]): T = - ## returns a random element from the openarray `a`. - result = a[random(a.low..a.len)] + result = d * T(RadPerDeg) -type - RunningStat* = object ## an accumulator for statistical data - n*: int ## number of pushed data - sum*, min*, max*, mean*: float ## self-explaining - oldM, oldS, newS: float - -{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} - -proc push*(s: var RunningStat, x: float) = - ## pushes a value `x` for processing - inc(s.n) - # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.n == 1: - s.min = x - s.max = x - s.oldM = x - s.mean = x - s.oldS = 0.0 - else: - if s.min > x: s.min = x - if s.max < x: s.max = x - s.mean = s.oldM + (x - s.oldM)/toFloat(s.n) - s.newS = s.oldS + (x - s.oldM)*(x - s.mean) - - # set up for next iteration: - s.oldM = s.mean - s.oldS = s.newS - s.sum = s.sum + x +func radToDeg*[T: float32|float64](d: T): T {.inline.} = + ## Converts from radians to degrees. + ## + ## **See also:** + ## * `degToRad func <#degToRad,T>`_ + runnableExamples: + doAssert almostEqual(radToDeg(2 * PI), 360.0) -proc push*(s: var RunningStat, x: int) = - ## pushes a value `x` for processing. `x` is simply converted to ``float`` - ## and the other push operation is called. - push(s, toFloat(x)) + result = d / T(RadPerDeg) -proc variance*(s: RunningStat): float = - ## computes the current variance of `s` - if s.n > 1: result = s.newS / (toFloat(s.n - 1)) +func sgn*[T: SomeNumber](x: T): int {.inline.} = + ## Sign function. + ## + ## Returns: + ## * `-1` for negative numbers and `NegInf`, + ## * `1` for positive numbers and `Inf`, + ## * `0` for positive zero, negative zero and `NaN` + runnableExamples: + doAssert sgn(5) == 1 + doAssert sgn(0) == 0 + doAssert sgn(-4.1) == -1 -proc standardDeviation*(s: RunningStat): float = - ## computes the current standard deviation of `s` - result = sqrt(variance(s)) + ord(T(0) < x) - ord(x < T(0)) {.pop.} {.pop.} -proc `^`*[T](x, y: T): T = - ## Computes ``x`` to the power ``y`. ``x`` must be non-negative, use - ## `pow <#pow,float,float>` for negative exponents. - assert y >= 0 - var (x, y) = (x, y) - result = 1 +func sum*[T](x: openArray[T]): T = + ## Computes the sum of the elements in `x`. + ## + ## If `x` is empty, 0 is returned. + ## + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ + runnableExamples: + doAssert sum([1, 2, 3, 4]) == 10 + doAssert sum([-4, 3, 5]) == 4 - while true: - if (y and 1) != 0: - result *= x - y = y shr 1 - if y == 0: - break - x *= x + for i in items(x): result = result + i -proc gcd*[T](x, y: T): T = - ## Computes the greatest common divisor of ``x`` and ``y``. +func prod*[T](x: openArray[T]): T = + ## Computes the product of the elements in `x`. + ## + ## If `x` is empty, 1 is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `fac func <#fac,int>`_ + runnableExamples: + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([-4, 3, 5]) == -60 + + result = T(1) + for i in items(x): result = result * i + +func cumsummed*[T](x: openArray[T]): seq[T] = + ## Returns the cumulative (aka prefix) summation of `x`. + ## + ## If `x` is empty, `@[]` is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version + runnableExamples: + doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] + + let xLen = x.len + if xLen == 0: + return @[] + result.setLen(xLen) + result[0] = x[0] + for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] + +func cumsum*[T](x: var openArray[T]) = + ## Transforms `x` in-place (must be declared as `var`) into its + ## cumulative (aka prefix) summation. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which + ## returns a cumsummed sequence + runnableExamples: + var a = [1, 2, 3, 4] + cumsum(a) + doAssert a == @[1, 3, 6, 10] + + for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] + +func `^`*[T: SomeNumber](x: T, y: Natural): T = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be non-negative, use + ## `pow <#pow,float64,float64>`_ for negative exponents. + ## + ## **See also:** + ## * `pow func <#pow,float64,float64>`_ for negative exponent or + ## floats + ## * `sqrt func <#sqrt,float64>`_ + ## * `cbrt func <#cbrt,float64>`_ + runnableExamples: + doAssert -3 ^ 0 == 1 + doAssert -3 ^ 1 == -3 + doAssert -3 ^ 2 == 9 + + case y + of 0: result = 1 + of 1: result = x + of 2: result = x * x + of 3: result = x * x * x + else: + var (x, y) = (x, y) + result = 1 + while true: + if (y and 1) != 0: + result *= x + y = y shr 1 + if y == 0: + break + x *= x + +func gcd*[T](x, y: T): T = + ## Computes the greatest common (positive) divisor of `x` and `y`. + ## ## Note that for floats, the result cannot always be interpreted as - ## "greatest decimal `z` such that ``z*N == x and z*M == y`` - ## where N and M are positive integers." - var (x,y) = (x,y) + ## "greatest decimal `z` such that `z*N == x and z*M == y` + ## where N and M are positive integers". + ## + ## **See also:** + ## * `gcd func <#gcd,SomeInteger,SomeInteger>`_ for an integer version + ## * `lcm func <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(13.5, 9.0) == 4.5 + + var (x, y) = (x, y) while y != 0: x = x mod y swap x, y abs x -proc lcm*[T](x, y: T): T = - ## Computes the least common multiple of ``x`` and ``y``. +when useBuiltins: + ## this func uses bitwise comparisons from C compilers, which are not always available. + func gcd*(x, y: SomeInteger): SomeInteger = + ## Computes the greatest common (positive) divisor of `x` and `y`, + ## using the binary GCD (aka Stein's) algorithm. + ## + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ for a float version + ## * `lcm func <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(12, 8) == 4 + doAssert gcd(17, 63) == 1 + + when x is SomeSignedInt: + var x = abs(x) + else: + var x = x + when y is SomeSignedInt: + var y = abs(y) + else: + var y = y + + if x == 0: + return y + if y == 0: + return x + + let shift = countTrailingZeroBits(x or y) + y = y shr countTrailingZeroBits(y) + while x != 0: + x = x shr countTrailingZeroBits(x) + if y > x: + swap y, x + x -= y + y shl shift + +func gcd*[T](x: openArray[T]): T {.since: (1, 1).} = + ## Computes the greatest common (positive) divisor of the elements of `x`. + ## + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ for a version with two arguments + runnableExamples: + doAssert gcd(@[13.5, 9.0]) == 4.5 + + result = x[0] + for i in 1 ..< x.len: + result = gcd(result, x[i]) + +func lcm*[T](x, y: T): T = + ## Computes the least common multiple of `x` and `y`. + ## + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ + runnableExamples: + doAssert lcm(24, 30) == 120 + doAssert lcm(13, 39) == 39 + x div gcd(x, y) * y -when isMainModule and not defined(JS): - proc gettime(dummy: ptr cint): cint {.importc: "time", header: "<time.h>".} - - # Verifies random seed initialization. - let seed = gettime(nil) - randomize(seed) - const SIZE = 10 - var buf : array[0..SIZE, int] - # Fill the buffer with random values - for i in 0..SIZE-1: - buf[i] = random(high(int)) - # Check that the second random calls are the same for each position. - randomize(seed) - for i in 0..SIZE-1: - assert buf[i] == random(high(int)), "non deterministic random seeding" - - when not defined(testing): - echo "random values equal after reseeding" - - # Check for no side effect annotation - proc mySqrt(num: float): float {.noSideEffect.} = - return sqrt(num) - - # check gamma function - assert($tgamma(5.0) == $24.0) # 4! - assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0 - assert(erf(6.0) > erf(5.0)) - assert(erfc(6.0) < erfc(5.0)) +func clamp*[T](val: T, bounds: Slice[T]): T {.since: (1, 5), inline.} = + ## Like `system.clamp`, but takes a slice, so you can easily clamp within a range. + runnableExamples: + assert clamp(10, 1 .. 5) == 5 + assert clamp(1, 1 .. 3) == 1 + type A = enum a0, a1, a2, a3, a4, a5 + assert a1.clamp(a2..a4) == a2 + assert clamp((3, 0), (1, 0) .. (2, 9)) == (2, 9) + doAssertRaises(AssertionDefect): discard clamp(1, 3..2) # invalid bounds + assert bounds.a <= bounds.b, $(bounds.a, bounds.b) + clamp(val, bounds.a, bounds.b) + +func lcm*[T](x: openArray[T]): T {.since: (1, 1).} = + ## Computes the least common multiple of the elements of `x`. + ## + ## **See also:** + ## * `lcm func <#lcm,T,T>`_ for a version with two arguments + runnableExamples: + doAssert lcm(@[24, 30]) == 120 + + result = x[0] + for i in 1 ..< x.len: + result = lcm(result, x[i]) diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 44b9ed0d4..9c3f6d51b 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -7,13 +7,28 @@ # distribution, for details about the copyright. # -## Module for computing MD5 checksums. +## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). +## +## This module also works at compile time and in JavaScript. +## +## See also +## ======== +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm +## * `hashes module<hashes.html>`_ for efficient computations of hash values +## for diverse Nim types + +{.deprecated: "use command `nimble install checksums` and import `checksums/md5` instead".} + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} type MD5State = array[0..3, uint32] MD5Block = array[0..15, uint32] MD5CBits = array[0..7, uint8] MD5Digest* = array[0..15, uint8] + ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. MD5Buffer = array[0..63, uint8] MD5Context* {.final.} = object state: MD5State @@ -21,15 +36,16 @@ type buffer: MD5Buffer const - padding: cstring = "\x80\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0" + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) @@ -66,7 +82,7 @@ proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = rot(a, s) a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: openArray[uint8]) = var j = 0 for i in 0..high(dest): dest[i] = uint32(ord(src[j])) or @@ -78,16 +94,41 @@ proc encode(dest: var MD5Block, src: cstring) = proc decode(dest: var openArray[uint8], src: openArray[uint32]) = var i = 0 for j in 0..high(src): - dest[i] = src[j] and 0xff'u32 - dest[i+1] = src[j] shr 8 and 0xff'u32 - dest[i+2] = src[j] shr 16 and 0xff'u32 - dest[i+3] = src[j] shr 24 and 0xff'u32 + dest[i] = uint8(src[j] and 0xff'u32) + dest[i+1] = uint8(src[j] shr 8 and 0xff'u32) + dest[i+2] = uint8(src[j] shr 16 and 0xff'u32) + dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +template slice(s: cstring, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + toOpenArrayByte($s, a, b) + else: + when defined(js): + # toOpenArrayByte for cstring is not implemented in JS + toOpenArrayByte($s, a, b) + else: + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + s.toOpenArray(a, b) + +const useMem = declared(copyMem) + +template memOrNot(withMem, withoutMem): untyped = + when nimvm: + withoutMem + else: + when useMem: + withMem + else: + withoutMem + +proc transform(buffer: openArray[uint8], state: var MD5State) = var myBlock: MD5Block - encode(myBlock, cast[cstring](buffer)) + encode(myBlock, buffer) var a = state[0] var b = state[1] var c = state[2] @@ -161,83 +202,134 @@ proc transform(buffer: pointer, state: var MD5State) = state[2] = state[2] + c state[3] = state[3] + d -proc md5Init*(c: var MD5Context) = - ## initializes a MD5Context - c.state[0] = 0x67452301'u32 - c.state[1] = 0xEFCDAB89'u32 - c.state[2] = 0x98BADCFE'u32 - c.state[3] = 0x10325476'u32 - c.count[0] = 0'u32 - c.count[1] = 0'u32 - zeroMem(addr(c.buffer), sizeof(MD5buffer)) +proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], + tags: [], gcsafe.} +proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: cstring, len: int) = - ## updates the MD5Context with the `input` data of length `len` - var input = input - var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(len) shl 3) - if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(len) shr 29) - var PartLen = 64 - Index - if len >= PartLen: - copyMem(addr(c.buffer[Index]), input, PartLen) - transform(addr(c.buffer), c.state) - var i = PartLen - while i + 63 < len: - transform(addr(input[i]), c.state) - inc(i, 64) - copyMem(addr(c.buffer[0]), addr(input[i]), len-i) - else: - copyMem(addr(c.buffer[Index]), addr(input[0]), len) +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) -proc md5Final*(c: var MD5Context, digest: var MD5Digest) = - ## finishes the MD5Context and stores the result in `digest` - var - Bits: MD5CBits - PadLen: int - decode(Bits, c.count) - var Index = int((c.count[0] shr 3) and 0x3F) - if Index < 56: PadLen = 56 - Index - else: PadLen = 120 - Index - md5Update(c, padding, PadLen) - md5Update(c, cast[cstring](addr(Bits)), 8) - decode(digest, c.state) - zeroMem(addr(c), sizeof(MD5Context)) proc toMD5*(s: string): MD5Digest = - ## computes the MD5Digest value for a string `s` + ## Computes the `MD5Digest` value for a string `s`. + ## + ## **See also:** + ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation + ## of the `MD5Digest` + ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string + runnableExamples: + assert $toMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" + var c: MD5Context md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, result) proc `$`*(d: MD5Digest): string = - ## converts a MD5Digest value into its string representation + ## Converts a `MD5Digest` value into its string representation. const digits = "0123456789abcdef" result = "" for i in 0..15: - add(result, digits[(d[i] shr 4) and 0xF]) - add(result, digits[d[i] and 0xF]) + add(result, digits[(d[i].int shr 4) and 0xF]) + add(result, digits[d[i].int and 0xF]) proc getMD5*(s: string): string = - ## computes an MD5 value of `s` and returns its string representation + ## Computes an MD5 value of `s` and returns its string representation. + ## + ## **See also:** + ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string + runnableExamples: + assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" + var c: MD5Context d: MD5Digest md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, d) result = $d proc `==`*(D1, D2: MD5Digest): bool = - ## checks if two MD5Digest values are identical + ## Checks if two `MD5Digest` values are identical. for i in 0..15: if D1[i] != D2[i]: return false return true -when isMainModule: - assert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "a3cca2b2aa1e3b5b3b5aad99a8529074") - assert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "7e716d0e702df0505fc72e2b89467910") - assert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") + +proc clearBuffer(c: var MD5Context) {.inline.} = + memOrNot: + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + do: + reset(c.buffer) + +proc md5Init*(c: var MD5Context) = + ## Initializes an `MD5Context`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + c.state[0] = 0x67452301'u32 + c.state[1] = 0xEFCDAB89'u32 + c.state[2] = 0x98BADCFE'u32 + c.state[3] = 0x10325476'u32 + c.count[0] = 0'u32 + c.count[1] = 0'u32 + clearBuffer(c) + +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + memOrNot: + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + do: + # cannot use system.`[]=` for arrays and openarrays as + # it can raise RangeDefect which gets tracked + for i in 0..<len: + c.buffer[index + i] = input[inputIndex + i] + +proc md5Update*(c: var MD5Context, input: openArray[uint8]) = + ## Updates the `MD5Context` with the `input` data. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + var Index = int((c.count[0] shr 3) and 0x3F) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) + var PartLen = 64 - Index + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) + var i = PartLen + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) + inc(i, 64) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) + +proc md5Final*(c: var MD5Context, digest: var MD5Digest) = + ## Finishes the `MD5Context` and stores the result in `digest`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + var + Bits: MD5CBits + PadLen: int + decode(Bits, c.count) + var Index = int((c.count[0] shr 3) and 0x3F) + if Index < 56: PadLen = 56 - Index + else: PadLen = 120 - Index + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) + decode(digest, c.state) + clearBuffer(c) + + +when defined(nimHasStyleChecks): + {.pop.} #{.push styleChecks: off.} \ No newline at end of file diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index b9c574944..8eec551c4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -16,60 +16,115 @@ ## other "line-like", variable length, delimited records). when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs elif defined(posix): - import posix + import std/posix else: {.error: "the memfiles module is not supported on your operating system!".} -import os +import std/streams +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + +proc newEIO(msg: string): ref IOError = + new(result) + result.msg = msg + +proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1, + ## allocating | freeing space from the file system. This routine returns the + ## last OSErrorCode found rather than raising to support old rollback/clean-up + ## code style. [ Should maybe move to std/osfiles. ] + if newFileSize < 0 or newFileSize == oldSize: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + if newFileSize > oldSize: # grow the file + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable BUT allows SEGV + elif e != 0: + result = osLastError() + else: # shrink the file + if ftruncate(fh.cint, newFileSize) == -1: + result = osLastError() type - MemFile* = object ## represents a memory mapped file - mem*: pointer ## a pointer to the memory mapped file. The pointer - ## can be used directly to change the contents of the - ## file, if it was opened with write access. - size*: int ## size of the memory mapped file + MemFile* = object ## represents a memory mapped file + mem*: pointer ## a pointer to the memory mapped file. The pointer + ## can be used directly to change the contents of the + ## file, if it was opened with write access. + size*: int ## size of the memory mapped file when defined(windows): - fHandle: Handle - mapHandle: Handle - wasOpened: bool ## only close if wasOpened + fHandle*: Handle ## **Caution**: Windows specific public field to allow + ## even more low level trickery. + mapHandle*: Handle ## **Caution**: Windows specific public field. + wasOpened*: bool ## **Caution**: Windows specific public field. else: - handle: cint - -{.deprecated: [TMemFile: MemFile].} + handle*: cint ## **Caution**: Posix specific public field. + flags: cint ## **Caution**: Platform specific private field. proc mapMem*(m: var MemFile, mode: FileMode = fmRead, - mappedSize = -1, offset = 0): pointer = + mappedSize = -1, offset = 0, mapFlags = cint(-1)): pointer = + ## returns a pointer to a mapped portion of MemFile `m` + ## + ## `mappedSize` of `-1` maps to the whole file, and + ## `offset` must be multiples of the PAGE SIZE of your OS + if mode == fmAppend: + raise newEIO("The append mode is not supported.") + var readonly = mode == fmRead when defined(windows): result = mapViewOfFileEx( m.mapHandle, - if readonly: FILE_MAP_READ else: FILE_MAP_WRITE, + if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE, int32(offset shr 32), int32(offset and 0xffffffff), - if mappedSize == -1: 0 else: mappedSize, + WinSizeT(if mappedSize == -1: 0 else: mappedSize), nil) if result == nil: raiseOSError(osLastError()) else: assert mappedSize > 0 + + m.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags + #Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set + if int(m.flags and MAP_PRIVATE) == 0: + m.flags = m.flags or MAP_SHARED + result = mmap( nil, mappedSize, if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE), + m.flags, m.handle, offset) if result == cast[pointer](MAP_FAILED): raiseOSError(osLastError()) proc unmapMem*(f: var MemFile, p: pointer, size: int) = - ## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`. + ## unmaps the memory region `(p, <p+size)` of the mapped file `f`. ## All changes are written back to the file system, if `f` was opened - ## with write access. ``size`` must be of exactly the size that was requested - ## via ``mapMem``. + ## with write access. + ## + ## `size` must be of exactly the size that was requested + ## via `mapMem`. when defined(windows): if unmapViewOfFile(p) == 0: raiseOSError(osLastError()) else: @@ -77,13 +132,29 @@ proc unmapMem*(f: var MemFile, p: pointer, size: int) = proc open*(filename: string, mode: FileMode = fmRead, - mappedSize = -1, offset = 0, newFileSize = -1): MemFile = - ## opens a memory mapped file. If this fails, ``EOS`` is raised. + mappedSize = -1, offset = 0, newFileSize = -1, + allowRemap = false, mapFlags = cint(-1)): MemFile = + ## opens a memory mapped file. If this fails, `OSError` is raised. + ## ## `newFileSize` can only be set if the file does not exist and is opened - ## with write access (e.g., with fmReadWrite). `mappedSize` and `offset` - ## can be used to map only a slice of the file. Example: + ## with write access (e.g., with fmReadWrite). + ## + ##`mappedSize` and `offset` + ## can be used to map only a slice of the file. + ## + ## `offset` must be multiples of the PAGE SIZE of your OS + ## (usually 4K or 8K but is unique to your OS) + ## + ## `allowRemap` only needs to be true if you want to call `mapMem` on + ## the resulting MemFile; else file handles are not kept open. + ## + ## `mapFlags` allows callers to override default choices for memory mapping + ## flags with a bitwise mask of a variety of likely platform-specific flags + ## which may be ignored or even cause `open` to fail if misspecified. ## - ## .. code-block:: nim + ## Example: + ## + ## ```nim ## var ## mm, mm_full, mm_half: MemFile ## @@ -95,8 +166,12 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## # Read the first 512 bytes ## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512) + ## ``` # The file can be resized only when write mode is used: + if mode == fmAppend: + raise newEIO("The append mode is not supported.") + assert newFileSize == -1 or mode != fmRead var readonly = mode == fmRead @@ -105,44 +180,37 @@ proc open*(filename: string, mode: FileMode = fmRead, result.size = 0 when defined(windows): - template fail(errCode: OSErrorCode, msg: expr) = + let desiredAccess = GENERIC_READ + let shareMode = FILE_SHARE_READ + let flags = FILE_FLAG_RANDOM_ACCESS + + template fail(errCode: OSErrorCode, msg: untyped) = rollback() if result.fHandle != 0: discard closeHandle(result.fHandle) if result.mapHandle != 0: discard closeHandle(result.mapHandle) raiseOSError(errCode) # return false - #raise newException(EIO, msg) + #raise newException(IOError, msg) - template callCreateFile(winApiProc, filename: expr): expr = + template callCreateFile(winApiProc, filename): untyped = winApiProc( filename, # GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE) - if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE, - FILE_SHARE_READ, + if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE, + if readonly: shareMode else: shareMode or FILE_SHARE_WRITE, nil, if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING, - if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY, + if readonly: FILE_ATTRIBUTE_READONLY or flags + else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -156,7 +224,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.mem = mapViewOfFileEx( result.mapHandle, - if readonly: FILE_MAP_READ else: FILE_MAP_WRITE, + if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE, int32(offset shr 32), int32(offset and 0xffffffff), if mappedSize == -1: 0 else: mappedSize, @@ -170,58 +238,134 @@ proc open*(filename: string, mode: FileMode = fmRead, if low == INVALID_FILE_SIZE: fail(osLastError(), "error getting file size") else: - var fileSize = (int64(hi) shr 32) or low + var fileSize = (int64(hi) shl 32) or int64(uint32(low)) if mappedSize != -1: result.size = min(fileSize, mappedSize).int else: result.size = fileSize.int result.wasOpened = true + if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: + if closeHandle(result.fHandle) != 0: + result.fHandle = INVALID_HANDLE_VALUE else: - template fail(errCode: OSErrorCode, msg: expr) = + template fail(errCode: OSErrorCode, msg: string) = rollback() - if result.handle != 0: discard close(result.handle) + if result.handle != -1: discard close(result.handle) raiseOSError(errCode) - var flags = if readonly: O_RDONLY else: O_RDWR + var flags = (if readonly: O_RDONLY else: O_RDWR) or O_CLOEXEC if newFileSize != -1: flags = flags or O_CREAT or O_TRUNC - var permissions_mode = S_IRUSR or S_IWUSR - result.handle = open(filename, flags, permissions_mode) + var permissionsMode = S_IRUSR or S_IWUSR + result.handle = open(filename, flags, permissionsMode) + if result.handle != -1: + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") else: result.handle = open(filename, flags) if result.handle == -1: - # XXX: errno is supposed to be set here - # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") - - if mappedSize != -1: - result.size = mappedSize - else: - var stat: Stat + if mappedSize != -1: #XXX Logic here differs from `when windows` branch .. + result.size = mappedSize #.. which always fstats&Uses min(mappedSize, st). + else: # if newFileSize!=-1: result.size=newFileSize # if trust setFileSize + var stat: Stat #^^.. BUT some FSes (eg. Linux HugeTLBfs) round to 2MiB. if fstat(result.handle, stat) != -1: - # XXX: Hmm, this could be unsafe - # Why is mmap taking int anyway? - result.size = int(stat.st_size) + result.size = stat.st_size.int # int may be 32-bit-unsafe for 2..<4 GiB else: fail(osLastError(), "error getting file size") - result.mem = mmap( - nil, - result.size, - if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE), - result.handle, - offset) + result.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags + # Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set + if int(result.flags and MAP_PRIVATE) == 0: + result.flags = result.flags or MAP_SHARED + let pr = if readonly: PROT_READ else: PROT_READ or PROT_WRITE + result.mem = mmap(nil, result.size, pr, result.flags, result.handle, offset) if result.mem == cast[pointer](MAP_FAILED): fail(osLastError(), "file mapping failed") + if not allowRemap and result.handle != -1: + if close(result.handle) == 0: + result.handle = -1 + +proc flush*(f: var MemFile; attempts: Natural = 3) = + ## Flushes `f`'s buffer for the number of attempts equal to `attempts`. + ## If were errors an exception `OSError` will be raised. + var res = false + var lastErr: OSErrorCode + when defined(windows): + for i in 1..attempts: + res = flushViewOfFile(f.mem, 0) != 0 + if res: + break + lastErr = osLastError() + if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode: + raiseOSError(lastErr) + else: + for i in 1..attempts: + res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0 + if res: + break + lastErr = osLastError() + if lastErr != EBUSY.OSErrorCode: + raiseOSError(lastErr, "error flushing mapping") + +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m + f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize, f.size); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "<sys/mman.h>".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize + proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the ## file system, if `f` was opened with write access. @@ -230,15 +374,19 @@ proc close*(f: var MemFile) = var lastErr: OSErrorCode when defined(windows): - if f.fHandle != INVALID_HANDLE_VALUE and f.wasOpened: + if f.wasOpened: error = unmapViewOfFile(f.mem) == 0 - lastErr = osLastError() - error = (closeHandle(f.mapHandle) == 0) or error - error = (closeHandle(f.fHandle) == 0) or error + if not error: + error = closeHandle(f.mapHandle) == 0 + if not error and f.fHandle != INVALID_HANDLE_VALUE: + discard closeHandle(f.fHandle) + f.fHandle = INVALID_HANDLE_VALUE + if error: + lastErr = osLastError() else: - if f.handle != 0: - error = munmap(f.mem, f.size) != 0 - lastErr = osLastError() + error = munmap(f.mem, f.size) != 0 + lastErr = osLastError() + if f.handle != -1: error = (close(f.handle) != 0) or error f.size = 0 @@ -249,25 +397,25 @@ proc close*(f: var MemFile) = f.mapHandle = 0 f.wasOpened = false else: - f.handle = 0 + f.handle = -1 if error: raiseOSError(lastErr) -type MemSlice* = object ## represent slice of a MemFile for iteration over delimited lines/records +type MemSlice* = object ## represent slice of a MemFile for iteration over delimited lines/records data*: pointer size*: int -proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".} +proc `==`*(x, y: MemSlice): bool = + ## Compare a pair of MemSlice for strict equality. + result = (x.size == y.size and equalMem(x.data, y.data, x.size)) proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. - var buf = newString(ms.size) - c_memcpy(addr(buf[0]), ms.data, ms.size) - buf[ms.size] = '\0' - result = buf + result.setLen(ms.size) + copyMem(result.cstring, ms.data, ms.size) -iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} = - ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`. +iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline.} = + ## Iterates over \[optional `eat`] `delim`-delimited slices in MemFile `mfile`. ## ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l) ## style on on a line-by-line basis. I.e., not every line needs the same ending. @@ -287,62 +435,134 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} = ## iterate over line-like records in a file. However, returned (data,size) ## objects are not Nim strings, bounds checked Nim arrays, or even terminated ## C strings. So, care is required to access the data (e.g., think C mem* - ## functions, not str* functions). Example: + ## functions, not str* functions). ## - ## .. code-block:: nim + ## Example: + ## ```nim ## var count = 0 ## for slice in memSlices(memfiles.open("foo")): ## if slice.size > 0 and cast[cstring](slice.data)[0] != '#': ## inc(count) ## echo count + ## ``` - proc c_memchr(cstr: pointer, c: char, n: csize): pointer {. - importc: "memchr", header: "<string.h>" .} + proc c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q) var ms: MemSlice var ending: pointer ms.data = mfile.mem var remaining = mfile.size while remaining > 0: - ending = c_memchr(ms.data, delim, remaining) - if ending == nil: # unterminated final slice - ms.size = remaining # Weird case..check eat? + ending = c_memchr(ms.data, delim, csize_t(remaining)) + if ending == nil: # unterminated final slice + ms.size = remaining # Weird case..check eat? yield ms break - ms.size = ending -! ms.data # delim is NOT included + ms.size = ending -! ms.data # delim is NOT included if eat != '\0' and ms.size > 0 and cast[cstring](ms.data)[ms.size - 1] == eat: - dec(ms.size) # trim pre-delim char + dec(ms.size) # trim pre-delim char yield ms - ms.data = cast[pointer](cast[int](ending) +% 1) # skip delim + ms.data = cast[pointer](cast[int](ending) +% 1) # skip delim remaining = mfile.size - (ms.data -! mfile.mem) -iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} = +iterator lines*(mfile: MemFile, buf: var string, delim = '\l', + eat = '\r'): string {.inline.} = ## Replace contents of passed buffer with each new line, like - ## `readLine(File) <system.html#readLine,File,TaintedString>`_. - ## `delim`, `eat`, and delimiting logic is exactly as for - ## `memSlices <#memSlices>`_, but Nim strings are returned. Example: + ## `readLine(File) <syncio.html#readLine,File,string>`_. + ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices + ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## - ## .. code-block:: nim - ## var buffer: TaintedString = "" + ## Example: + ## ```nim + ## var buffer: string = "" ## for line in lines(memfiles.open("foo"), buffer): ## echo line + ## ``` for ms in memSlices(mfile, delim, eat): - buf.setLen(ms.size) - c_memcpy(addr(buf[0]), ms.data, ms.size) - buf[ms.size] = '\0' + setLen(buf, ms.size) + if ms.size > 0: + copyMem(addr buf[0], ms.data, ms.size) yield buf -iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} = +iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## Return each line in a file as a Nim string, like - ## `lines(File) <system.html#lines.i,File>`_. - ## `delim`, `eat`, and delimiting logic is exactly as for - ## `memSlices <#memSlices>`_, but Nim strings are returned. Example: + ## `lines(File) <syncio.html#lines.i,File>`_. + ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices + ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## - ## .. code-block:: nim + ## Example: + ## ```nim ## for line in lines(memfiles.open("foo")): ## echo line + ## ``` - var buf = TaintedString(newStringOfCap(80)) + var buf = newStringOfCap(80) for line in lines(mfile, buf, delim, eat): yield buf + +type + MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile` + MemMapFileStreamObj* = object of Stream + mf: MemFile + mode: FileMode + pos: ByteAddress + +proc mmsClose(s: Stream) = + MemMapFileStream(s).pos = -1 + close(MemMapFileStream(s).mf) + +proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf) + +proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or + (MemMapFileStream(s).pos < 0) + +proc mmsSetPosition(s: Stream, pos: int) = + if pos > MemMapFileStream(s).mf.size or pos < 0: + raise newEIO("cannot set pos in stream") + MemMapFileStream(s).pos = pos + +proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos + +proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) + let l = min(bufLen, MemMapFileStream(s).mf.size - p) + moveMem(buffer, cast[pointer](startAddress + p), l) + result = l + +proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = mmsPeekData(s, buffer, bufLen) + inc(MemMapFileStream(s).pos, result) + +proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = + if MemMapFileStream(s).mode == fmRead: + raise newEIO("cannot write to read-only stream") + let size = MemMapFileStream(s).mf.size + if MemMapFileStream(s).pos + bufLen > size: + raise newEIO("cannot write to stream") + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) + moveMem(cast[pointer](p), buffer, bufLen) + inc(MemMapFileStream(s).pos, bufLen) + +proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, + fileSize: int = -1): MemMapFileStream = + ## creates a new stream from the file named `filename` with the mode `mode`. + ## Raises ## `OSError` if the file cannot be opened. See the `system + ## <system.html>`_ module for a list of available FileMode enums. + ## `fileSize` can only be set if the file does not exist and is opened + ## with write access (e.g., with fmReadWrite). + var mf: MemFile = open(filename, mode, newFileSize = fileSize) + new(result) + result.mode = mode + result.mf = mf + result.closeImpl = mmsClose + result.atEndImpl = mmsAtEnd + result.setPositionImpl = mmsSetPosition + result.getPositionImpl = mmsGetPosition + result.readDataImpl = mmsReadData + result.peekDataImpl = mmsPeekData + result.writeDataImpl = mmsWriteData + result.flushImpl = mmsFlush diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim deleted file mode 100644 index c8090dc6a..000000000 --- a/lib/pure/mersenne.nim +++ /dev/null @@ -1,39 +0,0 @@ -import unsigned - -type - MersenneTwister* = object - mt: array[0..623, uint32] - index: int - -{.deprecated: [TMersenneTwister: MersenneTwister].} - -proc newMersenneTwister*(seed: int): MersenneTwister = - result.index = 0 - result.mt[0]= uint32(seed) - for i in 1..623'u32: - result.mt[i]= (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) - -proc generateNumbers(m: var MersenneTwister) = - for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) - m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) - if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 - -proc getNum*(m: var MersenneTwister): int = - if m.index == 0: - generateNumbers(m) - var y = m.mt[m.index] - y = y xor (y shr 11'u32) - y = y xor ((7'u32 shl y) and 0x9d2c5680'u32) - y = y xor ((15'u32 shl y) and 0xefc60000'u32) - y = y xor (y shr 18'u32) - m.index = (m.index+1) mod 624 - return int(y) - -# Test -when not defined(testing) and isMainModule: - var mt = newMersenneTwister(2525) - - for i in 0..99: - echo mt.getNum diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 1e315afb4..ff639e8e5 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -8,515 +8,1064 @@ # ## This module implements a mimetypes database -import strtabs + +runnableExamples: + var m = newMimetypes() + doAssert m.getMimetype("mp4") == "video/mp4" + doAssert m.getExt("text/html") == "html" + ## Values can be uppercase too. + doAssert m.getMimetype("MP4") == "video/mp4" + doAssert m.getExt("TEXT/HTML") == "html" + ## If values are invalid then `default` is returned. + doAssert m.getMimetype("INVALID") == "text/plain" + doAssert m.getExt("INVALID/NONEXISTENT") == "txt" + doAssert m.getMimetype("") == "text/plain" + doAssert m.getExt("") == "txt" + ## Register new Mimetypes. + m.register(ext = "fakext", mimetype = "text/fakelang") + doAssert m.getMimetype("fakext") == "text/fakelang" + doAssert m.getMimetype("FaKeXT") == "text/fakelang" + +import std/tables +from std/strutils import startsWith, toLowerAscii, strip + +when defined(nimPreviewSlimSystem): + import std/assertions + + type MimeDB* = object - mimes: StringTableRef - -{.deprecated: [TMimeDB: MimeDB].} + mimes: OrderedTable[string, string] const mimes* = { - "ez": "application/andrew-inset", - "anx": "application/annodex", - "atom": "application/atom+xml", - "atomcat": "application/atomcat+xml", - "atomsrv": "application/atomserv+xml", - "lin": "application/bbolin", - "cap": "application/cap", - "pcap": "application/cap", - "cu": "application/cu-seeme", - "davmount": "application/davmount+xml", - "tsp": "application/dsptype", - "es": "application/ecmascript", - "spl": "application/futuresplash", - "hta": "application/hta", - "jar": "application/java-archive", - "ser": "application/java-serialized-object", - "class": "application/java-vm", - "js": "application/javascript", - "m3g": "application/m3g", - "hqx": "application/mac-binhex40", - "cpt": "application/mac-compactpro", - "nb": "application/mathematica", - "nbp": "application/mathematica", - "mdb": "application/msaccess", - "doc": "application/msword", - "dot": "application/msword", - "mxf": "application/mxf", - "bin": "application/octet-stream", - "oda": "application/oda", - "ogx": "application/ogg", - "pdf": "application/pdf", - "key": "application/pgp-keys", - "pgp": "application/pgp-signature", - "prf": "application/pics-rules", - "ps": "application/postscript", - "ai": "application/postscript", - "eps": "application/postscript", - "epsi": "application/postscript", - "epsf": "application/postscript", - "eps2": "application/postscript", - "eps3": "application/postscript", - "rar": "application/rar", - "rdf": "application/rdf+xml", - "rss": "application/rss+xml", - "rtf": "application/rtf", - "smi": "application/smil", - "smil": "application/smil", - "xhtml": "application/xhtml+xml", - "xht": "application/xhtml+xml", - "xml": "application/xml", - "xsl": "application/xml", - "xsd": "application/xml", - "xspf": "application/xspf+xml", - "zip": "application/zip", - "apk": "application/vnd.android.package-archive", - "cdy": "application/vnd.cinderella", - "kml": "application/vnd.google-earth.kml+xml", - "kmz": "application/vnd.google-earth.kmz", - "xul": "application/vnd.mozilla.xul+xml", - "xls": "application/vnd.ms-excel", - "xlb": "application/vnd.ms-excel", - "xlt": "application/vnd.ms-excel", - "cat": "application/vnd.ms-pki.seccat", - "stl": "application/vnd.ms-pki.stl", - "ppt": "application/vnd.ms-powerpoint", - "pps": "application/vnd.ms-powerpoint", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", - "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", - "cod": "application/vnd.rim.cod", - "mmf": "application/vnd.smaf", - "sis": "application/vnd.symbian.install", - "vsd": "application/vnd.visio", - "wbxml": "application/vnd.wap.wbxml", - "wmlc": "application/vnd.wap.wmlc", - "wmlsc": "application/vnd.wap.wmlscriptc", - "wpd": "application/vnd.wordperfect", - "wp5": "application/vnd.wordperfect5.1", - "wk": "application/x-123", - "7z": "application/x-7z-compressed", - "abw": "application/x-abiword", - "dmg": "application/x-apple-diskimage", - "bcpio": "application/x-bcpio", - "torrent": "application/x-bittorrent", - "cab": "application/x-cab", - "cbr": "application/x-cbr", - "cbz": "application/x-cbz", - "cdf": "application/x-cdf", - "cda": "application/x-cdf", - "vcd": "application/x-cdlink", - "pgn": "application/x-chess-pgn", - "cpio": "application/x-cpio", - "csh": "application/x-csh", - "deb": "application/x-debian-package", - "udeb": "application/x-debian-package", - "dcr": "application/x-director", - "dir": "application/x-director", - "dxr": "application/x-director", - "dms": "application/x-dms", - "wad": "application/x-doom", - "dvi": "application/x-dvi", - "rhtml": "application/x-httpd-eruby", - "pfa": "application/x-font", - "pfb": "application/x-font", - "gsf": "application/x-font", - "pcf": "application/x-font", - "pcf.Z": "application/x-font", - "mm": "application/x-freemind", - "spl": "application/x-futuresplash", - "gnumeric": "application/x-gnumeric", - "sgf": "application/x-go-sgf", - "gcf": "application/x-graphing-calculator", - "gtar": "application/x-gtar", - "tgz": "application/x-gtar", - "taz": "application/x-gtar", - "hdf": "application/x-hdf", - "phtml": "application/x-httpd-php", - "pht": "application/x-httpd-php", - "php": "application/x-httpd-php", - "phps": "application/x-httpd-php-source", - "php3": "application/x-httpd-php3", - "php3p": "application/x-httpd-php3-preprocessed", - "php4": "application/x-httpd-php4", - "php5": "application/x-httpd-php5", - "ica": "application/x-ica", - "info": "application/x-info", - "ins": "application/x-internet-signup", - "isp": "application/x-internet-signup", - "iii": "application/x-iphone", - "iso": "application/x-iso9660-image", - "jam": "application/x-jam", - "jnlp": "application/x-java-jnlp-file", - "jmz": "application/x-jmol", - "chrt": "application/x-kchart", - "kil": "application/x-killustrator", - "skp": "application/x-koan", - "skd": "application/x-koan", - "skt": "application/x-koan", - "skm": "application/x-koan", - "kpr": "application/x-kpresenter", - "kpt": "application/x-kpresenter", - "ksp": "application/x-kspread", - "kwd": "application/x-kword", - "kwt": "application/x-kword", - "latex": "application/x-latex", - "lha": "application/x-lha", - "lyx": "application/x-lyx", - "lzh": "application/x-lzh", - "lzx": "application/x-lzx", - "frm": "application/x-maker", - "maker": "application/x-maker", - "frame": "application/x-maker", - "fm": "application/x-maker", - "fb": "application/x-maker", - "book": "application/x-maker", - "fbdoc": "application/x-maker", - "mif": "application/x-mif", - "wmd": "application/x-ms-wmd", - "wmz": "application/x-ms-wmz", - "com": "application/x-msdos-program", - "exe": "application/x-msdos-program", - "bat": "application/x-msdos-program", - "dll": "application/x-msdos-program", - "msi": "application/x-msi", - "nc": "application/x-netcdf", - "pac": "application/x-ns-proxy-autoconfig", - "dat": "application/x-ns-proxy-autoconfig", - "nwc": "application/x-nwc", - "o": "application/x-object", - "oza": "application/x-oz-application", - "p7r": "application/x-pkcs7-certreqresp", - "crl": "application/x-pkcs7-crl", - "pyc": "application/x-python-code", - "pyo": "application/x-python-code", - "qgs": "application/x-qgis", - "shp": "application/x-qgis", - "shx": "application/x-qgis", - "qtl": "application/x-quicktimeplayer", - "rpm": "application/x-redhat-package-manager", - "rb": "application/x-ruby", - "sh": "application/x-sh", - "shar": "application/x-shar", - "swf": "application/x-shockwave-flash", - "swfl": "application/x-shockwave-flash", - "scr": "application/x-silverlight", - "sit": "application/x-stuffit", - "sitx": "application/x-stuffit", - "sv4cpio": "application/x-sv4cpio", - "sv4crc": "application/x-sv4crc", - "tar": "application/x-tar", - "tcl": "application/x-tcl", - "gf": "application/x-tex-gf", - "pk": "application/x-tex-pk", - "texinfo": "application/x-texinfo", - "texi": "application/x-texinfo", - "~": "application/x-trash", - "%": "application/x-trash", - "bak": "application/x-trash", - "old": "application/x-trash", - "sik": "application/x-trash", - "t": "application/x-troff", - "tr": "application/x-troff", - "roff": "application/x-troff", - "man": "application/x-troff-man", - "me": "application/x-troff-me", - "ms": "application/x-troff-ms", - "ustar": "application/x-ustar", - "src": "application/x-wais-source", - "wz": "application/x-wingz", - "crt": "application/x-x509-ca-cert", - "xcf": "application/x-xcf", - "fig": "application/x-xfig", - "xpi": "application/x-xpinstall", - "amr": "audio/amr", - "awb": "audio/amr-wb", - "amr": "audio/amr", - "awb": "audio/amr-wb", - "axa": "audio/annodex", - "au": "audio/basic", - "snd": "audio/basic", - "flac": "audio/flac", - "mid": "audio/midi", - "midi": "audio/midi", - "kar": "audio/midi", - "mpga": "audio/mpeg", - "mpega": "audio/mpeg", - "mp2": "audio/mpeg", - "mp3": "audio/mpeg", - "m4a": "audio/mpeg", - "m3u": "audio/mpegurl", - "oga": "audio/ogg", - "ogg": "audio/ogg", - "spx": "audio/ogg", - "sid": "audio/prs.sid", - "aif": "audio/x-aiff", - "aiff": "audio/x-aiff", - "aifc": "audio/x-aiff", - "gsm": "audio/x-gsm", - "m3u": "audio/x-mpegurl", - "wma": "audio/x-ms-wma", - "wax": "audio/x-ms-wax", - "ra": "audio/x-pn-realaudio", - "rm": "audio/x-pn-realaudio", - "ram": "audio/x-pn-realaudio", - "ra": "audio/x-realaudio", - "pls": "audio/x-scpls", - "sd2": "audio/x-sd2", - "wav": "audio/x-wav", - "alc": "chemical/x-alchemy", - "cac": "chemical/x-cache", - "cache": "chemical/x-cache", - "csf": "chemical/x-cache-csf", - "cbin": "chemical/x-cactvs-binary", - "cascii": "chemical/x-cactvs-binary", - "ctab": "chemical/x-cactvs-binary", - "cdx": "chemical/x-cdx", - "cer": "chemical/x-cerius", - "c3d": "chemical/x-chem3d", - "chm": "chemical/x-chemdraw", - "cif": "chemical/x-cif", - "cmdf": "chemical/x-cmdf", - "cml": "chemical/x-cml", - "cpa": "chemical/x-compass", - "bsd": "chemical/x-crossfire", - "csml": "chemical/x-csml", - "csm": "chemical/x-csml", - "ctx": "chemical/x-ctx", - "cxf": "chemical/x-cxf", - "cef": "chemical/x-cxf", - "smi": "#chemical/x-daylight-smiles", - "emb": "chemical/x-embl-dl-nucleotide", - "embl": "chemical/x-embl-dl-nucleotide", - "spc": "chemical/x-galactic-spc", - "inp": "chemical/x-gamess-input", - "gam": "chemical/x-gamess-input", - "gamin": "chemical/x-gamess-input", - "fch": "chemical/x-gaussian-checkpoint", - "fchk": "chemical/x-gaussian-checkpoint", - "cub": "chemical/x-gaussian-cube", - "gau": "chemical/x-gaussian-input", - "gjc": "chemical/x-gaussian-input", - "gjf": "chemical/x-gaussian-input", - "gal": "chemical/x-gaussian-log", - "gcg": "chemical/x-gcg8-sequence", - "gen": "chemical/x-genbank", - "hin": "chemical/x-hin", - "istr": "chemical/x-isostar", - "ist": "chemical/x-isostar", - "jdx": "chemical/x-jcamp-dx", - "dx": "chemical/x-jcamp-dx", - "kin": "chemical/x-kinemage", - "mcm": "chemical/x-macmolecule", - "mmd": "chemical/x-macromodel-input", - "mmod": "chemical/x-macromodel-input", - "mol": "chemical/x-mdl-molfile", - "rd": "chemical/x-mdl-rdfile", - "rxn": "chemical/x-mdl-rxnfile", - "sd": "chemical/x-mdl-sdfile", - "sdf": "chemical/x-mdl-sdfile", - "tgf": "chemical/x-mdl-tgf", - "mif": "#chemical/x-mif", - "mcif": "chemical/x-mmcif", - "mol2": "chemical/x-mol2", - "b": "chemical/x-molconn-Z", - "gpt": "chemical/x-mopac-graph", - "mop": "chemical/x-mopac-input", - "mopcrt": "chemical/x-mopac-input", - "mpc": "chemical/x-mopac-input", - "zmt": "chemical/x-mopac-input", - "moo": "chemical/x-mopac-out", - "mvb": "chemical/x-mopac-vib", - "asn": "chemical/x-ncbi-asn1", - "prt": "chemical/x-ncbi-asn1-ascii", - "ent": "chemical/x-ncbi-asn1-ascii", - "val": "chemical/x-ncbi-asn1-binary", - "aso": "chemical/x-ncbi-asn1-binary", - "asn": "chemical/x-ncbi-asn1-spec", - "pdb": "chemical/x-pdb", - "ent": "chemical/x-pdb", - "ros": "chemical/x-rosdal", - "sw": "chemical/x-swissprot", - "vms": "chemical/x-vamas-iso14976", - "vmd": "chemical/x-vmd", - "xtel": "chemical/x-xtel", - "xyz": "chemical/x-xyz", - "gif": "image/gif", - "ief": "image/ief", - "jpeg": "image/jpeg", - "jpg": "image/jpeg", - "jpe": "image/jpeg", - "pcx": "image/pcx", - "png": "image/png", - "svg": "image/svg+xml", - "svgz": "image/svg+xml", - "tiff": "image/tiff", - "tif": "image/tiff", - "djvu": "image/vnd.djvu", - "djv": "image/vnd.djvu", - "wbmp": "image/vnd.wap.wbmp", - "cr2": "image/x-canon-cr2", - "crw": "image/x-canon-crw", - "ras": "image/x-cmu-raster", - "cdr": "image/x-coreldraw", - "pat": "image/x-coreldrawpattern", - "cdt": "image/x-coreldrawtemplate", - "cpt": "image/x-corelphotopaint", - "erf": "image/x-epson-erf", - "ico": "image/x-icon", - "art": "image/x-jg", - "jng": "image/x-jng", - "bmp": "image/x-ms-bmp", - "nef": "image/x-nikon-nef", - "orf": "image/x-olympus-orf", - "psd": "image/x-photoshop", - "pnm": "image/x-portable-anymap", - "pbm": "image/x-portable-bitmap", - "pgm": "image/x-portable-graymap", - "ppm": "image/x-portable-pixmap", - "rgb": "image/x-rgb", - "xbm": "image/x-xbitmap", - "xpm": "image/x-xpixmap", - "xwd": "image/x-xwindowdump", - "eml": "message/rfc822", - "igs": "model/iges", - "iges": "model/iges", - "msh": "model/mesh", - "mesh": "model/mesh", - "silo": "model/mesh", - "wrl": "model/vrml", - "vrml": "model/vrml", - "x3dv": "model/x3d+vrml", - "x3d": "model/x3d+xml", - "x3db": "model/x3d+binary", - "manifest": "text/cache-manifest", - "ics": "text/calendar", - "icz": "text/calendar", - "css": "text/css", - "csv": "text/csv", - "323": "text/h323", - "html": "text/html", - "htm": "text/html", - "shtml": "text/html", - "uls": "text/iuls", - "mml": "text/mathml", - "asc": "text/plain", - "txt": "text/plain", - "text": "text/plain", - "pot": "text/plain", - "brf": "text/plain", - "rtx": "text/richtext", - "sct": "text/scriptlet", - "wsc": "text/scriptlet", - "tm": "text/texmacs", - "ts": "text/texmacs", - "tsv": "text/tab-separated-values", - "jad": "text/vnd.sun.j2me.app-descriptor", - "wml": "text/vnd.wap.wml", - "wmls": "text/vnd.wap.wmlscript", - "bib": "text/x-bibtex", - "boo": "text/x-boo", - "h++": "text/x-c++hdr", - "hpp": "text/x-c++hdr", - "hxx": "text/x-c++hdr", - "hh": "text/x-c++hdr", - "c++": "text/x-c++src", - "cpp": "text/x-c++src", - "cxx": "text/x-c++src", - "cc": "text/x-c++src", - "h": "text/x-chdr", - "htc": "text/x-component", - "csh": "text/x-csh", - "c": "text/x-csrc", - "d": "text/x-dsrc", - "diff": "text/x-diff", - "patch": "text/x-diff", - "hs": "text/x-haskell", - "java": "text/x-java", - "lhs": "text/x-literate-haskell", - "moc": "text/x-moc", - "p": "text/x-pascal", - "pas": "text/x-pascal", - "gcd": "text/x-pcs-gcd", - "pl": "text/x-perl", - "pm": "text/x-perl", - "py": "text/x-python", - "scala": "text/x-scala", - "etx": "text/x-setext", - "sh": "text/x-sh", - "tcl": "text/x-tcl", - "tk": "text/x-tcl", - "tex": "text/x-tex", - "ltx": "text/x-tex", - "sty": "text/x-tex", - "cls": "text/x-tex", - "vcs": "text/x-vcalendar", - "vcf": "text/x-vcard", - "3gp": "video/3gpp", - "axv": "video/annodex", - "dl": "video/dl", - "dif": "video/dv", - "dv": "video/dv", - "fli": "video/fli", - "gl": "video/gl", - "mpeg": "video/mpeg", - "mpg": "video/mpeg", - "mpe": "video/mpeg", - "mp4": "video/mp4", - "qt": "video/quicktime", - "mov": "video/quicktime", - "ogv": "video/ogg", - "mxu": "video/vnd.mpegurl", - "flv": "video/x-flv", - "lsf": "video/x-la-asf", - "lsx": "video/x-la-asf", - "mng": "video/x-mng", - "asf": "video/x-ms-asf", - "asx": "video/x-ms-asf", - "wm": "video/x-ms-wm", - "wmv": "video/x-ms-wmv", - "wmx": "video/x-ms-wmx", - "wvx": "video/x-ms-wvx", - "avi": "video/x-msvideo", - "movie": "video/x-sgi-movie", - "mpv": "video/x-matroska", - "mkv": "video/x-matroska", - "ice": "x-conference/x-cooltalk", - "sisx": "x-epoc/x-sisx-app", - "vrm": "x-world/x-vrml", - "vrml": "x-world/x-vrml", - "wrl": "x-world/x-vrml"} + "ez": "application/andrew-inset", + "aw": "application/applixware", + "atom": "application/atom+xml", + "atomcat": "application/atomcat+xml", + "atomsvc": "application/atomsvc+xml", + "ccxml": "application/ccxml+xml", + "cdmia": "application/cdmi-capability", + "cdmic": "application/cdmi-container", + "cdmid": "application/cdmi-domain", + "cdmio": "application/cdmi-object", + "cdmiq": "application/cdmi-queue", + "cu": "application/cu-seeme", + "davmount": "application/davmount+xml", + "dbk": "application/docbook+xml", + "dssc": "application/dssc+der", + "xdssc": "application/dssc+xml", + "ecma": "application/ecmascript", + "emma": "application/emma+xml", + "epub": "application/epub+zip", + "exi": "application/exi", + "pfr": "application/font-tdpfr", + "gml": "application/gml+xml", + "gpx": "application/gpx+xml", + "gxf": "application/gxf", + "stk": "application/hyperstudio", + "ink": "application/inkml+xml", + "inkml": "application/inkml+xml", + "ipfix": "application/ipfix", + "jar": "application/java-archive", + "ser": "application/java-serialized-object", + "class": "application/java-vm", + "json": "application/json", + "jsonml": "application/jsonml+json", + "lostxml": "application/lost+xml", + "hqx": "application/mac-binhex40", + "cpt": "application/mac-compactpro", + "mads": "application/mads+xml", + "mrc": "application/marc", + "mrcx": "application/marcxml+xml", + "ma": "application/mathematica", + "nb": "application/mathematica", + "mb": "application/mathematica", + "mathml": "application/mathml+xml", + "mbox": "application/mbox", + "mscml": "application/mediaservercontrol+xml", + "metalink": "application/metalink+xml", + "meta4": "application/metalink4+xml", + "mets": "application/mets+xml", + "mods": "application/mods+xml", + "m21": "application/mp21", + "mp21": "application/mp21", + "mp4s": "application/mp4", + "doc": "application/msword", + "dot": "application/msword", + "mxf": "application/mxf", + "bin": "application/octet-stream", + "dms": "application/octet-stream", + "lrf": "application/octet-stream", + "mar": "application/octet-stream", + "so": "application/octet-stream", + "dist": "application/octet-stream", + "distz": "application/octet-stream", + "pkg": "application/octet-stream", + "bpk": "application/octet-stream", + "dump": "application/octet-stream", + "elc": "application/octet-stream", + "deploy": "application/octet-stream", + "oda": "application/oda", + "opf": "application/oebps-package+xml", + "ogx": "application/ogg", + "omdoc": "application/omdoc+xml", + "onetoc": "application/onenote", + "onetoc2": "application/onenote", + "onetmp": "application/onenote", + "onepkg": "application/onenote", + "oxps": "application/oxps", + "xer": "application/patch-ops-error+xml", + "pdf": "application/pdf", + "pgp": "application/pgp-encrypted", + "asc": "application/pgp-signature", + "sig": "application/pgp-signature", + "prf": "application/pics-rules", + "p10": "application/pkcs10", + "p7m": "application/pkcs7-mime", + "p7c": "application/pkcs7-mime", + "p7s": "application/pkcs7-signature", + "p8": "application/pkcs8", + "ac": "application/pkix-attr-cert", + "cer": "application/pkix-cert", + "crl": "application/pkix-crl", + "pkipath": "application/pkix-pkipath", + "pki": "application/pkixcmp", + "pls": "application/pls+xml", + "ai": "application/postscript", + "eps": "application/postscript", + "ps": "application/postscript", + "cww": "application/prs.cww", + "pskcxml": "application/pskc+xml", + "rdf": "application/rdf+xml", + "rif": "application/reginfo+xml", + "rnc": "application/relax-ng-compact-syntax", + "rl": "application/resource-lists+xml", + "rld": "application/resource-lists-diff+xml", + "rs": "application/rls-services+xml", + "gbr": "application/rpki-ghostbusters", + "mft": "application/rpki-manifest", + "roa": "application/rpki-roa", + "rsd": "application/rsd+xml", + "rss": "application/rss+xml", + "rtf": "application/rtf", + "sbml": "application/sbml+xml", + "scq": "application/scvp-cv-request", + "scs": "application/scvp-cv-response", + "spq": "application/scvp-vp-request", + "spp": "application/scvp-vp-response", + "sdp": "application/sdp", + "setpay": "application/set-payment-initiation", + "setreg": "application/set-registration-initiation", + "shf": "application/shf+xml", + "smi": "application/smil+xml", + "smil": "application/smil+xml", + "rq": "application/sparql-query", + "srx": "application/sparql-results+xml", + "gram": "application/srgs", + "grxml": "application/srgs+xml", + "sru": "application/sru+xml", + "ssdl": "application/ssdl+xml", + "ssml": "application/ssml+xml", + "tei": "application/tei+xml", + "teicorpus": "application/tei+xml", + "tfi": "application/thraud+xml", + "tsd": "application/timestamped-data", + "plb": "application/vnd.3gpp.pic-bw-large", + "psb": "application/vnd.3gpp.pic-bw-small", + "pvb": "application/vnd.3gpp.pic-bw-var", + "tcap": "application/vnd.3gpp2.tcap", + "pwn": "application/vnd.3m.post-it-notes", + "aso": "application/vnd.accpac.simply.aso", + "imp": "application/vnd.accpac.simply.imp", + "acu": "application/vnd.acucobol", + "atc": "application/vnd.acucorp", + "acutc": "application/vnd.acucorp", + "air": "application/vnd.adobe.air-application-installer-package+zip", + "fcdt": "application/vnd.adobe.formscentral.fcdt", + "fxp": "application/vnd.adobe.fxp", + "fxpl": "application/vnd.adobe.fxp", + "xdp": "application/vnd.adobe.xdp+xml", + "xfdf": "application/vnd.adobe.xfdf", + "ahead": "application/vnd.ahead.space", + "azf": "application/vnd.airzip.filesecure.azf", + "azs": "application/vnd.airzip.filesecure.azs", + "azw": "application/vnd.amazon.ebook", + "acc": "application/vnd.americandynamics.acc", + "ami": "application/vnd.amiga.ami", + "apk": "application/vnd.android.package-archive", + "cii": "application/vnd.anser-web-certificate-issue-initiation", + "fti": "application/vnd.anser-web-funds-transfer-initiation", + "atx": "application/vnd.antix.game-component", + "mpkg": "application/vnd.apple.installer+xml", + "m3u8": "application/vnd.apple.mpegurl", + "swi": "application/vnd.aristanetworks.swi", + "iota": "application/vnd.astraea-software.iota", + "aep": "application/vnd.audiograph", + "mpm": "application/vnd.blueice.multipass", + "bmi": "application/vnd.bmi", + "rep": "application/vnd.businessobjects", + "cdxml": "application/vnd.chemdraw+xml", + "mmd": "application/vnd.chipnuts.karaoke-mmd", + "cdy": "application/vnd.cinderella", + "cla": "application/vnd.claymore", + "rp9": "application/vnd.cloanto.rp9", + "c4g": "application/vnd.clonk.c4group", + "c4d": "application/vnd.clonk.c4group", + "c4f": "application/vnd.clonk.c4group", + "c4p": "application/vnd.clonk.c4group", + "c4u": "application/vnd.clonk.c4group", + "c11amc": "application/vnd.cluetrust.cartomobile-config", + "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", + "csp": "application/vnd.commonspace", + "cdbcmsg": "application/vnd.contact.cmsg", + "cmc": "application/vnd.cosmocaller", + "clkx": "application/vnd.crick.clicker", + "clkk": "application/vnd.crick.clicker.keyboard", + "clkp": "application/vnd.crick.clicker.palette", + "clkt": "application/vnd.crick.clicker.template", + "clkw": "application/vnd.crick.clicker.wordbank", + "wbs": "application/vnd.criticaltools.wbs+xml", + "pml": "application/vnd.ctc-posml", + "ppd": "application/vnd.cups-ppd", + "car": "application/vnd.curl.car", + "pcurl": "application/vnd.curl.pcurl", + "dart": "application/vnd.dart", + "rdz": "application/vnd.data-vision.rdz", + "uvf": "application/vnd.dece.data", + "uvvf": "application/vnd.dece.data", + "uvd": "application/vnd.dece.data", + "uvvd": "application/vnd.dece.data", + "uvt": "application/vnd.dece.ttml+xml", + "uvvt": "application/vnd.dece.ttml+xml", + "uvx": "application/vnd.dece.unspecified", + "uvvx": "application/vnd.dece.unspecified", + "uvz": "application/vnd.dece.zip", + "uvvz": "application/vnd.dece.zip", + "fe_launch": "application/vnd.denovo.fcselayout-link", + "dna": "application/vnd.dna", + "mlp": "application/vnd.dolby.mlp", + "dpg": "application/vnd.dpgraph", + "dfac": "application/vnd.dreamfactory", + "kpxx": "application/vnd.ds-keypoint", + "ait": "application/vnd.dvb.ait", + "svc": "application/vnd.dvb.service", + "geo": "application/vnd.dynageo", + "mag": "application/vnd.ecowin.chart", + "nml": "application/vnd.enliven", + "esf": "application/vnd.epson.esf", + "msf": "application/vnd.epson.msf", + "qam": "application/vnd.epson.quickanime", + "slt": "application/vnd.epson.salt", + "ssf": "application/vnd.epson.ssf", + "es3": "application/vnd.eszigno3+xml", + "et3": "application/vnd.eszigno3+xml", + "ez2": "application/vnd.ezpix-album", + "ez3": "application/vnd.ezpix-package", + "fdf": "application/vnd.fdf", + "mseed": "application/vnd.fdsn.mseed", + "seed": "application/vnd.fdsn.seed", + "dataless": "application/vnd.fdsn.seed", + "gph": "application/vnd.flographit", + "ftc": "application/vnd.fluxtime.clip", + "fm": "application/vnd.framemaker", + "frame": "application/vnd.framemaker", + "maker": "application/vnd.framemaker", + "book": "application/vnd.framemaker", + "fnc": "application/vnd.frogans.fnc", + "ltf": "application/vnd.frogans.ltf", + "fsc": "application/vnd.fsc.weblaunch", + "oas": "application/vnd.fujitsu.oasys", + "oa2": "application/vnd.fujitsu.oasys2", + "oa3": "application/vnd.fujitsu.oasys3", + "fg5": "application/vnd.fujitsu.oasysgp", + "bh2": "application/vnd.fujitsu.oasysprs", + "ddd": "application/vnd.fujixerox.ddd", + "xdw": "application/vnd.fujixerox.docuworks", + "xbd": "application/vnd.fujixerox.docuworks.binder", + "fzs": "application/vnd.fuzzysheet", + "txd": "application/vnd.genomatix.tuxedo", + "ggb": "application/vnd.geogebra.file", + "ggs": "application/vnd.geogebra.slides", + "ggt": "application/vnd.geogebra.tool", + "gex": "application/vnd.geometry-explorer", + "gre": "application/vnd.geometry-explorer", + "gxt": "application/vnd.geonext", + "g2w": "application/vnd.geoplan", + "g3w": "application/vnd.geospace", + "gmx": "application/vnd.gmx", + "kml": "application/vnd.google-earth.kml+xml", + "kmz": "application/vnd.google-earth.kmz", + "gqf": "application/vnd.grafeq", + "gqs": "application/vnd.grafeq", + "gac": "application/vnd.groove-account", + "ghf": "application/vnd.groove-help", + "gim": "application/vnd.groove-identity-message", + "grv": "application/vnd.groove-injector", + "gtm": "application/vnd.groove-tool-message", + "tpl": "application/vnd.groove-tool-template", + "vcg": "application/vnd.groove-vcard", + "hal": "application/vnd.hal+xml", + "zmm": "application/vnd.handheld-entertainment+xml", + "hbci": "application/vnd.hbci", + "les": "application/vnd.hhe.lesson-player", + "hpgl": "application/vnd.hp-hpgl", + "hpid": "application/vnd.hp-hpid", + "hps": "application/vnd.hp-hps", + "jlt": "application/vnd.hp-jlyt", + "pcl": "application/vnd.hp-pcl", + "pclxl": "application/vnd.hp-pclxl", + "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "mpy": "application/vnd.ibm.minipay", + "afp": "application/vnd.ibm.modcap", + "listafp": "application/vnd.ibm.modcap", + "list3820": "application/vnd.ibm.modcap", + "irm": "application/vnd.ibm.rights-management", + "sc": "application/vnd.ibm.secure-container", + "icc": "application/vnd.iccprofile", + "icm": "application/vnd.iccprofile", + "igl": "application/vnd.igloader", + "ivp": "application/vnd.immervision-ivp", + "ivu": "application/vnd.immervision-ivu", + "igm": "application/vnd.insors.igm", + "xpw": "application/vnd.intercon.formnet", + "xpx": "application/vnd.intercon.formnet", + "i2g": "application/vnd.intergeo", + "qbo": "application/vnd.intu.qbo", + "qfx": "application/vnd.intu.qfx", + "rcprofile": "application/vnd.ipunplugged.rcprofile", + "irp": "application/vnd.irepository.package+xml", + "xpr": "application/vnd.is-xpr", + "fcs": "application/vnd.isac.fcs", + "jam": "application/vnd.jam", + "rms": "application/vnd.jcp.javame.midlet-rms", + "jisp": "application/vnd.jisp", + "joda": "application/vnd.joost.joda-archive", + "ktz": "application/vnd.kahootz", + "ktr": "application/vnd.kahootz", + "karbon": "application/vnd.kde.karbon", + "chrt": "application/vnd.kde.kchart", + "kfo": "application/vnd.kde.kformula", + "flw": "application/vnd.kde.kivio", + "kon": "application/vnd.kde.kontour", + "kpr": "application/vnd.kde.kpresenter", + "kpt": "application/vnd.kde.kpresenter", + "ksp": "application/vnd.kde.kspread", + "kwd": "application/vnd.kde.kword", + "kwt": "application/vnd.kde.kword", + "htke": "application/vnd.kenameaapp", + "kia": "application/vnd.kidspiration", + "kne": "application/vnd.kinar", + "knp": "application/vnd.kinar", + "skp": "application/vnd.koan", + "skd": "application/vnd.koan", + "skt": "application/vnd.koan", + "skm": "application/vnd.koan", + "sse": "application/vnd.kodak-descriptor", + "lasxml": "application/vnd.las.las+xml", + "lbd": "application/vnd.llamagraphics.life-balance.desktop", + "lbe": "application/vnd.llamagraphics.life-balance.exchange+xml", + "123": "application/vnd.lotus-1-2-3", + "apr": "application/vnd.lotus-approach", + "pre": "application/vnd.lotus-freelance", + "nsf": "application/vnd.lotus-notes", + "org": "application/vnd.lotus-organizer", + "scm": "application/vnd.lotus-screencam", + "lwp": "application/vnd.lotus-wordpro", + "portpkg": "application/vnd.macports.portpkg", + "mcd": "application/vnd.mcd", + "mc1": "application/vnd.medcalcdata", + "cdkey": "application/vnd.mediastation.cdkey", + "mwf": "application/vnd.mfer", + "mfm": "application/vnd.mfmp", + "flo": "application/vnd.micrografx.flo", + "igx": "application/vnd.micrografx.igx", + "mif": "application/vnd.mif", + "daf": "application/vnd.mobius.daf", + "dis": "application/vnd.mobius.dis", + "mbk": "application/vnd.mobius.mbk", + "mqy": "application/vnd.mobius.mqy", + "msl": "application/vnd.mobius.msl", + "plc": "application/vnd.mobius.plc", + "txf": "application/vnd.mobius.txf", + "mpn": "application/vnd.mophun.application", + "mpc": "application/vnd.mophun.certificate", + "xul": "application/vnd.mozilla.xul+xml", + "cil": "application/vnd.ms-artgalry", + "cab": "application/vnd.ms-cab-compressed", + "xls": "application/vnd.ms-excel", + "xlm": "application/vnd.ms-excel", + "xla": "application/vnd.ms-excel", + "xlc": "application/vnd.ms-excel", + "xlt": "application/vnd.ms-excel", + "xlw": "application/vnd.ms-excel", + "xlam": "application/vnd.ms-excel.addin.macroenabled.12", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", + "xltm": "application/vnd.ms-excel.template.macroenabled.12", + "eot": "application/vnd.ms-fontobject", + "chm": "application/vnd.ms-htmlhelp", + "ims": "application/vnd.ms-ims", + "lrm": "application/vnd.ms-lrm", + "thmx": "application/vnd.ms-officetheme", + "cat": "application/vnd.ms-pki.seccat", + "stl": "application/vnd.ms-pki.stl", + "ppt": "application/vnd.ms-powerpoint", + "pps": "application/vnd.ms-powerpoint", + "pot": "application/vnd.ms-powerpoint", + "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", + "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", + "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", + "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", + "mpp": "application/vnd.ms-project", + "mpt": "application/vnd.ms-project", + "docm": "application/vnd.ms-word.document.macroenabled.12", + "dotm": "application/vnd.ms-word.template.macroenabled.12", + "wps": "application/vnd.ms-works", + "wks": "application/vnd.ms-works", + "wcm": "application/vnd.ms-works", + "wdb": "application/vnd.ms-works", + "wpl": "application/vnd.ms-wpl", + "xps": "application/vnd.ms-xpsdocument", + "mseq": "application/vnd.mseq", + "mus": "application/vnd.musician", + "msty": "application/vnd.muvee.style", + "taglet": "application/vnd.mynfc", + "nlu": "application/vnd.neurolanguage.nlu", + "nim": "text/nim", + "nimble": "text/nimble", + "nimf": "text/nim", + "nims": "text/nim", + "ntf": "application/vnd.nitf", + "nitf": "application/vnd.nitf", + "nnd": "application/vnd.noblenet-directory", + "nns": "application/vnd.noblenet-sealer", + "nnw": "application/vnd.noblenet-web", + "ngdat": "application/vnd.nokia.n-gage.data", + "n-gage": "application/vnd.nokia.n-gage.symbian.install", + "rpst": "application/vnd.nokia.radio-preset", + "rpss": "application/vnd.nokia.radio-presets", + "edm": "application/vnd.novadigm.edm", + "edx": "application/vnd.novadigm.edx", + "ext": "application/vnd.novadigm.ext", + "odc": "application/vnd.oasis.opendocument.chart", + "otc": "application/vnd.oasis.opendocument.chart-template", + "odb": "application/vnd.oasis.opendocument.database", + "odf": "application/vnd.oasis.opendocument.formula", + "odft": "application/vnd.oasis.opendocument.formula-template", + "odg": "application/vnd.oasis.opendocument.graphics", + "otg": "application/vnd.oasis.opendocument.graphics-template", + "odi": "application/vnd.oasis.opendocument.image", + "oti": "application/vnd.oasis.opendocument.image-template", + "odp": "application/vnd.oasis.opendocument.presentation", + "otp": "application/vnd.oasis.opendocument.presentation-template", + "ods": "application/vnd.oasis.opendocument.spreadsheet", + "ots": "application/vnd.oasis.opendocument.spreadsheet-template", + "odt": "application/vnd.oasis.opendocument.text", + "odm": "application/vnd.oasis.opendocument.text-master", + "ott": "application/vnd.oasis.opendocument.text-template", + "oth": "application/vnd.oasis.opendocument.text-web", + "xo": "application/vnd.olpc-sugar", + "dd2": "application/vnd.oma.dd2+xml", + "oxt": "application/vnd.openofficeorg.extension", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "mgp": "application/vnd.osgeo.mapguide.package", + "dp": "application/vnd.osgi.dp", + "esa": "application/vnd.osgi.subsystem", + "pdb": "application/vnd.palm", + "pqa": "application/vnd.palm", + "oprc": "application/vnd.palm", + "paw": "application/vnd.pawaafile", + "str": "application/vnd.pg.format", + "ei6": "application/vnd.pg.osasli", + "efif": "application/vnd.picsel", + "wg": "application/vnd.pmi.widget", + "plf": "application/vnd.pocketlearn", + "pbd": "application/vnd.powerbuilder6", + "box": "application/vnd.previewsystems.box", + "mgz": "application/vnd.proteus.magazine", + "qps": "application/vnd.publishare-delta-tree", + "ptid": "application/vnd.pvi.ptid1", + "qxd": "application/vnd.quark.quarkxpress", + "qxt": "application/vnd.quark.quarkxpress", + "qwd": "application/vnd.quark.quarkxpress", + "qwt": "application/vnd.quark.quarkxpress", + "qxl": "application/vnd.quark.quarkxpress", + "qxb": "application/vnd.quark.quarkxpress", + "bed": "application/vnd.realvnc.bed", + "mxl": "application/vnd.recordare.musicxml", + "musicxml": "application/vnd.recordare.musicxml+xml", + "cryptonote": "application/vnd.rig.cryptonote", + "cod": "application/vnd.rim.cod", + "rm": "application/vnd.rn-realmedia", + "rmvb": "application/vnd.rn-realmedia-vbr", + "link66": "application/vnd.route66.link66+xml", + "st": "application/vnd.sailingtracker.track", + "see": "application/vnd.seemail", + "sema": "application/vnd.sema", + "semd": "application/vnd.semd", + "semf": "application/vnd.semf", + "ifm": "application/vnd.shana.informed.formdata", + "itp": "application/vnd.shana.informed.formtemplate", + "iif": "application/vnd.shana.informed.interchange", + "ipk": "application/vnd.shana.informed.package", + "twd": "application/vnd.simtech-mindmapper", + "twds": "application/vnd.simtech-mindmapper", + "mmf": "application/vnd.smaf", + "teacher": "application/vnd.smart.teacher", + "sdkm": "application/vnd.solent.sdkm+xml", + "sdkd": "application/vnd.solent.sdkm+xml", + "dxp": "application/vnd.spotfire.dxp", + "sfs": "application/vnd.spotfire.sfs", + "sdc": "application/vnd.stardivision.calc", + "sda": "application/vnd.stardivision.draw", + "sdd": "application/vnd.stardivision.impress", + "smf": "application/vnd.stardivision.math", + "sdw": "application/vnd.stardivision.writer", + "vor": "application/vnd.stardivision.writer", + "sgl": "application/vnd.stardivision.writer-global", + "smzip": "application/vnd.stepmania.package", + "sm": "application/vnd.stepmania.stepchart", + "sxc": "application/vnd.sun.xml.calc", + "stc": "application/vnd.sun.xml.calc.template", + "sxd": "application/vnd.sun.xml.draw", + "std": "application/vnd.sun.xml.draw.template", + "sxi": "application/vnd.sun.xml.impress", + "sti": "application/vnd.sun.xml.impress.template", + "sxm": "application/vnd.sun.xml.math", + "sxw": "application/vnd.sun.xml.writer", + "sxg": "application/vnd.sun.xml.writer.global", + "stw": "application/vnd.sun.xml.writer.template", + "sus": "application/vnd.sus-calendar", + "susp": "application/vnd.sus-calendar", + "svd": "application/vnd.svd", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "xsm": "application/vnd.syncml+xml", + "bdm": "application/vnd.syncml.dm+wbxml", + "xdm": "application/vnd.syncml.dm+xml", + "tao": "application/vnd.tao.intent-module-archive", + "pcap": "application/vnd.tcpdump.pcap", + "cap": "application/vnd.tcpdump.pcap", + "dmp": "application/vnd.tcpdump.pcap", + "tmo": "application/vnd.tmobile-livetv", + "tpt": "application/vnd.trid.tpt", + "mxs": "application/vnd.triscape.mxs", + "tra": "application/vnd.trueapp", + "ufd": "application/vnd.ufdl", + "ufdl": "application/vnd.ufdl", + "utz": "application/vnd.uiq.theme", + "umj": "application/vnd.umajin", + "unityweb": "application/vnd.unity", + "uoml": "application/vnd.uoml+xml", + "vcx": "application/vnd.vcx", + "vsd": "application/vnd.visio", + "vst": "application/vnd.visio", + "vss": "application/vnd.visio", + "vsw": "application/vnd.visio", + "vis": "application/vnd.visionary", + "vsf": "application/vnd.vsf", + "wbxml": "application/vnd.wap.wbxml", + "wmlc": "application/vnd.wap.wmlc", + "wmlsc": "application/vnd.wap.wmlscriptc", + "wtb": "application/vnd.webturbo", + "nbp": "application/vnd.wolfram.player", + "wpd": "application/vnd.wordperfect", + "wqd": "application/vnd.wqd", + "stf": "application/vnd.wt.stf", + "xar": "application/vnd.xara", + "xfdl": "application/vnd.xfdl", + "hvd": "application/vnd.yamaha.hv-dic", + "hvs": "application/vnd.yamaha.hv-script", + "hvp": "application/vnd.yamaha.hv-voice", + "osf": "application/vnd.yamaha.openscoreformat", + "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", + "saf": "application/vnd.yamaha.smaf-audio", + "spf": "application/vnd.yamaha.smaf-phrase", + "cmp": "application/vnd.yellowriver-custom-menu", + "zir": "application/vnd.zul", + "zirz": "application/vnd.zul", + "zaz": "application/vnd.zzazz.deck+xml", + "vxml": "application/voicexml+xml", + "wasm": "application/wasm", + "wgt": "application/widget", + "hlp": "application/winhlp", + "wsdl": "application/wsdl+xml", + "wspolicy": "application/wspolicy+xml", + "7z": "application/x-7z-compressed", + "abw": "application/x-abiword", + "ace": "application/x-ace-compressed", + "dmg": "application/x-apple-diskimage", + "aab": "application/x-authorware-bin", + "x32": "application/x-authorware-bin", + "u32": "application/x-authorware-bin", + "vox": "application/x-authorware-bin", + "aam": "application/x-authorware-map", + "aas": "application/x-authorware-seg", + "bcpio": "application/x-bcpio", + "torrent": "application/x-bittorrent", + "blb": "application/x-blorb", + "blorb": "application/x-blorb", + "bz": "application/x-bzip", + "bz2": "application/x-bzip2", + "boz": "application/x-bzip2", + "cbr": "application/x-cbr", + "cba": "application/x-cbr", + "cbt": "application/x-cbr", + "cbz": "application/x-cbr", + "cb7": "application/x-cbr", + "vcd": "application/x-cdlink", + "cfs": "application/x-cfs-compressed", + "chat": "application/x-chat", + "pgn": "application/x-chess-pgn", + "nsc": "application/x-conference", + "cpio": "application/x-cpio", + "csh": "application/x-csh", + "deb": "application/x-debian-package", + "udeb": "application/x-debian-package", + "dgc": "application/x-dgc-compressed", + "dir": "application/x-director", + "dcr": "application/x-director", + "dxr": "application/x-director", + "cst": "application/x-director", + "cct": "application/x-director", + "cxt": "application/x-director", + "w3d": "application/x-director", + "fgd": "application/x-director", + "swa": "application/x-director", + "wad": "application/x-doom", + "ncx": "application/x-dtbncx+xml", + "dtb": "application/x-dtbook+xml", + "res": "application/x-dtbresource+xml", + "dvi": "application/x-dvi", + "evy": "application/x-envoy", + "eva": "application/x-eva", + "bdf": "application/x-font-bdf", + "gsf": "application/x-font-ghostscript", + "psf": "application/x-font-linux-psf", + "pcf": "application/x-font-pcf", + "snf": "application/x-font-snf", + "pfa": "application/x-font-type1", + "pfb": "application/x-font-type1", + "pfm": "application/x-font-type1", + "afm": "application/x-font-type1", + "arc": "application/x-freearc", + "spl": "application/x-futuresplash", + "gca": "application/x-gca-compressed", + "ulx": "application/x-glulx", + "gnumeric": "application/x-gnumeric", + "gramps": "application/x-gramps-xml", + "gtar": "application/x-gtar", + "hdf": "application/x-hdf", + "install": "application/x-install-instructions", + "iso": "application/x-iso9660-image", + "jnlp": "application/x-java-jnlp-file", + "latex": "application/x-latex", + "lzh": "application/x-lzh-compressed", + "lha": "application/x-lzh-compressed", + "mie": "application/x-mie", + "prc": "application/x-mobipocket-ebook", + "mobi": "application/x-mobipocket-ebook", + "application": "application/x-ms-application", + "lnk": "application/x-ms-shortcut", + "wmd": "application/x-ms-wmd", + "wmz": "application/x-ms-wmz", + "xbap": "application/x-ms-xbap", + "mdb": "application/x-msaccess", + "obd": "application/x-msbinder", + "crd": "application/x-mscardfile", + "clp": "application/x-msclip", + "exe": "application/x-msdownload", + "dll": "application/x-msdownload", + "com": "application/x-msdownload", + "bat": "application/x-msdownload", + "msi": "application/x-msdownload", + "mvb": "application/x-msmediaview", + "m13": "application/x-msmediaview", + "m14": "application/x-msmediaview", + "wmf": "application/x-msmetafile", + "wmz": "application/x-msmetafile", + "emf": "application/x-msmetafile", + "emz": "application/x-msmetafile", + "mny": "application/x-msmoney", + "pub": "application/x-mspublisher", + "scd": "application/x-msschedule", + "trm": "application/x-msterminal", + "wri": "application/x-mswrite", + "nc": "application/x-netcdf", + "cdf": "application/x-netcdf", + "nzb": "application/x-nzb", + "p12": "application/x-pkcs12", + "pfx": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "spc": "application/x-pkcs7-certificates", + "p7r": "application/x-pkcs7-certreqresp", + "rar": "application/x-rar-compressed", + "ris": "application/x-research-info-systems", + "sh": "application/x-sh", + "shar": "application/x-shar", + "swf": "application/x-shockwave-flash", + "xap": "application/x-silverlight-app", + "sql": "application/x-sql", + "sit": "application/x-stuffit", + "sitx": "application/x-stuffitx", + "srt": "application/x-subrip", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "t3": "application/x-t3vm-image", + "gam": "application/x-tads", + "tar": "application/x-tar", + "tcl": "application/x-tcl", + "tex": "application/x-tex", + "tfm": "application/x-tex-tfm", + "texinfo": "application/x-texinfo", + "texi": "application/x-texinfo", + "obj": "application/x-tgif", + "ustar": "application/x-ustar", + "src": "application/x-wais-source", + "der": "application/x-x509-ca-cert", + "crt": "application/x-x509-ca-cert", + "fig": "application/x-xfig", + "xlf": "application/x-xliff+xml", + "xpi": "application/x-xpinstall", + "xz": "application/x-xz", + "z1": "application/x-zmachine", + "z2": "application/x-zmachine", + "z3": "application/x-zmachine", + "z4": "application/x-zmachine", + "z5": "application/x-zmachine", + "z6": "application/x-zmachine", + "z7": "application/x-zmachine", + "z8": "application/x-zmachine", + "xaml": "application/xaml+xml", + "xdf": "application/xcap-diff+xml", + "xenc": "application/xenc+xml", + "xhtml": "application/xhtml+xml", + "xht": "application/xhtml+xml", + "xml": "application/xml", + "xsl": "application/xml", + "dtd": "application/xml-dtd", + "xop": "application/xop+xml", + "xpl": "application/xproc+xml", + "xslt": "application/xslt+xml", + "xspf": "application/xspf+xml", + "mxml": "application/xv+xml", + "xhvml": "application/xv+xml", + "xvml": "application/xv+xml", + "xvm": "application/xv+xml", + "yang": "application/yang", + "yin": "application/yin+xml", + "zip": "application/zip", + "adp": "audio/adpcm", + "au": "audio/basic", + "snd": "audio/basic", + "mid": "audio/midi", + "midi": "audio/midi", + "kar": "audio/midi", + "rmi": "audio/midi", + "m4a": "audio/mp4", + "mp4a": "audio/mp4", + "mpga": "audio/mpeg", + "mp2": "audio/mpeg", + "mp2a": "audio/mpeg", + "mp3": "audio/mpeg", + "m2a": "audio/mpeg", + "m3a": "audio/mpeg", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "spx": "audio/ogg", + "opus": "audio/ogg", + "s3m": "audio/s3m", + "sil": "audio/silk", + "uva": "audio/vnd.dece.audio", + "uvva": "audio/vnd.dece.audio", + "eol": "audio/vnd.digital-winds", + "dra": "audio/vnd.dra", + "dts": "audio/vnd.dts", + "dtshd": "audio/vnd.dts.hd", + "lvp": "audio/vnd.lucent.voice", + "pya": "audio/vnd.ms-playready.media.pya", + "ecelp4800": "audio/vnd.nuera.ecelp4800", + "ecelp7470": "audio/vnd.nuera.ecelp7470", + "ecelp9600": "audio/vnd.nuera.ecelp9600", + "rip": "audio/vnd.rip", + "weba": "audio/webm", + "aac": "audio/x-aac", + "aif": "audio/x-aiff", + "aiff": "audio/x-aiff", + "aifc": "audio/x-aiff", + "caf": "audio/x-caf", + "flac": "audio/x-flac", + "mka": "audio/x-matroska", + "m3u": "audio/x-mpegurl", + "wax": "audio/x-ms-wax", + "wma": "audio/x-ms-wma", + "ram": "audio/x-pn-realaudio", + "ra": "audio/x-pn-realaudio", + "rmp": "audio/x-pn-realaudio-plugin", + "wav": "audio/x-wav", + "xm": "audio/xm", + "cdx": "chemical/x-cdx", + "cif": "chemical/x-cif", + "cmdf": "chemical/x-cmdf", + "cml": "chemical/x-cml", + "csml": "chemical/x-csml", + "xyz": "chemical/x-xyz", + "ttc": "font/collection", + "otf": "font/otf", + "ttf": "font/ttf", + "woff": "font/woff", + "woff2": "font/woff2", + "bmp": "image/bmp", + "cgm": "image/cgm", + "g3": "image/g3fax", + "gif": "image/gif", + "ief": "image/ief", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "jpe": "image/jpeg", + "ktx": "image/ktx", + "png": "image/png", + "btif": "image/prs.btif", + "sgi": "image/sgi", + "svg": "image/svg+xml", + "svgz": "image/svg+xml", + "tiff": "image/tiff", + "tif": "image/tiff", + "psd": "image/vnd.adobe.photoshop", + "uvi": "image/vnd.dece.graphic", + "uvvi": "image/vnd.dece.graphic", + "uvg": "image/vnd.dece.graphic", + "uvvg": "image/vnd.dece.graphic", + "djvu": "image/vnd.djvu", + "djv": "image/vnd.djvu", + "sub": "image/vnd.dvb.subtitle", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dxf", + "fbs": "image/vnd.fastbidsheet", + "fpx": "image/vnd.fpx", + "fst": "image/vnd.fst", + "mmr": "image/vnd.fujixerox.edmics-mmr", + "rlc": "image/vnd.fujixerox.edmics-rlc", + "mdi": "image/vnd.ms-modi", + "wdp": "image/vnd.ms-photo", + "npx": "image/vnd.net-fpx", + "wbmp": "image/vnd.wap.wbmp", + "xif": "image/vnd.xiff", + "webp": "image/webp", + "3ds": "image/x-3ds", + "ras": "image/x-cmu-raster", + "cmx": "image/x-cmx", + "fh": "image/x-freehand", + "fhc": "image/x-freehand", + "fh4": "image/x-freehand", + "fh5": "image/x-freehand", + "fh7": "image/x-freehand", + "ico": "image/x-icon", + "sid": "image/x-mrsid-image", + "pcx": "image/x-pcx", + "pic": "image/x-pict", + "pct": "image/x-pict", + "pnm": "image/x-portable-anymap", + "pbm": "image/x-portable-bitmap", + "pgm": "image/x-portable-graymap", + "ppm": "image/x-portable-pixmap", + "rgb": "image/x-rgb", + "tga": "image/x-tga", + "xbm": "image/x-xbitmap", + "xpm": "image/x-xpixmap", + "xwd": "image/x-xwindowdump", + "eml": "message/rfc822", + "mime": "message/rfc822", + "igs": "model/iges", + "iges": "model/iges", + "msh": "model/mesh", + "mesh": "model/mesh", + "silo": "model/mesh", + "dae": "model/vnd.collada+xml", + "dwf": "model/vnd.dwf", + "gdl": "model/vnd.gdl", + "gtw": "model/vnd.gtw", + "mts": "model/vnd.mts", + "vtu": "model/vnd.vtu", + "wrl": "model/vrml", + "vrml": "model/vrml", + "x3db": "model/x3d+binary", + "x3dbz": "model/x3d+binary", + "x3dv": "model/x3d+vrml", + "x3dvz": "model/x3d+vrml", + "x3d": "model/x3d+xml", + "x3dz": "model/x3d+xml", + "appcache": "text/cache-manifest", + "ics": "text/calendar", + "ifb": "text/calendar", + "css": "text/css", + "csv": "text/csv", + "html": "text/html", + "htm": "text/html", + "js": "text/javascript", + "mjs": "text/javascript", + "n3": "text/n3", + "txt": "text/plain", + "text": "text/plain", + "conf": "text/plain", + "def": "text/plain", + "list": "text/plain", + "log": "text/plain", + "in": "text/plain", + "dsc": "text/prs.lines.tag", + "rtx": "text/richtext", + "sgml": "text/sgml", + "sgm": "text/sgml", + "tsv": "text/tab-separated-values", + "t": "text/troff", + "tr": "text/troff", + "roff": "text/troff", + "man": "text/troff", + "me": "text/troff", + "ms": "text/troff", + "ttl": "text/turtle", + "uri": "text/uri-list", + "uris": "text/uri-list", + "urls": "text/uri-list", + "vcard": "text/vcard", + "curl": "text/vnd.curl", + "dcurl": "text/vnd.curl.dcurl", + "mcurl": "text/vnd.curl.mcurl", + "scurl": "text/vnd.curl.scurl", + "sub": "text/vnd.dvb.subtitle", + "fly": "text/vnd.fly", + "flx": "text/vnd.fmi.flexstor", + "gv": "text/vnd.graphviz", + "3dml": "text/vnd.in3d.3dml", + "spot": "text/vnd.in3d.spot", + "jad": "text/vnd.sun.j2me.app-descriptor", + "wml": "text/vnd.wap.wml", + "wmls": "text/vnd.wap.wmlscript", + "s": "text/x-asm", + "asm": "text/x-asm", + "c": "text/x-c", + "cc": "text/x-c", + "cxx": "text/x-c", + "cpp": "text/x-c", + "h": "text/x-c", + "hh": "text/x-c", + "dic": "text/x-c", + "f": "text/x-fortran", + "for": "text/x-fortran", + "f77": "text/x-fortran", + "f90": "text/x-fortran", + "java": "text/x-java-source", + "nfo": "text/x-nfo", + "opml": "text/x-opml", + "p": "text/x-pascal", + "pas": "text/x-pascal", + "etx": "text/x-setext", + "sfv": "text/x-sfv", + "uu": "text/x-uuencode", + "vcs": "text/x-vcalendar", + "vcf": "text/x-vcard", + "3gp": "video/3gpp", + "3g2": "video/3gpp2", + "h261": "video/h261", + "h263": "video/h263", + "h264": "video/h264", + "jpgv": "video/jpeg", + "jpm": "video/jpm", + "jpgm": "video/jpm", + "mj2": "video/mj2", + "mjp2": "video/mj2", + "mp4": "video/mp4", + "mp4v": "video/mp4", + "mpg4": "video/mp4", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpe": "video/mpeg", + "m1v": "video/mpeg", + "m2v": "video/mpeg", + "ogv": "video/ogg", + "qt": "video/quicktime", + "mov": "video/quicktime", + "uvh": "video/vnd.dece.hd", + "uvvh": "video/vnd.dece.hd", + "uvm": "video/vnd.dece.mobile", + "uvvm": "video/vnd.dece.mobile", + "uvp": "video/vnd.dece.pd", + "uvvp": "video/vnd.dece.pd", + "uvs": "video/vnd.dece.sd", + "uvvs": "video/vnd.dece.sd", + "uvv": "video/vnd.dece.video", + "uvvv": "video/vnd.dece.video", + "dvb": "video/vnd.dvb.file", + "fvt": "video/vnd.fvt", + "mxu": "video/vnd.mpegurl", + "m4u": "video/vnd.mpegurl", + "pyv": "video/vnd.ms-playready.media.pyv", + "uvu": "video/vnd.uvvu.mp4", + "uvvu": "video/vnd.uvvu.mp4", + "viv": "video/vnd.vivo", + "webm": "video/webm", + "f4v": "video/x-f4v", + "fli": "video/x-fli", + "flv": "video/x-flv", + "m4v": "video/x-m4v", + "mkv": "video/x-matroska", + "mk3d": "video/x-matroska", + "mks": "video/x-matroska", + "mng": "video/x-mng", + "asf": "video/x-ms-asf", + "asx": "video/x-ms-asf", + "vob": "video/x-ms-vob", + "wm": "video/x-ms-wm", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wvx": "video/x-ms-wvx", + "avi": "video/x-msvideo", + "movie": "video/x-sgi-movie", + "smv": "video/x-smv", + "ice": "x-conference/x-cooltalk", +} + -proc newMimetypes*(): MimeDB = +func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. - result.mimes = mimes.newStringTable() + {.cast(noSideEffect).}: + result.mimes = mimes.toOrderedTable() -proc getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = - ## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext`` - ## could not be found. - result = mimedb.mimes.getOrDefault(ext) +func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = + ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` + ## could not be found. `ext` can start with an optional dot which is ignored. + ## `ext` is lowercased before querying `mimedb`. + if ext.startsWith("."): + result = mimedb.mimes.getOrDefault(ext.toLowerAscii.substr(1)) + else: + result = mimedb.mimes.getOrDefault(ext.toLowerAscii()) if result == "": return default -proc getExt*(mimedb: MimeDB, mimetype: string, default = "txt"): string = - ## Gets extension which corresponds to ``mimetype``. Returns ``default`` if - ## ``mimetype`` could not be found. Extensions are returned without the - ## leading dot. +func getExt*(mimedb: MimeDB, mimetype: string, default = "txt"): string = + ## Gets extension which corresponds to `mimetype`. Returns `default` if + ## `mimetype` could not be found. Extensions are returned without the + ## leading dot. `mimetype` is lowercased before querying `mimedb`. result = default + let mimeLowered = mimetype.toLowerAscii() for e, m in mimedb.mimes: - if m == mimetype: + if m == mimeLowered: result = e + break -proc register*(mimedb: var MimeDB, ext: string, mimetype: string) = - ## Adds ``mimetype`` to the ``mimedb``. - mimedb.mimes[ext] = mimetype - -when isMainModule: - var m = newMimetypes() - assert m.getMimetype("mp4") == "video/mp4" - assert m.getExt("text/html") == "html" +func register*(mimedb: var MimeDB, ext: string, mimetype: string) = + ## Adds `mimetype` to the `mimedb`. + ## `mimetype` and `ext` are lowercased before registering on `mimedb`. + assert ext.strip.len > 0, "ext argument can not be empty string" + assert mimetype.strip.len > 0, "mimetype argument can not be empty string" + {.noSideEffect.}: + mimedb.mimes[ext.toLowerAscii()] = mimetype.toLowerAscii() diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 8a7780570..656c98a20 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -8,51 +8,66 @@ # ## This module implements a low-level cross-platform sockets interface. Look -## at the ``net`` module for the higher-level version. +## at the `net` module for the higher-level version. # TODO: Clean up the exports a bit and everything else in general. -import unsigned, os +import std/[os, options] +import std/private/since +import std/strbasics + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] when hostOS == "solaris": {.passl: "-lsocket -lnsl".} -const useWinVersion = defined(Windows) or defined(nimdoc) +const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: - import winlean + import std/winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, + WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR, WSAEDISCON, ERROR_NETNAME_DELETED else: - import posix + import std/posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, - EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET - export Sockaddr_storage + EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF + export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, - Sockaddr_in6, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto + Sockaddr_in6, Sockaddr_storage, + recv, `==`, connect, send, accept, recvfrom, sendto, + freeAddrInfo + +when not useNimNetLite: + export inet_ntoa export SO_ERROR, SOL_SOCKET, SOMAXCONN, SO_ACCEPTCONN, SO_BROADCAST, SO_DEBUG, SO_DONTROUTE, - SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, + SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, SO_REUSEPORT, MSG_PEEK when defined(macosx) and not defined(nimdoc): - export SO_NOSIGPIPE + export SO_NOSIGPIPE type - Port* = distinct uint16 ## port type - - Domain* = enum ## domain, which specifies the protocol family of the - ## created socket. Other domains than those that are listed - ## here are unsupported. - AF_UNIX, ## for local socket (using a file). Unsupported on Windows. - AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = 23 ## for network protocol IPv6. + Port* = distinct uint16 ## port type + + Domain* = enum ## \ + ## domain, which specifies the protocol family of the + ## created socket. Other domains than those that are listed + ## here are unsupported. + AF_UNSPEC = 0, ## unspecified domain (can be detected automatically by + ## some procedures, such as getaddrinfo) + AF_UNIX = 1, ## for local socket (using a file). Unsupported on Windows. + AF_INET = 2, ## for network protocol IPv4 or + AF_INET6 = when defined(macosx): 30 elif defined(windows): 23 else: 10 ## for network protocol IPv6. SockType* = enum ## second argument to `socket` proc SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets @@ -60,13 +75,14 @@ type SOCK_RAW = 3, ## raw protocols atop the network layer. SOCK_SEQPACKET = 5 ## reliable sequenced packet service - Protocol* = enum ## third argument to `socket` proc - IPPROTO_TCP = 6, ## Transmission control protocol. - IPPROTO_UDP = 17, ## User datagram protocol. - IPPROTO_IP, ## Internet protocol. Unsupported on Windows. - IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. - IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. - IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. + Protocol* = enum ## third argument to `socket` proc + IPPROTO_TCP = 6, ## Transmission control protocol. + IPPROTO_UDP = 17, ## User datagram protocol. + IPPROTO_IP, ## Internet protocol. + IPPROTO_IPV6, ## Internet Protocol Version 6. + IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. + IPPROTO_ICMP ## Internet Control message protocol. + IPPROTO_ICMPV6 ## Internet Control message protocol for IPv6. Servent* = object ## information about a service name*: string @@ -81,8 +97,7 @@ type length*: int addrList*: seq[string] -{.deprecated: [TPort: Port, TDomain: Domain, TType: SockType, - TProtocol: Protocol, TServent: Servent, THostent: Hostent].} +const IPPROTO_NONE* = IPPROTO_IP ## Use this if your socket type requires a protocol value of zero (e.g. Unix sockets). when useWinVersion: let @@ -92,7 +107,7 @@ when useWinVersion: IOCPARM_MASK* = 127 IOC_IN* = int(-2147483648) FIONBIO* = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or - (102 shl 8) or 126 + (102 shl 8) or 126 nativeAfInet = winlean.AF_INET nativeAfInet6 = winlean.AF_INET6 @@ -104,88 +119,166 @@ else: osInvalidSocket* = posix.INVALID_SOCKET nativeAfInet = posix.AF_INET nativeAfInet6 = posix.AF_INET6 + nativeAfUnix = posix.AF_UNIX proc `==`*(a, b: Port): bool {.borrow.} - ## ``==`` for ports. + ## `==` for ports. proc `$`*(p: Port): string {.borrow.} - ## returns the port number as a string + ## Returns the port number as a string proc toInt*(domain: Domain): cint - ## Converts the Domain enum to a platform-dependent ``cint``. + ## Converts the Domain enum to a platform-dependent `cint`. proc toInt*(typ: SockType): cint - ## Converts the SockType enum to a platform-dependent ``cint``. + ## Converts the SockType enum to a platform-dependent `cint`. proc toInt*(p: Protocol): cint - ## Converts the Protocol enum to a platform-dependent ``cint``. + ## Converts the Protocol enum to a platform-dependent `cint`. when not useWinVersion: proc toInt(domain: Domain): cint = case domain - of AF_UNIX: result = posix.AF_UNIX - of AF_INET: result = posix.AF_INET - of AF_INET6: result = posix.AF_INET6 - else: discard + of AF_UNSPEC: result = posix.AF_UNSPEC.cint + of AF_UNIX: result = posix.AF_UNIX.cint + of AF_INET: result = posix.AF_INET.cint + of AF_INET6: result = posix.AF_INET6.cint + + proc toKnownDomain*(family: cint): Option[Domain] = + ## Converts the platform-dependent `cint` to the Domain or none(), + ## if the `cint` is not known. + result = if family == posix.AF_UNSPEC: some(Domain.AF_UNSPEC) + elif family == posix.AF_UNIX: some(Domain.AF_UNIX) + elif family == posix.AF_INET: some(Domain.AF_INET) + elif family == posix.AF_INET6: some(Domain.AF_INET6) + else: none(Domain) proc toInt(typ: SockType): cint = case typ - of SOCK_STREAM: result = posix.SOCK_STREAM - of SOCK_DGRAM: result = posix.SOCK_DGRAM + of SOCK_STREAM: result = posix.SOCK_STREAM + of SOCK_DGRAM: result = posix.SOCK_DGRAM of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET - of SOCK_RAW: result = posix.SOCK_RAW - else: discard + of SOCK_RAW: result = posix.SOCK_RAW proc toInt(p: Protocol): cint = case p - of IPPROTO_TCP: result = posix.IPPROTO_TCP - of IPPROTO_UDP: result = posix.IPPROTO_UDP - of IPPROTO_IP: result = posix.IPPROTO_IP - of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 - of IPPROTO_RAW: result = posix.IPPROTO_RAW - of IPPROTO_ICMP: result = posix.IPPROTO_ICMP - else: discard + of IPPROTO_TCP: result = posix.IPPROTO_TCP + of IPPROTO_UDP: result = posix.IPPROTO_UDP + of IPPROTO_IP: result = posix.IPPROTO_IP + of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 + of IPPROTO_RAW: result = posix.IPPROTO_RAW + of IPPROTO_ICMP: result = posix.IPPROTO_ICMP + of IPPROTO_ICMPV6: result = posix.IPPROTO_ICMPV6 else: proc toInt(domain: Domain): cint = - result = toU16(ord(domain)) + result = cast[cint](uint32(ord(domain))) + + proc toKnownDomain*(family: cint): Option[Domain] = + ## Converts the platform-dependent `cint` to the Domain or none(), + ## if the `cint` is not known. + result = if family == winlean.AF_UNSPEC: some(Domain.AF_UNSPEC) + elif family == winlean.AF_INET: some(Domain.AF_INET) + elif family == winlean.AF_INET6: some(Domain.AF_INET6) + else: none(Domain) proc toInt(typ: SockType): cint = result = cint(ord(typ)) proc toInt(p: Protocol): cint = - result = cint(ord(p)) - + case p + of IPPROTO_IP: + result = 0.cint + of IPPROTO_ICMP: + result = 1.cint + of IPPROTO_TCP: + result = 6.cint + of IPPROTO_UDP: + result = 17.cint + of IPPROTO_IPV6: + result = 41.cint + of IPPROTO_ICMPV6: + result = 58.cint + else: + result = cint(ord(p)) + +proc toSockType*(protocol: Protocol): SockType = + result = case protocol + of IPPROTO_TCP: + SOCK_STREAM + of IPPROTO_UDP: + SOCK_DGRAM + of IPPROTO_IP, IPPROTO_IPV6, IPPROTO_RAW, IPPROTO_ICMP, IPPROTO_ICMPV6: + SOCK_RAW + +proc getProtoByName*(name: string): int {.since: (1, 3, 5).} = + ## Returns a protocol code from the database that matches the protocol `name`. + when useWinVersion: + let protoent = winlean.getprotobyname(name.cstring) + else: + let protoent = posix.getprotobyname(name.cstring) -proc newNativeSocket*(domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. - socket(toInt(domain), toInt(sockType), toInt(protocol)) + if protoent == nil: + raise newException(OSError, "protocol not found: " & name) -proc newNativeSocket*(domain: cint, sockType: cint, - protocol: cint): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. - ## - ## Use this overload if one of the enums specified above does - ## not contain what you need. - socket(domain, sockType, protocol) + result = protoent.p_proto.int proc close*(socket: SocketHandle) = - ## closes a socket. + ## Closes a socket. when useWinVersion: discard winlean.closesocket(socket) else: discard posix.close(socket) - # TODO: These values should not be discarded. An EOS should be raised. + # TODO: These values should not be discarded. An OSError should be raised. # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times -proc bindAddr*(socket: SocketHandle, name: ptr SockAddr, namelen: SockLen): cint = +when declared(setInheritable) or defined(nimdoc): + proc setInheritable*(s: SocketHandle, inheritable: bool): bool {.inline.} = + ## Set whether a socket is inheritable by child processes. Returns `true` + ## on success. + ## + ## This function is not implemented on all platform, test for availability + ## with `declared() <system.html#declared,untyped>`. + setInheritable(FileHandle s, inheritable) + +proc createNativeSocket*(domain: cint, sockType: cint, protocol: cint, + inheritable: bool = defined(nimInheritHandles)): SocketHandle = + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. + ## + ## `inheritable` decides if the resulting SocketHandle can be inherited + ## by child processes. + ## + ## Use this overload if one of the enums specified above does + ## not contain what you need. + let sockType = + when (defined(linux) or defined(bsd)) and not defined(nimdoc): + if inheritable: sockType and not SOCK_CLOEXEC else: sockType or SOCK_CLOEXEC + else: + sockType + result = socket(domain, sockType, protocol) + when declared(setInheritable) and not (defined(linux) or defined(bsd)): + if not setInheritable(result, inheritable): + close result + return osInvalidSocket + +proc createNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, + inheritable: bool = defined(nimInheritHandles)): SocketHandle = + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. + ## + ## `inheritable` decides if the resulting SocketHandle can be inherited + ## by child processes. + createNativeSocket(toInt(domain), toInt(sockType), toInt(protocol), inheritable) + +proc bindAddr*(socket: SocketHandle, name: ptr SockAddr, + namelen: SockLen): cint = result = bindSocket(socket, name, namelen) -proc listen*(socket: SocketHandle, backlog = SOMAXCONN): cint {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the +proc listen*(socket: SocketHandle, backlog = SOMAXCONN): cint {.tags: [ + ReadIOEffect].} = + ## Marks `socket` as accepting connections. + ## `Backlog` specifies the maximum length of the ## queue of pending connections. when useWinVersion: result = winlean.listen(socket, cint(backlog)) @@ -197,257 +290,447 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, protocol: Protocol = IPPROTO_TCP): ptr AddrInfo = ## ## - ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``! + ## .. warning:: The resulting `ptr AddrInfo` must be freed using `freeAddrInfo`! var hints: AddrInfo result = nil hints.ai_family = toInt(domain) hints.ai_socktype = toInt(sockType) hints.ai_protocol = toInt(protocol) # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED. - # FreeBSD doesn't support AI_V4MAPPED but defines the macro. + # FreeBSD, Haiku don't support AI_V4MAPPED but defines the macro. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092 - when not defined(freebsd) or defined(openbsd): + # https://dev.haiku-os.org/ticket/14323 + when not defined(freebsd) and not defined(openbsd) and not defined(netbsd) and + not defined(android) and not defined(haiku): if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED - var gaiResult = getaddrinfo(address, $port, addr(hints), result) + let socketPort = if sockType == SOCK_RAW: "" else: $port + var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion: + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) -proc dealloc*(ai: ptr AddrInfo) = - freeaddrinfo(ai) - -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. +proc ntohl*(x: uint32): uint32 = + ## Converts 32-bit unsigned integers from network to host byte order. ## On machines where the host byte order is the same as network byte order, ## this is a no-op; otherwise, it performs a 4-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) - -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. + else: result = (x shr 24'u32) or + (x shr 8'u32 and 0xff00'u32) or + (x shl 8'u32 and 0xff0000'u32) or + (x shl 24'u32) + +proc ntohs*(x: uint16): uint16 = + ## Converts 16-bit unsigned integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) + else: result = (x shr 8'u16) or (x shl 8'u16) -template htonl*(x: int32): expr = - ## Converts 32-bit integers from host to network byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 4-byte swap operation. +template htonl*(x: uint32): untyped = + ## Converts 32-bit unsigned integers from host to network byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. nativesockets.ntohl(x) -template htons*(x: int16): expr = - ## Converts 16-bit positive integers from host to network byte order. +template htons*(x: uint16): untyped = + ## Converts 16-bit unsigned integers from host to network byte order. ## On machines where the host byte order is the same as network byte ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by ``name`` matches the s_name member - ## and the protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by ``port`` matches the s_port member and the - ## protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - proc getSockDomain*(socket: SocketHandle): Domain = - ## returns the socket's domain (AF_INET or AF_INET6). - var name: SockAddr + ## Returns the socket's domain (AF_INET or AF_INET6). + var name: Sockaddr_in6 var namelen = sizeof(name).SockLen if getsockname(socket, cast[ptr SockAddr](addr(name)), addr(namelen)) == -1'i32: raiseOSError(osLastError()) - if name.sa_family == nativeAfInet: - result = AF_INET - elif name.sa_family == nativeAfInet6: - result = AF_INET6 - else: - raiseOSError(osLastError(), "unknown socket family in getSockFamily") - - -proc getAddrString*(sockAddr: ptr SockAddr): string = - ## return the string representation of address within sockAddr - if sockAddr.sa_family == nativeAfInet: - result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) - elif sockAddr.sa_family == nativeAfInet6: - when not useWinVersion: - # TODO: Windows - result = newString(posix.INET6_ADDRSTRLEN) - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - discard posix.inet_ntop(posix.AF_INET6, addr6, result.cstring, - result.len.int32) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result = result.substr("::ffff:".len) + let knownDomain = toKnownDomain(name.sin6_family.cint) + if knownDomain.isSome: + result = knownDomain.get() else: - raiseOSError(osLastError(), "unknown socket family in getAddrString") - - -proc getSockName*(socket: SocketHandle): Port = - ## returns the socket's associated port number. - var name: Sockaddr_in - when useWinVersion: - name.sin_family = int16(ord(AF_INET)) - else: - name.sin_family = posix.AF_INET - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(nativesockets.ntohs(name.sin_port)) - -proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## returns the socket's local address and port number. - ## - ## Similar to POSIX's `getsockname`:idx:. - case domain - of AF_INET: - var name: Sockaddr_in + raise newException(IOError, "Unknown socket family in getSockDomain") + +when not useNimNetLite: + proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the service name specified by `name` matches the s_name member + ## and the protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. when useWinVersion: - name.sin_family = int16(ord(AF_INET)) + var s = winlean.getservbyname(name, proto) else: - name.sin_family = posix.AF_INET - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 + var s = posix.getservbyname(name, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the port specified by `port` matches the s_port member and the + ## protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. when useWinVersion: - name.sin6_family = int16(ord(AF_INET6)) + var s = winlean.getservbyport(uint16(port).cint, proto) else: - name.sin6_family = posix.AF_INET6 - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - var buf: array[64, char] - if inet_ntop(name.sin6_family.cint, - addr name, buf.cstring, sizeof(buf).int32).isNil: - raiseOSError(osLastError()) - result = ($buf, Port(nativesockets.ntohs(name.sin6_port))) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + var s = posix.getservbyport(uint16(port).cint, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the hostname of an IP Address. + var + addrInfo = getAddrInfo(ip, Port(0), AF_UNSPEC) + myAddr: pointer + addrLen = 0 + family = 0 + + defer: freeAddrInfo(addrInfo) + + if addrInfo.ai_addr.sa_family.cint == nativeAfInet: + family = nativeAfInet + myAddr = addr cast[ptr Sockaddr_in](addrInfo.ai_addr).sin_addr + addrLen = 4 + elif addrInfo.ai_addr.sa_family.cint == nativeAfInet6: + family = nativeAfInet6 + myAddr = addr cast[ptr Sockaddr_in6](addrInfo.ai_addr).sin6_addr + addrLen = 16 + else: + raise newException(IOError, "Unknown socket family in `getHostByAddr()`") -proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## returns the socket's peer address and port number. - ## - ## Similar to POSIX's `getpeername`:idx: - case domain - of AF_INET: - var name: Sockaddr_in when useWinVersion: - name.sin_family = int16(ord(AF_INET)) + var s = winlean.gethostbyaddr(cast[ptr InAddr](myAddr), addrLen.cuint, + cint(family)) + if s == nil: raiseOSError(osLastError()) else: - name.sin_family = posix.AF_INET - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](myAddr), addrLen.cint, + cint(family)) + else: + posix.gethostbyaddr(myAddr, addrLen.SockLen, + cint(family)) + if s == nil: + raiseOSError(osLastError(), $hstrerror(h_errno)) + + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when useWinVersion: + result.addrtype = Domain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + let strAddrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 + var i = 0 + while not isNil(s.h_addr_list[i]): + var ipStr = newString(strAddrLen) + if inet_ntop(nativeAfInet6, cast[pointer](s.h_addr_list[i]), + cstring(ipStr), len(ipStr).int32) == nil: + raiseOSError(osLastError()) + when not useWinVersion: + if posix.IN6_IS_ADDR_V4MAPPED(cast[ptr In6Addr](s.h_addr_list[i])) != 0: + ipStr.setSlice("::ffff:".len..<strAddrLen) + setLen(ipStr, len(cstring(ipStr))) + result.addrList.add(ipStr) + inc(i) + result.length = int(s.h_length) + + proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the IP address of a hostname. + when useWinVersion: + var s = winlean.gethostbyname(name) + else: + var s = posix.gethostbyname(name) + if s == nil: raiseOSError(osLastError()) + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when useWinVersion: + result.addrtype = Domain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) + result.length = int(s.h_length) + + proc getHostname*(): string {.tags: [ReadIOEffect].} = + ## Returns the local hostname (not the FQDN) + # https://tools.ietf.org/html/rfc1035#section-2.3.1 + # https://tools.ietf.org/html/rfc2181#section-11 + const size = 256 + result = newString(size) + when useWinVersion: + let success = winlean.gethostname(result.cstring, size) + else: + # Posix + let success = posix.gethostname(result.cstring, size) + if success != 0.cint: raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 + let x = len(cstring(result)) + result.setLen(x) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + ## Returns the string representation of address within sockAddr + if sockAddr.sa_family.cint == nativeAfInet: + result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 # it's actually 46 in both cases + result = newString(addrLen) + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr result[0]), + result.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + result.setSlice("::ffff:".len..<addrLen) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr result[0]), + result.len.int32) == nil: + raiseOSError(osLastError()) + setLen(result, len(cstring(result))) + else: + when defined(posix) and not defined(nimdoc): + if sockAddr.sa_family.cint == nativeAfUnix: + return "unix" + raise newException(IOError, "Unknown socket family in getAddrString") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) = + ## Stores in `strAddress` the string representation of the address inside + ## `sockAddr` + ## + ## **Note** + ## * `strAddress` must be initialized to 46 in length. + const length = 46 + assert(length == len(strAddress), + "`strAddress` was not initialized correctly. 46 != `len(strAddress)`") + if sockAddr.sa_family.cint == nativeAfInet: + let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET, addr4, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + if winlean.inet_ntop(winlean.AF_INET, addr4, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + strAddress.setSlice("::ffff:".len..<length) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + raise newException(IOError, "Unknown socket family in getAddrString") + setLen(strAddress, len(cstring(strAddress))) + + when defined(posix) and not defined(nimdoc): + proc makeUnixAddr*(path: string): Sockaddr_un = + result.sun_family = AF_UNIX.TSa_Family + if path.len >= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + + proc getSockName*(socket: SocketHandle): Port = + ## Returns the socket's associated port number. + var name: Sockaddr_in when useWinVersion: - name.sin6_family = int16(ord(AF_INET6)) + name.sin_family = uint16(ord(AF_INET)) else: - name.sin6_family = posix.AF_INET6 + name.sin_family = TSa_Family(posix.AF_INET) + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - var buf: array[64, char] - if inet_ntop(name.sin6_family.cint, - addr name, buf.cstring, sizeof(buf).int32).isNil: + result = Port(nativesockets.ntohs(name.sin_port)) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's peer address and port number. + ## + ## Similar to POSIX's `getpeername`:idx: + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + +when useNimNetLite: + + when useWinVersion: + const + INET_ADDRSTRLEN = 16 + INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases + + proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} = + let af_family = sa.sa_family + var nl, v4Slice: cint + var si_addr: ptr InAddr + + if af_family == AF_INET.TSa_Family: + nl = INET_ADDRSTRLEN + si_addr = cast[ptr Sockaddr_in](sa).sin_addr.addr() + elif af_family == AF_INET6.TSa_Family: + nl = INET6_ADDRSTRLEN + let si6_addr = cast[ptr Sockaddr_in6](sa).sin6_addr.addr() + si_addr = cast[ptr InAddr](si6_addr) # let's us reuse logic below + when defined(posix) and not defined(nimdoc) and not defined(zephyr): + if posix.IN6_IS_ADDR_V4MAPPED(si6_addr) != 0: + v4Slice = "::ffff:".len() + else: + when defined(posix) and not defined(nimdoc): + if af_family.cint == nativeAfUnix: + return "unix" + return "" + + result = newString(nl) + let namePtr = result.cstring() + if namePtr == inet_ntop(af_family.cint, si_addr, namePtr, nl): + result.setLen(len(namePtr)) + if v4Slice > 0: result.setSlice(v4Slice.int ..< nl.int) + else: + return "" + + proc sockAddrToStr(sa: var Sockaddr_in | var Sockaddr_in6): string = + result = sockAddrToStr(cast[ptr SockAddr](unsafeAddr(sa))) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + result = sockAddrToStr(sockAddr) + if result.len() == 0: raiseOSError(osLastError()) - result = ($buf, Port(nativesockets.ntohs(name.sin6_port))) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) {.noinit.} = + strAddress = getAddrString(sockAddr) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + template sockGetNameOrRaiseError(socket: untyped, name: untyped) = + var namelen = sizeof(socket).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + case domain + of AF_INET: + var name = Sockaddr_in(sin_family: TSa_Family(posix.AF_INET)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name = Sockaddr_in6(sin6_family: TSa_Family(posix.AF_INET6)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin6_port))) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. tags: [ReadIOEffect].} = @@ -470,7 +753,7 @@ proc setSockOptInt*(socket: SocketHandle, level, optname, optval: int) {. proc setBlocking*(s: SocketHandle, blocking: bool) = ## Sets blocking mode on socket. ## - ## Raises EOS on error. + ## Raises OSError on error. when useWinVersion: var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking if ioctlsocket(s, FIONBIO, addr(mode)) == -1: @@ -487,8 +770,12 @@ proc setBlocking*(s: SocketHandle, blocking: bool) = proc timeValFromMilliseconds(timeout = 500): Timeval = if timeout != -1: var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + when useWinVersion: + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + else: + result.tv_sec = seconds.Time + result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds proc createFdSet(fd: var TFdSet, s: seq[SocketHandle], m: var int) = FD_ZERO(fd) @@ -507,15 +794,15 @@ proc pruneSocketSet(s: var seq[SocketHandle], fd: var TFdSet) = inc(i) setLen(s, L) -proc select*(readfds: var seq[SocketHandle], timeout = 500): int = - ## Traditional select function. This function will return the number of - ## sockets that are ready to be read from, written to, or which have errors. - ## If there are none; 0 is returned. - ## ``Timeout`` is in milliseconds and -1 can be specified for no timeout. +proc selectRead*(readfds: var seq[SocketHandle], timeout = 500): int = + ## When a socket in `readfds` is ready to be read from then a non-zero + ## value will be returned specifying the count of the sockets which can be + ## read from. The sockets which cannot be read from will also be removed + ## from `readfds`. ## - ## A socket is removed from the specific ``seq`` when it has data waiting to - ## be read/written to or has errors (``exceptfds``). - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) + ## `timeout` is specified in milliseconds and `-1` can be specified for + ## an unlimited time. + var tv {.noinit.}: Timeval = timeValFromMilliseconds(timeout) var rd: TFdSet var m = 0 @@ -530,14 +817,14 @@ proc select*(readfds: var seq[SocketHandle], timeout = 500): int = proc selectWrite*(writefds: var seq[SocketHandle], timeout = 500): int {.tags: [ReadIOEffect].} = - ## When a socket in ``writefds`` is ready to be written to then a non-zero + ## When a socket in `writefds` is ready to be written to then a non-zero ## value will be returned specifying the count of the sockets which can be - ## written to. The sockets which can be written to will also be removed - ## from ``writefds``. + ## written to. The sockets which cannot be written to will also be removed + ## from `writefds`. ## - ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for + ## `timeout` is specified in milliseconds and `-1` can be specified for ## an unlimited time. - var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout) + var tv {.noinit.}: Timeval = timeValFromMilliseconds(timeout) var wr: TFdSet var m = 0 @@ -550,6 +837,34 @@ proc selectWrite*(writefds: var seq[SocketHandle], pruneSocketSet(writefds, (wr)) -when defined(Windows): +proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (SocketHandle, string) = + ## Accepts a new client connection. + ## + ## `inheritable` decides if the resulting SocketHandle can be inherited by + ## child processes. + ## + ## Returns (osInvalidSocket, "") if an error occurred. + var sockAddress: SockAddr + var addrLen = sizeof(sockAddress).SockLen + var sock = + when (defined(linux) or defined(bsd)) and not defined(nimdoc): + accept4(fd, addr(sockAddress), addr(addrLen), + if inheritable: 0 else: SOCK_CLOEXEC) + else: + accept(fd, addr(sockAddress), addr(addrLen)) + when declared(setInheritable) and not (defined(linux) or defined(bsd)): + if not setInheritable(sock, inheritable): + close sock + sock = osInvalidSocket + if sock == osInvalidSocket: + return (osInvalidSocket, "") + else: + when useNimNetLite: + var name = sockAddrToStr(addr sockAddress) + return (sock, name) + else: + return (sock, $inet_ntoa(cast[Sockaddr_in](sockAddress).sin_addr)) + +when defined(windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index d1016011e..24c94b651 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -8,29 +8,128 @@ # ## This module implements a high-level cross-platform sockets interface. +## The procedures implemented in this module are primarily for blocking sockets. +## For asynchronous non-blocking sockets use the `asyncnet` module together +## with the `asyncdispatch` module. +## +## The first thing you will always need to do in order to start using sockets, +## is to create a new instance of the `Socket` type using the `newSocket` +## procedure. +## +## SSL +## ==== +## +## In order to use the SSL procedures defined in this module, you will need to +## compile your application with the `-d:ssl` flag. See the +## `newContext<net.html#newContext%2Cstring%2Cstring%2Cstring%2Cstring>`_ +## procedure for additional details. +## +## +## SSL on Windows +## ============== +## +## On Windows the SSL library checks for valid certificates. +## It uses the `cacert.pem` file for this purpose which was extracted +## from `https://curl.se/ca/cacert.pem`. Besides +## the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you +## also need to ship `cacert.pem` with your `.exe` file. +## +## +## Examples +## ======== +## +## Connecting to a server +## ---------------------- +## +## After you create a socket with the `newSocket` procedure, you can easily +## connect it to a server running at a known hostname (or IP address) and port. +## To do so over TCP, use the example below. + +runnableExamples("-r:off"): + let socket = newSocket() + socket.connect("google.com", Port(80)) + +## For SSL, use the following example: + +runnableExamples("-r:off -d:ssl"): + let socket = newSocket() + let ctx = newContext() + wrapSocket(ctx, socket) + socket.connect("google.com", Port(443)) + +## UDP is a connectionless protocol, so UDP sockets don't have to explicitly +## call the `connect <net.html#connect%2CSocket%2Cstring>`_ procedure. They can +## simply start sending data immediately. + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + socket.sendTo("192.168.0.1", Port(27960), "status\n") + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + let ip = parseIpAddress("192.168.0.1") + doAssert socket.sendTo(ip, Port(27960), "status\c\l") == 8 + +## Creating a server +## ----------------- +## +## After you create a socket with the `newSocket` procedure, you can create a +## TCP server by calling the `bindAddr` and `listen` procedures. + +runnableExamples("-r:off"): + let socket = newSocket() + socket.bindAddr(Port(1234)) + socket.listen() + + # You can then begin accepting connections using the `accept` procedure. + var client: Socket + var address = "" + while true: + socket.acceptAddr(client, address) + echo "Client connected from: ", address + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions -{.deadCodeElim: on.} -import nativesockets, os, strutils, unsigned, parseutils, times -export Port, `$`, `==` +import std/nativesockets +import std/[os, strutils, times, sets, options, monotimes] +import std/ssl_config +export nativesockets.Port, nativesockets.`$`, nativesockets.`==` +export Domain, SockType, Protocol, IPPROTO_NONE -const useWinVersion = defined(Windows) or defined(nimdoc) +const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) +const defineSsl = defined(ssl) or defined(nimdoc) -when defined(ssl): - import openssl +when useWinVersion: + from std/winlean import WSAESHUTDOWN + +when defineSsl: + import std/openssl + when not defined(nimDisableCertificateValidation): + from std/ssl_certs import scanSSLCertificates # Note: The enumerations are mapped to Window's constants. -when defined(ssl): +when defineSsl: type - SslError* = object of Exception + Certificate* = string ## DER encoded certificate + + SslError* = object of CatchableError SslCVerifyMode* = enum - CVerifyNone, CVerifyPeer + CVerifyNone, CVerifyPeer, CVerifyPeerUseEnvVars SslProtVersion* = enum protSSLv2, protSSLv3, protTLSv1, protSSLv23 - SslContext* = distinct SslCtx + SslContext* = ref object + context*: SslCtx + referencedData: HashSet[int] + extraInternal: SslContextExtraInternal SslAcceptResult* = enum AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess @@ -38,31 +137,37 @@ when defined(ssl): SslHandshakeType* = enum handshakeAsClient, handshakeAsServer - {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, - TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, - TSSLAcceptResult: SSLAcceptResult].} + SslClientGetPskFunc* = proc(hint: string): tuple[identity: string, psk: string] + + SslServerGetPskFunc* = proc(identity: string): string + + SslContextExtraInternal = ref object of RootRef + serverGetPskFunc: SslServerGetPskFunc + clientGetPskFunc: SslClientGetPskFunc + +else: + type + SslContext* = ref object # TODO: Workaround #4797. const BufferSize*: int = 4000 ## size of a buffered socket's buffer + MaxLineLength* = 1_000_000 type - SocketImpl* = object ## socket type + SocketImpl* = object ## socket type fd: SocketHandle - case isBuffered: bool # determines whether this socket is buffered. - of true: - buffer: array[0..BufferSize, char] - currPos: int # current index in buffer - bufLen: int # current length of buffer - of false: nil - when defined(ssl): - case isSsl: bool - of true: - sslHandle: SSLPtr - sslContext: SSLContext - sslNoHandshake: bool # True if needs handshake. - sslHasPeekChar: bool - sslPeekChar: char - of false: nil + isBuffered: bool # determines whether this socket is buffered. + buffer: array[0..BufferSize, char] + currPos: int # current index in buffer + bufLen: int # current length of buffer + when defineSsl: + isSsl: bool + sslHandle: SslPtr + sslContext: SslContext + sslNoHandshake: bool # True if needs handshake. + sslHasPeekChar: bool + sslPeekChar: char + sslNoShutdown: bool # True if shutdown shouldn't be done. lastError: OSErrorCode ## stores the last error on this socket domain: Domain sockType: SockType @@ -72,52 +177,81 @@ type SOBool* = enum ## Boolean socket options. OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr + OptOOBInline, OptReuseAddr, OptReusePort, OptNoDelay ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone - TimeoutError* = object of Exception + TimeoutError* = object of CatchableError SocketFlag* {.pure.} = enum Peek, SafeDisconn ## Ensures disconnection exceptions (ECONNRESET, EPIPE etc) are not thrown. -{.deprecated: [TSocketFlags: SocketFlag, ETimeout: TimeoutError, - TReadLineResult: ReadLineResult, TSOBool: SOBool, PSocket: Socket, - TSocketImpl: SocketImpl].} +when defined(nimHasStyleChecks): + {.push styleChecks: off.} type IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address - IPv6, ## IPv6 address - IPv4 ## IPv4 address + IPv6, ## IPv6 address + IPv4 ## IPv4 address - IpAddress* = object ## stores an arbitrary IP address - case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) + IpAddress* = object ## stores an arbitrary IP address + case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) of IpAddressFamily.IPv6: address_v6*: array[0..15, uint8] ## Contains the IP address in bytes in ## case of IPv6 of IpAddressFamily.IPv4: - address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in - ## case of IPv4 -{.deprecated: [TIpAddress: IpAddress].} + address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in + ## case of IPv4 +when defined(nimHasStyleChecks): + {.pop.} + + +when defined(posix) and not defined(lwip): + from std/posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds + + template monitorPollEvent(x: var SocketHandle, y: cint, timeout: int): int = + var tpollfd: TPollfd + tpollfd.fd = cast[cint](x) + tpollfd.events = y + posix.poll(addr(tpollfd), Tnfds(1), timeout) + +proc timeoutRead(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectRead(fds, timeout) + else: + monitorPollEvent(fd, POLLIN or POLLPRI, timeout) + +proc timeoutWrite(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectWrite(fds, timeout) + else: + monitorPollEvent(fd, POLLOUT or POLLWRBAND, timeout) -proc isIpAddress*(address_str: string): bool {.tags: [].} -proc parseIpAddress*(address_str: string): IpAddress proc socketError*(socket: Socket, err: int = -1, async = false, - lastError = (-1).OSErrorCode): void + lastError = (-1).OSErrorCode, + flags: set[SocketFlag] = {}) {.gcsafe.} proc isDisconnectionError*(flags: set[SocketFlag], lastError: OSErrorCode): bool = - ## Determines whether ``lastError`` is a disconnection error. Only does this - ## if flags contains ``SafeDisconn``. + ## Determines whether `lastError` is a disconnection error. Only does this + ## if flags contains `SafeDisconn`. when useWinVersion: SocketFlag.SafeDisconn in flags and - lastError.int32 in {WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, - WSAEDISCON, ERROR_NETNAME_DELETED} + (lastError.int32 == WSAECONNRESET or + lastError.int32 == WSAECONNABORTED or + lastError.int32 == WSAENETRESET or + lastError.int32 == WSAEDISCON or + lastError.int32 == WSAESHUTDOWN or + lastError.int32 == ERROR_NETNAME_DELETED) else: SocketFlag.SafeDisconn in flags and - lastError.int32 in {ECONNRESET, EPIPE, ENETRESET} + (lastError.int32 == ECONNRESET or + lastError.int32 == EPIPE or + lastError.int32 == ENETRESET) proc toOSFlags*(socketFlags: set[SocketFlag]): cint = ## Converts the flags into the underlying OS representation. @@ -129,66 +263,351 @@ proc toOSFlags*(socketFlags: set[SocketFlag]): cint = proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): Socket = + protocol: Protocol = IPPROTO_TCP, buffered = true): owned(Socket) = ## Creates a new socket as specified by the params. assert fd != osInvalidSocket - new(result) - result.fd = fd - result.isBuffered = buffered - result.domain = domain - result.sockType = sockType - result.protocol = protocol + result = Socket( + fd: fd, + isBuffered: buffered, + domain: domain, + sockType: sockType, + protocol: protocol) if buffered: result.currPos = 0 -proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket = + # Set SO_NOSIGPIPE on OS X. + when defined(macosx) and not defined(nimdoc): + setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1) + +proc newSocket*(domain, sockType, protocol: cint, buffered = true, + inheritable = defined(nimInheritHandles)): owned(Socket) = ## Creates a new socket. ## - ## If an error occurs EOS will be raised. - let fd = newNativeSocket(domain, sockType, protocol) + ## The SocketHandle associated with the resulting Socket will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. + ## + ## If an error occurs OSError will be raised. + let fd = createNativeSocket(domain, sockType, protocol, inheritable) if fd == osInvalidSocket: raiseOSError(osLastError()) result = newSocket(fd, domain.Domain, sockType.SockType, protocol.Protocol, buffered) proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): Socket = + protocol: Protocol = IPPROTO_TCP, buffered = true, + inheritable = defined(nimInheritHandles)): owned(Socket) = ## Creates a new socket. ## - ## If an error occurs EOS will be raised. - let fd = newNativeSocket(domain, sockType, protocol) + ## The SocketHandle associated with the resulting Socket will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. + ## + ## If an error occurs OSError will be raised. + let fd = createNativeSocket(domain, sockType, protocol, inheritable) if fd == osInvalidSocket: raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -when defined(ssl): - CRYPTO_malloc_init() - SslLibraryInit() - SslLoadErrorStrings() - ErrLoadBioStrings() - OpenSSL_add_all_algorithms() +proc parseIPv4Address(addressStr: string): IpAddress = + ## Parses IPv4 addresses + ## Raises ValueError on errors + var + byteCount = 0 + currentByte: uint16 = 0 + separatorValid = false + leadingZero = false + + result = IpAddress(family: IpAddressFamily.IPv4) + + for i in 0 .. high(addressStr): + if addressStr[i] in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers are not allowed") + currentByte = currentByte * 10 + + cast[uint16](ord(addressStr[i]) - ord('0')) + if currentByte == 0'u16: + leadingZero = true + elif currentByte > 255'u16: + raise newException(ValueError, + "Invalid IP Address. Value is out of range") + separatorValid = true + elif addressStr[i] == '.': # IPv4 address separator + if not separatorValid or byteCount >= 3: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + result.address_v4[byteCount] = cast[uint8](currentByte) + currentByte = 0 + byteCount.inc + separatorValid = false + leadingZero = false + else: + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not separatorValid: + raise newException(ValueError, "Invalid IP Address") + result.address_v4[byteCount] = cast[uint8](currentByte) + +proc parseIPv6Address(addressStr: string): IpAddress = + ## Parses IPv6 addresses + ## Raises ValueError on errors + result = IpAddress(family: IpAddressFamily.IPv6) + if addressStr.len < 2: + raise newException(ValueError, "Invalid IP Address") + + var + groupCount = 0 + currentGroupStart = 0 + currentShort: uint32 = 0 + separatorValid = true + dualColonGroup = -1 + lastWasColon = false + v4StartPos = -1 + byteCount = 0 + + for i, c in addressStr: + if c == ':': + if not separatorValid: + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid separator") + if lastWasColon: + if dualColonGroup != -1: + raise newException(ValueError, + "Invalid IP Address. Address contains more than one \"::\" separator") + dualColonGroup = groupCount + separatorValid = false + elif i != 0 and i != high(addressStr): + if groupCount >= 8: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + currentShort = 0 + groupCount.inc() + if dualColonGroup != -1: separatorValid = false + elif i == 0: # only valid if address starts with :: + if addressStr[1] != ':': + raise newException(ValueError, + "Invalid IP Address. Address may not start with \":\"") + else: # i == high(addressStr) - only valid if address ends with :: + if addressStr[high(addressStr)-1] != ':': + raise newException(ValueError, + "Invalid IP Address. Address may not end with \":\"") + lastWasColon = true + currentGroupStart = i + 1 + elif c == '.': # Switch to parse IPv4 mode + if i < 3 or not separatorValid or groupCount >= 7: + raise newException(ValueError, "Invalid IP Address") + v4StartPos = currentGroupStart + currentShort = 0 + separatorValid = false + break + elif c in strutils.HexDigits: + if c in strutils.Digits: # Normal digit + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) + elif c >= 'a' and c <= 'f': # Lower case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 + else: # Upper case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 + if currentShort > 65535'u32: + raise newException(ValueError, + "Invalid IP Address. Value is out of range") + lastWasColon = false + separatorValid = true + else: + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid character") + + + if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff + if separatorValid: # Copy remaining data + if groupCount >= 8: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + groupCount.inc() + else: # Must parse IPv4 address + var leadingZero = false + for i, c in addressStr[v4StartPos..high(addressStr)]: + if c in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers not allowed") + currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) + if currentShort == 0'u32: + leadingZero = true + elif currentShort > 255'u32: + raise newException(ValueError, + "Invalid IP Address. Value is out of range") + separatorValid = true + elif c == '.': # IPv4 address separator + if not separatorValid or byteCount >= 3: + raise newException(ValueError, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + currentShort = 0 + byteCount.inc() + separatorValid = false + leadingZero = false + else: # Invalid character + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not separatorValid: + raise newException(ValueError, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + groupCount += 2 + + # Shift and fill zeros in case of :: + if groupCount > 8: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + elif groupCount < 8: # must fill + if dualColonGroup == -1: + raise newException(ValueError, + "Invalid IP Address. The address consists of too few groups") + var toFill = 8 - groupCount # The number of groups to fill + var toShift = groupCount - dualColonGroup # Nr of known groups after :: + for i in 0..2*toShift-1: # shift + result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] + for i in 0..2*toFill-1: # fill with 0s + result.address_v6[dualColonGroup*2+i] = 0 + elif dualColonGroup != -1: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + +proc parseIpAddress*(addressStr: string): IpAddress = + ## Parses an IP address + ## + ## Raises ValueError on error. + ## + ## For IPv4 addresses, only the strict form as + ## defined in RFC 6943 is considered valid, see + ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. + if addressStr.len == 0: + raise newException(ValueError, "IP Address string is empty") + if addressStr.contains(':'): + return parseIPv6Address(addressStr) + else: + return parseIPv4Address(addressStr) + +proc isIpAddress*(addressStr: string): bool {.tags: [].} = + ## Checks if a string is an IP address + ## Returns true if it is, false otherwise + try: + discard parseIpAddress(addressStr) + except ValueError: + return false + return true - proc raiseSSLError*(s = "") = +proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, + sl: var SockLen) = + ## Converts `IpAddress` and `Port` to `SockAddr` and `SockLen` + let port = htons(uint16(port)) + case address.family + of IpAddressFamily.IPv4: + sl = sizeof(Sockaddr_in).SockLen + let s = cast[ptr Sockaddr_in](addr sa) + s.sin_family = typeof(s.sin_family)(toInt(AF_INET)) + s.sin_port = port + copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], + sizeof(s.sin_addr)) + of IpAddressFamily.IPv6: + sl = sizeof(Sockaddr_in6).SockLen + let s = cast[ptr Sockaddr_in6](addr sa) + s.sin6_family = typeof(s.sin6_family)(toInt(AF_INET6)) + s.sin6_port = port + copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], + sizeof(s.sin6_addr)) + +proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: SockLen, + address: var IpAddress, port: var Port) = + if sa.ss_family.cint == toInt(AF_INET) and sl == sizeof(Sockaddr_in).SockLen: + address = IpAddress(family: IpAddressFamily.IPv4) + let s = cast[ptr Sockaddr_in](sa) + copyMem(addr address.address_v4[0], addr s.sin_addr, + sizeof(address.address_v4)) + port = ntohs(s.sin_port).Port + elif sa.ss_family.cint == toInt(AF_INET6) and + sl == sizeof(Sockaddr_in6).SockLen: + address = IpAddress(family: IpAddressFamily.IPv6) + let s = cast[ptr Sockaddr_in6](sa) + copyMem(addr address.address_v6[0], addr s.sin6_addr, + sizeof(address.address_v6)) + port = ntohs(s.sin6_port).Port + else: + raise newException(ValueError, "Neither IPv4 nor IPv6") + +proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6, + sl: SockLen, address: var IpAddress, port: var Port) {.inline.} = + ## Converts `SockAddr` and `SockLen` to `IpAddress` and `Port`. Raises + ## `ObjectConversionDefect` in case of invalid `sa` and `sl` arguments. + fromSockAddrAux(cast[ptr Sockaddr_storage](unsafeAddr sa), sl, address, port) + +when defineSsl: + # OpenSSL >= 1.1.0 does not need explicit init. + when not useOpenssl3: + CRYPTO_malloc_init() + doAssert SslLibraryInit() == 1 + SSL_load_error_strings() + ERR_load_BIO_strings() + OpenSSL_add_all_algorithms() + + proc sslHandle*(self: Socket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + + proc raiseSSLError*(s = "") {.raises: [SslError].}= ## Raises a new SSL error. if s != "": - raise newException(SSLError, s) - let err = ErrPeekLastError() + raise newException(SslError, s) + let err = ERR_peek_last_error() if err == 0: - raise newException(SSLError, "No error reported.") - if err == -1: - raiseOSError(osLastError()) - var errStr = ErrErrorString(err, nil) - raise newException(SSLError, $errStr) + raise newException(SslError, "No error reported.") + var errStr = $ERR_error_string(err, nil) + case err + of 336032814, 336032784: + errStr = "Please upgrade your OpenSSL library, it does not support the " & + "necessary protocols. OpenSSL error is: " & errStr + else: + discard + raise newException(SslError, errStr) + + proc getExtraData*(ctx: SslContext, index: int): RootRef = + ## Retrieves arbitrary data stored inside SslContext. + if index notin ctx.referencedData: + raise newException(IndexDefect, "No data with that index.") + let res = ctx.context.SSL_CTX_get_ex_data(index.cint) + if cast[int](res) == 0: + raiseSSLError() + return cast[RootRef](res) + + proc setExtraData*(ctx: SslContext, index: int, data: RootRef) = + ## Stores arbitrary data inside SslContext. The unique `index` + ## should be retrieved using getSslContextExtraDataIndex. + if index in ctx.referencedData: + GC_unref(getExtraData(ctx, index)) + + if ctx.context.SSL_CTX_set_ex_data(index.cint, cast[pointer](data)) == -1: + raiseSSLError() + + if index notin ctx.referencedData: + ctx.referencedData.incl(index) + GC_ref(data) # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html - proc loadCertificates(ctx: SSL_CTX, certFile, keyFile: string) = - if certFile != "" and not existsFile(certFile): - raise newException(system.IOError, "Certificate file could not be found: " & certFile) - if keyFile != "" and not existsFile(keyFile): + proc loadCertificates(ctx: SslCtx, certFile, keyFile: string) = + if certFile != "" and not fileExists(certFile): + raise newException(system.IOError, + "Certificate file could not be found: " & certFile) + if keyFile != "" and not fileExists(keyFile): raise newException(system.IOError, "Key file could not be found: " & keyFile) if certFile != "": - var ret = SSLCTXUseCertificateChainFile(ctx, certFile) + var ret = SSL_CTX_use_certificate_chain_file(ctx, certFile) if ret != 1: raiseSSLError() @@ -202,90 +621,318 @@ when defined(ssl): raiseSSLError("Verification of private key file failed.") proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, - certFile = "", keyFile = ""): SSLContext = + certFile = "", keyFile = "", cipherList = CiphersIntermediate, + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## - ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 - ## are available with the addition of ``protSSLv23`` which allows for - ## compatibility with all of them. + ## Protocol version is currently ignored by default and TLS is used. + ## With `-d:openssl10`, only SSLv23 and TLSv1 may be used. + ## + ## There are three options for verify mode: + ## `CVerifyNone`: certificates are not verified; + ## `CVerifyPeer`: certificates are verified; + ## `CVerifyPeerUseEnvVars`: certificates are verified and the optional + ## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to + ## locate certificates ## - ## There are currently only two options for verify mode; - ## one is ``CVerifyNone`` and with it certificates will not be verified - ## the other is ``CVerifyPeer`` and certificates will be verified for - ## it, ``CVerifyPeer`` is the safest choice. + ## The `nimDisableCertificateValidation` define overrides verifyMode and + ## disables certificate verification globally! + ## + ## CA certificates will be loaded, in the following order, from: + ## + ## - caFile, caDir, parameters, if set + ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`, + ## the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used + ## - a set of files and directories from the `ssl_certs <ssl_certs.html>`_ file. ## ## The last two parameters specify the certificate file path and the key file ## path, a server socket will most likely not work without these. + ## ## Certificates can be generated using the following command: - ## ``openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.pem -out mycert.pem``. - var newCTX: SSL_CTX - case protVersion - of protSSLv23: - newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. - of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") - of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) - of protTLSv1: - newCTX = SSL_CTX_new(TLSv1_method()) - - if newCTX.SSLCTXSetCipherList("ALL") != 1: + ## - `openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout mykey.pem -out mycert.pem` + ## or using ECDSA: + ## - `openssl ecparam -out mykey.pem -name secp256k1 -genkey` + ## - `openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem` + var mtd: PSSL_METHOD + when defined(openssl10): + case protVersion + of protSSLv23: + mtd = SSLv23_method() + of protSSLv2: + raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") + of protSSLv3: + raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") + of protTLSv1: + mtd = TLSv1_method() + else: + mtd = TLS_method() + if mtd == nil: + raiseSSLError("Failed to create TLS context") + var newCTX = SSL_CTX_new(mtd) + if newCTX == nil: + raiseSSLError("Failed to create TLS context") + + if newCTX.SSL_CTX_set_cipher_list(cipherList) != 1: + raiseSSLError() + when not defined(openssl10) and not defined(libressl): + let sslVersion = getOpenSSLVersion() + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: + # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via + # this API. + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: + raiseSSLError() + # Automatically the best ECDH curve for client exchange. Without this, ECDH + # ciphers will be ignored by the server. + # + # From OpenSSL >= 1.1.0, this setting is set by default and can't be + # overridden. + if newCTX.SSL_CTX_set_ecdh_auto(1) != 1: raiseSSLError() - case verifyMode - of CVerifyPeer: - newCTX.SSLCTXSetVerify(SSLVerifyPeer, nil) - of CVerifyNone: - newCTX.SSLCTXSetVerify(SSLVerifyNone, nil) + + when defined(nimDisableCertificateValidation): + newCTX.SSL_CTX_set_verify(SSL_VERIFY_NONE, nil) + else: + case verifyMode + of CVerifyPeer, CVerifyPeerUseEnvVars: + newCTX.SSL_CTX_set_verify(SSL_VERIFY_PEER, nil) + of CVerifyNone: + newCTX.SSL_CTX_set_verify(SSL_VERIFY_NONE, nil) + if newCTX == nil: raiseSSLError() discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) newCTX.loadCertificates(certFile, keyFile) - return SSLContext(newCTX) - proc wrapSocket*(ctx: SSLContext, socket: Socket) = + const VerifySuccess = 1 # SSL_CTX_load_verify_locations returns 1 on success. + + when not defined(nimDisableCertificateValidation): + if verifyMode != CVerifyNone: + # Use the caDir and caFile parameters if set + if caDir != "" or caFile != "": + if newCTX.SSL_CTX_load_verify_locations(if caFile == "": nil else: caFile.cstring, if caDir == "": nil else: caDir.cstring) != VerifySuccess: + raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).") + + else: + # Scan for certs in known locations. For CVerifyPeerUseEnvVars also scan + # the SSL_CERT_FILE and SSL_CERT_DIR env vars + var found = false + let useEnvVars = (if verifyMode == CVerifyPeerUseEnvVars: true else: false) + for fn in scanSSLCertificates(useEnvVars = useEnvVars): + if fn.extractFilename == "": + if newCTX.SSL_CTX_load_verify_locations(nil, cstring(fn.normalizePathEnd(false))) == VerifySuccess: + found = true + break + elif newCTX.SSL_CTX_load_verify_locations(cstring(fn), nil) == VerifySuccess: + found = true + break + if not found: + raise newException(IOError, "No SSL/TLS CA certificates found.") + + result = SslContext(context: newCTX, referencedData: initHashSet[int](), + extraInternal: new(SslContextExtraInternal)) + + proc getExtraInternal(ctx: SslContext): SslContextExtraInternal = + return ctx.extraInternal + + proc destroyContext*(ctx: SslContext) = + ## Free memory referenced by SslContext. + + # We assume here that OpenSSL's internal indexes increase by 1 each time. + # That means we can assume that the next internal index is the length of + # extra data indexes. + for i in ctx.referencedData: + GC_unref(getExtraData(ctx, i)) + ctx.context.SSL_CTX_free() + + proc `pskIdentityHint=`*(ctx: SslContext, hint: string) = + ## Sets the identity hint passed to server. + ## + ## Only used in PSK ciphersuites. + if ctx.context.SSL_CTX_use_psk_identity_hint(hint) <= 0: + raiseSSLError() + + proc clientGetPskFunc*(ctx: SslContext): SslClientGetPskFunc = + return ctx.getExtraInternal().clientGetPskFunc + + proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; + max_identity_len: cuint; psk: ptr uint8; + max_psk_len: cuint): cuint {.cdecl.} = + let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) + let hintString = if hint == nil: "" else: $hint + let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) + if pskString.len.cuint > max_psk_len: + return 0 + if identityString.len.cuint >= max_identity_len: + return 0 + copyMem(identity, identityString.cstring, identityString.len + 1) # with the last zero byte + copyMem(psk, pskString.cstring, pskString.len) + + return pskString.len.cuint + + proc `clientGetPskFunc=`*(ctx: SslContext, fun: SslClientGetPskFunc) = + ## Sets function that returns the client identity and the PSK based on identity + ## hint from the server. + ## + ## Only used in PSK ciphersuites. + ctx.getExtraInternal().clientGetPskFunc = fun + ctx.context.SSL_CTX_set_psk_client_callback( + if fun == nil: nil else: pskClientCallback) + + proc serverGetPskFunc*(ctx: SslContext): SslServerGetPskFunc = + return ctx.getExtraInternal().serverGetPskFunc + + proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr uint8; + max_psk_len: cint): cuint {.cdecl.} = + let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) + let pskString = (ctx.serverGetPskFunc)($identity) + if pskString.len.cint > max_psk_len: + return 0 + copyMem(psk, pskString.cstring, pskString.len) + + return pskString.len.cuint + + proc `serverGetPskFunc=`*(ctx: SslContext, fun: SslServerGetPskFunc) = + ## Sets function that returns PSK based on the client identity. + ## + ## Only used in PSK ciphersuites. + ctx.getExtraInternal().serverGetPskFunc = fun + ctx.context.SSL_CTX_set_psk_server_callback(if fun == nil: nil + else: pskServerCallback) + + proc getPskIdentity*(socket: Socket): string = + ## Gets the PSK identity provided by the client. + assert socket.isSsl + return $(socket.sslHandle.SSL_get_psk_identity) + + proc wrapSocket*(ctx: SslContext, socket: Socket) = ## Wraps a socket in an SSL context. This function effectively turns - ## ``socket`` into an SSL socket. + ## `socket` into an SSL socket. ## ## This must be called on an unconnected socket; an SSL session will ## be started when the socket is connected. ## + ## FIXME: ## **Disclaimer**: This code is not well tested, may be very unsafe and ## prone to security vulnerabilities. - assert (not socket.isSSL) - socket.isSSL = true + assert(not socket.isSsl) + socket.isSsl = true socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) + socket.sslHandle = SSL_new(socket.sslContext.context) socket.sslNoHandshake = false socket.sslHasPeekChar = false + socket.sslNoShutdown = false if socket.sslHandle == nil: raiseSSLError() - if SSLSetFd(socket.sslHandle, socket.fd) != 1: + if SSL_set_fd(socket.sslHandle, socket.fd) != 1: raiseSSLError() - proc wrapConnectedSocket*(ctx: SSLContext, socket: Socket, - handshake: SslHandshakeType) = + proc checkCertName(socket: Socket, hostname: string) {.raises: [SslError], tags:[RootEffect].} = + ## Check if the certificate Subject Alternative Name (SAN) or Subject CommonName (CN) matches hostname. + ## Wildcards match only in the left-most label. + ## When name starts with a dot it will be matched by a certificate valid for any subdomain + when not defined(nimDisableCertificateValidation) and not defined(windows): + assert socket.isSsl + try: + let certificate = socket.sslHandle.SSL_get_peer_certificate() + if certificate.isNil: + raiseSSLError("No SSL certificate found.") + + const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint + # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html + let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil) + # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html + X509_free(certificate) + if match != 1: + raiseSSLError("SSL Certificate check failed.") + + except LibraryError: + raiseSSLError("SSL import failed") + + proc wrapConnectedSocket*(ctx: SslContext, socket: Socket, + handshake: SslHandshakeType, + hostname: string = "") = ## Wraps a connected socket in an SSL context. This function effectively - ## turns ``socket`` into an SSL socket. + ## turns `socket` into an SSL socket. + ## `hostname` should be specified so that the client knows which hostname + ## the server certificate should be validated against. ## ## This should be called on a connected socket, and will perform ## an SSL handshake immediately. ## + ## FIXME: ## **Disclaimer**: This code is not well tested, may be very unsafe and ## prone to security vulnerabilities. wrapSocket(ctx, socket) case handshake of handshakeAsClient: - let ret = SSLConnect(socket.sslHandle) + if hostname.len > 0 and not isIpAddress(hostname): + # Discard result in case OpenSSL version doesn't support SNI, or we're + # not using TLSv1+ + discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) + ErrClearError() + let ret = SSL_connect(socket.sslHandle) socketError(socket, ret) + when not defined(nimDisableCertificateValidation) and not defined(windows): + # FIXME: this should be skipped on CVerifyNone + if hostname.len > 0 and not isIpAddress(hostname): + socket.checkCertName(hostname) of handshakeAsServer: - let ret = SSLAccept(socket.sslHandle) + ErrClearError() + let ret = SSL_accept(socket.sslHandle) socketError(socket, ret) + proc getPeerCertificates*(sslHandle: SslPtr): seq[Certificate] {.since: (1, 1).} = + ## Returns the certificate chain received by the peer we are connected to + ## through the OpenSSL connection represented by `sslHandle`. + ## The handshake must have been completed and the certificate chain must + ## have been verified successfully or else an empty sequence is returned. + ## The chain is ordered from leaf certificate to root certificate. + result = newSeq[Certificate]() + if SSL_get_verify_result(sslHandle) != X509_V_OK: + return + let stack = SSL_get0_verified_chain(sslHandle) + if stack == nil: + return + let length = OPENSSL_sk_num(stack) + if length == 0: + return + for i in 0 .. length - 1: + let x509 = cast[PX509](OPENSSL_sk_value(stack, i)) + result.add(i2d_X509(x509)) + + proc getPeerCertificates*(socket: Socket): seq[Certificate] {.since: (1, 1).} = + ## Returns the certificate chain received by the peer we are connected to + ## through the given socket. + ## The handshake must have been completed and the certificate chain must + ## have been verified successfully or else an empty sequence is returned. + ## The chain is ordered from leaf certificate to root certificate. + if not socket.isSsl: + result = newSeq[Certificate]() + else: + result = getPeerCertificates(socket.sslHandle) + + proc `sessionIdContext=`*(ctx: SslContext, sidCtx: string) = + ## Sets the session id context in which a session can be reused. + ## Used for permitting clients to reuse a session id instead of + ## doing a new handshake. + ## + ## TLS clients might attempt to resume a session using the session id context, + ## thus it must be set if verifyMode is set to CVerifyPeer or CVerifyPeerUseEnvVars, + ## otherwise the connection will fail and SslError will be raised if resumption occurs. + ## + ## - Only useful if set server-side. + ## - Should be unique per-application to prevent clients from malfunctioning. + ## - sidCtx must be at most 32 characters in length. + if sidCtx.len > 32: + raiseSSLError("sessionIdContext must be shorter than 32 characters") + SSL_CTX_set_session_id_context(ctx.context, sidCtx, sidCtx.len) + proc getSocketError*(socket: Socket): OSErrorCode = - ## Checks ``osLastError`` for a valid error. If it has been reset it uses + ## Checks `osLastError` for a valid error. If it has been reset it uses ## the last error stored in the socket object. result = osLastError() if result == 0.OSErrorCode: @@ -294,18 +941,22 @@ proc getSocketError*(socket: Socket): OSErrorCode = raiseOSError(result, "No valid socket error code available") proc socketError*(socket: Socket, err: int = -1, async = false, - lastError = (-1).OSErrorCode) = - ## Raises an OSError based on the error code returned by ``SSLGetError`` - ## (for SSL sockets) and ``osLastError`` otherwise. + lastError = (-1).OSErrorCode, + flags: set[SocketFlag] = {}) = + ## Raises an OSError based on the error code returned by `SSL_get_error` + ## (for SSL sockets) and `osLastError` otherwise. ## - ## If ``async`` is ``true`` no error will be thrown in the case when the + ## If `async` is `true` no error will be thrown in the case when the ## error was caused by no data being available to be read. ## - ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): - if socket.isSSL: + ## If `err` is not lower than 0 no exception will be raised. + ## + ## If `flags` contains `SafeDisconn`, no exception will be raised + ## when the error was caused by a peer disconnection. + when defineSsl: + if socket.isSsl: if err <= 0: - var ret = SSLGetError(socket.sslHandle, err.cint) + var ret = SSL_get_error(socket.sslHandle, err.cint) case ret of SSL_ERROR_ZERO_RETURN: raiseSSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") @@ -320,141 +971,147 @@ proc socketError*(socket: Socket, err: int = -1, async = false, of SSL_ERROR_WANT_X509_LOOKUP: raiseSSLError("Function for x509 lookup has been called.") of SSL_ERROR_SYSCALL: - var errStr = "IO error has occurred " - let sslErr = ErrPeekLastError() - if sslErr == 0 and err == 0: - errStr.add "because an EOF was observed that violates the protocol" - elif sslErr == 0 and err == -1: - errStr.add "in the BIO layer" - else: - let errStr = $ErrErrorString(sslErr, nil) - raiseSSLError(errStr & ": " & errStr) + # SSL shutdown must not be done if a fatal error occurred. + socket.sslNoShutdown = true let osErr = osLastError() - raiseOSError(osErr, errStr) + if not flags.isDisconnectionError(osErr): + var errStr = "IO error has occurred " + let sslErr = ERR_peek_last_error() + if sslErr == 0 and err == 0: + errStr.add "because an EOF was observed that violates the protocol" + elif sslErr == 0 and err == -1: + errStr.add "in the BIO layer" + else: + let errStr = $ERR_error_string(sslErr, nil) + raiseSSLError(errStr & ": " & errStr) + raiseOSError(osErr, errStr) of SSL_ERROR_SSL: + # SSL shutdown must not be done if a fatal error occurred. + socket.sslNoShutdown = true raiseSSLError() else: raiseSSLError("Unknown Error") - if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if err == -1 and not (when defineSsl: socket.isSsl else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError - if async: - when useWinVersion: - if lastE.int32 == WSAEWOULDBLOCK: - return - else: raiseOSError(lastE) - else: - if lastE.int32 == EAGAIN or lastE.int32 == EWOULDBLOCK: - return - else: raiseOSError(lastE) - else: raiseOSError(lastE) + if not flags.isDisconnectionError(lastE): + if async: + when useWinVersion: + if lastE.int32 == WSAEWOULDBLOCK: + return + else: raiseOSError(lastE) + else: + if lastE.int32 == EAGAIN or lastE.int32 == EWOULDBLOCK: + return + else: raiseOSError(lastE) + else: raiseOSError(lastE) proc listen*(socket: Socket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the + ## Marks `socket` as accepting connections. + ## `Backlog` specifies the maximum length of the ## queue of pending connections. ## - ## Raises an EOS error upon failure. + ## Raises an OSError error upon failure. if nativesockets.listen(socket.fd, backlog) < 0'i32: raiseOSError(osLastError()) proc bindAddr*(socket: Socket, port = Port(0), address = "") {. tags: [ReadIOEffect].} = - ## Binds ``address``:``port`` to the socket. + ## Binds `address`:`port` to the socket. ## - ## If ``address`` is "" then ADDR_ANY will be bound. - - if address == "": - var name: Sockaddr_in - when useWinVersion: - name.sin_family = toInt(AF_INET).int16 + ## If `address` is "" then ADDR_ANY will be bound. + var realaddr = address + if realaddr == "": + case socket.domain + of AF_INET6: realaddr = "::" + of AF_INET: realaddr = "0.0.0.0" else: - name.sin_family = toInt(AF_INET) - name.sin_port = htons(int16(port)) - name.sin_addr.s_addr = htonl(INADDR_ANY) - if bindAddr(socket.fd, cast[ptr SockAddr](addr(name)), - sizeof(name).SockLen) < 0'i32: - raiseOSError(osLastError()) - else: - var aiList = getAddrInfo(address, port, socket.domain) - if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - dealloc(aiList) - raiseOSError(osLastError()) - dealloc(aiList) - -proc acceptAddr*(server: Socket, client: var Socket, address: var string, - flags = {SocketFlag.SafeDisconn}) {. - tags: [ReadIOEffect], gcsafe, locks: 0.} = + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") + + var aiList = getAddrInfo(realaddr, port, socket.domain) + if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: + freeAddrInfo(aiList) + var address2: string + address2.addQuoted address + raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) + freeAddrInfo(aiList) + +proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, + flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)) {. + tags: [ReadIOEffect], gcsafe.} = ## Blocks until a connection is being made from a client. When a connection - ## is made sets ``client`` to the client socket and ``address`` to the address + ## is made sets `client` to the client socket and `address` to the address ## of the connecting client. - ## This function will raise EOS if an error occurs. + ## This function will raise OSError if an error occurs. ## ## The resulting client will inherit any properties of the server socket. For ## example: whether the socket is buffered or not. ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. + ## The SocketHandle associated with the resulting client will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## The `accept` call may result in an error if the connecting socket + ## disconnects during the duration of the `accept`. If the `SafeDisconn` ## flag is specified then this error will not be raised and instead ## accept will be called again. - assert(client != nil) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) + if client.isNil: + new(client) + let ret = accept(server.fd, inheritable) + let sock = ret[0] if sock == osInvalidSocket: let err = osLastError() if flags.isDisconnectionError(err): - acceptAddr(server, client, address, flags) + acceptAddr(server, client, address, flags, inheritable) raiseOSError(err) else: + address = ret[1] client.fd = sock + client.domain = getSockDomain(sock) client.isBuffered = server.isBuffered # Handle SSL. - when defined(ssl): - if server.isSSL: + when defineSsl: + if server.isSsl: # We must wrap the client sock in a ssl context. server.sslContext.wrapSocket(client) - let ret = SSLAccept(client.sslHandle) + ErrClearError() + let ret = SSL_accept(client.sslHandle) socketError(client, ret, false) - # Client socket is set above. - address = $inet_ntoa(sockAddress.sin_addr) - -when false: #defined(ssl): +when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, - address: var string): SSLAcceptResult {. + address: var string): SSL_acceptResult {. tags: [ReadIOEffect].} = ## This procedure should only be used for non-blocking **SSL** sockets. ## It will immediately return with one of the following values: ## - ## ``AcceptSuccess`` will be returned when a client has been successfully + ## `AcceptSuccess` will be returned when a client has been successfully ## accepted and the handshake has been successfully performed between - ## ``server`` and the newly connected client. + ## `server` and the newly connected client. ## - ## ``AcceptNoHandshake`` will be returned when a client has been accepted + ## `AcceptNoHandshake` will be returned when a client has been accepted ## but no handshake could be performed. This can happen when the client ## connects but does not yet initiate a handshake. In this case - ## ``acceptAddrSSL`` should be called again with the same parameters. + ## `acceptAddrSSL` should be called again with the same parameters. ## - ## ``AcceptNoClient`` will be returned when no client is currently attempting + ## `AcceptNoClient` will be returned when no client is currently attempting ## to connect. - template doHandshake(): stmt = - when defined(ssl): - if server.isSSL: + template doHandshake(): untyped = + when defineSsl: + if server.isSsl: client.setBlocking(false) # We must wrap the client sock in a ssl context. - if not client.isSSL or client.sslHandle == nil: + if not client.isSsl or client.sslHandle == nil: server.sslContext.wrapSocket(client) - let ret = SSLAccept(client.sslHandle) + ErrClearError() + let ret = SSL_accept(client.sslHandle) while ret <= 0: - let err = SSLGetError(client.sslHandle, ret) + let err = SSL_get_error(client.sslHandle, ret) if err != SSL_ERROR_WANT_ACCEPT: case err of SSL_ERROR_ZERO_RETURN: @@ -471,50 +1128,152 @@ when false: #defined(ssl): raiseSSLError("Unknown error") client.sslNoHandshake = false - if client.isSSL and client.sslNoHandshake: + if client.isSsl and client.sslNoHandshake: doHandshake() return AcceptSuccess else: acceptAddrPlain(AcceptNoClient, AcceptSuccess): doHandshake() -proc accept*(server: Socket, client: var Socket, - flags = {SocketFlag.SafeDisconn}) {.tags: [ReadIOEffect].} = - ## Equivalent to ``acceptAddr`` but doesn't return the address, only the +proc accept*(server: Socket, client: var owned(Socket), + flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)) + {.tags: [ReadIOEffect].} = + ## Equivalent to `acceptAddr` but doesn't return the address, only the ## socket. ## - ## **Note**: ``client`` must be initialised (with ``new``), this function - ## makes no effort to initialise the ``client`` variable. + ## The SocketHandle associated with the resulting client will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## The `accept` call may result in an error if the connecting socket + ## disconnects during the duration of the `accept`. If the `SafeDisconn` ## flag is specified then this error will not be raised and instead ## accept will be called again. var addrDummy = "" acceptAddr(server, client, addrDummy, flags) -proc close*(socket: Socket) = +when defined(posix) and not defined(lwip): + from std/posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, + sigprocmask, pthread_sigmask, SIGPIPE, SIG_BLOCK, SIG_UNBLOCK + +template blockSigpipe(body: untyped): untyped = + ## Temporary block SIGPIPE within the provided code block. If SIGPIPE is + ## raised for the duration of the code block, it will be queued and will be + ## raised once the block ends. + ## + ## Within the block a `selectSigpipe()` template is provided which can be + ## used to remove SIGPIPE from the queue. Note that if SIGPIPE is **not** + ## raised at the time of call, it will block until SIGPIPE is raised. + ## + ## If SIGPIPE has already been blocked at the time of execution, the + ## signal mask is left as-is and `selectSigpipe()` will become a no-op. + ## + ## For convenience, this template is also available for non-POSIX system, + ## where `body` will be executed as-is. + when not defined(posix) or defined(lwip): + body + else: + template sigmask(how: cint, set, oset: var Sigset): untyped {.gensym.} = + ## Alias for pthread_sigmask or sigprocmask depending on the status + ## of --threads + when compileOption("threads"): + pthread_sigmask(how, set, oset) + else: + sigprocmask(how, set, oset) + + var oldSet, watchSet: Sigset + if sigemptyset(oldSet) == -1: + raiseOSError(osLastError()) + if sigemptyset(watchSet) == -1: + raiseOSError(osLastError()) + + if sigaddset(watchSet, SIGPIPE) == -1: + raiseOSError(osLastError(), "Couldn't add SIGPIPE to Sigset") + + if sigmask(SIG_BLOCK, watchSet, oldSet) == -1: + raiseOSError(osLastError(), "Couldn't block SIGPIPE") + + let alreadyBlocked = sigismember(oldSet, SIGPIPE) == 1 + + template selectSigpipe(): untyped {.used.} = + if not alreadyBlocked: + var signal: cint + let err = sigwait(watchSet, signal) + if err != 0: + raiseOSError(err.OSErrorCode, "Couldn't select SIGPIPE") + assert signal == SIGPIPE + + try: + body + finally: + if not alreadyBlocked: + if sigmask(SIG_UNBLOCK, watchSet, oldSet) == -1: + raiseOSError(osLastError(), "Couldn't unblock SIGPIPE") + +proc close*(socket: Socket, flags = {SocketFlag.SafeDisconn}) = ## Closes a socket. + ## + ## If `socket` is an SSL/TLS socket, this proc will also send a closure + ## notification to the peer. If `SafeDisconn` is in `flags`, failure to do so + ## due to disconnections will be ignored. This is generally safe in + ## practice. See + ## `here <https://security.stackexchange.com/a/82044>`_ for more details. try: - when defined(ssl): - if socket.isSSL: - ErrClearError() - # As we are closing the underlying socket immediately afterwards, - # it is valid, under the TLS standard, to perform a unidirectional - # shutdown i.e not wait for the peers "close notify" alert with a second - # call to SSLShutdown - let res = SSLShutdown(socket.sslHandle) - SSLFree(socket.sslHandle) - socket.sslHandle = nil - if res == 0: - discard - elif res != 1: - socketError(socket, res) + when defineSsl: + if socket.isSsl and socket.sslHandle != nil: + # Don't call SSL_shutdown if the connection has not been fully + # established, see: + # https://github.com/openssl/openssl/issues/710#issuecomment-253897666 + if not socket.sslNoShutdown and SSL_in_init(socket.sslHandle) == 0: + # As we are closing the underlying socket immediately afterwards, + # it is valid, under the TLS standard, to perform a unidirectional + # shutdown i.e not wait for the peers "close notify" alert with a second + # call to SSL_shutdown + blockSigpipe: + ErrClearError() + let res = SSL_shutdown(socket.sslHandle) + if res == 0: + discard + elif res != 1: + let + err = osLastError() + sslError = SSL_get_error(socket.sslHandle, res) + + # If a close notification is received, failures outside of the + # protocol will be returned as SSL_ERROR_ZERO_RETURN instead + # of SSL_ERROR_SYSCALL. This fact is deduced by digging into + # SSL_get_error() source code. + if sslError == SSL_ERROR_ZERO_RETURN or + sslError == SSL_ERROR_SYSCALL: + when defined(posix) and not defined(macosx) and + not defined(nimdoc): + if err == EPIPE.OSErrorCode: + # Clear the SIGPIPE that's been raised due to + # the disconnection. + selectSigpipe() + else: + discard + if not flags.isDisconnectionError(err): + socketError(socket, res, lastError = err, flags = flags) + else: + socketError(socket, res, lastError = err, flags = flags) finally: + when defineSsl: + if socket.isSsl and socket.sslHandle != nil: + SSL_free(socket.sslHandle) + socket.sslHandle = nil + socket.fd.close() + socket.fd = osInvalidSocket + +when defined(posix): + from std/posix import TCP_NODELAY +else: + from std/winlean import TCP_NODELAY proc toCInt*(opt: SOBool): cint = - ## Converts a ``SOBool`` into its Socket Option cint representation. + ## Converts a `SOBool` into its Socket Option cint representation. case opt of OptAcceptConn: SO_ACCEPTCONN of OptBroadcast: SO_BROADCAST @@ -523,10 +1282,12 @@ proc toCInt*(opt: SOBool): cint = of OptKeepAlive: SO_KEEPALIVE of OptOOBInline: SO_OOBINLINE of OptReuseAddr: SO_REUSEADDR + of OptReusePort: SO_REUSEPORT + of OptNoDelay: TCP_NODELAY proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = - ## Retrieves option ``opt`` as a boolean value. + ## Retrieves option `opt` as a boolean value. var res = getSockOptInt(socket.fd, cint(level), toCInt(opt)) result = res != 0 @@ -536,89 +1297,49 @@ proc getLocalAddr*(socket: Socket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: Socket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) - -proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {. - tags: [WriteIOEffect].} = - ## Sets option ``opt`` to a boolean value specified by ``value``. +when not useNimNetLite: + proc getPeerAddr*(socket: Socket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) + +proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, + level = SOL_SOCKET) {.tags: [WriteIOEffect].} = + ## Sets option `opt` to a boolean value specified by `value`. + runnableExamples("-r:off"): + let socket = newSocket() + socket.setSockOpt(OptReusePort, true) + socket.setSockOpt(OptNoDelay, true, level = IPPROTO_TCP.cint) var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -proc connect*(socket: Socket, address: string, - port = Port(0)) {.tags: [ReadIOEffect].} = - ## Connects socket to ``address``:``port``. ``Address`` can be an IP address or a - ## host name. If ``address`` is a host name, this function will try each IP - ## of that host name. ``htons`` is already performed on ``port`` so you must - ## not do it. - ## - ## If ``socket`` is an SSL socket a handshake will be automatically performed. - var aiList = getAddrInfo(address, port, socket.domain) - # try all possibilities: - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - if connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) == 0'i32: - success = true - break - else: lastError = osLastError() - it = it.ai_next - - dealloc(aiList) - if not success: raiseOSError(lastError) - - when defined(ssl): - if socket.isSSL: - # RFC3546 for SNI specifies that IP addresses are not allowed. - if not isIpAddress(address): - # Discard result in case OpenSSL version doesn't support SNI, or we're - # not using TLSv1+ - discard SSL_set_tlsext_host_name(socket.sslHandle, address) - - let ret = SSLConnect(socket.sslHandle) - socketError(socket, ret) - -when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = - ## This proc needs to be called on a socket after it connects. This is - ## only applicable when using ``connectAsync``. - ## This proc performs the SSL handshake. - ## - ## Returns ``False`` whenever the socket is not yet ready for a handshake, - ## ``True`` whenever handshake completed successfully. - ## - ## A ESSL error is raised on any other errors. - result = true - if socket.isSSL: - var ret = SSLConnect(socket.sslHandle) - if ret <= 0: - var errret = SSLGetError(socket.sslHandle, ret) - case errret - of SSL_ERROR_ZERO_RETURN: - raiseSSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") - of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT, - SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE: - return false - of SSL_ERROR_WANT_X509_LOOKUP: - raiseSSLError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: - raiseSSLError() - else: - raiseSSLError("Unknown Error") - socket.sslNoHandshake = false - else: - raiseSSLError("Socket is not an SSL socket.") - +when defined(nimdoc) or (defined(posix) and not useNimNetLite): + proc connectUnix*(socket: Socket, path: string) = + ## Connects to Unix socket on `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.connect(cast[ptr SockAddr](addr socketAddr), + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: + raiseOSError(osLastError()) + + proc bindUnix*(socket: Socket, path: string) = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: + raiseOSError(osLastError()) + +when defineSsl: proc gotHandshake*(socket: Socket): bool = - ## Determines whether a handshake has occurred between a client (``socket``) - ## and the server that ``socket`` is connected to. + ## Determines whether a handshake has occurred between a client (`socket`) + ## and the server that `socket` is connected to. ## - ## Throws ESSL if ``socket`` is not an SSL socket. - if socket.isSSL: + ## Throws SslError if `socket` is not an SSL socket. + if socket.isSsl: return not socket.sslNoHandshake else: raiseSSLError("Socket is not an SSL socket.") @@ -629,29 +1350,32 @@ proc hasDataBuffered*(s: Socket): bool = if s.isBuffered: result = s.bufLen > 0 and s.currPos != s.bufLen - when defined(ssl): - if s.isSSL and not result: + when defineSsl: + if s.isSsl and not result: result = s.sslHasPeekChar -proc select(readfd: Socket, timeout = 500): int = - ## Used for socket operation timeouts. - if readfd.hasDataBuffered: - return 1 +proc isClosed(socket: Socket): bool = + socket.fd == osInvalidSocket + +proc uniRecv(socket: Socket, buffer: pointer, size, flags: cint): int = + ## Handles SSL and non-ssl recv in a nice package. + ## + ## In particular handles the case where socket has been closed properly + ## for both SSL and non-ssl. + result = 0 + assert(not socket.isClosed, "Cannot `recv` on a closed socket") + when defineSsl: + if socket.isSsl: + ErrClearError() + return SSL_read(socket.sslHandle, buffer, size) - var fds = @[readfd.fd] - result = select(fds, timeout) + return recv(socket.fd, buffer, size, flags) proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defined(ssl): - if socket.isSSL: - result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) - else: - result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) + result = uniRecv(socket, addr(socket.buffer), socket.buffer.high, flags) if result < 0: - # Save it in case it gets reset (the Nim codegen occassionally may call + # Save it in case it gets reset (the Nim codegen occasionally may call # Win API functions which reset it). socket.lastError = osLastError() if result <= 0: @@ -669,11 +1393,12 @@ template retRead(flags, readBytes: int) {.dirty.} = else: return res -proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect].} = +proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ + ReadIOEffect].} = ## Receives data from a socket. ## ## **Note**: This is a low-level function, you may be interested in the higher - ## level versions of this function which are also named ``recv``. + ## level versions of this function which are also named `recv`. if size == 0: return if socket.isBuffered: if socket.bufLen == 0: @@ -693,18 +1418,18 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = read else: - when defined(ssl): - if socket.isSSL: - if socket.sslHasPeekChar: + when defineSsl: + if socket.isSsl: + if socket.sslHasPeekChar: # TODO: Merge this peek char mess into uniRecv copyMem(data, addr(socket.sslPeekChar), 1) socket.sslHasPeekChar = false if size-1 > 0: var d = cast[cstring](data) - result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 + result = uniRecv(socket, addr(d[1]), cint(size-1), 0'i32) + 1 else: result = 1 else: - result = SSLRead(socket.sslHandle, data, size) + result = uniRecv(socket, data, size.cint, 0'i32) else: result = recv(socket.fd, data, size.cint, 0'i32) else: @@ -713,44 +1438,47 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] # Save the error in case it gets reset. socket.lastError = osLastError() -proc waitFor(socket: Socket, waited: var float, timeout, size: int, +proc waitFor(socket: Socket, waited: var Duration, timeout, size: int, funcName: string): int {.tags: [TimeEffect].} = ## determines the amount of characters that can be read. Result will never - ## be larger than ``size``. For unbuffered sockets this will be ``1``. - ## For buffered sockets it can be as big as ``BufferSize``. + ## be larger than `size`. For unbuffered sockets this will be `1`. + ## For buffered sockets it can be as big as `BufferSize`. ## ## If this function does not determine that there is data on the socket - ## within ``timeout`` ms, an ETimeout error will be raised. + ## within `timeout` ms, a TimeoutError error will be raised. result = 1 if size <= 0: assert false if timeout == -1: return size - if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos: + if socket.isBuffered and socket.bufLen != 0 and + socket.bufLen != socket.currPos: result = socket.bufLen - socket.currPos result = min(result, size) else: - if timeout - int(waited * 1000.0) < 1: + if timeout - waited.inMilliseconds < 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - when defined(ssl): - if socket.isSSL: + when defineSsl: + if socket.isSsl: if socket.hasDataBuffered: # sslPeekChar is present. return 1 - let sslPending = SSLPending(socket.sslHandle) + let sslPending = SSL_pending(socket.sslHandle) if sslPending != 0: - return sslPending + return min(sslPending, size) - var startTime = epochTime() - let selRet = select(socket, timeout - int(waited * 1000.0)) + var startTime = getMonoTime() + let selRet = if socket.hasDataBuffered: 1 + else: + timeoutRead(socket.fd, (timeout - waited.inMilliseconds).int) if selRet < 0: raiseOSError(osLastError()) if selRet != 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - waited += (epochTime() - startTime) + waited += (getMonoTime() - startTime) proc recv*(socket: Socket, data: pointer, size: int, timeout: int): int {. tags: [ReadIOEffect, TimeEffect].} = - ## overload with a ``timeout`` parameter in milliseconds. - var waited = 0.0 # number of seconds already waited + ## overload with a `timeout` parameter in milliseconds. + var waited: Duration # duration already waited var read = 0 while read < size: @@ -767,27 +1495,63 @@ proc recv*(socket: Socket, data: pointer, size: int, timeout: int): int {. proc recv*(socket: Socket, data: var string, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): int = - ## Higher-level version of ``recv``. + ## Higher-level version of `recv`. + ## + ## Reads **up to** `size` bytes from `socket` into `data`. + ## + ## For buffered sockets this function will attempt to read all the requested + ## data. It will read this data in `BufferSize` chunks. + ## + ## For unbuffered sockets this function makes no effort to read + ## all the data requested. It will return as much data as the operating system + ## gives it. ## ## When 0 is returned the socket's connection has been closed. ## - ## This function will throw an EOS exception when an error occurs. A value + ## This function will throw an OSError exception when an error occurs. A value ## lower than 0 is never returned. ## ## A timeout may be specified in milliseconds, if enough data is not received - ## within the time specified an ETimeout exception will be raised. - ## - ## **Note**: ``data`` must be initialised. + ## within the time specified a TimeoutError exception will be raised. ## - ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + ## .. warning:: Only the `SafeDisconn` flag is currently supported. data.setLen(size) - result = recv(socket, cstring(data), size, timeout) + result = + if timeout == -1: + recv(socket, cstring(data), size) + else: + recv(socket, cstring(data), size, timeout) if result < 0: data.setLen(0) let lastError = getSocketError(socket) - if flags.isDisconnectionError(lastError): return - socket.socketError(result, lastError = lastError) - data.setLen(result) + socket.socketError(result, lastError = lastError, flags = flags) + else: + data.setLen(result) + +proc recv*(socket: Socket, size: int, timeout = -1, + flags = {SocketFlag.SafeDisconn}): string {.inline.} = + ## Higher-level version of `recv` which returns a string. + ## + ## Reads **up to** `size` bytes from `socket` into the result. + ## + ## For buffered sockets this function will attempt to read all the requested + ## data. It will read this data in `BufferSize` chunks. + ## + ## For unbuffered sockets this function makes no effort to read + ## all the data requested. It will return as much data as the operating system + ## gives it. + ## + ## When `""` is returned the socket's connection has been closed. + ## + ## This function will throw an OSError exception when an error occurs. + ## + ## A timeout may be specified in milliseconds, if enough data is not received + ## within the time specified a TimeoutError exception will be raised. + ## + ## + ## .. warning:: Only the `SafeDisconn` flag is currently supported. + result = newString(size) + discard recv(socket, result, size, timeout, flags) proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = if socket.isBuffered: @@ -799,52 +1563,57 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = c = socket.buffer[socket.currPos] else: - when defined(ssl): - if socket.isSSL: + when defineSsl: + if socket.isSsl: if not socket.sslHasPeekChar: - result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) + result = uniRecv(socket, addr(socket.sslPeekChar), 1, 0'i32) socket.sslHasPeekChar = true c = socket.sslPeekChar return result = recv(socket.fd, addr(c), 1, MSG_PEEK) -proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, - flags = {SocketFlag.SafeDisconn}) {. +proc readLine*(socket: Socket, line: var string, timeout = -1, + flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {. tags: [ReadIOEffect, TimeEffect].} = - ## Reads a line of data from ``socket``. + ## Reads a line of data from `socket`. ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## If a full line is read `\r\L` is not + ## added to `line`, however if solely `\r\L` is read then `line` ## will be set to it. ## - ## If the socket is disconnected, ``line`` will be set to ``""``. + ## If the socket is disconnected, `line` will be set to `""`. ## - ## An EOS exception will be raised in the case of a socket error. + ## An OSError exception will be raised in the case of a socket error. ## ## A timeout can be specified in milliseconds, if data is not received within - ## the specified time an ETimeout exception will be raised. + ## the specified time a TimeoutError exception will be raised. + ## + ## The `maxLength` parameter determines the maximum amount of characters + ## that can be read. The result is truncated after that. ## - ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + ## .. warning:: Only the `SafeDisconn` flag is currently supported. - template addNLIfEmpty(): stmt = + template addNLIfEmpty() = if line.len == 0: line.add("\c\L") - template raiseSockError(): stmt {.dirty, immediate.} = + template raiseSockError() {.dirty.} = let lastError = getSocketError(socket) - if flags.isDisconnectionError(lastError): setLen(line.string, 0); return - socket.socketError(n, lastError = lastError) + if flags.isDisconnectionError(lastError): + setLen(line, 0) + socket.socketError(n, lastError = lastError, flags = flags) + return - var waited = 0.0 + var waited: Duration - setLen(line.string, 0) + setLen(line, 0) while true: var c: char discard waitFor(socket, waited, timeout, 1, "readLine") var n = recv(socket, addr(c), 1) if n < 0: raiseSockError() - elif n == 0: setLen(line.string, 0); return + elif n == 0: setLen(line, 0); return if c == '\r': discard waitFor(socket, waited, timeout, 1, "readLine") n = peekChar(socket, c) @@ -856,44 +1625,90 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, elif c == '\L': addNLIfEmpty() return - add(line.string, c) + add(line, c) + + # Verify that this isn't a DOS attack: #3847. + if line.len > maxLength: break -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. +proc recvLine*(socket: Socket, timeout = -1, + flags = {SocketFlag.SafeDisconn}, + maxLength = MaxLineLength): string = + ## Reads a line of data from `socket`. + ## + ## If a full line is read `\r\L` is not + ## added to the result, however if solely `\r\L` is read then the result + ## will be set to it. + ## + ## If the socket is disconnected, the result will be set to `""`. + ## + ## An OSError exception will be raised in the case of a socket error. + ## + ## A timeout can be specified in milliseconds, if data is not received within + ## the specified time a TimeoutError exception will be raised. + ## + ## The `maxLength` parameter determines the maximum amount of characters + ## that can be read. The result is truncated after that. + ## + ## .. warning:: Only the `SafeDisconn` flag is currently supported. + result = "" + readLine(socket, result, timeout, flags, maxLength) + +proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: int, + address: var T, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = - ## Receives data from ``socket``. This function should normally be used with - ## connection-less sockets (UDP sockets). + ## Receives data from `socket`. This function should normally be used with + ## connection-less sockets (UDP sockets). The source address of the data + ## packet is stored in the `address` argument as either a string or an IpAddress. ## - ## If an error occurs an EOS exception will be raised. Otherwise the return + ## If an error occurs an OSError exception will be raised. Otherwise the return ## value will be the length of data received. ## - ## **Warning:** This function does not yet have a buffered implementation, - ## so when ``socket`` is buffered the non-buffered implementation will be - ## used. Therefore if ``socket`` contains something in its buffer this - ## function will make no effort to return it. + ## .. warning:: This function does not yet have a buffered implementation, + ## so when `socket` is buffered the non-buffered implementation will be + ## used. Therefore if `socket` contains something in its buffer this + ## function will make no effort to return it. + template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = + var addrLen = SockLen(sizeof(sockAddress)) + result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + + if result != -1: + data.setLen(result) + + when typeof(address) is string: + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) + when domain == AF_INET6: + port = ntohs(sockAddress.sin6_port).Port + else: + port = ntohs(sockAddress.sin_port).Port + else: + data.setLen(result) + sockAddress.fromSockAddr(addrLen, address, port) + else: + raiseOSError(osLastError()) + assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - if result != -1: - data.setLen(result) - address = $inet_ntoa(sockAddress.sin_addr) - port = ntohs(sockAddress.sin_port).Port + case socket.domain + of AF_INET6: + var sockAddress: Sockaddr_in6 + adaptRecvFromToDomain(sockAddress, AF_INET6) + of AF_INET: + var sockAddress: Sockaddr_in + adaptRecvFromToDomain(sockAddress, AF_INET) else: - raiseOSError(osLastError()) + raise newException(ValueError, "Unknown socket address family") proc skip*(socket: Socket, size: int, timeout = -1) = - ## Skips ``size`` amount of bytes. + ## Skips `size` amount of bytes. ## ## An optional timeout can be specified in milliseconds, if skipping the - ## bytes takes longer than specified an ETimeout exception will be raised. + ## bytes takes longer than specified a TimeoutError exception will be raised. ## ## Returns the number of skipped bytes. - var waited = 0.0 + var waited: Duration var dummy = alloc(size) var bytesSkipped = 0 while bytesSkipped != size: @@ -905,11 +1720,13 @@ proc send*(socket: Socket, data: pointer, size: int): int {. tags: [WriteIOEffect].} = ## Sends data to a socket. ## - ## **Note**: This is a low-level version of ``send``. You likely should use + ## **Note**: This is a low-level version of `send`. You likely should use ## the version below. - when defined(ssl): - if socket.isSSL: - return SSLWrite(socket.sslHandle, cast[cstring](data), size) + assert(not socket.isClosed, "Cannot `send` on a closed socket") + when defineSsl: + if socket.isSsl: + ErrClearError() + return SSL_write(socket.sslHandle, cast[cstring](data), size) when useWinVersion or defined(macosx): result = send(socket.fd, data, size.cint, 0'i32) @@ -919,39 +1736,67 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - if flags.isDisconnectionError(lastError): return - socketError(socket, lastError = lastError) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) + + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) + +template `&=`*(socket: Socket; data: typed) = + ## an alias for 'send'. + send(socket, data) proc trySend*(socket: Socket, data: string): bool {.tags: [WriteIOEffect].} = - ## Safe alternative to ``send``. Does not raise an EOS when an error occurs, - ## and instead returns ``false`` on failure. + ## Safe alternative to `send`. Does not raise an OSError when an error occurs, + ## and instead returns `false` on failure. result = send(socket, cstring(data), data.len) == data.len proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, - size: int, af: Domain = AF_INET, flags = 0'i32): int {. + size: int, af: Domain = AF_INET, flags = 0'i32) {. tags: [WriteIOEffect].} = - ## This proc sends ``data`` to the specified ``address``, + ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. + ## this function will try each IP of that hostname. This function + ## should normally be used with connection-less sockets (UDP sockets). ## + ## If an error occurs an OSError exception will be raised. ## ## **Note:** You may wish to use the high-level version of this function ## which is defined below. ## ## **Note:** This proc is not available for SSL sockets. - var aiList = getAddrInfo(address, port, af) - + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + var aiList = getAddrInfo(address, port, af, socket.sockType, socket.protocol) # try all possibilities: var success = false var it = aiList + var result = 0 while it != nil: result = sendto(socket.fd, data, size.cint, flags.cint, it.ai_addr, it.ai_addrlen.SockLen) @@ -960,83 +1805,73 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, break it = it.ai_next - dealloc(aiList) + let osError = osLastError() + freeAddrInfo(aiList) + + if not success: + raiseOSError(osError) proc sendTo*(socket: Socket, address: string, port: Port, - data: string): int {.tags: [WriteIOEffect].} = - ## This proc sends ``data`` to the specified ``address``, + data: string) {.tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. ## - ## This is the high-level version of the above ``sendTo`` function. - result = socket.sendTo(address, port, cstring(data), data.len) - -proc connectAsync(socket: Socket, name: string, port = Port(0), - af: Domain = AF_INET) {.tags: [ReadIOEffect].} = - ## A variant of ``connect`` for non-blocking sockets. + ## Generally for use with connection-less (UDP) sockets. ## - ## This procedure will immediately return, it will not block until a connection - ## is made. It is up to the caller to make sure the connection has been established - ## by checking (using ``select``) whether the socket is writeable. + ## If an error occurs an OSError exception will be raised. ## - ## **Note**: For SSL sockets, the ``handshake`` procedure must be called - ## whenever the socket successfully connects to a server. - var aiList = getAddrInfo(name, port, af) - # try all possibilities: - var success = false - var lastError: OSErrorCode - var it = aiList - while it != nil: - var ret = connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) - if ret == 0'i32: - success = true - break - else: - lastError = osLastError() - when useWinVersion: - # Windows EINTR doesn't behave same as POSIX. - if lastError.int32 == WSAEWOULDBLOCK: - success = true - break - else: - if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: - success = true - break + ## This is the high-level version of the above `sendTo` function. + socket.sendTo(address, port, cstring(data), data.len, socket.domain) + +proc sendTo*(socket: Socket, address: IpAddress, port: Port, + data: string, flags = 0'i32): int {. + discardable, tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `IpAddress` and returns + ## the number of bytes written. + ## + ## Generally for use with connection-less (UDP) sockets. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This is the high-level version of the above `sendTo` function. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") - it = it.ai_next + var sa: Sockaddr_storage + var sl: SockLen + toSockAddr(address, port, sa, sl) + result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, + cast[ptr SockAddr](addr sa), sl) - dealloc(aiList) - if not success: raiseOSError(lastError) + if result == -1'i32: + let osError = osLastError() + raiseOSError(osError) -proc connect*(socket: Socket, address: string, port = Port(0), - timeout: int) {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Connects to server as specified by ``address`` on port specified by ``port``. - ## - ## The ``timeout`` paremeter specifies the time in milliseconds to allow for - ## the connection to the server to be made. - socket.fd.setBlocking(false) - - socket.connectAsync(address, port, socket.domain) - var s = @[socket.fd] - if selectWrite(s, timeout) != 1: - raise newException(TimeoutError, "Call to 'connect' timed out.") - else: - when defined(ssl): - if socket.isSSL: - socket.fd.setBlocking(true) - doAssert socket.handshake() - socket.fd.setBlocking(true) proc isSsl*(socket: Socket): bool = - ## Determines whether ``socket`` is a SSL socket. - when defined(ssl): - result = socket.isSSL + ## Determines whether `socket` is a SSL socket. + when defineSsl: + result = socket.isSsl else: result = false proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor +when defined(zephyr) or defined(nimNetSocketExtras): # Remove in future + proc getDomain*(socket: Socket): Domain = return socket.domain + ## Returns the socket's domain + + proc getType*(socket: Socket): SockType = return socket.sockType + ## Returns the socket's type + + proc getProtocol*(socket: Socket): Protocol = return socket.protocol + ## Returns the socket's protocol + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + proc IPv4_any*(): IpAddress = ## Returns the IPv4 any address, which can be used to listen on all available ## network adapters @@ -1069,8 +1904,11 @@ proc IPv6_loopback*(): IpAddress = family: IpAddressFamily.IPv6, address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) +when defined(nimHasStyleChecks): + {.pop.} + proc `==`*(lhs, rhs: IpAddress): bool = - ## Compares two IpAddresses for Equality. Returns two if the addresses are equal + ## Compares two IpAddresses for Equality. Returns true if the addresses are equal if lhs.family != rhs.family: return false if lhs.family == IpAddressFamily.IPv4: for i in low(lhs.address_v4) .. high(lhs.address_v4): @@ -1082,14 +1920,18 @@ proc `==`*(lhs, rhs: IpAddress): bool = proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation - result = "" case address.family of IpAddressFamily.IPv4: - for i in 0 .. 3: - if i != 0: - result.add('.') - result.add($address.address_v4[i]) + result = newStringOfCap(15) + result.addInt address.address_v4[0] + result.add '.' + result.addInt address.address_v4[1] + result.add '.' + result.addInt address.address_v4[2] + result.add '.' + result.addInt address.address_v4[3] of IpAddressFamily.IPv6: + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 @@ -1115,7 +1957,7 @@ proc `$`*(address: IpAddress): string = else: # Print address var printedLastGroup = false for i in 0..7: - var word:uint16 = (cast[uint16](address.address_v6[i*2])) shl 8 + var word: uint16 = (cast[uint16](address.address_v6[i*2])) shl 8 word = word or cast[uint16](address.address_v6[i*2+1]) if biggestZeroCount != 0 and # Check if group is in skip group @@ -1138,179 +1980,196 @@ proc `$`*(address: IpAddress): string = result.add(chr(uint16(ord('a'))+val-0xA)) afterLeadingZeros = true mask = mask shr 4 + + if not afterLeadingZeros: + result.add '0' + printedLastGroup = true -proc parseIPv4Address(address_str: string): IpAddress = - ## Parses IPv4 adresses - ## Raises EInvalidValue on errors - var - byteCount = 0 - currentByte:uint16 = 0 - seperatorValid = false +proc dial*(address: string, port: Port, + protocol = IPPROTO_TCP, buffered = true): owned(Socket) + {.tags: [ReadIOEffect, WriteIOEffect].} = + ## Establishes connection to the specified `address`:`port` pair via the + ## specified protocol. The procedure iterates through possible + ## resolutions of the `address` until it succeeds, meaning that it + ## seamlessly works with both IPv4 and IPv6. + ## Returns Socket ready to send or receive data. + let sockType = protocol.toSockType() + + let aiList = getAddrInfo(address, port, AF_UNSPEC, sockType, protocol) + + var fdPerDomain: array[low(Domain).ord..high(Domain).ord, SocketHandle] + for i in low(fdPerDomain)..high(fdPerDomain): + fdPerDomain[i] = osInvalidSocket + template closeUnusedFds(domainToKeep = -1) {.dirty.} = + for i, fd in fdPerDomain: + if fd != osInvalidSocket and i != domainToKeep: + fd.close() - result.family = IpAddressFamily.IPv4 + var success = false + var lastError: OSErrorCode + var it = aiList + var domain: Domain + var lastFd: SocketHandle + while it != nil: + let domainOpt = it.ai_family.toKnownDomain() + if domainOpt.isNone: + it = it.ai_next + continue + domain = domainOpt.unsafeGet() + lastFd = fdPerDomain[ord(domain)] + if lastFd == osInvalidSocket: + lastFd = createNativeSocket(domain, sockType, protocol) + if lastFd == osInvalidSocket: + # we always raise if socket creation failed, because it means a + # network system problem (e.g. not enough FDs), and not an unreachable + # address. + let err = osLastError() + freeAddrInfo(aiList) + closeUnusedFds() + raiseOSError(err) + fdPerDomain[ord(domain)] = lastFd + if connect(lastFd, it.ai_addr, it.ai_addrlen.SockLen) == 0'i32: + success = true + break + lastError = osLastError() + it = it.ai_next + freeAddrInfo(aiList) + closeUnusedFds(ord(domain)) + + if success: + result = newSocket(lastFd, domain, sockType, protocol, buffered) + elif lastError != 0.OSErrorCode: + lastFd.close() + raiseOSError(lastError) + else: + lastFd.close() + raise newException(IOError, "Couldn't resolve address: " & address) - for i in 0 .. high(address_str): - if address_str[i] in strutils.Digits: # Character is a number - currentByte = currentByte * 10 + - cast[uint16](ord(address_str[i]) - ord('0')) - if currentByte > 255'u16: - raise newException(ValueError, - "Invalid IP Address. Value is out of range") - seperatorValid = true - elif address_str[i] == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - result.address_v4[byteCount] = cast[uint8](currentByte) - currentByte = 0 - byteCount.inc - seperatorValid = false - else: - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid character") +proc connect*(socket: Socket, address: string, + port = Port(0)) {.tags: [ReadIOEffect, RootEffect].} = + ## Connects socket to `address`:`port`. `Address` can be an IP address or a + ## host name. If `address` is a host name, this function will try each IP + ## of that host name. `htons` is already performed on `port` so you must + ## not do it. + ## + ## If `socket` is an SSL socket a handshake will be automatically performed. + var aiList = getAddrInfo(address, port, socket.domain) + # try all possibilities: + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + if connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) == 0'i32: + success = true + break + else: lastError = osLastError() + it = it.ai_next - if byteCount != 3 or not seperatorValid: - raise newException(ValueError, "Invalid IP Address") - result.address_v4[byteCount] = cast[uint8](currentByte) + freeAddrInfo(aiList) + if not success: raiseOSError(lastError) -proc parseIPv6Address(address_str: string): IpAddress = - ## Parses IPv6 adresses - ## Raises EInvalidValue on errors - result.family = IpAddressFamily.IPv6 - if address_str.len < 2: - raise newException(ValueError, "Invalid IP Address") + when defineSsl: + if socket.isSsl: + # RFC3546 for SNI specifies that IP addresses are not allowed. + if not isIpAddress(address): + # Discard result in case OpenSSL version doesn't support SNI, or we're + # not using TLSv1+ + discard SSL_set_tlsext_host_name(socket.sslHandle, address) - var - groupCount = 0 - currentGroupStart = 0 - currentShort:uint32 = 0 - seperatorValid = true - dualColonGroup = -1 - lastWasColon = false - v4StartPos = -1 - byteCount = 0 + ErrClearError() + let ret = SSL_connect(socket.sslHandle) + socketError(socket, ret) + when not defined(nimDisableCertificateValidation) and not defined(windows): + if not isIpAddress(address): + socket.checkCertName(address) - for i,c in address_str: - if c == ':': - if not seperatorValid: - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid seperator") - if lastWasColon: - if dualColonGroup != -1: - raise newException(ValueError, - "Invalid IP Address. Address contains more than one \"::\" seperator") - dualColonGroup = groupCount - seperatorValid = false - elif i != 0 and i != high(address_str): - if groupCount >= 8: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) - result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) - currentShort = 0 - groupCount.inc() - if dualColonGroup != -1: seperatorValid = false - elif i == 0: # only valid if address starts with :: - if address_str[1] != ':': - raise newException(ValueError, - "Invalid IP Address. Address may not start with \":\"") - else: # i == high(address_str) - only valid if address ends with :: - if address_str[high(address_str)-1] != ':': - raise newException(ValueError, - "Invalid IP Address. Address may not end with \":\"") - lastWasColon = true - currentGroupStart = i + 1 - elif c == '.': # Switch to parse IPv4 mode - if i < 3 or not seperatorValid or groupCount >= 7: - raise newException(ValueError, "Invalid IP Address") - v4StartPos = currentGroupStart - currentShort = 0 - seperatorValid = false +proc connectAsync(socket: Socket, name: string, port = Port(0), + af: Domain = AF_INET) {.tags: [ReadIOEffect].} = + ## A variant of `connect` for non-blocking sockets. + ## + ## This procedure will immediately return, it will not block until a connection + ## is made. It is up to the caller to make sure the connection has been established + ## by checking (using `select`) whether the socket is writeable. + ## + ## **Note**: For SSL sockets, the `handshake` procedure must be called + ## whenever the socket successfully connects to a server. + var aiList = getAddrInfo(name, port, af) + # try all possibilities: + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.fd, it.ai_addr, it.ai_addrlen.SockLen) + if ret == 0'i32: + success = true break - elif c in strutils.HexDigits: - if c in strutils.Digits: # Normal digit - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) - elif c >= 'a' and c <= 'f': # Lower case hex - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 - else: # Upper case hex - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 - if currentShort > 65535'u32: - raise newException(ValueError, - "Invalid IP Address. Value is out of range") - lastWasColon = false - seperatorValid = true else: - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid character") - + lastError = osLastError() + when useWinVersion: + # Windows EINTR doesn't behave same as POSIX. + if lastError.int32 == WSAEWOULDBLOCK: + success = true + break + else: + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + break - if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff - if seperatorValid: # Copy remaining data - if groupCount >= 8: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) - result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) - groupCount.inc() - else: # Must parse IPv4 address - for i,c in address_str[v4StartPos..high(address_str)]: - if c in strutils.Digits: # Character is a number - currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: - raise newException(ValueError, - "Invalid IP Address. Value is out of range") - seperatorValid = true - elif c == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: - raise newException(ValueError, "Invalid IP Address") - result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) - currentShort = 0 - byteCount.inc() - seperatorValid = false - else: # Invalid character - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid character") + it = it.ai_next - if byteCount != 3 or not seperatorValid: - raise newException(ValueError, "Invalid IP Address") - result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) - groupCount += 2 + freeAddrInfo(aiList) + if not success: raiseOSError(lastError) - # Shift and fill zeros in case of :: - if groupCount > 8: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - elif groupCount < 8: # must fill - if dualColonGroup == -1: - raise newException(ValueError, - "Invalid IP Address. The address consists of too few groups") - var toFill = 8 - groupCount # The number of groups to fill - var toShift = groupCount - dualColonGroup # Nr of known groups after :: - for i in 0..2*toShift-1: # shift - result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] - for i in 0..2*toFill-1: # fill with 0s - result.address_v6[dualColonGroup*2+i] = 0 - elif dualColonGroup != -1: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") +proc connect*(socket: Socket, address: string, port = Port(0), + timeout: int) {.tags: [ReadIOEffect, WriteIOEffect, RootEffect].} = + ## Connects to server as specified by `address` on port specified by `port`. + ## + ## The `timeout` parameter specifies the time in milliseconds to allow for + ## the connection to the server to be made. + socket.fd.setBlocking(false) -proc parseIpAddress(address_str: string): IpAddress = - ## Parses an IP address - ## Raises EInvalidValue on error - if address_str == nil: - raise newException(ValueError, "IP Address string is nil") - if address_str.contains(':'): - return parseIPv6Address(address_str) + socket.connectAsync(address, port, socket.domain) + if timeoutWrite(socket.fd, timeout) != 1: + raise newException(TimeoutError, "Call to 'connect' timed out.") else: - return parseIPv4Address(address_str) + let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR) + if res != 0: + raiseOSError(OSErrorCode(res)) + when defineSsl and not defined(nimdoc): + if socket.isSsl: + socket.fd.setBlocking(true) + # RFC3546 for SNI specifies that IP addresses are not allowed. + if not isIpAddress(address): + # Discard result in case OpenSSL version doesn't support SNI, or we're + # not using TLSv1+ + discard SSL_set_tlsext_host_name(socket.sslHandle, address) + ErrClearError() + let ret = SSL_connect(socket.sslHandle) + socketError(socket, ret) + when not defined(nimDisableCertificateValidation): + if not isIpAddress(address): + socket.checkCertName(address) + socket.fd.setBlocking(true) -proc isIpAddress(address_str: string): bool = - ## Checks if a string is an IP address - ## Returns true if it is, false otherwise +proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = + ## Finds the local IP address, usually assigned to eth0 on LAN or wlan0 on WiFi, + ## used to reach an external address. Useful to run local services. + ## + ## No traffic is sent. + ## + ## Supports IPv4 and v6. + ## Raises OSError if external networking is not set up. + runnableExamples("-r:off"): + echo getPrimaryIPAddr() # "192.168.1.2" + let socket = + if dest.family == IpAddressFamily.IPv4: + newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + else: + newSocket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP) try: - discard parseIpAddress(address_str) - except ValueError: - return false - return true + socket.connect($dest, 80.Port) + result = socket.getLocalAddr()[0].parseIpAddress() + finally: + socket.close() diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index cfe6bc40d..bf8367d1d 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -1,23 +1,29 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## Profiling support for Nim. This is an embedded profiler that requires -## ``--profiler:on``. You only need to import this module to get a profiling -## report at program exit. +## `--profiler:on`. You only need to import this module to get a profiling +## report at program exit. See `Embedded Stack Trace Profiler <estp.html>`_ +## for usage. when not defined(profiler) and not defined(memProfiler): - {.warning: "Profiling support is turned off!".} + {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} + +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} -import hashes, algorithm, strutils, tables, sets +import std/[hashes, algorithm, strutils, tables, sets] + +when defined(nimPreviewSlimSystem): + import std/[syncio, sysatomics] when not defined(memProfiler): include "system/timers" @@ -27,19 +33,20 @@ const tickCountCorrection = 50_000 when not declared(system.StackTrace): - type StackTrace = array [0..20, cstring] - {.deprecated: [TStackTrace: StackTrace].} + type StackTrace = object + lines: array[0..20, cstring] + files: array[0..20, cstring] + proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] # We use a simple hash table of bounded size to keep track of the stack traces: type ProfileEntry = object total: int st: StackTrace - ProfileData = array [0..64*1024-1, ptr ProfileEntry] -{.deprecated: [TProfileEntry: ProfileEntry, TProfileData: ProfileData].} + ProfileData = array[0..64*1024-1, ptr ProfileEntry] proc `==`(a, b: StackTrace): bool = - for i in 0 .. high(a): + for i in 0 .. high(a.lines): if a[i] != b[i]: return false result = true @@ -62,7 +69,7 @@ when not defined(memProfiler): else: interval = intervalInUs * 1000 - tickCountCorrection when withThreads: - import locks + import std/locks var profilingLock: Lock @@ -72,7 +79,7 @@ proc hookAux(st: StackTrace, costs: int) = # this is quite performance sensitive! when withThreads: acquire profilingLock inc totalCalls - var last = high(st) + var last = high(st.lines) while last > 0 and isNil(st[last]): dec last var h = hash(pointer(st[last])) and high(profileData) @@ -117,24 +124,38 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc hook(st: StackTrace, size: int) {.nimcall.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: - gTicker = -1 - when defined(ignoreAllocationSize): - hookAux(st, 1) - else: - hookAux(st, size) gTicker = SamplingInterval + result = true dec gTicker + proc hook(st: StackTrace, size: int) {.nimcall.} = + when defined(ignoreAllocationSize): + hookAux(st, 1) + else: + hookAux(st, size) + else: var t0 {.threadvar.}: Ticks + gTicker: int # we use an additional counter to + # avoid calling 'getTicks' too frequently + + proc requestedHook(): bool {.nimcall.} = + if interval == 0: result = true + elif gTicker == 0: + gTicker = 500 + if getTicks() - t0 > interval: + result = true + else: + dec gTicker proc hook(st: StackTrace) {.nimcall.} = + #echo "profiling! ", interval if interval == 0: hookAux(st, 1) - elif int64(t0) == 0 or getTicks() - t0 > interval: + else: hookAux(st, 1) t0 = getTicks() @@ -145,9 +166,10 @@ proc cmpEntries(a, b: ptr ProfileEntry): int = result = b.getTotal - a.getTotal proc `//`(a, b: int): string = - result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDefault, 2)) + result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDecimal, 2)) proc writeProfile() {.noconv.} = + system.profilingRequestedHook = nil when declared(system.StackTrace): system.profilerHook = nil const filename = "profile_results.txt" @@ -162,8 +184,8 @@ proc writeProfile() {.noconv.} = var perProc = initCountTable[string]() for i in 0..entries-1: - var dups = initSet[string]() - for ii in 0..high(StackTrace): + var dups = initHashSet[string]() + for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] if isNil(procname): break let p = $procname @@ -178,10 +200,12 @@ proc writeProfile() {.noconv.} = writeLine(f, "Entry: ", i+1, "/", entries, " Calls: ", profileData[i].total // totalCalls, " [sum: ", sum, "; ", sum // totalCalls, "]") - for ii in 0..high(StackTrace): + for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] + let filename = profileData[i].st.files[ii] if isNil(procname): break - writeLine(f, " ", procname, " ", perProc[$procname] // totalCalls) + writeLine(f, " ", $filename & ": " & $procname, " ", + perProc[$procname] // totalCalls) close(f) echo "... done" else: @@ -193,15 +217,17 @@ var proc disableProfiling*() = when declared(system.StackTrace): atomicDec disabled - system.profilerHook = nil + system.profilingRequestedHook = nil proc enableProfiling*() = when declared(system.StackTrace): if atomicInc(disabled) >= 0: - system.profilerHook = hook + system.profilingRequestedHook = requestedHook when declared(system.StackTrace): + import std/exitprocs + system.profilingRequestedHook = requestedHook system.profilerHook = hook - addQuitProc(writeProfile) + addExitProc(writeProfile) {.pop.} diff --git a/lib/pure/numeric.nim b/lib/pure/numeric.nim deleted file mode 100644 index 71adf19b3..000000000 --- a/lib/pure/numeric.nim +++ /dev/null @@ -1,84 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Robert Persson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -type OneVarFunction* = proc (x: float): float - -{.deprecated: [TOneVarFunction: OneVarFunction].} - -proc brent*(xmin,xmax:float, function:OneVarFunction, tol:float,maxiter=1000): - tuple[rootx, rooty: float, success: bool]= - ## Searches `function` for a root between `xmin` and `xmax` - ## using brents method. If the function value at `xmin`and `xmax` has the - ## same sign, `rootx`/`rooty` is set too the extrema value closest to x-axis - ## and succes is set to false. - ## Otherwise there exists at least one root and success is set to true. - ## This root is searched for at most `maxiter` iterations. - ## If `tol` tolerance is reached within `maxiter` iterations - ## the root refinement stops and success=true. - - # see http://en.wikipedia.org/wiki/Brent%27s_method - var - a=xmin - b=xmax - c=a - d=1.0e308 - fa=function(a) - fb=function(b) - fc=fa - s=0.0 - fs=0.0 - mflag:bool - i=0 - tmp2:float - - if fa*fb>=0: - if abs(fa)<abs(fb): - return (a,fa,false) - else: - return (b,fb,false) - - if abs(fa)<abs(fb): - swap(fa,fb) - swap(a,b) - - while fb!=0.0 and abs(a-b)>tol: - if fa!=fc and fb!=fc: # inverse quadratic interpolation - s = a * fb * fc / (fa - fb) / (fa - fc) + b * fa * fc / (fb - fa) / (fb - fc) + c * fa * fb / (fc - fa) / (fc - fb) - else: #secant rule - s = b - fb * (b - a) / (fb - fa) - tmp2 = (3.0 * a + b) / 4.0 - if not((s > tmp2 and s < b) or (s < tmp2 and s > b)) or - (mflag and abs(s - b) >= (abs(b - c) / 2.0)) or - (not mflag and abs(s - b) >= abs(c - d) / 2.0): - s=(a+b)/2.0 - mflag=true - else: - if (mflag and (abs(b - c) < tol)) or (not mflag and (abs(c - d) < tol)): - s=(a+b)/2.0 - mflag=true - else: - mflag=false - fs = function(s) - d = c - c = b - fc = fb - if fa * fs<0.0: - b=s - fb=fs - else: - a=s - fa=fs - if abs(fa)<abs(fb): - swap(a,b) - swap(fa,fb) - inc i - if i>maxiter: - break - - return (b,fb,true) diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fca10dab6..4d6ceefd7 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -9,89 +9,91 @@ ## Nim OID support. An OID is a global ID that consists of a timestamp, ## a unique counter and a random value. This combination should suffice to -## produce a globally distributed unique ID. This implementation was extracted -## from the Mongodb interface and it thus binary compatible with a Mongo OID. +## produce a globally distributed unique ID. ## -## This implementation calls ``math.randomize()`` for the first call of -## ``genOid``. +## This implementation calls `initRand()` for the first call of +## `genOid`. -import times, endians +import std/[hashes, times, endians, random] +from std/private/decode_helpers import handleHexChar -type - Oid* = object ## an OID - time: int32 ## - fuzz: int32 ## - count: int32 ## - -{.deprecated: [Toid: Oid].} - -proc `==`*(oid1: Oid, oid2: Oid): bool = - ## Compare two Mongo Object IDs for equality - return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) +when defined(nimPreviewSlimSystem): + import std/sysatomics -proc hexbyte*(hex: char): int = - case hex - of '0'..'9': result = (ord(hex) - ord('0')) - of 'a'..'f': result = (ord(hex) - ord('a') + 10) - of 'A'..'F': result = (ord(hex) - ord('A') + 10) - else: discard +type + Oid* = object ## An OID. + time: int64 + fuzz: int32 + count: int32 + +proc `==`*(oid1: Oid, oid2: Oid): bool {.inline.} = + ## Compares two OIDs for equality. + result = (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and + (oid1.count == oid2.count) + +proc hash*(oid: Oid): Hash = + ## Generates the hash of an OID for use in hashtables. + var h: Hash = 0 + h = h !& hash(oid.time) + h = h !& hash(oid.fuzz) + h = h !& hash(oid.count) + result = !$h + +proc hexbyte*(hex: char): int {.inline.} = + result = handleHexChar(hex) proc parseOid*(str: cstring): Oid = - ## parses an OID. - var bytes = cast[cstring](addr(result.time)) + ## Parses an OID. + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) inc(i) -proc oidToString*(oid: Oid, str: cstring) = +proc `$`*(oid: Oid): string = + ## Converts an OID to a string. const hex = "0123456789abcdef" - # work around a compiler bug: - var str = str + + result.setLen 24 + var o = oid - var bytes = cast[cstring](addr(o)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 while i < 12: let b = bytes[i].ord - str[2 * i] = hex[(b and 0xF0) shr 4] - str[2 * i + 1] = hex[b and 0xF] + result[2 * i] = hex[(b and 0xF0) shr 4] + result[2 * i + 1] = hex[b and 0xF] inc(i) - str[24] = '\0' -proc `$`*(oid: Oid): string = - result = newString(24) - oidToString(oid, result) +let + t = getTime().toUnix var - incr: int - fuzz: int32 + seed = initRand(t) + incr: int = seed.rand(int.high) -proc genOid*(): Oid = - ## generates a new OID. - proc rand(): cint {.importc: "rand", header: "<stdlib.h>", nodecl.} - proc gettime(dummy: ptr cint): cint {.importc: "time", header: "<time.h>".} - proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>", nodecl.} +let fuzz = cast[int32](seed.rand(high(int))) - var t = gettime(nil) - var i = int32(incr) - atomicInc(incr) +template genOid(result: var Oid, incr: var int, fuzz: int32) = + var time = getTime().toUnix + var i = cast[int32](atomicInc(incr)) - if fuzz == 0: - # racy, but fine semantically: - srand(t) - fuzz = rand() - bigEndian32(addr result.time, addr(t)) + bigEndian64(addr result.time, addr(time)) result.fuzz = fuzz bigEndian32(addr result.count, addr(i)) +proc genOid*(): Oid = + ## Generates a new OID. + runnableExamples: + doAssert ($genOid()).len == 24 + runnableExamples("-r:off"): + echo $genOid() # for example, "5fc7f546ddbbc84800006aaf" + genOid(result, incr, fuzz) + proc generatedTime*(oid: Oid): Time = - ## returns the generated timestamp of the OID. - var tmp: int32 + ## Returns the generated timestamp of the OID. + var tmp: int64 var dummy = oid.time - bigEndian32(addr(tmp), addr(dummy)) - result = Time(tmp) - -when not defined(testing) and isMainModule: - let xo = genOid() - echo xo.generatedTime + bigEndian64(addr(tmp), addr(dummy)) + result = fromUnix(tmp) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 2abb80016..b34ff72c0 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -7,216 +7,375 @@ # distribution, for details about the copyright. # -## Abstract -## ======== -## -## This module implements types which encapsulate an optional value. -## -## A value of type ``Option[T]`` either contains a value `x` (represented as -## ``some(x)``) or is empty (``none(T)``). -## -## This can be useful when you have a value that can be present or not. The -## absence of a value is often represented by ``nil``, but it is not always -## available, nor is it always a good solution. -## -## -## Tutorial -## ======== -## -## Let's start with an example: a procedure that finds the index of a character -## in a string. -## -## .. code-block:: nim -## -## import options -## -## proc find(haystack: string, needle: char): Option[int] = -## for i, c in haystack: -## if c == needle: -## return some(i) -## return none(int) # This line is actually optional, -## # because the default is empty -## -## .. code-block:: nim -## -## try: -## assert("abc".find('c').get() == 2) # Immediately extract the value -## except UnpackError: # If there is no value -## assert false # This will not be reached, because the value is present -## -## The ``get`` operation demonstrated above returns the underlying value, or -## raises ``UnpackError`` if there is no value. There is another option for -## obtaining the value: ``unsafeGet``, but you must only use it when you are -## absolutely sure the value is present (e.g. after checking ``isSome``). If -## you do not care about the tiny overhead that ``get`` causes, you should -## simply never use ``unsafeGet``. -## -## How to deal with an absence of a value: -## -## .. code-block:: nim -## -## let result = "team".find('i') -## -## # Nothing was found, so the result is `none`. -## assert(result == none(int)) -## # It has no value: -## assert(result.isNone) -## -## try: -## echo result.get() -## assert(false) # This will not be reached -## except UnpackError: # Because an exception is raised -## discard - -import typetraits +##[ +This module implements types which encapsulate an optional value. +A value of type `Option[T]` either contains a value `x` (represented as +`some(x)`) or is empty (`none(T)`). -type - Option*[T] = object - ## An optional type that stores its value and state separately in a boolean. - val: T - has: bool - UnpackError* = ref object of ValueError +This can be useful when you have a value that can be present or not. The +absence of a value is often represented by `nil`, but that is not always +available, nor is it always a good solution. -proc some*[T](val: T): Option[T] = - ## Returns a ``Option`` that has this value. - result.has = true - result.val = val +Basic usage +=========== -proc none*(T: typedesc): Option[T] = - ## Returns a ``Option`` for this type that has no value. - result.has = false +Let's start with an example: a procedure that finds the index of a character +in a string. +]## +runnableExamples: + proc find(haystack: string, needle: char): Option[int] = + for i, c in haystack: + if c == needle: + return some(i) + return none(int) # This line is actually optional, + # because the default is empty -proc isSome*[T](self: Option[T]): bool = - self.has + let found = "abc".find('c') + assert found.isSome and found.get() == 2 -proc isNone*[T](self: Option[T]): bool = - not self.has +##[ +The `get` operation demonstrated above returns the underlying value, or +raises `UnpackDefect` if there is no value. Note that `UnpackDefect` +inherits from `system.Defect` and should therefore never be caught. +Instead, rely on checking if the option contains a value with the +`isSome <#isSome,Option[T]>`_ and `isNone <#isNone,Option[T]>`_ procs. -proc unsafeGet*[T](self: Option[T]): T = - ## Returns the value of a ``some``. Behavior is undefined for ``none``. - assert self.isSome - self.val +Pattern matching +================ -proc get*[T](self: Option[T]): T = - ## Returns contents of the Option. If it is none, then an exception is - ## thrown. - if self.isNone: - raise UnpackError(msg : "Can't obtain a value from a `none`") - self.val +.. note:: This requires the [fusion](https://github.com/nim-lang/fusion) package. -proc get*[T](self: Option[T], otherwise: T): T = - ## Returns the contents of this option or `otherwise` if the option is none. - if self.isSome: - self.val - else: - otherwise +[fusion/matching](https://nim-lang.github.io/fusion/src/fusion/matching.html) +supports pattern matching on `Option`s, with the `Some(<pattern>)` and +`None()` patterns. + ```nim + {.experimental: "caseStmtMacros".} -proc map*[T](self: Option[T], callback: proc (input: T)) = - ## Applies a callback to the value in this Option - if self.has: - callback(self.val) - -proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] = - ## Applies a callback to the value in this Option and returns an option - ## containing the new value. If this option is None, None will be returned - if self.has: - some[R]( callback(self.val) ) - else: - none(R) + import fusion/matching -proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = - ## Applies a callback to the value in this Option. If the callback returns - ## `true`, the option is returned as a Some. If it returns false, it is - ## returned as a None. - if self.has and not callback(self.val): - none(T) - else: - self + case some(42) + of Some(@a): + assert a == 42 + of None(): + assert false + assertMatch(some(some(none(int))), Some(Some(None()))) + ``` +]## +# xxx pending https://github.com/timotheecour/Nim/issues/376 use `runnableExamples` and `whichModule` -proc `==`*(a, b: Option): bool = - ## Returns ``true`` if both ``Option``s are ``none``, - ## or if they have equal values - (a.has and b.has and a.val == b.val) or (not a.has and not b.has) +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} +import std/typetraits -proc `$`*[T]( self: Option[T] ): string = - ## Returns the contents of this option or `otherwise` if the option is none. - if self.has: - "Some(" & $self.val & ")" - else: - "None[" & T.name & "]" +when defined(nimPreviewSlimSystem): + import std/assertions -when isMainModule: - import unittest, sequtils +when (NimMajor, NimMinor) >= (1, 1): + type + SomePointer = ref | ptr | pointer | proc | iterator {.closure.} +else: + type + SomePointer = ref | ptr | pointer - suite "options": - # work around a bug in unittest - let intNone = none(int) - let stringNone = none(string) +type + Option*[T] = object + ## An optional type that may or may not contain a value of type `T`. + ## When `T` is a a pointer type (`ptr`, `pointer`, `ref`, `proc` or `iterator {.closure.}`), + ## `none(T)` is represented as `nil`. + when T is SomePointer: + val: T + else: + val: T + has: bool + + UnpackDefect* = object of Defect + UnpackError* {.deprecated: "See corresponding Defect".} = UnpackDefect + +proc option*[T](val: sink T): Option[T] {.inline.} = + ## Can be used to convert a pointer type (`ptr`, `pointer`, `ref` or `proc`) to an option type. + ## It converts `nil` to `none(T)`. When `T` is no pointer type, this is equivalent to `some(val)`. + ## + ## **See also:** + ## * `some proc <#some,T>`_ + ## * `none proc <#none,typedesc>`_ + runnableExamples: + type + Foo = ref object + a: int + b: string + + assert option[Foo](nil).isNone + assert option(42).isSome + + when T is SomePointer: + result = Option[T](val: val) + else: + result = Option[T](has: true, val: val) + +proc some*[T](val: sink T): Option[T] {.inline.} = + ## Returns an `Option` that has the value `val`. + ## + ## **See also:** + ## * `option proc <#option,T>`_ + ## * `none proc <#none,typedesc>`_ + ## * `isSome proc <#isSome,Option[T]>`_ + runnableExamples: + let a = some("abc") + + assert a.isSome + assert a.get == "abc" + + when T is SomePointer: + assert not val.isNil + result = Option[T](val: val) + else: + result = Option[T](has: true, val: val) + +proc none*(T: typedesc): Option[T] {.inline.} = + ## Returns an `Option` for this type that has no value. + ## + ## **See also:** + ## * `option proc <#option,T>`_ + ## * `some proc <#some,T>`_ + ## * `isNone proc <#isNone,Option[T]>`_ + runnableExamples: + assert none(int).isNone + + # the default is the none type + result = Option[T]() + +proc none*[T]: Option[T] {.inline.} = + ## Alias for `none(T) <#none,typedesc>`_. + none(T) + +proc isSome*[T](self: Option[T]): bool {.inline.} = + ## Checks if an `Option` contains a value. + ## + ## **See also:** + ## * `isNone proc <#isNone,Option[T]>`_ + ## * `some proc <#some,T>`_ + runnableExamples: + assert some(42).isSome + assert not none(string).isSome + + when T is SomePointer: + not self.val.isNil + else: + self.has + +proc isNone*[T](self: Option[T]): bool {.inline.} = + ## Checks if an `Option` is empty. + ## + ## **See also:** + ## * `isSome proc <#isSome,Option[T]>`_ + ## * `none proc <#none,typedesc>`_ + runnableExamples: + assert not some(42).isNone + assert none(string).isNone + + when T is SomePointer: + self.val.isNil + else: + not self.has + +proc get*[T](self: Option[T]): lent T {.inline.} = + ## Returns the content of an `Option`. If it has no value, + ## an `UnpackDefect` exception is raised. + ## + ## **See also:** + ## * `get proc <#get,Option[T],T>`_ with a default return value + runnableExamples: + assert some(42).get == 42 + doAssertRaises(UnpackDefect): + echo none(string).get - test "example": - proc find(haystack: string, needle: char): Option[int] = - for i, c in haystack: - if c == needle: - return some i + if self.isNone: + raise newException(UnpackDefect, "Can't obtain a value from a `none`") + result = self.val - check("abc".find('c').get() == 2) +proc get*[T](self: Option[T], otherwise: T): T {.inline.} = + ## Returns the content of the `Option` or `otherwise` if + ## the `Option` has no value. + runnableExamples: + assert some(42).get(9999) == 42 + assert none(int).get(9999) == 9999 - let result = "team".find('i') + if self.isSome: + self.val + else: + otherwise - check result == intNone - check result.isNone +proc get*[T](self: var Option[T]): var T {.inline.} = + ## Returns the content of the `var Option` mutably. If it has no value, + ## an `UnpackDefect` exception is raised. + runnableExamples: + var + a = some(42) + b = none(string) + inc(a.get) + assert a.get == 43 + doAssertRaises(UnpackDefect): + echo b.get - test "some": - check some(6).get() == 6 - check some("a").unsafeGet() == "a" - check some(6).isSome - check some("a").isSome + if self.isNone: + raise newException(UnpackDefect, "Can't obtain a value from a `none`") + return self.val + +proc map*[T](self: Option[T], callback: proc (input: T)) {.inline, effectsOf: callback.} = + ## Applies a `callback` function to the value of the `Option`, if it has one. + ## + ## **See also:** + ## * `map proc <#map,Option[T],proc(T)_2>`_ for a version with a callback + ## which returns a value + runnableExamples: + var d = 0 + proc saveDouble(x: int) = + d = 2 * x + + none(int).map(saveDouble) + assert d == 0 + some(42).map(saveDouble) + assert d == 84 - test "none": - expect UnpackError: - discard none(int).get() - check(none(int).isNone) - check(not none(string).isSome) + if self.isSome: + callback(self.val) - test "equality": - check some("a") == some("a") - check some(7) != some(6) - check some("a") != stringNone - check intNone == intNone +proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline, effectsOf: callback.} = + ## Applies a `callback` function to the value of the `Option` and returns an + ## `Option` containing the new value. + ## + ## If the `Option` has no value, `none(R)` will be returned. + ## + ## **See also:** + ## * `map proc <#map,Option[T],proc(T)>`_ + ## * `flatMap proc <#flatMap,Option[T],proc(T)>`_ for a version with a + ## callback that returns an `Option` + runnableExamples: + proc isEven(x: int): bool = + x mod 2 == 0 + + assert some(42).map(isEven) == some(true) + assert none(int).map(isEven) == none(bool) - when compiles(some("a") == some(5)): - check false - when compiles(none(string) == none(int)): - check false + if self.isSome: + some[R](callback(self.val)) + else: + none(R) - test "get with a default value": - check( some("Correct").get("Wrong") == "Correct" ) - check( stringNone.get("Correct") == "Correct" ) +proc flatten*[T](self: Option[Option[T]]): Option[T] {.inline.} = + ## Remove one level of structure in a nested `Option`. + ## + ## **See also:** + ## * `flatMap proc <#flatMap,Option[T],proc(T)>`_ + runnableExamples: + assert flatten(some(some(42))) == some(42) + assert flatten(none(Option[int])) == none(int) - test "$": - check( $(some("Correct")) == "Some(Correct)" ) - check( $(stringNone) == "None[string]" ) + if self.isSome: + self.val + else: + none(T) - test "map with a void result": - var procRan = 0 - some(123).map(proc (v: int) = procRan = v) - check procRan == 123 - intNone.map(proc (v: int) = check false) +proc flatMap*[T, R](self: Option[T], + callback: proc (input: T): Option[R]): Option[R] {.inline, effectsOf: callback.} = + ## Applies a `callback` function to the value of the `Option` and returns the new value. + ## + ## If the `Option` has no value, `none(R)` will be returned. + ## + ## This is similar to `map`, with the difference that the `callback` returns an + ## `Option`, not a raw value. This allows multiple procs with a + ## signature of `A -> Option[B]` to be chained together. + ## + ## See also: + ## * `flatten proc <#flatten,Option[Option[A]]>`_ + ## * `filter proc <#filter,Option[T],proc(T)>`_ + runnableExamples: + proc doublePositives(x: int): Option[int] = + if x > 0: + some(2 * x) + else: + none(int) + + assert some(42).flatMap(doublePositives) == some(84) + assert none(int).flatMap(doublePositives) == none(int) + assert some(-11).flatMap(doublePositives) == none(int) + + map(self, callback).flatten() + +proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline, effectsOf: callback.} = + ## Applies a `callback` to the value of the `Option`. + ## + ## If the `callback` returns `true`, the option is returned as `some`. + ## If it returns `false`, it is returned as `none`. + ## + ## **See also:** + ## * `flatMap proc <#flatMap,Option[A],proc(A)>`_ + runnableExamples: + proc isEven(x: int): bool = + x mod 2 == 0 + + assert some(42).filter(isEven) == some(42) + assert none(int).filter(isEven) == none(int) + assert some(-11).filter(isEven) == none(int) + + if self.isSome and not callback(self.val): + none(T) + else: + self - test "map": - check( some(123).map(proc (v: int): int = v * 2) == some(246) ) - check( intNone.map(proc (v: int): int = v * 2).isNone ) +proc `==`*[T](a, b: Option[T]): bool {.inline.} = + ## Returns `true` if both `Option`s are `none`, + ## or if they are both `some` and have equal values. + runnableExamples: + let + a = some(42) + b = none(int) + c = some(42) + d = none(int) + + assert a == c + assert b == d + assert not (a == b) + + when T is SomePointer: + a.val == b.val + else: + (a.isSome and b.isSome and a.val == b.val) or (a.isNone and b.isNone) - test "filter": - check( some(123).filter(proc (v: int): bool = v == 123) == some(123) ) - check( some(456).filter(proc (v: int): bool = v == 123).isNone ) - check( intNone.filter(proc (v: int): bool = check false).isNone ) +proc `$`*[T](self: Option[T]): string = + ## Get the string representation of the `Option`. + runnableExamples: + assert $some(42) == "some(42)" + assert $none(int) == "none(int)" + if self.isSome: + when defined(nimLagacyOptionsDollar): + result = "Some(" + else: + result = "some(" + result.addQuoted self.val + result.add ")" + else: + when defined(nimLagacyOptionsDollar): + result = "None[" & name(T) & "]" + else: + result = "none(" & name(T) & ")" + +proc unsafeGet*[T](self: Option[T]): lent T {.inline.}= + ## Returns the value of a `some`. The behavior is undefined for `none`. + ## + ## **Note:** Use this only when you are **absolutely sure** the value is present + ## (e.g. after checking with `isSome <#isSome,Option[T]>`_). + ## Generally, using the `get proc <#get,Option[T]>`_ is preferred. + assert self.isSome + result = self.val diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1e00f92b1..78ebb1c88 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -8,1329 +8,589 @@ # ## This module contains basic operating system facilities like -## retrieving environment variables, reading command line arguments, -## working with directories, running shell commands, etc. -{.deadCodeElim: on.} +## retrieving environment variables, working with directories, +## running shell commands, etc. -{.push debugger: off.} +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim -include "system/inclrtl" +runnableExamples: + let myFile = "/path/to/my/file.nim" + assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim") + when defined(posix): + assert parentDir(myFile) == "/path/to/my" + assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim") + assert myFile.changeFileExt("c") == "/path/to/my/file.c" + +## **See also:** +## * `paths <paths.html>`_ and `files <files.html>`_ modules for high-level file manipulation +## * `osproc module <osproc.html>`_ for process communication beyond +## `execShellCmd proc`_ +## * `uri module <uri.html>`_ +## * `distros module <distros.html>`_ +## * `dynlib module <dynlib.html>`_ +## * `streams module <streams.html>`_ +import std/private/ospaths2 +export ospaths2 + +import std/private/osfiles +export osfiles + +import std/private/osdirs +export osdirs + +import std/private/ossymlinks +export ossymlinks + +import std/private/osappdirs +export osappdirs + +import std/private/oscommon + +include system/inclrtl +import std/private/since + +import std/cmdline +export cmdline + +import std/[strutils, pathnorm] -import - strutils, times +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] -when defined(windows): - import winlean +const weirdTarget = defined(nimscript) or defined(js) + +since (1, 1): + const + invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ + ## Characters that may produce invalid filenames across Linux, Windows and Mac. + ## You can check if your filename contains any of these chars and strip them for safety. + ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. + invalidFilenames* = [ + "CON", "PRN", "AUX", "NUL", + "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"] ## \ + ## Filenames that may be invalid across Linux, Windows, Mac, etc. + ## You can check if your filename match these and rename it for safety + ## (Currently all invalid filenames are from Windows only). + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] elif defined(posix): - import posix + import std/[posix, times] + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) else: {.error: "OS module not ported to your operating system!".} -include "system/ansi_c" -include ospaths +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} -when defined(posix): - when NoFakeVars: - const pathMax = 5000 # doesn't matter really. The concept of PATH_MAX - # doesn't work anymore on modern OSes. - else: - var - pathMax {.importc: "PATH_MAX", header: "<stdlib.h>".}: cint +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} -proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Retrieves the operating system's error flag, ``errno``. - ## On Windows ``GetLastError`` is checked before ``errno``. - ## Returns "" if no error occurred. - ## - ## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc. - result = "" - when defined(Windows): - var err = getLastError() - if err != 0'i32: - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, - nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, - nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - if errno != 0'i32: - result = $os.strerror(errno) - -{.push warning[deprecated]: off.} -proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1", - deprecated.} = - ## raises an OSError exception with the given message ``msg``. - ## If ``msg == ""``, the operating system's error flag - ## (``errno``) is converted to a readable error message. On Windows - ## ``GetLastError`` is checked before ``errno``. - ## If no error flag is set, the message ``unknown OS error`` is used. - ## - ## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc. - if len(msg) == 0: - var m = osErrorMsg() - raise newException(OSError, if m.len > 0: m else: "unknown OS error") - else: - raise newException(OSError, msg) -{.pop.} +import std/oserrors +export oserrors +import std/envvars +export envvars -when not defined(nimfix): - {.deprecated: [osError: raiseOSError].} +import std/private/osseps +export osseps -proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} -proc `$`*(err: OSErrorCode): string {.borrow.} -proc osErrorMsg*(errorCode: OSErrorCode): string = - ## Converts an OS error code into a human readable string. - ## - ## The error code can be retrieved using the ``osLastError`` proc. - ## - ## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be - ## returned. - ## - ## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to - ## make this procedure use the non-unicode Win API calls to retrieve the - ## message. - result = "" - when defined(Windows): - if errorCode != OSErrorCode(0'i32): - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - else: - if errorCode != OSErrorCode(0'i32): - result = $os.strerror(errorCode.int32) -proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = - ## Raises an ``OSError`` exception. The ``errorCode`` will determine the - ## message, ``osErrorMsg`` will be used to get this message. +proc expandTilde*(path: string): string {. + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with `getHomeDir()`_ (otherwise returns ``path`` unmodified). ## - ## The error code can be retrieved using the ``osLastError`` proc. + ## Windows: this is still supported despite the Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. ## - ## If the error code is ``0`` or an error message could not be retrieved, - ## the message ``unknown OS error`` will be used. - var e: ref OSError; new(e) - e.errorCode = errorCode.int32 - if additionalInfo.len == 0: - e.msg = osErrorMsg(errorCode) + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + runnableExamples: + assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" + assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" + assert expandTilde("/foo/bar") == "/foo/bar" + + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): + result = getHomeDir() / path.substr(2) else: - e.msg = additionalInfo & " " & osErrorMsg(errorCode) - if e.msg == "": - e.msg = "unknown OS error" - raise e - -{.push stackTrace:off.} -proc osLastError*(): OSErrorCode = - ## Retrieves the last operating system error code. - ## - ## This procedure is useful in the event when an OS call fails. In that case - ## this procedure will return the error code describing the reason why the - ## OS call failed. The ``OSErrorMsg`` procedure can then be used to convert - ## this code into a string. - ## - ## **Warning**: - ## The behaviour of this procedure varies between Windows and POSIX systems. - ## On Windows some OS calls can reset the error code to ``0`` causing this - ## procedure to return ``0``. It is therefore advised to call this procedure - ## immediately after an OS call fails. On POSIX systems this is not a problem. + # TODO: handle `~bob` and `~bob/` which means home of bob + result = path - when defined(windows): - result = OSErrorCode(getLastError()) - else: - result = OSErrorCode(errno) -{.pop.} - -when defined(windows): - when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: expr) {.immediate.} = - var varname = winApiProc(newWideCString(arg)) - - template wrapBinary(varname, winApiProc, arg, arg2: expr) {.immediate.} = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: expr): expr = findNextFileW(a, b) - template getCommandLine(): expr = getCommandLineW() - - template getFilename(f: expr): expr = - $cast[WideCString](addr(f.cFilename[0])) - else: - template findFirstFile(a, b: expr): expr = findFirstFileA(a, b) - template findNextFile(a, b: expr): expr = findNextFileA(a, b) - template getCommandLine(): expr = getCommandLineA() - - template getFilename(f: expr): expr = $f.cFilename - - proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = - # Note - takes advantage of null delimiter in the cstring - const dot = ord('.') - result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or - f.cFileName[1].int == dot and f.cFileName[2].int == 0) - -proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect].} = - ## Returns true if the file exists, false otherwise. - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) +proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote `s`, so it can be safely passed to Windows API. + ## + ## Based on Python's `subprocess.list2cmdline`. + ## See `this link <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_ + ## for more details. + let needQuote = {' ', '\t'} in s or s.len == 0 + result = "" + var backslashBuff = "" + if needQuote: + result.add("\"") + + for c in s: + if c == '\\': + backslashBuff.add(c) + elif c == '\"': + for i in 0..<backslashBuff.len*2: + result.add('\\') + backslashBuff.setLen(0) + result.add("\\\"") else: - var a = getFileAttributesA(filename) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 + if backslashBuff.len != 0: + result.add(backslashBuff) + backslashBuff.setLen(0) + result.add(c) + + if backslashBuff.len > 0: + result.add(backslashBuff) + if needQuote: + result.add(backslashBuff) + result.add("\"") + + +proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote ``s``, so it can be safely passed to POSIX shell. + const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', + '0'..'9', 'A'..'Z', 'a'..'z'} + if s.len == 0: + result = "''" + elif s.allCharsInSet(safeUnixChars): + result = s else: - var res: Stat - return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) + result = "'" & s.replace("'", "'\"'\"'") & "'" -proc existsDir*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns true iff the directory `dir` exists. If `dir` is a file, false - ## is returned. - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) +when defined(windows) or defined(posix) or defined(nintendoswitch): + proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote ``s``, so it can be safely passed to shell. + ## + ## When on Windows, it calls `quoteShellWindows proc`_. + ## Otherwise, calls `quoteShellPosix proc`_. + when defined(windows): + result = quoteShellWindows(s) else: - var a = getFileAttributesA(dir) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + result = quoteShellPosix(s) + + proc quoteShellCommand*(args: openArray[string]): string = + ## Concatenates and quotes shell arguments `args`. + runnableExamples: + when defined(posix): + assert quoteShellCommand(["aaa", "", "c d"]) == "aaa '' 'c d'" + when defined(windows): + assert quoteShellCommand(["aaa", "", "c d"]) == "aaa \"\" \"c d\"" + + # can't use `map` pending https://github.com/nim-lang/Nim/issues/8303 + result = "" + for i in 0..<args.len: + if i > 0: result.add " " + result.add quoteShell(args[i]) + +when not weirdTarget: + proc c_system(cmd: cstring): cint {. + importc: "system", header: "<stdlib.h>".} + + when not defined(windows): + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + + +const + ExeExts* = ## Platform specific file extension for executables. + ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``. + when defined(windows): ["exe", "cmd", "bat"] else: [""] + +proc findExe*(exe: string, followSymlinks: bool = true; + extensions: openArray[string]=ExeExts): string {. + tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect], noNimJs.} = + ## Searches for `exe` in the current working directory and then + ## in directories listed in the ``PATH`` environment variable. + ## + ## Returns `""` if the `exe` cannot be found. `exe` + ## is added the `ExeExts`_ file extensions if it has none. + ## + ## If the system supports symlinks it also resolves them until it + ## meets the actual file. This behavior can be disabled if desired + ## by setting `followSymlinks = false`. + + if exe.len == 0: return + template checkCurrentDir() = + for ext in extensions: + result = addFileExt(exe, ext) + if fileExists(result): return + when defined(posix): + if '/' in exe: checkCurrentDir() else: - var res: Stat - return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) - -proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect].} = - ## Returns true iff the symlink `link` exists. Will return true - ## regardless of whether the link points to a directory or file. - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) + checkCurrentDir() + let path = getEnv("PATH") + for candidate in split(path, PathSep): + if candidate.len == 0: continue + when defined(windows): + var x = (if candidate[0] == '"' and candidate[^1] == '"': + substr(candidate, 1, candidate.len-2) else: candidate) / + exe else: - var a = getFileAttributesA(link) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 - else: - var res: Stat - return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) - -proc fileExists*(filename: string): bool {.inline.} = - ## Synonym for existsFile - existsFile(filename) + var x = expandTilde(candidate) / exe + for ext in extensions: + var x = addFileExt(x, ext) + if fileExists(x): + when not (defined(windows) or defined(nintendoswitch)): + while followSymlinks: # doubles as if here + if x.symlinkExists: + var r = newString(maxSymlinkLen) + var len = readlink(x.cstring, r.cstring, maxSymlinkLen) + if len < 0: + raiseOSError(osLastError(), exe) + if len > maxSymlinkLen: + r = newString(len+1) + len = readlink(x.cstring, r.cstring, len) + setLen(r, len) + if isAbsolute(r): + x = r + else: + x = parentDir(x) / r + else: + break + return x + result = "" -proc dirExists*(dir: string): bool {.inline.} = - ## Synonym for existsDir - existsDir(dir) +when weirdTarget: + const times = "fake const" + template Time(x: untyped): untyped = string -proc getLastModificationTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns the `file`'s last modification time. + ## + ## See also: + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat - if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_mtime + if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) + result = res.st_mtim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) - if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)) + if h == -1'i32: raiseOSError(osLastError(), file) + result = fromWinTime(rdFileTime(f.ftLastWriteTime)) findClose(h) -proc getLastAccessTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns the `file`'s last read or write access time. + ## + ## See also: + ## * `getLastModificationTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat - if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_atime + if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) + result = res.st_atim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) - if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)) + if h == -1'i32: raiseOSError(osLastError(), file) + result = fromWinTime(rdFileTime(f.ftLastAccessTime)) findClose(h) -proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} = +proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns the `file`'s creation time. ## ## **Note:** Under POSIX OS's, the returned time may actually be the time at ## which the file's attribute's were last modified. See ## `here <https://github.com/nim-lang/Nim/issues/1058>`_ for details. + ## + ## See also: + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat - if stat(file, res) < 0'i32: raiseOSError(osLastError()) - return res.st_ctime + if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) + result = res.st_ctim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) - if h == -1'i32: raiseOSError(osLastError()) - result = winTimeToUnixTime(rdFileTime(f.ftCreationTime)) + if h == -1'i32: raiseOSError(osLastError(), file) + result = fromWinTime(rdFileTime(f.ftCreationTime)) findClose(h) -proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} = +proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns true if the file `a` is newer than file `b`, i.e. if `a`'s ## modification time is later than `b`'s. - when defined(posix): - result = getLastModificationTime(a) - getLastModificationTime(b) >= 0 - # Posix's resolution sucks so, we use '>=' for posix. - else: - result = getLastModificationTime(a) - getLastModificationTime(b) > 0 - -proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = - ## Returns the `current working directory`:idx:. - const bufsize = 512 # should be enough - when defined(windows): - when useWinUnicode: - var res = newWideCString("", bufsize) - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: raiseOSError(osLastError()) - result = res$L - else: - result = newString(bufsize) - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: raiseOSError(osLastError()) - setLen(result, L) - else: - result = newString(bufsize) - if getcwd(result, bufsize) != nil: - setLen(result, c_strlen(result)) - else: - raiseOSError(osLastError()) - -proc setCurrentDir*(newDir: string) {.inline, tags: [].} = - ## Sets the `current working directory`:idx:; `OSError` is raised if - ## `newDir` cannot been set. - when defined(Windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError()) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError()) - else: - if chdir(newDir) != 0'i32: raiseOSError(osLastError()) - -proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", - tags: [ReadDirEffect].} = - ## Returns the full path of `filename`, raises OSError in case of an error. - when defined(windows): - const bufsize = 3072'i32 - when useWinUnicode: - var unused: WideCString - var res = newWideCString("", bufsize div 2) - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L <= 0'i32 or L >= bufsize: - raiseOSError(osLastError()) - result = res$L - else: - var unused: cstring - result = newString(bufsize) - var L = getFullPathNameA(filename, bufsize, result, unused) - if L <= 0'i32 or L >= bufsize: raiseOSError(osLastError()) - setLen(result, L) - else: - # careful, realpath needs to take an allocated buffer according to Posix: - result = newString(pathMax) - var r = realpath(filename, result) - if r.isNil: raiseOSError(osLastError()) - setLen(result, c_strlen(result)) - -when defined(Windows): - proc openHandle(path: string, followSymlink=true): Handle = - var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL - if not followSymlink: - flags = flags or FILE_FLAG_OPEN_REPARSE_POINT - - when useWinUnicode: - result = createFileW( - newWideCString(path), 0'i32, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, 0'i32, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - -proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect].} = - ## Returns true if both pathname arguments refer to the same physical - ## file or directory. Raises an exception if any of the files does not - ## exist or information about it can not be obtained. ## - ## This proc will return true if given two alternative hard-linked or - ## sym-linked paths to the same file or directory. - when defined(Windows): - var success = true - var f1 = openHandle(path1) - var f2 = openHandle(path2) - - var lastErr: OSErrorCode - if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: - var fi1, fi2: BY_HANDLE_FILE_INFORMATION - - if getFileInformationByHandle(f1, addr(fi1)) != 0 and - getFileInformationByHandle(f2, addr(fi2)) != 0: - result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and - fi1.nFileIndexHigh == fi2.nFileIndexHigh and - fi1.nFileIndexLow == fi2.nFileIndexLow - else: - lastErr = osLastError() - success = false - else: - lastErr = osLastError() - success = false - - discard closeHandle(f1) - discard closeHandle(f2) - - if not success: raiseOSError(lastErr) - else: - var a, b: Stat - if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: - raiseOSError(osLastError()) - else: - result = a.st_dev == b.st_dev and a.st_ino == b.st_ino - -proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - ## Returns true if both pathname arguments refer to files with identical - ## binary content. - const - bufSize = 8192 # 8K buffer - var - a, b: File - if not open(a, path1): return false - if not open(b, path2): - close(a) - return false - var bufA = alloc(bufSize) - var bufB = alloc(bufSize) - while true: - var readA = readBuffer(a, bufA, bufSize) - var readB = readBuffer(b, bufB, bufSize) - if readA != readB: - result = false - break - if readA == 0: - result = true - break - result = equalMem(bufA, bufB, readA) - if not result: break - if readA != bufSize: break # end of file - dealloc(bufA) - dealloc(bufB) - close(a) - close(b) - -type - FilePermission* = enum ## file access permission; modelled after UNIX - fpUserExec, ## execute access for the file owner - fpUserWrite, ## write access for the file owner - fpUserRead, ## read access for the file owner - fpGroupExec, ## execute access for the group - fpGroupWrite, ## write access for the group - fpGroupRead, ## read access for the group - fpOthersExec, ## execute access for others - fpOthersWrite, ## write access for others - fpOthersRead ## read access for others - -{.deprecated: [TFilePermission: FilePermission].} - -proc getFilePermissions*(filename: string): set[FilePermission] {. - rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## retrieves file permissions for `filename`. `OSError` is raised in case of - ## an error. On Windows, only the ``readonly`` flag is checked, every other - ## permission is available in any case. + ## See also: + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ when defined(posix): - var a: Stat - if stat(filename, a) < 0'i32: raiseOSError(osLastError()) - result = {} - if (a.st_mode and S_IRUSR) != 0'i32: result.incl(fpUserRead) - if (a.st_mode and S_IWUSR) != 0'i32: result.incl(fpUserWrite) - if (a.st_mode and S_IXUSR) != 0'i32: result.incl(fpUserExec) - - if (a.st_mode and S_IRGRP) != 0'i32: result.incl(fpGroupRead) - if (a.st_mode and S_IWGRP) != 0'i32: result.incl(fpGroupWrite) - if (a.st_mode and S_IXGRP) != 0'i32: result.incl(fpGroupExec) - - if (a.st_mode and S_IROTH) != 0'i32: result.incl(fpOthersRead) - if (a.st_mode and S_IWOTH) != 0'i32: result.incl(fpOthersWrite) - if (a.st_mode and S_IXOTH) != 0'i32: result.incl(fpOthersExec) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError()) - if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: - result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, - fpOthersExec, fpOthersRead} + # If we don't have access to nanosecond resolution, use '>=' + when not StatHasNanoseconds: + result = getLastModificationTime(a) >= getLastModificationTime(b) else: - result = {fpUserExec..fpOthersRead} + result = getLastModificationTime(a) > getLastModificationTime(b) + else: + result = getLastModificationTime(a) > getLastModificationTime(b) -proc setFilePermissions*(filename: string, permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [WriteDirEffect].} = - ## sets the file permissions for `filename`. `OSError` is raised in case of - ## an error. On Windows, only the ``readonly`` flag is changed, depending on - ## ``fpUserWrite``. - when defined(posix): - var p = 0'i32 - if fpUserRead in permissions: p = p or S_IRUSR - if fpUserWrite in permissions: p = p or S_IWUSR - if fpUserExec in permissions: p = p or S_IXUSR - if fpGroupRead in permissions: p = p or S_IRGRP - if fpGroupWrite in permissions: p = p or S_IWGRP - if fpGroupExec in permissions: p = p or S_IXGRP +proc isAdmin*: bool {.noWeirdTarget.} = + ## Returns whether the caller's process is a member of the Administrators local + ## group (on Windows) or a root (on POSIX), via `geteuid() == 0`. + when defined(windows): + # Rewrite of the example from Microsoft Docs: + # https://docs.microsoft.com/en-us/windows/win32/api/securitybaseapi/nf-securitybaseapi-checktokenmembership#examples + # and corresponding PostgreSQL function: + # https://doxygen.postgresql.org/win32security_8c.html#ae6b61e106fa5d6c5d077a9d14ee80569 + var ntAuthority = SID_IDENTIFIER_AUTHORITY(value: SECURITY_NT_AUTHORITY) + var administratorsGroup: PSID + if not isSuccess(allocateAndInitializeSid(addr ntAuthority, + BYTE(2), + SECURITY_BUILTIN_DOMAIN_RID, + DOMAIN_ALIAS_RID_ADMINS, + 0, 0, 0, 0, 0, 0, + addr administratorsGroup)): + raiseOSError(osLastError(), "could not get SID for Administrators group") - if fpOthersRead in permissions: p = p or S_IROTH - if fpOthersWrite in permissions: p = p or S_IWOTH - if fpOthersExec in permissions: p = p or S_IXOTH + try: + var b: WINBOOL + if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): + raiseOSError(osLastError(), "could not check access token membership") - if chmod(filename, p) != 0: raiseOSError(osLastError()) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError()) - if fpUserWrite in permissions: - res = res and not FILE_ATTRIBUTE_READONLY - else: - res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) - if res2 == - 1'i32: raiseOSError(osLastError()) + result = isSuccess(b) + finally: + if freeSid(administratorsGroup) != nil: + raiseOSError(osLastError(), "failed to free SID for Administrators group") -proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadIOEffect, WriteIOEffect].} = - ## Copies a file from `source` to `dest`. - ## - ## If this fails, `OSError` is raised. On the Windows platform this proc will - ## copy the source file's attributes into dest. On other platforms you need - ## to use `getFilePermissions() <#getFilePermissions>`_ and - ## `setFilePermissions() <#setFilePermissions>`_ to copy them by hand (or use - ## the convenience `copyFileWithPermissions() <#copyFileWithPermissions>`_ - ## proc), otherwise `dest` will inherit the default permissions of a newly - ## created file for the user. If `dest` already exists, the file attributes - ## will be preserved and the content overwritten. - when defined(Windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: raiseOSError(osLastError()) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError()) else: - # generic version of copyFile which works for any platform: - const bufSize = 8000 # better for memory manager - var d, s: File - if not open(s, source): raiseOSError(osLastError()) - if not open(d, dest, fmWrite): - close(s) - raiseOSError(osLastError()) - var buf = alloc(bufSize) - while true: - var bytesread = readBuffer(s, buf, bufSize) - if bytesread > 0: - var byteswritten = writeBuffer(d, buf, bytesread) - if bytesread != byteswritten: - dealloc(buf) - close(s) - close(d) - raiseOSError(osLastError()) - if bytesread != bufSize: break - dealloc(buf) - close(s) - close(d) - -proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadIOEffect, WriteIOEffect].} = - ## Moves a file from `source` to `dest`. If this fails, `OSError` is raised. - when defined(Windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if moveFileW(s, d, 0'i32) == 0'i32: raiseOSError(osLastError()) - else: - if moveFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError()) - else: - if c_rename(source, dest) != 0'i32: - raiseOSError(osLastError(), $strerror(errno)) + result = geteuid() == 0 -when not declared(ENOENT) and not defined(Windows): - when NoFakeVars: - const ENOENT = cint(2) # 2 on most systems including Solaris - else: - var ENOENT {.importc, header: "<errno.h>".}: cint -when defined(Windows): - when useWinUnicode: - template deleteFile(file: expr): expr {.immediate.} = deleteFileW(file) - template setFileAttributes(file, attrs: expr): expr {.immediate.} = - setFileAttributesW(file, attrs) - else: - template deleteFile(file: expr): expr {.immediate.} = deleteFileA(file) - template setFileAttributes(file, attrs: expr): expr {.immediate.} = - setFileAttributesA(file, attrs) - -proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].} = - ## Removes the `file`. If this fails, `OSError` is raised. This does not fail - ## if the file never existed in the first place. - ## On Windows, ignores the read-only attribute. - when defined(Windows): - when useWinUnicode: - let f = newWideCString(file) +proc exitStatusLikeShell*(status: cint): cint = + ## Converts exit code from `c_system` into a shell exit code. + when defined(posix) and not weirdTarget: + if WIFSIGNALED(status): + # like the shell! + 128 + WTERMSIG(status) else: - let f = file - if deleteFile(f) == 0: - if getLastError() == ERROR_ACCESS_DENIED: - if setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) == 0: - raiseOSError(osLastError()) - if deleteFile(f) == 0: - raiseOSError(osLastError()) + WEXITSTATUS(status) else: - if c_remove(file) != 0'i32 and errno != ENOENT: - raiseOSError(osLastError(), $strerror(errno)) + status proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", - tags: [ExecIOEffect].} = + tags: [ExecIOEffect], noWeirdTarget.} = ## Executes a `shell command`:idx:. ## ## Command has the form 'program args' where args are the command ## line arguments given to program. The proc returns the error code - ## of the shell when it has finished. The proc does not return until - ## the process has finished. To execute a program without having a - ## shell involved, use the `execProcess` proc of the `osproc` - ## module. - when defined(linux): - result = c_system(command) shr 8 - else: - result = c_system(command) - -# Environment handling cannot be put into RTL, because the ``envPairs`` -# iterator depends on ``environment``. - -var - envComputed {.threadvar.}: bool - environment {.threadvar.}: seq[string] - -when defined(windows): - # because we support Windows GUI applications, things get really - # messy here... - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "<string.h>".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {. - importc: "strchr", header: "<string.h>".} - - proc getEnvVarsC() = - if not envComputed: - environment = @[] - when useWinUnicode: - var - env = getEnvironmentStringsW() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[WideCString](cast[ByteAddress](eend)+2) - if eend[1].int == 0: break - discard freeEnvironmentStringsW(env) - else: - var - env = getEnvironmentStringsA() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[cstring](cast[ByteAddress](eend)+1) - if eend[1] == '\0': break - discard freeEnvironmentStringsA(env) - envComputed = true - -else: - const - useNSGetEnviron = defined(macosx) - - when useNSGetEnviron: - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # <crt_externs.h>, can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {. - importc: "_NSGetEnviron", header: "<crt_externs.h>".} - else: - var gEnv {.importc: "environ".}: cstringArray - - proc getEnvVarsC() = - # retrieves the variables of char** env of C's main proc - if not envComputed: - environment = @[] - when useNSGetEnviron: - var gEnv = NSGetEnviron()[] - var i = 0 - while true: - if gEnv[i] == nil: break - add environment, $gEnv[i] - inc(i) - envComputed = true - -proc findEnvVar(key: string): int = - getEnvVarsC() - var temp = key & '=' - for i in 0..high(environment): - if startsWith(environment[i], temp): return i - return -1 - -proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. + ## of the shell when it has finished (zero if there is no error). + ## The proc does not return until the process has finished. ## - ## If the variable does not exist, "" is returned. To distinguish - ## whether a variable exists or it's value is just "", call - ## `existsEnv(key)`. - var i = findEnvVar(key) - if i >= 0: - return TaintedString(substr(environment[i], find(environment[i], '=')+1)) - else: - var env = c_getenv(key) - if env == nil: return TaintedString("") - result = TaintedString($env) - -proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - if c_getenv(key) != nil: return true - else: return findEnvVar(key) >= 0 - -proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `EInvalidEnvVar` is raised. - - # Note: by storing the string in the environment sequence, - # we guarantee that we don't free the memory before the program - # ends (this is needed for POSIX compliance). It is also needed so that - # the process itself may access its modified environment variables! - var indx = findEnvVar(key) - if indx >= 0: - environment[indx] = key & '=' & val - else: - add environment, (key & '=' & val) - indx = high(environment) - when defined(unix): - if c_putenv(environment[indx]) != 0'i32: - raiseOSError(osLastError()) - else: - when useWinUnicode: - var k = newWideCString(key) - var v = newWideCString(val) - if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError()) - -iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. In the first component - ## of the tuple is the name of the current variable stored, in the second - ## its value. - getEnvVarsC() - for i in 0..high(environment): - var p = find(environment[i], '=') - yield (TaintedString(substr(environment[i], 0, p-1)), - TaintedString(substr(environment[i], p+1))) - -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = - ## Iterate over all the files that match the `pattern`. On POSIX this uses - ## the `glob`:idx: call. + ## To execute a program without having a shell involved, use `osproc.execProcess proc + ## <osproc.html#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. ## - ## `pattern` is OS dependent, but at least the "\*.ext" - ## notation is supported. - when defined(windows): - var - f: WIN32_FIND_DATA - res: int - res = findFirstFile(pattern, f) - if res != -1: - while true: - if not skipFindData(f) and - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32: - # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check - # that the file extensions have the same length ... - let ff = getFilename(f) - let dotPos = searchExtPos(pattern) - let idx = ff.len - pattern.len + dotPos - if dotPos < 0 or idx >= ff.len or ff[idx] == '.' or - pattern[dotPos+1] == '*': - yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: break - findClose(res) - else: # here we use glob - var - f: Glob - res: int - f.gl_offs = 0 - f.gl_pathc = 0 - f.gl_pathv = nil - res = glob(pattern, 0, nil, addr(f)) - if res == 0: - for i in 0.. f.gl_pathc - 1: - assert(f.gl_pathv[i] != nil) - yield $f.gl_pathv[i] - globfree(addr(f)) - -type - PathComponent* = enum ## Enumeration specifying a path component. - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory + ## **Examples:** + ## ```Nim + ## discard execShellCmd("ls -la") + ## ``` + result = exitStatusLikeShell(c_system(command)) -{.deprecated: [TPathComponent: PathComponent].} - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent, path: string]] = - discard - -iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. - tags: [ReadDirEffect].} = - ## walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item is returned. - ## Walking is not recursive. If ``relative`` is true the resulting path is - ## shortened to be relative to ``dir``. - ## Example: This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: - ## - ## .. code-block:: Nim - ## for kind, path in walkDir("dirA"): - ## echo(path) - ## - ## produces this output (but not necessarily in this order!):: - ## dirA/dirB - ## dirA/dirC - ## dirA/fileA1.txt - ## dirA/fileA2.txt - when nimvm: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - else: - when defined(windows): - var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h != -1: - while true: - var k = pcFile - if not skipFindData(f): - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - yield (k, xx) - if findNextFile(h, f) == 0'i32: break - findClose(h) - else: - var d = opendir(dir) - if d != nil: - while true: - var x = readdir(d) - if x == nil: break - var y = $x.d_name - if y != "." and y != "..": - var s: Stat - if not relative: - y = dir / y - var k = pcFile - - when defined(linux) or defined(macosx) or defined(bsd): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: k = pcDir - if x.d_type == DT_LNK: - if dirExists(y): k = pcLinkToDir - else: k = succ(k) - yield (k, y) - continue - - if lstat(y, s) < 0'i32: break - if S_ISDIR(s.st_mode): k = pcDir - if S_ISLNK(s.st_mode): k = succ(k) - yield (k, y) - discard closedir(d) - -iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {. - tags: [ReadDirEffect].} = - ## walks over the directory `dir` and yields for each file in `dir`. The - ## full path for each file is returned. - ## **Warning**: - ## Modifying the directory structure while the iterator - ## is traversing may result in undefined behavior! - ## - ## Walking is recursive. `filter` controls the behaviour of the iterator: - ## - ## --------------------- --------------------------------------------- - ## filter meaning - ## --------------------- --------------------------------------------- - ## ``pcFile`` yield real files - ## ``pcLinkToFile`` yield symbolic links to files - ## ``pcDir`` follow real directories - ## ``pcLinkToDir`` follow symbolic links to directories - ## --------------------- --------------------------------------------- +proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noWeirdTarget.} = + ## Returns the full (`absolute`:idx:) path of an existing file `filename`. ## - var stack = @[dir] - while stack.len > 0: - for k,p in walkDir(stack.pop()): - if k in filter: - case k - of pcFile, pcLinkToFile: yield p - of pcDir, pcLinkToDir: stack.add(p) - -proc rawRemoveDir(dir: string) = + ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) - let lastError = osLastError() - if res == 0'i32 and lastError.int32 != 3'i32 and - lastError.int32 != 18'i32 and lastError.int32 != 2'i32: - raiseOSError(lastError) - else: - if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError()) - -proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ - WriteDirEffect, ReadDirEffect], benign.} = - ## Removes the directory `dir` including all subdirectories and files - ## in `dir` (recursively). - ## - ## If this fails, `OSError` is raised. This does not fail if the directory never - ## existed in the first place. - for kind, path in walkDir(dir): - case kind - of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) - of pcDir: removeDir(path) - rawRemoveDir(dir) - -proc rawCreateDir(dir: string) = - when defined(solaris): - if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST and errno != ENOSYS: - raiseOSError(osLastError()) - elif defined(unix): - if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST: - raiseOSError(osLastError()) + var bufsize = MAX_PATH.int32 + var unused: WideCString = nil + var res = newWideCString(bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break + # getFullPathName doesn't do case corrections, so we have to use this convoluted + # way of retrieving the true filename + for x in walkFiles(result): + result = x + if not fileExists(result) and not dirExists(result): + # consider using: `raiseOSError(osLastError(), result)` + raise newException(OSError, "file '" & result & "' does not exist") else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) + # according to Posix we don't need to allocate space for result pathname. + # But we need to free return value with free(3). + var r = realpath(filename, nil) + if r.isNil: + raiseOSError(osLastError(), filename) else: - var res = createDirectoryA(dir) - if res == 0'i32 and getLastError() != 183'i32: - raiseOSError(osLastError()) + result = $r + c_free(cast[pointer](r)) -proc createDir*(dir: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].} = - ## Creates the `directory`:idx: `dir`. +proc getCurrentCompilerExe*(): string {.compileTime.} = discard + ## Returns the path of the currently running Nim compiler or nimble executable. ## - ## The directory may contain several subdirectories that do not exist yet. - ## The full path is created. If this fails, `OSError` is raised. It does **not** - ## fail if the path already exists because for most usages this does not - ## indicate an error. - var omitNext = false - when doslike: - omitNext = isAbsolute(dir) - for i in 1.. dir.len-1: - if dir[i] in {DirSep, AltSep}: - if omitNext: - omitNext = false - else: - rawCreateDir(substr(dir, 0, i-1)) - rawCreateDir(dir) - -proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect], benign.} = - ## Copies a directory from `source` to `dest`. - ## - ## If this fails, `OSError` is raised. On the Windows platform this proc will - ## copy the attributes from `source` into `dest`. On other platforms created - ## files and directories will inherit the default permissions of a newly - ## created file/directory for the user. To preserve attributes recursively on - ## these platforms use `copyDirWithPermissions() <#copyDirWithPermissions>`_. - createDir(dest) - for kind, path in walkDir(source): - var noSource = path.substr(source.len()+1) - case kind - of pcFile: - copyFile(path, dest / noSource) - of pcDir: - copyDir(path, dest / noSource) - else: discard - -proc createSymlink*(src, dest: string) = - ## Create a symbolic link at `dest` which points to the item specified - ## by `src`. On most operating systems, will fail if a lonk - ## - ## **Warning**: - ## Some OS's (such as Microsoft Windows) restrict the creation - ## of symlinks to root users (administrators). - when defined(Windows): - let flag = dirExists(src).int32 - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError()) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError()) - else: - if symlink(src, dest) != 0: - raiseOSError(osLastError()) + ## Can be used to retrieve the currently executing + ## Nim compiler from a Nim or nimscript program, or the nimble binary + ## inside a nimble program (likewise with other binaries built from + ## compiler API). -proc createHardlink*(src, dest: string) = +proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified ## by `src`. ## - ## **Warning**: Most OS's restrict the creation of hard links to - ## root users (administrators) . - when defined(Windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError()) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError()) + ## .. warning:: Some OS's restrict the creation of hard links to + ## root users (administrators). + ## + ## See also: + ## * `createSymlink proc`_ + when defined(windows): + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: - raiseOSError(osLastError()) - -proc parseCmdLine*(c: string): seq[string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a command line into several components; - ## This proc is only occasionally useful, better use the `parseopt` module. - ## - ## On Windows, it uses the following parsing rules - ## (see http://msdn.microsoft.com/en-us/library/17w5ykft.aspx ): - ## - ## * Arguments are delimited by white space, which is either a space or a tab. - ## * The caret character (^) is not recognized as an escape character or - ## delimiter. The character is handled completely by the command-line parser - ## in the operating system before being passed to the argv array in the - ## program. - ## * A string surrounded by double quotation marks ("string") is interpreted - ## as a single argument, regardless of white space contained within. A - ## quoted string can be embedded in an argument. - ## * A double quotation mark preceded by a backslash (\") is interpreted as a - ## literal double quotation mark character ("). - ## * Backslashes are interpreted literally, unless they immediately precede - ## a double quotation mark. - ## * If an even number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is interpreted as a string delimiter. - ## * If an odd number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is "escaped" by the remaining backslash, - ## causing a literal double quotation mark (") to be placed in argv. - ## - ## On Posix systems, it uses the following parsing rules: - ## Components are separated by whitespace unless the whitespace - ## occurs within ``"`` or ``'`` quotes. - result = @[] - var i = 0 - var a = "" - while true: - setLen(a, 0) - # eat all delimiting whitespace - while c[i] == ' ' or c[i] == '\t' or c [i] == '\l' or c [i] == '\r' : inc(i) - when defined(windows): - # parse a single argument according to the above rules: - if c[i] == '\0': break - var inQuote = false - while true: - case c[i] - of '\0': break - of '\\': - var j = i - while c[j] == '\\': inc(j) - if c[j] == '"': - for k in 1..(j-i) div 2: a.add('\\') - if (j-i) mod 2 == 0: - i = j - else: - a.add('"') - i = j+1 - else: - a.add(c[i]) - inc(i) - of '"': - inc(i) - if not inQuote: inQuote = true - elif c[i] == '"': - a.add(c[i]) - inc(i) - else: - inQuote = false - break - of ' ', '\t': - if not inQuote: break - a.add(c[i]) - inc(i) - else: - a.add(c[i]) - inc(i) - else: - case c[i] - of '\'', '\"': - var delim = c[i] - inc(i) # skip ' or " - while c[i] != '\0' and c[i] != delim: - add a, c[i] - inc(i) - if c[i] != '\0': inc(i) - of '\0': break - else: - while c[i] > ' ': - add(a, c[i]) - inc(i) - add(result, a) - -proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true) = - ## Copies a file from `source` to `dest` preserving file permissions. - ## - ## This is a wrapper proc around `copyFile() <#copyFile>`_, - ## `getFilePermissions() <#getFilePermissions>`_ and `setFilePermissions() - ## <#setFilePermissions>`_ on non Windows platform. On Windows this proc is - ## just a wrapper for `copyFile() <#copyFile>`_ since that proc already - ## copies attributes. - ## - ## On non Windows systems permissions are copied after the file itself has - ## been copied, which won't happen atomically and could lead to a race - ## condition. If `ignorePermissionErrors` is true, errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - copyFile(source, dest) - when not defined(Windows): - try: - setFilePermissions(dest, getFilePermissions(source)) - except: - if not ignorePermissionErrors: - raise - -proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect], benign.} = - ## Copies a directory from `source` to `dest` preserving file permissions. - ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir() - ## <#copyDir>`_ and `copyFileWithPermissions() <#copyFileWithPermissions>`_ - ## on non Windows platforms. On Windows this proc is just a wrapper for - ## `copyDir() <#copyDir>`_ since that proc already copies attributes. - ## - ## On non Windows systems permissions are copied after the file or directory - ## itself has been copied, which won't happen atomically and could lead to a - ## race condition. If `ignorePermissionErrors` is true, errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - createDir(dest) - when not defined(Windows): - try: - setFilePermissions(dest, getFilePermissions(source)) - except: - if not ignorePermissionErrors: - raise - for kind, path in walkDir(source): - var noSource = path.substr(source.len()+1) - case kind - of pcFile: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors) - of pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) - else: discard + raiseOSError(osLastError(), $(src, dest)) proc inclFilePermissions*(filename: string, permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect].} = - ## a convenience procedure for: - ## - ## .. code-block:: nim + rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = + ## A convenience proc for: + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)+permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)+permissions) proc exclFilePermissions*(filename: string, permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect].} = - ## a convenience procedure for: - ## - ## .. code-block:: nim + rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = + ## A convenience proc for: + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)-permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)-permissions) -include ospaths - -proc expandSymlink*(symlinkPath: string): string = - ## Returns a string representing the path to which the symbolic link points. - ## - ## On Windows this is a noop, ``symlinkPath`` is simply returned. - when defined(windows): - result = symlinkPath +when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netbsd)): + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, + newp: pointer, newplen: csize_t): cint + {.importc: "sysctl",header: """#include <sys/types.h> + #include <sys/sysctl.h>""".} + const + CTL_KERN = 1 + KERN_PROC = 14 + MAX_PATH = 1024 + + when defined(freebsd): + const KERN_PROC_PATHNAME = 12 + elif defined(netbsd): + const KERN_PROC_ARGS = 48 + const KERN_PROC_PATHNAME = 5 else: - result = newString(256) - var len = readlink(symlinkPath, result, 256) - if len < 0: - raiseOSError(osLastError()) - if len > 256: - result = newString(len+1) - len = readlink(symlinkPath, result, len) - setLen(result, len) + const KERN_PROC_PATHNAME = 9 -when defined(nimdoc): - # Common forward declaration docstring block for parameter retrieval procs. - proc paramCount*(): int {.tags: [ReadIOEffect].} = - ## Returns the number of `command line arguments`:idx: given to the - ## application. - ## - ## If your binary was called without parameters this will return zero. You - ## can later query each individual paramater with `paramStr() <#paramStr>`_ - ## or retrieve all of them in one go with `commandLineParams() - ## <#commandLineParams>`_. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() <system.html#declared>`_. - ## Example: - ## - ## .. code-block:: nim - ## when declared(paramCount): - ## # Use paramCount() here - ## else: - ## # Do something else! - - proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} = - ## Returns the `i`-th `command line argument`:idx: given to the application. - ## - ## `i` should be in the range `1..paramCount()`, the `EInvalidIndex` - ## exception will be raised for invalid values. Instead of iterating over - ## `paramCount() <#paramCount>`_ with this proc you can call the - ## convenience `commandLineParams() <#commandLineParams>`_. - ## - ## It is possible to call ``paramStr(0)`` but this will return OS specific - ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename() <#getAppFilename>`_ instead. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() <system.html#declared>`_. - ## Example: - ## - ## .. code-block:: nim - ## when declared(paramStr): - ## # Use paramStr() here - ## else: - ## # Do something else! + proc getApplFreebsd(): string = + var pathLength = csize_t(0) -elif defined(windows): - # Since we support GUI applications with Nim, we sometimes generate - # a WinMain entry proc. But a WinMain proc has no access to the parsed - # command line arguments. The way to get them differs. Thus we parse them - # ourselves. This has the additional benefit that the program's behaviour - # is always the same -- independent of the used C compiler. - var - ownArgv {.threadvar.}: seq[string] + when defined(netbsd): + var req = [CTL_KERN.cint, KERN_PROC_ARGS.cint, -1.cint, KERN_PROC_PATHNAME.cint] + else: + var req = [CTL_KERN.cint, KERN_PROC.cint, KERN_PROC_PATHNAME.cint, -1.cint] - proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if isNil(ownArgv): ownArgv = parseCmdLine($getCommandLine()) - result = ownArgv.len-1 + # first call to get the required length + var res = sysctl(addr req[0], 4, nil, addr pathLength, nil, 0) - proc paramStr*(i: int): TaintedString {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if isNil(ownArgv): ownArgv = parseCmdLine($getCommandLine()) - return TaintedString(ownArgv[i]) + if res < 0: + return "" -elif not defined(createNimRtl): - # On Posix, there is no portable way to get the command line from a DLL. - var - cmdCount {.importc: "cmdCount".}: cint - cmdLine {.importc: "cmdLine".}: cstringArray + result.setLen(pathLength) + res = sysctl(addr req[0], 4, addr result[0], addr pathLength, nil, 0) - proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if i < cmdCount and i >= 0: return TaintedString($cmdLine[i]) - raise newException(IndexError, "invalid index") + if res < 0: + return "" - proc paramCount*(): int {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - result = cmdCount-1 + let realLen = len(cstring(result)) + setLen(result, realLen) -when declared(paramCount) or defined(nimdoc): - proc commandLineParams*(): seq[TaintedString] = - ## Convenience proc which returns the command line parameters. - ## - ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename() <#getAppFilename>`_. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() <system.html#declared>`_. - ## Example: - ## - ## .. code-block:: nim - ## when declared(commandLineParams): - ## # Use commandLineParams() here - ## else: - ## # Do something else! - result = @[] - for i in 1..paramCount(): - result.add(paramStr(i)) - -when defined(linux) or defined(solaris) or defined(bsd) or defined(aix): +when not weirdTarget and (defined(linux) or defined(solaris) or defined(bsd) or defined(aix)): proc getApplAux(procPath: string): string = - result = newString(256) - var len = readlink(procPath, result, 256) - if len > 256: + result = newString(maxSymlinkLen) + var len = readlink(procPath, result.cstring, maxSymlinkLen) + if len > maxSymlinkLen: result = newString(len+1) - len = readlink(procPath, result, len) + len = readlink(procPath, result.cstring, len) setLen(result, len) -when not (defined(windows) or defined(macosx)): +when not weirdTarget and defined(openbsd): + proc getApplOpenBsd(): string = + # similar to getApplHeuristic, but checks current working directory + when declared(paramStr): + result = "" + + # POSIX guaranties that this contains the executable + # as it has been executed by the calling process + let exePath = paramStr(0) + + if len(exePath) == 0: + return "" + + if exePath[0] == DirSep: + # path is absolute + result = exePath + else: + # not an absolute path, check if it's relative to the current working directory + for i in 1..<len(exePath): + if exePath[i] == DirSep: + result = joinPath(getCurrentDir(), exePath) + break + + if len(result) > 0: + return expandFilename(result) + + # search in path + for p in split(getEnv("PATH"), {PathSep}): + var x = joinPath(p, exePath) + if fileExists(x): + return expandFilename(x) + else: + result = "" + +when not (defined(windows) or defined(macosx) or weirdTarget): proc getApplHeuristic(): string = when declared(paramStr): - result = string(paramStr(0)) + result = paramStr(0) # POSIX guaranties that this contains the executable # as it has been executed by the calling process if len(result) > 0 and result[0] != DirSep: # not an absolute path? # iterate over any path in the $PATH environment variable - for p in split(string(getEnv("PATH")), {PathSep}): + for p in split(getEnv("PATH"), {PathSep}): var x = joinPath(p, result) - if existsFile(x): return x + if fileExists(x): return x else: result = "" @@ -1346,94 +606,120 @@ when defined(macosx): proc getExecPath2(c: cstring, size: var cuint32): bool {. importc: "_NSGetExecutablePath", header: "<mach-o/dyld.h>".} -proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = +when defined(haiku): + const + PATH_MAX = 1024 + B_FIND_PATH_IMAGE_PATH = 1000 + + proc find_path(codePointer: pointer, baseDirectory: cint, subPath: cstring, + pathBuffer: cstring, bufferSize: csize_t): int32 + {.importc, header: "<FindDirectory.h>".} + + proc getApplHaiku(): string = + result = newString(PATH_MAX) + + if find_path(nil, B_FIND_PATH_IMAGE_PATH, nil, result, PATH_MAX) == 0: + let realLen = len(cstring(result)) + setLen(result, realLen) + else: + result = "" + +proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget, raises: [].} = ## Returns the filename of the application's executable. + ## This proc will resolve symlinks. ## - ## This procedure will resolve symlinks. + ## Returns empty string when name is unavailable ## - ## **Note**: This does not work reliably on BSD. + ## See also: + ## * `getAppDir proc`_ + ## * `getCurrentCompilerExe proc`_ # Linux: /proc/<pid>/exe # Solaris: # /proc/<pid>/object/a.out (filename only) # /proc/<pid>/path/a.out (complete pathname) - # *BSD (and maybe Darwin too): - # /proc/<pid>/file when defined(windows): - when useWinUnicode: - var buf = newWideCString("", 256) - var len = getModuleFileNameW(0, buf, 256) - result = buf$len - else: - result = newString(256) - var len = getModuleFileNameA(0, result, 256) - setlen(result, int(len)) - elif defined(linux) or defined(aix): - result = getApplAux("/proc/self/exe") - if result.len == 0: result = getApplHeuristic() - elif defined(solaris): - result = getApplAux("/proc/" & $getpid() & "/path/a.out") - if result.len == 0: result = getApplHeuristic() - elif defined(freebsd): - result = getApplAux("/proc/" & $getpid() & "/file") - if result.len == 0: result = getApplHeuristic() + var bufsize = int32(MAX_PATH) + var buf = newWideCString(bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString(L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): - var size: cuint32 + var size = cuint32(0) getExecPath1(nil, size) result = newString(int(size)) - if getExecPath2(result, size): + if getExecPath2(result.cstring, size): result = "" # error! if result.len > 0: - result = result.expandFilename + try: + result = result.expandFilename + except OSError: + result = "" else: - # little heuristic that may work on other POSIX-like systems: - result = string(getEnv("_")) - if result.len == 0: result = getApplHeuristic() - -proc getApplicationFilename*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Returns the filename of the application's executable. - ## **Deprecated since version 0.8.12**: use ``getAppFilename`` - ## instead. - result = getAppFilename() + when defined(linux) or defined(aix): + result = getApplAux("/proc/self/exe") + elif defined(solaris): + result = getApplAux("/proc/" & $getpid() & "/path/a.out") + elif defined(genode): + result = "" # Not supported + elif defined(freebsd) or defined(dragonfly) or defined(netbsd): + result = getApplFreebsd() + elif defined(haiku): + result = getApplHaiku() + elif defined(openbsd): + result = try: getApplOpenBsd() except OSError: "" + elif defined(nintendoswitch): + result = "" -proc getApplicationDir*(): string {.rtl, extern: "nos$1", deprecated.} = - ## Returns the directory of the application's executable. - ## **Deprecated since version 0.8.12**: use ``getAppDir`` - ## instead. - result = splitFile(getAppFilename()).dir + # little heuristic that may work on other POSIX-like systems: + if result.len == 0: + result = try: getApplHeuristic() except OSError: "" -proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = +proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = ## Returns the directory of the application's executable. - ## **Note**: This does not work reliably on BSD. + ## + ## See also: + ## * `getAppFilename proc`_ result = splitFile(getAppFilename()).dir -proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect].} = - ## sleeps `milsecs` milliseconds. +proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = + ## Sleeps `milsecs` milliseconds. + ## A negative `milsecs` causes sleep to return immediately. when defined(windows): + if milsecs < 0: + return # fixes #23732 winlean.sleep(int32(milsecs)) else: var a, b: Timespec - a.tv_sec = Time(milsecs div 1000) + a.tv_sec = posix.Time(milsecs div 1000) a.tv_nsec = (milsecs mod 1000) * 1000 * 1000 discard posix.nanosleep(a, b) proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - ## returns the file size of `file`. Can raise ``OSError``. + tags: [ReadIOEffect], noWeirdTarget.} = + ## Returns the file size of `file` (in bytes). ``OSError`` is + ## raised in case of an error. when defined(windows): var a: WIN32_FIND_DATA var resA = findFirstFile(file, a) - if resA == -1: raiseOSError(osLastError()) + if resA == -1: raiseOSError(osLastError(), file) result = rdFileSize(a) findClose(resA) else: - var f: File - if open(f, file): - result = getFileSize(f) - close(f) - else: raiseOSError(osLastError()) + var rawInfo: Stat + if stat(file, rawInfo) < 0'i32: + raiseOSError(osLastError(), file) + rawInfo.st_size -when defined(Windows): +when defined(windows) or weirdTarget: type DeviceId* = int32 FileId* = int64 @@ -1445,57 +731,74 @@ else: type FileInfo* = object ## Contains information associated with a file object. - id*: tuple[device: DeviceId, file: FileId] # Device and file id. - kind*: PathComponent # Kind of file object - directory, symlink, etc. - size*: BiggestInt # Size of file. - permissions*: set[FilePermission] # File permissions - linkCount*: BiggestInt # Number of hard links the file object has. - lastAccessTime*: Time # Time file was last accessed. - lastWriteTime*: Time # Time file was last modified/written to. - creationTime*: Time # Time file was created. Not supported on all systems! - -template rawToFormalFileInfo(rawInfo, formalInfo): expr = + ## + ## See also: + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ + id*: tuple[device: DeviceId, file: FileId] ## Device and file id. + kind*: PathComponent ## Kind of file object - directory, symlink, etc. + size*: BiggestInt ## Size of file. + permissions*: set[FilePermission] ## File permissions + linkCount*: BiggestInt ## Number of hard links the file object has. + lastAccessTime*: times.Time ## Time file was last accessed. + lastWriteTime*: times.Time ## Time file was last modified/written to. + creationTime*: times.Time ## Time file was created. Not supported on all systems! + blockSize*: int ## Preferred I/O block size for this object. + ## In some filesystems, this may vary from file to file. + isSpecial*: bool ## Is file special? (on Unix some "files" + ## can be special=non-regular like FIFOs, + ## devices); for directories `isSpecial` + ## is always `false`, for symlinks it is + ## the same as for the link's target. + +template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. - ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, + ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix - when defined(Windows): - template toTime(e): expr = winTimeToUnixTime(rdFileTime(e)) - template merge(a, b): expr = a or (b shl 32) + when defined(windows): + template merge[T](a, b): untyped = + cast[T]( + (uint64(cast[uint32](a))) or + (uint64(cast[uint32](b)) shl 32) + ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber - formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) - formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) + formalInfo.id.file = merge[FileId](rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) + formalInfo.size = merge[BiggestInt](rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks - formalInfo.lastAccessTime = toTime(rawInfo.ftLastAccessTime) - formalInfo.lastWriteTime = toTime(rawInfo.ftLastWriteTime) - formalInfo.creationTime = toTime(rawInfo.ftCreationTime) + formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) + formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) + formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) + formalInfo.blockSize = 8192 # xxx use Windows API instead of hardcoding # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: formalInfo.permissions = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, fpOthersExec, fpOthersRead} else: - result.permissions = {fpUserExec..fpOthersRead} + formalInfo.permissions = {fpUserExec..fpOthersRead} # Retrieve basic file kind - result.kind = pcFile if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: formalInfo.kind = pcDir + else: + formalInfo.kind = pcFile if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - formalInfo.kind = succ(result.kind) - + formalInfo.kind = succ(formalInfo.kind) else: - template checkAndIncludeMode(rawMode, formalMode: expr) = - if (rawInfo.st_mode and rawMode) != 0'i32: + template checkAndIncludeMode(rawMode, formalMode: untyped) = + if (rawInfo.st_mode and rawMode.Mode) != 0.Mode: formalInfo.permissions.incl(formalMode) formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) formalInfo.size = rawInfo.st_size - formalInfo.linkCount = rawInfo.st_Nlink - formalInfo.lastAccessTime = rawInfo.st_atime - formalInfo.lastWriteTime = rawInfo.st_mtime - formalInfo.creationTime = rawInfo.st_ctime + formalInfo.linkCount = rawInfo.st_nlink.BiggestInt + formalInfo.lastAccessTime = rawInfo.st_atim.toTime + formalInfo.lastWriteTime = rawInfo.st_mtim.toTime + formalInfo.creationTime = rawInfo.st_ctim.toTime + formalInfo.blockSize = rawInfo.st_blksize - result.permissions = {} + formalInfo.permissions = {} checkAndIncludeMode(S_IRUSR, fpUserRead) checkAndIncludeMode(S_IWUSR, fpUserWrite) checkAndIncludeMode(S_IXUSR, fpUserExec) @@ -1508,93 +811,222 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = checkAndIncludeMode(S_IWOTH, fpOthersWrite) checkAndIncludeMode(S_IXOTH, fpOthersExec) - formalInfo.kind = pcFile - if S_ISDIR(rawInfo.st_mode): formalInfo.kind = pcDir - if S_ISLNK(rawInfo.st_mode): formalInfo.kind.inc() + (formalInfo.kind, formalInfo.isSpecial) = + if S_ISDIR(rawInfo.st_mode): + (pcDir, false) + elif S_ISLNK(rawInfo.st_mode): + assert(path != "") # symlinks can't occur for file handles + getSymlinkFileKind(path) + else: + (pcFile, not S_ISREG(rawInfo.st_mode)) -proc getFileInfo*(handle: FileHandle): FileInfo = +when defined(js): + when not declared(FileHandle): + type FileHandle = distinct int32 + when not declared(File): + type File = object + +proc getFileInfo*(handle: FileHandle): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object represented by the given ## handle. ## ## If the information cannot be retrieved, such as when the file handle - ## is invalid, an error will be thrown. + ## is invalid, `OSError` is raised. + ## + ## See also: + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ + # Done: ID, Kind, Size, Permissions, Link Count - when defined(Windows): + when defined(windows): var rawInfo: BY_HANDLE_FILE_INFORMATION # We have to use the super special '_get_osfhandle' call (wrapped above) - # To transform the C file descripter to a native file handle. + # To transform the C file descriptor to a native file handle. var realHandle = get_osfhandle(handle) if getFileInformationByHandle(realHandle, addr rawInfo) == 0: - raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + raiseOSError(osLastError(), $handle) + rawToFormalFileInfo(rawInfo, "", result) else: var rawInfo: Stat if fstat(handle, rawInfo) < 0'i32: - raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + raiseOSError(osLastError(), $handle) + rawToFormalFileInfo(rawInfo, "", result) -proc getFileInfo*(file: File): FileInfo = +proc getFileInfo*(file: File): FileInfo {.noWeirdTarget.} = + ## Retrieves file information for the file object. + ## + ## See also: + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) -proc getFileInfo*(path: string, followSymlink = true): FileInfo = +proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo` structure will be slightly different - ## across platforms, and in some cases, incomplete or inaccurate. + ## contained by the returned `FileInfo object`_ will be slightly + ## different across platforms, and in some cases, incomplete or inaccurate. ## - ## When `followSymlink` is true, symlinks are followed and the information - ## retrieved is information related to the symlink's target. Otherwise, - ## information on the symlink itself is retrieved. + ## When `followSymlink` is true (default), symlinks are followed and the + ## information retrieved is information related to the symlink's target. + ## Otherwise, information on the symlink itself is retrieved (however, + ## field `isSpecial` is still determined from the target on Unix). ## ## If the information cannot be retrieved, such as when the path doesn't ## exist, or when permission restrictions prevent the program from retrieving - ## file information, an error will be thrown. - when defined(Windows): + ## file information, `OSError` is raised. + ## + ## See also: + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + when defined(windows): var handle = openHandle(path, followSymlink) rawInfo: BY_HANDLE_FILE_INFORMATION if handle == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) + raiseOSError(osLastError(), path) if getFileInformationByHandle(handle, addr rawInfo) == 0: - raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + raiseOSError(osLastError(), path) + rawToFormalFileInfo(rawInfo, path, result) discard closeHandle(handle) else: var rawInfo: Stat if followSymlink: if stat(path, rawInfo) < 0'i32: - raiseOSError(osLastError()) + raiseOSError(osLastError(), path) else: if lstat(path, rawInfo) < 0'i32: - raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + raiseOSError(osLastError(), path) + rawToFormalFileInfo(rawInfo, path, result) -proc isHidden*(path: string): bool = - ## Determines whether a given path is hidden or not. Returns false if the - ## file doesn't exist. The given path must be accessible from the current - ## working directory of the program. +proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadIOEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to files with identical + ## binary content. ## - ## On Windows, a file is hidden if the file's 'hidden' attribute is set. - ## On Unix-like systems, a file is hidden if it starts with a '.' (period) - ## and is not *just* '.' or '..' ' ." - when defined(Windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + ## See also: + ## * `sameFile proc`_ + var + a, b: File + if not open(a, path1): return false + if not open(b, path2): + close(a) + return false + let bufSize = getFileInfo(a).blockSize + var bufA = alloc(bufSize) + var bufB = alloc(bufSize) + while true: + var readA = readBuffer(a, bufA, bufSize) + var readB = readBuffer(b, bufB, bufSize) + if readA != readB: + result = false + break + if readA == 0: + result = true + break + result = equalMem(bufA, bufB, readA) + if not result: break + if readA != bufSize: break # end of file + dealloc(bufA) + dealloc(bufB) + close(a) + close(b) + +proc isHidden*(path: string): bool {.noWeirdTarget.} = + ## Determines whether ``path`` is hidden or not, using `this + ## reference <https://en.wikipedia.org/wiki/Hidden_file_and_hidden_directory>`_. + ## + ## On Windows: returns true if it exists and its "hidden" attribute is set. + ## + ## On posix: returns true if ``lastPathPart(path)`` starts with ``.`` and is + ## not ``.`` or ``..``. + ## + ## **Note**: paths are not normalized to determine `isHidden`. + runnableExamples: + when defined(posix): + assert ".foo".isHidden + assert not ".foo/bar".isHidden + assert not ".".isHidden + assert not "..".isHidden + assert not "".isHidden + assert ".foo/".isHidden + + when defined(windows): + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: - if fileExists(path): - let - fileName = extractFilename(path) - nameLen = len(fileName) - if nameLen == 2: - result = (fileName[0] == '.') and (fileName[1] != '.') - elif nameLen > 2: - result = (fileName[0] == '.') and (fileName[3] != '.') - -{.pop.} + let fileName = lastPathPart(path) + result = len(fileName) >= 2 and fileName[0] == '.' and fileName != ".." + +proc getCurrentProcessId*(): int {.noWeirdTarget.} = + ## Return current process ID. + ## + ## See also: + ## * `osproc.processID(p: Process) <osproc.html#processID,Process>`_ + when defined(windows): + proc GetCurrentProcessId(): DWORD {.stdcall, dynlib: "kernel32", + importc: "GetCurrentProcessId".} + result = GetCurrentProcessId().int + else: + result = getpid() + +proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = + ## Sets the `file`'s last modification time. `OSError` is raised in case of + ## an error. + when defined(posix): + let unixt = posix.Time(t.toUnix) + let micro = convert(Nanoseconds, Microseconds, t.nanosecond) + var timevals = [Timeval(tv_sec: unixt, tv_usec: micro), + Timeval(tv_sec: unixt, tv_usec: micro)] # [last access, last modification] + if utimes(file, timevals.addr) != 0: raiseOSError(osLastError(), file) + else: + let h = openHandle(path = file, writeAccess = true) + if h == INVALID_HANDLE_VALUE: raiseOSError(osLastError(), file) + var ft = t.toWinTime.toFILETIME + let res = setFileTime(h, nil, nil, ft.addr) + discard h.closeHandle + if res == 0'i32: raiseOSError(osLastError(), file) + + +func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = + ## Returns `true` if `filename` is valid for crossplatform use. + ## + ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. + ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. + ## + ## See also: + ## + ## * https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception + ## * https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + ## * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + ## + ## .. warning:: This only checks filenames, not whole paths + ## (because basically you can mount anything as a path on Linux). + runnableExamples: + assert not isValidFilename(" foo") # Leading white space + assert not isValidFilename("foo ") # Trailing white space + assert not isValidFilename("foo.") # Ends with dot + assert not isValidFilename("con.txt") # "CON" is invalid (Windows) + assert not isValidFilename("OwO:UwU") # ":" is invalid (Mac) + assert not isValidFilename("aux.bat") # "AUX" is invalid (Windows) + assert not isValidFilename("") # Empty string + assert not isValidFilename("foo/") # Filename is empty + + result = true + let f = filename.splitFile() + if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or + f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or + find(f.name, invalidFilenameChars) != -1): return false + for invalid in invalidFilenames: + if cmpIgnoreCase(f.name, invalid) == 0: return false + + +# deprecated declarations +when not weirdTarget: + template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = + fileExists(args) + template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = + dirExists(args) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim deleted file mode 100644 index e9f5bee0a..000000000 --- a/lib/pure/ospaths.nim +++ /dev/null @@ -1,577 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Included by the ``os`` module but a module in its own right for NimScript -# support. - -when isMainModule: - {.pragma: rtl.} - import strutils - -when defined(nimscript) or (defined(nimdoc) and not declared(os)): - {.pragma: rtl.} - {.push hint[ConvFromXtoItselfNotNeeded]:off.} - -when not declared(getEnv) or defined(nimscript): - type - ReadEnvEffect* = object of ReadIOEffect ## effect that denotes a read - ## from an environment variable - WriteEnvEffect* = object of WriteIOEffect ## effect that denotes a write - ## to an environment variable - - ReadDirEffect* = object of ReadIOEffect ## effect that denotes a write - ## operation to the directory - ## structure - WriteDirEffect* = object of WriteIOEffect ## effect that denotes a write - ## operation to - ## the directory structure - - OSErrorCode* = distinct int32 ## Specifies an OS Error Code. - - {.deprecated: [FReadEnv: ReadEnvEffect, FWriteEnv: WriteEnvEffect, - FReadDir: ReadDirEffect, - FWriteDir: WriteDirEffect, - TOSErrorCode: OSErrorCode - ].} - const - doslike = defined(windows) or defined(OS2) or defined(DOS) - # DOS-like filesystem - - when defined(Nimdoc): # only for proper documentation: - const - CurDir* = '.' - ## The constant string used by the operating system to refer to the - ## current directory. - ## - ## For example: '.' for POSIX or ':' for the classic Macintosh. - - ParDir* = ".." - ## The constant string used by the operating system to refer to the - ## parent directory. - ## - ## For example: ".." for POSIX or "::" for the classic Macintosh. - - DirSep* = '/' - ## The character used by the operating system to separate pathname - ## components, for example, '/' for POSIX or ':' for the classic - ## Macintosh. - - AltSep* = '/' - ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep` if only one separator - ## character exists. This is set to '/' on Windows systems - ## where `DirSep` is a backslash. - - PathSep* = ':' - ## The character conventionally used by the operating system to separate - ## search patch components (as in PATH), such as ':' for POSIX - ## or ';' for Windows. - - FileSystemCaseSensitive* = true - ## true if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths` to compare filenames properly. - - ExeExt* = "" - ## The file extension of native executables. For example: - ## "" for POSIX, "exe" on Windows. - - ScriptExt* = "" - ## The file extension of a script file. For example: "" for POSIX, - ## "bat" on Windows. - - DynlibFormat* = "lib$1.so" - ## The format string to turn a filename into a `DLL`:idx: file (also - ## called `shared object`:idx: on some operating systems). - - elif defined(macos): - const - CurDir* = ':' - ParDir* = "::" - DirSep* = ':' - AltSep* = Dirsep - PathSep* = ',' - FileSystemCaseSensitive* = false - ExeExt* = "" - ScriptExt* = "" - DynlibFormat* = "$1.dylib" - - # MacOS paths - # =========== - # MacOS directory separator is a colon ":" which is the only character not - # allowed in filenames. - # - # A path containing no colon or which begins with a colon is a partial - # path. - # E.g. ":kalle:petter" ":kalle" "kalle" - # - # All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:" - # When generating paths, one is safe if one ensures that all partial paths - # begin with a colon, and all full paths end with a colon. - # In full paths the first name (e g HD above) is the name of a mounted - # volume. - # These names are not unique, because, for instance, two diskettes with the - # same names could be inserted. This means that paths on MacOS are not - # waterproof. In case of equal names the first volume found will do. - # Two colons "::" are the relative path to the parent. Three is to the - # grandparent etc. - elif doslike: - const - CurDir* = '.' - ParDir* = ".." - DirSep* = '\\' # seperator within paths - AltSep* = '/' - PathSep* = ';' # seperator between paths - FileSystemCaseSensitive* = false - ExeExt* = "exe" - ScriptExt* = "bat" - DynlibFormat* = "$1.dll" - elif defined(PalmOS) or defined(MorphOS): - const - DirSep* = '/' - AltSep* = Dirsep - PathSep* = ';' - ParDir* = ".." - FileSystemCaseSensitive* = false - ExeExt* = "" - ScriptExt* = "" - DynlibFormat* = "$1.prc" - elif defined(RISCOS): - const - DirSep* = '.' - AltSep* = '.' - ParDir* = ".." # is this correct? - PathSep* = ',' - FileSystemCaseSensitive* = true - ExeExt* = "" - ScriptExt* = "" - DynlibFormat* = "lib$1.so" - else: # UNIX-like operating system - const - CurDir* = '.' - ParDir* = ".." - DirSep* = '/' - AltSep* = DirSep - PathSep* = ':' - FileSystemCaseSensitive* = true - ExeExt* = "" - ScriptExt* = "" - DynlibFormat* = when defined(macosx): "lib$1.dylib" else: "lib$1.so" - - const - ExtSep* = '.' - ## The character which separates the base filename from the extension; - ## for example, the '.' in ``os.nim``. - - - proc joinPath*(head, tail: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Joins two directory names to one. - ## - ## For example on Unix: - ## - ## .. code-block:: nim - ## joinPath("usr", "lib") - ## - ## results in: - ## - ## .. code-block:: nim - ## "usr/lib" - ## - ## If head is the empty string, tail is returned. If tail is the empty - ## string, head is returned with a trailing path separator. If tail starts - ## with a path separator it will be removed when concatenated to head. Other - ## path separators not located on boundaries won't be modified. More - ## examples on Unix: - ## - ## .. code-block:: nim - ## assert joinPath("usr", "") == "usr/" - ## assert joinPath("", "lib") == "lib" - ## assert joinPath("", "/lib") == "/lib" - ## assert joinPath("usr/", "/lib") == "usr/lib" - if len(head) == 0: - result = tail - elif head[len(head)-1] in {DirSep, AltSep}: - if tail[0] in {DirSep, AltSep}: - result = head & substr(tail, 1) - else: - result = head & tail - else: - if tail[0] in {DirSep, AltSep}: - result = head & tail - else: - result = head & DirSep & tail - - proc joinPath*(parts: varargs[string]): string {.noSideEffect, - rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail)`, but works with any number of - ## directory parts. You need to pass at least one element or the proc - ## will assert in debug builds and crash on release builds. - result = parts[0] - for i in 1..high(parts): - result = joinPath(result, parts[i]) - - proc `/` * (head, tail: string): string {.noSideEffect.} = - ## The same as ``joinPath(head, tail)`` - ## - ## Here are some examples for Unix: - ## - ## .. code-block:: nim - ## assert "usr" / "" == "usr/" - ## assert "" / "lib" == "lib" - ## assert "" / "/lib" == "/lib" - ## assert "usr/" / "/lib" == "usr/lib" - return joinPath(head, tail) - - proc splitPath*(path: string): tuple[head, tail: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a directory into (head, tail), so that - ## ``head / tail == path`` (except for edge cases like "/usr"). - ## - ## Examples: - ## - ## .. code-block:: nim - ## splitPath("usr/local/bin") -> ("usr/local", "bin") - ## splitPath("usr/local/bin/") -> ("usr/local/bin", "") - ## splitPath("bin") -> ("", "bin") - ## splitPath("/bin") -> ("", "bin") - ## splitPath("") -> ("", "") - var sepPos = -1 - for i in countdown(len(path)-1, 0): - if path[i] in {DirSep, AltSep}: - sepPos = i - break - if sepPos >= 0: - result.head = substr(path, 0, sepPos-1) - result.tail = substr(path, sepPos+1) - else: - result.head = "" - result.tail = path - - proc parentDirPos(path: string): int = - var q = 1 - if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 - for i in countdown(len(path)-q, 0): - if path[i] in {DirSep, AltSep}: return i - result = -1 - - proc parentDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the parent directory of `path`. - ## - ## This is often the same as the ``head`` result of ``splitPath``. - ## If there is no parent, "" is returned. - ## | Example: ``parentDir("/usr/local/bin") == "/usr/local"``. - ## | Example: ``parentDir("/usr/local/bin/") == "/usr/local"``. - let sepPos = parentDirPos(path) - if sepPos >= 0: - result = substr(path, 0, sepPos-1) - else: - result = "" - - proc tailDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the tail part of `path`.. - ## - ## | Example: ``tailDir("/usr/local/bin") == "local/bin"``. - ## | Example: ``tailDir("usr/local/bin/") == "local/bin"``. - ## | Example: ``tailDir("bin") == ""``. - var q = 1 - if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 - for i in 0..len(path)-q: - if path[i] in {DirSep, AltSep}: - return substr(path, i+1) - result = "" - - proc isRootDir*(path: string): bool {. - noSideEffect, rtl, extern: "nos$1".} = - ## Checks whether a given `path` is a root directory - result = parentDirPos(path) < 0 - - iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = - ## Walks over all parent directories of a given `path` - ## - ## If `fromRoot` is set, the traversal will start from the file system root - ## diretory. If `inclusive` is set, the original argument will be included - ## in the traversal. - ## - ## Relative paths won't be expanded by this proc. Instead, it will traverse - ## only the directories appearing in the relative path. - if not fromRoot: - var current = path - if inclusive: yield path - while true: - if current.isRootDir: break - current = current.parentDir - yield current - else: - for i in countup(0, path.len - 2): # ignore the last / - # deal with non-normalized paths such as /foo//bar//baz - if path[i] in {DirSep, AltSep} and - (i == 0 or path[i-1] notin {DirSep, AltSep}): - yield path.substr(0, i) - - if inclusive: yield path - - proc `/../` * (head, tail: string): string {.noSideEffect.} = - ## The same as ``parentDir(head) / tail`` unless there is no parent - ## directory. Then ``head / tail`` is performed instead. - let sepPos = parentDirPos(head) - if sepPos >= 0: - result = substr(head, 0, sepPos-1) / tail - else: - result = head / tail - - proc normExt(ext: string): string = - if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here - else: result = ExtSep & ext - - proc searchExtPos(s: string): int = - # BUGFIX: do not search until 0! .DS_Store is no file extension! - result = -1 - for i in countdown(len(s)-1, 1): - if s[i] == ExtSep: - result = i - break - elif s[i] in {DirSep, AltSep}: - break # do not skip over path - - proc splitFile*(path: string): tuple[dir, name, ext: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into (dir, filename, extension). - ## `dir` does not end in `DirSep`. - ## `extension` includes the leading dot. - ## - ## Example: - ## - ## .. code-block:: nim - ## var (dir, name, ext) = splitFile("usr/local/nimc.html") - ## assert dir == "usr/local" - ## assert name == "nimc" - ## assert ext == ".html" - ## - ## If `path` has no extension, `ext` is the empty string. - ## If `path` has no directory component, `dir` is the empty string. - ## If `path` has no filename component, `name` and `ext` are empty strings. - if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: - result = (path, "", "") - else: - var sepPos = -1 - var dotPos = path.len - for i in countdown(len(path)-1, 0): - if path[i] == ExtSep: - if dotPos == path.len and i > 0 and - path[i-1] notin {DirSep, AltSep}: dotPos = i - elif path[i] in {DirSep, AltSep}: - sepPos = i - break - result.dir = substr(path, 0, sepPos-1) - result.name = substr(path, sepPos+1, dotPos-1) - result.ext = substr(path, dotPos) - - proc extractFilename*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Extracts the filename of a given `path`. This is the same as - ## ``name & ext`` from ``splitFile(path)``. - if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: - result = "" - else: - result = splitPath(path).tail - - - proc changeFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Changes the file extension to `ext`. - ## - ## If the `filename` has no extension, `ext` will be added. - ## If `ext` == "" then any extension is removed. - ## `Ext` should be given without the leading '.', because some - ## filesystems may use a different character. (Although I know - ## of none such beast.) - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = substr(filename, 0, extPos-1) & normExt(ext) - - proc addFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Adds the file extension `ext` to `filename`, unless - ## `filename` already has an extension. - ## - ## `Ext` should be given without the leading '.', because some - ## filesystems may use a different character. - ## (Although I know of none such beast.) - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = filename - - proc cmpPaths*(pathA, pathB: string): int {. - noSideEffect, rtl, extern: "nos$1".} = - ## Compares two paths. - ## - ## On a case-sensitive filesystem this is done - ## case-sensitively otherwise case-insensitively. Returns: - ## - ## | 0 iff pathA == pathB - ## | < 0 iff pathA < pathB - ## | > 0 iff pathA > pathB - if FileSystemCaseSensitive: - result = cmp(pathA, pathB) - else: - when defined(nimscript): - result = cmpic(pathA, pathB) - elif defined(nimdoc): discard - else: - result = cmpIgnoreCase(pathA, pathB) - - proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} = - ## Checks whether a given `path` is absolute. - ## - ## On Windows, network paths are considered absolute too. - when doslike: - var len = len(path) - result = (len > 1 and path[0] in {'/', '\\'}) or - (len > 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') - elif defined(macos): - result = path.len > 0 and path[0] != ':' - elif defined(RISCOS): - result = path[0] == '$' - elif defined(posix): - result = path[0] == '/' - - proc unixToNativePath*(path: string, drive=""): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Converts an UNIX-like path to a native one. - ## - ## On an UNIX system this does nothing. Else it converts - ## '/', '.', '..' to the appropriate things. - ## - ## On systems with a concept of "drives", `drive` is used to determine - ## which drive label to use during absolute path conversion. - ## `drive` defaults to the drive of the current working directory, and is - ## ignored on systems that do not have a concept of "drives". - - when defined(unix): - result = path - else: - var start: int - if path[0] == '/': - # an absolute path - when doslike: - if drive != "": - result = drive & ":" & DirSep - else: - result = $DirSep - elif defined(macos): - result = "" # must not start with ':' - else: - result = $DirSep - start = 1 - elif path[0] == '.' and path[1] == '/': - # current directory - result = $CurDir - start = 2 - else: - result = "" - start = 0 - - var i = start - while i < len(path): # ../../../ --> :::: - if path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': - # parent directory - when defined(macos): - if result[high(result)] == ':': - add result, ':' - else: - add result, ParDir - else: - add result, ParDir & DirSep - inc(i, 3) - elif path[i] == '/': - add result, DirSep - inc(i) - else: - add result, path[i] - inc(i) - -when defined(nimdoc) and not declared(os): - proc getEnv(x: string): string = discard - proc existsFile(x: string): bool = discard - -when declared(getEnv) or defined(nimscript): - proc getHomeDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the home directory of the current user. - ## - ## This proc is wrapped by the expandTilde proc for the convenience of - ## processing paths coming from user configuration files. - when defined(windows): return string(getEnv("USERPROFILE")) & "\\" - else: return string(getEnv("HOME")) & "/" - - proc getConfigDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the config directory of the current user for applications. - when defined(windows): return string(getEnv("APPDATA")) & "\\" - else: return string(getEnv("HOME")) & "/.config/" - - proc getTempDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the temporary directory of the current user for applications to - ## save temporary files in. - when defined(windows): return string(getEnv("TEMP")) & "\\" - else: return "/tmp/" - - proc expandTilde*(path: string): string {. - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Expands a path starting with ``~/`` to a full path. - ## - ## If `path` starts with the tilde character and is followed by `/` or `\\` - ## this proc will return the reminder of the path appended to the result of - ## the getHomeDir() proc, otherwise the input path will be returned without - ## modification. - ## - ## The behaviour of this proc is the same on the Windows platform despite - ## not having this convention. Example: - ## - ## .. code-block:: nim - ## let configFile = expandTilde("~" / "appname.cfg") - ## echo configFile - ## # --> C:\Users\amber\appname.cfg - if len(path) > 1 and path[0] == '~' and (path[1] == '/' or path[1] == '\\'): - result = getHomeDir() / path.substr(2) - else: - result = path - - when not declared(split): - iterator split(s: string, sep: char): string = - var last = 0 - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and s[last] != sep: inc(last) - yield substr(s, first, last-1) - inc(last) - - proc findExe*(exe: string): string {. - tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} = - ## Searches for `exe` in the current working directory and then - ## in directories listed in the ``PATH`` environment variable. - ## Returns "" if the `exe` cannot be found. On DOS-like platforms, `exe` - ## is added the `ExeExt <#ExeExt>`_ file extension if it has none. - result = addFileExt(exe, ExeExt) - if existsFile(result): return - var path = string(getEnv("PATH")) - for candidate in split(path, PathSep): - when defined(windows): - var x = candidate / result - else: - var x = expandTilde(candidate) / result - if existsFile(x): return x - result = "" - -when defined(nimscript) or (defined(nimdoc) and not declared(os)): - {.pop.} # hint[ConvFromXtoItselfNotNeeded]:off diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 7a1e14a57..c304ecca6 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -9,153 +9,131 @@ ## This module implements an advanced facility for executing OS processes ## and process communication. +## +## **See also:** +## * `os module <os.html>`_ +## * `streams module <streams.html>`_ +## * `memfiles module <memfiles.html>`_ include "system/inclrtl" import - strutils, os, strtabs, streams, cpuinfo + std/[strutils, os, strtabs, streams, cpuinfo, streamwrapper, + private/since] + +export quoteShell, quoteShellWindows, quoteShellPosix when defined(windows): - import winlean + import std/winlean else: - import posix + import std/posix + +when defined(linux) and defined(useClone): + import std/linux + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + when defined(windows): + import std/widestrs -when defined(linux): - import linux type - ProcessOption* = enum ## options that can be passed `startProcess` - poEchoCmd, ## echo the command before execution - poUsePath, ## Asks system to search for executable using PATH environment - ## variable. - ## On Windows, this is the default. - poEvalCommand, ## Pass `command` directly to the shell, without quoting. - ## Use it only if `command` comes from trusted source. - poStdErrToStdOut, ## merge stdout and stderr to the stdout stream - poParentStreams, ## use the parent's streams - poInteractive, ## optimize the buffer handling for responsiveness for - ## UI applications. Currently this only affects - ## Windows: Named pipes are used so that you can peek - ## at the process' output streams. - poDemon ## Windows: The program creates no Window. + ProcessOption* = enum ## Options that can be passed to `startProcess proc + ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. + poEchoCmd, ## Echo the command before execution. + poUsePath, ## Asks system to search for executable using PATH environment + ## variable. + ## On Windows, this is the default. + poEvalCommand, ## Pass `command` directly to the shell, without quoting. + ## Use it only if `command` comes from trusted source. + poStdErrToStdOut, ## Merge stdout and stderr to the stdout stream. + poParentStreams, ## Use the parent's streams. + poInteractive, ## Optimize the buffer handling for responsiveness for + ## UI applications. Currently this only affects + ## Windows: Named pipes are used so that you can peek + ## at the process' output streams. + poDaemon ## Windows: The program creates no Window. + ## Unix: Start the program as a daemon. This is still + ## work in progress! ProcessObj = object of RootObj when defined(windows): fProcessHandle: Handle + fThreadHandle: Handle inHandle, outHandle, errHandle: FileHandle id: Handle else: inHandle, outHandle, errHandle: FileHandle - inStream, outStream, errStream: Stream id: Pid - exitCode: cint + inStream, outStream, errStream: owned(Stream) + exitStatus: cint + exitFlag: bool options: set[ProcessOption] - Process* = ref ProcessObj ## represents an operating system process - - -{.deprecated: [TProcess: ProcessObj, PProcess: Process, - TProcessOption: ProcessOption].} + Process* = ref ProcessObj ## Represents an operating system process. -const poUseShell* {.deprecated.} = poUsePath - ## Deprecated alias for poUsePath. -proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to Windows API. - ## Based on Python's subprocess.list2cmdline - ## See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - let needQuote = {' ', '\t'} in s or s.len == 0 - - result = "" - var backslashBuff = "" - if needQuote: - result.add("\"") - - for c in s: - if c == '\\': - backslashBuff.add(c) - elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) - backslashBuff.setLen(0) - result.add("\\\"") - else: - if backslashBuff.len != 0: - result.add(backslashBuff) - backslashBuff.setLen(0) - result.add(c) - - if needQuote: - result.add("\"") - -proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to POSIX shell. - ## Based on Python's pipes.quote - const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', - '0'..'9', 'A'..'Z', 'a'..'z'} - if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s - else: - return "'" & s.replace("'", "'\"'\"'") & "'" - -proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to shell. - when defined(Windows): - return quoteShellWindows(s) - elif defined(posix): - return quoteShellPosix(s) - else: - {.error:"quoteShell is not supported on your system".} - -proc execProcess*(command: string, - args: openArray[string] = [], - env: StringTableRef = nil, - options: set[ProcessOption] = {poStdErrToStdOut, - poUsePath, - poEvalCommand}): TaintedString {. - rtl, extern: "nosp$1", - tags: [ExecIOEffect, ReadIOEffect].} +proc execProcess*(command: string, workingDir: string = "", + args: openArray[string] = [], env: StringTableRef = nil, + options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], + tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. - ## WARNING: this function uses poEvalCommand by default for backward compatibility. - ## Make sure to pass options explicitly. ## - ## .. code-block:: Nim + ## .. warning:: This function uses `poEvalCommand` by default for backwards + ## compatibility. Make sure to pass options explicitly. + ## + ## See also: + ## * `startProcess proc + ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ + ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process)>`_ + ## * `execCmd proc <#execCmd,string>`_ + ## + ## Example: + ## ```Nim + ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) + ## let outp_shell = execProcess("nim c -r mytestfile.nim") + ## # Note: outp may have an interleave of text from the nim compile + ## # and any output from mytestfile when it runs + ## ``` + +proc execCmd*(command: string): int {.rtl, extern: "nosp$1", + tags: [ExecIOEffect, ReadIOEffect, RootEffect].} + ## Executes ``command`` and returns its error code. ## - ## let outp = execProcess("nim c -r mytestfile.nim") - ## # Note: outp may have an interleave of text from the nim compile - ## # and any output from mytestfile when it runs - -proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect].} - ## Executes ``command`` and returns its error code. Standard input, output, - ## error streams are inherited from the calling process. This operation - ## is also often called `system`:idx:. + ## Standard input, output, error streams are inherited from the calling process. + ## This operation is also often called `system`:idx:. ## - ## .. code-block:: Nim + ## See also: + ## * `execCmdEx proc <#execCmdEx,string,set[ProcessOption],StringTableRef,string,string>`_ + ## * `startProcess proc + ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ + ## * `execProcess proc + ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## - ## let errC = execCmd("nim c -r mytestfile.nim") - -proc startProcess*(command: string, - workingDir: string = "", - args: openArray[string] = [], - env: StringTableRef = nil, - options: set[ProcessOption] = {poStdErrToStdOut}): - Process {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadEnvEffect].} + ## Example: + ## ```Nim + ## let errC = execCmd("nim c -r mytestfile.nim") + ## ``` + +proc startProcess*(command: string, workingDir: string = "", + args: openArray[string] = [], env: StringTableRef = nil, + options: set[ProcessOption] = {poStdErrToStdOut}): + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], + tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory - ## is used. `args` are the command line arguments that are passed to the + ## is used (default). `args` are the command line arguments that are passed to the ## process. On many operating systems, the first command line argument is the - ## name of the executable. `args` should not contain this argument! + ## name of the executable. `args` should *not* contain this argument! ## `env` is the environment that will be passed to the process. - ## If ``env == nil`` the environment is inherited of + ## If ``env == nil`` (default) the environment is inherited of ## the parent process. `options` are additional flags that may be passed - ## to `startProcess`. See the documentation of ``ProcessOption`` for the - ## meaning of these flags. You need to `close` the process when done. + ## to `startProcess`. See the documentation of `ProcessOption<#ProcessOption>`_ + ## for the meaning of these flags. + ## + ## You need to `close <#close,Process>`_ the process when done. ## ## Note that you can't pass any `args` if you use the option ## ``poEvalCommand``, which invokes the system shell to run the specified @@ -166,207 +144,418 @@ proc startProcess*(command: string, ## invocation if possible as it leads to non portable software. ## ## Return value: The newly created process object. Nil is never returned, - ## but ``EOS`` is raised in case of an error. - -proc startCmd*(command: string, options: set[ProcessOption] = { - poStdErrToStdOut, poUsePath}): Process {. - tags: [ExecIOEffect, ReadEnvEffect], deprecated.} = - ## Deprecated - use `startProcess` directly. - result = startProcess(command=command, options=options + {poEvalCommand}) - -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [].} - ## When the process has finished executing, cleanup related handles + ## but ``OSError`` is raised in case of an error. + ## + ## See also: + ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process)>`_ + ## * `execProcess proc + ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ + ## * `execCmd proc <#execCmd,string>`_ + +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} + ## When the process has finished executing, cleanup related handles. + ## + ## .. warning:: If the process has not finished executing, this will forcibly + ## terminate the process. Doing so may result in zombie processes and + ## `pty leaks <http://stackoverflow.com/questions/27021641/how-to-fix-request-failed-on-channel-0>`_. proc suspend*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Suspends the process `p`. + ## + ## See also: + ## * `resume proc <#resume,Process>`_ + ## * `terminate proc <#terminate,Process>`_ + ## * `kill proc <#kill,Process>`_ + proc resume*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Resumes the process `p`. + ## + ## See also: + ## * `suspend proc <#suspend,Process>`_ + ## * `terminate proc <#terminate,Process>`_ + ## * `kill proc <#kill,Process>`_ proc terminate*(p: Process) {.rtl, extern: "nosp$1", tags: [].} - ## Stop the process `p`. On Posix OSes the procedure sends ``SIGTERM`` - ## to the process. On Windows the Win32 API function ``TerminateProcess()`` + ## Stop the process `p`. + ## + ## On Posix OSes the procedure sends ``SIGTERM`` to the process. + ## On Windows the Win32 API function ``TerminateProcess()`` ## is called to stop the process. + ## + ## See also: + ## * `suspend proc <#suspend,Process>`_ + ## * `resume proc <#resume,Process>`_ + ## * `kill proc <#kill,Process>`_ + ## * `posix_utils.sendSignal(pid: Pid, signal: int) <posix_utils.html#sendSignal,Pid,int>`_ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} - ## Kill the process `p`. On Posix OSes the procedure sends ``SIGKILL`` to - ## the process. On Windows ``kill()`` is simply an alias for ``terminate()``. + ## Kill the process `p`. + ## + ## On Posix OSes the procedure sends ``SIGKILL`` to the process. + ## On Windows ``kill`` is simply an alias for `terminate() <#terminate,Process>`_. + ## + ## See also: + ## * `suspend proc <#suspend,Process>`_ + ## * `resume proc <#resume,Process>`_ + ## * `terminate proc <#terminate,Process>`_ + ## * `posix_utils.sendSignal(pid: Pid, signal: int) <posix_utils.html#sendSignal,Pid,int>`_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} - ## Returns true iff the process `p` is still running. Returns immediately. +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} + ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = - ## returns `p`'s process ID. + ## Returns `p`'s process ID. + ## + ## See also: + ## * `os.getCurrentProcessId proc <os.html#getCurrentProcessId>`_ return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} - ## waits for the process to finish and returns `p`'s error code. + extern: "nosp$1", raises: [OSError, ValueError], tags: [TimeEffect].} + ## Waits for the process to finish and returns `p`'s error code. + ## + ## .. warning:: Be careful when using `waitForExit` for processes created without + ## `poParentStreams` because they may fill output buffers, causing deadlock. + ## + ## On posix, if the process has exited because of a signal, 128 + signal + ## number will be returned. ## - ## **Warning**: Be careful when using waitForExit for processes created without - ## poParentStreams because they may fill output buffers, causing deadlock. + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. -proc peekExitCode*(p: Process): int {.tags: [].} - ## return -1 if the process is still running. Otherwise the process' exit code +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} + ## Return `-1` if the process is still running. Otherwise the process' exit code. + ## + ## On posix, if the process has exited because of a signal, 128 + signal + ## number will be returned. proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} - ## returns ``p``'s input stream for writing to. + ## Returns ``p``'s input stream for writing to. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. ## - ## **Warning**: The returned `Stream` should not be closed manually as it - ## is closed when closing the Process ``p``. + ## See also: + ## * `outputStream proc <#outputStream,Process>`_ + ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} - ## returns ``p``'s output stream for reading from. +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} + ## Returns ``p``'s output stream for reading from. + ## + ## You cannot perform peek/write/setOption operations to this stream. + ## Use `peekableOutputStream proc <#peekableOutputStream,Process>`_ + ## if you need to peek stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. ## - ## **Warning**: The returned `Stream` should not be closed manually as it - ## is closed when closing the Process ``p``. + ## See also: + ## * `inputStream proc <#inputStream,Process>`_ + ## * `errorStream proc <#errorStream,Process>`_ proc errorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} - ## returns ``p``'s error stream for reading from. + ## Returns ``p``'s error stream for reading from. ## - ## **Warning**: The returned `Stream` should not be closed manually as it - ## is closed when closing the Process ``p``. + ## You cannot perform peek/write/setOption operations to this stream. + ## Use `peekableErrorStream proc <#peekableErrorStream,Process>`_ + ## if you need to peek stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `inputStream proc <#inputStream,Process>`_ + ## * `outputStream proc <#outputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc peekableOutputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], since: (1, 3).} + ## Returns ``p``'s output stream for reading from. + ## + ## You can peek returned stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `outputStream proc <#outputStream,Process>`_ + ## * `peekableErrorStream proc <#peekableErrorStream,Process>`_ + +proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], since: (1, 3).} + ## Returns ``p``'s error stream for reading from. + ## + ## You can run peek operation to returned stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `errorStream proc <#errorStream,Process>`_ + ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ + +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = - ## returns ``p``'s input file handle for writing to. + ## Returns ``p``'s input file handle for writing to. + ## + ## .. warning:: The returned `FileHandle` should not be closed manually as + ## it is closed when closing the Process ``p``. ## - ## **Warning**: The returned `FileHandle` should not be closed manually as - ## it is closed when closing the Process ``p``. + ## See also: + ## * `outputHandle proc <#outputHandle,Process>`_ + ## * `errorHandle proc <#errorHandle,Process>`_ result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = - ## returns ``p``'s output file handle for reading from. + raises: [], tags: [].} = + ## Returns ``p``'s output file handle for reading from. + ## + ## .. warning:: The returned `FileHandle` should not be closed manually as + ## it is closed when closing the Process ``p``. ## - ## **Warning**: The returned `FileHandle` should not be closed manually as - ## it is closed when closing the Process ``p``. + ## See also: + ## * `inputHandle proc <#inputHandle,Process>`_ + ## * `errorHandle proc <#errorHandle,Process>`_ result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = - ## returns ``p``'s error file handle for reading from. + raises: [], tags: [].} = + ## Returns ``p``'s error file handle for reading from. + ## + ## .. warning:: The returned `FileHandle` should not be closed manually as + ## it is closed when closing the Process ``p``. ## - ## **Warning**: The returned `FileHandle` should not be closed manually as - ## it is closed when closing the Process ``p``. + ## See also: + ## * `inputHandle proc <#inputHandle,Process>`_ + ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = - ## returns the numer of the processors/cores the machine has. +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = + ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. + ## It is implemented just calling `cpuinfo.countProcessors`. result = cpuinfo.countProcessors() +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc execProcesses*(cmds: openArray[string], - options = {poStdErrToStdOut, poParentStreams}, - n = countProcessors(), - beforeRunEvent: proc(idx: int) = nil, - afterRunEvent: proc(idx: int, p: Process) = nil): int - {.rtl, extern: "nosp$1", - tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect]} = - ## executes the commands `cmds` in parallel. Creates `n` processes - ## that execute in parallel. The highest return value of all processes - ## is returned. Runs `beforeRunEvent` before running each command. - when defined(posix): - # poParentStreams causes problems on Posix, so we simply disable it: - var options = options - {poParentStreams} + options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), + beforeRunEvent: proc(idx: int) = nil, + afterRunEvent: proc(idx: int, p: Process) = nil): + int {.rtl, extern: "nosp$1", + raises: [ValueError, OSError, IOError], + tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], + effectsOf: [beforeRunEvent, afterRunEvent].} = + ## Executes the commands `cmds` in parallel. + ## Creates `n` processes that execute in parallel. + ## + ## The highest (absolute) return value of all processes is returned. + ## Runs `beforeRunEvent` before running each command. assert n > 0 if n > 1: - var q: seq[Process] - newSeq(q, n) - var m = min(n, cmds.len) - for i in 0..m-1: + var i = 0 + var q = newSeq[Process](n) + var idxs = newSeq[int](n) # map process index to cmds index + + when defined(windows): + var w: WOHandleArray + var m = min(min(n, MAXIMUM_WAIT_OBJECTS), cmds.len) + var wcount = m + else: + var m = min(n, cmds.len) + + while i < m: if beforeRunEvent != nil: beforeRunEvent(i) - q[i] = startProcess(cmds[i], options=options + {poEvalCommand}) - when defined(noBusyWaiting): - var r = 0 - for i in m..high(cmds): - when defined(debugExecProcesses): - var err = "" - var outp = outputStream(q[r]) - while running(q[r]) or not atEnd(outp): - err.add(outp.readLine()) - err.add("\n") - echo(err) - result = max(waitForExit(q[r]), result) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - r = (r + 1) mod n - else: - var i = m - while i <= high(cmds): - sleep(50) - for r in 0..n-1: - if not running(q[r]): - #echo(outputStream(q[r]).readLine()) - result = max(waitForExit(q[r]), result) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - inc(i) - if i > high(cmds): break - for j in 0..m-1: - result = max(waitForExit(q[j]), result) - if afterRunEvent != nil: afterRunEvent(j, q[j]) - if q[j] != nil: close(q[j]) + q[i] = startProcess(cmds[i], options = options + {poEvalCommand}) + idxs[i] = i + when defined(windows): + w[i] = q[i].fProcessHandle + inc(i) + + var ecount = len(cmds) + while ecount > 0: + var rexit = -1 + when defined(windows): + # waiting for all children, get result if any child exits + var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32, + INFINITE) + if ret == WAIT_TIMEOUT: + # must not be happen + discard + elif ret == WAIT_FAILED: + raiseOSError(osLastError()) + else: + var status: int32 + for r in 0..m-1: + if not isNil(q[r]) and q[r].fProcessHandle == w[ret]: + discard getExitCodeProcess(q[r].fProcessHandle, status) + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break + else: + var status: cint = 1 + # waiting for all children, get result if any child exits + let res = waitpid(-1, status, 0) + if res > 0: + for r in 0..m-1: + if not isNil(q[r]) and q[r].id == res: + if WIFEXITED(status) or WIFSIGNALED(status): + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break + else: + let err = osLastError() + if err == OSErrorCode(ECHILD): + # some child exits, we need to check our childs exit codes + for r in 0..m-1: + if (not isNil(q[r])) and (not running(q[r])): + q[r].exitFlag = true + q[r].exitStatus = status + rexit = r + break + elif err == OSErrorCode(EINTR): + # signal interrupted our syscall, lets repeat it + continue + else: + # all other errors are exceptions + raiseOSError(err) + + if rexit >= 0: + when defined(windows): + let processHandle = q[rexit].fProcessHandle + result = max(result, abs(q[rexit].peekExitCode())) + if afterRunEvent != nil: afterRunEvent(idxs[rexit], q[rexit]) + close(q[rexit]) + if i < len(cmds): + if beforeRunEvent != nil: beforeRunEvent(i) + q[rexit] = startProcess(cmds[i], + options = options + {poEvalCommand}) + idxs[rexit] = i + when defined(windows): + w[rexit] = q[rexit].fProcessHandle + inc(i) + else: + when defined(windows): + for k in 0..wcount - 1: + if w[k] == processHandle: + w[k] = w[wcount - 1] + w[wcount - 1] = 0 + dec(wcount) + break + q[rexit] = nil + dec(ecount) else: for i in 0..high(cmds): if beforeRunEvent != nil: beforeRunEvent(i) - var p = startProcess(cmds[i], options=options + {poEvalCommand}) - result = max(waitForExit(p), result) + var p = startProcess(cmds[i], options = options + {poEvalCommand}) + result = max(abs(waitForExit(p)), result) if afterRunEvent != nil: afterRunEvent(i, p) close(p) -proc select*(readfds: var seq[Process], timeout = 500): int {.benign.} - ## `select` with a sensible Nim interface. `timeout` is in milliseconds. - ## Specify -1 for no timeout. Returns the number of processes that are - ## ready to read from. The processes that are ready to be read from are - ## removed from `readfds`. +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = + ## Convenience iterator for working with `startProcess` to read data from a + ## background process. + ## + ## See also: + ## * `readLines proc <#readLines,Process>`_ ## - ## **Warning**: This function may give unexpected or completely wrong - ## results on Windows. + ## Example: + ## ```Nim + ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} + ## var ps: seq[Process] + ## for prog in ["a", "b"]: # run 2 progs in parallel + ## ps.add startProcess("nim", "", ["r", prog], nil, opts) + ## for p in ps: + ## var i = 0 + ## for line in p.lines: + ## echo line + ## i.inc + ## if i > 100: break + ## p.close + ## ``` + var outp = p.outputStream + var line = newStringOfCap(120) + while outp.readLine(line): + if keepNewLines: + line.add("\n") + yield line + discard waitForExit(p) + +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = + ## Convenience function for working with `startProcess` to read data from a + ## background process. + ## + ## See also: + ## * `lines iterator <#lines.i,Process>`_ + ## + ## Example: + ## ```Nim + ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} + ## var ps: seq[Process] + ## for prog in ["a", "b"]: # run 2 progs in parallel + ## ps.add startProcess("nim", "", ["r", prog], nil, opts) + ## for p in ps: + ## let (lines, exCode) = p.readLines + ## if exCode != 0: + ## for line in lines: echo line + ## p.close + ## ``` + for line in p.lines: result[0].add(line) + result[1] = p.peekExitCode when not defined(useNimRtl): - proc execProcess(command: string, - args: openArray[string] = [], - env: StringTableRef = nil, - options: set[ProcessOption] = {poStdErrToStdOut, - poUsePath, - poEvalCommand}): TaintedString = - var p = startProcess(command, args=args, env=env, options=options) + proc execProcess(command: string, workingDir: string = "", + args: openArray[string] = [], env: StringTableRef = nil, + options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, + poEvalCommand}): + string = + + var p = startProcess(command, workingDir = workingDir, args = args, + env = env, options = options) var outp = outputStream(p) - result = TaintedString"" - var line = newStringOfCap(120).TaintedString + result = "" + var line = newStringOfCap(120) + # consider `p.lines(keepNewLines=true)` to circumvent `running` busy-wait while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): - result.string.add(line.string) - result.string.add("\n") + result.add(line) + result.add("\n") elif not running(p): break close(p) +template streamAccess(p) = + assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams" -when defined(Windows) and not defined(useNimRtl): +when defined(windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: type - PFileHandleStream = ref FileHandleStream - FileHandleStream = object of StreamObj + FileHandleStream = ref object of StreamObj handle: Handle atTheEnd: bool - {.deprecated: [TFileHandleStream: FileHandleStream].} - proc hsClose(s: Stream) = discard # nothing to do here - proc hsAtEnd(s: Stream): bool = return PFileHandleStream(s).atTheEnd + proc closeHandleCheck(handle: Handle) {.inline.} = + if handle.closeHandle() == 0: + raiseOSError(osLastError()) + + proc fileClose[T: Handle | FileHandle](h: var T) {.inline.} = + if h > 4: + closeHandleCheck(h) + h = INVALID_HANDLE_VALUE.T + + proc hsClose(s: Stream) = + FileHandleStream(s).handle.fileClose() + + proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd proc hsReadData(s: Stream, buffer: pointer, bufLen: int): int = - var s = PFileHandleStream(s) + var s = FileHandleStream(s) if s.atTheEnd: return 0 var br: int32 var a = winlean.readFile(s.handle, buffer, bufLen.cint, addr br, nil) @@ -378,37 +567,32 @@ when defined(Windows) and not defined(useNimRtl): result = br proc hsWriteData(s: Stream, buffer: pointer, bufLen: int) = - var s = PFileHandleStream(s) + var s = FileHandleStream(s) var bytesWritten: int32 var a = winlean.writeFile(s.handle, buffer, bufLen.cint, addr bytesWritten, nil) if a == 0: raiseOSError(osLastError()) - proc newFileHandleStream(handle: Handle): PFileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData - - proc buildCommandLine(a: string, args: openArray[string]): cstring = - var res = quoteShell(a) + proc newFileHandleStream(handle: Handle): owned FileHandleStream = + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) + + proc buildCommandLine(a: string, args: openArray[string]): string = + result = quoteShell(a) for i in 0..high(args): - res.add(' ') - res.add(quoteShell(args[i])) - result = cast[cstring](alloc0(res.len+1)) - copyMem(result, cstring(res), res.len) + result.add(' ') + result.add(quoteShell(args[i])) - proc buildEnv(env: StringTableRef): cstring = + proc buildEnv(env: StringTableRef): tuple[str: cstring, len: int] = var L = 0 for key, val in pairs(env): inc(L, key.len + val.len + 2) - result = cast[cstring](alloc0(L+2)) + var str = cast[cstring](alloc0(L+2)) L = 0 for key, val in pairs(env): var x = key & "=" & val - copyMem(addr(result[L]), cstring(x), x.len+1) # copy \0 + copyMem(addr(str[L]), cstring(x), x.len+1) # copy \0 inc(L, x.len+1) + (str, L) #proc open_osfhandle(osh: Handle, mode: int): int {. # importc: "_open_osfhandle", header: "<fcntl.h>".} @@ -416,60 +600,59 @@ when defined(Windows) and not defined(useNimRtl): #var # O_WRONLY {.importc: "_O_WRONLY", header: "<fcntl.h>".}: int # O_RDONLY {.importc: "_O_RDONLY", header: "<fcntl.h>".}: int - proc myDup(h: Handle; inherit: WinBool=1): Handle = + proc myDup(h: Handle; inherit: WINBOOL = 1): Handle = let thisProc = getCurrentProcess() - if duplicateHandle(thisProc, h, - thisProc, addr result,0,inherit, + if duplicateHandle(thisProc, h, thisProc, addr result, 0, inherit, DUPLICATE_SAME_ACCESS) == 0: raiseOSError(osLastError()) proc createAllPipeHandles(si: var STARTUPINFO; - stdin, stdout, stderr: var Handle) = + stdin, stdout, stderr: var Handle; hash: int) = var sa: SECURITY_ATTRIBUTES sa.nLength = sizeof(SECURITY_ATTRIBUTES).cint sa.lpSecurityDescriptor = nil sa.bInheritHandle = 1 - let pipeOutName = newWideCString(r"\\.\pipe\stdout") - let pipeInName = newWideCString(r"\\.\pipe\stdin") + let pipeOutName = newWideCString(r"\\.\pipe\stdout" & $hash) + let pipeInName = newWideCString(r"\\.\pipe\stdin" & $hash) let pipeOut = createNamedPipe(pipeOutName, - dwOpenMode=PIPE_ACCESS_INBOUND or FILE_FLAG_WRITE_THROUGH, - dwPipeMode=PIPE_NOWAIT, - nMaxInstances=1, - nOutBufferSize=1024, nInBufferSize=1024, - nDefaultTimeOut=0,addr sa) + dwOpenMode = PIPE_ACCESS_INBOUND or FILE_FLAG_WRITE_THROUGH, + dwPipeMode = PIPE_NOWAIT, + nMaxInstances = 1, + nOutBufferSize = 1024, nInBufferSize = 1024, + nDefaultTimeOut = 0, addr sa) if pipeOut == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) let pipeIn = createNamedPipe(pipeInName, - dwOpenMode=PIPE_ACCESS_OUTBOUND or FILE_FLAG_WRITE_THROUGH, - dwPipeMode=PIPE_NOWAIT, - nMaxInstances=1, - nOutBufferSize=1024, nInBufferSize=1024, - nDefaultTimeOut=0,addr sa) + dwOpenMode = PIPE_ACCESS_OUTBOUND or FILE_FLAG_WRITE_THROUGH, + dwPipeMode = PIPE_NOWAIT, + nMaxInstances = 1, + nOutBufferSize = 1024, nInBufferSize = 1024, + nDefaultTimeOut = 0, addr sa) if pipeIn == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) si.hStdOutput = createFileW(pipeOutName, FILE_WRITE_DATA or SYNCHRONIZE, 0, addr sa, OPEN_EXISTING, # very important flag! - FILE_ATTRIBUTE_NORMAL, - 0 # no template file for OPEN_EXISTING - ) + FILE_ATTRIBUTE_NORMAL, + 0 # no template file for OPEN_EXISTING + ) if si.hStdOutput == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) si.hStdError = myDup(si.hStdOutput) si.hStdInput = createFileW(pipeInName, FILE_READ_DATA or SYNCHRONIZE, 0, addr sa, OPEN_EXISTING, # very important flag! - FILE_ATTRIBUTE_NORMAL, - 0 # no template file for OPEN_EXISTING - ) - if si.hStdOutput == INVALID_HANDLE_VALUE: + FILE_ATTRIBUTE_NORMAL, + 0 # no template file for OPEN_EXISTING + ) + if si.hStdInput == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) stdin = myDup(pipeIn, 0) stdout = myDup(pipeOut, 0) - discard closeHandle(pipeIn) - discard closeHandle(pipeOut) + closeHandleCheck(pipeIn) + closeHandleCheck(pipeOut) stderr = stdout proc createPipeHandles(rdHandle, wrHandle: var Handle) = @@ -477,17 +660,13 @@ when defined(Windows) and not defined(useNimRtl): sa.nLength = sizeof(SECURITY_ATTRIBUTES).cint sa.lpSecurityDescriptor = nil sa.bInheritHandle = 1 - if createPipe(rdHandle, wrHandle, sa, 1024) == 0'i32: + if createPipe(rdHandle, wrHandle, sa, 0) == 0'i32: raiseOSError(osLastError()) - proc fileClose(h: Handle) {.inline.} = - if h > 4: discard closeHandle(h) - - proc startProcess(command: string, - workingDir: string = "", - args: openArray[string] = [], - env: StringTableRef = nil, - options: set[ProcessOption] = {poStdErrToStdOut}): Process = + proc startProcess(command: string, workingDir: string = "", + args: openArray[string] = [], env: StringTableRef = nil, + options: set[ProcessOption] = {poStdErrToStdOut}): + owned Process = var si: STARTUPINFO procInfo: PROCESS_INFORMATION @@ -495,6 +674,7 @@ when defined(Windows) and not defined(useNimRtl): hi, ho, he: Handle new(result) result.options = options + result.exitFlag = true si.cb = sizeof(si).cint if poParentStreams notin options: si.dwFlags = STARTF_USESTDHANDLES # STARTF_USESHOWWINDOW or @@ -506,8 +686,14 @@ when defined(Windows) and not defined(useNimRtl): he = ho else: createPipeHandles(he, si.hStdError) + if setHandleInformation(he, DWORD(1), DWORD(0)) == 0'i32: + raiseOSError(osLastError()) + if setHandleInformation(hi, DWORD(1), DWORD(0)) == 0'i32: + raiseOSError(osLastError()) + if setHandleInformation(ho, DWORD(1), DWORD(0)) == 0'i32: + raiseOSError(osLastError()) else: - createAllPipeHandles(si, hi, ho, he) + createAllPipeHandles(si, hi, ho, he, cast[int](result)) result.inHandle = FileHandle(hi) result.outHandle = FileHandle(ho) result.errHandle = FileHandle(he) @@ -520,27 +706,27 @@ when defined(Windows) and not defined(useNimRtl): result.errHandle = FileHandle(si.hStdError) var cmdl: cstring + var cmdRoot: string if poEvalCommand in options: cmdl = command assert args.len == 0 else: - cmdl = buildCommandLine(command, args) + cmdRoot = buildCommandLine(command, args) + cmdl = cstring(cmdRoot) var wd: cstring = nil - var e: cstring = nil + var e = (str: nil.cstring, len: -1) if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = newWideCString(e) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, e, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -549,38 +735,59 @@ when defined(Windows) and not defined(useNimRtl): if poStdErrToStdOut notin options: fileClose(si.hStdError) - if e != nil: dealloc(e) + if e.str != nil: dealloc(e.str) if success == 0: if poInteractive in result.options: close(result) const errInvalidParameter = 87.int const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) - # Close the handle now so anyone waiting is woken: - discard closeHandle(procInfo.hThread) result.fProcessHandle = procInfo.hProcess + result.fThreadHandle = procInfo.hThread result.id = procInfo.dwProcessId + result.exitFlag = false + + proc closeThreadAndProcessHandle(p: Process) = + if p.fThreadHandle != 0: + closeHandleCheck(p.fThreadHandle) + p.fThreadHandle = 0 + + if p.fProcessHandle != 0: + closeHandleCheck(p.fProcessHandle) + p.fProcessHandle = 0 proc close(p: Process) = - if poInteractive in p.options: - # somehow this is not always required on Windows: - discard closeHandle(p.inHandle) - discard closeHandle(p.outHandle) - discard closeHandle(p.errHandle) - #discard closeHandle(p.FProcessHandle) + if poParentStreams notin p.options: + if p.inStream == nil: + p.inHandle.fileClose() + else: + # p.inHandle can be already closed via inputStream. + p.inStream.close + + # You may NOT close outputStream and errorStream. + assert p.outStream == nil or FileHandleStream(p.outStream).handle != INVALID_HANDLE_VALUE + assert p.errStream == nil or FileHandleStream(p.errStream).handle != INVALID_HANDLE_VALUE + + if p.outHandle != p.errHandle: + p.errHandle.fileClose() + p.outHandle.fileClose() + p.closeThreadAndProcessHandle() proc suspend(p: Process) = - discard suspendThread(p.fProcessHandle) + discard suspendThread(p.fThreadHandle) proc resume(p: Process) = - discard resumeThread(p.fProcessHandle) + discard resumeThread(p.fThreadHandle) proc running(p: Process): bool = - var x = waitForSingleObject(p.fProcessHandle, 50) - return x == WAIT_TIMEOUT + if p.exitFlag: + return false + else: + var x = waitForSingleObject(p.fProcessHandle, 0) + return x == WAIT_TIMEOUT proc terminate(p: Process) = if running(p): @@ -590,29 +797,65 @@ when defined(Windows) and not defined(useNimRtl): terminate(p) proc waitForExit(p: Process, timeout: int = -1): int = - discard waitForSingleObject(p.fProcessHandle, timeout.int32) - - var res: int32 - discard getExitCodeProcess(p.fProcessHandle, res) - result = res - discard closeHandle(p.fProcessHandle) + if p.exitFlag: + return p.exitStatus + + let res = waitForSingleObject(p.fProcessHandle, timeout.int32) + if res == WAIT_TIMEOUT: + terminate(p) + var status: int32 + discard getExitCodeProcess(p.fProcessHandle, status) + if status != STILL_ACTIVE: + p.exitFlag = true + p.exitStatus = status + p.closeThreadAndProcessHandle() + result = status + else: + result = -1 proc peekExitCode(p: Process): int = - var b = waitForSingleObject(p.fProcessHandle, 50) == WAIT_TIMEOUT - if b: result = -1 - else: - var res: int32 - discard getExitCodeProcess(p.fProcessHandle, res) - return res + if p.exitFlag: + return p.exitStatus + + result = -1 + var b = waitForSingleObject(p.fProcessHandle, 0) == WAIT_TIMEOUT + if not b: + var status: int32 + discard getExitCodeProcess(p.fProcessHandle, status) + p.exitFlag = true + p.exitStatus = status + p.closeThreadAndProcessHandle() + result = status proc inputStream(p: Process): Stream = - result = newFileHandleStream(p.inHandle) + streamAccess(p) + if p.inStream == nil: + p.inStream = newFileHandleStream(p.inHandle) + result = p.inStream proc outputStream(p: Process): Stream = - result = newFileHandleStream(p.outHandle) + streamAccess(p) + if p.outStream == nil: + p.outStream = newFileHandleStream(p.outHandle) + result = p.outStream proc errorStream(p: Process): Stream = - result = newFileHandleStream(p.errHandle) + streamAccess(p) + if p.errStream == nil: + p.errStream = newFileHandleStream(p.errHandle) + result = p.errStream + + proc peekableOutputStream(p: Process): Stream = + streamAccess(p) + if p.outStream == nil: + p.outStream = newFileHandleStream(p.outHandle).newPipeOutStream + result = p.outStream + + proc peekableErrorStream(p: Process): Stream = + streamAccess(p) + if p.errStream == nil: + p.errStream = newFileHandleStream(p.errHandle).newPipeOutStream + result = p.errStream proc execCmd(command: string): int = var @@ -624,13 +867,9 @@ when defined(Windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: @@ -663,7 +902,7 @@ when defined(Windows) and not defined(useNimRtl): proc hasData*(p: Process): bool = var x: int32 - if peekNamedPipe(p.outHandle, lpTotalBytesAvail=addr x): + if peekNamedPipe(p.outHandle, lpTotalBytesAvail = addr x): result = x > 0 elif not defined(useNimRtl): @@ -671,6 +910,9 @@ elif not defined(useNimRtl): readIdx = 0 writeIdx = 1 + proc isExitStatus(status: cint): bool = + WIFEXITED(status) or WIFSIGNALED(status) + proc envToCStringArray(t: StringTableRef): cstringArray = result = cast[cstringArray](alloc0((t.len + 1) * sizeof(cstring))) var i = 0 @@ -686,58 +928,59 @@ elif not defined(useNimRtl): result = cast[cstringArray](alloc0((counter + 1) * sizeof(cstring))) var i = 0 for key, val in envPairs(): - var x = key.string & "=" & val.string + var x = key & "=" & val result[i] = cast[cstring](alloc(x.len+1)) copyMem(result[i], addr(x[0]), x.len+1) inc(i) - type StartProcessData = object - sysCommand: cstring - sysArgs: cstringArray - sysEnv: cstringArray - workingDir: cstring - pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] - optionPoUsePath: bool - optionPoParentStreams: bool - optionPoStdErrToStdOut: bool - {.deprecated: [TStartProcessData: StartProcessData].} + type + StartProcessData = object + sysCommand: string + sysArgs: cstringArray + sysEnv: cstringArray + workingDir: cstring + pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] + options: set[ProcessOption] const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect], gcsafe.} - {.push stacktrace: off, profiler: off.} - proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect], cdecl, gcsafe.} - {.pop.} - - proc startProcess(command: string, - workingDir: string = "", - args: openArray[string] = [], - env: StringTableRef = nil, - options: set[ProcessOption] = {poStdErrToStdOut}): Process = + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + {.push stacktrace: off, profiler: off.} + proc startProcessAfterFork(data: ptr StartProcessData) {. + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + {.pop.} + + proc startProcess(command: string, workingDir: string = "", + args: openArray[string] = [], env: StringTableRef = nil, + options: set[ProcessOption] = {poStdErrToStdOut}): + owned Process = var - pStdin, pStdout, pStderr: array [0..1, cint] + pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options - result.exitCode = -3 # for ``waitForExit`` + result.exitFlag = true + if poParentStreams notin options: if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or pipe(pStderr) != 0'i32: raiseOSError(osLastError()) - var sysCommand: string + var data: StartProcessData var sysArgsRaw: seq[string] if poEvalCommand in options: - sysCommand = "/bin/sh" - sysArgsRaw = @[sysCommand, "-c", command] + const useShPath {.strdefine.} = + when not defined(android): "/bin/sh" + else: "/system/bin/sh" + data.sysCommand = useShPath + sysArgsRaw = @[useShPath, "-c", command] assert args.len == 0, "`args` has to be empty when using poEvalCommand." else: - sysCommand = command + data.sysCommand = command sysArgsRaw = @[command] for arg in args.items: sysArgsRaw.add arg @@ -754,20 +997,19 @@ elif not defined(useNimRtl): defer: deallocCStringArray(sysEnv) - var data: StartProcessData - data.sysCommand = sysCommand data.sysArgs = sysArgs data.sysEnv = sysEnv data.pStdin = pStdin data.pStdout = pStdout data.pStderr = pStderr - data.optionPoParentStreams = poParentStreams in options - data.optionPoUsePath = poUsePath in options - data.optionPoStdErrToStdOut = poStdErrToStdOut in options data.workingDir = workingDir + data.options = options when useProcessAuxSpawn: + var currentDir = getCurrentDir() pid = startProcessAuxSpawn(data) + if workingDir.len > 0: + setCurrentDir(currentDir) else: pid = startProcessAuxFork(data) @@ -775,6 +1017,7 @@ elif not defined(useNimRtl): if poEchoCmd in options: echo(command, " ", join(args, " ")) result.id = pid + result.exitFlag = false if poParentStreams in options: # does not make much sense, but better than nothing: @@ -801,7 +1044,7 @@ elif not defined(useNimRtl): var attr: Tposix_spawnattr var fops: Tposix_spawn_file_actions - template chck(e: expr) = + template chck(e: untyped) = if e != 0'i32: raiseOSError(osLastError()) chck posix_spawn_file_actions_init(fops) @@ -810,37 +1053,42 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - chck posix_spawnattr_setpgroup(attr, 0'i32) - - chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or - POSIX_SPAWN_SETSIGMASK or - POSIX_SPAWN_SETPGROUP) - - if not data.optionPoParentStreams: + when not defined(nuttx): + if poDaemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) + + var flags = POSIX_SPAWN_USEVFORK or + POSIX_SPAWN_SETSIGMASK + when not defined(nuttx): + if poDaemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP + chck posix_spawnattr_setflags(attr, flags) + + if not (poParentStreams in data.options): chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx) chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if poStdErrToStdOut in data.options: chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) var res: cint - # FIXME: chdir is global to process if data.workingDir.len > 0: setCurrentDir($data.workingDir) var pid: Pid - if data.optionPoUsePath: - res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + if (poUsePath in data.options): + res = posix_spawnp(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) else: - res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawn(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) - chck res + if res != 0'i32: raiseOSError(OSErrorCode(res), data.sysCommand) + return pid else: proc startProcessAuxFork(data: StartProcessData): Pid = @@ -875,145 +1123,355 @@ elif not defined(useNimRtl): var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): - raiseOSError("Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + raiseOSError(OSErrorCode(error), + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid - {.push stacktrace: off, profiler: off.} - proc startProcessFail(data: ptr StartProcessData) = - var error: cint = errno - discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) - exitnow(1) - - when defined(macosx) or defined(freebsd): - var environ {.importc.}: cstringArray - - proc startProcessAfterFork(data: ptr StartProcessData) = - # Warning: no GC here! - # Or anything that touches global structures - all called nim procs - # must be marked with stackTrace:off. Inspect C code after making changes. - if not data.optionPoParentStreams: - discard close(data.pStdin[writeIdx]) - if dup2(data.pStdin[readIdx], readIdx) < 0: - startProcessFail(data) - discard close(data.pStdout[readIdx]) - if dup2(data.pStdout[writeIdx], writeIdx) < 0: - startProcessFail(data) - discard close(data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: - if dup2(data.pStdout[writeIdx], 2) < 0: + {.push stacktrace: off, profiler: off.} + proc startProcessFail(data: ptr StartProcessData, error: cint = errno) = + discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) + exitnow(1) + + when not defined(uClibc) and (not defined(linux) or defined(android)) and + not defined(haiku): + var environ {.importc.}: cstringArray + + proc startProcessAfterFork(data: ptr StartProcessData) = + # Warning: no GC here! + # Or anything that touches global structures - all called nim procs + # must be marked with stackTrace:off. Inspect C code after making changes. + if not (poParentStreams in data.options): + discard close(data.pStdin[writeIdx]) + if dup2(data.pStdin[readIdx], readIdx) < 0: startProcessFail(data) - else: - if dup2(data.pStderr[writeIdx], 2) < 0: + discard close(data.pStdout[readIdx]) + if dup2(data.pStdout[writeIdx], writeIdx) < 0: startProcessFail(data) + discard close(data.pStderr[readIdx]) + if (poStdErrToStdOut in data.options): + if dup2(data.pStdout[writeIdx], 2) < 0: + startProcessFail(data) + else: + if dup2(data.pStderr[writeIdx], 2) < 0: + startProcessFail(data) - if data.workingDir.len > 0: - if chdir(data.workingDir) < 0: - startProcessFail(data) + if data.workingDir.len > 0: + if chdir(data.workingDir) < 0: + startProcessFail(data) - discard close(data.pErrorPipe[readIdx]) - discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) + discard close(data.pErrorPipe[readIdx]) + discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) - if data.optionPoUsePath: - when defined(macosx) or defined(freebsd): - # MacOSX doesn't have execvpe, so we need workaround. - # On MacOSX we can arrive here only from fork, so this is safe: - environ = data.sysEnv - discard execvp(data.sysCommand, data.sysArgs) - else: - when defined(uClibc): + if (poUsePath in data.options): + when defined(uClibc) or defined(linux) or defined(haiku): # uClibc environment (OpenWrt included) doesn't have the full execvpe - discard execve(data.sysCommand, data.sysArgs, data.sysEnv) + var exe: string + try: + exe = findExe(data.sysCommand) + except OSError as e: + startProcessFail(data, e.errorCode) + discard execve(exe.cstring, data.sysArgs, data.sysEnv) else: - discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv) - else: - discard execve(data.sysCommand, data.sysArgs, data.sysEnv) + # MacOSX doesn't have execvpe, so we need workaround. + # On MacOSX we can arrive here only from fork, so this is safe: + environ = data.sysEnv + discard execvp(data.sysCommand.cstring, data.sysArgs) + else: + discard execve(data.sysCommand.cstring, data.sysArgs, data.sysEnv) - startProcessFail(data) - {.pop} + startProcessFail(data) + {.pop.} proc close(p: Process) = - if p.inStream != nil: close(p.inStream) - if p.outStream != nil: close(p.outStream) - if p.errStream != nil: close(p.errStream) if poParentStreams notin p.options: - discard close(p.inHandle) - discard close(p.outHandle) - discard close(p.errHandle) + if p.inStream != nil: + close(p.inStream) + else: + discard close(p.inHandle) + + if p.outStream != nil: + close(p.outStream) + else: + discard close(p.outHandle) + + if p.errStream != nil: + close(p.errStream) + else: + discard close(p.errHandle) proc suspend(p: Process) = - if kill(p.id, SIGSTOP) != 0'i32: raiseOsError(osLastError()) + if kill(p.id, SIGSTOP) != 0'i32: raiseOSError(osLastError()) proc resume(p: Process) = - if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError()) + if kill(p.id, SIGCONT) != 0'i32: raiseOSError(osLastError()) proc running(p: Process): bool = - var ret : int - when not defined(freebsd): - ret = waitpid(p.id, p.exitCode, WNOHANG) + if p.exitFlag: + return false else: - var status : cint = 1 - ret = waitpid(p.id, status, WNOHANG) - if WIFEXITED(status): - p.exitCode = status - if ret == 0: return true # Can't establish status. Assume running. - result = ret == int(p.id) + var status: cint = 1 + let ret = waitpid(p.id, status, WNOHANG) + if ret == int(p.id): + if isExitStatus(status): + p.exitFlag = true + p.exitStatus = status + return false + else: + return true + elif ret == 0: + return true # Can't establish status. Assume running. + else: + raiseOSError(osLastError()) proc terminate(p: Process) = if kill(p.id, SIGTERM) != 0'i32: - raiseOsError(osLastError()) + raiseOSError(osLastError()) proc kill(p: Process) = if kill(p.id, SIGKILL) != 0'i32: - raiseOsError(osLastError()) - - proc waitForExit(p: Process, timeout: int = -1): int = - #if waitPid(p.id, p.exitCode, 0) == int(p.id): - # ``waitPid`` fails if the process is not running anymore. But then - # ``running`` probably set ``p.exitCode`` for us. Since ``p.exitCode`` is - # initialized with -3, wrong success exit codes are prevented. - if p.exitCode != -3: return p.exitCode - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 raiseOSError(osLastError()) - result = int(p.exitCode) shr 8 + + when defined(macosx) or defined(freebsd) or defined(netbsd) or + defined(openbsd) or defined(dragonfly): + import std/kqueue + + proc waitForExit(p: Process, timeout: int = -1): int = + if p.exitFlag: + return exitStatusLikeShell(p.exitStatus) + + if timeout == -1: + var status: cint = 1 + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + else: + var kqFD = kqueue() + if kqFD == -1: + raiseOSError(osLastError()) + + var kevIn = KEvent(ident: p.id.uint, filter: EVFILT_PROC, + flags: EV_ADD, fflags: NOTE_EXIT) + var kevOut: KEvent + var tmspec: Timespec + + if timeout >= 1000: + tmspec.tv_sec = posix.Time(timeout div 1_000) + tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tmspec.tv_sec = posix.Time(0) + tmspec.tv_nsec = (timeout * 1_000_000) + + try: + while true: + var status: cint = 1 + var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, + addr(tmspec)) + if count < 0: + let err = osLastError() + if err.cint != EINTR: + raiseOSError(osLastError()) + elif count == 0: + # timeout expired, so we trying to kill process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + break + else: + if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + break + else: + raiseOSError(osLastError()) + finally: + discard posix.close(kqFD) + + result = exitStatusLikeShell(p.exitStatus) + elif defined(haiku): + const + B_OBJECT_TYPE_THREAD = 3 + B_EVENT_INVALID = 0x1000 + B_RELATIVE_TIMEOUT = 0x8 + + type + ObjectWaitInfo {.importc: "object_wait_info", header: "OS.h".} = object + obj {.importc: "object".}: int32 + typ {.importc: "type".}: uint16 + events: uint16 + + proc waitForObjects(infos: ptr ObjectWaitInfo, numInfos: cint, flags: uint32, + timeout: int64): clong + {.importc: "wait_for_objects_etc", header: "OS.h".} + + proc waitForExit(p: Process, timeout: int = -1): int = + if p.exitFlag: + return exitStatusLikeShell(p.exitStatus) + + if timeout == -1: + var status: cint = 1 + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + else: + var info = ObjectWaitInfo( + obj: p.id, # Haiku's PID is actually the main thread ID. + typ: B_OBJECT_TYPE_THREAD, + events: B_EVENT_INVALID # notify when the thread die. + ) + + while true: + var status: cint = 1 + let count = waitForObjects(addr info, 1, B_RELATIVE_TIMEOUT, timeout) + + if count < 0: + let err = count.cint + if err == ETIMEDOUT: + # timeout expired, so we try to kill the process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + break + elif err != EINTR: + raiseOSError(err.OSErrorCode) + elif count > 0: + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + break + else: + raiseAssert "unreachable!" + + result = exitStatusLikeShell(p.exitStatus) + + else: + import std/times except getTime + import std/monotimes + + proc waitForExit(p: Process, timeout: int = -1): int = + if p.exitFlag: + return exitStatusLikeShell(p.exitStatus) + + if timeout < 0: + # Backwards compatibility with previous verison to + # handle cases where timeout == -1, but extend + # to handle cases where timeout < 0 + var status: cint + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + else: + # Max 50ms delay + const maxWait = initDuration(milliseconds = 50) + let wait = initDuration(milliseconds = timeout) + let deadline = getMonoTime() + wait + # starting 50μs delay + var delay = initDuration(microseconds = 50) + + while true: + var status: cint + let pid = waitpid(p.id, status, WNOHANG) + if p.id == pid : + p.exitFlag = true + p.exitStatus = status + break + elif pid.int == -1: + raiseOsError(osLastError()) + else: + # Continue waiting if needed + if getMonoTime() >= deadline: + # Previous version of `waitForExit` + # foricibly killed the process. + # We keep this so we don't break programs + # that depend on this behavior + if posix.kill(p.id, SIGKILL) < 0: + raiseOSError(osLastError()) + else: + const max = 1_000_000_000 + let + newWait = getMonoTime() + delay + ticks = newWait.ticks() + ns = ticks mod max + secs = ticks div max + var + waitSpec: TimeSpec + unused: Timespec + waitSpec.tv_sec = posix.Time(secs) + waitSpec.tv_nsec = clong ns + discard posix.clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, waitSpec, unused) + let remaining = deadline - getMonoTime() + delay = min([delay * 2, remaining, maxWait]) + + result = exitStatusLikeShell(p.exitStatus) proc peekExitCode(p: Process): int = - if p.exitCode != -3: return p.exitCode - var ret = waitpid(p.id, p.exitCode, WNOHANG) - var b = ret == int(p.id) - if b: result = -1 - if not WIFEXITED(p.exitCode): result = -1 - else: result = p.exitCode.int shr 8 - - proc createStream(stream: var Stream, handle: var FileHandle, - fileMode: FileMode) = + var status = cint(0) + result = -1 + if p.exitFlag: + return exitStatusLikeShell(p.exitStatus) + + var ret = waitpid(p.id, status, WNOHANG) + if ret > 0: + if isExitStatus(status): + p.exitFlag = true + p.exitStatus = status + result = exitStatusLikeShell(status) + + proc createStream(handle: var FileHandle, + fileMode: FileMode): owned FileStream = var f: File if not open(f, handle, fileMode): raiseOSError(osLastError()) - stream = newFileStream(f) + return newFileStream(f) proc inputStream(p: Process): Stream = + streamAccess(p) if p.inStream == nil: - createStream(p.inStream, p.inHandle, fmWrite) + p.inStream = createStream(p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = + streamAccess(p) if p.outStream == nil: - createStream(p.outStream, p.outHandle, fmRead) + p.outStream = createStream(p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = + streamAccess(p) if p.errStream == nil: - createStream(p.errStream, p.errHandle, fmRead) + p.errStream = createStream(p.errHandle, fmRead) + return p.errStream + + proc peekableOutputStream(p: Process): Stream = + streamAccess(p) + if p.outStream == nil: + p.outStream = createStream(p.outHandle, fmRead).newPipeOutStream + return p.outStream + + proc peekableErrorStream(p: Process): Stream = + streamAccess(p) + if p.errStream == nil: + p.errStream = createStream(p.errHandle, fmRead).newPipeOutStream return p.errStream proc csystem(cmd: cstring): cint {.nodecl, importc: "system", header: "<stdlib.h>".} proc execCmd(command: string): int = - when defined(linux): - result = csystem(command) shr 8 + when defined(posix): + let tmp = csystem(command) + result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: result = csystem(command) @@ -1036,8 +1494,8 @@ elif not defined(useNimRtl): proc select(readfds: var seq[Process], timeout = 500): int = var tv: Timeval - tv.tv_sec = 0 - tv.tv_usec = timeout * 1000 + tv.tv_sec = posix.Time(0) + tv.tv_usec = Suseconds(timeout * 1000) var rd: TFdSet var m = 0 @@ -1061,37 +1519,63 @@ elif not defined(useNimRtl): proc execCmdEx*(command: string, options: set[ProcessOption] = { - poStdErrToStdOut, poUsePath}): tuple[ - output: TaintedString, - exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} = - ## a convenience proc that runs the `command`, grabs all its output and - ## exit code and returns both. + poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, + workingDir = "", input = ""): tuple[ + output: string, + exitCode: int] {.raises: [OSError, IOError], tags: + [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = + ## A convenience proc that runs the `command`, and returns its `output` and + ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. + ## If `input.len > 0`, it is passed as stdin. + ## + ## Note: this could block if `input.len` is greater than your OS's maximum + ## pipe buffer size. ## - ## .. code-block:: Nim + ## See also: + ## * `execCmd proc <#execCmd,string>`_ + ## * `startProcess proc + ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ + ## * `execProcess proc + ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## - ## let (outp, errC) = execCmdEx("nim c -r mytestfile.nim") - var p = startProcess(command, options=options + {poEvalCommand}) + ## Example: + ## ```Nim + ## var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") + ## import std/[strutils, strtabs] + ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any + ## doAssert result == ("12", 0) + ## doAssert execCmdEx("ls --nonexistent").exitCode != 0 + ## when defined(posix): + ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) + ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + ## ``` + + when (NimMajor, NimMinor, NimPatch) < (1, 3, 5): + doAssert input.len == 0 + doAssert workingDir.len == 0 + doAssert env == nil + + var p = startProcess(command, options = options + {poEvalCommand}, + workingDir = workingDir, env = env) var outp = outputStream(p) - result = (TaintedString"", -1) - var line = newStringOfCap(120).TaintedString + + if input.len > 0: + # There is no way to provide input for the child process + # anymore. Closing it will create EOF on stdin instead of eternal + # blocking. + # Writing in chunks would require a selectors (eg kqueue/epoll) to avoid + # blocking on io. + inputStream(p).write(input) + close inputStream(p) + + # consider `p.lines(keepNewLines=true)` to avoid exit code test + result = ("", -1) + var line = newStringOfCap(120) while true: if outp.readLine(line): - result[0].string.add(line.string) - result[0].string.add("\n") + result[0].add(line) + result[0].add("\n") else: result[1] = peekExitCode(p) if result[1] != -1: break close(p) - -when isMainModule: - assert quoteShellWindows("aaa") == "aaa" - assert quoteShellWindows("aaa\"") == "aaa\\\"" - assert quoteShellWindows("") == "\"\"" - - assert quoteShellPosix("aaa") == "aaa" - assert quoteShellPosix("aaa a") == "'aaa a'" - assert quoteShellPosix("") == "''" - assert quoteShellPosix("a'a") == "'a'\"'\"'a'" - - when defined(posix): - assert quoteShell("") == "''" diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim deleted file mode 100644 index 000fe25a3..000000000 --- a/lib/pure/oswalkdir.nim +++ /dev/null @@ -1,27 +0,0 @@ - -## Compile-time only version for walkDir if you need it at compile-time -## for JavaScript. - -type - PathComponent* = enum ## Enumeration specifying a path component. - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent, path: string]] = - discard - -iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] = - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - -iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string = - var stack = @[dir] - while stack.len > 0: - for k,p in walkDir(stack.pop()): - if k in filter: - case k - of pcFile, pcLinkToFile: yield p - of pcDir, pcLinkToDir: stack.add(p) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index f680d4fef..8a43daf54 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -7,76 +7,224 @@ # distribution, for details about the copyright. # -## The ``parsecfg`` module implements a high performance configuration file -## parser. The configuration file's syntax is similar to the Windows ``.ini`` +## The `parsecfg` module implements a high performance configuration file +## parser. The configuration file's syntax is similar to the Windows `.ini` ## format, but much more powerful, as it is not a line based parser. String ## literals, raw string literals and triple quoted string literals are supported ## as in the Nim programming language. - -## This is an example of how a configuration file may look like: ## -## .. include:: doc/mytest.cfg +## Example of how a configuration file may look like: +## +## .. include:: ../../doc/mytest.cfg ## :literal: -## The file ``examples/parsecfgex.nim`` demonstrates how to use the -## configuration file parser: ## -## .. code-block:: nim -## :file: examples/parsecfgex.nim +## Here is an example of how to use the configuration file parser: +runnableExamples("-r:off"): + import std/[strutils, streams] + + let configFile = "example.ini" + var f = newFileStream(configFile, fmRead) + assert f != nil, "cannot open " & configFile + var p: CfgParser + open(p, f, configFile) + while true: + var e = next(p) + case e.kind + of cfgEof: break + of cfgSectionStart: ## a `[section]` has been parsed + echo "new section: " & e.section + of cfgKeyValuePair: + echo "key-value-pair: " & e.key & ": " & e.value + of cfgOption: + echo "command: " & e.key & ": " & e.value + of cfgError: + echo e.msg + close(p) + +##[ +## Configuration file example +]## + +## ```none +## charset = "utf-8" +## [Package] +## name = "hello" +## --threads:on +## [Author] +## name = "nim-lang" +## website = "nim-lang.org" +## ``` + +##[ +## Creating a configuration file +]## + +runnableExamples: + var dict = newConfig() + dict.setSectionKey("","charset", "utf-8") + dict.setSectionKey("Package", "name", "hello") + dict.setSectionKey("Package", "--threads", "on") + dict.setSectionKey("Author", "name", "nim-lang") + dict.setSectionKey("Author", "website", "nim-lang.org") + assert $dict == """ +charset=utf-8 +[Package] +name=hello +--threads:on +[Author] +name=nim-lang +website=nim-lang.org +""" + +##[ +## Reading a configuration file +]## + +runnableExamples("-r:off"): + let dict = loadConfig("config.ini") + let charset = dict.getSectionValue("","charset") + let threads = dict.getSectionValue("Package","--threads") + let pname = dict.getSectionValue("Package","name") + let name = dict.getSectionValue("Author","name") + let website = dict.getSectionValue("Author","website") + echo pname & "\n" & name & "\n" & website + +##[ +## Modifying a configuration file +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.setSectionKey("Author", "name", "nim-lang") + dict.writeConfig("config.ini") + +##[ +## Deleting a section key in a configuration file +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.delSectionKey("Author", "website") + dict.writeConfig("config.ini") + +##[ +## Supported INI File structure +]## +# taken from https://docs.python.org/3/library/configparser.html#supported-ini-file-structure +runnableExamples: + import std/streams -import - hashes, strutils, lexbase, streams + var dict = loadConfig(newStringStream("""[Simple Values] + key=value + spaces in keys=allowed + spaces in values=allowed as well + spaces around the delimiter = obviously + you can also use : to delimit keys from values + [All Values Are Strings] + values like this: 19990429 + or this: 3.14159265359 + are they treated as numbers : no + integers floats and booleans are held as: strings + can use the API to get converted values directly: true + [No Values] + key_without_value + # empty string value is not allowed = + [ Seletion A ] + space around section name will be ignored + [You can use comments] + # like this + ; or this + # By default only in an empty line. + # Inline comments can be harmful because they prevent users + # from using the delimiting characters as parts of values. + # That being said, this can be customized. + [Sections Can Be Indented] + can_values_be_as_well = True + does_that_mean_anything_special = False + purpose = formatting for readability + # Did I mention we can indent comments, too? + """) + ) + + let section1 = "Simple Values" + assert dict.getSectionValue(section1, "key") == "value" + assert dict.getSectionValue(section1, "spaces in keys") == "allowed" + assert dict.getSectionValue(section1, "spaces in values") == "allowed as well" + assert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" + assert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" + + let section2 = "All Values Are Strings" + assert dict.getSectionValue(section2, "values like this") == "19990429" + assert dict.getSectionValue(section2, "or this") == "3.14159265359" + assert dict.getSectionValue(section2, "are they treated as numbers") == "no" + assert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" + assert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" + + let section3 = "Seletion A" + assert dict.getSectionValue(section3, + "space around section name will be ignored", "not an empty value") == "" + + let section4 = "Sections Can Be Indented" + assert dict.getSectionValue(section4, "can_values_be_as_well") == "True" + assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" + assert dict.getSectionValue(section4, "purpose") == "formatting for readability" + +import std/[strutils, lexbase, streams, tables] +import std/private/decode_helpers +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/syncio include "system/inclrtl" + type CfgEventKind* = enum ## enumeration of all events that may occur when parsing - cfgEof, ## end of file reached - cfgSectionStart, ## a ``[section]`` has been parsed - cfgKeyValuePair, ## a ``key=value`` pair has been detected - cfgOption, ## a ``--key=value`` command line option - cfgError ## an error occurred during parsing + cfgEof, ## end of file reached + cfgSectionStart, ## a `[section]` has been parsed + cfgKeyValuePair, ## a `key=value` pair has been detected + cfgOption, ## a `--key=value` command line option + cfgError ## an error occurred during parsing CfgEvent* = object of RootObj ## describes a parsing event case kind*: CfgEventKind ## the kind of the event of cfgEof: nil of cfgSectionStart: - section*: string ## `section` contains the name of the - ## parsed section start (syntax: ``[section]``) + section*: string ## `section` contains the name of the + ## parsed section start (syntax: `[section]`) of cfgKeyValuePair, cfgOption: - key*, value*: string ## contains the (key, value) pair if an option - ## of the form ``--key: value`` or an ordinary - ## ``key= value`` pair has been parsed. - ## ``value==""`` if it was not specified in the - ## configuration file. - of cfgError: ## the parser encountered an error: `msg` - msg*: string ## contains the error message. No exceptions - ## are thrown if a parse error occurs. + key*, value*: string ## contains the (key, value) pair if an option + ## of the form `--key: value` or an ordinary + ## `key= value` pair has been parsed. + ## `value==""` if it was not specified in the + ## configuration file. + of cfgError: ## the parser encountered an error: `msg` + msg*: string ## contains the error message. No exceptions + ## are thrown if a parse error occurs. TokKind = enum tkInvalid, tkEof, tkSymbol, tkEquals, tkColon, tkBracketLe, tkBracketRi, tkDashDash - Token = object # a token - kind: TokKind # the type of the token - literal: string # the parsed (string) literal + Token = object # a token + kind: TokKind # the type of the token + literal: string # the parsed (string) literal CfgParser* = object of BaseLexer ## the parser object. tok: Token filename: string -{.deprecated: [TCfgEventKind: CfgEventKind, TCfgEvent: CfgEvent, - TTokKind: TokKind, TToken: Token, TCfgParser: CfgParser].} - # implementation const - SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\'} + SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', ' ', '\x80'..'\xFF', '.', '/', '\\', '-'} proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe.} proc open*(c: var CfgParser, input: Stream, filename: string, lineOffset = 0) {.rtl, extern: "npc$1".} = - ## initializes the parser with an input stream. `Filename` is only used + ## Initializes the parser with an input stream. `Filename` is only used ## for nice error messages. `lineOffset` can be used to influence the line ## number information in the generated error messages. lexbase.open(c, input) @@ -87,42 +235,28 @@ proc open*(c: var CfgParser, input: Stream, filename: string, rawGetTok(c, c.tok) proc close*(c: var CfgParser) {.rtl, extern: "npc$1".} = - ## closes the parser `c` and its associated input stream. + ## Closes the parser `c` and its associated input stream. lexbase.close(c) proc getColumn*(c: CfgParser): int {.rtl, extern: "npc$1".} = - ## get the current column the parser has arrived at. + ## Gets the current column the parser has arrived at. result = getColNumber(c, c.bufpos) proc getLine*(c: CfgParser): int {.rtl, extern: "npc$1".} = - ## get the current line the parser has arrived at. + ## Gets the current line the parser has arrived at. result = c.lineNumber proc getFilename*(c: CfgParser): string {.rtl, extern: "npc$1".} = - ## get the filename of the file that the parser processes. + ## Gets the filename of the file that the parser processes. result = c.filename -proc handleHexChar(c: var CfgParser, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: - discard - proc handleDecChars(c: var CfgParser, xi: var int) = while c.buf[c.bufpos] in {'0'..'9'}: xi = (xi * 10) + (ord(c.buf[c.bufpos]) - ord('0')) inc(c.bufpos) proc getEscapedChar(c: var CfgParser, tok: var Token) = - inc(c.bufpos) # skip '\' + inc(c.bufpos) # skip '\' case c.buf[c.bufpos] of 'n', 'N': add(tok.literal, "\n") @@ -160,8 +294,10 @@ proc getEscapedChar(c: var CfgParser, tok: var Token) = of 'x', 'X': inc(c.bufpos) var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) add(tok.literal, chr(xi)) of '0'..'9': var xi = 0 @@ -177,38 +313,35 @@ proc handleCRLF(c: var CfgParser, pos: int): int = else: result = pos proc getString(c: var CfgParser, tok: var Token, rawMode: bool) = - var pos = c.bufpos + 1 # skip " - var buf = c.buf # put `buf` in a register + var pos = c.bufpos + 1 # skip " tok.kind = tkSymbol - if (buf[pos] == '"') and (buf[pos + 1] == '"'): + if (c.buf[pos] == '"') and (c.buf[pos + 1] == '"'): # long string literal: - inc(pos, 2) # skip "" + inc(pos, 2) # skip "" # skip leading newline: pos = handleCRLF(c, pos) - buf = c.buf while true: - case buf[pos] + case c.buf[pos] of '"': - if (buf[pos + 1] == '"') and (buf[pos + 2] == '"'): break + if (c.buf[pos + 1] == '"') and (c.buf[pos + 2] == '"'): break add(tok.literal, '"') inc(pos) of '\c', '\L': pos = handleCRLF(c, pos) - buf = c.buf add(tok.literal, "\n") of lexbase.EndOfFile: tok.kind = tkInvalid break else: - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) - c.bufpos = pos + 3 # skip the three """ + c.bufpos = pos + 3 # skip the three """ else: # ordinary string literal while true: - var ch = buf[pos] + var ch = c.buf[pos] if ch == '"': - inc(pos) # skip '"' + inc(pos) # skip '"' break if ch in {'\c', '\L', lexbase.EndOfFile}: tok.kind = tkInvalid @@ -224,28 +357,29 @@ proc getString(c: var CfgParser, tok: var Token, rawMode: bool) = proc getSymbol(c: var CfgParser, tok: var Token) = var pos = c.bufpos - var buf = c.buf while true: - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) - if not (buf[pos] in SymChars): break + if not (c.buf[pos] in SymChars): break + + while tok.literal.len > 0 and tok.literal[^1] == ' ': + tok.literal.setLen(tok.literal.len - 1) + c.bufpos = pos tok.kind = tkSymbol proc skip(c: var CfgParser) = var pos = c.bufpos - var buf = c.buf while true: - case buf[pos] + case c.buf[pos] of ' ', '\t': inc(pos) of '#', ';': - while not (buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos) + while not (c.buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos) of '\c', '\L': pos = handleCRLF(c, pos) - buf = c.buf else: - break # EndOfFile also leaves the loop + break # EndOfFile also leaves the loop c.bufpos = pos proc rawGetTok(c: var CfgParser, tok: var Token) = @@ -259,9 +393,13 @@ proc rawGetTok(c: var CfgParser, tok: var Token) = tok.literal = "=" of '-': inc(c.bufpos) - if c.buf[c.bufpos] == '-': inc(c.bufpos) - tok.kind = tkDashDash - tok.literal = "--" + if c.buf[c.bufpos] == '-': + inc(c.bufpos) + tok.kind = tkDashDash + tok.literal = "--" + else: + dec(c.bufpos) + getSymbol(c, tok) of ':': tok.kind = tkColon inc(c.bufpos) @@ -275,7 +413,7 @@ proc rawGetTok(c: var CfgParser, tok: var Token) = of '[': tok.kind = tkBracketLe inc(c.bufpos) - tok.literal = "]" + tok.literal = "[" of ']': tok.kind = tkBracketRi inc(c.bufpos) @@ -288,19 +426,19 @@ proc rawGetTok(c: var CfgParser, tok: var Token) = else: getSymbol(c, tok) proc errorStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} = - ## returns a properly formated error message containing current line and + ## Returns a properly formatted error message containing current line and ## column information. result = `%`("$1($2, $3) Error: $4", - [c.filename, $getLine(c), $getColumn(c), msg]) + [c.filename, $getLine(c), $getColumn(c), msg]) proc warningStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} = - ## returns a properly formated warning message containing current line and + ## Returns a properly formatted warning message containing current line and ## column information. result = `%`("$1($2, $3) Warning: $4", - [c.filename, $getLine(c), $getColumn(c), msg]) + [c.filename, $getLine(c), $getColumn(c), msg]) proc ignoreMsg*(c: CfgParser, e: CfgEvent): string {.rtl, extern: "npc$1".} = - ## returns a properly formated warning message containing that + ## Returns a properly formatted warning message containing that ## an entry is ignored. case e.kind of cfgSectionStart: result = c.warningStr("section ignored: " & e.section) @@ -312,29 +450,29 @@ proc ignoreMsg*(c: CfgParser, e: CfgEvent): string {.rtl, extern: "npc$1".} = proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent = if c.tok.kind == tkSymbol: - result.kind = kind - result.key = c.tok.literal - result.value = "" + case kind + of cfgOption, cfgKeyValuePair: + result = CfgEvent(kind: kind, key: c.tok.literal.move, value: "") + else: discard rawGetTok(c, c.tok) if c.tok.kind in {tkEquals, tkColon}: rawGetTok(c, c.tok) if c.tok.kind == tkSymbol: result.value = c.tok.literal else: - reset result - result.kind = cfgError - result.msg = errorStr(c, "symbol expected, but found: " & c.tok.literal) + result = CfgEvent(kind: cfgError, + msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) rawGetTok(c, c.tok) else: - result.kind = cfgError - result.msg = errorStr(c, "symbol expected, but found: " & c.tok.literal) + result = CfgEvent(kind: cfgError, + msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) rawGetTok(c, c.tok) proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = - ## retrieves the first/next event. This controls the parser. + ## Retrieves the first/next event. This controls the parser. case c.tok.kind of tkEof: - result.kind = cfgEof + result = CfgEvent(kind: cfgEof) of tkDashDash: rawGetTok(c, c.tok) result = getKeyValPair(c, cfgOption) @@ -343,19 +481,179 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = of tkBracketLe: rawGetTok(c, c.tok) if c.tok.kind == tkSymbol: - result.kind = cfgSectionStart - result.section = c.tok.literal + result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal.move) else: - result.kind = cfgError - result.msg = errorStr(c, "symbol expected, but found: " & c.tok.literal) + result = CfgEvent(kind: cfgError, + msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) rawGetTok(c, c.tok) if c.tok.kind == tkBracketRi: rawGetTok(c, c.tok) else: - reset(result) - result.kind = cfgError - result.msg = errorStr(c, "']' expected, but found: " & c.tok.literal) + result = CfgEvent(kind: cfgError, + msg: errorStr(c, "']' expected, but found: " & c.tok.literal)) of tkInvalid, tkEquals, tkColon, tkBracketRi: - result.kind = cfgError - result.msg = errorStr(c, "invalid token: " & c.tok.literal) + result = CfgEvent(kind: cfgError, + msg: errorStr(c, "invalid token: " & c.tok.literal)) rawGetTok(c, c.tok) + +# ---------------- Configuration file related operations ---------------- +type + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] + +proc newConfig*(): Config = + ## Creates a new configuration table. + ## Useful when wanting to create a configuration file. + result = newOrderedTable[string, OrderedTableRef[string, string]]() + +proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = + ## Loads the specified configuration from stream into a new Config instance. + ## `filename` parameter is only used for nicer error messages. + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() + var curSection = "" ## Current section, + ## the default value of the current section is "", + ## which means that the current section is a common + var p: CfgParser + open(p, stream, filename) + while true: + var e = next(p) + case e.kind + of cfgEof: + break + of cfgSectionStart: # Only look for the first time the Section + curSection = e.section + of cfgKeyValuePair: + var t = newOrderedTable[string, string]() + if dict.hasKey(curSection): + t = dict[curSection] + t[e.key] = e.value + dict[curSection] = t + of cfgOption: + var c = newOrderedTable[string, string]() + if dict.hasKey(curSection): + c = dict[curSection] + c["--" & e.key] = e.value + dict[curSection] = c + of cfgError: + break + close(p) + result = dict + +proc loadConfig*(filename: string): Config = + ## Loads the specified configuration file into a new Config instance. + let file = open(filename, fmRead) + let fileStream = newFileStream(file) + defer: fileStream.close() + result = fileStream.loadConfig(filename) + +proc replace(s: string): string = + var d = "" + var i = 0 + while i < s.len(): + if s[i] == '\\': + d.add(r"\\") + elif s[i] == '\c' and s[i+1] == '\l': + d.add(r"\c\l") + inc(i) + elif s[i] == '\c': + d.add(r"\n") + elif s[i] == '\l': + d.add(r"\n") + else: + d.add(s[i]) + inc(i) + result = d + +proc writeConfig*(dict: Config, stream: Stream) = + ## Writes the contents of the table to the specified stream. + ## + ## .. note:: Comment statement will be ignored. + for section, sectionData in dict.pairs(): + if section != "": ## Not general section + if not allCharsInSet(section, SymChars): ## Non system character + stream.writeLine("[\"" & section & "\"]") + else: + stream.writeLine("[" & section & "]") + for key, value in sectionData.pairs(): + var kv, segmentChar: string + if key.len > 1 and key[0] == '-' and key[1] == '-': ## If it is a command key + segmentChar = ":" + if not allCharsInSet(key[2..key.len()-1], SymChars): + kv.add("--\"") + kv.add(key[2..key.len()-1]) + kv.add("\"") + else: + kv = key + else: + segmentChar = "=" + kv = key + if value != "": ## If the key is not empty + if not allCharsInSet(value, SymChars): + if find(value, '"') == -1: + kv.add(segmentChar) + kv.add("\"") + kv.add(replace(value)) + kv.add("\"") + else: + kv.add(segmentChar) + kv.add("\"\"\"") + kv.add(replace(value)) + kv.add("\"\"\"") + else: + kv.add(segmentChar) + kv.add(value) + stream.writeLine(kv) + +proc `$`*(dict: Config): string = + ## Writes the contents of the table to string. + ## + ## .. note:: Comment statement will be ignored. + let stream = newStringStream() + defer: stream.close() + dict.writeConfig(stream) + result = stream.data + +proc writeConfig*(dict: Config, filename: string) = + ## Writes the contents of the table to the specified configuration file. + ## + ## .. note:: Comment statement will be ignored. + let file = open(filename, fmWrite) + defer: file.close() + let fileStream = newFileStream(file) + dict.writeConfig(fileStream) + +proc getSectionValue*(dict: Config, section, key: string, defaultVal = ""): string = + ## Gets the key value of the specified Section. + ## Returns the specified default value if the specified key does not exist. + if dict.hasKey(section): + if dict[section].hasKey(key): + result = dict[section][key] + else: + result = defaultVal + else: + result = defaultVal + +proc setSectionKey*(dict: var Config, section, key, value: string) = + ## Sets the Key value of the specified Section. + var t = newOrderedTable[string, string]() + if dict.hasKey(section): + t = dict[section] + t[key] = value + dict[section] = t + +proc delSection*(dict: var Config, section: string) = + ## Deletes the specified section and all of its sub keys. + dict.del(section) + +proc delSectionKey*(dict: var Config, section, key: string) = + ## Deletes the key of the specified section. + if dict.hasKey(section): + if dict[section].hasKey(key): + if dict[section].len == 1: + dict.del(section) + else: + dict[section].del(key) + +iterator sections*(dict: Config): lent string {.since: (1, 5).} = + ## Iterates through the sections in the `dict`. + for section in dict.keys: + yield section diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index af51e1201..c7bf0c9c1 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -10,13 +10,18 @@ ## This module implements a simple high performance `CSV`:idx: ## (`comma separated value`:idx:) parser. ## -## Example: How to use the parser -## ============================== +## Basic usage +## =========== +## +## ```nim +## import std/parsecsv +## from std/os import paramStr +## from std/streams import newFileStream ## -## .. code-block:: nim -## import os, parsecsv, streams ## var s = newFileStream(paramStr(1), fmRead) -## if s == nil: quit("cannot open the file" & paramStr(1)) +## if s == nil: +## quit("cannot open the file" & paramStr(1)) +## ## var x: CsvParser ## open(x, s, paramStr(1)) ## while readRow(x): @@ -24,150 +29,327 @@ ## for val in items(x.row): ## echo "##", val, "##" ## close(x) +## ``` +## +## For CSV files with a header row, the header can be read and then used as a +## reference for item access with `rowEntry <#rowEntry,CsvParser,string>`_: +## +## ```nim +## import std/parsecsv +## +## # Prepare a file +## let content = """One,Two,Three,Four +## 1,2,3,4 +## 10,20,30,40 +## 100,200,300,400 +## """ +## writeFile("temp.csv", content) +## +## var p: CsvParser +## p.open("temp.csv") +## p.readHeaderRow() +## while p.readRow(): +## echo "new row: " +## for col in items(p.headers): +## echo "##", col, ":", p.rowEntry(col), "##" +## p.close() +## ``` +## +## See also +## ======== ## +## * `streams module <streams.html>`_ for using +## `open proc <#open,CsvParser,Stream,string,char,char,char>`_ +## and other stream processing (like `close proc <streams.html#close,Stream>`_) +## * `parseopt module <parseopt.html>`_ for a command line parser +## * `parsecfg module <parsecfg.html>`_ for a configuration file parser +## * `parsexml module <parsexml.html>`_ for a XML / HTML parser +## * `parsesql module <parsesql.html>`_ for a SQL parser +## * `other parsers <lib.html#pure-libraries-parsers>`_ for other parsers -import - lexbase, streams +import std/[lexbase, streams] + +when defined(nimPreviewSlimSystem): + import std/syncio type - CsvRow* = seq[string] ## a row in a CSV file - CsvParser* = object of BaseLexer ## the parser object. - row*: CsvRow ## the current row + CsvRow* = seq[string] ## A row in a CSV file. + CsvParser* = object of BaseLexer ## The parser object. + ## + ## It consists of two public fields: + ## * `row` is the current row + ## * `headers` are the columns that are defined in the csv file + ## (read using `readHeaderRow <#readHeaderRow,CsvParser>`_). + ## Used with `rowEntry <#rowEntry,CsvParser,string>`_). + row*: CsvRow filename: string sep, quote, esc: char skipWhite: bool currRow: int + headers*: seq[string] - CsvError* = object of IOError ## exception that is raised if - ## a parsing error occurs - -{.deprecated: [TCsvRow: CsvRow, TCsvParser: CsvParser, EInvalidCsv: CsvError].} + CsvError* = object of IOError ## An exception that is raised if + ## a parsing error occurs. proc raiseEInvalidCsv(filename: string, line, col: int, msg: string) {.noreturn.} = var e: ref CsvError new(e) - e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg + if filename.len == 0: + e.msg = "Error: " & msg + else: + e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg raise e -proc error(my: CsvParser, pos: int, msg: string) = - raiseEInvalidCsv(my.filename, my.lineNumber, getColNumber(my, pos), msg) +proc error(self: CsvParser, pos: int, msg: string) = + raiseEInvalidCsv(self.filename, self.lineNumber, getColNumber(self, pos), msg) -proc open*(my: var CsvParser, input: Stream, filename: string, +proc open*(self: var CsvParser, input: Stream, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = - ## initializes the parser with an input stream. `Filename` is only used + ## Initializes the parser with an input stream. `Filename` is only used ## for nice error messages. The parser's behaviour can be controlled by ## the diverse optional parameters: ## - `separator`: character used to separate fields ## - `quote`: Used to quote fields containing special characters like - ## `separator`, `quote` or new-line characters. '\0' disables the parsing + ## `separator`, `quote` or new-line characters. '\\0' disables the parsing ## of quotes. ## - `escape`: removes any special meaning from the following character; - ## '\0' disables escaping; if escaping is disabled and `quote` is not '\0', + ## '\\0' disables escaping; if escaping is disabled and `quote` is not '\\0', ## two `quote` characters are parsed one literal `quote` character. ## - `skipInitialSpace`: If true, whitespace immediately following the ## `separator` is ignored. - lexbase.open(my, input) - my.filename = filename - my.sep = separator - my.quote = quote - my.esc = escape - my.skipWhite = skipInitialSpace - my.row = @[] - my.currRow = 0 - -proc parseField(my: var CsvParser, a: var string) = - var pos = my.bufpos - var buf = my.buf - if my.skipWhite: - while buf[pos] in {' ', '\t'}: inc(pos) + ## + ## See also: + ## * `open proc <#open,CsvParser,string,char,char,char>`_ which creates the + ## file stream for you + runnableExamples: + import std/streams + var strm = newStringStream("One,Two,Three\n1,2,3\n10,20,30") + var parser: CsvParser + parser.open(strm, "tmp.csv") + parser.close() + strm.close() + + lexbase.open(self, input) + self.filename = filename + self.sep = separator + self.quote = quote + self.esc = escape + self.skipWhite = skipInitialSpace + +proc open*(self: var CsvParser, filename: string, + separator = ',', quote = '"', escape = '\0', + skipInitialSpace = false) = + ## Similar to the `other open proc<#open,CsvParser,Stream,string,char,char,char>`_, + ## but creates the file stream for you. + runnableExamples: + from std/os import removeFile + writeFile("tmp.csv", "One,Two,Three\n1,2,3\n10,20,300") + var parser: CsvParser + parser.open("tmp.csv") + parser.close() + removeFile("tmp.csv") + + var s = newFileStream(filename, fmRead) + if s == nil: self.error(0, "cannot open: " & filename) + open(self, s, filename, separator, + quote, escape, skipInitialSpace) + +proc parseField(self: var CsvParser, a: var string) = + var pos = self.bufpos + if self.skipWhite: + while self.buf[pos] in {' ', '\t'}: inc(pos) setLen(a, 0) # reuse memory - if buf[pos] == my.quote and my.quote != '\0': + if self.buf[pos] == self.quote and self.quote != '\0': inc(pos) while true: - var c = buf[pos] + let c = self.buf[pos] if c == '\0': - my.bufpos = pos # can continue after exception? - error(my, pos, my.quote & " expected") + self.bufpos = pos # can continue after exception? + error(self, pos, self.quote & " expected") break - elif c == my.quote: - if my.esc == '\0' and buf[pos+1] == my.quote: - add(a, my.quote) + elif c == self.quote: + if self.esc == '\0' and self.buf[pos + 1] == self.quote: + add(a, self.quote) inc(pos, 2) else: inc(pos) break - elif c == my.esc: - add(a, buf[pos+1]) + elif c == self.esc: + add(a, self.buf[pos + 1]) inc(pos, 2) else: case c of '\c': - pos = handleCR(my, pos) - buf = my.buf + pos = handleCR(self, pos) add(a, "\n") of '\l': - pos = handleLF(my, pos) - buf = my.buf + pos = handleLF(self, pos) add(a, "\n") else: add(a, c) inc(pos) else: while true: - var c = buf[pos] - if c == my.sep: break + let c = self.buf[pos] + if c == self.sep: break if c in {'\c', '\l', '\0'}: break add(a, c) inc(pos) - my.bufpos = pos + self.bufpos = pos + +proc processedRows*(self: var CsvParser): int {.inline.} = + ## Returns number of the processed rows. + ## + ## But even if `readRow <#readRow,CsvParser,int>`_ arrived at EOF then + ## processed rows counter is incremented. + runnableExamples: + import std/streams -proc processedRows*(my: var CsvParser): int = - ## returns number of the processed rows - return my.currRow + var strm = newStringStream("One,Two,Three\n1,2,3") + var parser: CsvParser + parser.open(strm, "tmp.csv") + doAssert parser.readRow() + doAssert parser.processedRows() == 1 + doAssert parser.readRow() + doAssert parser.processedRows() == 2 + ## Even if `readRow` arrived at EOF then `processedRows` is incremented. + doAssert parser.readRow() == false + doAssert parser.processedRows() == 3 + doAssert parser.readRow() == false + doAssert parser.processedRows() == 4 + parser.close() + strm.close() -proc readRow*(my: var CsvParser, columns = 0): bool = - ## reads the next row; if `columns` > 0, it expects the row to have + self.currRow + +proc readRow*(self: var CsvParser, columns = 0): bool = + ## Reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. + ## + ## Blank lines are skipped. + runnableExamples: + import std/streams + var strm = newStringStream("One,Two,Three\n1,2,3\n\n10,20,30") + var parser: CsvParser + parser.open(strm, "tmp.csv") + doAssert parser.readRow() + doAssert parser.row == @["One", "Two", "Three"] + doAssert parser.readRow() + doAssert parser.row == @["1", "2", "3"] + ## Blank lines are skipped. + doAssert parser.readRow() + doAssert parser.row == @["10", "20", "30"] + + var emptySeq: seq[string] + doAssert parser.readRow() == false + doAssert parser.row == emptySeq + doAssert parser.readRow() == false + doAssert parser.row == emptySeq + + parser.close() + strm.close() + var col = 0 # current column - var oldpos = my.bufpos - while my.buf[my.bufpos] != '\0': - var oldlen = my.row.len - if oldlen < col+1: - setLen(my.row, col+1) - my.row[col] = "" - parseField(my, my.row[col]) + let oldpos = self.bufpos + # skip initial empty lines #8365 + while true: + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) + else: break + while self.buf[self.bufpos] != '\0': + let oldlen = self.row.len + if oldlen < col + 1: + setLen(self.row, col + 1) + self.row[col] = "" + parseField(self, self.row[col]) inc(col) - if my.buf[my.bufpos] == my.sep: - inc(my.bufpos) + if self.buf[self.bufpos] == self.sep: + inc(self.bufpos) else: - case my.buf[my.bufpos] + case self.buf[self.bufpos] of '\c', '\l': # skip empty lines: while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break of '\0': discard - else: error(my, my.bufpos, my.sep & " expected") + else: error(self, self.bufpos, self.sep & " expected") break - setLen(my.row, col) + setLen(self.row, col) result = col > 0 if result and col != columns and columns > 0: - error(my, oldpos+1, $columns & " columns expected, but found " & + error(self, oldpos + 1, $columns & " columns expected, but found " & $col & " columns") - inc(my.currRow) + inc(self.currRow) + +proc close*(self: var CsvParser) {.inline.} = + ## Closes the parser `self` and its associated input stream. + lexbase.close(self) + +proc readHeaderRow*(self: var CsvParser) = + ## Reads the first row and creates a look-up table for column numbers + ## See also: + ## * `rowEntry proc <#rowEntry,CsvParser,string>`_ + runnableExamples: + import std/streams -proc close*(my: var CsvParser) {.inline.} = - ## closes the parser `my` and its associated input stream. - lexbase.close(my) + var strm = newStringStream("One,Two,Three\n1,2,3") + var parser: CsvParser + parser.open(strm, "tmp.csv") + + parser.readHeaderRow() + doAssert parser.headers == @["One", "Two", "Three"] + doAssert parser.row == @["One", "Two", "Three"] + + doAssert parser.readRow() + doAssert parser.headers == @["One", "Two", "Three"] + doAssert parser.row == @["1", "2", "3"] + + parser.close() + strm.close() + + let present = self.readRow() + if present: + self.headers = self.row + +proc rowEntry*(self: var CsvParser, entry: string): var string = + ## Accesses a specified `entry` from the current row. + ## + ## Assumes that `readHeaderRow <#readHeaderRow,CsvParser>`_ has already been + ## called. + ## + ## If specified `entry` does not exist, raises KeyError. + runnableExamples: + import std/streams + var strm = newStringStream("One,Two,Three\n1,2,3\n\n10,20,30") + var parser: CsvParser + parser.open(strm, "tmp.csv") + ## Requires calling `readHeaderRow`. + parser.readHeaderRow() + doAssert parser.readRow() + doAssert parser.rowEntry("One") == "1" + doAssert parser.rowEntry("Two") == "2" + doAssert parser.rowEntry("Three") == "3" + doAssertRaises(KeyError): + discard parser.rowEntry("NonexistentEntry") + parser.close() + strm.close() + + let index = self.headers.find(entry) + if index >= 0: + result = self.row[index] + else: + raise newException(KeyError, "Entry `" & entry & "` doesn't exist") when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: CsvParser @@ -177,4 +359,3 @@ when not defined(testing) and isMainModule: for val in items(x.row): echo "##", val, "##" close(x) - diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim new file mode 100644 index 000000000..9292a8596 --- /dev/null +++ b/lib/pure/parsejson.nim @@ -0,0 +1,522 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a json parser. It is used +## and exported by the `json` standard library +## module, but can also be used in its own right. + +import std/[strutils, lexbase, streams, unicode] +import std/private/decode_helpers + +when defined(nimPreviewSlimSystem): + import std/assertions + +type + JsonEventKind* = enum ## enumeration of all events that may occur when parsing + jsonError, ## an error occurred during parsing + jsonEof, ## end of file reached + jsonString, ## a string literal + jsonInt, ## an integer literal + jsonFloat, ## a float literal + jsonTrue, ## the value `true` + jsonFalse, ## the value `false` + jsonNull, ## the value `null` + jsonObjectStart, ## start of an object: the `{` token + jsonObjectEnd, ## end of an object: the `}` token + jsonArrayStart, ## start of an array: the `[` token + jsonArrayEnd ## end of an array: the `]` token + + TokKind* = enum # must be synchronized with TJsonEventKind! + tkError, + tkEof, + tkString, + tkInt, + tkFloat, + tkTrue, + tkFalse, + tkNull, + tkCurlyLe, + tkCurlyRi, + tkBracketLe, + tkBracketRi, + tkColon, + tkComma + + JsonError* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errInvalidToken, ## invalid token + errStringExpected, ## string expected + errColonExpected, ## `:` expected + errCommaExpected, ## `,` expected + errBracketRiExpected, ## `]` expected + errCurlyRiExpected, ## `}` expected + errQuoteExpected, ## `"` or `'` expected + errEOC_Expected, ## `*/` expected + errEofExpected, ## EOF expected + errExprExpected ## expr expected + + ParserState = enum + stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, + stateExpectObjectComma, stateExpectColon, stateExpectValue + + JsonParser* = object of BaseLexer ## the parser object. + a*: string + tok*: TokKind + kind: JsonEventKind + err: JsonError + state: seq[ParserState] + filename: string + rawStringLiterals: bool + + JsonKindError* = object of ValueError ## raised by the `to` macro if the + ## JSON kind is incorrect. + JsonParsingError* = object of ValueError ## is raised for a JSON error + +const + errorMessages*: array[JsonError, string] = [ + "no error", + "invalid token", + "string expected", + "':' expected", + "',' expected", + "']' expected", + "'}' expected", + "'\"' or \"'\" expected", + "'*/' expected", + "EOF expected", + "expression expected" + ] + tokToStr: array[TokKind, string] = [ + "invalid token", + "EOF", + "string literal", + "int literal", + "float literal", + "true", + "false", + "null", + "{", "}", "[", "]", ":", "," + ] + +proc open*(my: var JsonParser, input: Stream, filename: string; + rawStringLiterals = false) = + ## initializes the parser with an input stream. `Filename` is only used + ## for nice error messages. If `rawStringLiterals` is true, string literals + ## are kept with their surrounding quotes and escape sequences in them are + ## left untouched too. + lexbase.open(my, input) + my.filename = filename + my.state = @[stateStart] + my.kind = jsonError + my.a = "" + my.rawStringLiterals = rawStringLiterals + +proc close*(my: var JsonParser) {.inline.} = + ## closes the parser `my` and its associated input stream. + lexbase.close(my) + +proc str*(my: JsonParser): string {.inline.} = + ## returns the character data for the events: `jsonInt`, `jsonFloat`, + ## `jsonString` + assert(my.kind in {jsonInt, jsonFloat, jsonString}) + return my.a + +proc getInt*(my: JsonParser): BiggestInt {.inline.} = + ## returns the number for the event: `jsonInt` + assert(my.kind == jsonInt) + return parseBiggestInt(my.a) + +proc getFloat*(my: JsonParser): float {.inline.} = + ## returns the number for the event: `jsonFloat` + assert(my.kind == jsonFloat) + return parseFloat(my.a) + +proc kind*(my: JsonParser): JsonEventKind {.inline.} = + ## returns the current event type for the JSON parser + return my.kind + +proc getColumn*(my: JsonParser): int {.inline.} = + ## get the current column the parser has arrived at. + result = getColNumber(my, my.bufpos) + +proc getLine*(my: JsonParser): int {.inline.} = + ## get the current line the parser has arrived at. + result = my.lineNumber + +proc getFilename*(my: JsonParser): string {.inline.} = + ## get the filename of the file that the parser processes. + result = my.filename + +proc errorMsg*(my: JsonParser): string = + ## returns a helpful error message for the event `jsonError` + assert(my.kind == jsonError) + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] + +proc errorMsgExpected*(my: JsonParser, e: string): string = + ## returns an error message "`e` expected" in the same format as the + ## other error messages + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), e & " expected"] + +proc parseEscapedUTF16*(buf: cstring, pos: var int): int = + result = 0 + #UTF-16 escape is always 4 bytes. + for _ in 0..3: + # if char in '0' .. '9', 'a' .. 'f', 'A' .. 'F' + if handleHexChar(buf[pos], result): + inc(pos) + else: + return -1 + +proc parseString(my: var JsonParser): TokKind = + result = tkString + var pos = my.bufpos + 1 + if my.rawStringLiterals: + add(my.a, '"') + while true: + case my.buf[pos] + of '\0': + my.err = errQuoteExpected + result = tkError + break + of '"': + if my.rawStringLiterals: + add(my.a, '"') + inc(pos) + break + of '\\': + if my.rawStringLiterals: + add(my.a, '\\') + case my.buf[pos+1] + of '\\', '"', '\'', '/': + add(my.a, my.buf[pos+1]) + inc(pos, 2) + of 'b': + add(my.a, '\b') + inc(pos, 2) + of 'f': + add(my.a, '\f') + inc(pos, 2) + of 'n': + add(my.a, '\L') + inc(pos, 2) + of 'r': + add(my.a, '\C') + inc(pos, 2) + of 't': + add(my.a, '\t') + inc(pos, 2) + of 'v': + add(my.a, '\v') + inc(pos, 2) + of 'u': + if my.rawStringLiterals: + add(my.a, 'u') + inc(pos, 2) + var pos2 = pos + var r = parseEscapedUTF16(cstring(my.buf), pos) + if r < 0: + my.err = errInvalidToken + break + # Deal with surrogates + if (r and 0xfc00) == 0xd800: + if my.buf[pos] != '\\' or my.buf[pos+1] != 'u': + my.err = errInvalidToken + break + inc(pos, 2) + var s = parseEscapedUTF16(cstring(my.buf), pos) + if (s and 0xfc00) == 0xdc00 and s > 0: + r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) + else: + my.err = errInvalidToken + break + if my.rawStringLiterals: + let length = pos - pos2 + for i in 1 .. length: + if my.buf[pos2] in {'0'..'9', 'A'..'F', 'a'..'f'}: + add(my.a, my.buf[pos2]) + inc pos2 + else: + break + else: + add(my.a, toUTF8(Rune(r))) + else: + # don't bother with the error + add(my.a, my.buf[pos]) + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + add(my.a, '\c') + of '\L': + pos = lexbase.handleLF(my, pos) + add(my.a, '\L') + else: + add(my.a, my.buf[pos]) + inc(pos) + my.bufpos = pos # store back + +proc skip(my: var JsonParser) = + var pos = my.bufpos + while true: + case my.buf[pos] + of '/': + if my.buf[pos+1] == '/': + # skip line comment: + inc(pos, 2) + while true: + case my.buf[pos] + of '\0': + break + of '\c': + pos = lexbase.handleCR(my, pos) + break + of '\L': + pos = lexbase.handleLF(my, pos) + break + else: + inc(pos) + elif my.buf[pos+1] == '*': + # skip long comment: + inc(pos, 2) + while true: + case my.buf[pos] + of '\0': + my.err = errEOC_Expected + break + of '\c': + pos = lexbase.handleCR(my, pos) + of '\L': + pos = lexbase.handleLF(my, pos) + of '*': + inc(pos) + if my.buf[pos] == '/': + inc(pos) + break + else: + inc(pos) + else: + break + of ' ', '\t': + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + of '\L': + pos = lexbase.handleLF(my, pos) + else: + break + my.bufpos = pos + +proc parseNumber(my: var JsonParser) = + var pos = my.bufpos + if my.buf[pos] == '-': + add(my.a, '-') + inc(pos) + if my.buf[pos] == '.': + add(my.a, "0.") + inc(pos) + else: + while my.buf[pos] in Digits: + add(my.a, my.buf[pos]) + inc(pos) + if my.buf[pos] == '.': + add(my.a, '.') + inc(pos) + # digits after the dot: + while my.buf[pos] in Digits: + add(my.a, my.buf[pos]) + inc(pos) + if my.buf[pos] in {'E', 'e'}: + add(my.a, my.buf[pos]) + inc(pos) + if my.buf[pos] in {'+', '-'}: + add(my.a, my.buf[pos]) + inc(pos) + while my.buf[pos] in Digits: + add(my.a, my.buf[pos]) + inc(pos) + my.bufpos = pos + +proc parseName(my: var JsonParser) = + var pos = my.bufpos + if my.buf[pos] in IdentStartChars: + while my.buf[pos] in IdentChars: + add(my.a, my.buf[pos]) + inc(pos) + my.bufpos = pos + +proc getTok*(my: var JsonParser): TokKind = + setLen(my.a, 0) + skip(my) # skip whitespace, comments + case my.buf[my.bufpos] + of '-', '.', '0'..'9': + parseNumber(my) + if {'.', 'e', 'E'} in my.a: + result = tkFloat + else: + result = tkInt + of '"': + result = parseString(my) + of '[': + inc(my.bufpos) + result = tkBracketLe + of '{': + inc(my.bufpos) + result = tkCurlyLe + of ']': + inc(my.bufpos) + result = tkBracketRi + of '}': + inc(my.bufpos) + result = tkCurlyRi + of ',': + inc(my.bufpos) + result = tkComma + of ':': + inc(my.bufpos) + result = tkColon + of '\0': + result = tkEof + of 'a'..'z', 'A'..'Z', '_': + parseName(my) + case my.a + of "null": result = tkNull + of "true": result = tkTrue + of "false": result = tkFalse + else: result = tkError + else: + inc(my.bufpos) + result = tkError + my.tok = result + + +proc next*(my: var JsonParser) = + ## retrieves the first/next event. This controls the parser. + var tk = getTok(my) + var i = my.state.len-1 + # the following code is a state machine. If we had proper coroutines, + # the code could be much simpler. + case my.state[i] + of stateEof: + if tk == tkEof: + my.kind = jsonEof + else: + my.kind = jsonError + my.err = errEofExpected + of stateStart: + # tokens allowed? + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state[i] = stateEof # expect EOF next! + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateArray) # we expect any + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkEof: + my.kind = jsonEof + else: + my.kind = jsonError + my.err = errEofExpected + of stateObject: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state.add(stateExpectColon) + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateExpectColon) + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateExpectColon) + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkCurlyRi: + my.kind = jsonObjectEnd + discard my.state.pop() + else: + my.kind = jsonError + my.err = errCurlyRiExpected + of stateArray: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state.add(stateExpectArrayComma) # expect value next! + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state.add(stateExpectArrayComma) + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state.add(stateExpectArrayComma) + my.state.add(stateObject) + my.kind = jsonObjectStart + of tkBracketRi: + my.kind = jsonArrayEnd + discard my.state.pop() + else: + my.kind = jsonError + my.err = errBracketRiExpected + of stateExpectArrayComma: + case tk + of tkComma: + discard my.state.pop() + next(my) + of tkBracketRi: + my.kind = jsonArrayEnd + discard my.state.pop() # pop stateExpectArrayComma + discard my.state.pop() # pop stateArray + else: + my.kind = jsonError + my.err = errBracketRiExpected + of stateExpectObjectComma: + case tk + of tkComma: + discard my.state.pop() + next(my) + of tkCurlyRi: + my.kind = jsonObjectEnd + discard my.state.pop() # pop stateExpectObjectComma + discard my.state.pop() # pop stateObject + else: + my.kind = jsonError + my.err = errCurlyRiExpected + of stateExpectColon: + case tk + of tkColon: + my.state[i] = stateExpectValue + next(my) + else: + my.kind = jsonError + my.err = errColonExpected + of stateExpectValue: + case tk + of tkString, tkInt, tkFloat, tkTrue, tkFalse, tkNull: + my.state[i] = stateExpectObjectComma + my.kind = JsonEventKind(ord(tk)) + of tkBracketLe: + my.state[i] = stateExpectObjectComma + my.state.add(stateArray) + my.kind = jsonArrayStart + of tkCurlyLe: + my.state[i] = stateExpectObjectComma + my.state.add(stateObject) + my.kind = jsonObjectStart + else: + my.kind = jsonError + my.err = errExprExpected + +proc raiseParseErr*(p: JsonParser, msg: string) {.noinline, noreturn.} = + ## raises an `EJsonParsingError` exception. + raise newException(JsonParsingError, errorMsgExpected(p, msg)) + +proc eat*(p: var JsonParser, tok: TokKind) = + if p.tok == tok: discard getTok(p) + else: raiseParseErr(p, tokToStr[tok]) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 218f5ab81..03f151b66 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -11,168 +11,515 @@ ## It supports one convenience iterator over all command line options and some ## lower-level features. ## -## Supported syntax: +## Supported Syntax +## ================ ## -## 1. short options - ``-abcd``, where a, b, c, d are names -## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` -## 3. argument - everything else +## The following syntax is supported when arguments for the `shortNoVal` and +## `longNoVal` parameters, which are +## `described later<#nimshortnoval-and-nimlongnoval>`_, are not provided: +## +## 1. Short options: `-abcd`, `-e:5`, `-e=5` +## 2. Long options: `--foo:bar`, `--foo=bar`, `--foo` +## 3. Arguments: everything that does not start with a `-` +## +## These three kinds of tokens are enumerated in the +## `CmdLineKind enum<#CmdLineKind>`_. +## +## When option values begin with ':' or '=', they need to be doubled up (as in +## `--delim::`) or alternated (as in `--delim=:`). +## +## The `--` option, commonly used to denote that every token that follows is +## an argument, is interpreted as a long option, and its name is the empty +## string. +## +## Parsing +## ======= +## +## Use an `OptParser<#OptParser>`_ to parse command line options. It can be +## created with `initOptParser<#initOptParser,string,set[char],seq[string]>`_, +## and `next<#next,OptParser>`_ advances the parser by one token. +## +## For each token, the parser's `kind`, `key`, and `val` fields give +## information about that token. If the token is a long or short option, `key` +## is the option's name, and `val` is either the option's value, if provided, +## or the empty string. For arguments, the `key` field contains the argument +## itself, and `val` is unused. To check if the end of the command line has +## been reached, check if `kind` is equal to `cmdEnd`. +## +## Here is an example: +## +## ```Nim +## import std/parseopt +## +## var p = initOptParser("-ab -e:5 --foo --bar=20 file.txt") +## while true: +## p.next() +## case p.kind +## of cmdEnd: break +## of cmdShortOption, cmdLongOption: +## if p.val == "": +## echo "Option: ", p.key +## else: +## echo "Option and value: ", p.key, ", ", p.val +## of cmdArgument: +## echo "Argument: ", p.key +## +## # Output: +## # Option: a +## # Option: b +## # Option and value: e, 5 +## # Option: foo +## # Option and value: bar, 20 +## # Argument: file.txt +## ``` +## +## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for +## convenience, can be used to iterate through all command line options as well. +## +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. +## +## Here is an example: +## +## ```Nim +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName:<value> in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ``` +## +## `shortNoVal` and `longNoVal` +## ============================ +## +## The optional `shortNoVal` and `longNoVal` parameters present in +## `initOptParser<#initOptParser,string,set[char],seq[string]>`_ are for +## specifying which short and long options do not accept values. +## +## When `shortNoVal` is non-empty, users are not required to separate short +## options and their values with a ':' or '=' since the parser knows which +## options accept values and which ones do not. This behavior also applies for +## long options if `longNoVal` is non-empty. For short options, `-j4` +## becomes supported syntax, and for long options, `--foo bar` becomes +## supported. This is in addition to the `previously mentioned +## syntax<#supported-syntax>`_. Users can still separate options and their +## values with ':' or '=', but that becomes optional. +## +## As more options which do not accept values are added to your program, +## remember to amend `shortNoVal` and `longNoVal` accordingly. +## +## The following example illustrates the difference between having an empty +## `shortNoVal` and `longNoVal`, which is the default, and providing +## arguments for those two parameters: +## +## ```Nim +## import std/parseopt +## +## proc printToken(kind: CmdLineKind, key: string, val: string) = +## case kind +## of cmdEnd: doAssert(false) # Doesn't happen with getopt() +## of cmdShortOption, cmdLongOption: +## if val == "": +## echo "Option: ", key +## else: +## echo "Option and value: ", key, ", ", val +## of cmdArgument: +## echo "Argument: ", key +## +## let cmdLine = "-j4 --first bar" +## +## var emptyNoVal = initOptParser(cmdLine) +## for kind, key, val in emptyNoVal.getopt(): +## printToken(kind, key, val) +## +## # Output: +## # Option: j +## # Option: 4 +## # Option: first +## # Argument: bar +## +## var withNoVal = initOptParser(cmdLine, shortNoVal = {'c'}, +## longNoVal = @["second"]) +## for kind, key, val in withNoVal.getopt(): +## printToken(kind, key, val) +## +## # Output: +## # Option and value: j, 4 +## # Option and value: first, bar +## ``` +## +## See also +## ======== +## +## * `os module<os.html>`_ for lower-level command line parsing procs +## * `parseutils module<parseutils.html>`_ for helpers that parse tokens, +## numbers, identifiers, etc. +## * `strutils module<strutils.html>`_ for common string handling operations +## * `json module<json.html>`_ for a JSON parser +## * `parsecfg module<parsecfg.html>`_ for a configuration file parser +## * `parsecsv module<parsecsv.html>`_ for a simple CSV (comma separated value) +## parser +## * `parsexml module<parsexml.html>`_ for a XML / HTML parser +## * `other parsers<lib.html#pure-libraries-parsers>`_ for more parsers {.push debugger: off.} include "system/inclrtl" -import - os, strutils +import std/strutils +import std/os type - CmdLineKind* = enum ## the detected command line token - cmdEnd, ## end of command line reached - cmdArgument, ## argument detected - cmdLongOption, ## a long option ``--option`` detected - cmdShortOption ## a short option ``-c`` detected - OptParser* = - object of RootObj ## this object implements the command line parser - cmd: string + CmdLineKind* = enum ## The detected command line token. + cmdEnd, ## End of command line reached + cmdArgument, ## An argument such as a filename + cmdLongOption, ## A long option such as --option + cmdShortOption ## A short option such as -c + OptParser* = object of RootObj ## \ + ## Implementation of the command line parser. + ## + ## To initialize it, use the + ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. pos: int inShortState: bool - kind*: CmdLineKind ## the dected command line token - key*, val*: TaintedString ## key and value pair; ``key`` is the option - ## or the argument, ``value`` is not "" if - ## the option was given a value - -{.deprecated: [TCmdLineKind: CmdLineKind, TOptParser: OptParser].} + allowWhitespaceAfterColon: bool + shortNoVal: set[char] + longNoVal: seq[string] + cmds: seq[string] + idx: int + kind*: CmdLineKind ## The detected command line token + key*, val*: string ## Key and value pair; the key is the option + ## or the argument, and the value is not "" if + ## the option was given a value proc parseWord(s: string, i: int, w: var string, - delim: set[char] = {'\x09', ' ', '\0'}): int = + delim: set[char] = {'\t', ' '}): int = result = i - if s[result] == '\"': + if result < s.len and s[result] == '\"': inc(result) - while not (s[result] in {'\0', '\"'}): + while result < s.len: + if s[result] == '"': + inc result + break add(w, s[result]) inc(result) - if s[result] == '\"': inc(result) else: - while not (s[result] in delim): + while result < s.len and s[result] notin delim: add(w, s[result]) inc(result) -when declared(os.paramCount): - proc quote(s: string): string = - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': - if s[0] == '-': - result = newStringOfCap(s.len) - var i = parseWord(s, 0, result, {'\0', ' ', '\x09', ':', '='}) - if s[i] in {':','='}: - result.add s[i] - inc i - result.add '"' - while i < s.len: - result.add s[i] - inc i - result.add '"' +proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline.len == 0`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## Behavior of the other parameters remains the same as in + ## `initOptParser(string, ...) + ## <#initOptParser,string,set[char],seq[string]>`_. + ## + ## See also: + ## * `getopt iterator<#getopt.i,seq[string],set[char],seq[string]>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"]) + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"], + shortNoVal = {'l'}, longNoVal = @["left"]) + + result.pos = 0 + result.idx = 0 + result.inShortState = false + result.shortNoVal = shortNoVal + result.longNoVal = longNoVal + result.allowWhitespaceAfterColon = allowWhitespaceAfterColon + if cmdline.len != 0: + result.cmds = newSeq[string](cmdline.len) + for i in 0..<cmdline.len: + result.cmds[i] = cmdline[i] + else: + when declared(paramCount): + when defined(nimscript): + var ctr = 0 + var firstNimsFound = false + for i in countup(0, paramCount()): + if firstNimsFound: + result.cmds[ctr] = paramStr(i) + inc ctr, 1 + if paramStr(i).endsWith(".nims") and not firstNimsFound: + firstNimsFound = true + result.cmds = newSeq[string](paramCount()-i) else: - result = '"' & s & '"' - else: - result = s - - # we cannot provide this for NimRtl creation on Posix, because we can't - # access the command line arguments then! - - proc initOptParser*(cmdline = ""): OptParser = - ## inits the option parser. If ``cmdline == ""``, the real command line - ## (as provided by the ``OS`` module) is taken. - result.pos = 0 - result.inShortState = false - if cmdline != "": - result.cmd = cmdline + result.cmds = newSeq[string](paramCount()) + for i in countup(1, paramCount()): + result.cmds[i-1] = paramStr(i) else: - result.cmd = "" - for i in countup(1, paramCount()): - result.cmd.add quote(paramStr(i).string) - result.cmd.add ' ' - result.kind = cmdEnd - result.key = TaintedString"" - result.val = TaintedString"" - -proc handleShortOption(p: var OptParser) = + # we cannot provide this for NimRtl creation on Posix, because we can't + # access the command line arguments then! + raiseAssert "empty command line given but" & + " real command line is not accessible" + result.kind = cmdEnd + result.key = "" + result.val = "" + +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline == ""`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) + + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) + +proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption - add(p.key.string, p.cmd[i]) - inc(i) + if i < cmd.len: + add(p.key, cmd[i]) + inc(i) p.inShortState = true - while p.cmd[i] in {'\x09', ' '}: + while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) p.inShortState = false - if p.cmd[i] in {':', '='}: - inc(i) + if i < cmd.len and (cmd[i] in {':', '='} or + card(p.shortNoVal) > 0 and p.key[0] notin p.shortNoVal): + if i < cmd.len and cmd[i] in {':', '='}: + inc(i) p.inShortState = false - while p.cmd[i] in {'\x09', ' '}: inc(i) - i = parseWord(p.cmd, i, p.val.string) - if p.cmd[i] == '\0': p.inShortState = false - p.pos = i + while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) + p.val = substr(cmd, i) + p.pos = 0 + inc p.idx + else: + p.pos = i + if i >= cmd.len: + p.inShortState = false + p.pos = 0 + inc p.idx proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = - ## parses the first or next option; ``p.kind`` describes what token has been - ## parsed. ``p.key`` and ``p.val`` are set accordingly. + ## Parses the next token. + ## + ## `p.kind` describes what kind of token has been parsed. `p.key` and + ## `p.val` are set accordingly. + runnableExamples: + var p = initOptParser("--left -r:2 file.txt") + p.next() + doAssert p.kind == cmdLongOption and p.key == "left" + p.next() + doAssert p.kind == cmdShortOption and p.key == "r" and p.val == "2" + p.next() + doAssert p.kind == cmdArgument and p.key == "file.txt" + p.next() + doAssert p.kind == cmdEnd + + if p.idx >= p.cmds.len: + p.kind = cmdEnd + return + var i = p.pos - while p.cmd[i] in {'\x09', ' '}: inc(i) + while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) p.pos = i - setLen(p.key.string, 0) - setLen(p.val.string, 0) + setLen(p.key, 0) + setLen(p.val, 0) if p.inShortState: - handleShortOption(p) - return - case p.cmd[i] - of '\0': - p.kind = cmdEnd - of '-': + p.inShortState = false + if i >= p.cmds[p.idx].len: + inc(p.idx) + p.pos = 0 + if p.idx >= p.cmds.len: + p.kind = cmdEnd + return + else: + handleShortOption(p, p.cmds[p.idx]) + return + + if i < p.cmds[p.idx].len and p.cmds[p.idx][i] == '-': inc(i) - if p.cmd[i] == '-': - p.kind = cmdLongoption + if i < p.cmds[p.idx].len and p.cmds[p.idx][i] == '-': + p.kind = cmdLongOption inc(i) - i = parseWord(p.cmd, i, p.key.string, {'\0', ' ', '\x09', ':', '='}) - while p.cmd[i] in {'\x09', ' '}: inc(i) - if p.cmd[i] in {':', '='}: + i = parseWord(p.cmds[p.idx], i, p.key, {' ', '\t', ':', '='}) + while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) + if i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {':', '='}: inc(i) - while p.cmd[i] in {'\x09', ' '}: inc(i) - p.pos = parseWord(p.cmd, i, p.val.string) + while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) + # if we're at the end, use the next command line option: + if i >= p.cmds[p.idx].len and p.idx < p.cmds.len and + p.allowWhitespaceAfterColon: + inc p.idx + i = 0 + if p.idx < p.cmds.len: + p.val = p.cmds[p.idx].substr(i) + elif len(p.longNoVal) > 0 and p.key notin p.longNoVal and p.idx+1 < p.cmds.len: + p.val = p.cmds[p.idx+1] + inc p.idx else: - p.pos = i + p.val = "" + inc p.idx + p.pos = 0 else: p.pos = i - handleShortOption(p) + handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.pos = parseWord(p.cmd, i, p.key.string) - -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} = - ## retrieves the rest of the command line that has not been parsed yet. - result = strip(substr(p.cmd, p.pos, len(p.cmd) - 1)).TaintedString + p.key = p.cmds[p.idx] + inc p.idx + p.pos = 0 -when declared(initOptParser): - iterator getopt*(): tuple[kind: CmdLineKind, key, val: TaintedString] = - ## This is an convenience iterator for iterating over the command line. - ## This uses the OptParser object. Example: +when declared(quoteShellCommand): + proc cmdLineRest*(p: OptParser): string {.rtl, extern: "npo$1".} = + ## Retrieves the rest of the command line that has not been parsed yet. ## - ## .. code-block:: nim - ## var - ## filename = "" - ## for kind, key, val in getopt(): - ## case kind - ## of cmdArgument: - ## filename = key - ## of cmdLongOption, cmdShortOption: - ## case key - ## of "help", "h": writeHelp() - ## of "version", "v": writeVersion() - ## of cmdEnd: assert(false) # cannot happen - ## if filename == "": - ## # no filename has been given, so we show the help: - ## writeHelp() - var p = initOptParser() - while true: - next(p) - if p.kind == cmdEnd: break - yield (p.kind, p.key, p.val) + ## See also: + ## * `remainingArgs proc<#remainingArgs,OptParser>`_ + ## + ## **Examples:** + ## ```Nim + ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") + ## while true: + ## p.next() + ## if p.kind == cmdLongOption and p.key == "": # Look for "--" + ## break + ## doAssert p.cmdLineRest == "foo.txt bar.txt" + ## ``` + result = p.cmds[p.idx .. ^1].quoteShellCommand + +proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = + ## Retrieves a sequence of the arguments that have not been parsed yet. + ## + ## See also: + ## * `cmdLineRest proc<#cmdLineRest,OptParser>`_ + ## + ## **Examples:** + ## ```Nim + ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") + ## while true: + ## p.next() + ## if p.kind == cmdLongOption and p.key == "": # Look for "--" + ## break + ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] + ## ``` + result = @[] + for i in p.idx..<p.cmds.len: result.add p.cmds[i] + +iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, + val: string] = + ## Convenience iterator for iterating over the given + ## `OptParser<#OptParser>`_. + ## + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. + ## + ## See also: + ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## # these are placeholders, of course + ## proc writeHelp() = discard + ## proc writeVersion() = discard + ## + ## var filename: string + ## var p = initOptParser("--left --debug:3 -l -r:2") + ## + ## for kind, key, val in p.getopt(): + ## case kind + ## of cmdArgument: + ## filename = key + ## of cmdLongOption, cmdShortOption: + ## case key + ## of "help", "h": writeHelp() + ## of "version", "v": writeVersion() + ## of cmdEnd: assert(false) # cannot happen + ## if filename == "": + ## # no filename has been given, so we show the help + ## writeHelp() + ## ``` + p.pos = 0 + p.idx = 0 + while true: + next(p) + if p.kind == cmdEnd: break + yield (p.kind, p.key, p.val) + +iterator getopt*(cmdline: seq[string] = @[], + shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]): + tuple[kind: CmdLineKind, key, val: string] = + ## Convenience iterator for iterating over command line arguments. + ## + ## This creates a new `OptParser<#OptParser>`_. If no command line + ## arguments are provided, the real command line as provided by the + ## `os` module is retrieved instead. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. + ## + ## See also: + ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## # these are placeholders, of course + ## proc writeHelp() = discard + ## proc writeVersion() = discard + ## + ## var filename: string + ## let params = @["--left", "--debug:3", "-l", "-r:2"] + ## + ## for kind, key, val in getopt(params): + ## case kind + ## of cmdArgument: + ## filename = key + ## of cmdLongOption, cmdShortOption: + ## case key + ## of "help", "h": writeHelp() + ## of "version", "v": writeVersion() + ## of cmdEnd: assert(false) # cannot happen + ## if filename == "": + ## # no filename has been written, so we show the help + ## writeHelp() + ## ``` + var p = initOptParser(cmdline, shortNoVal = shortNoVal, + longNoVal = longNoVal) + while true: + next(p) + if p.kind == cmdEnd: break + yield (p.kind, p.key, p.val) {.pop.} diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim deleted file mode 100644 index 7fd9c60fe..000000000 --- a/lib/pure/parseopt2.nim +++ /dev/null @@ -1,152 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module provides the standard Nim command line parser. -## It supports one convenience iterator over all command line options and some -## lower-level features. -## -## Supported syntax: -## -## 1. short options - ``-abcd``, where a, b, c, d are names -## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` -## 3. argument - everything else - -{.push debugger: off.} - -include "system/inclrtl" - -import - os, strutils - -type - CmdLineKind* = enum ## the detected command line token - cmdEnd, ## end of command line reached - cmdArgument, ## argument detected - cmdLongOption, ## a long option ``--option`` detected - cmdShortOption ## a short option ``-c`` detected - OptParser* = - object of RootObj ## this object implements the command line parser - cmd: seq[string] - pos: int - remainingShortOptions: string - kind*: CmdLineKind ## the dected command line token - key*, val*: TaintedString ## key and value pair; ``key`` is the option - ## or the argument, ``value`` is not "" if - ## the option was given a value - -{.deprecated: [TCmdLineKind: CmdLineKind, TOptParser: OptParser].} - -proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} = - ## Initalizes option parses with cmdline. cmdline should not contain - ## argument 0 - program name. - ## If cmdline == nil default to current command line arguments. - result.remainingShortOptions = "" - when not defined(createNimRtl): - if cmdline == nil: - result.cmd = commandLineParams() - return - else: - assert cmdline != nil, "Cannot determine command line arguments." - - result.cmd = @cmdline - -proc initOptParser*(cmdline: string): OptParser {.rtl, deprecated.} = - ## Initalizes option parses with cmdline. Splits cmdline in on spaces - ## and calls initOptParser(openarray[string]) - ## Do not use. - if cmdline == "": # backward compatibility - return initOptParser(seq[string](nil)) - else: - return initOptParser(cmdline.split) - -when not defined(createNimRtl): - proc initOptParser*(): OptParser = - ## Initializes option parser from current command line arguments. - return initOptParser(commandLineParams()) - -proc next*(p: var OptParser) {.rtl, extern: "npo2$1".} - -proc nextOption(p: var OptParser, token: string, allowEmpty: bool) = - for splitchar in [':', '=']: - if splitchar in token: - let pos = token.find(splitchar) - p.key = token[0..pos-1] - p.val = token[pos+1..token.len-1] - return - - p.key = token - if allowEmpty: - p.val = "" - else: - p.remainingShortOptions = token[0..token.len-1] - p.next() - -proc next(p: var OptParser) = - if p.remainingShortOptions.len != 0: - p.kind = cmdShortOption - p.key = TaintedString(p.remainingShortOptions[0..0]) - p.val = "" - p.remainingShortOptions = p.remainingShortOptions[1..p.remainingShortOptions.len-1] - return - - if p.pos >= p.cmd.len: - p.kind = cmdEnd - return - - let token = p.cmd[p.pos] - p.pos += 1 - - if token.startsWith("--"): - p.kind = cmdLongOption - nextOption(p, token[2..token.len-1], allowEmpty=true) - elif token.startsWith("-"): - p.kind = cmdShortOption - nextOption(p, token[1..token.len-1], allowEmpty=true) - else: - p.kind = cmdArgument - p.key = token - p.val = "" - -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecated.} = - ## Returns part of command line string that has not been parsed yet. - ## Do not use - does not correctly handle whitespace. - return p.cmd[p.pos..p.cmd.len-1].join(" ") - -type - GetoptResult* = tuple[kind: CmdLineKind, key, val: TaintedString] - -{.deprecated: [TGetoptResult: GetoptResult].} - -when declared(paramCount): - iterator getopt*(): GetoptResult = - ## This is an convenience iterator for iterating over the command line. - ## This uses the OptParser object. Example: - ## - ## .. code-block:: nim - ## var - ## filename = "" - ## for kind, key, val in getopt(): - ## case kind - ## of cmdArgument: - ## filename = key - ## of cmdLongOption, cmdShortOption: - ## case key - ## of "help", "h": writeHelp() - ## of "version", "v": writeVersion() - ## of cmdEnd: assert(false) # cannot happen - ## if filename == "": - ## # no filename has been given, so we show the help: - ## writeHelp() - var p = initOptParser() - while true: - next(p) - if p.kind == cmdEnd: break - yield (p.kind, p.key, p.val) - -{.pop.} diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index d6fafec08..a7c938d01 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -7,21 +7,26 @@ # distribution, for details about the copyright. # -## The ``parsesql`` module implements a high performance SQL file +## The `parsesql` module implements a high performance SQL file ## parser. It parses PostgreSQL syntax and the SQL ANSI standard. +## +## Unstable API. -import - hashes, strutils, lexbase, streams +import std/[strutils, lexbase] +import std/private/decode_helpers + +when defined(nimPreviewSlimSystem): + import std/assertions # ------------------- scanner ------------------------------------------------- type - TokKind = enum ## enumeration of all SQL tokens - tkInvalid, ## invalid token - tkEof, ## end of file reached - tkIdentifier, ## abc - tkQuotedIdentifier, ## "abc" - tkStringConstant, ## 'abc' + TokKind = enum ## enumeration of all SQL tokens + tkInvalid, ## invalid token + tkEof, ## end of file reached + tkIdentifier, ## abc + tkQuotedIdentifier, ## "abc" + tkStringConstant, ## 'abc' tkEscapeConstant, ## e'abc' tkDollarQuotedConstant, ## $tag$abc$tag$ tkBitStringConstant, ## B'00011' @@ -38,15 +43,13 @@ type tkBracketRi, ## ']' tkDot ## '.' - Token = object # a token - kind: TokKind # the type of the token - literal: string # the parsed (string) literal + Token = object # a token + kind: TokKind # the type of the token + literal: string # the parsed (string) literal SqlLexer* = object of BaseLexer ## the parser object. filename: string -{.deprecated: [TToken: Token, TSqlLexer: SqlLexer].} - const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", "identifier", "quoted identifier", "string constant", @@ -55,9 +58,12 @@ const ";", ":", ",", "(", ")", "[", "]", "." ] -proc open(L: var SqlLexer, input: Stream, filename: string) = - lexbase.open(L, input) - L.filename = filename + reservedKeywords = @[ + # statements + "select", "from", "where", "group", "limit", "offset", "having", + # functions + "count", + ] proc close(L: var SqlLexer) = lexbase.close(L) @@ -69,20 +75,6 @@ proc getColumn(L: SqlLexer): int = proc getLine(L: SqlLexer): int = result = L.lineNumber -proc handleHexChar(c: var SqlLexer, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: - discard - proc handleOctChar(c: var SqlLexer, xi: var int) = if c.buf[c.bufpos] in {'0'..'7'}: xi = (xi shl 3) or (ord(c.buf[c.bufpos]) - ord('0')) @@ -127,8 +119,10 @@ proc getEscapedChar(c: var SqlLexer, tok: var Token) = of 'x', 'X': inc(c.bufpos) var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) add(tok.literal, chr(xi)) of '0'..'7': var xi = 0 @@ -147,35 +141,33 @@ proc handleCRLF(c: var SqlLexer, pos: int): int = proc skip(c: var SqlLexer) = var pos = c.bufpos - var buf = c.buf var nested = 0 while true: - case buf[pos] + case c.buf[pos] of ' ', '\t': inc(pos) of '-': - if buf[pos+1] == '-': - while not (buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos) + if c.buf[pos+1] == '-': + while not (c.buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos) else: break of '/': - if buf[pos+1] == '*': - inc(pos,2) + if c.buf[pos+1] == '*': + inc(pos, 2) while true: - case buf[pos] + case c.buf[pos] of '\0': break of '\c', '\L': pos = handleCRLF(c, pos) - buf = c.buf of '*': - if buf[pos+1] == '/': + if c.buf[pos+1] == '/': inc(pos, 2) if nested <= 0: break dec(nested) else: inc(pos) of '/': - if buf[pos+1] == '*': + if c.buf[pos+1] == '*': inc(pos, 2) inc(nested) else: @@ -184,21 +176,19 @@ proc skip(c: var SqlLexer) = else: break of '\c', '\L': pos = handleCRLF(c, pos) - buf = c.buf else: - break # EndOfFile also leaves the loop + break # EndOfFile also leaves the loop c.bufpos = pos proc getString(c: var SqlLexer, tok: var Token, kind: TokKind) = var pos = c.bufpos + 1 - var buf = c.buf tok.kind = kind block parseLoop: while true: while true: - var ch = buf[pos] + var ch = c.buf[pos] if ch == '\'': - if buf[pos+1] == '\'': + if c.buf[pos+1] == '\'': inc(pos, 2) add(tok.literal, '\'') else: @@ -220,30 +210,27 @@ proc getString(c: var SqlLexer, tok: var Token, kind: TokKind) = if c.lineNumber > line: # a new line whitespace has been parsed, so we check if the string # continues after the whitespace: - buf = c.buf # may have been reallocated pos = c.bufpos - if buf[pos] == '\'': inc(pos) + if c.buf[pos] == '\'': inc(pos) else: break parseLoop else: break parseLoop c.bufpos = pos proc getDollarString(c: var SqlLexer, tok: var Token) = var pos = c.bufpos + 1 - var buf = c.buf tok.kind = tkDollarQuotedConstant var tag = "$" - while buf[pos] in IdentChars: - add(tag, buf[pos]) + while c.buf[pos] in IdentChars: + add(tag, c.buf[pos]) inc(pos) - if buf[pos] == '$': inc(pos) + if c.buf[pos] == '$': inc(pos) else: tok.kind = tkInvalid return while true: - case buf[pos] + case c.buf[pos] of '\c', '\L': pos = handleCRLF(c, pos) - buf = c.buf add(tok.literal, "\L") of '\0': tok.kind = tkInvalid @@ -251,39 +238,38 @@ proc getDollarString(c: var SqlLexer, tok: var Token) = of '$': inc(pos) var tag2 = "$" - while buf[pos] in IdentChars: - add(tag2, buf[pos]) + while c.buf[pos] in IdentChars: + add(tag2, c.buf[pos]) inc(pos) - if buf[pos] == '$': inc(pos) + if c.buf[pos] == '$': inc(pos) if tag2 == tag: break add(tok.literal, tag2) add(tok.literal, '$') else: - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) c.bufpos = pos proc getSymbol(c: var SqlLexer, tok: var Token) = var pos = c.bufpos - var buf = c.buf while true: - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) - if buf[pos] notin {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}: + if c.buf[pos] notin {'a'..'z', 'A'..'Z', '0'..'9', '_', '$', + '\128'..'\255'}: break c.bufpos = pos tok.kind = tkIdentifier -proc getQuotedIdentifier(c: var SqlLexer, tok: var Token) = +proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote = '\"') = var pos = c.bufpos + 1 - var buf = c.buf tok.kind = tkQuotedIdentifier while true: - var ch = buf[pos] - if ch == '\"': - if buf[pos+1] == '\"': + var ch = c.buf[pos] + if ch == quote: + if c.buf[pos+1] == quote: inc(pos, 2) - add(tok.literal, '\"') + add(tok.literal, quote) else: inc(pos) break @@ -297,11 +283,10 @@ proc getQuotedIdentifier(c: var SqlLexer, tok: var Token) = proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) = var pos = c.bufpos + 1 - var buf = c.buf block parseLoop: while true: while true: - var ch = buf[pos] + var ch = c.buf[pos] if ch in validChars: add(tok.literal, ch) inc(pos) @@ -317,9 +302,8 @@ proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) = if c.lineNumber > line: # a new line whitespace has been parsed, so we check if the string # continues after the whitespace: - buf = c.buf # may have been reallocated pos = c.bufpos - if buf[pos] == '\'': inc(pos) + if c.buf[pos] == '\'': inc(pos) else: break parseLoop else: break parseLoop c.bufpos = pos @@ -327,29 +311,28 @@ proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) = proc getNumeric(c: var SqlLexer, tok: var Token) = tok.kind = tkInteger var pos = c.bufpos - var buf = c.buf - while buf[pos] in Digits: - add(tok.literal, buf[pos]) + while c.buf[pos] in Digits: + add(tok.literal, c.buf[pos]) inc(pos) - if buf[pos] == '.': + if c.buf[pos] == '.': tok.kind = tkNumeric - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) - while buf[pos] in Digits: - add(tok.literal, buf[pos]) + while c.buf[pos] in Digits: + add(tok.literal, c.buf[pos]) inc(pos) - if buf[pos] in {'E', 'e'}: + if c.buf[pos] in {'E', 'e'}: tok.kind = tkNumeric - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) - if buf[pos] == '+': + if c.buf[pos] == '+': inc(pos) - elif buf[pos] == '-': - add(tok.literal, buf[pos]) + elif c.buf[pos] == '-': + add(tok.literal, c.buf[pos]) inc(pos) - if buf[pos] in Digits: - while buf[pos] in Digits: - add(tok.literal, buf[pos]) + if c.buf[pos] in Digits: + while c.buf[pos] in Digits: + add(tok.literal, c.buf[pos]) inc(pos) else: tok.kind = tkInvalid @@ -360,24 +343,23 @@ proc getOperator(c: var SqlLexer, tok: var Token) = '^', '&', '|', '`', '?'} tok.kind = tkOperator var pos = c.bufpos - var buf = c.buf var trailingPlusMinus = false while true: - case buf[pos] + case c.buf[pos] of '-': - if buf[pos] == '-': break - if not trailingPlusMinus and buf[pos+1] notin operators and + if c.buf[pos] == '-': break + if not trailingPlusMinus and c.buf[pos+1] notin operators and tok.literal.len > 0: break of '/': - if buf[pos] == '*': break + if c.buf[pos] == '*': break of '~', '!', '@', '#', '%', '^', '&', '|', '`', '?': trailingPlusMinus = true of '+': - if not trailingPlusMinus and buf[pos+1] notin operators and + if not trailingPlusMinus and c.buf[pos+1] notin operators and tok.literal.len > 0: break of '*', '<', '>', '=': discard else: break - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) c.bufpos = pos @@ -413,7 +395,7 @@ proc getTok(c: var SqlLexer, tok: var Token) = of 'x', 'X': if c.buf[c.bufpos + 1] == '\'': tok.kind = tkHexStringConstant - getBitHexString(c, tok, {'a'..'f','A'..'F','0'..'9'}) + getBitHexString(c, tok, {'a'..'f', 'A'..'F', '0'..'9'}) else: getSymbol(c, tok) of '$': getDollarString(c, tok) @@ -442,7 +424,8 @@ proc getTok(c: var SqlLexer, tok: var Token) = add(tok.literal, '.') of '0'..'9': getNumeric(c, tok) of '\'': getString(c, tok, tkStringConstant) - of '"': getQuotedIdentifier(c, tok) + of '"': getQuotedIdentifier(c, tok, '"') + of '`': getQuotedIdentifier(c, tok, '`') of lexbase.EndOfFile: tok.kind = tkEof tok.literal = "[EOF]" @@ -450,7 +433,7 @@ proc getTok(c: var SqlLexer, tok: var Token) = '\128'..'\255': getSymbol(c, tok) of '+', '-', '*', '/', '<', '>', '=', '~', '!', '@', '#', '%', - '^', '&', '|', '`', '?': + '^', '&', '|', '?': getOperator(c, tok) else: add(tok.literal, c.buf[c.bufpos]) @@ -462,32 +445,33 @@ proc errorStr(L: SqlLexer, msg: string): string = # ----------------------------- parser ---------------------------------------- -# Operator/Element Associativity Description -# . left table/column name separator -# :: left PostgreSQL-style typecast -# [ ] left array element selection -# - right unary minus -# ^ left exponentiation -# * / % left multiplication, division, modulo -# + - left addition, subtraction -# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL -# ISNULL test for null -# NOTNULL test for not null -# (any other) left all other native and user-defined oprs -# IN set membership -# BETWEEN range containment -# OVERLAPS time interval overlap -# LIKE ILIKE SIMILAR string pattern matching -# < > less than, greater than -# = right equality, assignment -# NOT right logical negation -# AND left logical conjunction -# OR left logical disjunction +# Operator/Element Associativity Description +# . left table/column name separator +# :: left PostgreSQL-style typecast +# [ ] left array element selection +# - right unary minus +# ^ left exponentiation +# * / % left multiplication, division, modulo +# + - left addition, subtraction +# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL +# ISNULL test for null +# NOTNULL test for not null +# (any other) left all other native and user-defined oprs +# IN set membership +# BETWEEN range containment +# OVERLAPS time interval overlap +# LIKE ILIKE SIMILAR string pattern matching +# < > less than, greater than +# = right equality, assignment +# NOT right logical negation +# AND left logical conjunction +# OR left logical disjunction type SqlNodeKind* = enum ## kind of SQL abstract syntax tree nkNone, nkIdent, + nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, @@ -496,6 +480,7 @@ type nkPrimaryKey, nkForeignKey, nkNotNull, + nkNull, nkStmtList, nkDot, @@ -503,6 +488,7 @@ type nkPrefix, nkInfix, nkCall, + nkPrGroup, nkColumnReference, nkReferences, nkDefault, @@ -510,18 +496,23 @@ type nkConstraint, nkUnique, nkIdentity, - nkColumnDef, ## name, datatype, constraints + nkColumnDef, ## name, datatype, constraints nkInsert, nkUpdate, nkDelete, nkSelect, nkSelectDistinct, nkSelectColumns, + nkSelectPair, nkAsgn, nkFrom, + nkFromItemPair, nkGroup, + nkLimit, + nkOffset, nkHaving, nkOrder, + nkJoin, nkDesc, nkUnion, nkIntersect, @@ -537,39 +528,54 @@ type nkCreateIndexIfNotExists, nkEnumDef +const + LiteralNodes = { + nkIdent, nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, + nkIntegerLit, nkNumericLit + } + type SqlParseError* = object of ValueError ## Invalid SQL encountered - SqlNode* = ref SqlNodeObj ## an SQL abstract syntax tree node - SqlNodeObj* = object ## an SQL abstract syntax tree node - case kind*: SqlNodeKind ## kind of syntax tree - of nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit, - nkIntegerLit, nkNumericLit: - strVal*: string ## AST leaf: the identifier, numeric literal - ## string literal, etc. + SqlNode* = ref SqlNodeObj ## an SQL abstract syntax tree node + SqlNodeObj* = object ## an SQL abstract syntax tree node + case kind*: SqlNodeKind ## kind of syntax tree + of LiteralNodes: + strVal*: string ## AST leaf: the identifier, numeric literal + ## string literal, etc. else: - sons*: seq[SqlNode] ## the node's children + sons*: seq[SqlNode] ## the node's children SqlParser* = object of SqlLexer ## SQL parser object tok: Token -{.deprecated: [EInvalidSql: SqlParseError, PSqlNode: SqlNode, - TSqlNode: SqlNodeObj, TSqlParser: SqlParser, TSqlNodeKind: SqlNodeKind].} - -proc newNode(k: SqlNodeKind): SqlNode = - new(result) - result.kind = k +proc newNode*(k: SqlNodeKind): SqlNode = + when defined(js): # bug #14117 + case k + of LiteralNodes: + result = SqlNode(kind: k, strVal: "") + else: + result = SqlNode(kind: k, sons: @[]) + else: + result = SqlNode(kind: k) -proc newNode(k: SqlNodeKind, s: string): SqlNode = - new(result) - result.kind = k +proc newNode*(k: SqlNodeKind, s: string): SqlNode = + result = SqlNode(kind: k) result.strVal = s +proc newNode*(k: SqlNodeKind, sons: seq[SqlNode]): SqlNode = + result = SqlNode(kind: k) + result.sons = sons + proc len*(n: SqlNode): int = - if isNil(n.sons): result = 0 - else: result = n.sons.len + if n.kind in LiteralNodes: + result = 0 + else: + result = n.sons.len + +proc `[]`*(n: SqlNode; i: int): SqlNode = n.sons[i] +proc `[]`*(n: SqlNode; i: BackwardsIndex): SqlNode = n.sons[n.len - int(i)] proc add*(father, n: SqlNode) = - if isNil(father.sons): father.sons = @[] add(father.sons, n) proc getTok(p: var SqlParser) = @@ -611,7 +617,10 @@ proc eat(p: var SqlParser, keyw: string) = if isKeyw(p, keyw): getTok(p) else: - sqlError(p, keyw.toUpper() & " expected") + sqlError(p, keyw.toUpperAscii() & " expected") + +proc opt(p: var SqlParser, kind: TokKind) = + if p.tok.kind == kind: getTok(p) proc parseDataType(p: var SqlParser): SqlNode = if isKeyw(p, "enum"): @@ -648,11 +657,13 @@ proc getPrecedence(p: SqlParser): int = result = 5 elif isOpr(p, "=") or isOpr(p, "<") or isOpr(p, ">") or isOpr(p, ">=") or isOpr(p, "<=") or isOpr(p, "<>") or isOpr(p, "!=") or isKeyw(p, "is") or - isKeyw(p, "like"): - result = 3 + isKeyw(p, "like") or isKeyw(p, "in"): + result = 4 elif isKeyw(p, "and"): - result = 2 + result = 3 elif isKeyw(p, "or"): + result = 2 + elif isKeyw(p, "between"): result = 1 elif p.tok.kind == tkOperator: # user-defined operator: @@ -660,11 +671,15 @@ proc getPrecedence(p: SqlParser): int = else: result = - 1 -proc parseExpr(p: var SqlParser): SqlNode +proc parseExpr(p: var SqlParser): SqlNode {.gcsafe.} +proc parseSelect(p: var SqlParser): SqlNode {.gcsafe.} proc identOrLiteral(p: var SqlParser): SqlNode = case p.tok.kind - of tkIdentifier, tkQuotedIdentifier: + of tkQuotedIdentifier: + result = newNode(nkQuotedIdent, p.tok.literal) + getTok(p) + of tkIdentifier: result = newNode(nkIdent, p.tok.literal) getTok(p) of tkStringConstant, tkEscapeConstant, tkDollarQuotedConstant: @@ -684,14 +699,23 @@ proc identOrLiteral(p: var SqlParser): SqlNode = getTok(p) of tkParLe: getTok(p) - result = parseExpr(p) + result = newNode(nkPrGroup) + while true: + result.add(parseExpr(p)) + if p.tok.kind != tkComma: break + getTok(p) eat(p, tkParRi) else: - sqlError(p, "expression expected") - getTok(p) # we must consume a token here to prevend endless loops! + if p.tok.literal == "*": + result = newNode(nkIdent, p.tok.literal) + getTok(p) + else: + sqlError(p, "expression expected") + getTok(p) # we must consume a token here to prevent endless loops! proc primary(p: var SqlParser): SqlNode = - if p.tok.kind == tkOperator or isKeyw(p, "not"): + if (p.tok.kind == tkOperator and (p.tok.literal == "+" or p.tok.literal == + "-")) or isKeyw(p, "not"): result = newNode(nkPrefix) result.add(newNode(nkIdent, p.tok.literal)) getTok(p) @@ -705,7 +729,7 @@ proc primary(p: var SqlParser): SqlNode = result = newNode(nkCall) result.add(a) getTok(p) - while true: + while p.tok.kind != tkParRi: result.add(parseExpr(p)) if p.tok.kind == tkComma: getTok(p) else: break @@ -736,7 +760,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int = result = opPred while opPred > limit: node = newNode(nkInfix) - opNode = newNode(nkIdent, p.tok.literal) + opNode = newNode(nkIdent, p.tok.literal.toLowerAscii()) getTok(p) result = lowestExprAux(p, v2, opPred) node.add(opNode) @@ -776,9 +800,19 @@ proc parseConstraint(p: var SqlParser): SqlNode = expectIdent(p) result.add(newNode(nkIdent, p.tok.literal)) getTok(p) - eat(p, "check") + optKeyw(p, "check") result.add(parseExpr(p)) +proc parseParIdentList(p: var SqlParser, father: SqlNode) = + eat(p, tkParLe) + while true: + expectIdent(p) + father.add(newNode(nkIdent, p.tok.literal)) + getTok(p) + if p.tok.kind != tkComma: break + getTok(p) + eat(p, tkParRi) + proc parseColumnConstraints(p: var SqlParser, result: SqlNode) = while true: if isKeyw(p, "default"): @@ -795,6 +829,9 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) = getTok(p) eat(p, "null") result.add(newNode(nkNotNull)) + elif isKeyw(p, "null"): + getTok(p) + result.add(newNode(nkNull)) elif isKeyw(p, "identity"): getTok(p) result.add(newNode(nkIdentity)) @@ -807,6 +844,7 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) = elif isKeyw(p, "constraint"): result.add(parseConstraint(p)) elif isKeyw(p, "unique"): + getTok(p) result.add(newNode(nkUnique)) else: break @@ -829,16 +867,6 @@ proc parseIfNotExists(p: var SqlParser, k: SqlNodeKind): SqlNode = else: result = newNode(k) -proc parseParIdentList(p: var SqlParser, father: SqlNode) = - eat(p, tkParLe) - while true: - expectIdent(p) - father.add(newNode(nkIdent, p.tok.literal)) - getTok(p) - if p.tok.kind != tkComma: break - getTok(p) - eat(p, tkParRi) - proc parseTableConstraint(p: var SqlParser): SqlNode = if isKeyw(p, "primary"): getTok(p) @@ -866,20 +894,34 @@ proc parseTableConstraint(p: var SqlParser): SqlNode = else: sqlError(p, "column definition expected") +proc parseUnique(p: var SqlParser): SqlNode = + result = parseExpr(p) + if result.kind == nkCall: result.kind = nkUnique + proc parseTableDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateTable) expectIdent(p) result.add(newNode(nkIdent, p.tok.literal)) getTok(p) if p.tok.kind == tkParLe: - while true: - getTok(p) - if p.tok.kind == tkIdentifier or p.tok.kind == tkQuotedIdentifier: + getTok(p) + while p.tok.kind != tkParRi: + if isKeyw(p, "constraint"): + result.add parseConstraint(p) + elif isKeyw(p, "primary") or isKeyw(p, "foreign"): + result.add parseTableConstraint(p) + elif isKeyw(p, "unique"): + result.add parseUnique(p) + elif p.tok.kind == tkIdentifier or p.tok.kind == tkQuotedIdentifier: result.add(parseColumnDef(p)) else: result.add(parseTableConstraint(p)) if p.tok.kind != tkComma: break + getTok(p) eat(p, tkParRi) + # skip additional crap after 'create table (...) crap;' + while p.tok.kind notin {tkSemicolon, tkEof}: + getTok(p) proc parseTypeDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateType) @@ -894,6 +936,19 @@ proc parseWhere(p: var SqlParser): SqlNode = result = newNode(nkWhere) result.add(parseExpr(p)) +proc parseFromItem(p: var SqlParser): SqlNode = + result = newNode(nkFromItemPair) + if p.tok.kind == tkParLe: + getTok(p) + var select = parseSelect(p) + result.add(select) + eat(p, tkParRi) + else: + result.add(parseExpr(p)) + if isKeyw(p, "as"): + getTok(p) + result.add(parseExpr(p)) + proc parseIndexDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateIndex) if isKeyw(p, "primary"): @@ -929,8 +984,9 @@ proc parseInsert(p: var SqlParser): SqlNode = if p.tok.kind == tkParLe: var n = newNode(nkColumnList) parseParIdentList(p, n) + result.add n else: - result.add(nil) + result.add(newNode(nkNone)) if isKeyw(p, "default"): getTok(p) eat(p, "values") @@ -965,17 +1021,19 @@ proc parseUpdate(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseDelete(p: var SqlParser): SqlNode = getTok(p) + if isOpr(p, "*"): + getTok(p) result = newNode(nkDelete) eat(p, "from") result.add(primary(p)) if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseSelect(p: var SqlParser): SqlNode = getTok(p) @@ -991,7 +1049,12 @@ proc parseSelect(p: var SqlParser): SqlNode = a.add(newNode(nkIdent, "*")) getTok(p) else: - a.add(parseExpr(p)) + var pair = newNode(nkSelectPair) + pair.add(parseExpr(p)) + a.add(pair) + if isKeyw(p, "as"): + getTok(p) + pair.add(parseExpr(p)) if p.tok.kind != tkComma: break getTok(p) result.add(a) @@ -999,7 +1062,7 @@ proc parseSelect(p: var SqlParser): SqlNode = var f = newNode(nkFrom) while true: getTok(p) - f.add(parseExpr(p)) + f.add(parseFromItem(p)) if p.tok.kind != tkComma: break result.add(f) if isKeyw(p, "where"): @@ -1013,6 +1076,23 @@ proc parseSelect(p: var SqlParser): SqlNode = if p.tok.kind != tkComma: break getTok(p) result.add(g) + if isKeyw(p, "order"): + getTok(p) + eat(p, "by") + var n = newNode(nkOrder) + while true: + var e = parseExpr(p) + if isKeyw(p, "asc"): + getTok(p) # is default + elif isKeyw(p, "desc"): + getTok(p) + var x = newNode(nkDesc) + x.add(e) + e = x + n.add(e) + if p.tok.kind != tkComma: break + getTok(p) + result.add(n) if isKeyw(p, "having"): var h = newNode(nkHaving) while true: @@ -1029,24 +1109,31 @@ proc parseSelect(p: var SqlParser): SqlNode = elif isKeyw(p, "except"): result.add(newNode(nkExcept)) getTok(p) - if isKeyw(p, "order"): - getTok(p) - eat(p, "by") - var n = newNode(nkOrder) - while true: - var e = parseExpr(p) - if isKeyw(p, "asc"): getTok(p) # is default - elif isKeyw(p, "desc"): - getTok(p) - var x = newNode(nkDesc) - x.add(e) - e = x - n.add(e) - if p.tok.kind != tkComma: break + if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"): + var join = newNode(nkJoin) + result.add(join) + if isKeyw(p, "join"): + join.add(newNode(nkIdent, "")) getTok(p) - result.add(n) + else: + join.add(newNode(nkIdent, p.tok.literal.toLowerAscii())) + getTok(p) + eat(p, "join") + join.add(parseFromItem(p)) + eat(p, "on") + join.add(parseExpr(p)) + if isKeyw(p, "limit"): + getTok(p) + var l = newNode(nkLimit) + l.add(parseExpr(p)) + result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) -proc parseStmt(p: var SqlParser): SqlNode = +proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): getTok(p) optKeyw(p, "cached") @@ -1058,71 +1145,108 @@ proc parseStmt(p: var SqlParser): SqlNode = optKeyw(p, "unique") optKeyw(p, "hash") if isKeyw(p, "table"): - result = parseTableDef(p) + parent.add parseTableDef(p) elif isKeyw(p, "type"): - result = parseTypeDef(p) + parent.add parseTypeDef(p) elif isKeyw(p, "index"): - result = parseIndexDef(p) + parent.add parseIndexDef(p) else: sqlError(p, "TABLE expected") elif isKeyw(p, "insert"): - result = parseInsert(p) + parent.add parseInsert(p) elif isKeyw(p, "update"): - result = parseUpdate(p) + parent.add parseUpdate(p) elif isKeyw(p, "delete"): - result = parseDelete(p) + parent.add parseDelete(p) elif isKeyw(p, "select"): - result = parseSelect(p) + parent.add parseSelect(p) + elif isKeyw(p, "begin"): + getTok(p) else: - sqlError(p, "CREATE expected") - -proc open(p: var SqlParser, input: Stream, filename: string) = - ## opens the parser `p` and assigns the input stream `input` to it. - ## `filename` is only used for error messages. - open(SqlLexer(p), input, filename) - p.tok.kind = tkInvalid - p.tok.literal = "" - getTok(p) + sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected") proc parse(p: var SqlParser): SqlNode = ## parses the content of `p`'s input stream and returns the SQL AST. - ## Syntax errors raise an `EInvalidSql` exception. + ## Syntax errors raise an `SqlParseError` exception. result = newNode(nkStmtList) while p.tok.kind != tkEof: - var s = parseStmt(p) + parseStmt(p, result) + if p.tok.kind == tkEof: + break eat(p, tkSemicolon) - result.add(s) - if result.len == 1: - result = result.sons[0] proc close(p: var SqlParser) = ## closes the parser `p`. The associated input stream is closed too. close(SqlLexer(p)) -proc parseSQL*(input: Stream, filename: string): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `EInvalidSql` exception. - var p: SqlParser - open(p, input, filename) - try: - result = parse(p) - finally: - close(p) +type + SqlWriter = object + indent: int + upperCase: bool + buffer: string + +proc add(s: var SqlWriter, thing: char) = + s.buffer.add(thing) -proc ra(n: SqlNode, s: var string, indent: int) +proc prepareAdd(s: var SqlWriter) {.inline.} = + if s.buffer.len > 0 and s.buffer[^1] notin {' ', '\L', '(', '.'}: + s.buffer.add(" ") -proc rs(n: SqlNode, s: var string, indent: int, - prefix = "(", suffix = ")", - sep = ", ") = +proc add(s: var SqlWriter, thing: string) = + s.prepareAdd + s.buffer.add(thing) + +proc addKeyw(s: var SqlWriter, thing: string) = + var keyw = thing + if s.upperCase: + keyw = keyw.toUpperAscii() + s.add(keyw) + +proc addIden(s: var SqlWriter, thing: string) = + var iden = thing + if iden.toLowerAscii() in reservedKeywords: + iden = '"' & iden & '"' + s.add(iden) + +proc ra(n: SqlNode, s: var SqlWriter) {.gcsafe.} + +proc rs(n: SqlNode, s: var SqlWriter, prefix = "(", suffix = ")", sep = ", ") = if n.len > 0: s.add(prefix) for i in 0 .. n.len-1: if i > 0: s.add(sep) - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(suffix) -proc ra(n: SqlNode, s: var string, indent: int) = +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',') = + if n.len > 0: + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = + if n.len > 0: + s.add(prefix) + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + s.add(suffix) + +proc quoted(s: string): string = + "\"" & replace(s, "\"", "\"\"") & "\"" + +func escape(result: var string; s: string) = + result.add('\'') + for c in items(s): + case c + of '\0'..'\31': + result.add("\\x") + result.add(toHex(ord(c), 2)) + of '\'': result.add("''") + else: result.add(c) + result.add('\'') + +proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard @@ -1130,9 +1254,12 @@ proc ra(n: SqlNode, s: var string, indent: int) = if allCharsInSet(n.strVal, {'\33'..'\127'}): s.add(n.strVal) else: - s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") + s.add(quoted(n.strVal)) + of nkQuotedIdent: + s.add(quoted(n.strVal)) of nkStringLit: - s.add(escape(n.strVal, "e'", "'")) + s.prepareAdd + s.buffer.escape(n.strVal) of nkBitStringLit: s.add("b'" & n.strVal & "'") of nkHexStringLit: @@ -1140,211 +1267,257 @@ proc ra(n: SqlNode, s: var string, indent: int) = of nkIntegerLit, nkNumericLit: s.add(n.strVal) of nkPrimaryKey: - s.add(" primary key") - rs(n, s, indent) + s.addKeyw("primary key") + rs(n, s) of nkForeignKey: - s.add(" foreign key") - rs(n, s, indent) + s.addKeyw("foreign key") + rs(n, s) of nkNotNull: - s.add(" not null") + s.addKeyw("not null") + of nkNull: + s.addKeyw("null") of nkDot: - ra(n.sons[0], s, indent) - s.add(".") - ra(n.sons[1], s, indent) + ra(n.sons[0], s) + s.add('.') + ra(n.sons[1], s) of nkDotDot: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(". .") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkPrefix: - s.add('(') - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(' ') - ra(n.sons[1], s, indent) - s.add(')') + ra(n.sons[1], s) of nkInfix: - s.add('(') - ra(n.sons[1], s, indent) + ra(n.sons[1], s) s.add(' ') - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(' ') - ra(n.sons[2], s, indent) - s.add(')') + ra(n.sons[2], s) of nkCall, nkColumnReference: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") - ra(n.sons[i], s, indent) + if i > 1: s.add(',') + ra(n.sons[i], s) + s.add(')') + of nkPrGroup: + s.add('(') + s.addMulti(n) s.add(')') of nkReferences: - s.add(" references ") - ra(n.sons[0], s, indent) + s.addKeyw("references") + ra(n.sons[0], s) of nkDefault: - s.add(" default ") - ra(n.sons[0], s, indent) + s.addKeyw("default") + ra(n.sons[0], s) of nkCheck: - s.add(" check ") - ra(n.sons[0], s, indent) + s.addKeyw("check") + ra(n.sons[0], s) of nkConstraint: - s.add(" constraint ") - ra(n.sons[0], s, indent) - s.add(" check ") - ra(n.sons[1], s, indent) + s.addKeyw("constraint") + ra(n.sons[0], s) + s.addKeyw("check") + ra(n.sons[1], s) of nkUnique: - s.add(" unique") - rs(n, s, indent) + s.addKeyw("unique") + rs(n, s) of nkIdentity: - s.add(" identity") + s.addKeyw("identity") of nkColumnDef: - s.add("\n ") - rs(n, s, indent, "", "", " ") + rs(n, s, "", "", " ") of nkStmtList: for i in 0..n.len-1: - ra(n.sons[i], s, indent) - s.add("\n") + ra(n.sons[i], s) + s.add(';') of nkInsert: assert n.len == 3 - s.add("insert into ") - ra(n.sons[0], s, indent) - ra(n.sons[1], s, indent) + s.addKeyw("insert into") + ra(n.sons[0], s) + s.add(' ') + ra(n.sons[1], s) if n.sons[2].kind == nkDefault: - s.add("default values") + s.addKeyw("default values") else: - s.add("\nvalues ") - ra(n.sons[2], s, indent) - s.add(';') + ra(n.sons[2], s) of nkUpdate: - s.add("update ") - ra(n.sons[0], s, indent) - s.add(" set ") + s.addKeyw("update") + ra(n.sons[0], s) + s.addKeyw("set") var L = n.len for i in 1 .. L-2: if i > 1: s.add(", ") var it = n.sons[i] assert it.kind == nkAsgn - ra(it, s, indent) - ra(n.sons[L-1], s, indent) - s.add(';') + ra(it, s) + ra(n.sons[L-1], s) of nkDelete: - s.add("delete from ") - ra(n.sons[0], s, indent) - ra(n.sons[1], s, indent) - s.add(';') + s.addKeyw("delete from") + ra(n.sons[0], s) + ra(n.sons[1], s) of nkSelect, nkSelectDistinct: - s.add("select ") + s.addKeyw("select") if n.kind == nkSelectDistinct: - s.add("distinct ") - rs(n.sons[0], s, indent, "", "", ", ") - for i in 1 .. n.len-1: ra(n.sons[i], s, indent) - s.add(';') + s.addKeyw("distinct") + for i in 0 ..< n.len: + ra(n.sons[i], s) of nkSelectColumns: - assert(false) + for i, column in n.sons: + if i > 0: s.add(',') + ra(column, s) + of nkSelectPair: + ra(n.sons[0], s) + if n.sons.len == 2: + s.addKeyw("as") + ra(n.sons[1], s) + of nkFromItemPair: + if n.sons[0].kind in {nkIdent, nkQuotedIdent}: + ra(n.sons[0], s) + else: + assert n.sons[0].kind == nkSelect + s.add('(') + ra(n.sons[0], s) + s.add(')') + if n.sons.len == 2: + s.addKeyw("as") + ra(n.sons[1], s) of nkAsgn: - ra(n.sons[0], s, indent) + ra(n.sons[0], s) s.add(" = ") - ra(n.sons[1], s, indent) + ra(n.sons[1], s) of nkFrom: - s.add("\nfrom ") - rs(n, s, indent, "", "", ", ") + s.addKeyw("from") + s.addMulti(n) of nkGroup: - s.add("\ngroup by") - rs(n, s, indent, "", "", ", ") + s.addKeyw("group by") + s.addMulti(n) + of nkLimit: + s.addKeyw("limit") + s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: - s.add("\nhaving") - rs(n, s, indent, "", "", ", ") + s.addKeyw("having") + s.addMulti(n) of nkOrder: - s.add("\norder by ") - rs(n, s, indent, "", "", ", ") + s.addKeyw("order by") + s.addMulti(n) + of nkJoin: + var joinType = n.sons[0].strVal + if joinType == "": + joinType = "join" + else: + joinType &= " " & "join" + s.addKeyw(joinType) + ra(n.sons[1], s) + s.addKeyw("on") + ra(n.sons[2], s) of nkDesc: - ra(n.sons[0], s, indent) - s.add(" desc") + ra(n.sons[0], s) + s.addKeyw("desc") of nkUnion: - s.add(" union") + s.addKeyw("union") of nkIntersect: - s.add(" intersect") + s.addKeyw("intersect") of nkExcept: - s.add(" except") + s.addKeyw("except") of nkColumnList: - rs(n, s, indent) + rs(n, s) of nkValueList: - s.add("values ") - rs(n, s, indent) + s.addKeyw("values") + rs(n, s) of nkWhere: - s.add("\nwhere ") - ra(n.sons[0], s, indent) + s.addKeyw("where") + ra(n.sons[0], s) of nkCreateTable, nkCreateTableIfNotExists: - s.add("create table ") + s.addKeyw("create table") if n.kind == nkCreateTableIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") - ra(n.sons[i], s, indent) + if i > 1: s.add(',') + ra(n.sons[i], s) s.add(");") of nkCreateType, nkCreateTypeIfNotExists: - s.add("create type ") + s.addKeyw("create type") if n.kind == nkCreateTypeIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) - s.add(" as ") - ra(n.sons[1], s, indent) - s.add(';') + s.addKeyw("if not exists") + ra(n.sons[0], s) + s.addKeyw("as") + ra(n.sons[1], s) of nkCreateIndex, nkCreateIndexIfNotExists: - s.add("create index ") + s.addKeyw("create index") if n.kind == nkCreateIndexIfNotExists: - s.add("if not exists ") - ra(n.sons[0], s, indent) - s.add(" on ") - ra(n.sons[1], s, indent) + s.addKeyw("if not exists") + ra(n.sons[0], s) + s.addKeyw("on") + ra(n.sons[1], s) s.add('(') for i in 2..n.len-1: if i > 2: s.add(", ") - ra(n.sons[i], s, indent) + ra(n.sons[i], s) s.add(");") of nkEnumDef: - s.add("enum ") - rs(n, s, indent) + s.addKeyw("enum") + rs(n, s) -# What I want: -# -#select(columns = [T1.all, T2.name], -# fromm = [T1, T2], -# where = T1.name ==. T2.name, -# orderby = [name]): -# -#for row in dbQuery(db, """select x, y, z -# from a, b -# where a.name = b.name"""): -# +proc renderSql*(n: SqlNode, upperCase = false): string = + ## Converts an SQL abstract syntax tree to its string representation. + var s: SqlWriter + s.buffer = "" + s.upperCase = upperCase + ra(n, s) + return s.buffer + +proc `$`*(n: SqlNode): string = + ## an alias for `renderSql`. + renderSql(n) + +proc treeReprAux(s: SqlNode, level: int, result: var string) = + result.add('\n') + for i in 0 ..< level: result.add(" ") + + result.add($s.kind) + if s.kind in LiteralNodes: + result.add(' ') + result.add(s.strVal) + else: + for son in s.sons: + treeReprAux(son, level + 1, result) -#select x, y, z: -# fromm: Table1, Table2 -# where: x.name == y.name -#db.select(fromm = [t1, t2], where = t1.name == t2.name): -#for x, y, z in db.select(fromm = a, b where = a.name == b.name): -# writeLine x, y, z +proc treeRepr*(s: SqlNode): string = + result = newStringOfCap(128) + treeReprAux(s, 0, result) -proc renderSQL*(n: SqlNode): string = - ## Converts an SQL abstract syntax tree to its string representation. - result = "" - ra(n, result, 0) - -when not defined(testing) and isMainModule: - echo(renderSQL(parseSQL(newStringStream(""" - CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); - CREATE TABLE holidays ( - num_weeks int, - happiness happiness - ); - CREATE INDEX table1_attr1 ON table1(attr1); - - SELECT * FROM myTab WHERE col1 = 'happy'; - """), "stdin"))) - -# CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); -# CREATE TABLE holidays ( -# num_weeks int, -# happiness happiness -# ); -# CREATE INDEX table1_attr1 ON table1(attr1) +import std/streams + +proc open(L: var SqlLexer, input: Stream, filename: string) = + lexbase.open(L, input) + L.filename = filename + +proc open(p: var SqlParser, input: Stream, filename: string) = + ## opens the parser `p` and assigns the input stream `input` to it. + ## `filename` is only used for error messages. + open(SqlLexer(p), input, filename) + p.tok.kind = tkInvalid + p.tok.literal = "" + getTok(p) + +proc parseSql*(input: Stream, filename: string): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + var p: SqlParser + open(p, input, filename) + try: + result = parse(p) + finally: + close(p) + +proc parseSql*(input: string, filename = ""): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + parseSql(newStringStream(input), "") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index b3708838a..2ca255fa0 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -7,15 +7,50 @@ # distribution, for details about the copyright. # -## This module contains helpers for parsing tokens, numbers, identifiers, etc. +## This module contains helpers for parsing tokens, numbers, integers, floats, +## identifiers, etc. +## +## To unpack raw bytes look at the `streams <streams.html>`_ module. +## +## ```nim test +## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] +## var outp: seq[string] +## +## for log in logs: +## var res: string +## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 +## outp.add(res & " - " & captureBetween(log, ' ', '_')) +## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ``` +## +## ```nim test +## from std/strutils import Digits, parseInt +## +## let +## input1 = "2019 school start" +## input2 = "3 years back" +## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 +## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 +## examYear = parseInt(startYear) + parseInt(yearsBack) +## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## ``` +## +## **See also:** +## * `strutils module<strutils.html>`_ for combined and identical parsing proc's +## * `json module<json.html>`_ for a JSON parser +## * `parsecfg module<parsecfg.html>`_ for a configuration file parser +## * `parsecsv module<parsecsv.html>`_ for a simple CSV (comma separated value) parser +## * `parseopt module<parseopt.html>`_ for a command line parser +## * `parsexml module<parsexml.html>`_ for a XML / HTML parser +## * `other parsers<lib.html#pure-libraries-parsers>`_ for other parsers -{.deadCodeElim: on.} - -{.push debugger:off .} # the user does not want to trace a part +{.push debugger: off.} # the user does not want to trace a part # of the standard library! include "system/inclrtl" +template toOa(s: string): openArray[char] = openArray[char](s) + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} @@ -25,318 +60,1028 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseHex*(s: string, number: var int, start = 0): int {. - rtl, extern: "npuParseHex", noSideEffect.} = - ## Parses a hexadecimal number and stores its value in ``number``. +proc parseBin*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = + ## Parses a binary number and stores its value in ``number``. ## - ## Returns the number of the parsed characters or 0 in case of an error. This - ## proc is sensitive to the already existing value of ``number`` and will - ## likely not do what you want unless you make sure ``number`` is zero. You - ## can use this feature to *chain* calls, though the result int will quickly - ## overflow. Example: + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. ## - ## .. code-block:: nim - ## var value = 0 - ## discard parseHex("0x38", value) - ## assert value == 56 - ## discard parseHex("0x34", value) - ## assert value == 56 * 256 + 52 - ## value = -1 - ## discard parseHex("0x38", value) - ## assert value == -200 + ## If ``maxLen == 0``, the parsing continues until the first non-bin character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. ## - var i = start + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29 + doAssert num == 5138925 + doAssert parseBin("3", num) == 0 + var num8: int8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32 + doAssert num8 == 0b1110_1101'i8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9 + doAssert num8 == 0b0100_1110'i8 + var num8u: uint8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32 + doAssert num8u == 237 + var num64: int64 + doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 + doAssert num64 == 336784608873 + var i = 0 + var output = T(0) var foundDigit = false - if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) - elif s[i] == '#': inc(i) - while true: + let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) + if i + 1 < last and s[i] == '0' and (s[i+1] in {'b', 'B'}): inc(i, 2) + while i < last: case s[i] of '_': discard - of '0'..'9': - number = number shl 4 or (ord(s[i]) - ord('0')) - foundDigit = true - of 'a'..'f': - number = number shl 4 or (ord(s[i]) - ord('a') + 10) - foundDigit = true - of 'A'..'F': - number = number shl 4 or (ord(s[i]) - ord('A') + 10) + of '0'..'1': + output = output shl 1 or T(ord(s[i]) - ord('0')) foundDigit = true else: break inc(i) - if foundDigit: result = i-start + if foundDigit: + number = output + result = i -proc parseOct*(s: string, number: var int, start = 0): int {. - rtl, extern: "npuParseOct", noSideEffect.} = - ## parses an octal number and stores its value in ``number``. Returns - ## the number of the parsed characters or 0 in case of an error. - var i = start +proc parseOct*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = + ## Parses an octal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-oct character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseOct("0o23464755", num) == 10 + doAssert num == 5138925 + doAssert parseOct("8", num) == 0 + var num8: int8 + doAssert parseOct("0o_1464_755", num8) == 11 + doAssert num8 == -19 + doAssert parseOct("0o_1464_755", num8, 3, 3) == 3 + doAssert num8 == 102 + var num8u: uint8 + doAssert parseOct("1464755", num8u) == 7 + doAssert num8u == 237 + var num64: int64 + doAssert parseOct("2346475523464755", num64) == 16 + doAssert num64 == 86216859871725 + var i = 0 + var output = T(0) var foundDigit = false - if s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) - while true: + let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) + if i + 1 < last and s[i] == '0' and (s[i+1] in {'o', 'O'}): inc(i, 2) + while i < last: case s[i] of '_': discard of '0'..'7': - number = number shl 3 or (ord(s[i]) - ord('0')) + output = output shl 3 or T(ord(s[i]) - ord('0')) foundDigit = true else: break inc(i) - if foundDigit: result = i-start + if foundDigit: + number = output + result = i -proc parseIdent*(s: string, ident: var string, start = 0): int = - ## parses an identifier and stores it in ``ident``. Returns +proc parseHex*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = + ## Parses a hexadecimal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-hex character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseHex("4E_69_ED", num) == 8 + doAssert num == 5138925 + doAssert parseHex("X", num) == 0 + doAssert parseHex("#ABC", num) == 4 + var num8: int8 + doAssert parseHex("0x_4E_69_ED", num8) == 11 + doAssert num8 == 0xED'i8 + doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2 + doAssert num8 == 0x4E'i8 + var num8u: uint8 + doAssert parseHex("0x_4E_69_ED", num8u) == 11 + doAssert num8u == 237 + var num64: int64 + doAssert parseHex("4E69ED4E69ED", num64) == 12 + doAssert num64 == 86216859871725 + var i = 0 + var output = T(0) + var foundDigit = false + let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) + if i + 1 < last and s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2) + elif i < last and s[i] == '#': inc(i) + while i < last: + case s[i] + of '_': discard + of '0'..'9': + output = output shl 4 or T(ord(s[i]) - ord('0')) + foundDigit = true + of 'a'..'f': + output = output shl 4 or T(ord(s[i]) - ord('a') + 10) + foundDigit = true + of 'A'..'F': + output = output shl 4 or T(ord(s[i]) - ord('A') + 10) + foundDigit = true + else: break + inc(i) + if foundDigit: + number = output + result = i + +proc parseIdent*(s: openArray[char], ident: var string): int = + ## Parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. - var i = start - if s[i] in IdentStartChars: + ## If error, the value of `ident` is not changed. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" + var i = 0 + if i < s.len and s[i] in IdentStartChars: inc(i) - while s[i] in IdentChars: inc(i) - ident = substr(s, start, i-1) - result = i-start + while i < s.len and s[i] in IdentChars: inc(i) + ident = substr(s.toOpenArray(0, i-1)) + result = i -proc parseIdent*(s: string, start = 0): string = - ## parses an identifier and stores it in ``ident``. - ## Returns the parsed identifier or an empty string in case of an error. +proc parseIdent*(s: openArray[char]): string = + ## Parses an identifier and returns it or an empty string in + ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" result = "" - var i = start - - if s[i] in IdentStartChars: + var i = 0 + if i < s.len and s[i] in IdentStartChars: inc(i) - while s[i] in IdentChars: inc(i) + while i < s.len and s[i] in IdentChars: inc(i) + result = substr(s.toOpenArray(0, i - 1)) - result = substr(s, start, i-1) +proc parseChar*(s: openArray[char], c: var char): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + if s.len > 0: + c = s[0] + result = 1 -proc parseToken*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline, deprecated.} = - ## parses a token and stores it in ``token``. Returns - ## the number of the parsed characters or 0 in case of an error. A token - ## consists of the characters in `validChars`. - ## - ## **Deprecated since version 0.8.12**: Use ``parseWhile`` instead. - var i = start - while s[i] in validChars: inc(i) - result = i-start - token = substr(s, start, i-1) - -proc skipWhitespace*(s: string, start = 0): int {.inline.} = - ## skips the whitespace starting at ``s[start]``. Returns the number of +proc skipWhitespace*(s: openArray[char]): int {.inline.} = + ## Skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. - while s[start+result] in Whitespace: inc(result) + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 + result = 0 + while result < s.len and s[result] in Whitespace: inc(result) -proc skip*(s, token: string, start = 0): int {.inline.} = - ## skips the `token` starting at ``s[start]``. Returns the length of `token` +proc skip*(s, token: openArray[char]): int {.inline.} = + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. - while result < token.len and s[result+start] == token[result]: inc(result) + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 + result = 0 + while result < s.len and result < token.len and + s[result] == token[result]: + inc(result) if result != token.len: result = 0 -proc skipIgnoreCase*(s, token: string, start = 0): int = - ## same as `skip` but case is ignored for token matching. - while result < token.len and - toLower(s[result+start]) == toLower(token[result]): inc(result) +proc skipIgnoreCase*(s, token: openArray[char]): int = + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 + result = 0 + while result < s.len and result < token.len and + toLower(s[result]) == toLower(token[result]): inc(result) if result != token.len: result = 0 -proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: set[char]): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. - while s[result+start] notin until and s[result+start] != '\0': inc(result) + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 + result = 0 + while result < s.len and s[result] notin until: inc(result) -proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: char): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. - while s[result+start] != until and s[result+start] != '\0': inc(result) + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 + result = 0 + while result < s.len and s[result] != until: inc(result) -proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = - ## Skips all characters while one char from the set `token` is found. +proc skipWhile*(s: openArray[char], toSkip: set[char]): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. ## Returns number of characters skipped. - while s[result+start] in toSkip and s[result+start] != '\0': inc(result) + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 + result = 0 + while result < s.len and s[result] in toSkip: inc(result) -proc parseUntil*(s: string, token: var string, until: set[char], - start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns +proc fastSubstr(s: openArray[char]; token: var string; length: int) = + token.setLen length + for i in 0 ..< length: token[i] = s[i] + +proc parseUntil*(s: openArray[char], token: var string, until: set[char]): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters notin `until`. - var i = start + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " + var i = 0 while i < s.len and s[i] notin until: inc(i) - result = i-start - token = substr(s, start, i-1) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: char, - start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns +proc parseUntil*(s: openArray[char], token: var string, until: char): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that is not the `until` character. - var i = start + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" + var i = 0 while i < s.len and s[i] != until: inc(i) - result = i-start - token = substr(s, start, i-1) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc parseWhile*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline.} = - ## parses a token and stores it in ``token``. Returns +proc parseUntil*(s: openArray[char], token: var string, until: string): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " + when (NimMajor, NimMinor) <= (1, 0): + if until.len == 0: + token.setLen(0) + return 0 + var i = 0 + while i < s.len: + if until.len > 0 and s[i] == until[0]: + var u = 1 + while i+u < s.len and u < until.len and s[i+u] == until[u]: + inc u + if u >= until.len: break + inc(i) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) + +proc parseWhile*(s: openArray[char], token: var string, validChars: set[char]): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. - var i = start - while s[i] in validChars: inc(i) - result = i-start - token = substr(s, start, i-1) + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" + var i = 0 + while i < s.len and s[i] in validChars: inc(i) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = +proc captureBetween*(s: openArray[char], first: char, second = '\0'): string = ## Finds the first occurrence of ``first``, then returns everything from there - ## up to ``second``(if ``second`` is '\0', then ``first`` is used). - var i = skipUntil(s, first, start)+1+start + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World".toOpenArray(6, "Hello World".high), 'l') == "d" + var i = skipUntil(s, first) + 1 result = "" - discard s.parseUntil(result, if second == '\0': first else: second, i) + discard parseUntil(s.toOpenArray(i, s.high), result, if second == '\0': first else: second) + +proc integerOutOfRangeError() {.noinline.} = + raise newException(ValueError, "Parsed integer outside of valid range") -{.push overflowChecks: on.} -# this must be compiled with overflow checking turned on: -proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = +# See #6752 +when defined(js): + {.push overflowChecks: off.} + +proc rawParseInt(s: openArray[char], b: var BiggestInt): int = var sign: BiggestInt = -1 - i = start - if s[i] == '+': inc(i) - elif s[i] == '-': - inc(i) - sign = 1 - if s[i] in {'0'..'9'}: + i = 0 + if i < s.len: + if s[i] == '+': inc(i) + elif s[i] == '-': + inc(i) + sign = 1 + if i < s.len and s[i] in {'0'..'9'}: b = 0 - while s[i] in {'0'..'9'}: - b = b * 10 - (ord(s[i]) - ord('0')) + while i < s.len and s[i] in {'0'..'9'}: + let c = ord(s[i]) - ord('0') + if b >= (low(BiggestInt) + c) div 10: + b = b * 10 - c + else: + integerOutOfRangeError() inc(i) - while s[i] == '_': inc(i) # underscores are allowed and ignored - b = b * sign - result = i - start -{.pop.} # overflowChecks - -proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. - rtl, extern: "npuParseBiggestInt", noSideEffect.} = - ## parses an integer starting at `start` and stores the value into `number`. + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored + if sign == -1 and b == low(BiggestInt): + integerOutOfRangeError() + else: + b = b * sign + result = i + +when defined(js): + {.pop.} # overflowChecks: off + +proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {. + rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. - ## `EOverflow` is raised if an overflow occurs. - var res: BiggestInt + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res) == 19 + doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 + var res = BiggestInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseInt(s, res, start) - number = res + result = rawParseInt(s, res) + if result != 0: + number = res -proc parseInt*(s: string, number: var int, start = 0): int {. - rtl, extern: "npuParseInt", noSideEffect.} = - ## parses an integer starting at `start` and stores the value into `number`. +proc parseInt*(s: openArray[char], number: var int): int {. + rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. - ## `EOverflow` is raised if an overflow occurs. - var res: BiggestInt - result = parseBiggestInt(s, res, start) - if (sizeof(int) <= 4) and - ((res < low(int)) or (res > high(int))): - raise newException(OverflowError, "overflow") - elif result != 0: + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 + var res = BiggestInt(0) + result = parseBiggestInt(s, res) + when sizeof(int) <= 4: + if res < low(int) or res > high(int): + integerOutOfRangeError() + if result != 0: number = int(res) -proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. +proc parseSaturatedNatural*(s: openArray[char], b: var int): int {. + raises: [].} = + ## Parses a natural number into ``b``. This cannot raise an overflow + ## error. ``high(int)`` is returned for an overflow. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 + var i = 0 + if i < s.len and s[i] == '+': inc(i) + if i < s.len and s[i] in {'0'..'9'}: + b = 0 + while i < s.len and s[i] in {'0'..'9'}: + let c = ord(s[i]) - ord('0') + if b <= (high(int) - c) div 10: + b = b * 10 + c + else: + b = high(int) + inc(i) + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored + result = i + +proc rawParseUInt(s: openArray[char], b: var BiggestUInt): int = + var + res = 0.BiggestUInt + prev = 0.BiggestUInt + i = 0 + if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: + integerOutOfRangeError() + if i < s.len and s[i] == '+': inc(i) # Allow + if i < s.len and s[i] in {'0'..'9'}: + b = 0 + while i < s.len and s[i] in {'0'..'9'}: + prev = res + res = res * 10 + (ord(s[i]) - ord('0')).BiggestUInt + if prev > res: + integerOutOfRangeError() + inc(i) + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored + b = res + result = i + +proc parseBiggestUInt*(s: openArray[char], number: var BiggestUInt): int {. + rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 + var res = BiggestUInt(0) + # use 'res' for exception safety (don't write to 'number' in case of an + # overflow exception): + result = rawParseUInt(s, res) + if result != 0: + number = res + +proc parseUInt*(s: openArray[char], number: var uint): int {. + rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 + var res = BiggestUInt(0) + result = parseBiggestUInt(s, res) + when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: + if res > 0xFFFF_FFFF'u64: + integerOutOfRangeError() + if result != 0: + number = uint(res) + +proc parseBiggestFloat*(s: openArray[char], number: var BiggestFloat): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} - ## parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if a parsing error ## occurred. -proc parseFloat*(s: string, number: var float, start = 0): int {. +proc parseFloat*(s: openArray[char], number: var float): int {. rtl, extern: "npuParseFloat", noSideEffect.} = - ## parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if there occurred a parsing ## error. - var bf: BiggestFloat - result = parseBiggestFloat(s, bf, start) + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 + var bf = BiggestFloat(0.0) + result = parseBiggestFloat(s, bf) if result != 0: number = bf -type - InterpolatedKind* = enum ## describes for `interpolatedFragments` - ## which part of the interpolated string is - ## yielded; for example in "str$$$var${expr}" - ikStr, ## ``str`` part of the interpolated string - ikDollar, ## escaped ``$`` part of the interpolated string - ikVar, ## ``var`` part of the interpolated string - ikExpr ## ``expr`` part of the interpolated string +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c -{.deprecated: [TInterpolatedKind: InterpolatedKind].} - -iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, - value: string] = - ## Tokenizes the string `s` into substrings for interpolation purposes. +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). ## - ## Example: + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). ## - ## .. code-block:: nim - ## for k, v in interpolatedFragments(" $this is ${an example} $$"): - ## echo "(", k, ", \"", v, "\")" + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). ## - ## Results in: + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module<strutils.html>`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + +type + InterpolatedKind* = enum ## Describes for `interpolatedFragments` + ## which part of the interpolated string is + ## yielded; for example in "str$$$var${expr}" + ikStr, ## ``str`` part of the interpolated string + ikDollar, ## escaped ``$`` part of the interpolated string + ikVar, ## ``var`` part of the interpolated string + ikExpr ## ``expr`` part of the interpolated string + +iterator interpolatedFragments*(s: openArray[char]): tuple[kind: InterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. ## - ## .. code-block:: nim - ## (ikString, " ") - ## (ikExpr, "this") - ## (ikString, " is ") - ## (ikExpr, "an example") - ## (ikString, " ") - ## (ikDollar, "$") + runnableExamples: + var outp: seq[tuple[kind: InterpolatedKind, value: string]] + for k, v in interpolatedFragments(" $this is ${an example} $$"): + outp.add (k, v) + doAssert outp == @[(ikStr, " "), + (ikVar, "this"), + (ikStr, " is "), + (ikExpr, "an example"), + (ikStr, " "), + (ikDollar, "$")] + var i = 0 var kind: InterpolatedKind while true: var j = i - if s[j] == '$': - if s[j+1] == '{': + if j < s.len and s[j] == '$': + if j+1 < s.len and s[j+1] == '{': inc j, 2 var nesting = 0 - while true: - case s[j] - of '{': inc nesting - of '}': - if nesting == 0: - inc j - break - dec nesting - of '\0': - raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) - else: discard - inc j + block curlies: + while j < s.len: + case s[j] + of '{': inc nesting + of '}': + if nesting == 0: + inc j + break curlies + dec nesting + else: discard + inc j + raise newException(ValueError, + "Expected closing '}': " & substr(s.toOpenArray(i, s.high))) inc i, 2 # skip ${ kind = ikExpr - elif s[j+1] in IdentStartChars: + elif j+1 < s.len and s[j+1] in IdentStartChars: inc j, 2 - while s[j] in IdentChars: inc(j) + while j < s.len and s[j] in IdentChars: inc(j) inc i # skip $ kind = ikVar - elif s[j+1] == '$': + elif j+1 < s.len and s[j+1] == '$': inc j, 2 inc i # skip $ kind = ikDollar else: raise newException(ValueError, - "Unable to parse a varible name at " & substr(s, i, s.high)) + "Unable to parse a variable name at " & substr(s.toOpenArray(i, s.high))) else: while j < s.len and s[j] != '$': inc j kind = ikStr if j > i: # do not copy the trailing } for ikExpr: - yield (kind, substr(s, i, j-1-ord(kind == ikExpr))) + yield (kind, substr(s.toOpenArray(i, j-1-ord(kind == ikExpr)))) else: break i = j -when isMainModule: - import sequtils - let input = "$test{} $this is ${an{ example}} " - let expected = @[(ikVar, "test"), (ikStr, "{} "), (ikVar, "this"), - (ikStr, " is "), (ikExpr, "an{ example}"), (ikStr, " ")] - assert toSeq(interpolatedFragments(input)) == expected +{.pop.} - var value = 0 - discard parseHex("0x38", value) - assert value == 56 - discard parseHex("0x34", value) - assert value == 56 * 256 + 52 - value = -1 - discard parseHex("0x38", value) - assert value == -200 +proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a binary number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-bin character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29 + doAssert num == 5138925 + doAssert parseBin("3", num) == 0 + var num8: int8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32 + doAssert num8 == 0b1110_1101'i8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9 + doAssert num8 == 0b0100_1110'i8 + var num8u: uint8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32 + doAssert num8u == 237 + var num64: int64 + doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 + doAssert num64 == 336784608873 + parseBin(s.toOpenArray(start, s.high), number, maxLen) + +proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses an octal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-oct character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseOct("0o23464755", num) == 10 + doAssert num == 5138925 + doAssert parseOct("8", num) == 0 + var num8: int8 + doAssert parseOct("0o_1464_755", num8) == 11 + doAssert num8 == -19 + doAssert parseOct("0o_1464_755", num8, 3, 3) == 3 + doAssert num8 == 102 + var num8u: uint8 + doAssert parseOct("1464755", num8u) == 7 + doAssert num8u == 237 + var num64: int64 + doAssert parseOct("2346475523464755", num64) == 16 + doAssert num64 == 86216859871725 + parseOct(s.toOpenArray(start, s.high), number, maxLen) + +proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a hexadecimal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-hex character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseHex("4E_69_ED", num) == 8 + doAssert num == 5138925 + doAssert parseHex("X", num) == 0 + doAssert parseHex("#ABC", num) == 4 + var num8: int8 + doAssert parseHex("0x_4E_69_ED", num8) == 11 + doAssert num8 == 0xED'i8 + doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2 + doAssert num8 == 0x4E'i8 + var num8u: uint8 + doAssert parseHex("0x_4E_69_ED", num8u) == 11 + doAssert num8u == 237 + var num64: int64 + doAssert parseHex("4E69ED4E69ED", num64) == 12 + doAssert num64 == 86216859871725 + parseHex(s.toOpenArray(start, s.high), number, maxLen) + +proc parseIdent*(s: string, ident: var string, start = 0): int = + ## Parses an identifier and stores it in ``ident``. Returns + ## the number of the parsed characters or 0 in case of an error. + ## If error, the value of `ident` is not changed. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" + parseIdent(s.toOpenArray(start, s.high), ident) + +proc parseIdent*(s: string, start = 0): string = + ## Parses an identifier and returns it or an empty string in + ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" + parseIdent(s.toOpenArray(start, s.high)) + +proc parseChar*(s: string, c: var char, start = 0): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + parseChar(s.toOpenArray(start, s.high), c) + +proc skipWhitespace*(s: string, start = 0): int {.inline.} = + ## Skips the whitespace starting at ``s[start]``. Returns the number of + ## skipped characters. + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 + skipWhitespace(s.toOpenArray(start, s.high)) + +proc skip*(s, token: string, start = 0): int {.inline.} = + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` + ## or 0 if there was no `token` at ``s[start]``. + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 + skip(s.toOpenArray(start, s.high), token) + +proc skipIgnoreCase*(s, token: string, start = 0): int = + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 + skipIgnoreCase(s.toOpenArray(start, s.high), token) + +proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = + ## Skips all characters until one char from the set `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = + ## Skips all characters until the char `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 + skipWhile(s.toOpenArray(start, s.high), toSkip) + +proc parseUntil*(s: string, token: var string, until: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters notin `until`. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: char, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that is not the `until` character. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseWhile*(s: string, token: var string, validChars: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters in `validChars`. + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" + parseWhile(s.toOpenArray(start, s.high), token, validChars) + +proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = + ## Finds the first occurrence of ``first``, then returns everything from there + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World", 'l', start = 6) == "d" + captureBetween(s.toOpenArray(start, s.high), first, second) + +proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 + doAssert parseBiggestInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseBiggestInt(s.toOpenArray(start, s.high), number) + +proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 + doAssert parseInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseInt(s.toOpenArray(start, s.high), number) + + +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. + raises: [].} = + ## Parses a natural number into ``b``. This cannot raise an overflow + ## error. ``high(int)`` is returned for an overflow. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 + parseSaturatedNatural(s.toOpenArray(start, s.high), b) + + +proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 + parseBiggestUInt(s.toOpenArray(start, s.high), number) + +proc parseUInt*(s: string, number: var uint, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 + parseUInt(s.toOpenArray(start, s.high), number) + +proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if a parsing error + ## occurred. + parseFloat(s.toOpenArray(start, s.high), number) + +proc parseFloat*(s: string, number: var float, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there occurred a parsing + ## error. + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 + parseFloat(s.toOpenArray(start, s.high), number) + +iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. + ## + runnableExamples: + var outp: seq[tuple[kind: InterpolatedKind, value: string]] + for k, v in interpolatedFragments(" $this is ${an example} $$"): + outp.add (k, v) + doAssert outp == @[(ikStr, " "), + (ikVar, "this"), + (ikStr, " is "), + (ikExpr, "an example"), + (ikStr, " "), + (ikDollar, "$")] + for x in s.toOa.interpolatedFragments: + yield x -{.pop.} diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index f8b2c3d8d..c760799a2 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -26,81 +26,183 @@ ## creates. ## ## -## Example 1: Retrieve HTML title -## ============================== -## -## The file ``examples/htmltitle.nim`` demonstrates how to use the -## XML parser to accomplish a simple task: To determine the title of an HTML -## document. -## -## .. code-block:: nim -## :file: examples/htmltitle.nim -## -## -## Example 2: Retrieve all HTML links -## ================================== -## -## The file ``examples/htmlrefs.nim`` demonstrates how to use the -## XML parser to accomplish another simple task: To determine all the links -## an HTML document contains. -## -## .. code-block:: nim -## :file: examples/htmlrefs.nim -## + +##[ + +Example 1: Retrieve HTML title +============================== + +The file ``examples/htmltitle.nim`` demonstrates how to use the +XML parser to accomplish a simple task: To determine the title of an HTML +document. + + ```nim + # Example program to show the parsexml module + # This program reads an HTML file and writes its title to stdout. + # Errors and whitespace are ignored. + + import std/[os, streams, parsexml, strutils] + + if paramCount() < 1: + quit("Usage: htmltitle filename[.html]") + + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + while true: + x.next() + case x.kind + of xmlElementStart: + if cmpIgnoreCase(x.elementName, "title") == 0: + var title = "" + x.next() # skip "<title>" + while x.kind == xmlCharData: + title.add(x.charData) + x.next() + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: + echo("Title: " & title) + quit(0) # Success! + else: + echo(x.errorMsgExpected("/title")) + + of xmlEof: break # end of file reached + else: discard # ignore other events + + x.close() + quit("Could not determine title!") + ``` + +]## + +##[ + +Example 2: Retrieve all HTML links +================================== + +The file ``examples/htmlrefs.nim`` demonstrates how to use the +XML parser to accomplish another simple task: To determine all the links +an HTML document contains. + + ```nim + # Example program to show the new parsexml module + # This program reads an HTML file and writes all its used links to stdout. + # Errors and whitespace are ignored. + + import std/[os, streams, parsexml, strutils] + + proc `=?=` (a, b: string): bool = + # little trick: define our own comparator that ignores case + return cmpIgnoreCase(a, b) == 0 + + if paramCount() < 1: + quit("Usage: htmlrefs filename[.html]") + + var links = 0 # count the number of links + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + next(x) # get first event + block mainLoop: + while true: + case x.kind + of xmlElementOpen: + # the <a href = "xyz"> tag we are interested in always has an attribute, + # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` + if x.elementName =?= "a": + x.next() + if x.kind == xmlAttribute: + if x.attrKey =?= "href": + var link = x.attrValue + inc(links) + # skip until we have an ``xmlElementClose`` event + while true: + x.next() + case x.kind + of xmlEof: break mainLoop + of xmlElementClose: break + else: discard + x.next() # skip ``xmlElementClose`` + # now we have the description for the ``a`` element + var desc = "" + while x.kind == xmlCharData: + desc.add(x.charData) + x.next() + echo(desc & ": " & link) + else: + x.next() + of xmlEof: break # end of file reached + of xmlError: + echo(errorMsg(x)) + x.next() + else: x.next() # skip other events + + echo($links & " link(s) found!") + x.close() + ``` + +]## import - hashes, strutils, lexbase, streams, unicode + std/[strutils, lexbase, streams, unicode] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] # the parser treats ``<br />`` as ``<br></br>`` # xmlElementCloseEnd, ## ``/>`` type - XmlEventKind* = enum ## enumation of all events that may occur when parsing - xmlError, ## an error occurred during parsing - xmlEof, ## end of file reached - xmlCharData, ## character data - xmlWhitespace, ## whitespace has been parsed - xmlComment, ## a comment has been parsed - xmlPI, ## processing instruction (``<?name something ?>``) - xmlElementStart, ## ``<elem>`` - xmlElementEnd, ## ``</elem>`` - xmlElementOpen, ## ``<elem - xmlAttribute, ## ``key = "value"`` pair - xmlElementClose, ## ``>`` - xmlCData, ## ``<![CDATA[`` ... data ... ``]]>`` - xmlEntity, ## &entity; - xmlSpecial ## ``<! ... data ... >`` - - XmlErrorKind* = enum ## enumeration that lists all errors that can occur - errNone, ## no error - errEndOfCDataExpected, ## ``]]>`` expected - errNameExpected, ## name expected - errSemicolonExpected, ## ``;`` expected - errQmGtExpected, ## ``?>`` expected - errGtExpected, ## ``>`` expected - errEqExpected, ## ``=`` expected - errQuoteExpected, ## ``"`` or ``'`` expected - errEndOfCommentExpected ## ``-->`` expected + XmlEventKind* = enum ## enumeration of all events that may occur when parsing + xmlError, ## an error occurred during parsing + xmlEof, ## end of file reached + xmlCharData, ## character data + xmlWhitespace, ## whitespace has been parsed + xmlComment, ## a comment has been parsed + xmlPI, ## processing instruction (``<?name something ?>``) + xmlElementStart, ## ``<elem>`` + xmlElementEnd, ## ``</elem>`` + xmlElementOpen, ## ``<elem + xmlAttribute, ## ``key = "value"`` pair + xmlElementClose, ## ``>`` + xmlCData, ## ``<![CDATA[`` ... data ... ``]]>`` + xmlEntity, ## &entity; + xmlSpecial ## ``<! ... data ... >`` + + XmlErrorKind* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errEndOfCDataExpected, ## ``]]>`` expected + errNameExpected, ## name expected + errSemicolonExpected, ## ``;`` expected + errQmGtExpected, ## ``?>`` expected + errGtExpected, ## ``>`` expected + errEqExpected, ## ``=`` expected + errQuoteExpected, ## ``"`` or ``'`` expected + errEndOfCommentExpected ## ``-->`` expected + errAttributeValueExpected ## non-empty attribute value expected ParserState = enum stateStart, stateNormal, stateAttr, stateEmptyElementTag, stateError - XmlParseOption* = enum ## options for the XML parser - reportWhitespace, ## report whitespace - reportComments ## report comments + XmlParseOption* = enum ## options for the XML parser + reportWhitespace, ## report whitespace + reportComments ## report comments + allowUnquotedAttribs ## allow unquoted attribute values (for HTML) + allowEmptyAttribs ## allow empty attributes (without explicit value) XmlParser* = object of BaseLexer ## the parser object. a, b, c: string kind: XmlEventKind err: XmlErrorKind state: ParserState + cIsEmpty: bool filename: string options: set[XmlParseOption] -{.deprecated: [TXmlParser: XmlParser, TXmlParseOptions: XmlParseOption, - TXmlError: XmlErrorKind, TXmlEventKind: XmlEventKind].} - const errorMessages: array[XmlErrorKind, string] = [ "no error", @@ -111,7 +213,8 @@ const "'>' expected", "'=' expected", "'\"' or \"'\" expected", - "'-->' expected" + "'-->' expected", + "attribute value expected" ] proc open*(my: var XmlParser, input: Stream, filename: string, @@ -128,7 +231,8 @@ proc open*(my: var XmlParser, input: Stream, filename: string, my.kind = xmlError my.a = "" my.b = "" - my.c = nil + my.c = "" + my.cIsEmpty = true my.options = options proc close*(my: var XmlParser) {.inline.} = @@ -142,6 +246,9 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) my.a @@ -149,43 +256,61 @@ template charData*(my: XmlParser): string = template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) my.a template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlEntity``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlEntity) my.a template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.a template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.b template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.a template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.b -proc rawData*(my: XmlParser): string {.inline.} = +proc rawData*(my: var XmlParser): lent string {.inline.} = ## returns the underlying 'data' string by reference. ## This is only used for speed hacks. - shallowCopy(result, my.a) + result = my.a -proc rawData2*(my: XmlParser): string {.inline.} = +proc rawData2*(my: var XmlParser): lent string {.inline.} = ## returns the underlying second 'data' string by reference. ## This is only used for speed hacks. - shallowCopy(result, my.b) + result = my.b proc getColumn*(my: XmlParser): int {.inline.} = ## get the current column the parser has arrived at. @@ -223,11 +348,10 @@ proc markError(my: var XmlParser, kind: XmlErrorKind) {.inline.} = proc parseCDATA(my: var XmlParser) = var pos = my.bufpos + len("<![CDATA[") - var buf = my.buf while true: - case buf[pos] + case my.buf[pos] of ']': - if buf[pos+1] == ']' and buf[pos+2] == '>': + if my.buf[pos+1] == ']' and my.buf[pos+2] == '>': inc(pos, 3) break add(my.a, ']') @@ -237,29 +361,25 @@ proc parseCDATA(my: var XmlParser) = break of '\c': pos = lexbase.handleCR(my, pos) - buf = my.buf add(my.a, '\L') of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf add(my.a, '\L') of '/': pos = lexbase.handleRefillChar(my, pos) - buf = my.buf add(my.a, '/') else: - add(my.a, buf[pos]) + add(my.a, my.buf[pos]) inc(pos) my.bufpos = pos # store back my.kind = xmlCData proc parseComment(my: var XmlParser) = var pos = my.bufpos + len("<!--") - var buf = my.buf while true: - case buf[pos] + case my.buf[pos] of '-': - if buf[pos+1] == '-' and buf[pos+2] == '>': + if my.buf[pos+1] == '-' and my.buf[pos+2] == '>': inc(pos, 3) break if my.options.contains(reportComments): add(my.a, '-') @@ -269,38 +389,32 @@ proc parseComment(my: var XmlParser) = break of '\c': pos = lexbase.handleCR(my, pos) - buf = my.buf if my.options.contains(reportComments): add(my.a, '\L') of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf if my.options.contains(reportComments): add(my.a, '\L') of '/': pos = lexbase.handleRefillChar(my, pos) - buf = my.buf if my.options.contains(reportComments): add(my.a, '/') else: - if my.options.contains(reportComments): add(my.a, buf[pos]) + if my.options.contains(reportComments): add(my.a, my.buf[pos]) inc(pos) my.bufpos = pos my.kind = xmlComment -proc parseWhitespace(my: var XmlParser, skip=false) = +proc parseWhitespace(my: var XmlParser, skip = false) = var pos = my.bufpos - var buf = my.buf while true: - case buf[pos] + case my.buf[pos] of ' ', '\t': - if not skip: add(my.a, buf[pos]) + if not skip: add(my.a, my.buf[pos]) inc(pos) of '\c': # the specification says that CR-LF, CR are to be transformed to LF pos = lexbase.handleCR(my, pos) - buf = my.buf if not skip: add(my.a, '\L') of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf if not skip: add(my.a, '\L') else: break @@ -312,82 +426,82 @@ const proc parseName(my: var XmlParser, dest: var string) = var pos = my.bufpos - var buf = my.buf - if buf[pos] in NameStartChar: + if my.buf[pos] in NameStartChar: while true: - add(dest, buf[pos]) + add(dest, my.buf[pos]) inc(pos) - if buf[pos] notin NameChar: break + if my.buf[pos] notin NameChar: break my.bufpos = pos else: markError(my, errNameExpected) proc parseEntity(my: var XmlParser, dest: var string) = var pos = my.bufpos+1 - var buf = my.buf my.kind = xmlCharData - if buf[pos] == '#': + if my.buf[pos] == '#': var r: int inc(pos) - if buf[pos] == 'x': + if my.buf[pos] == 'x': inc(pos) while true: - case buf[pos] - of '0'..'9': r = (r shl 4) or (ord(buf[pos]) - ord('0')) - of 'a'..'f': r = (r shl 4) or (ord(buf[pos]) - ord('a') + 10) - of 'A'..'F': r = (r shl 4) or (ord(buf[pos]) - ord('A') + 10) + case my.buf[pos] + of '0'..'9': r = (r shl 4) or (ord(my.buf[pos]) - ord('0')) + of 'a'..'f': r = (r shl 4) or (ord(my.buf[pos]) - ord('a') + 10) + of 'A'..'F': r = (r shl 4) or (ord(my.buf[pos]) - ord('A') + 10) else: break inc(pos) else: - while buf[pos] in {'0'..'9'}: - r = r * 10 + (ord(buf[pos]) - ord('0')) + while my.buf[pos] in {'0'..'9'}: + r = r * 10 + (ord(my.buf[pos]) - ord('0')) inc(pos) add(dest, toUTF8(Rune(r))) - elif buf[pos] == 'l' and buf[pos+1] == 't' and buf[pos+2] == ';': + elif my.buf[pos] == 'l' and my.buf[pos+1] == 't' and my.buf[pos+2] == ';': add(dest, '<') inc(pos, 2) - elif buf[pos] == 'g' and buf[pos+1] == 't' and buf[pos+2] == ';': + elif my.buf[pos] == 'g' and my.buf[pos+1] == 't' and my.buf[pos+2] == ';': add(dest, '>') inc(pos, 2) - elif buf[pos] == 'a' and buf[pos+1] == 'm' and buf[pos+2] == 'p' and - buf[pos+3] == ';': + elif my.buf[pos] == 'a' and my.buf[pos+1] == 'm' and my.buf[pos+2] == 'p' and + my.buf[pos+3] == ';': add(dest, '&') inc(pos, 3) - elif buf[pos] == 'a' and buf[pos+1] == 'p' and buf[pos+2] == 'o' and - buf[pos+3] == 's' and buf[pos+4] == ';': + elif my.buf[pos] == 'a' and my.buf[pos+1] == 'p' and my.buf[pos+2] == 'o' and + my.buf[pos+3] == 's' and my.buf[pos+4] == ';': add(dest, '\'') inc(pos, 4) - elif buf[pos] == 'q' and buf[pos+1] == 'u' and buf[pos+2] == 'o' and - buf[pos+3] == 't' and buf[pos+4] == ';': + elif my.buf[pos] == 'q' and my.buf[pos+1] == 'u' and my.buf[pos+2] == 'o' and + my.buf[pos+3] == 't' and my.buf[pos+4] == ';': add(dest, '"') inc(pos, 4) else: my.bufpos = pos - parseName(my, dest) + var name = "" + parseName(my, name) pos = my.bufpos - if my.err != errNameExpected: + if my.err != errNameExpected and my.buf[pos] == ';': my.kind = xmlEntity else: add(dest, '&') - if buf[pos] == ';': + add(dest, name) + if my.buf[pos] == ';': inc(pos) else: - markError(my, errSemicolonExpected) + my.err = errSemicolonExpected + # do not overwrite 'my.state' here, it's a benign error my.bufpos = pos proc parsePI(my: var XmlParser) = inc(my.bufpos, "<?".len) parseName(my, my.a) var pos = my.bufpos - var buf = my.buf setLen(my.b, 0) while true: - case buf[pos] + case my.buf[pos] of '\0': markError(my, errQmGtExpected) break of '?': - if buf[pos+1] == '>': + if my.buf[pos+1] == '>': inc(pos, 2) break add(my.b, '?') @@ -395,18 +509,15 @@ proc parsePI(my: var XmlParser) = of '\c': # the specification says that CR-LF, CR are to be transformed to LF pos = lexbase.handleCR(my, pos) - buf = my.buf add(my.b, '\L') of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf add(my.b, '\L') of '/': pos = lexbase.handleRefillChar(my, pos) - buf = my.buf add(my.b, '/') else: - add(my.b, buf[pos]) + add(my.b, my.buf[pos]) inc(pos) my.bufpos = pos my.kind = xmlPI @@ -414,10 +525,9 @@ proc parsePI(my: var XmlParser) = proc parseSpecial(my: var XmlParser) = # things that start with <! var pos = my.bufpos + 2 - var buf = my.buf var opentags = 0 while true: - case buf[pos] + case my.buf[pos] of '\0': markError(my, errGtExpected) break @@ -434,18 +544,15 @@ proc parseSpecial(my: var XmlParser) = add(my.a, '>') of '\c': pos = lexbase.handleCR(my, pos) - buf = my.buf add(my.a, '\L') of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf add(my.a, '\L') of '/': pos = lexbase.handleRefillChar(my, pos) - buf = my.buf add(my.b, '/') else: - add(my.a, buf[pos]) + add(my.a, my.buf[pos]) inc(pos) my.bufpos = pos my.kind = xmlSpecial @@ -458,12 +565,13 @@ proc parseTag(my: var XmlParser) = my.kind = xmlCharData add(my.a, '<') return - parseWhitespace(my, skip=true) + parseWhitespace(my, skip = true) if my.buf[my.bufpos] in NameStartChar: # an attribute follows: my.kind = xmlElementOpen my.state = stateAttr my.c = my.a # save for later + my.cIsEmpty = false else: my.kind = xmlElementStart let slash = my.buf[my.bufpos] == '/' @@ -472,7 +580,8 @@ proc parseTag(my: var XmlParser) = if slash and my.buf[my.bufpos] == '>': inc(my.bufpos) my.state = stateEmptyElementTag - my.c = nil + my.c = "" + my.cIsEmpty = true elif my.buf[my.bufpos] == '>': inc(my.bufpos) else: @@ -482,7 +591,7 @@ proc parseEndTag(my: var XmlParser) = my.bufpos = lexbase.handleRefillChar(my, my.bufpos+1) #inc(my.bufpos, 2) parseName(my, my.a) - parseWhitespace(my, skip=true) + parseWhitespace(my, skip = true) if my.buf[my.bufpos] == '>': inc(my.bufpos) else: @@ -498,21 +607,25 @@ proc parseAttribute(my: var XmlParser) = if my.a.len == 0: markError(my, errGtExpected) return - parseWhitespace(my, skip=true) + + let startPos = my.bufpos + parseWhitespace(my, skip = true) if my.buf[my.bufpos] != '=': - markError(my, errEqExpected) + if allowEmptyAttribs notin my.options or + (my.buf[my.bufpos] != '>' and my.bufpos == startPos): + markError(my, errEqExpected) return + inc(my.bufpos) - parseWhitespace(my, skip=true) + parseWhitespace(my, skip = true) var pos = my.bufpos - var buf = my.buf - if buf[pos] in {'\'', '"'}: - var quote = buf[pos] + if my.buf[pos] in {'\'', '"'}: + var quote = my.buf[pos] var pendingSpace = false inc(pos) while true: - case buf[pos] + case my.buf[pos] of '\0': markError(my, errQuoteExpected) break @@ -529,52 +642,67 @@ proc parseAttribute(my: var XmlParser) = inc(pos) of '\c': pos = lexbase.handleCR(my, pos) - buf = my.buf pendingSpace = true of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf pendingSpace = true of '/': pos = lexbase.handleRefillChar(my, pos) - buf = my.buf add(my.b, '/') else: - if buf[pos] == quote: + if my.buf[pos] == quote: inc(pos) break else: if pendingSpace: add(my.b, ' ') pendingSpace = false - add(my.b, buf[pos]) + add(my.b, my.buf[pos]) inc(pos) + elif allowUnquotedAttribs in my.options: + const disallowedChars = {'"', '\'', '`', '=', '<', '>', ' ', + '\0', '\t', '\L', '\F', '\f'} + let startPos = pos + while (let c = my.buf[pos]; c notin disallowedChars): + if c == '&': + my.bufpos = pos + parseEntity(my, my.b) + my.kind = xmlAttribute # parseEntity overwrites my.kind! + pos = my.bufpos + elif c == '/': + pos = lexbase.handleRefillChar(my, pos) + add(my.b, '/') + else: + add(my.b, c) + inc(pos) + if pos == startPos: + markError(my, errAttributeValueExpected) else: markError(my, errQuoteExpected) + # error corrections: guess what was meant + while my.buf[pos] != '>' and my.buf[pos] > ' ': + add(my.b, my.buf[pos]) + inc pos my.bufpos = pos - parseWhitespace(my, skip=true) + parseWhitespace(my, skip = true) proc parseCharData(my: var XmlParser) = var pos = my.bufpos - var buf = my.buf while true: - case buf[pos] + case my.buf[pos] of '\0', '<', '&': break of '\c': # the specification says that CR-LF, CR are to be transformed to LF pos = lexbase.handleCR(my, pos) - buf = my.buf add(my.a, '\L') of '\L': pos = lexbase.handleLF(my, pos) - buf = my.buf add(my.a, '\L') of '/': pos = lexbase.handleRefillChar(my, pos) - buf = my.buf add(my.a, '/') else: - add(my.a, buf[pos]) + add(my.a, my.buf[pos]) inc(pos) my.bufpos = pos my.kind = xmlCharData @@ -583,18 +711,18 @@ proc rawGetTok(my: var XmlParser) = my.kind = xmlError setLen(my.a, 0) var pos = my.bufpos - var buf = my.buf - case buf[pos] + case my.buf[pos] of '<': - case buf[pos+1] + case my.buf[pos+1] of '/': parseEndTag(my) of '!': - if buf[pos+2] == '[' and buf[pos+3] == 'C' and buf[pos+4] == 'D' and - buf[pos+5] == 'A' and buf[pos+6] == 'T' and buf[pos+7] == 'A' and - buf[pos+8] == '[': + if my.buf[pos+2] == '[' and my.buf[pos+3] == 'C' and + my.buf[pos+4] == 'D' and my.buf[pos+5] == 'A' and + my.buf[pos+6] == 'T' and my.buf[pos+7] == 'A' and + my.buf[pos+8] == '[': parseCDATA(my) - elif buf[pos+2] == '-' and buf[pos+3] == '-': + elif my.buf[pos+2] == '-' and my.buf[pos+3] == '-': parseComment(my) else: parseSpecial(my) @@ -615,12 +743,15 @@ proc rawGetTok(my: var XmlParser) = proc getTok(my: var XmlParser) = while true: + let lastKind = my.kind rawGetTok(my) case my.kind of xmlComment: if my.options.contains(reportComments): break of xmlWhitespace: - if my.options.contains(reportWhitespace): break + if my.options.contains(reportWhitespace) or lastKind in {xmlCharData, + xmlComment, xmlEntity}: + break else: break proc next*(my: var XmlParser) = @@ -654,14 +785,14 @@ proc next*(my: var XmlParser) = of stateEmptyElementTag: my.state = stateNormal my.kind = xmlElementEnd - if not my.c.isNil: + if not my.cIsEmpty: my.a = my.c of stateError: my.kind = xmlError my.state = stateNormal when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: XmlParser @@ -691,4 +822,3 @@ when not defined(testing) and isMainModule: of xmlSpecial: echo("SPECIAL: " & x.charData) close(x) - diff --git a/lib/pure/pathnorm.nim b/lib/pure/pathnorm.nim new file mode 100644 index 000000000..4cdc02303 --- /dev/null +++ b/lib/pure/pathnorm.nim @@ -0,0 +1,121 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## OS-Path normalization. Used by `os.nim` but also +## generally useful for dealing with paths. +## +## Unstable API. + +# Yes, this uses import here, not include so that +# we don't end up exporting these symbols from pathnorm and os: +import std/private/osseps + +type + PathIter* = object + i, prev: int + notFirst: bool + +proc hasNext*(it: PathIter; x: string): bool = + it.i < x.len + +proc next*(it: var PathIter; x: string): (int, int) = + it.prev = it.i + if not it.notFirst and x[it.i] in {DirSep, AltSep}: + # absolute path: + inc it.i + else: + while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i + if it.i > it.prev: + result = (it.prev, it.i-1) + elif hasNext(it, x): + result = next(it, x) + # skip all separators: + while it.i < x.len and x[it.i] in {DirSep, AltSep}: inc it.i + it.notFirst = true + +iterator dirs(x: string): (int, int) = + var it = default PathIter + while hasNext(it, x): yield next(it, x) + +proc isDot(x: string; bounds: (int, int)): bool = + bounds[1] == bounds[0] and x[bounds[0]] == '.' + +proc isDotDot(x: string; bounds: (int, int)): bool = + bounds[1] == bounds[0] + 1 and x[bounds[0]] == '.' and x[bounds[0]+1] == '.' + +proc isSlash(x: string; bounds: (int, int)): bool = + bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep} + +when doslikeFileSystem: + import std/private/ntpath + +proc addNormalizePath*(x: string; result: var string; state: var int; + dirSep = DirSep) = + ## Low level proc. Undocumented. + + when doslikeFileSystem: # Add Windows drive at start without normalization + var x = x + if result == "": + let (drive, file) = splitDrive(x) + x = file + result.add drive + for c in result.mitems: + if c in {DirSep, AltSep}: + c = dirSep + + # state: 0th bit set if isAbsolute path. Other bits count + # the number of path components. + var it: PathIter + it.notFirst = (state shr 1) > 0 + if it.notFirst: + while it.i < x.len and x[it.i] in {DirSep, AltSep}: inc it.i + while hasNext(it, x): + let b = next(it, x) + if (state shr 1 == 0) and isSlash(x, b): + if result.len == 0 or result[result.len - 1] notin {DirSep, AltSep}: + result.add dirSep + state = state or 1 + elif isDotDot(x, b): + if (state shr 1) >= 1: + var d = result.len + # f/.. + # We could handle stripping trailing sep here: foo// => foo like this: + # while (d-1) > (state and 1) and result[d-1] in {DirSep, AltSep}: dec d + # but right now we instead handle it inside os.joinPath + + # strip path component: foo/bar => foo + while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}: + dec d + if d > 0: + setLen(result, d-1) + dec state, 2 + else: + if result.len > 0 and result[result.len - 1] notin {DirSep, AltSep}: + result.add dirSep + result.add substr(x, b[0], b[1]) + elif isDot(x, b): + discard "discard the dot" + elif b[1] >= b[0]: + if result.len > 0 and result[result.len - 1] notin {DirSep, AltSep}: + result.add dirSep + result.add substr(x, b[0], b[1]) + inc state, 2 + if result == "" and x != "": result = "." + +proc normalizePath*(path: string; dirSep = DirSep): string = + runnableExamples: + when defined(posix): + doAssert normalizePath("./foo//bar/../baz") == "foo/baz" + + ## - Turns multiple slashes into single slashes. + ## - Resolves `'/foo/../bar'` to `'/bar'`. + ## - Removes `'./'` from the path, but `"foo/.."` becomes `"."`. + result = newStringOfCap(path.len) + var state = 0 + addNormalizePath(path, result, state, dirSep) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index fead66de2..2969fd6d7 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -12,19 +12,22 @@ ## Matching performance is hopefully competitive with optimized regular ## expression engines. ## -## .. include:: ../doc/pegdocs.txt +## .. include:: ../../doc/pegdocs.txt ## include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] const useUnicode = true ## change this to deactivate proper UTF-8 support -import - strutils +import std/[strutils, macros] +import std/private/decode_helpers when useUnicode: - import unicode + import std/unicode + export unicode.`==` const InlineThreshold = 5 ## number of leaves; -1 to disable inlining @@ -32,102 +35,147 @@ const ## can be captured. More subpatterns cannot be captured! type - PegKind = enum + PegKind* = enum pkEmpty, - pkAny, ## any character (.) - pkAnyRune, ## any Unicode character (_) - pkNewLine, ## CR-LF, LF, CR - pkLetter, ## Unicode letter - pkLower, ## Unicode lower case letter - pkUpper, ## Unicode upper case letter - pkTitle, ## Unicode title character - pkWhitespace, ## Unicode whitespace character + pkAny, ## any character (.) + pkAnyRune, ## any Unicode character (_) + pkNewLine, ## CR-LF, LF, CR + pkLetter, ## Unicode letter + pkLower, ## Unicode lower case letter + pkUpper, ## Unicode upper case letter + pkTitle, ## Unicode title character + pkWhitespace, ## Unicode whitespace character pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, - pkChar, ## single character to match + pkChar, ## single character to match pkCharChoice, pkNonTerminal, - pkSequence, ## a b c ... --> Internal DSL: peg(a, b, c) - pkOrderedChoice, ## a / b / ... --> Internal DSL: a / b or /[a, b, c] - pkGreedyRep, ## a* --> Internal DSL: *a - ## a+ --> (a a*) - pkGreedyRepChar, ## x* where x is a single character (superop) - pkGreedyRepSet, ## [set]* (superop) - pkGreedyAny, ## .* or _* (superop) - pkOption, ## a? --> Internal DSL: ?a - pkAndPredicate, ## &a --> Internal DSL: &a - pkNotPredicate, ## !a --> Internal DSL: !a - pkCapture, ## {a} --> Internal DSL: capture(a) - pkBackRef, ## $i --> Internal DSL: backref(i) + pkSequence, ## a b c ... --> Internal DSL: peg(a, b, c) + pkOrderedChoice, ## a / b / ... --> Internal DSL: a / b or /[a, b, c] + pkGreedyRep, ## a* --> Internal DSL: *a + ## a+ --> (a a*) + pkGreedyRepChar, ## x* where x is a single character (superop) + pkGreedyRepSet, ## [set]* (superop) + pkGreedyAny, ## .* or _* (superop) + pkOption, ## a? --> Internal DSL: ?a + pkAndPredicate, ## &a --> Internal DSL: &a + pkNotPredicate, ## !a --> Internal DSL: !a + pkCapture, ## {a} --> Internal DSL: capture(a) + pkBackRef, ## $i --> Internal DSL: backref(i) pkBackRefIgnoreCase, pkBackRefIgnoreStyle, - pkSearch, ## @a --> Internal DSL: !*a - pkCapturedSearch, ## {@} a --> Internal DSL: !*\a - pkRule, ## a <- b - pkList, ## a, b - pkStartAnchor ## ^ --> Internal DSL: startAnchor() - NonTerminalFlag = enum + pkSearch, ## @a --> Internal DSL: !*a + pkCapturedSearch, ## {@} a --> Internal DSL: !*\a + pkRule, ## a <- b + pkList, ## a, b + pkStartAnchor ## ^ --> Internal DSL: startAnchor() + NonTerminalFlag* = enum ntDeclared, ntUsed - NonTerminalObj = object ## represents a non terminal symbol - name: string ## the name of the symbol - line: int ## line the symbol has been declared/used in - col: int ## column the symbol has been declared/used in - flags: set[NonTerminalFlag] ## the nonterminal's flags - rule: Node ## the rule that the symbol refers to - Node {.shallow.} = object + NonTerminalObj = object ## represents a non terminal symbol + name: string ## the name of the symbol + line: int ## line the symbol has been declared/used in + col: int ## column the symbol has been declared/used in + flags: set[NonTerminalFlag] ## the nonterminal's flags + rule: Peg ## the rule that the symbol refers to + Peg* {.shallow.} = object ## type that represents a PEG case kind: PegKind of pkEmpty..pkWhitespace: nil of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle: term: string of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] - else: sons: seq[Node] + of pkBackRef..pkBackRefIgnoreStyle: index: range[-MaxSubpatterns..MaxSubpatterns-1] + else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj - Peg* = Node ## type that represents a PEG +func kind*(p: Peg): PegKind = p.kind + ## Returns the *PegKind* of a given *Peg* object. -{.deprecated: [TPeg: Peg, TNode: Node].} +func term*(p: Peg): string = p.term + ## Returns the *string* representation of a given *Peg* variant object + ## where present. -proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} = +func ch*(p: Peg): char = p.ch + ## Returns the *char* representation of a given *Peg* variant object + ## where present. + +func charChoice*(p: Peg): ref set[char] = p.charChoice + ## Returns the *charChoice* field of a given *Peg* variant object + ## where present. + +func nt*(p: Peg): NonTerminal = p.nt + ## Returns the *NonTerminal* object of a given *Peg* variant object + ## where present. + +func index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index + ## Returns the back-reference index of a captured sub-pattern in the + ## *Captures* object for a given *Peg* variant object where present. + +iterator items*(p: Peg): Peg {.inline.} = + ## Yields the child nodes of a *Peg* variant object where present. + for s in p.sons: + yield s + +iterator pairs*(p: Peg): (int, Peg) {.inline.} = + ## Yields the indices and child nodes of a *Peg* variant object where present. + for i in 0 ..< p.sons.len: + yield (i, p.sons[i]) + +func name*(nt: NonTerminal): string = nt.name + ## Gets the name of the symbol represented by the parent *Peg* object variant + ## of a given *NonTerminal*. + +func line*(nt: NonTerminal): int = nt.line + ## Gets the line number of the definition of the parent *Peg* object variant + ## of a given *NonTerminal*. + +func col*(nt: NonTerminal): int = nt.col + ## Gets the column number of the definition of the parent *Peg* object variant + ## of a given *NonTerminal*. + +func flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags + ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant + ## object of a given *NonTerminal*. + +func rule*(nt: NonTerminal): Peg = nt.rule + ## Gets the *Peg* object representing the rule definition of the parent *Peg* + ## object variant of a given *NonTerminal*. + +func term*(t: string): Peg {.rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: - result.kind = pkTerminal - result.term = t + result = Peg(kind: pkTerminal, term: t) else: - result.kind = pkChar - result.ch = t[0] + result = Peg(kind: pkChar, ch: t[0]) -proc termIgnoreCase*(t: string): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = +func termIgnoreCase*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore case for matching - result.kind = pkTerminalIgnoreCase - result.term = t + result = Peg(kind: pkTerminalIgnoreCase, term: t) -proc termIgnoreStyle*(t: string): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = +func termIgnoreStyle*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore style for matching - result.kind = pkTerminalIgnoreStyle - result.term = t + result = Peg(kind: pkTerminalIgnoreStyle, term: t) -proc term*(t: char): Peg {.nosideEffect, rtl, extern: "npegs$1Char".} = +func term*(t: char): Peg {.rtl, extern: "npegs$1Char".} = ## constructs a PEG from a terminal char assert t != '\0' - result.kind = pkChar - result.ch = t + result = Peg(kind: pkChar, ch: t) -proc charSet*(s: set[char]): Peg {.nosideEffect, rtl, extern: "npegs$1".} = +func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s - result.kind = pkCharChoice - new(result.charChoice) - result.charChoice[] = s + result = Peg(kind: pkCharChoice) + {.cast(noSideEffect).}: + new(result.charChoice) + result.charChoice[] = s -proc len(a: Peg): int {.inline.} = return a.sons.len -proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) +func len(a: Peg): int {.inline.} = return a.sons.len +func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) -proc addChoice(dest: var Peg, elem: Peg) = +func addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! @@ -139,9 +187,8 @@ proc addChoice(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -template multipleOp(k: PegKind, localOpt: expr) = - result.kind = k - result.sons = @[] +template multipleOp(k: PegKind, localOpt: untyped) = + result = Peg(kind: k, sons: @[]) for x in items(a): if x.kind == k: for y in items(x.sons): @@ -151,12 +198,12 @@ template multipleOp(k: PegKind, localOpt: expr) = if result.len == 1: result = result.sons[0] -proc `/`*(a: varargs[Peg]): Peg {. - nosideEffect, rtl, extern: "npegsOrderedChoice".} = +func `/`*(a: varargs[Peg]): Peg {. + rtl, extern: "npegsOrderedChoice".} = ## constructs an ordered choice with the PEGs in `a` multipleOp(pkOrderedChoice, addChoice) -proc addSequence(dest: var Peg, elem: Peg) = +func addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! @@ -168,12 +215,12 @@ proc addSequence(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -proc sequence*(a: varargs[Peg]): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = +func sequence*(a: varargs[Peg]): Peg {. + rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) -proc `?`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsOptional".} = +func `?`*(a: Peg): Peg {.rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, pkGreedyRepSet}: @@ -181,125 +228,116 @@ proc `?`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsOptional".} = # a? ? --> a? result = a else: - result.kind = pkOption - result.sons = @[a] + result = Peg(kind: pkOption, sons: @[a]) -proc `*`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsGreedyRep".} = +func `*`*(a: Peg): Peg {.rtl, extern: "npegsGreedyRep".} = ## constructs a "greedy repetition" for the PEG `a` case a.kind of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: assert false # produces endless loop! of pkChar: - result.kind = pkGreedyRepChar - result.ch = a.ch + result = Peg(kind: pkGreedyRepChar, ch: a.ch) of pkCharChoice: - result.kind = pkGreedyRepSet - result.charChoice = a.charChoice # copying a reference suffices! + result = Peg(kind: pkGreedyRepSet, charChoice: a.charChoice) of pkAny, pkAnyRune: - result.kind = pkGreedyAny + result = Peg(kind: pkGreedyAny) else: - result.kind = pkGreedyRep - result.sons = @[a] + result = Peg(kind: pkGreedyRep, sons: @[a]) -proc `!*`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsSearch".} = +func `!*`*(a: Peg): Peg {.rtl, extern: "npegsSearch".} = ## constructs a "search" for the PEG `a` - result.kind = pkSearch - result.sons = @[a] + result = Peg(kind: pkSearch, sons: @[a]) -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +func `!*\`*(a: Peg): Peg {.rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` - result.kind = pkCapturedSearch - result.sons = @[a] + result = Peg(kind: pkCapturedSearch, sons: @[a]) -proc `+`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsGreedyPosRep".} = +func `+`*(a: Peg): Peg {.rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) -proc `&`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsAndPredicate".} = +func `&`*(a: Peg): Peg {.rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` - result.kind = pkAndPredicate - result.sons = @[a] + result = Peg(kind: pkAndPredicate, sons: @[a]) -proc `!`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsNotPredicate".} = +func `!`*(a: Peg): Peg {.rtl, extern: "npegsNotPredicate".} = ## constructs a "not predicate" with the PEG `a` - result.kind = pkNotPredicate - result.sons = @[a] + result = Peg(kind: pkNotPredicate, sons: @[a]) -proc any*: Peg {.inline.} = +func any*: Peg {.inline.} = ## constructs the PEG `any character`:idx: (``.``) - result.kind = pkAny + result = Peg(kind: pkAny) -proc anyRune*: Peg {.inline.} = +func anyRune*: Peg {.inline.} = ## constructs the PEG `any rune`:idx: (``_``) - result.kind = pkAnyRune + result = Peg(kind: pkAnyRune) -proc newLine*: Peg {.inline.} = +func newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) - result.kind = pkNewLine + result = Peg(kind: pkNewLine) -proc unicodeLetter*: Peg {.inline.} = +func unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. - result.kind = pkLetter + result = Peg(kind: pkLetter) -proc unicodeLower*: Peg {.inline.} = +func unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. - result.kind = pkLower + result = Peg(kind: pkLower) -proc unicodeUpper*: Peg {.inline.} = +func unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. - result.kind = pkUpper + result = Peg(kind: pkUpper) -proc unicodeTitle*: Peg {.inline.} = +func unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. - result.kind = pkTitle + result = Peg(kind: pkTitle) -proc unicodeWhitespace*: Peg {.inline.} = +func unicodeWhitespace*: Peg {.inline.} = ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. - result.kind = pkWhitespace + result = Peg(kind: pkWhitespace) -proc startAnchor*: Peg {.inline.} = +func startAnchor*: Peg {.inline.} = ## constructs the PEG ``^`` which matches the start of the input. - result.kind = pkStartAnchor + result = Peg(kind: pkStartAnchor) -proc endAnchor*: Peg {.inline.} = +func endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsCapture".} = +func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` - result.kind = pkCapture - result.sons = @[a] + result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns]): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = +func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result.kind = pkBackRef - result.index = index-1 + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. + result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result.kind = pkBackRefIgnoreCase - result.index = index-1 + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores case for matching. + result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. - nosideEffect, rtl, extern: "npegs$1".}= +func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result.kind = pkBackRefIgnoreStyle - result.index = index-1 + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores style for matching. + result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) -proc spaceCost(n: Peg): int = +func spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: + pkAny..pkWhitespace, pkGreedyAny, pkBackRef..pkBackRefIgnoreStyle: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -309,57 +347,53 @@ proc spaceCost(n: Peg): int = inc(result, spaceCost(n.sons[i])) if result >= InlineThreshold: break -proc nonterminal*(n: NonTerminal): Peg {. - nosideEffect, rtl, extern: "npegs$1".} = +func nonterminal*(n: NonTerminal): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: when false: echo "inlining symbol: ", n.name result = n.rule # inlining of rule enables better optimizations else: - result.kind = pkNonTerminal - result.nt = n + result = Peg(kind: pkNonTerminal, nt: n) -proc newNonTerminal*(name: string, line, column: int): NonTerminal {. - nosideEffect, rtl, extern: "npegs$1".} = +func newNonTerminal*(name: string, line, column: int): NonTerminal {. + rtl, extern: "npegs$1".} = ## constructs a nonterminal symbol - new(result) - result.name = name - result.line = line - result.col = column + result = NonTerminal(name: name, line: line, col: column) -template letters*: expr = +template letters*: Peg = ## expands to ``charset({'A'..'Z', 'a'..'z'})`` charSet({'A'..'Z', 'a'..'z'}) -template digits*: expr = +template digits*: Peg = ## expands to ``charset({'0'..'9'})`` charSet({'0'..'9'}) -template whitespace*: expr = +template whitespace*: Peg = ## expands to ``charset({' ', '\9'..'\13'})`` charSet({' ', '\9'..'\13'}) -template identChars*: expr = +template identChars*: Peg = ## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})`` charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}) -template identStartChars*: expr = +template identStartChars*: Peg = ## expands to ``charset({'A'..'Z', 'a'..'z', '_'})`` charSet({'a'..'z', 'A'..'Z', '_'}) -template ident*: expr = +template ident*: Peg = ## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier sequence(charSet({'a'..'z', 'A'..'Z', '_'}), *charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})) -template natural*: expr = +template natural*: Peg = ## same as ``\d+`` +digits # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +func esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -371,18 +405,18 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = of '\a': result = "\\a" of '\\': result = "\\\\" of 'a'..'z', 'A'..'Z', '0'..'9', '_': result = $c - elif c < ' ' or c >= '\128': result = '\\' & $ord(c) + elif c < ' ' or c >= '\127': result = '\\' & $ord(c) elif c in reserved: result = '\\' & c else: result = $c -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" +func singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +func singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' -proc charSetEscAux(cc: set[char]): string = +func charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 @@ -399,13 +433,13 @@ proc charSetEscAux(cc: set[char]): string = c1 = c2 inc(c1) -proc charSetEsc(cc: set[char]): string = +func charSetEsc(cc: set[char]): string = if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' else: result = '[' & charSetEscAux(cc) & ']' -proc toStrAux(r: Peg, res: var string) = +func toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -491,7 +525,7 @@ proc toStrAux(r: Peg, res: var string) = of pkStartAnchor: add(res, '^') -proc `$` *(r: Peg): string {.nosideEffect, rtl, extern: "npegsToString".} = +func `$` *(r: Peg): string {.rtl, extern: "npegsToString".} = ## converts a PEG to its string representation result = "" toStrAux(r, result) @@ -504,9 +538,7 @@ type ml: int origStart: int -{.deprecated: [TCaptures: Captures].} - -proc bounds*(c: Captures, +func bounds*(c: Captures, i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] @@ -514,238 +546,543 @@ proc bounds*(c: Captures, when not useUnicode: type Rune = char - template fastRuneAt(s, i, ch: expr) = + template fastRuneAt(s, i, ch) = ch = s[i] inc(i) - template runeLenAt(s, i: expr): expr = 1 - - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z','A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} - -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. - nosideEffect, rtl, extern: "npegs$1".} = - ## low-level matching proc that implements the PEG interpreter. Use this - ## for maximum efficiency (every other PEG operation ends up calling this - ## proc). - ## Returns -1 if it does not match, else the length of the match - case p.kind - of pkEmpty: result = 0 # match of length 0 - of pkAny: - if s[start] != '\0': result = 1 - else: result = -1 - of pkAnyRune: - if s[start] != '\0': - result = runeLenAt(s, start) - else: - result = -1 - of pkLetter: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isAlpha(a): dec(result, start) + template runeLenAt(s, i): untyped = 1 + + func isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} + func isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + func isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + func isTitle(a: char): bool {.inline.} = return false + func isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + +template matchOrParse(mopProc: untyped) = + # Used to make the main matcher proc *rawMatch* as well as event parser + # procs. For the former, *enter* and *leave* event handler code generators + # are provided which just return *discard*. + + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.gcsafe, raises: [].} = + proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = + # Parse handler code must run in an *of* clause of its own for each + # *PegKind*, so we encapsulate the identical clause body for + # *pkBackRef..pkBackRefIgnoreStyle* here. + var index = p.index + if index < 0: index.inc(c.ml) + if index < 0 or index >= c.ml: return -1 + var (a, b) = c.matches[index] + var n: Peg + case p.kind + of pkBackRef: + n = Peg(kind: pkTerminal, term: s.substr(a, b)) + of pkBackRefIgnoreStyle: + n = Peg(kind: pkTerminalIgnoreStyle, term: s.substr(a, b)) + of pkBackRefIgnoreCase: + n = Peg(kind: pkTerminalIgnoreCase, term: s.substr(a, b)) + else: assert(false, "impossible case") + mopProc(s, n, start, c) + + case p.kind + of pkEmpty: + enter(pkEmpty, s, p, start) + result = 0 # match of length 0 + leave(pkEmpty, s, p, start, result) + of pkAny: + enter(pkAny, s, p, start) + if start < s.len: result = 1 else: result = -1 - else: - result = -1 - of pkLower: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isLower(a): dec(result, start) + leave(pkAny, s, p, start, result) + of pkAnyRune: + enter(pkAnyRune, s, p, start) + if start < s.len: + result = runeLenAt(s, start) + else: + result = -1 + leave(pkAnyRune, s, p, start, result) + of pkLetter: + enter(pkLetter, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isAlpha(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkLetter, s, p, start, result) + of pkLower: + enter(pkLower, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isLower(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkLower, s, p, start, result) + of pkUpper: + enter(pkUpper, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isUpper(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkUpper, s, p, start, result) + of pkTitle: + enter(pkTitle, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isTitle(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkTitle, s, p, start, result) + of pkWhitespace: + enter(pkWhitespace, s, p, start) + if start < s.len: + var a: Rune + result = start + fastRuneAt(s, result, a) + if isWhiteSpace(a): dec(result, start) + else: result = -1 + else: + result = -1 + leave(pkWhitespace, s, p, start, result) + of pkGreedyAny: + enter(pkGreedyAny, s, p, start) + result = len(s) - start + leave(pkGreedyAny, s, p, start, result) + of pkNewLine: + enter(pkNewLine, s, p, start) + if start < s.len and s[start] == '\L': result = 1 + elif start < s.len and s[start] == '\C': + if start+1 < s.len and s[start+1] == '\L': result = 2 + else: result = 1 else: result = -1 - else: - result = -1 - of pkUpper: - if s[start] != '\0': - var a: Rune + leave(pkNewLine, s, p, start, result) + of pkTerminal: + enter(pkTerminal, s, p, start) + result = len(p.term) + for i in 0..result-1: + if start+i >= s.len or p.term[i] != s[start+i]: + result = -1 + break + leave(pkTerminal, s, p, start, result) + of pkTerminalIgnoreCase: + enter(pkTerminalIgnoreCase, s, p, start) + var + i = 0 + a, b: Rune result = start - fastRuneAt(s, result, a) - if isUpper(a): dec(result, start) - else: result = -1 - else: - result = -1 - of pkTitle: - if s[start] != '\0': - var a: Rune + while i < len(p.term): + if result >= s.len: + result = -1 + break + fastRuneAt(p.term, i, a) + fastRuneAt(s, result, b) + if toLower(a) != toLower(b): + result = -1 + break + dec(result, start) + leave(pkTerminalIgnoreCase, s, p, start, result) + of pkTerminalIgnoreStyle: + enter(pkTerminalIgnoreStyle, s, p, start) + var + i = 0 + a, b: Rune result = start - fastRuneAt(s, result, a) - if isTitle(a): dec(result, start) + while i < len(p.term): + while i < len(p.term): + fastRuneAt(p.term, i, a) + if a != Rune('_'): break + while result < s.len: + fastRuneAt(s, result, b) + if b != Rune('_'): break + if result >= s.len: + if i >= p.term.len: break + else: + result = -1 + break + elif toLower(a) != toLower(b): + result = -1 + break + dec(result, start) + leave(pkTerminalIgnoreStyle, s, p, start, result) + of pkChar: + enter(pkChar, s, p, start) + if start < s.len and p.ch == s[start]: result = 1 else: result = -1 - else: - result = -1 - of pkWhitespace: - if s[start] != '\0': - var a: Rune - result = start - fastRuneAt(s, result, a) - if isWhiteSpace(a): dec(result, start) + leave(pkChar, s, p, start, result) + of pkCharChoice: + enter(pkCharChoice, s, p, start) + if start < s.len and contains(p.charChoice[], s[start]): result = 1 else: result = -1 - else: - result = -1 - of pkGreedyAny: - result = len(s) - start - of pkNewLine: - if s[start] == '\L': result = 1 - elif s[start] == '\C': - if s[start+1] == '\L': result = 2 - else: result = 1 - else: result = -1 - of pkTerminal: - result = len(p.term) - for i in 0..result-1: - if p.term[i] != s[start+i]: - result = -1 - break - of pkTerminalIgnoreCase: - var - i = 0 - a, b: Rune - result = start - while i < len(p.term): - fastRuneAt(p.term, i, a) - fastRuneAt(s, result, b) - if toLower(a) != toLower(b): - result = -1 - break - dec(result, start) - of pkTerminalIgnoreStyle: - var - i = 0 - a, b: Rune - result = start - while i < len(p.term): - while true: - fastRuneAt(p.term, i, a) - if a != Rune('_'): break - while true: - fastRuneAt(s, result, b) - if b != Rune('_'): break - if toLower(a) != toLower(b): - result = -1 - break - dec(result, start) - of pkChar: - if p.ch == s[start]: result = 1 - else: result = -1 - of pkCharChoice: - if contains(p.charChoice[], s[start]): result = 1 - else: result = -1 - of pkNonTerminal: - var oldMl = c.ml - when false: echo "enter: ", p.nt.name - result = rawMatch(s, p.nt.rule, start, c) - when false: echo "leave: ", p.nt.name - if result < 0: c.ml = oldMl - of pkSequence: - var oldMl = c.ml - result = 0 - for i in 0..high(p.sons): - var x = rawMatch(s, p.sons[i], start+result, c) - if x < 0: + leave(pkCharChoice, s, p, start, result) + of pkNonTerminal: + enter(pkNonTerminal, s, p, start) + var oldMl = c.ml + when false: echo "enter: ", p.nt.name + result = mopProc(s, p.nt.rule, start, c) + when false: echo "leave: ", p.nt.name + if result < 0: c.ml = oldMl + leave(pkNonTerminal, s, p, start, result) + of pkSequence: + enter(pkSequence, s, p, start) + var oldMl = c.ml + result = 0 + for i in 0..high(p.sons): + var x = mopProc(s, p.sons[i], start+result, c) + if x < 0: + c.ml = oldMl + result = -1 + break + else: inc(result, x) + leave(pkSequence, s, p, start, result) + of pkOrderedChoice: + enter(pkOrderedChoice, s, p, start) + var oldMl = c.ml + for i in 0..high(p.sons): + result = mopProc(s, p.sons[i], start, c) + if result >= 0: break c.ml = oldMl - result = -1 - break - else: inc(result, x) - of pkOrderedChoice: - var oldMl = c.ml - for i in 0..high(p.sons): - result = rawMatch(s, p.sons[i], start, c) - if result >= 0: break - c.ml = oldMl - of pkSearch: - var oldMl = c.ml - result = 0 - while start+result < s.len: - var x = rawMatch(s, p.sons[0], start+result, c) - if x >= 0: - inc(result, x) - return - inc(result) - result = -1 - c.ml = oldMl - of pkCapturedSearch: - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = 0 - while start+result < s.len: - var x = rawMatch(s, p.sons[0], start+result, c) - if x >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture - inc(result, x) - return - inc(result) - result = -1 - c.ml = idx - of pkGreedyRep: - result = 0 - while true: - var x = rawMatch(s, p.sons[0], start+result, c) - # if x == 0, we have an endless loop; so the correct behaviour would be - # not to break. But endless loops can be easily introduced: - # ``(comment / \w*)*`` is such an example. Breaking for x == 0 does the - # expected thing in this case. - if x <= 0: break - inc(result, x) - of pkGreedyRepChar: - result = 0 - var ch = p.ch - while ch == s[start+result]: inc(result) - of pkGreedyRepSet: - result = 0 - while contains(p.charChoice[], s[start+result]): inc(result) - of pkOption: - result = max(0, rawMatch(s, p.sons[0], start, c)) - of pkAndPredicate: - var oldMl = c.ml - result = rawMatch(s, p.sons[0], start, c) - if result >= 0: result = 0 # do not consume anything - else: c.ml = oldMl - of pkNotPredicate: - var oldMl = c.ml - result = rawMatch(s, p.sons[0], start, c) - if result < 0: result = 0 - else: + leave(pkOrderedChoice, s, p, start, result) + of pkSearch: + enter(pkSearch, s, p, start) + var oldMl = c.ml + result = 0 + while start+result <= s.len: + var x = mopProc(s, p.sons[0], start+result, c) + if x >= 0: + inc(result, x) + leave(pkSearch, s, p, start, result) + return + inc(result) + result = -1 c.ml = oldMl + leave(pkSearch, s, p, start, result) + of pkCapturedSearch: + enter(pkCapturedSearch, s, p, start) + var idx = c.ml # reserve a slot for the subpattern + inc(c.ml) + result = 0 + while start+result <= s.len: + var x = mopProc(s, p.sons[0], start+result, c) + if x >= 0: + if idx < MaxSubpatterns: + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture + inc(result, x) + leave(pkCapturedSearch, s, p, start, result) + return + inc(result) result = -1 - of pkCapture: - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = rawMatch(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture - else: c.ml = idx - of pkBackRef..pkBackRefIgnoreStyle: - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] - var n: Peg - n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) - n.term = s.substr(a, b) - result = rawMatch(s, n, start, c) - of pkStartAnchor: - if c.origStart == start: result = 0 - else: result = -1 - of pkRule, pkList: assert false + leave(pkCapturedSearch, s, p, start, result) + of pkGreedyRep: + enter(pkGreedyRep, s, p, start) + result = 0 + while true: + var x = mopProc(s, p.sons[0], start+result, c) + # if x == 0, we have an endless loop; so the correct behaviour would be + # not to break. But endless loops can be easily introduced: + # ``(comment / \w*)*`` is such an example. Breaking for x == 0 does the + # expected thing in this case. + if x <= 0: break + inc(result, x) + leave(pkGreedyRep, s, p, start, result) + of pkGreedyRepChar: + enter(pkGreedyRepChar, s, p, start) + result = 0 + var ch = p.ch + while start+result < s.len and ch == s[start+result]: inc(result) + leave(pkGreedyRepChar, s, p, start, result) + of pkGreedyRepSet: + enter(pkGreedyRepSet, s, p, start) + result = 0 + while start+result < s.len and contains(p.charChoice[], s[start+result]): + inc(result) + leave(pkGreedyRepSet, s, p, start, result) + of pkOption: + enter(pkOption, s, p, start) + result = max(0, mopProc(s, p.sons[0], start, c)) + leave(pkOption, s, p, start, result) + of pkAndPredicate: + enter(pkAndPredicate, s, p, start) + var oldMl = c.ml + result = mopProc(s, p.sons[0], start, c) + if result >= 0: result = 0 # do not consume anything + else: c.ml = oldMl + leave(pkAndPredicate, s, p, start, result) + of pkNotPredicate: + enter(pkNotPredicate, s, p, start) + var oldMl = c.ml + result = mopProc(s, p.sons[0], start, c) + if result < 0: result = 0 + else: + c.ml = oldMl + result = -1 + leave(pkNotPredicate, s, p, start, result) + of pkCapture: + enter(pkCapture, s, p, start) + if p.sons.len == 0 or p.sons[0].kind == pkEmpty: + # empty capture removes last match + dec(c.ml) + c.matches[c.ml] = (0, 0) + result = 0 # match of length 0 + else: + var idx = c.ml # reserve a slot for the subpattern + result = mopProc(s, p.sons[0], start, c) + if result >= 0: + if idx < MaxSubpatterns: + if idx != c.ml: + for i in countdown(c.ml, idx): + c.matches[i+1] = c.matches[i] + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture + inc(c.ml) + leave(pkCapture, s, p, start, result) + of pkBackRef: + enter(pkBackRef, s, p, start) + result = matchBackRef(s, p, start, c) + leave(pkBackRef, s, p, start, result) + of pkBackRefIgnoreCase: + enter(pkBackRefIgnoreCase, s, p, start) + result = matchBackRef(s, p, start, c) + leave(pkBackRefIgnoreCase, s, p, start, result) + of pkBackRefIgnoreStyle: + enter(pkBackRefIgnoreStyle, s, p, start) + result = matchBackRef(s, p, start, c) + leave(pkBackRefIgnoreStyle, s, p, start, result) + of pkStartAnchor: + enter(pkStartAnchor, s, p, start) + if c.origStart == start: result = 0 + else: result = -1 + leave(pkStartAnchor, s, p, start, result) + of pkRule, pkList: assert false + +func rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.rtl, extern: "npegs$1".} = + ## low-level matching proc that implements the PEG interpreter. Use this + ## for maximum efficiency (every other PEG operation ends up calling this + ## proc). + ## Returns -1 if it does not match, else the length of the match + + # Set the handler generators to produce do-nothing handlers. + template enter(pk, s, p, start) = + discard + template leave(pk, s, p, start, length) = + discard + matchOrParse(matchIt) + {.cast(noSideEffect).}: + # This cast is allowed because the `matchOrParse` template is used for + # both matching and parsing, but side effects are only possible when it's + # used by `eventParser`. + result = matchIt(s, p, start, c) + +macro mkHandlerTplts(handlers: untyped): untyped = + # Transforms the handler spec in *handlers* into handler templates. + # The AST structure of *handlers[0]*: + # + # ``` + # StmtList + # Call + # Ident "pkNonTerminal" + # StmtList + # Call + # Ident "enter" + # StmtList + # <handler code block> + # Call + # Ident "leave" + # StmtList + # <handler code block> + # Call + # Ident "pkChar" + # StmtList + # Call + # Ident "leave" + # StmtList + # <handler code block> + # ... + # ``` + func mkEnter(hdName, body: NimNode): NimNode = + template helper(hdName, body) {.dirty.} = + template hdName(s, p, start) = + let s {.inject.} = s + let p {.inject.} = p + let start {.inject.} = start + body + result = getAst(helper(hdName, body)) + + template mkLeave(hdPostf, body) {.dirty.} = + # this has to be dirty to be able to capture *result* as *length* in + # *leaveXX* calls. + template `leave hdPostf`(s, p, start, length) = + body + + result = newStmtList() + for topCall in handlers[0]: + if topCall.kind notin nnkCallKinds: + error("Call syntax expected.", topCall) + let pegKind = topCall[0] + if pegKind.kind notin {nnkIdent, nnkSym}: + error("PegKind expected.", pegKind) + if 2 == topCall.len: + for hdDef in topCall[1]: + if hdDef.kind notin nnkCallKinds: + error("Call syntax expected.", hdDef) + if hdDef[0].kind notin {nnkIdent, nnkSym}: + error("Handler identifier expected.", hdDef[0]) + if 2 == hdDef.len: + let hdPostf = substr(pegKind.strVal, 2) + case hdDef[0].strVal + of "enter": + result.add mkEnter(newIdentNode("enter" & hdPostf), hdDef[1]) + of "leave": + result.add getAst(mkLeave(ident(hdPostf), hdDef[1])) + else: + error( + "Unsupported handler identifier, expected 'enter' or 'leave'.", + hdDef[0] + ) + +template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = + ## Generates an interpreting event parser *proc* according to the specified + ## PEG AST and handler code blocks. The *proc* can be called with a string + ## to be parsed and will execute the handler code blocks whenever their + ## associated grammar element is matched. It returns -1 if the string does not + ## match, else the length of the total match. The following example code + ## evaluates an arithmetic expression defined by a simple PEG: + ## + ## ```nim + ## import std/[strutils, pegs] + ## + ## let + ## pegAst = """ + ## Expr <- Sum + ## Sum <- Product (('+' / '-')Product)* + ## Product <- Value (('*' / '/')Value)* + ## Value <- [0-9]+ / '(' Expr ')' + ## """.peg + ## txt = "(5+3)/2-7*22" + ## + ## var + ## pStack: seq[string] = @[] + ## valStack: seq[float] = @[] + ## opStack = "" + ## let + ## parseArithExpr = pegAst.eventParser: + ## pkNonTerminal: + ## enter: + ## pStack.add p.nt.name + ## leave: + ## pStack.setLen pStack.high + ## if length > 0: + ## let matchStr = s.substr(start, start+length-1) + ## case p.nt.name + ## of "Value": + ## try: + ## valStack.add matchStr.parseFloat + ## echo valStack + ## except ValueError: + ## discard + ## of "Sum", "Product": + ## try: + ## let val = matchStr.parseFloat + ## except ValueError: + ## if valStack.len > 1 and opStack.len > 0: + ## valStack[^2] = case opStack[^1] + ## of '+': valStack[^2] + valStack[^1] + ## of '-': valStack[^2] - valStack[^1] + ## of '*': valStack[^2] * valStack[^1] + ## else: valStack[^2] / valStack[^1] + ## valStack.setLen valStack.high + ## echo valStack + ## opStack.setLen opStack.high + ## echo opStack + ## pkChar: + ## leave: + ## if length == 1 and "Value" != pStack[^1]: + ## let matchChar = s[start] + ## opStack.add matchChar + ## echo opStack + ## + ## let pLen = parseArithExpr(txt) + ## ``` + ## + ## The *handlers* parameter consists of code blocks for *PegKinds*, + ## which define the grammar elements of interest. Each block can contain + ## handler code to be executed when the parser enters and leaves text + ## matching the grammar element. An *enter* handler can access the specific + ## PEG AST node being matched as *p*, the entire parsed string as *s* + ## and the position of the matched text segment in *s* as *start*. A *leave* + ## handler can access *p*, *s*, *start* and also the length of the matched + ## text segment as *length*. For an unsuccessful match, the *enter* and + ## *leave* handlers will be executed, with *length* set to -1. + ## + ## Symbols declared in an *enter* handler can be made visible in the + ## corresponding *leave* handler by annotating them with an *inject* pragma. + proc rawParse(s: string, p: Peg, start: int, c: var Captures): int + {.gensym.} = + + # binding from *macros* + bind strVal + + mkHandlerTplts: + handlers + + macro enter(pegKind, s, pegNode, start: untyped): untyped = + # This is called by the matcher code in *matchOrParse* at the + # start of the code for a grammar element of kind *pegKind*. + # Expands to a call to the handler template if one was generated + # by *mkHandlerTplts*. + template mkDoEnter(hdPostf, s, pegNode, start) = + when declared(`enter hdPostf`): + `enter hdPostf`(s, pegNode, start) + else: + discard + let hdPostf = ident(substr($pegKind, 2)) + getAst(mkDoEnter(hdPostf, s, pegNode, start)) + + macro leave(pegKind, s, pegNode, start, length: untyped): untyped = + # Like *enter*, but called at the end of the matcher code for + # a grammar element of kind *pegKind*. + template mkDoLeave(hdPostf, s, pegNode, start, length) = + when declared(`leave hdPostf`): + `leave hdPostf`(s, pegNode, start, length) + else: + discard + let hdPostf = ident(substr($pegKind, 2)) + getAst(mkDoLeave(hdPostf, s, pegNode, start, length)) + + matchOrParse(parseIt) + parseIt(s, p, start, c) -template fillMatches(s, caps, c: expr) = + proc parser(s: string): int {.gensym.} = + # the proc to be returned + var + ms: array[MaxSubpatterns, (int, int)] + cs = Captures(matches: ms, ml: 0, origStart: 0) + rawParse(s, pegAst, 0, cs) + parser + +template fillMatches(s, caps, c) = for k in 0..c.ml-1: let startIdx = c.matches[k][0] let endIdx = c.matches[k][1] if startIdx != -1: caps[k] = substr(s, startIdx, endIdx) else: - caps[k] = nil + caps[k] = "" -proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = +func matchLen*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -755,8 +1092,8 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, - start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = +func matchLen*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -765,22 +1102,22 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1Capture".} = +func match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. result = matchLen(s, pattern, matches, start) != -1 -proc match*(s: string, pattern: Peg, - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} = +func match*(s: string, pattern: Peg, + start = 0): bool {.rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. result = matchLen(s, pattern, start) != -1 -proc find*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = +func find*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and -1 is returned. @@ -794,9 +1131,9 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return -1 # could also use the pattern here: (!P .)* P -proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], +func findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. - nosideEffect, rtl, extern: "npegs$1Capture".} = + rtl, extern: "npegs$1Capture".} = ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing @@ -811,8 +1148,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], return (i, i+L-1) return (-1, 0) -proc find*(s: string, pattern: Peg, - start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = +func find*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. var c: Captures @@ -835,21 +1172,18 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = yield substr(s, i, i+L-1) inc(i, L) -proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - nosideEffect, rtl, extern: "npegs$1".} = +func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. + rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. - accumulateResult(findAll(s, pattern, start)) - -when not defined(nimhygiene): - {.pragma: inject.} + ## If it does not match, `@[]` is returned. + result = @[] + for it in findAll(s, pattern, start): result.add it template `=~`*(s: string, pattern: Peg): bool = ## This calls ``match`` with an implicit declared ``matches`` array that ## can be used in the scope of the ``=~`` call: ## - ## .. code-block:: nim - ## + ## ```nim ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## # matches a key=value pair: ## echo("Key: ", matches[0]) @@ -861,50 +1195,51 @@ template `=~`*(s: string, pattern: Peg): bool = ## echo("comment: ", matches[0]) ## else: ## echo("syntax error") - ## + ## ``` bind MaxSubpatterns when not declaredInScope(matches): - var matches {.inject.}: array[0..MaxSubpatterns-1, string] + var matches {.inject.} = default(array[0..MaxSubpatterns-1, string]) match(s, pattern, matches) # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Peg, start = 0): bool {. - nosideEffect, rtl, extern: "npegs$1".} = +func contains*(s: string, pattern: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## same as ``find(s, pattern, start) >= 0`` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.nosideEffect, rtl, extern: "npegs$1Capture".} = +func contains*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## same as ``find(s, pattern, matches, start) >= 0`` return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Peg, start = 0): bool {. - nosideEffect, rtl, extern: "npegs$1".} = +func startsWith*(s: string, prefix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix, start) >= 0 -proc endsWith*(s: string, suffix: Peg, start = 0): bool {. - nosideEffect, rtl, extern: "npegs$1".} = +func endsWith*(s: string, suffix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` ends with the pattern `suffix` var c: Captures c.origStart = start for i in start .. s.len-1: if rawMatch(s, suffix, i, c) == s.len - i: return true -proc replacef*(s: string, sub: Peg, by: string): string {. - nosideEffect, rtl, extern: "npegs$1".} = +func replacef*(s: string, sub: Peg, by: string): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## - ## .. code-block:: nim + ## ```nim ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1<-keykey; val2<-key2key2" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -921,8 +1256,8 @@ proc replacef*(s: string, sub: Peg, by: string): string {. inc(i, x) add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, by = ""): string {. - nosideEffect, rtl, extern: "npegs$1".} = +func replace*(s: string, sub: Peg, by = ""): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed ## in `by`. result = "" @@ -938,9 +1273,9 @@ proc replace*(s: string, sub: Peg, by = ""): string {. inc(i, x) add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: varargs[ +func parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - nosideEffect, rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -962,14 +1297,65 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) -proc transformFile*(infile, outfile: string, - subs: varargs[tuple[pattern: Peg, repl: string]]) {. - rtl, extern: "npegs$1".} = - ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an - ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +func replace*(s: string, sub: Peg, cb: proc( + match: int, cnt: int, caps: openArray[string]): string): string {. + rtl, extern: "npegs$1cb", effectsOf: cb.} = + ## Replaces `sub` in `s` by the resulting strings from the callback. + ## The callback proc receives the index of the current match (starting with 0), + ## the count of captures and an open array with the captures of each match. Examples: + ## + ## ```nim + ## func handleMatches*(m: int, n: int, c: openArray[string]): string = + ## result = "" + ## if m > 0: + ## result.add ", " + ## result.add case n: + ## of 2: c[0].toLower & ": '" & c[1] & "'" + ## of 1: c[0].toLower & ": ''" + ## else: "" + ## + ## let s = "Var1=key1;var2=Key2; VAR3" + ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## ``` + ## + ## Results in: + ## + ## ```nim + ## "var1: 'key1', var2: 'Key2', var3: ''" + ## ``` + result = "" + var i = 0 + var caps: array[0..MaxSubpatterns-1, string] + var c: Captures + var m = 0 + while i < s.len: + c.ml = 0 + var x = rawMatch(s, sub, i, c) + if x <= 0: + add(result, s[i]) + inc(i) + else: + fillMatches(s, caps, c) + add(result, cb(m, c.ml, caps)) + inc(i, x) + inc(m) + add(result, substr(s, i)) + +when not defined(js): + proc transformFile*(infile, outfile: string, + subs: varargs[tuple[pattern: Peg, repl: string]]) {. + rtl, extern: "npegs$1".} = + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises ``IOError`` if an + ## error occurs. This is supposed to be used for quick scripting. + ## + ## **Note**: this proc does not exist while using the JS backend. + var x = readFile(infile) + writeFile(outfile, x.parallelReplace(subs)) + iterator split*(s: string, sep: Peg): string = ## Splits the string `s` into substrings. @@ -977,18 +1363,19 @@ iterator split*(s: string, sep: Peg): string = ## Substrings are separated by the PEG `sep`. ## Examples: ## - ## .. code-block:: nim + ## ```nim ## for word in split("00232this02939is39an22example111", peg"\d+"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "this" ## "is" ## "an" ## "example" - ## + ## ``` var c: Captures var first = 0 @@ -1006,10 +1393,11 @@ iterator split*(s: string, sep: Peg): string = if first < last: yield substr(s, first, last-1) -proc split*(s: string, sep: Peg): seq[string] {. - nosideEffect, rtl, extern: "npegs$1".} = +func split*(s: string, sep: Peg): seq[string] {. + rtl, extern: "npegs$1".} = ## Splits the string `s` into substrings. - accumulateResult(split(s, sep)) + result = @[] + for it in split(s, sep): result.add it # ------------------- scanner ------------------------------------------------- @@ -1019,70 +1407,71 @@ type modVerbatim, modIgnoreCase, modIgnoreStyle - TokKind = enum ## enumeration of all tokens - tkInvalid, ## invalid token - tkEof, ## end of file reached - tkAny, ## . - tkAnyRune, ## _ - tkIdentifier, ## abc - tkStringLit, ## "abc" or 'abc' - tkCharSet, ## [^A-Z] - tkParLe, ## '(' - tkParRi, ## ')' - tkCurlyLe, ## '{' - tkCurlyRi, ## '}' - tkCurlyAt, ## '{@}' - tkArrow, ## '<-' - tkBar, ## '/' - tkStar, ## '*' - tkPlus, ## '+' - tkAmp, ## '&' - tkNot, ## '!' - tkOption, ## '?' - tkAt, ## '@' - tkBuiltin, ## \identifier - tkEscaped, ## \\ - tkBackref, ## '$' - tkDollar, ## '$' - tkHat ## '^' - - Token {.final.} = object ## a token - kind: TokKind ## the type of the token + TokKind = enum ## enumeration of all tokens + tkInvalid, ## invalid token + tkEof, ## end of file reached + tkAny, ## . + tkAnyRune, ## _ + tkIdentifier, ## abc + tkStringLit, ## "abc" or 'abc' + tkCharSet, ## [^A-Z] + tkParLe, ## '(' + tkParRi, ## ')' + tkCurlyLe, ## '{' + tkCurlyRi, ## '}' + tkCurlyAt, ## '{@}' + tkEmptyCurl, ## '{}' + tkArrow, ## '<-' + tkBar, ## '/' + tkStar, ## '*' + tkPlus, ## '+' + tkAmp, ## '&' + tkNot, ## '!' + tkOption, ## '?' + tkAt, ## '@' + tkBuiltin, ## \identifier + tkEscaped, ## \\ + tkBackref, ## '$' + tkDollar, ## '$' + tkHat ## '^' + + Token {.final.} = object ## a token + kind: TokKind ## the type of the token modifier: Modifier - literal: string ## the parsed (string) literal - charset: set[char] ## if kind == tkCharSet - index: int ## if kind == tkBackref - - PegLexer {.inheritable.} = object ## the lexer object. - bufpos: int ## the current position within the buffer - buf: cstring ## the buffer itself - lineNumber: int ## the current line number - lineStart: int ## index of last line start in buffer - colOffset: int ## column to add + literal: string ## the parsed (string) literal + charset: set[char] ## if kind == tkCharSet + index: int ## if kind == tkBackref + + PegLexer {.inheritable.} = object ## the lexer object. + bufpos: int ## the current position within the buffer + buf: string ## the buffer itself + lineNumber: int ## the current line number + lineStart: int ## index of last line start in buffer + colOffset: int ## column to add filename: string const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", + "character set", "(", ")", "{", "}", "{@}", "{}", "<-", "/", "*", "+", "&", "!", "?", "@", "built-in", "escaped", "$", "$", "^" ] -proc handleCR(L: var PegLexer, pos: int): int = +func handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 - if L.buf[result] == '\L': inc(result) + if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result -proc handleLF(L: var PegLexer, pos: int): int = +func handleLF(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\L') inc(L.lineNumber) result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +func init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1090,32 +1479,22 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +func getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = +func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc handleHexChar(c: var PegLexer, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: discard - -proc getEscapedChar(c: var PegLexer, tok: var Token) = +func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) + if c.bufpos >= len(c.buf): + tok.kind = tkInvalid + return case c.buf[c.bufpos] of 'r', 'R', 'c', 'C': add(tok.literal, '\c') @@ -1143,16 +1522,21 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) of 'x', 'X': inc(c.bufpos) + if c.bufpos >= len(c.buf): + tok.kind = tkInvalid + return var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) if xi == 0: tok.kind = tkInvalid else: add(tok.literal, chr(xi)) of '0'..'9': var val = ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) var i = 1 - while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): + while (c.bufpos < len(c.buf)) and (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): val = val * 10 + ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) inc(i) @@ -1166,32 +1550,29 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc skip(c: var PegLexer) = +func skip(c: var PegLexer) = var pos = c.bufpos - var buf = c.buf - while true: - case buf[pos] + while pos < c.buf.len: + case c.buf[pos] of ' ', '\t': inc(pos) of '#': - while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) + while (pos < c.buf.len) and + not (c.buf[pos] in {'\c', '\L', '\0'}): inc(pos) of '\c': pos = handleCR(c, pos) - buf = c.buf of '\L': pos = handleLF(c, pos) - buf = c.buf else: - break # EndOfFile also leaves the loop + break # EndOfFile also leaves the loop c.bufpos = pos -proc getString(c: var PegLexer, tok: var Token) = +func getString(c: var PegLexer, tok: var Token) = tok.kind = tkStringLit var pos = c.bufpos + 1 - var buf = c.buf - var quote = buf[pos-1] - while true: - case buf[pos] + var quote = c.buf[pos-1] + while pos < c.buf.len: + case c.buf[pos] of '\\': c.bufpos = pos getEscapedChar(c, tok) @@ -1199,90 +1580,102 @@ proc getString(c: var PegLexer, tok: var Token) = of '\c', '\L', '\0': tok.kind = tkInvalid break - elif buf[pos] == quote: + elif c.buf[pos] == quote: inc(pos) break else: - add(tok.literal, buf[pos]) + add(tok.literal, c.buf[pos]) inc(pos) c.bufpos = pos -proc getDollar(c: var PegLexer, tok: var Token) = +func getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 - var buf = c.buf - if buf[pos] in {'0'..'9'}: + var neg = false + if pos < c.buf.len and c.buf[pos] == '^': + neg = true + inc(pos) + if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 - while buf[pos] in {'0'..'9'}: - tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') + while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: + tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0') inc(pos) + if neg: + tok.index = -tok.index else: + if neg: + dec(pos) tok.kind = tkDollar c.bufpos = pos -proc getCharSet(c: var PegLexer, tok: var Token) = +func getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 - var buf = c.buf var caret = false - if buf[pos] == '^': - inc(pos) - caret = true - while true: - var ch: char - case buf[pos] - of ']': + if pos < c.buf.len: + if c.buf[pos] == '^': inc(pos) - break - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: - ch = buf[pos] - inc(pos) - incl(tok.charset, ch) - if buf[pos] == '-': - if buf[pos+1] == ']': - incl(tok.charset, '-') - inc(pos) + caret = true + while pos < c.buf.len: + var ch: char + case c.buf[pos] + of ']': + if pos < c.buf.len: inc(pos) + break + of '\\': + c.bufpos = pos + getEscapedChar(c, tok) + pos = c.bufpos + ch = tok.literal[tok.literal.len-1] + of '\C', '\L', '\0': + tok.kind = tkInvalid + break else: + ch = c.buf[pos] inc(pos) - var ch2: char - case buf[pos] - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch2 = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: - ch2 = buf[pos] + incl(tok.charset, ch) + if c.buf[pos] == '-': + if pos+1 < c.buf.len and c.buf[pos+1] == ']': + incl(tok.charset, '-') inc(pos) - for i in ord(ch)+1 .. ord(ch2): - incl(tok.charset, chr(i)) + else: + if pos+1 < c.buf.len: + inc(pos) + else: + break + var ch2: char + case c.buf[pos] + of '\\': + c.bufpos = pos + getEscapedChar(c, tok) + pos = c.bufpos + ch2 = tok.literal[tok.literal.len-1] + of '\C', '\L', '\0': + tok.kind = tkInvalid + break + else: + if pos+1 < c.buf.len: + ch2 = c.buf[pos] + inc(pos) + else: + break + for i in ord(ch)+1 .. ord(ch2): + incl(tok.charset, chr(i)) c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset -proc getSymbol(c: var PegLexer, tok: var Token) = +func getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos - var buf = c.buf - while true: - add(tok.literal, buf[pos]) + while pos < c.buf.len: + add(tok.literal, c.buf[pos]) inc(pos) - if buf[pos] notin strutils.IdentChars: break + if pos < c.buf.len and c.buf[pos] notin strutils.IdentChars: break c.bufpos = pos tok.kind = tkIdentifier -proc getBuiltin(c: var PegLexer, tok: var Token) = - if c.buf[c.bufpos+1] in strutils.Letters: +func getBuiltin(c: var PegLexer, tok: var Token) = + if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) tok.kind = tkBuiltin @@ -1290,18 +1683,31 @@ proc getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) skip(c) + + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" + add(tok.literal, '\0') + inc(c.bufpos) + return + case c.buf[c.bufpos] of '{': inc(c.bufpos) - if c.buf[c.bufpos] == '@' and c.buf[c.bufpos+1] == '}': + if c.buf[c.bufpos] == '@' and c.bufpos+2 < c.buf.len and + c.buf[c.bufpos+1] == '}': tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") + elif c.buf[c.bufpos] == '}' and c.bufpos < c.buf.len: + tok.kind = tkEmptyCurl + inc(c.bufpos) + add(tok.literal, "{}") else: tok.kind = tkCurlyLe add(tok.literal, '{') @@ -1331,13 +1737,13 @@ proc getTok(c: var PegLexer, tok: var Token) = getBuiltin(c, tok) of '\'', '"': getString(c, tok) of '$': getDollar(c, tok) - of '\0': - tok.kind = tkEof - tok.literal = "[EOF]" of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) + if c.bufpos >= c.buf.len: + return if c.buf[c.bufpos] in {'\'', '"'} or - c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and + c.buf[c.bufpos+1] in {'^', '0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1358,7 +1764,7 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '+') of '<': - if c.buf[c.bufpos+1] == '-': + if c.bufpos+2 < c.buf.len and c.buf[c.bufpos+1] == '-': inc(c.bufpos, 2) tok.kind = tkArrow add(tok.literal, "<-") @@ -1393,20 +1799,25 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '^') else: + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc arrowIsNextTok(c: PegLexer): bool = +func arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos - while c.buf[pos] in {'\t', ' '}: inc(pos) + while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) + if pos+1 >= c.buf.len: + return result = c.buf[pos] == '<' and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- type EInvalidPeg* = object of ValueError ## raised if an invalid - ## PEG has been detected + ## PEG has been detected PegParser = object of PegLexer ## the PEG parser object tok: Token nonterms: seq[NonTerminal] @@ -1415,23 +1826,21 @@ type identIsVerbatim: bool skip: Peg -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = - var e: ref EInvalidPeg - new(e) - e.msg = errorStr(p, msg, line, col) +func pegError(p: PegParser, msg: string, line = -1, col = -1) = + var e = (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) raise e -proc getTok(p: var PegParser) = +func getTok(p: var PegParser) = getTok(p, p.tok) - if p.tok.kind == tkInvalid: pegError(p, "invalid token") + if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") -proc eat(p: var PegParser, kind: TokKind) = +func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -proc parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.gcsafe.} -proc getNonTerminal(p: var PegParser, name: string): NonTerminal = +func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): result = p.nonterms[i] if cmpIgnoreStyle(result.name, name) == 0: return @@ -1439,19 +1848,22 @@ proc getNonTerminal(p: var PegParser, name: string): NonTerminal = result = newNonTerminal(name, getLine(p), getColumn(p)) add(p.nonterms, result) -proc modifiedTerm(s: string, m: Modifier): Peg = +func modifiedTerm(s: string, m: Modifier): Peg = case m of modNone, modVerbatim: result = term(s) of modIgnoreCase: result = termIgnoreCase(s) of modIgnoreStyle: result = termIgnoreStyle(s) -proc modifiedBackref(s: int, m: Modifier): Peg = +func modifiedBackref(s: int, m: Modifier): Peg = + var + reverse = s < 0 + index = if reverse: -s else: s case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) + of modNone, modVerbatim: result = backref(index, reverse) + of modIgnoreCase: result = backrefIgnoreCase(index, reverse) + of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) -proc builtin(p: var PegParser): Peg = +func builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous case p.tok.literal of "n": result = newLine() @@ -1460,7 +1872,7 @@ proc builtin(p: var PegParser): Peg = of "s": result = charSet({' ', '\9'..'\13'}) of "S": result = charSet({'\1'..'\xff'} - {' ', '\9'..'\13'}) of "w": result = charSet({'a'..'z', 'A'..'Z', '_', '0'..'9'}) - of "W": result = charSet({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'}) + of "W": result = charSet({'\1'..'\xff'} - {'a'..'z', 'A'..'Z', '_', '0'..'9'}) of "a": result = charSet({'a'..'z', 'A'..'Z'}) of "A": result = charSet({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'}) of "ident": result = pegs.ident @@ -1471,11 +1883,11 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -proc primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg = case p.tok.kind of tkAmp: getTok(p) @@ -1499,7 +1911,8 @@ proc primary(p: var PegParser): Peg = getTok(p) elif not arrowIsNextTok(p): var nt = getNonTerminal(p, p.tok.literal) - incl(nt.flags, ntUsed) + {.cast(noSideEffect).}: + incl(nt.flags, ntUsed) result = nonterminal(nt).token(p) getTok(p) else: @@ -1523,6 +1936,9 @@ proc primary(p: var PegParser): Peg = result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) + of tkEmptyCurl: + result = capture() + getTok(p) of tkAny: result = any().token(p) getTok(p) @@ -1542,11 +1958,11 @@ proc primary(p: var PegParser): Peg = result = startAnchor() getTok(p) of tkBackref: + if abs(p.tok.index) > p.captures or p.tok.index == 0: + pegError(p, "invalid back reference index: " & $p.tok.index) var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) @@ -1564,13 +1980,13 @@ proc primary(p: var PegParser): Peg = getTok(p) else: break -proc seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg = result = primary(p) while true: case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: + tkHat, tkCurlyAt, tkEmptyCurl: result = sequence(result, primary(p)) of tkIdentifier: if not arrowIsNextTok(p): @@ -1578,27 +1994,29 @@ proc seqExpr(p: var PegParser): Peg = else: break else: break -proc parseExpr(p: var PegParser): Peg = +func parseExpr(p: var PegParser): Peg = result = seqExpr(p) while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) -proc parseRule(p: var PegParser): NonTerminal = +func parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: pegError(p, "attempt to redefine: " & result.name) - result.line = getLine(p) - result.col = getColumn(p) + {.cast(noSideEffect).}: + result.line = getLine(p) + result.col = getColumn(p) getTok(p) eat(p, tkArrow) - result.rule = parseExpr(p) - incl(result.flags, ntDeclared) # NOW inlining may be attempted + {.cast(noSideEffect).}: + result.rule = parseExpr(p) + incl(result.flags, ntDeclared) # NOW inlining may be attempted else: pegError(p, "rule expected, but found: " & p.tok.literal) -proc rawParse(p: var PegParser): Peg = +func rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: case p.tok.literal @@ -1628,7 +2046,7 @@ proc rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -1643,14 +2061,14 @@ proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -proc peg*(pattern: string): Peg = +func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been - ## chosen to encourage its use as a raw string modifier:: + ## chosen to encourage its use as a raw string modifier: ## - ## peg"{\ident} \s* '=' \s* {.*}" + ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") -proc escapePeg*(s: string): string = +func escapePeg*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a peg. result = "" var inQuote = false @@ -1668,124 +2086,3 @@ proc escapePeg*(s: string): string = inQuote = true result.add(c) if inQuote: result.add('\'') - -when isMainModule: - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", peg"\d+"): - accum.add(word) - assert(accum == @["this", "is", "an", "example"]) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false - - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - accum = @[] - for x in findAll("abcdef", peg".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) - - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" - - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false - - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) - - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") - - assert match("prefix/start", peg"^start$", 7) - - if "foo" =~ peg"{'a'}?.*": - assert matches[0] == nil - else: assert false - - if "foo" =~ peg"{''}.*": - assert matches[0] == "" - else: assert false - - if "foo" =~ peg"{'foo'}": - assert matches[0] == "foo" - else: assert false - - let empty_test = peg"^\d*" - let str = "XYZ" - - assert(str.find(empty_test) == 0) - assert(str.match(empty_test)) diff --git a/lib/pure/poly.nim b/lib/pure/poly.nim deleted file mode 100644 index c52300400..000000000 --- a/lib/pure/poly.nim +++ /dev/null @@ -1,368 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Robert Persson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import math -import strutils -import numeric - -type - Poly* = object - cofs:seq[float] - -{.deprecated: [TPoly: Poly].} - -proc degree*(p:Poly):int= - ## Returns the degree of the polynomial, - ## that is the number of coefficients-1 - return p.cofs.len-1 - - -proc eval*(p:Poly,x:float):float= - ## Evaluates a polynomial function value for `x` - ## quickly using Horners method - var n=p.degree - result=p.cofs[n] - dec n - while n>=0: - result = result*x+p.cofs[n] - dec n - -proc `[]` *(p:Poly;idx:int):float= - ## Gets a coefficient of the polynomial. - ## p[2] will returns the quadric term, p[3] the cubic etc. - ## Out of bounds index will return 0.0. - if idx<0 or idx>p.degree: - return 0.0 - return p.cofs[idx] - -proc `[]=` *(p:var Poly;idx:int,v:float)= - ## Sets an coefficient of the polynomial by index. - ## p[2] set the quadric term, p[3] the cubic etc. - ## If index is out of range for the coefficients, - ## the polynomial grows to the smallest needed degree. - assert(idx>=0) - - if idx>p.degree: #polynomial must grow - var oldlen=p.cofs.len - p.cofs.setLen(idx+1) - for q in oldlen.. <high(p.cofs): - p.cofs[q]=0.0 #new-grown coefficients set to zero - - p.cofs[idx]=v - - -iterator items*(p:Poly):float= - ## Iterates through the coefficients of the polynomial. - var i=p.degree - while i>=0: - yield p[i] - dec i - -proc clean*(p:var Poly;zerotol=0.0)= - ## Removes leading zero coefficients of the polynomial. - ## An optional tolerance can be given for what's considered zero. - var n=p.degree - var relen=false - - while n>0 and abs(p[n])<=zerotol: # >0 => keep at least one coefficient - dec n - relen=true - - if relen: p.cofs.setLen(n+1) - - -proc `$` *(p:Poly):string = - ## Gets a somewhat reasonable string representation of the polynomial - ## The format should be compatible with most online function plotters, - ## for example directly in google search - result="" - var first=true #might skip + sign if first coefficient - - for idx in countdown(p.degree,0): - let a=p[idx] - - if a==0.0: - continue - - if a>= 0.0 and not first: - result.add('+') - first=false - - if a!=1.0 or idx==0: - result.add(formatFloat(a,ffDefault,0)) - if idx>=2: - result.add("x^" & $idx) - elif idx==1: - result.add("x") - - if result=="": - result="0" - - -proc derivative*(p: Poly): Poly= - ## Returns a new polynomial, which is the derivative of `p` - newSeq[float](result.cofs,p.degree) - for idx in 0..high(result.cofs): - result.cofs[idx]=p.cofs[idx+1]*float(idx+1) - -proc diff*(p:Poly,x:float):float= - ## Evaluates the differentiation of a polynomial with - ## respect to `x` quickly using a modifed Horners method - var n=p.degree - result=p[n]*float(n) - dec n - while n>=1: - result = result*x+p[n]*float(n) - dec n - -proc integral*(p:Poly):Poly= - ## Returns a new polynomial which is the indefinite - ## integral of `p`. The constant term is set to 0.0 - newSeq(result.cofs,p.cofs.len+1) - result.cofs[0]=0.0 #constant arbitrary term, use 0.0 - for i in 1..high(result.cofs): - result.cofs[i]=p.cofs[i-1]/float(i) - - -proc integrate*(p:Poly;xmin,xmax:float):float= - ## Computes the definite integral of `p` between `xmin` and `xmax` - ## quickly using a modified version of Horners method - var - n=p.degree - s1=p[n]/float(n+1) - s2=s1 - fac:float - - dec n - while n>=0: - fac=p[n]/float(n+1) - s1 = s1*xmin+fac - s2 = s2*xmax+fac - dec n - - result=s2*xmax-s1*xmin - -proc initPoly*(cofs:varargs[float]):Poly= - ## Initializes a polynomial with given coefficients. - ## The most significant coefficient is first, so to create x^2-2x+3: - ## intiPoly(1.0,-2.0,3.0) - if len(cofs)<=0: - result.cofs= @[0.0] #need at least one coefficient - else: - # reverse order of coefficients so indexing matches degree of - # coefficient... - result.cofs= @[] - for idx in countdown(cofs.len-1,0): - result.cofs.add(cofs[idx]) - - result.clean #remove leading zero terms - - -proc divMod*(p,d:Poly;q,r:var Poly)= - ## Divides `p` with `d`, and stores the quotinent in `q` and - ## the remainder in `d` - var - pdeg=p.degree - ddeg=d.degree - power=p.degree-d.degree - ratio:float - - r.cofs = p.cofs #initial remainder=numerator - if power<0: #denominator is larger than numerator - q.cofs= @ [0.0] #quotinent is 0.0 - return # keep remainder as numerator - - q.cofs=newSeq[float](power+1) - - for i in countdown(pdeg,ddeg): - ratio=r.cofs[i]/d.cofs[ddeg] - - q.cofs[i-ddeg]=ratio - r.cofs[i]=0.0 - - for j in countup(0,<ddeg): - var idx=i-ddeg+j - r.cofs[idx] = r.cofs[idx] - d.cofs[j]*ratio - - r.clean # drop zero coefficients in remainder - -proc `+` *(p1:Poly,p2:Poly):Poly= - ## Adds two polynomials - var n=max(p1.cofs.len,p2.cofs.len) - newSeq(result.cofs,n) - - for idx in countup(0,n-1): - result[idx]=p1[idx]+p2[idx] - - result.clean # drop zero coefficients in remainder - -proc `*` *(p1:Poly,p2:Poly):Poly= - ## Multiplies the polynomial `p1` with `p2` - var - d1=p1.degree - d2=p2.degree - n=d1+d2 - idx:int - - newSeq(result.cofs,n) - - for i1 in countup(0,d1): - for i2 in countup(0,d2): - idx=i1+i2 - result[idx]=result[idx]+p1[i1]*p2[i2] - - result.clean - -proc `*` *(p:Poly,f:float):Poly= - ## Multiplies the polynomial `p` with a real number - newSeq(result.cofs,p.cofs.len) - for i in 0..high(p.cofs): - result[i]=p.cofs[i]*f - result.clean - -proc `*` *(f:float,p:Poly):Poly= - ## Multiplies a real number with a polynomial - return p*f - -proc `-`*(p:Poly):Poly= - ## Negates a polynomial - result=p - for i in countup(0,<result.cofs.len): - result.cofs[i]= -result.cofs[i] - -proc `-` *(p1:Poly,p2:Poly):Poly= - ## Subtract `p1` with `p2` - var n=max(p1.cofs.len,p2.cofs.len) - newSeq(result.cofs,n) - - for idx in countup(0,n-1): - result[idx]=p1[idx]-p2[idx] - - result.clean # drop zero coefficients in remainder - -proc `/`*(p:Poly,f:float):Poly= - ## Divides polynomial `p` with a real number `f` - newSeq(result.cofs,p.cofs.len) - for i in 0..high(p.cofs): - result[i]=p.cofs[i]/f - result.clean - -proc `/` *(p,q:Poly):Poly= - ## Divides polynomial `p` with polynomial `q` - var dummy:Poly - p.divMod(q,result,dummy) - -proc `mod` *(p,q:Poly):Poly= - ## Computes the polynomial modulo operation, - ## that is the remainder of `p`/`q` - var dummy:Poly - p.divMod(q,dummy,result) - - -proc normalize*(p:var Poly)= - ## Multiplies the polynomial inplace by a term so that - ## the leading term is 1.0. - ## This might lead to an unstable polynomial - ## if the leading term is zero. - p=p/p[p.degree] - - -proc solveQuadric*(a,b,c:float;zerotol=0.0):seq[float]= - ## Solves the quadric equation `ax^2+bx+c`, with a possible - ## tolerance `zerotol` to find roots of curves just 'touching' - ## the x axis. Returns sequence with 0,1 or 2 solutions. - - var p,q,d:float - - p=b/(2.0*a) - - if p==Inf or p==NegInf: #linear equation.. - var linrt= -c/b - if linrt==Inf or linrt==NegInf: #constant only - return @[] - return @[linrt] - - q=c/a - d=p*p-q - - if d<0.0: - #check for inside zerotol range for neg. roots - var err=a*p*p-b*p+c #evaluate error at parabola center axis - if(err<=zerotol): return @[-p] - return @[] - else: - var sr=sqrt(d) - result= @[-sr-p,sr-p] - -proc getRangeForRoots(p:Poly):tuple[xmin,xmax:float]= - ## helper function for `roots` function - ## quickly computes a range, guaranteed to contain - ## all the real roots of the polynomial - # see http://www.mathsisfun.com/algebra/polynomials-bounds-zeros.html - - var deg=p.degree - var d=p[deg] - var bound1,bound2:float - - for i in countup(0,deg): - var c=abs(p.cofs[i]/d) - bound1=max(bound1,c+1.0) - bound2=bound2+c - - bound2=max(1.0,bound2) - result.xmax=min(bound1,bound2) - result.xmin= -result.xmax - - -proc addRoot(p:Poly,res:var seq[float],xp0,xp1,tol,zerotol,mergetol:float,maxiter:int)= - ## helper function for `roots` function - ## try to do a numeric search for a single root in range xp0-xp1, - ## adding it to `res` (allocating `res` if nil) - var br=brent(xp0,xp1, proc(x:float):float=p.eval(x),tol) - if br.success: - if res.len==0 or br.rootx>=res[high(res)]+mergetol: #dont add equal roots. - res.add(br.rootx) - else: - #this might be a 'touching' case, check function value against - #zero tolerance - if abs(br.rooty)<=zerotol: - if res.len==0 or br.rootx>=res[high(res)]+mergetol: #dont add equal roots. - res.add(br.rootx) - - -proc roots*(p:Poly,tol=1.0e-9,zerotol=1.0e-6,mergetol=1.0e-12,maxiter=1000):seq[float]= - ## Computes the real roots of the polynomial `p` - ## `tol` is the tolerance used to break searching for each root when reached. - ## `zerotol` is the tolerance, which is 'close enough' to zero to be considered a root - ## and is used to find roots for curves that only 'touch' the x-axis. - ## `mergetol` is the tolerance, of which two x-values are considered beeing the same root. - ## `maxiter` can be used to limit the number of iterations for each root. - ## Returns a (possibly empty) sorted sequence with the solutions. - var deg=p.degree - if deg<=0: #constant only => no roots - return @[] - elif p.degree==1: #linear - var linrt= -p.cofs[0]/p.cofs[1] - if linrt==Inf or linrt==NegInf: - return @[] #constant only => no roots - return @[linrt] - elif p.degree==2: - return solveQuadric(p.cofs[2],p.cofs[1],p.cofs[0],zerotol) - else: - # degree >=3 , find min/max points of polynomial with recursive - # derivative and do a numerical search for root between each min/max - var range=p.getRangeForRoots() - var minmax=p.derivative.roots(tol,zerotol,mergetol) - result= @[] - if minmax!=nil: #ie. we have minimas/maximas in this function - for x in minmax.items: - addRoot(p,result,range.xmin,x,tol,zerotol,mergetol,maxiter) - range.xmin=x - addRoot(p,result,range.xmin,range.xmax,tol,zerotol,mergetol,maxiter) - diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim new file mode 100644 index 000000000..9428f29eb --- /dev/null +++ b/lib/pure/prelude.nim @@ -0,0 +1,28 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimdoc) and isMainModule: + from std/compileSettings import nil + when compileSettings.querySetting(compileSettings.SingleValueSetting.projectFull) == currentSourcePath: + ## This is an include file that simply imports common modules for your convenience. + runnableExamples: + include std/prelude + # same as: + # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] + let x = 1 + assert "foo $# $#" % [$x, "bar"] == "foo 1 bar" + assert toSeq(1..3) == @[1, 2, 3] + when not defined(js) or defined(nodejs): + assert getCurrentDir().len > 0 + assert ($now()).startsWith "20" + + # xxx `nim doc -b:js -d:nodejs --doccmd:-d:nodejs lib/pure/prelude.nim` fails for some reason + # specific to `nim doc`, but the code otherwise works with nodejs. + +import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] diff --git a/lib/pure/random.nim b/lib/pure/random.nim new file mode 100644 index 000000000..3ec77d37e --- /dev/null +++ b/lib/pure/random.nim @@ -0,0 +1,766 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim's standard random number generator (RNG). +## +## Its implementation is based on the `xoroshiro128+` +## (xor/rotate/shift/rotate) library. +## * More information: http://xoroshiro.di.unimi.it +## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c +## +## **Do not use this module for cryptographic purposes!** +## +## Basic usage +## =========== +## +runnableExamples: + # Call randomize() once to initialize the default random number generator. + # If this is not called, the same results will occur every time these + # examples are run. + randomize() + + # Pick a number in 0..100. + let num = rand(100) + doAssert num in 0..100 + + # Roll a six-sided die. + let roll = rand(1..6) + doAssert roll in 1..6 + + # Pick a marble from a bag. + let marbles = ["red", "blue", "green", "yellow", "purple"] + let pick = sample(marbles) + doAssert pick in marbles + + # Shuffle some cards. + var cards = ["Ace", "King", "Queen", "Jack", "Ten"] + shuffle(cards) + doAssert cards.len == 5 + +## These examples all use the default RNG. The +## `Rand type <#Rand>`_ represents the state of an RNG. +## For convenience, this module contains a default Rand state that corresponds +## to the default RNG. Most procs in this module which do +## not take in a Rand parameter, including those called in the above examples, +## use the default generator. Those procs are **not** thread-safe. +## +## Note that the default generator always starts in the same state. +## The `randomize proc <#randomize>`_ can be called to initialize the default +## generator with a seed based on the current time, and it only needs to be +## called once before the first usage of procs from this module. If +## `randomize` is not called, the default generator will always produce +## the same results. +## +## RNGs that are independent of the default one can be created with the +## `initRand proc <#initRand,int64>`_. +## +## Again, it is important to remember that this module must **not** be used for +## cryptographic applications. +## +## See also +## ======== +## * `std/sysrand module <sysrand.html>`_ for a cryptographically secure pseudorandom number generator +## * `math module <math.html>`_ for basic math routines +## * `stats module <stats.html>`_ for statistical analysis +## * `list of cryptographic and hashing modules <lib.html#pure-libraries-hashing>`_ +## in the standard library + +import std/[algorithm, math] +import std/private/[since, jsutils] + +when defined(nimPreviewSlimSystem): + import std/[assertions] + +include system/inclrtl +{.push debugger: off.} +template whenHasBigInt64(yes64, no64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + yes64 + + +whenHasBigInt64: + type Ui = uint64 + + const randMax = 18_446_744_073_709_551_615u64 +do: + type Ui = uint32 + + const randMax = 4_294_967_295u32 + + +type + Rand* = object ## State of a random number generator. + ## + ## Create a new Rand state using the `initRand proc <#initRand,int64>`_. + ## + ## The module contains a default Rand state for convenience. + ## It corresponds to the default RNG's state. + ## The default Rand state always starts with the same values, but the + ## `randomize proc <#randomize>`_ can be used to seed the default generator + ## with a value based on the current time. + ## + ## Many procs have two variations: one that takes in a Rand parameter and + ## another that uses the default generator. The procs that use the default + ## generator are **not** thread-safe! + a0, a1: Ui + +whenHasBigInt64: + const DefaultRandSeed = Rand( + a0: 0x69B4C98CB8530805u64, + a1: 0xFED1DD3004688D67CAu64) + + # racy for multi-threading but good enough for now: + var state = DefaultRandSeed # global for backwards compatibility +do: + var state = Rand( + a0: 0x69B4C98Cu32, + a1: 0xFED1DD30u32) # global for backwards compatibility + +func isValid(r: Rand): bool {.inline.} = + ## Check whether state of `r` is valid. + ## + ## In `xoroshiro128+`, if all bits of `a0` and `a1` are zero, + ## they are always zero after calling `next(r: var Rand)`. + not (r.a0 == 0 and r.a1 == 0) + +since (1, 5): + template randState*(): untyped = + ## Makes the default Rand state accessible from other modules. + ## Useful for module authors. + state + +proc rotl(x, k: Ui): Ui = + result = (x shl k) or (x shr (Ui(64) - k)) + +proc next*(r: var Rand): uint64 = + ## Computes a random `uint64` number using the given state. + ## + ## **See also:** + ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer between zero and + ## a given upper bound + ## * `rand proc<#rand,Rand,range[]>`_ that returns a float + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + ## * `skipRandomNumbers proc<#skipRandomNumbers,Rand>`_ + runnableExamples("-r:off"): + var r = initRand(2019) + assert r.next() == 13223559681708962501'u64 # implementation defined + assert r.next() == 7229677234260823147'u64 # ditto + + let s0 = r.a0 + var s1 = r.a1 + result = s0 + s1 + s1 = s1 xor s0 + r.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b + r.a1 = rotl(s1, 36) # c + +proc skipRandomNumbers*(s: var Rand) = + ## The jump function for the generator. + ## + ## This proc is equivalent to `2^64` calls to `next <#next,Rand>`_, and it can + ## be used to generate `2^64` non-overlapping subsequences for parallel + ## computations. + ## + ## When multiple threads are generating random numbers, each thread must + ## own the `Rand <#Rand>`_ state it is using so that the thread can safely + ## obtain random numbers. However, if each thread creates its own Rand state, + ## the subsequences of random numbers that each thread generates may overlap, + ## even if the provided seeds are unique. This is more likely to happen as the + ## number of threads and amount of random numbers generated increases. + ## + ## If many threads will generate random numbers concurrently, it is better to + ## create a single Rand state and pass it to each thread. After passing the + ## Rand state to a thread, call this proc before passing it to the next one. + ## By using the Rand state this way, the subsequences of random numbers + ## generated in each thread will never overlap as long as no thread generates + ## more than `2^64` random numbers. + ## + ## **See also:** + ## * `next proc<#next,Rand>`_ + runnableExamples("--threads:on"): + import std/random + + const numbers = 100000 + + var + thr: array[0..3, Thread[(Rand, int)]] + vals: array[0..3, int] + + proc randomSum(params: tuple[r: Rand, index: int]) {.thread.} = + var r = params.r + var s = 0 # avoid cache thrashing + for i in 1..numbers: + s += r.rand(0..10) + vals[params.index] = s + + var r = initRand(2019) + for i in 0..<thr.len: + createThread(thr[i], randomSum, (r, i)) + r.skipRandomNumbers() + + joinThreads(thr) + + for val in vals: + doAssert abs(val - numbers * 5) / numbers < 0.1 + + doAssert vals == [501737, 497901, 500683, 500157] + + + whenHasBigInt64: + const helper = [0xbeac0467eba5facbu64, 0xd86b048b86aa9922u64] + do: + const helper = [0xbeac0467u32, 0xd86b048bu32] + var + s0 = Ui 0 + s1 = Ui 0 + for i in 0..high(helper): + for b in 0 ..< 64: + if (helper[i] and (Ui(1) shl Ui(b))) != 0: + s0 = s0 xor s.a0 + s1 = s1 xor s.a1 + discard next(s) + s.a0 = s0 + s.a1 = s1 + +proc rand[T: uint | uint64](r: var Rand; max: T): T = + # xxx export in future work + if max == 0: return + else: + let max = uint64(max) + when T.high.uint64 == uint64.high: + if max == uint64.high: return T(next(r)) + var iters = 0 + while true: + let x = next(r) + # avoid `mod` bias + if x <= randMax - (randMax mod max) or iters > 20: + return T(x mod (max + 1)) + else: + inc iters + +proc rand*(r: var Rand; max: Natural): int {.benign.} = + ## Returns a random integer in the range `0..max` using the given state. + ## + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer using the default RNG + ## * `rand proc<#rand,Rand,range[]>`_ that returns a float + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + runnableExamples: + var r = initRand(123) + if false: + assert r.rand(100) == 96 # implementation defined + # bootstrap: can't use `runnableExamples("-r:off")` + cast[int](rand(r, uint64(max))) + # xxx toUnsigned pending https://github.com/nim-lang/Nim/pull/18445 + +proc rand*(max: int): int {.benign.} = + ## Returns a random integer in the range `0..max`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer using a + ## provided state + ## * `rand proc<#rand,float>`_ that returns a float + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + runnableExamples("-r:off"): + randomize(123) + assert [rand(100), rand(100)] == [96, 63] # implementation defined + + rand(state, max) + +proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} = + ## Returns a random floating point number in the range `0.0..max` + ## using the given state. + ## + ## **See also:** + ## * `rand proc<#rand,float>`_ that returns a float using the default RNG + ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + runnableExamples: + var r = initRand(234) + let f = r.rand(1.0) # 8.717181376738381e-07 + + let x = next(r) + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + result = (float(x) / float(high(uint64))) * max + else: + result = (float(x) / float(high(uint32))) * max + else: + result = (float(x) / float(high(uint32))) * max + else: + let u = (0x3FFu64 shl 52u64) or (x shr 12u64) + result = (cast[float](u) - 1.0) * max + +proc rand*(max: float): float {.benign.} = + ## Returns a random floating point number in the range `0.0..max`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `rand proc<#rand,Rand,range[]>`_ that returns a float using a + ## provided state + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + runnableExamples: + randomize(234) + let f = rand(1.0) # 8.717181376738381e-07 + + rand(state, max) + +proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T = + ## For a slice `a..b`, returns a value in the range `a..b` using the given + ## state. + ## + ## Allowed types for `T` are integers, floats, and enums without holes. + ## + ## **See also:** + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice and uses the default RNG + ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer + ## * `rand proc<#rand,Rand,range[]>`_ that returns a float + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + runnableExamples: + var r = initRand(345) + assert r.rand(1..5) <= 5 + assert r.rand(-1.1 .. 1.2) >= -1.1 + assert x.a <= x.b + when T is SomeFloat: + result = rand(r, x.b - x.a) + x.a + else: # Integers and Enum types + whenJsNoBigInt64: + result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a)) + do: + result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a)) + +proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = + ## For a slice `a..b`, returns a value in the range `a..b`. + ## + ## Allowed types for `T` are integers, floats, and enums without holes. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice and uses a provided state + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,float>`_ that returns a floating point number + ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type + runnableExamples: + randomize(345) + assert rand(1..6) <= 6 + + result = rand(state, x) + +proc rand*[T: Ordinal](r: var Rand; t: typedesc[T]): T {.since: (1, 7, 1).} = + ## Returns a random Ordinal in the range `low(T)..high(T)`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,float>`_ that returns a floating point number + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + when T is range or T is enum: + result = rand(r, low(T)..high(T)) + elif T is bool: + result = r.next < randMax div 2 + else: + whenJsNoBigInt64: + result = cast[T](r.next shr (sizeof(uint)*8 - sizeof(T)*8)) + do: + result = cast[T](r.next shr (sizeof(uint64)*8 - sizeof(T)*8)) + +proc rand*[T: Ordinal](t: typedesc[T]): T = + ## Returns a random Ordinal in the range `low(T)..high(T)`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,float>`_ that returns a floating point number + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + runnableExamples: + randomize(567) + type E = enum a, b, c, d + + assert rand(E) in a..d + assert rand(char) in low(char)..high(char) + assert rand(int8) in low(int8)..high(int8) + assert rand(uint32) in low(uint32)..high(uint32) + assert rand(range[1..16]) in 1..16 + + result = rand(state, t) + +proc sample*[T](r: var Rand; s: set[T]): T = + ## Returns a random element from the set `s` using the given state. + ## + ## **See also:** + ## * `sample proc<#sample,set[T]>`_ that uses the default RNG + ## * `sample proc<#sample,Rand,openArray[T]>`_ for `openArray`s + ## * `sample proc<#sample,Rand,openArray[T],openArray[U]>`_ that uses a + ## cumulative distribution function + runnableExamples: + var r = initRand(987) + let s = {1, 3, 5, 7, 9} + assert r.sample(s) in s + + assert card(s) != 0 + var i = rand(r, card(s) - 1) + for e in s: + if i == 0: return e + dec(i) + +proc sample*[T](s: set[T]): T = + ## Returns a random element from the set `s`. + ## + ## If `randomize <#randomize>`_ has not been called, the order of outcomes + ## from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `sample proc<#sample,Rand,set[T]>`_ that uses a provided state + ## * `sample proc<#sample,openArray[T]>`_ for `openArray`s + ## * `sample proc<#sample,openArray[T],openArray[U]>`_ that uses a + ## cumulative distribution function + runnableExamples: + randomize(987) + let s = {1, 3, 5, 7, 9} + assert sample(s) in s + + sample(state, s) + +proc sample*[T](r: var Rand; a: openArray[T]): T = + ## Returns a random element from `a` using the given state. + ## + ## **See also:** + ## * `sample proc<#sample,openArray[T]>`_ that uses the default RNG + ## * `sample proc<#sample,Rand,openArray[T],openArray[U]>`_ that uses a + ## cumulative distribution function + ## * `sample proc<#sample,Rand,set[T]>`_ for sets + runnableExamples: + let marbles = ["red", "blue", "green", "yellow", "purple"] + var r = initRand(456) + assert r.sample(marbles) in marbles + + result = a[r.rand(a.low..a.high)] + +proc sample*[T](a: openArray[T]): lent T = + ## Returns a random element from `a`. + ## + ## If `randomize <#randomize>`_ has not been called, the order of outcomes + ## from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `sample proc<#sample,Rand,openArray[T]>`_ that uses a provided state + ## * `sample proc<#sample,openArray[T],openArray[U]>`_ that uses a + ## cumulative distribution function + ## * `sample proc<#sample,set[T]>`_ for sets + runnableExamples: + let marbles = ["red", "blue", "green", "yellow", "purple"] + randomize(456) + assert sample(marbles) in marbles + + result = a[rand(a.low..a.high)] + +proc sample*[T, U](r: var Rand; a: openArray[T]; cdf: openArray[U]): T = + ## Returns an element from `a` using a cumulative distribution function + ## (CDF) and the given state. + ## + ## The `cdf` argument does not have to be normalized, and it could contain + ## any type of elements that can be converted to a `float`. It must be + ## the same length as `a`. Each element in `cdf` should be greater than + ## or equal to the previous element. + ## + ## The outcome of the `cumsum<math.html#cumsum,openArray[T]>`_ proc and the + ## return value of the `cumsummed<math.html#cumsummed,openArray[T]>`_ proc, + ## which are both in the math module, can be used as the `cdf` argument. + ## + ## **See also:** + ## * `sample proc<#sample,openArray[T],openArray[U]>`_ that also utilizes + ## a CDF but uses the default RNG + ## * `sample proc<#sample,Rand,openArray[T]>`_ that does not use a CDF + ## * `sample proc<#sample,Rand,set[T]>`_ for sets + runnableExamples: + from std/math import cumsummed + + let marbles = ["red", "blue", "green", "yellow", "purple"] + let count = [1, 6, 8, 3, 4] + let cdf = count.cumsummed + var r = initRand(789) + assert r.sample(marbles, cdf) in marbles + + assert(cdf.len == a.len) # Two basic sanity checks. + assert(float(cdf[^1]) > 0.0) + # While we could check cdf[i-1] <= cdf[i] for i in 1..cdf.len, that could get + # awfully expensive even in debugging modes. + let u = r.rand(float(cdf[^1])) + a[cdf.upperBound(U(u))] + +proc sample*[T, U](a: openArray[T]; cdf: openArray[U]): T = + ## Returns an element from `a` using a cumulative distribution function + ## (CDF). + ## + ## This proc works similarly to + ## `sample <#sample,Rand,openArray[T],openArray[U]>`_. + ## See that proc's documentation for more details. + ## + ## If `randomize <#randomize>`_ has not been called, the order of outcomes + ## from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `sample proc<#sample,Rand,openArray[T],openArray[U]>`_ that also utilizes + ## a CDF but uses a provided state + ## * `sample proc<#sample,openArray[T]>`_ that does not use a CDF + ## * `sample proc<#sample,set[T]>`_ for sets + runnableExamples: + from std/math import cumsummed + + let marbles = ["red", "blue", "green", "yellow", "purple"] + let count = [1, 6, 8, 3, 4] + let cdf = count.cumsummed + randomize(789) + assert sample(marbles, cdf) in marbles + + state.sample(a, cdf) + +proc gauss*(r: var Rand; mu = 0.0; sigma = 1.0): float {.since: (1, 3).} = + ## Returns a Gaussian random variate, + ## with mean `mu` and standard deviation `sigma` + ## using the given state. + # Ratio of uniforms method for normal + # https://www2.econ.osaka-u.ac.jp/~tanizaki/class/2013/econome3/13.pdf + const K = sqrt(2 / E) + var + a = 0.0 + b = 0.0 + while true: + a = rand(r, 1.0) + b = (2.0 * rand(r, 1.0) - 1.0) * K + if b * b <= -4.0 * a * a * ln(a): break + result = mu + sigma * (b / a) + +proc gauss*(mu = 0.0, sigma = 1.0): float {.since: (1, 3).} = + ## Returns a Gaussian random variate, + ## with mean `mu` and standard deviation `sigma`. + ## + ## If `randomize <#randomize>`_ has not been called, the order of outcomes + ## from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + result = gauss(state, mu, sigma) + +proc initRand*(seed: int64): Rand = + ## Initializes a new `Rand <#Rand>`_ state using the given seed. + ## + ## Providing a specific seed will produce the same results for that seed each time. + ## + ## The resulting state is independent of the default RNG's state. When `seed == 0`, + ## we internally set the seed to an implementation defined non-zero value. + ## + ## **See also:** + ## * `initRand proc<#initRand>`_ that uses the current time + ## * `randomize proc<#randomize,int64>`_ that accepts a seed for the default RNG + ## * `randomize proc<#randomize>`_ that initializes the default RNG using the current time + runnableExamples: + from std/times import getTime, toUnix, nanosecond + + var r1 = initRand(123) + let now = getTime() + var r2 = initRand(now.toUnix * 1_000_000_000 + now.nanosecond) + const seedFallback0 = int32.high # arbitrary + let seed = if seed != 0: seed else: seedFallback0 # because 0 is a fixed point + result.a0 = Ui(seed shr 16) + result.a1 = Ui(seed and 0xffff) + when not defined(nimLegacyRandomInitRand): + # calling `discard next(result)` (even a few times) would still produce + # skewed numbers for the 1st call to `rand()`. + skipRandomNumbers(result) + discard next(result) + +proc randomize*(seed: int64) {.benign.} = + ## Initializes the default random number generator with the given seed. + ## + ## Providing a specific seed will produce the same results for that seed each time. + ## + ## **See also:** + ## * `initRand proc<#initRand,int64>`_ that initializes a Rand state + ## with a given seed + ## * `randomize proc<#randomize>`_ that uses the current time instead + ## * `initRand proc<#initRand>`_ that initializes a Rand state using + ## the current time + runnableExamples: + from std/times import getTime, toUnix, nanosecond + + randomize(123) + + let now = getTime() + randomize(now.toUnix * 1_000_000_000 + now.nanosecond) + + state = initRand(seed) + +proc shuffle*[T](r: var Rand; x: var openArray[T]) = + ## Shuffles a sequence of elements in-place using the given state. + ## + ## **See also:** + ## * `shuffle proc<#shuffle,openArray[T]>`_ that uses the default RNG + runnableExamples: + var cards = ["Ace", "King", "Queen", "Jack", "Ten"] + var r = initRand(678) + r.shuffle(cards) + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] + + for i in countdown(x.high, 1): + let j = r.rand(i) + swap(x[i], x[j]) + +proc shuffle*[T](x: var openArray[T]) = + ## Shuffles a sequence of elements in-place. + ## + ## If `randomize <#randomize>`_ has not been called, the order of outcomes + ## from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + ## + ## **See also:** + ## * `shuffle proc<#shuffle,Rand,openArray[T]>`_ that uses a provided state + runnableExamples: + var cards = ["Ace", "King", "Queen", "Jack", "Ten"] + randomize(678) + shuffle(cards) + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] + + shuffle(state, x) + +when not defined(standalone): + when defined(js): + import std/times + else: + when defined(nimscript): + import std/hashes + else: + import std/[hashes, os, sysrand, monotimes] + + when compileOption("threads"): + import std/locks + var baseSeedLock: Lock + baseSeedLock.initLock + + var baseState: Rand + + proc initRand(): Rand = + ## Initializes a new Rand state. + ## + ## The resulting state is independent of the default RNG's state. + ## + ## **Note:** Does not work for the compile-time VM. + ## + ## See also: + ## * `initRand proc<#initRand,int64>`_ that accepts a seed for a new Rand state + ## * `randomize proc<#randomize>`_ that initializes the default RNG using the current time + ## * `randomize proc<#randomize,int64>`_ that accepts a seed for the default RNG + when defined(js): + let time = int64(times.epochTime() * 1000) and 0x7fff_ffff + result = initRand(time) + else: + proc getRandomState(): Rand = + when defined(nimscript): + result = Rand( + a0: CompileTime.hash.Ui, + a1: CompileDate.hash.Ui) + if not result.isValid: + result = DefaultRandSeed + else: + var urand: array[sizeof(Rand), byte] + + for i in 0 .. 7: + if sysrand.urandom(urand): + copyMem(result.addr, urand[0].addr, sizeof(Rand)) + if result.isValid: + break + + if not result.isValid: + # Don't try to get alternative random values from other source like time or process/thread id, + # because such code would be never tested and is a liability for security. + quit("Failed to initializes baseState in random module as sysrand.urandom doesn't work.") + + when compileOption("threads"): + baseSeedLock.withLock: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + else: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + + since (1, 5, 1): + export initRand + + proc randomize*() {.benign.} = + ## Initializes the default random number generator with a seed based on + ## random number source. + ## + ## This proc only needs to be called once, and it should be called before + ## the first usage of procs from this module that use the default RNG. + ## + ## **Note:** Does not work for the compile-time VM. + ## + ## **See also:** + ## * `randomize proc<#randomize,int64>`_ that accepts a seed + ## * `initRand proc<#initRand>`_ that initializes a Rand state using + ## the current time + ## * `initRand proc<#initRand,int64>`_ that initializes a Rand state + ## with a given seed + state = initRand() + +{.pop.} diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index 72c64befc..5f806bd70 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -8,207 +8,213 @@ # -## This module implements rational numbers, consisting of a numerator `num` and -## a denominator `den`, both of type int. The denominator can not be 0. +## This module implements rational numbers, consisting of a numerator and +## a denominator. The denominator can not be 0. -import math -import hashes +runnableExamples: + let + r1 = 1 // 2 + r2 = -3 // 4 + + doAssert r1 + r2 == -1 // 4 + doAssert r1 - r2 == 5 // 4 + doAssert r1 * r2 == -3 // 8 + doAssert r1 / r2 == -2 // 3 + +import std/[math, hashes] +when defined(nimPreviewSlimSystem): + import std/assertions type Rational*[T] = object - ## a rational number, consisting of a numerator and denominator + ## A rational number, consisting of a numerator `num` and a denominator `den`. num*, den*: T -proc initRational*[T:SomeInteger](num, den: T): Rational[T] = - ## Create a new rational number. - assert(den != 0, "a denominator of zero value is invalid") +func reduce*[T: SomeInteger](x: var Rational[T]) = + ## Reduces the rational number `x`, so that the numerator and denominator + ## have no common divisors other than 1 (and -1). + ## If `x` is 0, raises `DivByZeroDefect`. + ## + ## **Note:** This is called automatically by the various operations on rationals. + runnableExamples: + var r = Rational[int](num: 2, den: 4) # 1/2 + reduce(r) + doAssert r.num == 1 + doAssert r.den == 2 + if x.den == 0: + raise newException(DivByZeroDefect, "division by zero") + let common = gcd(x.num, x.den) + if x.den > 0: + x.num = x.num div common + x.den = x.den div common + when T isnot SomeUnsignedInt: + if x.den < 0: + x.num = -x.num div common + x.den = -x.den div common + +func initRational*[T: SomeInteger](num, den: T): Rational[T] = + ## Creates a new rational number with numerator `num` and denominator `den`. + ## `den` must not be 0. + ## + ## **Note:** `den != 0` is not checked when assertions are turned off. + assert(den != 0, "a denominator of zero is invalid") result.num = num result.den = den + reduce(result) -proc `//`*[T](num, den: T): Rational[T] = initRational[T](num, den) - ## A friendlier version of `initRational`. Example usage: - ## - ## .. code-block:: nim - ## var x = 1//3 + 1//5 +func `//`*[T](num, den: T): Rational[T] = + ## A friendlier version of `initRational <#initRational,T,T>`_. + runnableExamples: + let x = 1 // 3 + 1 // 5 + doAssert x == 8 // 15 + + initRational[T](num, den) + +func `$`*[T](x: Rational[T]): string = + ## Turns a rational number into a string. + runnableExamples: + doAssert $(1 // 2) == "1/2" -proc `$`*[T](x: Rational[T]): string = - ## Turn a rational number into a string. result = $x.num & "/" & $x.den -proc toRational*[T:SomeInteger](x: T): Rational[T] = - ## Convert some integer `x` to a rational number. +func toRational*[T: SomeInteger](x: T): Rational[T] = + ## Converts some integer `x` to a rational number. + runnableExamples: + doAssert toRational(42) == 42 // 1 + result.num = x result.den = 1 -proc toRationalSub(x: float, n: int): Rational[int] = - var - a = 0 - b, c, d = 1 - result = 0 // 1 # rational 0 - while b <= n and d <= n: - let ac = (a+c) - let bd = (b+d) - # scale by 1000 so not overflow for high precision - let mediant = (ac/1000) / (bd/1000) - if x == mediant: - if bd <= n: - result.num = ac - result.den = bd - return result - elif d > b: - result.num = c - result.den = d - return result - else: - result.num = a - result.den = b - return result - elif x > mediant: - a = ac - b = bd - else: - c = ac - d = bd - if (b > n): - return initRational(c, d) - return initRational(a, b) - -proc toRational*(x: float, n: int = high(int)): Rational[int] = - ## Calculate the best rational numerator and denominator - ## that approximates to `x`, where the denominator is - ## smaller than `n` (default is the largest possible - ## int to give maximum resolution) - ## - ## The algorithm is based on the Farey sequence named - ## after John Farey +func toRational*(x: float, + n: int = high(int) shr (sizeof(int) div 2 * 8)): Rational[int] = + ## Calculates the best rational approximation of `x`, + ## where the denominator is smaller than `n` + ## (default is the largest possible `int` for maximal resolution). ## - ## .. code-block:: Nim - ## import math, rationals - ## for i in 1..10: - ## let t = (10 ^ (i+3)).int - ## let x = toRational(PI, t) - ## let newPI = x.num / x.den - ## echo x, " ", newPI, " error: ", PI - newPI, " ", t - if x > 1: - result = toRationalSub(1.0/x, n) - swap(result.num, result.den) - elif x == 1.0: - result = 1 // 1 - else: - result = toRationalSub(x, n) + ## The algorithm is based on the theory of continued fractions. + # David Eppstein / UC Irvine / 8 Aug 1993 + # With corrections from Arno Formella, May 2008 + runnableExamples: + let x = 1.2 + doAssert x.toRational.toFloat == x -proc toFloat*[T](x: Rational[T]): float = - ## Convert a rational number `x` to a float. + var + m11, m22 = 1 + m12, m21 = 0 + ai = int(x) + x = x + while m21 * ai + m22 <= n: + swap m12, m11 + swap m22, m21 + m11 = m12 * ai + m11 + m21 = m22 * ai + m21 + if x == float(ai): break # division by zero + x = 1 / (x - float(ai)) + if x > float(high(int32)): break # representation failure + ai = int(x) + result = m11 // m21 + +func toFloat*[T](x: Rational[T]): float = + ## Converts a rational number `x` to a `float`. x.num / x.den -proc toInt*[T](x: Rational[T]): int = - ## Convert a rational number `x` to an int. Conversion rounds towards 0 if +func toInt*[T](x: Rational[T]): int = + ## Converts a rational number `x` to an `int`. Conversion rounds towards 0 if ## `x` does not contain an integer value. x.num div x.den -proc reduce*[T:SomeInteger](x: var Rational[T]) = - ## Reduce rational `x`. - let common = gcd(x.num, x.den) - if x.den > 0: - x.num = x.num div common - x.den = x.den div common - elif x.den < 0: - x.num = -x.num div common - x.den = -x.den div common - else: - raise newException(DivByZeroError, "division by zero") - -proc `+` *[T](x, y: Rational[T]): Rational[T] = - ## Add two rational numbers. +func `+`*[T](x, y: Rational[T]): Rational[T] = + ## Adds two rational numbers. let common = lcm(x.den, y.den) result.num = common div x.den * x.num + common div y.den * y.num result.den = common reduce(result) -proc `+` *[T](x: Rational[T], y: T): Rational[T] = - ## Add rational `x` to int `y`. +func `+`*[T](x: Rational[T], y: T): Rational[T] = + ## Adds the rational `x` to the int `y`. result.num = x.num + y * x.den result.den = x.den -proc `+` *[T](x: T, y: Rational[T]): Rational[T] = - ## Add int `x` to rational `y`. +func `+`*[T](x: T, y: Rational[T]): Rational[T] = + ## Adds the int `x` to the rational `y`. result.num = x * y.den + y.num result.den = y.den -proc `+=` *[T](x: var Rational[T], y: Rational[T]) = - ## Add rational `y` to rational `x`. +func `+=`*[T](x: var Rational[T], y: Rational[T]) = + ## Adds the rational `y` to the rational `x` in-place. let common = lcm(x.den, y.den) x.num = common div x.den * x.num + common div y.den * y.num x.den = common reduce(x) -proc `+=` *[T](x: var Rational[T], y: T) = - ## Add int `y` to rational `x`. +func `+=`*[T](x: var Rational[T], y: T) = + ## Adds the int `y` to the rational `x` in-place. x.num += y * x.den -proc `-` *[T](x: Rational[T]): Rational[T] = +func `-`*[T](x: Rational[T]): Rational[T] = ## Unary minus for rational numbers. result.num = -x.num result.den = x.den -proc `-` *[T](x, y: Rational[T]): Rational[T] = - ## Subtract two rational numbers. +func `-`*[T](x, y: Rational[T]): Rational[T] = + ## Subtracts two rational numbers. let common = lcm(x.den, y.den) result.num = common div x.den * x.num - common div y.den * y.num result.den = common reduce(result) -proc `-` *[T](x: Rational[T], y: T): Rational[T] = - ## Subtract int `y` from rational `x`. +func `-`*[T](x: Rational[T], y: T): Rational[T] = + ## Subtracts the int `y` from the rational `x`. result.num = x.num - y * x.den result.den = x.den -proc `-` *[T](x: T, y: Rational[T]): Rational[T] = - ## Subtract rational `y` from int `x`. - result.num = - x * y.den + y.num +func `-`*[T](x: T, y: Rational[T]): Rational[T] = + ## Subtracts the rational `y` from the int `x`. + result.num = x * y.den - y.num result.den = y.den -proc `-=` *[T](x: var Rational[T], y: Rational[T]) = - ## Subtract rational `y` from rational `x`. +func `-=`*[T](x: var Rational[T], y: Rational[T]) = + ## Subtracts the rational `y` from the rational `x` in-place. let common = lcm(x.den, y.den) x.num = common div x.den * x.num - common div y.den * y.num x.den = common reduce(x) -proc `-=` *[T](x: var Rational[T], y: T) = - ## Subtract int `y` from rational `x`. +func `-=`*[T](x: var Rational[T], y: T) = + ## Subtracts the int `y` from the rational `x` in-place. x.num -= y * x.den -proc `*` *[T](x, y: Rational[T]): Rational[T] = - ## Multiply two rational numbers. +func `*`*[T](x, y: Rational[T]): Rational[T] = + ## Multiplies two rational numbers. result.num = x.num * y.num result.den = x.den * y.den reduce(result) -proc `*` *[T](x: Rational[T], y: T): Rational[T] = - ## Multiply rational `x` with int `y`. +func `*`*[T](x: Rational[T], y: T): Rational[T] = + ## Multiplies the rational `x` with the int `y`. result.num = x.num * y result.den = x.den reduce(result) -proc `*` *[T](x: T, y: Rational[T]): Rational[T] = - ## Multiply int `x` with rational `y`. +func `*`*[T](x: T, y: Rational[T]): Rational[T] = + ## Multiplies the int `x` with the rational `y`. result.num = x * y.num result.den = y.den reduce(result) -proc `*=` *[T](x: var Rational[T], y: Rational[T]) = - ## Multiply rationals `y` to `x`. +func `*=`*[T](x: var Rational[T], y: Rational[T]) = + ## Multiplies the rational `x` by `y` in-place. x.num *= y.num x.den *= y.den reduce(x) -proc `*=` *[T](x: var Rational[T], y: T) = - ## Multiply int `y` to rational `x`. +func `*=`*[T](x: var Rational[T], y: T) = + ## Multiplies the rational `x` by the int `y` in-place. x.num *= y reduce(x) -proc reciprocal*[T](x: Rational[T]): Rational[T] = - ## Calculate the reciprocal of `x`. (1/x) +func reciprocal*[T](x: Rational[T]): Rational[T] = + ## Calculates the reciprocal of `x` (`1/x`). + ## If `x` is 0, raises `DivByZeroDefect`. if x.num > 0: result.num = x.den result.den = x.num @@ -216,56 +222,94 @@ proc reciprocal*[T](x: Rational[T]): Rational[T] = result.num = -x.den result.den = -x.num else: - raise newException(DivByZeroError, "division by zero") + raise newException(DivByZeroDefect, "division by zero") -proc `/`*[T](x, y: Rational[T]): Rational[T] = - ## Divide rationals `x` by `y`. +func `/`*[T](x, y: Rational[T]): Rational[T] = + ## Divides the rational `x` by the rational `y`. result.num = x.num * y.den result.den = x.den * y.num reduce(result) -proc `/`*[T](x: Rational[T], y: T): Rational[T] = - ## Divide rational `x` by int `y`. +func `/`*[T](x: Rational[T], y: T): Rational[T] = + ## Divides the rational `x` by the int `y`. result.num = x.num result.den = x.den * y reduce(result) -proc `/`*[T](x: T, y: Rational[T]): Rational[T] = - ## Divide int `x` by Rational `y`. +func `/`*[T](x: T, y: Rational[T]): Rational[T] = + ## Divides the int `x` by the rational `y`. result.num = x * y.den result.den = y.num reduce(result) -proc `/=`*[T](x: var Rational[T], y: Rational[T]) = - ## Divide rationals `x` by `y` in place. +func `/=`*[T](x: var Rational[T], y: Rational[T]) = + ## Divides the rational `x` by the rational `y` in-place. x.num *= y.den x.den *= y.num reduce(x) -proc `/=`*[T](x: var Rational[T], y: T) = - ## Divide rational `x` by int `y` in place. +func `/=`*[T](x: var Rational[T], y: T) = + ## Divides the rational `x` by the int `y` in-place. x.den *= y reduce(x) -proc cmp*(x, y: Rational): int {.procvar.} = - ## Compares two rationals. +func cmp*(x, y: Rational): int = + ## Compares two rationals. Returns + ## * a value less than zero, if `x < y` + ## * a value greater than zero, if `x > y` + ## * zero, if `x == y` (x - y).num -proc `<` *(x, y: Rational): bool = +func `<`*(x, y: Rational): bool = + ## Returns true if `x` is less than `y`. (x - y).num < 0 -proc `<=` *(x, y: Rational): bool = +func `<=`*(x, y: Rational): bool = + ## Returns tue if `x` is less than or equal to `y`. (x - y).num <= 0 -proc `==` *(x, y: Rational): bool = +func `==`*(x, y: Rational): bool = + ## Compares two rationals for equality. (x - y).num == 0 -proc abs*[T](x: Rational[T]): Rational[T] = +func abs*[T](x: Rational[T]): Rational[T] = + ## Returns the absolute value of `x`. + runnableExamples: + doAssert abs(1 // 2) == 1 // 2 + doAssert abs(-1 // 2) == 1 // 2 + result.num = abs x.num result.den = abs x.den -proc hash*[T](x: Rational[T]): Hash = - ## Computes hash for rational `x` +func `div`*[T: SomeInteger](x, y: Rational[T]): T = + ## Computes the rational truncated division. + (x.num * y.den) div (y.num * x.den) + +func `mod`*[T: SomeInteger](x, y: Rational[T]): Rational[T] = + ## Computes the rational modulo by truncated division (remainder). + ## This is same as `x - (x div y) * y`. + result = ((x.num * y.den) mod (y.num * x.den)) // (x.den * y.den) + reduce(result) + +func floorDiv*[T: SomeInteger](x, y: Rational[T]): T = + ## Computes the rational floor division. + ## + ## Floor division is conceptually defined as `floor(x / y)`. + ## This is different from the `div` operator, which is defined + ## as `trunc(x / y)`. That is, `div` rounds towards 0 and `floorDiv` + ## rounds down. + floorDiv(x.num * y.den, y.num * x.den) + +func floorMod*[T: SomeInteger](x, y: Rational[T]): Rational[T] = + ## Computes the rational modulo by floor division (modulo). + ## + ## This is same as `x - floorDiv(x, y) * y`. + ## This func behaves the same as the `%` operator in Python. + result = floorMod(x.num * y.den, y.num * x.den) // (x.den * y.den) + reduce(result) + +func hash*[T](x: Rational[T]): Hash = + ## Computes the hash for the rational `x`. # reduce first so that hash(x) == hash(y) for x == y var copy = x reduce(copy) @@ -275,78 +319,22 @@ proc hash*[T](x: Rational[T]): Hash = h = h !& hash(copy.den) result = !$h -when isMainModule: - var - z = Rational[int](num: 0, den: 1) - o = initRational(num=1, den=1) - a = initRational(1, 2) - b = -1 // -2 - m1 = -1 // 1 - tt = 10 // 2 - - assert( a == a ) - assert( (a-a) == z ) - assert( (a+b) == o ) - assert( (a/b) == o ) - assert( (a*b) == 1 // 4 ) - assert( (3/a) == 6 // 1 ) - assert( (a/3) == 1 // 6 ) - assert( a*b == 1 // 4 ) - assert( tt*z == z ) - assert( 10*a == tt ) - assert( a*10 == tt ) - assert( tt/10 == a ) - assert( a-m1 == 3 // 2 ) - assert( a+m1 == -1 // 2 ) - assert( m1+tt == 16 // 4 ) - assert( m1-tt == 6 // -1 ) - - assert( z < o ) - assert( z <= o ) - assert( z == z ) - assert( cmp(z, o) < 0 ) - assert( cmp(o, z) > 0 ) - - assert( o == o ) - assert( o >= o ) - assert( not(o > o) ) - assert( cmp(o, o) == 0 ) - assert( cmp(z, z) == 0 ) - assert( hash(o) == hash(o) ) - - assert( a == b ) - assert( a >= b ) - assert( not(b > a) ) - assert( cmp(a, b) == 0 ) - assert( hash(a) == hash(b) ) - - var x = 1//3 - - x *= 5//1 - assert( x == 5//3 ) - x += 2 // 9 - assert( x == 17//9 ) - x -= 9//18 - assert( x == 25//18 ) - x /= 1//2 - assert( x == 50//18 ) - - var y = 1//3 - - y *= 4 - assert( y == 4//3 ) - y += 5 - assert( y == 19//3 ) - y -= 2 - assert( y == 13//3 ) - y /= 9 - assert( y == 13//27 ) - - assert toRational(5) == 5//1 - assert abs(toFloat(y) - 0.4814814814814815) < 1.0e-7 - assert toInt(z) == 0 - - assert toRational(0.98765432) == 12345679 // 12500000 - assert toRational(0.1, 1000000) == 1 // 10 - assert toRational(0.9, 1000000) == 9 // 10 - assert toRational(PI) == 80143857 // 25510582 +func `^`*[T: SomeInteger](x: Rational[T], y: T): Rational[T] = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be an integer. Negative exponents are supported + ## but floating point exponents are not. + runnableExamples: + doAssert (-3 // 5) ^ 0 == (1 // 1) + doAssert (-3 // 5) ^ 1 == (-3 // 5) + doAssert (-3 // 5) ^ 2 == (9 // 25) + doAssert (-3 // 5) ^ -2 == (25 // 9) + + if y >= 0: + result.num = x.num ^ y + result.den = x.den ^ y + else: + result.num = x.den ^ -y + result.den = x.num ^ -y + # Note that all powers of reduced rationals are already reduced, + # so we don't need to call reduce() here diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim deleted file mode 100644 index 6741d3c8e..000000000 --- a/lib/pure/redis.nim +++ /dev/null @@ -1,1096 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a redis client. It allows you to connect to a -## redis-server instance, send commands and receive replies. -## -## **Beware**: Most (if not all) functions that return a ``RedisString`` may -## return ``redisNil``, and functions which return a ``RedisList`` -## may return ``nil``. - -import net, os, strutils, parseutils - -const - redisNil* = "\0\0" - -type - Pipeline = ref object - enabled: bool - buffer: string - expected: int ## number of replies expected if pipelined - -type - SendMode = enum - normal, pipelined, multiple - -type - Redis* = object - socket: net.Socket - connected: bool - pipeline: Pipeline - - RedisStatus* = string - RedisInteger* = BiggestInt - RedisString* = string ## Bulk reply - RedisList* = seq[RedisString] ## Multi-bulk reply - - ReplyError* = object of IOError ## Invalid reply from redis - RedisError* = object of IOError ## Error in redis - -{.deprecated: [TSendMode: SendMode, TRedis: Redis, TRedisStatus: RedisStatus, - TRedisInteger: RedisInteger, TRedisString: RedisString, - TRedisList: RedisList, EInvalidReply: ReplyError, ERedis: RedisError].} - -proc newPipeline(): Pipeline = - new(result) - result.buffer = "" - result.enabled = false - result.expected = 0 - -proc open*(host = "localhost", port = 6379.Port): Redis = - ## Opens a connection to the redis server. - result.socket = newSocket(buffered = false) - - result.socket.connect(host, port) - result.pipeline = newPipeline() - -proc raiseInvalidReply(expected, got: char) = - raise newException(ReplyError, - "Expected '$1' at the beginning of a status reply got '$2'" % - [$expected, $got]) - -proc raiseNoOK(status: string, pipelineEnabled: bool) = - if pipelineEnabled and not (status == "QUEUED" or status == "PIPELINED"): - raise newException(ReplyError, "Expected \"QUEUED\" or \"PIPELINED\" got \"$1\"" % status) - elif not pipelineEnabled and status != "OK": - raise newException(ReplyError, "Expected \"OK\" got \"$1\"" % status) - -template readSocket(r: Redis, dummyVal:expr): stmt = - var line {.inject.}: TaintedString = "" - if r.pipeline.enabled: - return dummyVal - else: - readLine(r.socket, line) - -proc parseStatus(r: Redis, line: string = ""): RedisStatus = - if r.pipeline.enabled: - return "PIPELINED" - - if line == "": - raise newException(RedisError, "Server closed connection prematurely") - - if line[0] == '-': - raise newException(RedisError, strip(line)) - if line[0] != '+': - raiseInvalidReply('+', line[0]) - - return line.substr(1) # Strip '+' - -proc readStatus(r:Redis): RedisStatus = - r.readSocket("PIPELINED") - return r.parseStatus(line) - -proc parseInteger(r: Redis, line: string = ""): RedisInteger = - if r.pipeline.enabled: return -1 - - #if line == "+QUEUED": # inside of multi - # return -1 - - if line == "": - raise newException(RedisError, "Server closed connection prematurely") - - if line[0] == '-': - raise newException(RedisError, strip(line)) - if line[0] != ':': - raiseInvalidReply(':', line[0]) - - # Strip ':' - if parseBiggestInt(line, result, 1) == 0: - raise newException(ReplyError, "Unable to parse integer.") - -proc readInteger(r: Redis): RedisInteger = - r.readSocket(-1) - return r.parseInteger(line) - -proc recv(sock: Socket, size: int): TaintedString = - result = newString(size).TaintedString - if sock.recv(cstring(result), size) != size: - raise newException(ReplyError, "recv failed") - -proc parseSingleString(r: Redis, line:string, allowMBNil = false): RedisString = - if r.pipeline.enabled: return "" - - # Error. - if line[0] == '-': - raise newException(RedisError, strip(line)) - - # Some commands return a /bulk/ value or a /multi-bulk/ nil. Odd. - if allowMBNil: - if line == "*-1": - return redisNil - - if line[0] != '$': - raiseInvalidReply('$', line[0]) - - var numBytes = parseInt(line.substr(1)) - if numBytes == -1: - return redisNil - - var s = r.socket.recv(numBytes+2) - result = strip(s.string) - -proc readSingleString(r: Redis): RedisString = - r.readSocket("") - return r.parseSingleString(line) - -proc readNext(r: Redis): RedisList - -proc parseArrayLines(r: Redis, countLine:string): RedisList = - if countLine.string[0] != '*': - raiseInvalidReply('*', countLine.string[0]) - - var numElems = parseInt(countLine.string.substr(1)) - if numElems == -1: return nil - result = @[] - - for i in 1..numElems: - var parsed = r.readNext() - if not isNil(parsed): - for item in parsed: - result.add(item) - -proc readArrayLines(r: Redis): RedisList = - r.readSocket(nil) - return r.parseArrayLines(line) - -proc parseBulkString(r: Redis, allowMBNil = false, line:string = ""): RedisString = - if r.pipeline.enabled: return "" - - return r.parseSingleString(line, allowMBNil) - -proc readBulkString(r: Redis, allowMBNil = false): RedisString = - r.readSocket("") - return r.parseBulkString(allowMBNil, line) - -proc readArray(r: Redis): RedisList = - r.readSocket(@[]) - return r.parseArrayLines(line) - -proc readNext(r: Redis): RedisList = - r.readSocket(@[]) - - var res = case line[0] - of '+', '-': @[r.parseStatus(line)] - of ':': @[$(r.parseInteger(line))] - of '$': @[r.parseBulkString(true,line)] - of '*': r.parseArrayLines(line) - else: - raise newException(ReplyError, "readNext failed on line: " & line) - nil - r.pipeline.expected -= 1 - return res - -proc flushPipeline*(r: Redis, wasMulti = false): RedisList = - ## Send buffered commands, clear buffer, return results - if r.pipeline.buffer.len > 0: - r.socket.send(r.pipeline.buffer) - r.pipeline.buffer = "" - - r.pipeline.enabled = false - result = @[] - - var tot = r.pipeline.expected - - for i in 0..tot-1: - var ret = r.readNext() - for item in ret: - if not (item.contains("OK") or item.contains("QUEUED")): - result.add(item) - - r.pipeline.expected = 0 - -proc startPipelining*(r: Redis) = - ## Enable command pipelining (reduces network roundtrips). - ## Note that when enabled, you must call flushPipeline to actually send commands, except - ## for multi/exec() which enable and flush the pipeline automatically. - ## Commands return immediately with dummy values; actual results returned from - ## flushPipeline() or exec() - r.pipeline.expected = 0 - r.pipeline.enabled = true - -proc sendCommand(r: Redis, cmd: string, args: varargs[string]) = - var request = "*" & $(1 + args.len()) & "\c\L" - request.add("$" & $cmd.len() & "\c\L") - request.add(cmd & "\c\L") - for i in items(args): - request.add("$" & $i.len() & "\c\L") - request.add(i & "\c\L") - - if r.pipeline.enabled: - r.pipeline.buffer.add(request) - r.pipeline.expected += 1 - else: - r.socket.send(request) - -proc sendCommand(r: Redis, cmd: string, arg1: string, - args: varargs[string]) = - var request = "*" & $(2 + args.len()) & "\c\L" - request.add("$" & $cmd.len() & "\c\L") - request.add(cmd & "\c\L") - request.add("$" & $arg1.len() & "\c\L") - request.add(arg1 & "\c\L") - for i in items(args): - request.add("$" & $i.len() & "\c\L") - request.add(i & "\c\L") - - if r.pipeline.enabled: - r.pipeline.expected += 1 - r.pipeline.buffer.add(request) - else: - r.socket.send(request) - -# Keys - -proc del*(r: Redis, keys: varargs[string]): RedisInteger = - ## Delete a key or multiple keys - r.sendCommand("DEL", keys) - return r.readInteger() - -proc exists*(r: Redis, key: string): bool = - ## Determine if a key exists - r.sendCommand("EXISTS", key) - return r.readInteger() == 1 - -proc expire*(r: Redis, key: string, seconds: int): bool = - ## Set a key's time to live in seconds. Returns `false` if the key could - ## not be found or the timeout could not be set. - r.sendCommand("EXPIRE", key, $seconds) - return r.readInteger() == 1 - -proc expireAt*(r: Redis, key: string, timestamp: int): bool = - ## Set the expiration for a key as a UNIX timestamp. Returns `false` - ## if the key could not be found or the timeout could not be set. - r.sendCommand("EXPIREAT", key, $timestamp) - return r.readInteger() == 1 - -proc keys*(r: Redis, pattern: string): RedisList = - ## Find all keys matching the given pattern - r.sendCommand("KEYS", pattern) - return r.readArray() - -proc scan*(r: Redis, cursor: var BiggestInt): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using default Redis values for MATCH and COUNT parameters - r.sendCommand("SCAN", $cursor) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc scan*(r: Redis, cursor: var BiggestInt, pattern: string): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using cursor as a client query identifier. Using default Redis value for COUNT argument - r.sendCommand("SCAN", $cursor, ["MATCH", pattern]) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc scan*(r: Redis, cursor: var BiggestInt, pattern: string, count: int): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using cursor as a client query identifier. - r.sendCommand("SCAN", $cursor, ["MATCH", pattern, "COUNT", $count]) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc move*(r: Redis, key: string, db: int): bool = - ## Move a key to another database. Returns `true` on a successful move. - r.sendCommand("MOVE", key, $db) - return r.readInteger() == 1 - -proc persist*(r: Redis, key: string): bool = - ## Remove the expiration from a key. - ## Returns `true` when the timeout was removed. - r.sendCommand("PERSIST", key) - return r.readInteger() == 1 - -proc randomKey*(r: Redis): RedisString = - ## Return a random key from the keyspace - r.sendCommand("RANDOMKEY") - return r.readBulkString() - -proc rename*(r: Redis, key, newkey: string): RedisStatus = - ## Rename a key. - ## - ## **WARNING:** Overwrites `newkey` if it exists! - r.sendCommand("RENAME", key, newkey) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc renameNX*(r: Redis, key, newkey: string): bool = - ## Same as ``rename`` but doesn't continue if `newkey` exists. - ## Returns `true` if key was renamed. - r.sendCommand("RENAMENX", key, newkey) - return r.readInteger() == 1 - -proc ttl*(r: Redis, key: string): RedisInteger = - ## Get the time to live for a key - r.sendCommand("TTL", key) - return r.readInteger() - -proc keyType*(r: Redis, key: string): RedisStatus = - ## Determine the type stored at key - r.sendCommand("TYPE", key) - return r.readStatus() - - -# Strings - -proc append*(r: Redis, key, value: string): RedisInteger = - ## Append a value to a key - r.sendCommand("APPEND", key, value) - return r.readInteger() - -proc decr*(r: Redis, key: string): RedisInteger = - ## Decrement the integer value of a key by one - r.sendCommand("DECR", key) - return r.readInteger() - -proc decrBy*(r: Redis, key: string, decrement: int): RedisInteger = - ## Decrement the integer value of a key by the given number - r.sendCommand("DECRBY", key, $decrement) - return r.readInteger() - -proc get*(r: Redis, key: string): RedisString = - ## Get the value of a key. Returns `redisNil` when `key` doesn't exist. - r.sendCommand("GET", key) - return r.readBulkString() - -proc getBit*(r: Redis, key: string, offset: int): RedisInteger = - ## Returns the bit value at offset in the string value stored at key - r.sendCommand("GETBIT", key, $offset) - return r.readInteger() - -proc getRange*(r: Redis, key: string, start, stop: int): RedisString = - ## Get a substring of the string stored at a key - r.sendCommand("GETRANGE", key, $start, $stop) - return r.readBulkString() - -proc getSet*(r: Redis, key: string, value: string): RedisString = - ## Set the string value of a key and return its old value. Returns `redisNil` - ## when key doesn't exist. - r.sendCommand("GETSET", key, value) - return r.readBulkString() - -proc incr*(r: Redis, key: string): RedisInteger = - ## Increment the integer value of a key by one. - r.sendCommand("INCR", key) - return r.readInteger() - -proc incrBy*(r: Redis, key: string, increment: int): RedisInteger = - ## Increment the integer value of a key by the given number - r.sendCommand("INCRBY", key, $increment) - return r.readInteger() - -proc setk*(r: Redis, key, value: string) = - ## Set the string value of a key. - ## - ## NOTE: This function had to be renamed due to a clash with the `set` type. - r.sendCommand("SET", key, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc setNX*(r: Redis, key, value: string): bool = - ## Set the value of a key, only if the key does not exist. Returns `true` - ## if the key was set. - r.sendCommand("SETNX", key, value) - return r.readInteger() == 1 - -proc setBit*(r: Redis, key: string, offset: int, - value: string): RedisInteger = - ## Sets or clears the bit at offset in the string value stored at key - r.sendCommand("SETBIT", key, $offset, value) - return r.readInteger() - -proc setEx*(r: Redis, key: string, seconds: int, value: string): RedisStatus = - ## Set the value and expiration of a key - r.sendCommand("SETEX", key, $seconds, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc setRange*(r: Redis, key: string, offset: int, - value: string): RedisInteger = - ## Overwrite part of a string at key starting at the specified offset - r.sendCommand("SETRANGE", key, $offset, value) - return r.readInteger() - -proc strlen*(r: Redis, key: string): RedisInteger = - ## Get the length of the value stored in a key. Returns 0 when key doesn't - ## exist. - r.sendCommand("STRLEN", key) - return r.readInteger() - -# Hashes -proc hDel*(r: Redis, key, field: string): bool = - ## Delete a hash field at `key`. Returns `true` if the field was removed. - r.sendCommand("HDEL", key, field) - return r.readInteger() == 1 - -proc hExists*(r: Redis, key, field: string): bool = - ## Determine if a hash field exists. - r.sendCommand("HEXISTS", key, field) - return r.readInteger() == 1 - -proc hGet*(r: Redis, key, field: string): RedisString = - ## Get the value of a hash field - r.sendCommand("HGET", key, field) - return r.readBulkString() - -proc hGetAll*(r: Redis, key: string): RedisList = - ## Get all the fields and values in a hash - r.sendCommand("HGETALL", key) - return r.readArray() - -proc hIncrBy*(r: Redis, key, field: string, incr: int): RedisInteger = - ## Increment the integer value of a hash field by the given number - r.sendCommand("HINCRBY", key, field, $incr) - return r.readInteger() - -proc hKeys*(r: Redis, key: string): RedisList = - ## Get all the fields in a hash - r.sendCommand("HKEYS", key) - return r.readArray() - -proc hLen*(r: Redis, key: string): RedisInteger = - ## Get the number of fields in a hash - r.sendCommand("HLEN", key) - return r.readInteger() - -proc hMGet*(r: Redis, key: string, fields: varargs[string]): RedisList = - ## Get the values of all the given hash fields - r.sendCommand("HMGET", key, fields) - return r.readArray() - -proc hMSet*(r: Redis, key: string, - fieldValues: openArray[tuple[field, value: string]]) = - ## Set multiple hash fields to multiple values - var args = @[key] - for field, value in items(fieldValues): - args.add(field) - args.add(value) - r.sendCommand("HMSET", args) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc hSet*(r: Redis, key, field, value: string): RedisInteger = - ## Set the string value of a hash field - r.sendCommand("HSET", key, field, value) - return r.readInteger() - -proc hSetNX*(r: Redis, key, field, value: string): RedisInteger = - ## Set the value of a hash field, only if the field does **not** exist - r.sendCommand("HSETNX", key, field, value) - return r.readInteger() - -proc hVals*(r: Redis, key: string): RedisList = - ## Get all the values in a hash - r.sendCommand("HVALS", key) - return r.readArray() - -# Lists - -proc bLPop*(r: Redis, keys: varargs[string], timeout: int): RedisList = - ## Remove and get the *first* element in a list, or block until - ## one is available - var args: seq[string] = @[] - for i in items(keys): args.add(i) - args.add($timeout) - r.sendCommand("BLPOP", args) - return r.readArray() - -proc bRPop*(r: Redis, keys: varargs[string], timeout: int): RedisList = - ## Remove and get the *last* element in a list, or block until one - ## is available. - var args: seq[string] = @[] - for i in items(keys): args.add(i) - args.add($timeout) - r.sendCommand("BRPOP", args) - return r.readArray() - -proc bRPopLPush*(r: Redis, source, destination: string, - timeout: int): RedisString = - ## Pop a value from a list, push it to another list and return it; or - ## block until one is available. - ## - ## http://redis.io/commands/brpoplpush - r.sendCommand("BRPOPLPUSH", source, destination, $timeout) - return r.readBulkString(true) # Multi-Bulk nil allowed. - -proc lIndex*(r: Redis, key: string, index: int): RedisString = - ## Get an element from a list by its index - r.sendCommand("LINDEX", key, $index) - return r.readBulkString() - -proc lInsert*(r: Redis, key: string, before: bool, pivot, value: string): - RedisInteger = - ## Insert an element before or after another element in a list - var pos = if before: "BEFORE" else: "AFTER" - r.sendCommand("LINSERT", key, pos, pivot, value) - return r.readInteger() - -proc lLen*(r: Redis, key: string): RedisInteger = - ## Get the length of a list - r.sendCommand("LLEN", key) - return r.readInteger() - -proc lPop*(r: Redis, key: string): RedisString = - ## Remove and get the first element in a list - r.sendCommand("LPOP", key) - return r.readBulkString() - -proc lPush*(r: Redis, key, value: string, create: bool = true): RedisInteger = - ## Prepend a value to a list. Returns the length of the list after the push. - ## The ``create`` param specifies whether a list should be created if it - ## doesn't exist at ``key``. More specifically if ``create`` is true, `LPUSH` - ## will be used, otherwise `LPUSHX`. - if create: - r.sendCommand("LPUSH", key, value) - else: - r.sendCommand("LPUSHX", key, value) - return r.readInteger() - -proc lRange*(r: Redis, key: string, start, stop: int): RedisList = - ## Get a range of elements from a list. Returns `nil` when `key` - ## doesn't exist. - r.sendCommand("LRANGE", key, $start, $stop) - return r.readArray() - -proc lRem*(r: Redis, key: string, value: string, count: int = 0): RedisInteger = - ## Remove elements from a list. Returns the number of elements that have been - ## removed. - r.sendCommand("LREM", key, $count, value) - return r.readInteger() - -proc lSet*(r: Redis, key: string, index: int, value: string) = - ## Set the value of an element in a list by its index - r.sendCommand("LSET", key, $index, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc lTrim*(r: Redis, key: string, start, stop: int) = - ## Trim a list to the specified range - r.sendCommand("LTRIM", key, $start, $stop) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc rPop*(r: Redis, key: string): RedisString = - ## Remove and get the last element in a list - r.sendCommand("RPOP", key) - return r.readBulkString() - -proc rPopLPush*(r: Redis, source, destination: string): RedisString = - ## Remove the last element in a list, append it to another list and return it - r.sendCommand("RPOPLPUSH", source, destination) - return r.readBulkString() - -proc rPush*(r: Redis, key, value: string, create: bool = true): RedisInteger = - ## Append a value to a list. Returns the length of the list after the push. - ## The ``create`` param specifies whether a list should be created if it - ## doesn't exist at ``key``. More specifically if ``create`` is true, `RPUSH` - ## will be used, otherwise `RPUSHX`. - if create: - r.sendCommand("RPUSH", key, value) - else: - r.sendCommand("RPUSHX", key, value) - return r.readInteger() - -# Sets - -proc sadd*(r: Redis, key: string, member: string): RedisInteger = - ## Add a member to a set - r.sendCommand("SADD", key, member) - return r.readInteger() - -proc scard*(r: Redis, key: string): RedisInteger = - ## Get the number of members in a set - r.sendCommand("SCARD", key) - return r.readInteger() - -proc sdiff*(r: Redis, keys: varargs[string]): RedisList = - ## Subtract multiple sets - r.sendCommand("SDIFF", keys) - return r.readArray() - -proc sdiffstore*(r: Redis, destination: string, - keys: varargs[string]): RedisInteger = - ## Subtract multiple sets and store the resulting set in a key - r.sendCommand("SDIFFSTORE", destination, keys) - return r.readInteger() - -proc sinter*(r: Redis, keys: varargs[string]): RedisList = - ## Intersect multiple sets - r.sendCommand("SINTER", keys) - return r.readArray() - -proc sinterstore*(r: Redis, destination: string, - keys: varargs[string]): RedisInteger = - ## Intersect multiple sets and store the resulting set in a key - r.sendCommand("SINTERSTORE", destination, keys) - return r.readInteger() - -proc sismember*(r: Redis, key: string, member: string): RedisInteger = - ## Determine if a given value is a member of a set - r.sendCommand("SISMEMBER", key, member) - return r.readInteger() - -proc smembers*(r: Redis, key: string): RedisList = - ## Get all the members in a set - r.sendCommand("SMEMBERS", key) - return r.readArray() - -proc smove*(r: Redis, source: string, destination: string, - member: string): RedisInteger = - ## Move a member from one set to another - r.sendCommand("SMOVE", source, destination, member) - return r.readInteger() - -proc spop*(r: Redis, key: string): RedisString = - ## Remove and return a random member from a set - r.sendCommand("SPOP", key) - return r.readBulkString() - -proc srandmember*(r: Redis, key: string): RedisString = - ## Get a random member from a set - r.sendCommand("SRANDMEMBER", key) - return r.readBulkString() - -proc srem*(r: Redis, key: string, member: string): RedisInteger = - ## Remove a member from a set - r.sendCommand("SREM", key, member) - return r.readInteger() - -proc sunion*(r: Redis, keys: varargs[string]): RedisList = - ## Add multiple sets - r.sendCommand("SUNION", keys) - return r.readArray() - -proc sunionstore*(r: Redis, destination: string, - key: varargs[string]): RedisInteger = - ## Add multiple sets and store the resulting set in a key - r.sendCommand("SUNIONSTORE", destination, key) - return r.readInteger() - -# Sorted sets - -proc zadd*(r: Redis, key: string, score: int, member: string): RedisInteger = - ## Add a member to a sorted set, or update its score if it already exists - r.sendCommand("ZADD", key, $score, member) - return r.readInteger() - -proc zcard*(r: Redis, key: string): RedisInteger = - ## Get the number of members in a sorted set - r.sendCommand("ZCARD", key) - return r.readInteger() - -proc zcount*(r: Redis, key: string, min: string, max: string): RedisInteger = - ## Count the members in a sorted set with scores within the given values - r.sendCommand("ZCOUNT", key, min, max) - return r.readInteger() - -proc zincrby*(r: Redis, key: string, increment: string, - member: string): RedisString = - ## Increment the score of a member in a sorted set - r.sendCommand("ZINCRBY", key, increment, member) - return r.readBulkString() - -proc zinterstore*(r: Redis, destination: string, numkeys: string, - keys: openArray[string], weights: openArray[string] = [], - aggregate: string = ""): RedisInteger = - ## Intersect multiple sorted sets and store the resulting sorted set in - ## a new key - var args = @[destination, numkeys] - for i in items(keys): args.add(i) - - if weights.len != 0: - args.add("WITHSCORE") - for i in items(weights): args.add(i) - if aggregate.len != 0: - args.add("AGGREGATE") - args.add(aggregate) - - r.sendCommand("ZINTERSTORE", args) - - return r.readInteger() - -proc zrange*(r: Redis, key: string, start: string, stop: string, - withScores: bool): RedisList = - ## Return a range of members in a sorted set, by index - if not withScores: - r.sendCommand("ZRANGE", key, start, stop) - else: - r.sendCommand("ZRANGE", "WITHSCORES", key, start, stop) - return r.readArray() - -proc zrangebyscore*(r: Redis, key: string, min: string, max: string, - withScore: bool = false, limit: bool = false, - limitOffset: int = 0, limitCount: int = 0): RedisList = - ## Return a range of members in a sorted set, by score - var args = @[key, min, max] - - if withScore: args.add("WITHSCORE") - if limit: - args.add("LIMIT") - args.add($limitOffset) - args.add($limitCount) - - r.sendCommand("ZRANGEBYSCORE", args) - return r.readArray() - -proc zrank*(r: Redis, key: string, member: string): RedisString = - ## Determine the index of a member in a sorted set - r.sendCommand("ZRANK", key, member) - return r.readBulkString() - -proc zrem*(r: Redis, key: string, member: string): RedisInteger = - ## Remove a member from a sorted set - r.sendCommand("ZREM", key, member) - return r.readInteger() - -proc zremrangebyrank*(r: Redis, key: string, start: string, - stop: string): RedisInteger = - ## Remove all members in a sorted set within the given indexes - r.sendCommand("ZREMRANGEBYRANK", key, start, stop) - return r.readInteger() - -proc zremrangebyscore*(r: Redis, key: string, min: string, - max: string): RedisInteger = - ## Remove all members in a sorted set within the given scores - r.sendCommand("ZREMRANGEBYSCORE", key, min, max) - return r.readInteger() - -proc zrevrange*(r: Redis, key: string, start: string, stop: string, - withScore: bool): RedisList = - ## Return a range of members in a sorted set, by index, - ## with scores ordered from high to low - if withScore: - r.sendCommand("ZREVRANGE", "WITHSCORE", key, start, stop) - else: r.sendCommand("ZREVRANGE", key, start, stop) - return r.readArray() - -proc zrevrangebyscore*(r: Redis, key: string, min: string, max: string, - withScore: bool = false, limit: bool = false, - limitOffset: int = 0, limitCount: int = 0): RedisList = - ## Return a range of members in a sorted set, by score, with - ## scores ordered from high to low - var args = @[key, min, max] - - if withScore: args.add("WITHSCORE") - if limit: - args.add("LIMIT") - args.add($limitOffset) - args.add($limitCount) - - r.sendCommand("ZREVRANGEBYSCORE", args) - return r.readArray() - -proc zrevrank*(r: Redis, key: string, member: string): RedisString = - ## Determine the index of a member in a sorted set, with - ## scores ordered from high to low - r.sendCommand("ZREVRANK", key, member) - return r.readBulkString() - -proc zscore*(r: Redis, key: string, member: string): RedisString = - ## Get the score associated with the given member in a sorted set - r.sendCommand("ZSCORE", key, member) - return r.readBulkString() - -proc zunionstore*(r: Redis, destination: string, numkeys: string, - keys: openArray[string], weights: openArray[string] = [], - aggregate: string = ""): RedisInteger = - ## Add multiple sorted sets and store the resulting sorted set in a new key - var args = @[destination, numkeys] - for i in items(keys): args.add(i) - - if weights.len != 0: - args.add("WEIGHTS") - for i in items(weights): args.add(i) - if aggregate.len != 0: - args.add("AGGREGATE") - args.add(aggregate) - - r.sendCommand("ZUNIONSTORE", args) - - return r.readInteger() - -# HyperLogLog - -proc pfadd*(r: Redis, key: string, elements: varargs[string]): RedisInteger = - ## Add variable number of elements into special 'HyperLogLog' set type - r.sendCommand("PFADD", key, elements) - return r.readInteger() - -proc pfcount*(r: Redis, key: string): RedisInteger = - ## Count approximate number of elements in 'HyperLogLog' - r.sendCommand("PFCOUNT", key) - return r.readInteger() - -proc pfmerge*(r: Redis, destination: string, sources: varargs[string]) = - ## Merge several source HyperLogLog's into one specified by destKey - r.sendCommand("PFMERGE", destination, sources) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -# Pub/Sub - -# TODO: pub/sub -- I don't think this will work synchronously. -discard """ -proc psubscribe*(r: Redis, pattern: openarray[string]): ???? = - ## Listen for messages published to channels matching the given patterns - r.socket.send("PSUBSCRIBE $#\c\L" % pattern) - return ??? - -proc publish*(r: Redis, channel: string, message: string): RedisInteger = - ## Post a message to a channel - r.socket.send("PUBLISH $# $#\c\L" % [channel, message]) - return r.readInteger() - -proc punsubscribe*(r: Redis, [pattern: openarray[string], : string): ???? = - ## Stop listening for messages posted to channels matching the given patterns - r.socket.send("PUNSUBSCRIBE $# $#\c\L" % [[pattern.join(), ]) - return ??? - -proc subscribe*(r: Redis, channel: openarray[string]): ???? = - ## Listen for messages published to the given channels - r.socket.send("SUBSCRIBE $#\c\L" % channel.join) - return ??? - -proc unsubscribe*(r: Redis, [channel: openarray[string], : string): ???? = - ## Stop listening for messages posted to the given channels - r.socket.send("UNSUBSCRIBE $# $#\c\L" % [[channel.join(), ]) - return ??? - -""" - -# Transactions - -proc discardMulti*(r: Redis) = - ## Discard all commands issued after MULTI - r.sendCommand("DISCARD") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc exec*(r: Redis): RedisList = - ## Execute all commands issued after MULTI - r.sendCommand("EXEC") - r.pipeline.enabled = false - # Will reply with +OK for MULTI/EXEC and +QUEUED for every command - # between, then with the results - return r.flushPipeline(true) - - -proc multi*(r: Redis) = - ## Mark the start of a transaction block - r.startPipelining() - r.sendCommand("MULTI") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc unwatch*(r: Redis) = - ## Forget about all watched keys - r.sendCommand("UNWATCH") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc watch*(r: Redis, key: varargs[string]) = - ## Watch the given keys to determine execution of the MULTI/EXEC block - r.sendCommand("WATCH", key) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -# Connection - -proc auth*(r: Redis, password: string) = - ## Authenticate to the server - r.sendCommand("AUTH", password) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc echoServ*(r: Redis, message: string): RedisString = - ## Echo the given string - r.sendCommand("ECHO", message) - return r.readBulkString() - -proc ping*(r: Redis): RedisStatus = - ## Ping the server - r.sendCommand("PING") - return r.readStatus() - -proc quit*(r: Redis) = - ## Close the connection - r.sendCommand("QUIT") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - r.socket.close() - -proc select*(r: Redis, index: int): RedisStatus = - ## Change the selected database for the current connection - r.sendCommand("SELECT", $index) - return r.readStatus() - -# Server - -proc bgrewriteaof*(r: Redis) = - ## Asynchronously rewrite the append-only file - r.sendCommand("BGREWRITEAOF") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc bgsave*(r: Redis) = - ## Asynchronously save the dataset to disk - r.sendCommand("BGSAVE") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc configGet*(r: Redis, parameter: string): RedisList = - ## Get the value of a configuration parameter - r.sendCommand("CONFIG", "GET", parameter) - return r.readArray() - -proc configSet*(r: Redis, parameter: string, value: string) = - ## Set a configuration parameter to the given value - r.sendCommand("CONFIG", "SET", parameter, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc configResetStat*(r: Redis) = - ## Reset the stats returned by INFO - r.sendCommand("CONFIG", "RESETSTAT") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc dbsize*(r: Redis): RedisInteger = - ## Return the number of keys in the selected database - r.sendCommand("DBSIZE") - return r.readInteger() - -proc debugObject*(r: Redis, key: string): RedisStatus = - ## Get debugging information about a key - r.sendCommand("DEBUG", "OBJECT", key) - return r.readStatus() - -proc debugSegfault*(r: Redis) = - ## Make the server crash - r.sendCommand("DEBUG", "SEGFAULT") - -proc flushall*(r: Redis): RedisStatus = - ## Remove all keys from all databases - r.sendCommand("FLUSHALL") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc flushdb*(r: Redis): RedisStatus = - ## Remove all keys from the current database - r.sendCommand("FLUSHDB") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc info*(r: Redis): RedisString = - ## Get information and statistics about the server - r.sendCommand("INFO") - return r.readBulkString() - -proc lastsave*(r: Redis): RedisInteger = - ## Get the UNIX time stamp of the last successful save to disk - r.sendCommand("LASTSAVE") - return r.readInteger() - -discard """ -proc monitor*(r: Redis) = - ## Listen for all requests received by the server in real time - r.socket.send("MONITOR\c\L") - raiseNoOK(r.readStatus(), r.pipeline.enabled) -""" - -proc save*(r: Redis) = - ## Synchronously save the dataset to disk - r.sendCommand("SAVE") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc shutdown*(r: Redis) = - ## Synchronously save the dataset to disk and then shut down the server - r.sendCommand("SHUTDOWN") - var s = "".TaintedString - r.socket.readLine(s) - if s.string.len != 0: raise newException(RedisError, s.string) - -proc slaveof*(r: Redis, host: string, port: string) = - ## Make the server a slave of another instance, or promote it as master - r.sendCommand("SLAVEOF", host, port) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -iterator hPairs*(r: Redis, key: string): tuple[key, value: string] = - ## Iterator for keys and values in a hash. - var - contents = r.hGetAll(key) - k = "" - for i in items(contents): - if k == "": - k = i - else: - yield (k, i) - k = "" - -proc someTests(r: Redis, how: SendMode):seq[string] = - var list:seq[string] = @[] - - if how == pipelined: - r.startPipelining() - elif how == multiple: - r.multi() - - r.setk("nim:test", "Testing something.") - r.setk("nim:utf8", "こんにちは") - r.setk("nim:esc", "\\ths ągt\\") - r.setk("nim:int", "1") - list.add(r.get("nim:esc")) - list.add($(r.incr("nim:int"))) - list.add(r.get("nim:int")) - list.add(r.get("nim:utf8")) - list.add($(r.hSet("test1", "name", "A Test"))) - var res = r.hGetAll("test1") - for r in res: - list.add(r) - list.add(r.get("invalid_key")) - list.add($(r.lPush("mylist","itema"))) - list.add($(r.lPush("mylist","itemb"))) - r.lTrim("mylist",0,1) - var p = r.lRange("mylist", 0, -1) - - for i in items(p): - if not isNil(i): - list.add(i) - - list.add(r.debugObject("mylist")) - - r.configSet("timeout", "299") - var g = r.configGet("timeout") - for i in items(g): - list.add(i) - - list.add(r.echoServ("BLAH")) - - case how - of normal: - return list - of pipelined: - return r.flushPipeline() - of multiple: - return r.exec() - -proc assertListsIdentical(listA, listB: seq[string]) = - assert(listA.len == listB.len) - var i = 0 - for item in listA: - assert(item == listB[i]) - i = i + 1 - -when not defined(testing) and isMainModule: - when false: - var r = open() - - # Test with no pipelining - var listNormal = r.someTests(normal) - - # Test with pipelining enabled - var listPipelined = r.someTests(pipelined) - assertListsIdentical(listNormal, listPipelined) - - # Test with multi/exec() (automatic pipelining) - var listMulti = r.someTests(multiple) - assertListsIdentical(listNormal, listMulti) diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim new file mode 100644 index 000000000..ffa0128dc --- /dev/null +++ b/lib/pure/reservedmem.nim @@ -0,0 +1,229 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## :Authors: Zahary Karadjov +## +## This module provides utilities for reserving portions of the +## address space of a program without consuming physical memory. +## It can be used to implement a dynamically resizable buffer that +## is guaranteed to remain in the same memory location. The buffer +## will be able to grow up to the size of the initially reserved +## portion of the address space. +## +## Unstable API. + +from std/oserrors import raiseOSError, osLastError + +when defined(nimPreviewSlimSystem): + import std/assertions + +template distance*(lhs, rhs: pointer): int = + cast[int](rhs) - cast[int](lhs) + +template shift*(p: pointer, distance: int): pointer = + cast[pointer](cast[int](p) + distance) + +type + MemAccessFlags* = int + + ReservedMem* = object + memStart: pointer + usedMemEnd: pointer + committedMemEnd: pointer + memEnd: pointer + maxCommittedAndUnusedPages: int + accessFlags: MemAccessFlags + + ReservedMemSeq*[T] = object + mem: ReservedMem + +when defined(windows): + import std/winlean + import std/private/win_getsysteminfo + + proc getAllocationGranularity: uint = + var sysInfo: SystemInfo + getSystemInfo(addr sysInfo) + return uint(sysInfo.dwAllocationGranularity) + + let allocationGranularity = getAllocationGranularity().int + + const + memNoAccess = MemAccessFlags(PAGE_NOACCESS) + memExec* = MemAccessFlags(PAGE_EXECUTE) + memExecRead* = MemAccessFlags(PAGE_EXECUTE_READ) + memExecReadWrite* = MemAccessFlags(PAGE_EXECUTE_READWRITE) + memRead* = MemAccessFlags(PAGE_READONLY) + memReadWrite* = MemAccessFlags(PAGE_READWRITE) + + template check(expr) = + let r = expr + if r == cast[typeof(r)](0): + raiseOSError(osLastError()) + +else: + import std/posix + + let allocationGranularity = sysconf(SC_PAGESIZE) + + let + memNoAccess = MemAccessFlags(PROT_NONE) + memExec* = MemAccessFlags(PROT_EXEC) + memExecRead* = MemAccessFlags(PROT_EXEC or PROT_READ) + memExecReadWrite* = MemAccessFlags(PROT_EXEC or PROT_READ or PROT_WRITE) + memRead* = MemAccessFlags(PROT_READ) + memReadWrite* = MemAccessFlags(PROT_READ or PROT_WRITE) + + template check(expr) = + if not expr: + raiseOSError(osLastError()) + +func nextAlignedOffset(n, alignment: int): int = + result = n + let m = n mod alignment + if m != 0: result += alignment - m + + +when defined(windows): + const + MEM_DECOMMIT = 0x4000 + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32): cint {.header: "<windows.h>", stdcall, + importc: "VirtualFree".} + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "<windows.h>", stdcall, importc: "VirtualAlloc".} + +proc init*(T: type ReservedMem, + maxLen: Natural, + initLen: Natural = 0, + initCommitLen = initLen, + memStart = pointer(nil), + accessFlags = memReadWrite, + maxCommittedAndUnusedPages = 3): ReservedMem = + + assert initLen <= initCommitLen + let commitSize = nextAlignedOffset(initCommitLen, allocationGranularity) + + when defined(windows): + result.memStart = virtualAlloc(memStart, maxLen, MEM_RESERVE, + accessFlags.cint) + check result.memStart + if commitSize > 0: + check virtualAlloc(result.memStart, commitSize, MEM_COMMIT, + accessFlags.cint) + else: + var allocFlags = MAP_PRIVATE or MAP_ANONYMOUS # or MAP_NORESERVE + # if memStart != nil: + # allocFlags = allocFlags or MAP_FIXED_NOREPLACE + result.memStart = mmap(memStart, maxLen, PROT_NONE, allocFlags, -1, 0) + check result.memStart != MAP_FAILED + if commitSize > 0: + check mprotect(result.memStart, commitSize, cint(accessFlags)) == 0 + + result.usedMemEnd = result.memStart.shift(initLen) + result.committedMemEnd = result.memStart.shift(commitSize) + result.memEnd = result.memStart.shift(maxLen) + result.accessFlags = accessFlags + result.maxCommittedAndUnusedPages = maxCommittedAndUnusedPages + +func len*(m: ReservedMem): int = + distance(m.memStart, m.usedMemEnd) + +func commitedLen*(m: ReservedMem): int = + distance(m.memStart, m.committedMemEnd) + +func maxLen*(m: ReservedMem): int = + distance(m.memStart, m.memEnd) + +proc setLen*(m: var ReservedMem, newLen: int) = + let len = m.len + m.usedMemEnd = m.memStart.shift(newLen) + if newLen > len: + let d = distance(m.committedMemEnd, m.usedMemEnd) + if d > 0: + let commitExtensionSize = nextAlignedOffset(d, allocationGranularity) + when defined(windows): + check virtualAlloc(m.committedMemEnd, commitExtensionSize, + MEM_COMMIT, m.accessFlags.cint) + else: + check mprotect(m.committedMemEnd, commitExtensionSize, + m.accessFlags.cint) == 0 + else: + let d = distance(m.usedMemEnd, m.committedMemEnd) - + m.maxCommittedAndUnusedPages * allocationGranularity + if d > 0: + let commitSizeShrinkage = nextAlignedOffset(d, allocationGranularity) + let newCommitEnd = m.committedMemEnd.shift(-commitSizeShrinkage) + + when defined(windows): + check virtualFree(newCommitEnd, commitSizeShrinkage, MEM_DECOMMIT) + else: + check posix_madvise(newCommitEnd, commitSizeShrinkage, + POSIX_MADV_DONTNEED) == 0 + + m.committedMemEnd = newCommitEnd + +proc init*(SeqType: type ReservedMemSeq, + maxLen: Natural, + initLen: Natural = 0, + initCommitLen: Natural = 0, + memStart = pointer(nil), + accessFlags = memReadWrite, + maxCommittedAndUnusedPages = 3): SeqType = + + let elemSize = sizeof(SeqType.T) + result.mem = ReservedMem.init(maxLen * elemSize, + initLen * elemSize, + initCommitLen * elemSize, + memStart, accessFlags, + maxCommittedAndUnusedPages) + +func `[]`*[T](s: ReservedMemSeq[T], pos: Natural): lent T = + let elemAddr = s.mem.memStart.shift(pos * sizeof(T)) + rangeCheck elemAddr < s.mem.usedMemEnd + result = (cast[ptr T](elemAddr))[] + +func `[]`*[T](s: var ReservedMemSeq[T], pos: Natural): var T = + let elemAddr = s.mem.memStart.shift(pos * sizeof(T)) + rangeCheck elemAddr < s.mem.usedMemEnd + result = (cast[ptr T](elemAddr))[] + +func `[]`*[T](s: ReservedMemSeq[T], rpos: BackwardsIndex): lent T = + return s[int(s.len) - int(rpos)] + +func `[]`*[T](s: var ReservedMemSeq[T], rpos: BackwardsIndex): var T = + return s[int(s.len) - int(rpos)] + +func len*[T](s: ReservedMemSeq[T]): int = + s.mem.len div sizeof(T) + +proc setLen*[T](s: var ReservedMemSeq[T], newLen: int) = + # TODO call destructors + s.mem.setLen(newLen * sizeof(T)) + +proc add*[T](s: var ReservedMemSeq[T], val: T) = + let len = s.len + s.setLen(len + 1) + s[len] = val + +proc pop*[T](s: var ReservedMemSeq[T]): T = + assert s.usedMemEnd != s.memStart + let lastIdx = s.len - 1 + result = s[lastIdx] + s.setLen(lastIdx) + +func commitedLen*[T](s: ReservedMemSeq[T]): int = + s.mem.commitedLen div sizeof(T) + +func maxLen*[T](s: ReservedMemSeq[T]): int = + s.mem.maxLen div sizeof(T) + diff --git a/lib/pure/romans.nim b/lib/pure/romans.nim deleted file mode 100644 index 18c04ef58..000000000 --- a/lib/pure/romans.nim +++ /dev/null @@ -1,56 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2011 Philippe Lhoste -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Module for converting an integer to a Roman numeral. -## See http://en.wikipedia.org/wiki/Roman_numerals for reference. - -const - RomanNumeralDigits* = {'I', 'i', 'V', 'v', 'X', 'x', 'L', 'l', 'C', 'c', - 'D', 'd', 'M', 'm'} ## set of all characters a Roman numeral may consist of - -proc romanToDecimal*(romanVal: string): int = - ## Converts a Roman numeral to its int representation. - result = 0 - var prevVal = 0 - for i in countdown(romanVal.len - 1, 0): - var val = 0 - case romanVal[i] - of 'I', 'i': val = 1 - of 'V', 'v': val = 5 - of 'X', 'x': val = 10 - of 'L', 'l': val = 50 - of 'C', 'c': val = 100 - of 'D', 'd': val = 500 - of 'M', 'm': val = 1000 - else: - raise newException(EInvalidValue, "invalid roman numeral: " & $romanVal) - if val >= prevVal: - inc(result, val) - else: - dec(result, val) - prevVal = val - -proc decimalToRoman*(number: range[1..3_999]): string = - ## Converts a number to a Roman numeral. - const romanComposites = [ - ("M", 1000), ("CM", 900), - ("D", 500), ("CD", 400), ("C", 100), - ("XC", 90), ("L", 50), ("XL", 40), ("X", 10), ("IX", 9), - ("V", 5), ("IV", 4), ("I", 1)] - result = "" - var decVal: int = number - for key, val in items(romanComposites): - while decVal >= val: - dec(decVal, val) - result.add(key) - -when isMainModule: - for i in 1 .. 3_999: - assert i == i.decimalToRoman.romanToDecimal - diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index df7071642..8750aca87 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -8,19 +8,21 @@ # ## This module contains support for a `rope`:idx: data type. -## Ropes can represent very long strings efficiently; especially concatenation +## Ropes can represent very long strings efficiently; in particular, concatenation ## is done in O(1) instead of O(n). They are essentially concatenation ## trees that are only flattened when converting to a native Nim -## string. The empty string is represented by ``nil``. Ropes are immutable and +## string. The empty string is represented by `nil`. Ropes are immutable and ## subtrees can be shared without copying. ## Leaves can be cached for better memory efficiency at the cost of ## runtime efficiency. -include "system/inclrtl" +include system/inclrtl +import std/streams -{.deadCodeElim: on.} +when defined(nimPreviewSlimSystem): + import std/[syncio, formatfloat, assertions] -{.push debugger:off .} # the user does not want to trace a part +{.push debugger: off.} # the user does not want to trace a part # of the standard library! const @@ -30,15 +32,11 @@ var cacheEnabled = false type - Rope* = ref RopeObj ## empty rope is represented by nil - RopeObj {.acyclic.} = object + Rope* {.acyclic.} = ref object + ## A rope data type. The empty rope is represented by `nil`. left, right: Rope length: int - data: string # != nil if a leaf - -{.deprecated: [PRope: Rope].} - -proc isConc(r: Rope): bool {.inline.} = return isNil(r.data) + data: string # not empty if a leaf # Note that the left and right pointers are not needed for leafs. # Leaves have relatively high memory overhead (~30 bytes on a 32 @@ -49,9 +47,8 @@ proc isConc(r: Rope): bool {.inline.} = return isNil(r.data) # pointers. proc len*(a: Rope): int {.rtl, extern: "nro$1".} = - ## the rope's length - if a == nil: result = 0 - else: result = a.length + ## The rope's length. + if a == nil: 0 else: a.length proc newRope(): Rope = new(result) proc newRope(data: string): Rope = @@ -60,8 +57,8 @@ proc newRope(data: string): Rope = result.data = data var - cache {.threadvar.}: Rope # the root of the cache tree - N {.threadvar.}: Rope # dummy rope needed for splay algorithm + cache {.threadvar.}: Rope # the root of the cache tree + N {.threadvar.}: Rope # dummy rope needed for splay algorithm when countCacheMisses: var misses, hits: int @@ -70,7 +67,7 @@ proc splay(s: string, tree: Rope, cmpres: var int): Rope = var c: int var t = tree N.left = nil - N.right = nil # reset to nil + N.right = nil # reset to nil var le = N var r = N while true: @@ -130,22 +127,39 @@ proc insertInCache(s: string, tree: Rope): Rope = result.left = t t.right = nil -proc rope*(s: string): Rope {.rtl, extern: "nro$1Str".} = +proc rope*(s: string = ""): Rope {.rtl, extern: "nro$1Str".} = ## Converts a string to a rope. + runnableExamples: + let r = rope("I'm a rope") + doAssert $r == "I'm a rope" + if s.len == 0: result = nil - elif cacheEnabled: - result = insertInCache(s, cache) - cache = result else: - result = newRope(s) + when nimvm: + # No caching in VM context + result = newRope(s) + else: + if cacheEnabled: + result = insertInCache(s, cache) + cache = result + else: + result = newRope(s) proc rope*(i: BiggestInt): Rope {.rtl, extern: "nro$1BiggestInt".} = ## Converts an int to a rope. + runnableExamples: + let r = rope(429) + doAssert $r == "429" + result = rope($i) proc rope*(f: BiggestFloat): Rope {.rtl, extern: "nro$1BiggestFloat".} = ## Converts a float to a rope. + runnableExamples: + let r = rope(4.29) + doAssert $r == "4.29" + result = rope($f) proc enableCache*() {.rtl, extern: "nro$1".} = @@ -154,12 +168,16 @@ proc enableCache*() {.rtl, extern: "nro$1".} = cacheEnabled = true proc disableCache*() {.rtl, extern: "nro$1".} = - ## the cache is discarded and disabled. The GC will reuse its used memory. + ## The cache is discarded and disabled. The GC will reuse its used memory. cache = nil cacheEnabled = false proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = - ## the concatenation operator for ropes. + ## The concatenation operator for ropes. + runnableExamples: + let r = rope("Hello, ") & rope("Nim!") + doAssert $r == "Hello, Nim!" + if a == nil: result = b elif b == nil: @@ -167,134 +185,127 @@ proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = else: result = newRope() result.length = a.length + b.length - when false: - # XXX rebalancing would be nice, but is too expensive. - result.left = a.left - var x = newRope() - x.left = a.right - x.right = b - result.right = x - else: - result.left = a - result.right = b + result.left = a + result.right = b proc `&`*(a: Rope, b: string): Rope {.rtl, extern: "nroConcRopeStr".} = - ## the concatenation operator for ropes. + ## The concatenation operator for ropes. + runnableExamples: + let r = rope("Hello, ") & "Nim!" + doAssert $r == "Hello, Nim!" + result = a & rope(b) proc `&`*(a: string, b: Rope): Rope {.rtl, extern: "nroConcStrRope".} = - ## the concatenation operator for ropes. + ## The concatenation operator for ropes. + runnableExamples: + let r = "Hello, " & rope("Nim!") + doAssert $r == "Hello, Nim!" + result = rope(a) & b proc `&`*(a: openArray[Rope]): Rope {.rtl, extern: "nroConcOpenArray".} = - ## the concatenation operator for an openarray of ropes. - for i in countup(0, high(a)): result = result & a[i] + ## The concatenation operator for an `openArray` of ropes. + runnableExamples: + let r = &[rope("Hello, "), rope("Nim"), rope("!")] + doAssert $r == "Hello, Nim!" + + for item in a: result = result & item proc add*(a: var Rope, b: Rope) {.rtl, extern: "nro$1Rope".} = - ## adds `b` to the rope `a`. + ## Adds `b` to the rope `a`. + runnableExamples: + var r = rope("Hello, ") + r.add(rope("Nim!")) + doAssert $r == "Hello, Nim!" + a = a & b proc add*(a: var Rope, b: string) {.rtl, extern: "nro$1Str".} = - ## adds `b` to the rope `a`. + ## Adds `b` to the rope `a`. + runnableExamples: + var r = rope("Hello, ") + r.add("Nim!") + doAssert $r == "Hello, Nim!" + a = a & b proc `[]`*(r: Rope, i: int): char {.rtl, extern: "nroCharAt".} = - ## returns the character at position `i` in the rope `r`. This is quite - ## expensive! Worst-case: O(n). If ``i >= r.len``, ``\0`` is returned. + ## Returns the character at position `i` in the rope `r`. This is quite + ## expensive! Worst-case: O(n). If `i >= r.len or i < 0`, `\0` is returned. + runnableExamples: + let r = rope("Hello, Nim!") + + doAssert r[0] == 'H' + doAssert r[7] == 'N' + doAssert r[22] == '\0' + var x = r var j = i - if x == nil: return + if x == nil or i < 0 or i >= r.len: return while true: - if not isConc(x): - if x.data.len <% j: return x.data[j] - return '\0' + if x != nil and x.data.len > 0: + # leaf + return x.data[j] else: - if x.left.len >% j: + if x.left.length > j: x = x.left else: + dec(j, x.left.length) x = x.right - dec(j, x.len) iterator leaves*(r: Rope): string = - ## iterates over any leaf string in the rope `r`. + ## Iterates over any leaf string in the rope `r`. + runnableExamples: + let r = rope("Hello") & rope(", Nim!") + let s = ["Hello", ", Nim!"] + var index = 0 + for leave in r.leaves: + doAssert leave == s[index] + inc(index) + if r != nil: var stack = @[r] while stack.len > 0: var it = stack.pop - while isConc(it): + while it.left != nil: + assert(it.right != nil) stack.add(it.right) it = it.left assert(it != nil) - assert(it.data != nil) yield it.data iterator items*(r: Rope): char = - ## iterates over any character in the rope `r`. + ## Iterates over any character in the rope `r`. for s in leaves(r): for c in items(s): yield c proc write*(f: File, r: Rope) {.rtl, extern: "nro$1".} = - ## writes a rope to a file. + ## Writes a rope to a file. for s in leaves(r): write(f, s) -proc `$`*(r: Rope): string {.rtl, extern: "nroToString".}= - ## converts a rope back to a string. - result = newString(r.len) - setLen(result, 0) +proc write*(s: Stream, r: Rope) {.rtl, extern: "nroWriteStream".} = + ## Writes a rope to a stream. + for rs in leaves(r): write(s, rs) + +proc `$`*(r: Rope): string {.rtl, extern: "nroToString".} = + ## Converts a rope back to a string. + result = newStringOfCap(r.len) for s in leaves(r): add(result, s) -when false: - # Format string caching seems reasonable: All leaves can be shared and format - # string parsing has to be done only once. A compiled format string is stored - # as a rope. A negative length is used for the index into the args array. - proc compiledArg(idx: int): Rope = - new(result) - result.length = -idx - - proc compileFrmt(frmt: string): Rope = - var i = 0 - var length = len(frmt) - result = nil - var num = 0 - while i < length: - if frmt[i] == '$': - inc(i) - case frmt[i] - of '$': - add(result, "$") - inc(i) - of '#': - inc(i) - add(result, compiledArg(num+1)) - inc(num) - of '0'..'9': - var j = 0 - while true: - j = j * 10 + ord(frmt[i]) - ord('0') - inc(i) - if frmt[i] notin {'0'..'9'}: break - add(s, compiledArg(j)) - of '{': - inc(i) - var j = 0 - while frmt[i] in {'0'..'9'}: - j = j * 10 + ord(frmt[i]) - ord('0') - inc(i) - if frmt[i] == '}': inc(i) - else: raise newException(EInvalidValue, "invalid format string") - add(s, compiledArg(j)) - else: raise newException(EInvalidValue, "invalid format string") - var start = i - while i < length: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: - add(result, substr(frmt, start, i-1)) - -proc `%`*(frmt: string, args: openArray[Rope]): Rope {. - rtl, extern: "nroFormat".} = - ## `%` substitution operator for ropes. Does not support the ``$identifier`` - ## nor ``${identifier}`` notations. +proc `%`*(frmt: string, args: openArray[Rope]): Rope {.rtl, extern: "nroFormat".} = + ## `%` substitution operator for ropes. Does not support the `$identifier` + ## nor `${identifier}` notations. + runnableExamples: + let r1 = "$1 $2 $3" % [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r1 == "Nim is a great language" + + let r2 = "$# $# $#" % [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r2 == "Nim is a great language" + + let r3 = "${1} ${2} ${3}" % [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r3 == "Nim is a great language" + var i = 0 var length = len(frmt) result = nil @@ -315,7 +326,7 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {. while true: j = j * 10 + ord(frmt[i]) - ord('0') inc(i) - if frmt[i] notin {'0'..'9'}: break + if i >= frmt.len or frmt[i] notin {'0'..'9'}: break add(result, args[j-1]) of '{': inc(i) @@ -335,51 +346,54 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {. if i - 1 >= start: add(result, substr(frmt, start, i - 1)) -proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {. - rtl, extern: "nro$1".} = - ## shortcut for ``add(c, frmt % args)``. - add(c, frmt % args) - -const - bufSize = 1024 # 1 KB is reasonable +proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {.rtl, extern: "nro$1".} = + ## Shortcut for `add(c, frmt % args)`. + runnableExamples: + var r = rope("Dash: ") + r.addf "$1 $2 $3", [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r == "Dash: Nim is a great language" -proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} = - ## returns true if the contents of the file `f` equal `r`. - var - buf: array[bufSize, char] - bpos = buf.len - blen = buf.len + add(c, frmt % args) - for s in leaves(r): - var spos = 0 - let slen = s.len - while spos < slen: - if bpos == blen: - # Read more data - bpos = 0 - blen = readBuffer(f, addr(buf[0]), buf.len) - if blen == 0: # no more data in file - result = false - return - let n = min(blen - bpos, slen - spos) - # TODO There's gotta be a better way of comparing here... - if not equalMem(addr(buf[bpos]), - cast[pointer](cast[int](cstring(s))+spos), n): - result = false - return - spos += n - bpos += n - - result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all - -proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} = - ## returns true if the contents of the file `f` equal `r`. If `f` does not - ## exist, false is returned. - var f: File - result = open(f, filename) - if result: - result = equalsFile(r, f) - close(f) +when not defined(js) and not defined(nimscript): + const + bufSize = 1024 # 1 KB is reasonable + + proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} = + ## Returns true if the contents of the file `f` equal `r`. + var + buf: array[bufSize, char] + bpos = buf.len + blen = buf.len + + for s in leaves(r): + var spos = 0 + let slen = s.len + while spos < slen: + if bpos == blen: + # Read more data + bpos = 0 + blen = readBuffer(f, addr(buf[0]), buf.len) + if blen == 0: # no more data in file + return false + let n = min(blen - bpos, slen - spos) + # TODO: There's gotta be a better way of comparing here... + if not equalMem(addr(buf[bpos]), + cast[pointer](cast[int](cstring(s)) + spos), n): + return false + spos += n + bpos += n + + result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all + + proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} = + ## Returns true if the contents of the file `f` equal `r`. If `f` does not + ## exist, false is returned. + var f: File + result = open(f, filename) + if result: + result = equalsFile(r, f) + close(f) new(N) # init dummy node for splay algorithm diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim deleted file mode 100644 index 711e4a897..000000000 --- a/lib/pure/scgi.nim +++ /dev/null @@ -1,298 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf, Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements helper procs for SCGI applications. Example: -## -## .. code-block:: Nim -## -## import strtabs, sockets, scgi -## -## var counter = 0 -## proc handleRequest(client: Socket, input: string, -## headers: StringTableRef): bool {.procvar.} = -## inc(counter) -## client.writeStatusOkTextContent() -## client.send("Hello for the $#th time." % $counter & "\c\L") -## return false # do not stop processing -## -## run(handleRequest) -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. -## -## **Warning:** This module only supports the old asynchronous interface. -## You may wish to use the `asynchttpserver <asynchttpserver.html>`_ -## instead for web applications. - -include "system/inclrtl" - -import sockets, strutils, os, strtabs, asyncio - -type - ScgiError* = object of IOError ## the exception that is raised, if a SCGI error occurs - -proc raiseScgiError*(msg: string) {.noreturn.} = - ## raises an ScgiError exception with message `msg`. - var e: ref ScgiError - new(e) - e.msg = msg - raise e - -proc parseWord(inp: string, outp: var string, start: int): int = - result = start - while inp[result] != '\0': inc(result) - outp = substr(inp, start, result-1) - -proc parseHeaders(s: string, L: int): StringTableRef = - result = newStringTable() - var i = 0 - while i < L: - var key, val: string - i = parseWord(s, key, i)+1 - i = parseWord(s, val, i)+1 - result[key] = val - if s[i] == ',': inc(i) - else: raiseScgiError("',' after netstring expected") - -proc recvChar(s: Socket): char = - var c: char - if recv(s, addr(c), sizeof(c)) == sizeof(c): - result = c - -type - ScgiState* = object of RootObj ## SCGI state object - server: Socket - bufLen: int - client*: Socket ## the client socket to send data to - headers*: StringTableRef ## the parsed headers - input*: string ## the input buffer - - - # Async - - ClientMode = enum - ClientReadChar, ClientReadHeaders, ClientReadContent - - AsyncClient = ref object - c: AsyncSocket - mode: ClientMode - dataLen: int - headers: StringTableRef ## the parsed headers - input: string ## the input buffer - - AsyncScgiStateObj = object - handleRequest: proc (client: AsyncSocket, - input: string, - headers: StringTableRef) {.closure, gcsafe.} - asyncServer: AsyncSocket - disp: Dispatcher - AsyncScgiState* = ref AsyncScgiStateObj - -{.deprecated: [EScgi: ScgiError, TScgiState: ScgiState, - PAsyncScgiState: AsyncScgiState, scgiError: raiseScgiError].} - -proc recvBuffer(s: var ScgiState, L: int) = - if L > s.bufLen: - s.bufLen = L - s.input = newString(L) - if L > 0 and recv(s.client, cstring(s.input), L) != L: - raiseScgiError("could not read all data") - setLen(s.input, L) - -proc open*(s: var ScgiState, port = Port(4000), address = "127.0.0.1", - reuseAddr = false) = - ## opens a connection. - s.bufLen = 4000 - s.input = newString(s.bufLen) # will be reused - - s.server = socket() - if s.server == invalidSocket: raiseOSError(osLastError()) - new(s.client) # Initialise s.client for `next` - if s.server == invalidSocket: raiseScgiError("could not open socket") - #s.server.connect(connectionName, port) - if reuseAddr: - s.server.setSockOpt(OptReuseAddr, true) - bindAddr(s.server, port, address) - listen(s.server) - -proc close*(s: var ScgiState) = - ## closes the connection. - s.server.close() - -proc next*(s: var ScgiState, timeout: int = -1): bool = - ## proceed to the first/next request. Waits ``timeout`` milliseconds for a - ## request, if ``timeout`` is `-1` then this function will never time out. - ## Returns `true` if a new request has been processed. - var rsocks = @[s.server] - if select(rsocks, timeout) == 1 and rsocks.len == 1: - new(s.client) - accept(s.server, s.client) - var L = 0 - while true: - var d = s.client.recvChar() - if d == '\0': - s.client.close() - return false - if d notin strutils.Digits: - if d != ':': raiseScgiError("':' after length expected") - break - L = L * 10 + ord(d) - ord('0') - recvBuffer(s, L+1) - s.headers = parseHeaders(s.input, L) - if s.headers.getOrDefault("SCGI") != "1": raiseScgiError("SCGI Version 1 expected") - L = parseInt(s.headers.getOrDefault("CONTENT_LENGTH")) - recvBuffer(s, L) - return true - -proc writeStatusOkTextContent*(c: Socket, contentType = "text/html") = - ## sends the following string to the socket `c`:: - ## - ## Status: 200 OK\r\LContent-Type: text/html\r\L\r\L - ## - ## You should send this before sending your HTML page, for example. - c.send("Status: 200 OK\r\L" & - "Content-Type: $1\r\L\r\L" % contentType) - -proc run*(handleRequest: proc (client: Socket, input: string, - headers: StringTableRef): bool {.nimcall,gcsafe.}, - port = Port(4000)) = - ## encapsulates the SCGI object and main loop. - var s: ScgiState - s.open(port) - var stop = false - while not stop: - if next(s): - stop = handleRequest(s.client, s.input, s.headers) - s.client.close() - s.close() - -# -- AsyncIO start - -proc recvBufferAsync(client: AsyncClient, L: int): ReadLineResult = - result = ReadPartialLine - var data = "" - if L < 1: - raiseScgiError("Cannot read negative or zero length: " & $L) - let ret = recvAsync(client.c, data, L) - if ret == 0 and data == "": - client.c.close() - return ReadDisconnected - if ret == -1: - return ReadNone # No more data available - client.input.add(data) - if ret == L: - return ReadFullLine - -proc checkCloseSocket(client: AsyncClient) = - if not client.c.isClosed: - if client.c.isSendDataBuffered: - client.c.setHandleWrite do (s: AsyncSocket): - if not s.isClosed and not s.isSendDataBuffered: - s.close() - s.delHandleWrite() - else: client.c.close() - -proc handleClientRead(client: AsyncClient, s: AsyncScgiState) = - case client.mode - of ClientReadChar: - while true: - var d = "" - let ret = client.c.recvAsync(d, 1) - if d == "" and ret == 0: - # Disconnected - client.c.close() - return - if ret == -1: - return # No more data available - if d[0] notin strutils.Digits: - if d[0] != ':': raiseScgiError("':' after length expected") - break - client.dataLen = client.dataLen * 10 + ord(d[0]) - ord('0') - client.mode = ClientReadHeaders - handleClientRead(client, s) # Allow progression - of ClientReadHeaders: - let ret = recvBufferAsync(client, (client.dataLen+1)-client.input.len) - case ret - of ReadFullLine: - client.headers = parseHeaders(client.input, client.input.len-1) - if client.headers.getOrDefault("SCGI") != "1": raiseScgiError("SCGI Version 1 expected") - client.input = "" # For next part - - let contentLen = parseInt(client.headers.getOrDefault("CONTENT_LENGTH")) - if contentLen > 0: - client.mode = ClientReadContent - else: - s.handleRequest(client.c, client.input, client.headers) - checkCloseSocket(client) - of ReadPartialLine, ReadDisconnected, ReadNone: return - of ClientReadContent: - let L = parseInt(client.headers.getOrDefault("CONTENT_LENGTH")) - - client.input.len - if L > 0: - let ret = recvBufferAsync(client, L) - case ret - of ReadFullLine: - s.handleRequest(client.c, client.input, client.headers) - checkCloseSocket(client) - of ReadPartialLine, ReadDisconnected, ReadNone: return - else: - s.handleRequest(client.c, client.input, client.headers) - checkCloseSocket(client) - -proc handleAccept(sock: AsyncSocket, s: AsyncScgiState) = - var client: AsyncSocket - new(client) - accept(s.asyncServer, client) - var asyncClient = AsyncClient(c: client, mode: ClientReadChar, dataLen: 0, - headers: newStringTable(), input: "") - client.handleRead = - proc (sock: AsyncSocket) = - handleClientRead(asyncClient, s) - s.disp.register(client) - -proc open*(handleRequest: proc (client: AsyncSocket, - input: string, headers: StringTableRef) {. - closure, gcsafe.}, - port = Port(4000), address = "127.0.0.1", - reuseAddr = false): AsyncScgiState = - ## Creates an ``AsyncScgiState`` object which serves as a SCGI server. - ## - ## After the execution of ``handleRequest`` the client socket will be closed - ## automatically unless it has already been closed. - var cres: AsyncScgiState - new(cres) - cres.asyncServer = asyncSocket() - cres.asyncServer.handleAccept = proc (s: AsyncSocket) = handleAccept(s, cres) - if reuseAddr: - cres.asyncServer.setSockOpt(OptReuseAddr, true) - bindAddr(cres.asyncServer, port, address) - listen(cres.asyncServer) - cres.handleRequest = handleRequest - result = cres - -proc register*(d: Dispatcher, s: AsyncScgiState): Delegate {.discardable.} = - ## Registers ``s`` with dispatcher ``d``. - result = d.register(s.asyncServer) - s.disp = d - -proc close*(s: AsyncScgiState) = - ## Closes the ``AsyncScgiState``. - s.asyncServer.close() - -when false: - var counter = 0 - proc handleRequest(client: Socket, input: string, - headers: StringTableRef): bool {.procvar.} = - inc(counter) - client.writeStatusOkTextContent() - client.send("Hello for the $#th time." % $counter & "\c\L") - return false # do not stop processing - - run(handleRequest) - diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim deleted file mode 100644 index 8ac6acb0e..000000000 --- a/lib/pure/securehash.nim +++ /dev/null @@ -1,199 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Nim Contributers -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import - strutils, unsigned - -const Sha1DigestSize = 20 - -type - Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] - SecureHash* = distinct Sha1Digest - -proc sha1(src: string) : Sha1Digest - -proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) -proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) -proc `$`*(self: SecureHash): string = - result = "" - for v in Sha1Digest(self): - result.add(toHex(int(v), 2)) - -proc parseSecureHash*(hash: string): SecureHash = - for i in 0.. <Sha1DigestSize: - Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) - -proc `==`*(a, b: SecureHash): bool = - # Not a constant-time comparison, but that's acceptable in this context - Sha1Digest(a) == Sha1Digest(b) - - -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 - - -# Copyright (c) 2011, Micael Hildenborg -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Micael Hildenborg nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Ported to Nim by Erik O'Leary - -type - Sha1State = array[0 .. 5-1, uint32] - Sha1Buffer = array[0 .. 80-1, uint32] - -template clearBuffer(w: Sha1Buffer, len = 16) = - zeroMem(addr(w), len * sizeof(uint32)) - -proc init(result: var Sha1State) = - result[0] = 0x67452301'u32 - result[1] = 0xefcdab89'u32 - result[2] = 0x98badcfe'u32 - result[3] = 0x10325476'u32 - result[4] = 0xc3d2e1f0'u32 - -proc innerHash(state: var Sha1State, w: var Sha1Buffer) = - var - a = state[0] - b = state[1] - c = state[2] - d = state[3] - e = state[4] - - var round = 0 - - template rot(value, bits: uint32): uint32 {.immediate.} = - (value shl bits) or (value shr (32 - bits)) - - template sha1(fun, val: uint32): stmt = - let t = rot(a, 5) + fun + e + val + w[round] - e = d - d = c - c = rot(b, 30) - b = a - a = t - - template process(body: stmt): stmt = - w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) - body - inc(round) - - template wrap(dest, value: expr): stmt {.immediate.} = - let v = dest + value - dest = v - - while round < 16: - sha1((b and c) or (not b and d), 0x5a827999'u32) - inc(round) - - while round < 20: - process: - sha1((b and c) or (not b and d), 0x5a827999'u32) - - while round < 40: - process: - sha1(b xor c xor d, 0x6ed9eba1'u32) - - while round < 60: - process: - sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) - - while round < 80: - process: - sha1(b xor c xor d, 0xca62c1d6'u32) - - wrap state[0], a - wrap state[1], b - wrap state[2], c - wrap state[3], d - wrap state[4], e - -template computeInternal(src: expr): stmt {.immediate.} = - #Initialize state - var state: Sha1State - init(state) - - #Create w buffer - var w: Sha1Buffer - - #Loop through all complete 64byte blocks. - let byteLen = src.len - let endOfFullBlocks = byteLen - 64 - var endCurrentBlock = 0 - var currentBlock = 0 - - while currentBlock <= endOfFullBlocks: - endCurrentBlock = currentBlock + 64 - - var i = 0 - while currentBlock < endCurrentBlock: - w[i] = uint32(src[currentBlock+3]) or - uint32(src[currentBlock+2]) shl 8'u32 or - uint32(src[currentBlock+1]) shl 16'u32 or - uint32(src[currentBlock]) shl 24'u32 - currentBlock += 4 - inc(i) - - innerHash(state, w) - - #Handle last and not full 64 byte block if existing - endCurrentBlock = byteLen - currentBlock - clearBuffer(w) - var lastBlockBytes = 0 - - while lastBlockBytes < endCurrentBlock: - - var value = uint32(src[lastBlockBytes + currentBlock]) shl - ((3'u32 - (lastBlockBytes and 3)) shl 3) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value - inc(lastBlockBytes) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( - 0x80'u32 shl ((3'u32 - (lastBlockBytes and 3)) shl 3) - ) - - if endCurrentBlock >= 56: - innerHash(state, w) - clearBuffer(w) - - w[15] = uint32(byteLen) shl 3 - innerHash(state, w) - - # Store hash in result pointer, and make sure we get in in the correct order - # on both endian models. - for i in 0 .. Sha1DigestSize-1: - result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) - -proc sha1(src: string) : Sha1Digest = - ## Calculate SHA1 from input string - computeInternal(src) diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim new file mode 100644 index 000000000..65b059e86 --- /dev/null +++ b/lib/pure/segfaults.nim @@ -0,0 +1,88 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This modules registers a signal handler that turns access violations / +## segfaults into a ``NilAccessDefect`` exception. To be able to catch +## a NilAccessDefect all you have to do is to import this module. +## +## Tested on these OSes: Linux, Windows, OSX + +# xxx possibly broken on arm64, see bug #17178 + +{.used.} + +# do allocate memory upfront: +var se: ref NilAccessDefect +new(se) +se.name = "NilAccessDefect" +se.msg = "Could not access value because it is nil." + +when defined(windows): + include "../system/ansi_c" + + import std/winlean + + const + EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) + EXCEPTION_CONTINUE_SEARCH = LONG(0) + + type + PEXCEPTION_RECORD = ptr object + exceptionCode: DWORD # other fields left out + + PEXCEPTION_POINTERS = ptr object + exceptionRecord: PEXCEPTION_RECORD + contextRecord: pointer + + VectoredHandler = proc (p: PEXCEPTION_POINTERS): LONG {.stdcall.} + proc addVectoredExceptionHandler(firstHandler: ULONG, + handler: VectoredHandler): pointer {. + importc: "AddVectoredExceptionHandler", stdcall, dynlib: "kernel32.dll".} + + {.push stackTrace: off.} + proc segfaultHandler(p: PEXCEPTION_POINTERS): LONG {.stdcall.} = + if p.exceptionRecord.exceptionCode == EXCEPTION_ACCESS_VIOLATION: + {.gcsafe.}: + raise se + else: + result = EXCEPTION_CONTINUE_SEARCH + {.pop.} + + discard addVectoredExceptionHandler(0, segfaultHandler) + + when false: + {.push stackTrace: off.} + proc segfaultHandler(sig: cint) {.noconv.} = + {.gcsafe.}: + rawRaise se + {.pop.} + c_signal(SIGSEGV, segfaultHandler) + +else: + import std/posix + + var sa: Sigaction + + var SEGV_MAPERR {.importc, header: "<signal.h>".}: cint + + {.push stackTrace: off.} + proc segfaultHandler(sig: cint, y: ptr SigInfo, z: pointer) {.noconv.} = + if y.si_code == SEGV_MAPERR: + {.gcsafe.}: + raise se + else: + quit(1) + {.pop.} + + discard sigemptyset(sa.sa_mask) + + sa.sa_sigaction = segfaultHandler + sa.sa_flags = SA_SIGINFO or SA_NODEFER + + discard sigaction(SIGSEGV, sa) diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 832f5f4f9..ac180e2bd 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -1,414 +1,375 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta +# (c) Copyright 2016 Eugene Kabanov # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# TODO: Docs. - -import os, unsigned, hashes - -when defined(linux): - import posix, epoll -elif defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd): - import posix, kqueue, times -elif defined(windows): - import winlean -else: - import posix - -const MultiThreaded = defined(useStdlibThreading) - -when MultiThreaded: - import sharedtables - - type SelectorData = pointer -else: - import tables - - type SelectorData = RootRef - -proc hash*(x: SocketHandle): Hash {.borrow.} -proc `$`*(x: SocketHandle): string {.borrow.} - -type - Event* = enum - EvRead, EvWrite, EvError - - SelectorKey* = object - fd*: SocketHandle - events*: set[Event] ## The events which ``fd`` listens for. - data*: SelectorData ## User object. - - ReadyInfo* = tuple[key: SelectorKey, events: set[Event]] +## This module allows high-level and efficient I/O multiplexing. +## +## Supported OS primitives: `epoll`, `kqueue`, `poll` and +## Windows `select`. +## +## To use threadsafe version of this module, it needs to be compiled +## with both `-d:threadsafe` and `--threads:on` options. +## +## Supported features: files, sockets, pipes, timers, processes, signals +## and user events. +## +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux (except +## for Android). +## +## Partially supported OS: Windows (only sockets and user events), +## Solaris (files, sockets, handles and user events). +## Android (files, sockets, handles and user events). +## +## TODO: `/dev/poll`, `event ports` and filesystem events. + +import std/nativesockets +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/assertions + +const hasThreadSupport = compileOption("threads") and defined(threadsafe) + +const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(dragonfly) or defined(nuttx) or + (defined(linux) and not defined(android) and not defined(emscripten)) + ## This constant is used to determine whether the destination platform is + ## fully supported by `ioselectors` module. + +const bsdPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(dragonfly) when defined(nimdoc): type - Selector* = ref object - ## An object which holds file descriptors to be checked for read/write - ## status. - - proc register*(s: Selector, fd: SocketHandle, events: set[Event], - data: SelectorData): SelectorKey {.discardable.} = - ## Registers file descriptor ``fd`` to selector ``s`` with a set of Event - ## ``events``. + Selector*[T] = ref object + ## An object which holds descriptors to be checked for read/write status + + IOSelectorsException* = object of CatchableError + ## Exception that is raised if an IOSelectors error occurs. + + Event* {.pure.} = enum + ## An enum which hold event types + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## BSD specific file change + User, ## User event is raised + Error, ## Error occurred while waiting for descriptor + VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) + VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) + VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) + VnodeAttrib, ## NOTE_ATTRIB (BSD specific, file attributes changed) + VnodeLink, ## NOTE_LINK (BSD specific, file link count changed) + VnodeRename, ## NOTE_RENAME (BSD specific, file renamed) + VnodeRevoke ## NOTE_REVOKE (BSD specific, file revoke occurred) + + ReadyKey* = object + ## An object which holds result for descriptor + fd* : int ## file/socket descriptor + events*: set[Event] ## set of events + errorCode*: OSErrorCode ## additional error code information for + ## Error events + + SelectEvent* = object + ## An object which holds user defined event + + proc newSelector*[T](): Selector[T] = + ## Creates a new selector - proc update*(s: Selector, fd: SocketHandle, - events: set[Event]): SelectorKey {.discardable.} = - ## Updates the events which ``fd`` wants notifications for. + proc close*[T](s: Selector[T]) = + ## Closes the selector. + + proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + ## Registers file/socket descriptor `fd` to selector `s` + ## with events set in `events`. The `data` is application-defined + ## data, which will be passed when an event is triggered. + + proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + ## Update file/socket descriptor `fd`, registered in selector + ## `s` with new events set `event`. + + proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + ## Registers timer notification with `timeout` (in milliseconds) + ## to selector `s`. + ## + ## If `oneshot` is `true`, timer will be notified only once. + ## + ## Set `oneshot` to `false` if you want periodic notifications. + ## + ## The `data` is application-defined data, which will be passed, when + ## the timer is triggered. + ## + ## Returns the file descriptor for the registered timer. + + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + ## Registers Unix signal notification with `signal` to selector + ## `s`. + ## + ## The `data` is application-defined data, which will be + ## passed when signal raises. + ## + ## Returns the file descriptor for the registered signal. + ## + ## **Note:** This function is not supported on `Windows`. + + proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + ## Registers a process id (pid) notification (when process has + ## exited) in selector `s`. + ## + ## The `data` is application-defined data, which will be passed when + ## process with `pid` has exited. + ## + ## Returns the file descriptor for the registered signal. + + proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + ## Registers selector event `ev` in selector `s`. + ## + ## The `data` is application-defined data, which will be passed when + ## `ev` happens. + + proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], + data: T) = + ## Registers selector BSD/MacOSX specific vnode events for file + ## descriptor `fd` and events `events`. + ## `data` application-defined data, which to be passed, when + ## vnode event happens. + ## + ## **Note:** This function is supported only by BSD and MacOSX. + + proc newSelectEvent*(): SelectEvent = + ## Creates a new user-defined event. + + proc trigger*(ev: SelectEvent) = + ## Trigger event `ev`. + + proc close*(ev: SelectEvent) = + ## Closes user-defined event `ev`. + + proc unregister*[T](s: Selector[T], ev: SelectEvent) = + ## Unregisters user-defined event `ev` from selector `s`. + + proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = + ## Unregisters file/socket descriptor `fd` from selector `s`. + + proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int = + ## Waits for events registered in selector `s`. + ## + ## The `timeout` argument specifies the maximum number of milliseconds + ## the function will be blocked for if no events are ready. Specifying a + ## timeout of `-1` causes the function to block indefinitely. + ## All available events will be stored in `results` array. + ## + ## Returns number of triggered events. + + proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + ## Waits for events registered in selector `s`. + ## + ## The `timeout` argument specifies the maximum number of milliseconds + ## the function will be blocked for if no events are ready. Specifying a + ## timeout of `-1` causes the function to block indefinitely. + ## + ## Returns a list of triggered events. + + proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + ## Retrieves application-defined `data` associated with descriptor `fd`. + ## If specified descriptor `fd` is not registered, empty/default value + ## will be returned. + + proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: var T): bool = + ## Associate application-defined `data` with descriptor `fd`. + ## + ## Returns `true`, if data was successfully updated, `false` otherwise. + + template isEmpty*[T](s: Selector[T]): bool = # TODO: Why is this a template? + ## Returns `true`, if there are no registered events or descriptors + ## in selector. + + template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + ## Retrieves the application-data assigned with descriptor `fd` + ## to `value`. This `value` can be modified in the scope of + ## the `withData` call. + ## + ## ```nim + ## s.withData(fd, value) do: + ## # block is executed only if `fd` registered in selector `s` + ## value.uid = 1000 + ## ``` + + template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body1, body2: untyped) = + ## Retrieves the application-data assigned with descriptor `fd` + ## to `value`. This `value` can be modified in the scope of + ## the `withData` call. + ## + ## ```nim + ## s.withData(fd, value) do: + ## # block is executed only if `fd` registered in selector `s`. + ## value.uid = 1000 + ## do: + ## # block is executed if `fd` not registered in selector `s`. + ## raise + ## ``` + + proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + ## Determines whether selector contains a file descriptor. - proc unregister*(s: Selector, fd: SocketHandle): SelectorKey {.discardable.} = - ## Unregisters file descriptor ``fd`` from selector ``s``. + proc getFd*[T](s: Selector[T]): int = + ## Retrieves the underlying selector's file descriptor. + ## + ## For *poll* and *select* selectors `-1` is returned. - proc close*(s: Selector) = - ## Closes the selector +else: + import std/strutils + when hasThreadSupport: + import std/locks - proc select*(s: Selector, timeout: int): seq[ReadyInfo] = - ## The ``events`` field of the returned ``key`` contains the original events - ## for which the ``fd`` was bound. This is contrary to the ``events`` field - ## of the ``ReadyInfo`` tuple which determines which events are ready - ## on the ``fd``. + type + SharedArray[T] = UncheckedArray[T] - proc newSelector*(): Selector = - ## Creates a new selector + proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - proc contains*(s: Selector, fd: SocketHandle): bool = - ## Determines whether selector contains a file descriptor. + proc reallocSharedArray[T](sa: ptr SharedArray[T], oldsize, nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared0(sa, oldsize * sizeof(T), sizeof(T) * nsize)) - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - ## Retrieves the selector key for ``fd``. + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = + deallocShared(cast[pointer](sa)) + type + Event* {.pure.} = enum + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, + Finished, VnodeWrite, VnodeDelete, VnodeExtend, VnodeAttrib, VnodeLink, + VnodeRename, VnodeRevoke -elif defined(linux): type - Selector* = object - epollFD: cint - events: array[64, epoll_event] - when MultiThreaded: - fds: SharedTable[SocketHandle, SelectorKey] - else: - fds: Table[SocketHandle, SelectorKey] - - proc createEventStruct(events: set[Event], fd: SocketHandle): epoll_event = - if EvRead in events: - result.events = EPOLLIN - if EvWrite in events: - result.events = result.events or EPOLLOUT - result.events = result.events or EPOLLRDHUP - result.data.fd = fd.cint - - proc register*(s: var Selector, fd: SocketHandle, events: set[Event], - data: SelectorData) = - var event = createEventStruct(events, fd) - if events != {}: - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: - raiseOSError(osLastError()) - - s.fds[fd] = SelectorKey(fd: fd, events: events, data: data) - - proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = - if s.fds[fd].events != events: - if events == {}: - # This fd is idle -- it should not be registered to epoll. - # But it should remain a part of this selector instance. - # This is to prevent epoll_wait from returning immediately - # because its got fds which are waiting for no events and - # are therefore constantly ready. (leading to 100% CPU usage). - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: - raiseOSError(osLastError()) - s.fds.mget(fd).events = events - else: - var event = createEventStruct(events, fd) - if s.fds[fd].events == {}: - # This fd is idle. It's not a member of this epoll instance and must - # be re-registered. - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: - raiseOSError(osLastError()) - else: - if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: - raiseOSError(osLastError()) - s.fds.mget(fd).events = events - - proc unregister*(s: var Selector, fd: SocketHandle) = - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: - let err = osLastError() - if err.cint notin {ENOENT, EBADF}: - # TODO: Why do we sometimes get an EBADF? Is this normal? - raiseOSError(err) - s.fds.del(fd) - - proc close*(s: var Selector) = - when MultiThreaded: deinitSharedTable(s.fds) - if s.epollFD.close() != 0: raiseOSError(osLastError()) - - proc epollHasFd(s: Selector, fd: SocketHandle): bool = - result = true - var event = createEventStruct(s.fds[fd].events, fd) - if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: - let err = osLastError() - if err.cint in {ENOENT, EBADF}: - return false - raiseOSError(err) - - proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = - result = @[] - let evNum = epoll_wait(s.epollFD, addr s.events[0], 64.cint, timeout.cint) - if evNum < 0: - let err = osLastError() - if err.cint == EINTR: - return @[] - raiseOSError(err) - if evNum == 0: return @[] - for i in 0 .. <evNum: - let fd = s.events[i].data.fd.SocketHandle - - var evSet: set[Event] = {} - if (s.events[i].events and EPOLLERR) != 0 or (s.events[i].events and EPOLLHUP) != 0: evSet = evSet + {EvError} - if (s.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead} - if (s.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite} - let selectorKey = s.fds[fd] - assert selectorKey.fd != 0.SocketHandle - result.add((selectorKey, evSet)) - - #echo("Epoll: ", result[i].key.fd, " ", result[i].events, " ", result[i].key.events) - - proc newSelector*(): Selector = - result.epollFD = epoll_create(64) - if result.epollFD < 0: - raiseOSError(osLastError()) - when MultiThreaded: - result.fds = initSharedTable[SocketHandle, SelectorKey]() + IOSelectorsException* = object of CatchableError + + ReadyKey* = object + fd*: int + events*: set[Event] + errorCode*: OSErrorCode + + SelectorKey[T] = object + ident: int + events: set[Event] + param: int + data: T + + const + InvalidIdent = -1 + + proc raiseIOSelectorsError[T](message: T) = + var msg = "" + when T is string: + msg.add(message) + elif T is OSErrorCode: + msg.add(osErrorMsg(message) & " (code: " & $int(message) & ")") else: - result.fds = initTable[SocketHandle, SelectorKey]() - - proc contains*(s: Selector, fd: SocketHandle): bool = - ## Determines whether selector contains a file descriptor. - if s.fds.hasKey(fd): - # Ensure the underlying epoll instance still contains this fd. - if s.fds[fd].events != {}: - result = epollHasFd(s, fd) + msg.add("Internal Error\n") + var err = newException(IOSelectorsException, msg) + raise err + + proc setNonBlocking(fd: cint) {.inline.} = + setBlocking(fd.SocketHandle, false) + + when not defined(windows): + import std/posix + + template setKey(s, pident, pevents, pparam, pdata: untyped) = + var skey = addr(s.fds[pident]) + skey.ident = pident + skey.events = pevents + skey.param = pparam + skey.data = pdata + + when ioselSupportedPlatform: + template blockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) else: - result = true - else: - return false - - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - ## Retrieves the selector key for ``fd``. - return s.fds[fd] + if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) -elif defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd): - type - Selector* = object - kqFD: cint - events: array[64, KEvent] - when MultiThreaded: - fds: SharedTable[SocketHandle, SelectorKey] + template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) else: - fds: Table[SocketHandle, SelectorKey] - - template modifyKQueue(kqFD: cint, fd: SocketHandle, event: Event, - op: cushort) = - var kev = KEvent(ident: fd.cuint, - filter: if event == EvRead: EVFILT_READ else: EVFILT_WRITE, - flags: op) - if kevent(kqFD, addr kev, 1, nil, 0, nil) == -1: - raiseOSError(osLastError()) - - proc register*(s: var Selector, fd: SocketHandle, events: set[Event], - data: SelectorData) = - for event in events: - modifyKQueue(s.kqFD, fd, event, EV_ADD) - s.fds[fd] = SelectorKey(fd: fd, events: events, data: data) - - proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = - let previousEvents = s.fds[fd].events - if previousEvents != events: - for event in events-previousEvents: - modifyKQueue(s.kqFD, fd, event, EV_ADD) - for event in previousEvents-events: - modifyKQueue(s.kqFD, fd, event, EV_DELETE) - s.fds.mget(fd).events = events - - proc unregister*(s: var Selector, fd: SocketHandle) = - for event in s.fds[fd].events: - modifyKQueue(s.kqFD, fd, event, EV_DELETE) - s.fds.del(fd) - - proc close*(s: var Selector) = - when MultiThreaded: deinitSharedTable(s.fds) - if s.kqFD.close() != 0: raiseOSError(osLastError()) - - proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = - result = @[] - var tv = - if timeout >= 1000: Timespec(tv_sec: (timeout div 1000).Time, tv_nsec: 0) - else: Timespec(tv_sec: 0.Time, tv_nsec: timeout * 1000000) - let evNum = kevent(s.kqFD, nil, 0, addr s.events[0], 64.cint, addr tv) - if evNum < 0: - let err = osLastError() - if err.cint == EINTR: - return @[] - raiseOSError(err) - if evNum == 0: return @[] - for i in 0 .. <evNum: - let fd = s.events[i].ident.SocketHandle - - var evSet: set[Event] = {} - if (s.events[i].flags and EV_EOF) != 0: evSet = evSet + {EvError} - if s.events[i].filter == EVFILT_READ: evSet = evSet + {EvRead} - elif s.events[i].filter == EVFILT_WRITE: evSet = evSet + {EvWrite} - let selectorKey = s.fds[fd] - assert selectorKey.fd != 0.SocketHandle - result.add((selectorKey, evSet)) - - proc newSelector*(): Selector = - result.kqFD = kqueue() - if result.kqFD < 0: - raiseOSError(osLastError()) - when MultiThreaded: - result.fds = initSharedTable[SocketHandle, SelectorKey]() - else: - result.fds = initTable[SocketHandle, SelectorKey]() - - proc contains*(s: Selector, fd: SocketHandle): bool = - ## Determines whether selector contains a file descriptor. - s.fds.hasKey(fd) # and s.fds[fd].events != {} - - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - ## Retrieves the selector key for ``fd``. - return s.fds[fd] - -elif not defined(nimdoc): - # TODO: kqueue for bsd/mac os x. - type - Selector* = object - when MultiThreaded: - fds: SharedTable[SocketHandle, SelectorKey] + if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseIOSelectorsError(osLastError()) + + template clearKey[T](key: ptr SelectorKey[T]) = + var empty: T + key.ident = InvalidIdent + key.events = {} + key.data = empty + + proc verifySelectParams(timeout: int) = + # Timeout of -1 means: wait forever + # Anything higher is the time to wait in milliseconds. + doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) + + when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): + template maxDescriptors*(): int = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. + when defined(windows): + 16_700_000 + elif defined(zephyr) or defined(freertos): + FD_MAX else: - fds: Table[SocketHandle, SelectorKey] - - proc register*(s: var Selector, fd: SocketHandle, events: set[Event], - data: SelectorData) = - let result = SelectorKey(fd: fd, events: events, data: data) - if s.fds.hasKeyOrPut(fd, result): - raise newException(ValueError, "File descriptor already exists.") - - proc update*(s: var Selector, fd: SocketHandle, events: set[Event]) = - #if not s.fds.hasKey(fd): - # raise newException(ValueError, "File descriptor not found.") - s.fds.mget(fd).events = events - - proc unregister*(s: var Selector, fd: SocketHandle) = - s.fds.del(fd) - - proc close*(s: var Selector) = - when MultiThreaded: deinitSharedTable(s.fds) - - proc timeValFromMilliseconds(timeout: int): TimeVal = - if timeout != -1: - var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - - proc createFdSet(rd, wr: var TFdSet, s: Selector, m: var int) = - FD_ZERO(rd); FD_ZERO(wr) - for k, v in pairs(s.fds): - if EvRead in v.events: - m = max(m, int(k)) - FD_SET(k, rd) - if EvWrite in v.events: - m = max(m, int(k)) - FD_SET(k, wr) - - proc getReadyFDs(rd, wr: var TFdSet, - s: var Selector): seq[ReadyInfo] = - result = @[] - for k, v in pairs(s.fds): - var events: set[Event] = {} - if FD_ISSET(k, rd) != 0'i32: - events = events + {EvRead} - if FD_ISSET(k, wr) != 0'i32: - events = events + {EvWrite} - result.add((v, events)) - - proc select*(s: var Selector, timeout: int): seq[ReadyInfo] = - var tv {.noInit.}: TimeVal = timeValFromMilliseconds(timeout) - - var rd, wr: TFdSet - var m = 0 - createFdSet(rd, wr, s, m) - - var retCode = 0 - if timeout != -1: - retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv))) + var fdLim: RLimit + var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) + if res >= 0: + res = int(fdLim.rlim_cur) - 1 + res + + when defined(nimIoselector): + when nimIoselector == "epoll": + include ioselects/ioselectors_epoll + elif nimIoselector == "kqueue": + include ioselects/ioselectors_kqueue + elif nimIoselector == "poll": + include ioselects/ioselectors_poll + elif nimIoselector == "select": + include ioselects/ioselectors_select else: - retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, nil)) - - if retCode < 0: - raiseOSError(osLastError()) - elif retCode == 0: - return @[] - else: - return getReadyFDs(rd, wr, s) - - proc newSelector*(): Selector = - when MultiThreaded: - result.fds = initSharedTable[SocketHandle, SelectorKey]() - else: - result.fds = initTable[SocketHandle, SelectorKey]() - - proc contains*(s: Selector, fd: SocketHandle): bool = - return s.fds.hasKey(fd) - - proc `[]`*(s: Selector, fd: SocketHandle): SelectorKey = - return s.fds[fd] - -proc contains*(s: Selector, key: SelectorKey): bool = - ## Determines whether selector contains this selector key. More accurate - ## than checking if the file descriptor is in the selector because it - ## ensures that the keys are equal. File descriptors may not always be - ## unique especially when an fd is closed and then a new one is opened, - ## the new one may have the same value. - when not defined(nimdoc): - return key.fd in s and s.fds[key.fd] == key - -{.deprecated: [TEvent: Event, PSelectorKey: SelectorKey, - TReadyInfo: ReadyInfo, PSelector: Selector].} - - -when not defined(testing) and isMainModule and not defined(nimdoc): - # Select() - import sockets - - when MultiThreaded: - type - SockWrapper = object - sock: Socket - else: - type - SockWrapper = ref object of RootObj - sock: Socket - - var sock = socket() - if sock == sockets.invalidSocket: raiseOSError(osLastError()) - #sock.setBlocking(false) - sock.connect("irc.freenode.net", Port(6667)) - - var selector = newSelector() - var data = SockWrapper(sock: sock) - when MultiThreaded: - selector.register(sock.getFD, {EvWrite}, addr data) + {.fatal: "Unknown nimIoselector specified by define.".} + elif defined(linux) and not defined(emscripten): + include ioselects/ioselectors_epoll + elif bsdPlatform: + include ioselects/ioselectors_kqueue + elif defined(windows): + include ioselects/ioselectors_select + elif defined(solaris): + include ioselects/ioselectors_poll # need to replace it with event ports + elif defined(genode): + include ioselects/ioselectors_select # TODO: use the native VFS layer + elif defined(nintendoswitch): + include ioselects/ioselectors_select + elif defined(freertos) or defined(lwip): + include ioselects/ioselectors_select + elif defined(zephyr): + include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: - selector.register(sock.getFD, {EvWrite}, data) - var i = 0 - while true: - let ready = selector.select(1000) - echo ready.len - if ready.len > 0: echo ready[0].events - i.inc - if i == 6: - selector.unregister(sock.getFD) - selector.close() - break + include ioselects/ioselectors_poll diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim deleted file mode 100644 index b2adac2f3..000000000 --- a/lib/pure/smtp.nim +++ /dev/null @@ -1,277 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements the SMTP client protocol as specified by RFC 5321, -## this can be used to send mail to any SMTP Server. -## -## This module also implements the protocol used to format messages, -## as specified by RFC 2822. -## -## Example gmail use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## var smtp = connect("smtp.gmail.com", 465, true, true) -## smtp.auth("username", "password") -## smtp.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## For SSL support this module relies on OpenSSL. If you want to -## enable SSL, compile with ``-d:ssl``. - -import net, strutils, strtabs, base64, os -import asyncnet, asyncdispatch - -type - Smtp* = object - sock: Socket - debug: bool - - Message* = object - msgTo: seq[string] - msgCc: seq[string] - msgSubject: string - msgOtherHeaders: StringTableRef - msgBody: string - - ReplyError* = object of IOError - - AsyncSmtp* = ref object - sock: AsyncSocket - address: string - port: Port - useSsl: bool - debug: bool - -{.deprecated: [EInvalidReply: ReplyError, TMessage: Message, TSMTP: Smtp].} - -proc debugSend(smtp: Smtp, cmd: string) = - if smtp.debug: - echo("C:" & cmd) - smtp.sock.send(cmd) - -proc debugRecv(smtp: var Smtp): TaintedString = - var line = TaintedString"" - smtp.sock.readLine(line) - - if smtp.debug: - echo("S:" & line.string) - return line - -proc quitExcpt(smtp: Smtp, msg: string) = - smtp.debugSend("QUIT") - raise newException(ReplyError, msg) - -proc checkReply(smtp: var Smtp, reply: string) = - var line = smtp.debugRecv() - if not line.string.startswith(reply): - quitExcpt(smtp, "Expected " & reply & " reply, got: " & line.string) - -const compiledWithSsl = defined(ssl) - -when not defined(ssl): - type PSSLContext = ref object - let defaultSSLContext: PSSLContext = nil -else: - let defaultSSLContext = newContext(verifyMode = CVerifyNone) - -proc connect*(address: string, port = Port(25), - ssl = false, debug = false, - sslContext = defaultSSLContext): Smtp = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - result.sock = newSocket() - if ssl: - when compiledWithSsl: - sslContext.wrapSocket(result.sock) - else: - raise newException(ESystem, - "SMTP module compiled without SSL support") - result.sock.connect(address, port) - result.debug = debug - - result.checkReply("220") - result.debugSend("HELO " & address & "\c\L") - result.checkReply("250") - -proc auth*(smtp: var Smtp, username, password: string) = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - smtp.debugSend("AUTH LOGIN\c\L") - smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - smtp.debugSend(encode(username) & "\c\L") - smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - smtp.debugSend(encode(password) & "\c\L") - smtp.checkReply("235") # Check whether the authentification was successful. - -proc sendmail*(smtp: var Smtp, fromaddr: string, - toaddrs: seq[string], msg: string) = - ## Sends `msg` from `fromaddr` to `toaddr`. - ## Messages may be formed using ``createMessage`` by converting the - ## Message into a string. - - smtp.debugSend("MAIL FROM:<" & fromaddr & ">\c\L") - smtp.checkReply("250") - for address in items(toaddrs): - smtp.debugSend("RCPT TO:<" & address & ">\c\L") - smtp.checkReply("250") - - # Send the message - smtp.debugSend("DATA " & "\c\L") - smtp.checkReply("354") - smtp.debugSend(msg & "\c\L") - smtp.debugSend(".\c\L") - smtp.checkReply("250") - -proc close*(smtp: Smtp) = - ## Disconnects from the SMTP server and closes the socket. - smtp.debugSend("QUIT\c\L") - smtp.sock.close() - -proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], - otherHeaders: openarray[tuple[name, value: string]]): Message = - ## Creates a new MIME compliant message. - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - for n, v in items(otherHeaders): - result.msgOtherHeaders[n] = v - -proc createMessage*(mSubject, mBody: string, mTo, - mCc: seq[string] = @[]): Message = - ## Alternate version of the above. - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - -proc `$`*(msg: Message): string = - ## stringify for ``Message``. - result = "" - if msg.msgTo.len() > 0: - result = "TO: " & msg.msgTo.join(", ") & "\c\L" - if msg.msgCc.len() > 0: - result.add("CC: " & msg.msgCc.join(", ") & "\c\L") - # TODO: Folding? i.e when a line is too long, shorten it... - result.add("Subject: " & msg.msgSubject & "\c\L") - for key, value in pairs(msg.msgOtherHeaders): - result.add(key & ": " & value & "\c\L") - - result.add("\c\L") - result.add(msg.msgBody) - -proc newAsyncSmtp*(address: string, port: Port, useSsl = false, - sslContext = defaultSslContext): AsyncSmtp = - ## Creates a new ``AsyncSmtp`` instance. - new result - result.address = address - result.port = port - result.useSsl = useSsl - - result.sock = newAsyncSocket() - if useSsl: - when compiledWithSsl: - sslContext.wrapSocket(result.sock) - else: - raise newException(ESystem, - "SMTP module compiled without SSL support") - -proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = - var retFuture = newFuture[void]() - var sendFut = smtp.sock.send("QUIT") - sendFut.callback = - proc () = - # TODO: Fix this in async procs. - raise newException(ReplyError, msg) - return retFuture - -proc checkReply(smtp: AsyncSmtp, reply: string) {.async.} = - var line = await smtp.sock.recvLine() - if not line.string.startswith(reply): - await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line.string) - -proc connect*(smtp: AsyncSmtp) {.async.} = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - await smtp.sock.connect(smtp.address, smtp.port) - - await smtp.checkReply("220") - await smtp.sock.send("HELO " & smtp.address & "\c\L") - await smtp.checkReply("250") - -proc auth*(smtp: AsyncSmtp, username, password: string) {.async.} = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - await smtp.sock.send("AUTH LOGIN\c\L") - await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - await smtp.sock.send(encode(username) & "\c\L") - await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - await smtp.sock.send(encode(password) & "\c\L") - await smtp.checkReply("235") # Check whether the authentification was successful. - -proc sendMail*(smtp: AsyncSmtp, fromAddr: string, - toAddrs: seq[string], msg: string) {.async.} = - ## Sends ``msg`` from ``fromAddr`` to the addresses specified in ``toAddrs``. - ## Messages may be formed using ``createMessage`` by converting the - ## Message into a string. - - await smtp.sock.send("MAIL FROM:<" & fromAddr & ">\c\L") - await smtp.checkReply("250") - for address in items(toAddrs): - await smtp.sock.send("RCPT TO:<" & address & ">\c\L") - await smtp.checkReply("250") - - # Send the message - await smtp.sock.send("DATA " & "\c\L") - await smtp.checkReply("354") - await smtp.sock.send(msg & "\c\L") - await smtp.sock.send(".\c\L") - await smtp.checkReply("250") - -proc close*(smtp: AsyncSmtp) {.async.} = - ## Disconnects from the SMTP server and closes the socket. - await smtp.sock.send("QUIT\c\L") - smtp.sock.close() - -when not defined(testing) and isMainModule: - #var msg = createMessage("Test subject!", - # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) - #echo(msg) - - #var smtp = connect("localhost", 25, False, True) - #smtp.sendmail("root@localhost", @["dominik@localhost"], $msg) - - #echo(decode("a17sm3701420wbe.12")) - proc main() {.async.} = - var client = newAsyncSmtp("smtp.gmail.com", Port(465), true) - await client.connect() - await client.auth("johndoe", "foo") - var msg = createMessage("Hello from Nim's SMTP!", - "Hello!!!!.\n Is this awesome or what?", - @["blah@gmail.com"]) - echo(msg) - await client.sendMail("blah@gmail.com", @["blah@gmail.com"], $msg) - - await client.close() - - waitFor main() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg deleted file mode 100644 index 521e21de4..000000000 --- a/lib/pure/smtp.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:ssl diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim new file mode 100644 index 000000000..d60cd22eb --- /dev/null +++ b/lib/pure/ssl_certs.nim @@ -0,0 +1,172 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Scan for SSL/TLS CA certificates on disk +## The default locations can be overridden using the SSL_CERT_FILE and +## SSL_CERT_DIR environment variables. + +import std/[os, strutils] + +# FWIW look for files before scanning entire dirs. + +when defined(macosx): + const certificatePaths = [ + "/etc/ssl/cert.pem", + "/System/Library/OpenSSL/certs/cert.pem" + ] +elif defined(linux): + const certificatePaths = [ + # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo + # NetBSD (security/mozilla-rootcerts) + # SLES10/SLES11, https://golang.org/issue/12139 + "/etc/ssl/certs/ca-certificates.crt", + # OpenSUSE + "/etc/ssl/ca-bundle.pem", + # Red Hat 5+, Fedora, Centos + "/etc/pki/tls/certs/ca-bundle.crt", + # Red Hat 4 + "/usr/share/ssl/certs/ca-bundle.crt", + # Fedora/RHEL + "/etc/pki/tls/certs", + # Android + "/data/data/com.termux/files/usr/etc/tls/cert.pem", + "/system/etc/security/cacerts", + ] +elif defined(bsd): + const certificatePaths = [ + # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo + # NetBSD (security/mozilla-rootcerts) + # SLES10/SLES11, https://golang.org/issue/12139 + "/etc/ssl/certs/ca-certificates.crt", + # FreeBSD (security/ca-root-nss package) + "/usr/local/share/certs/ca-root-nss.crt", + # OpenBSD, FreeBSD (optional symlink) + "/etc/ssl/cert.pem", + # FreeBSD + "/usr/local/share/certs", + # NetBSD + "/etc/openssl/certs", + ] +else: + const certificatePaths = [ + # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo + # NetBSD (security/mozilla-rootcerts) + # SLES10/SLES11, https://golang.org/issue/12139 + "/etc/ssl/certs/ca-certificates.crt", + # OpenSUSE + "/etc/ssl/ca-bundle.pem", + # Red Hat 5+, Fedora, Centos + "/etc/pki/tls/certs/ca-bundle.crt", + # Red Hat 4 + "/usr/share/ssl/certs/ca-bundle.crt", + # FreeBSD (security/ca-root-nss package) + "/usr/local/share/certs/ca-root-nss.crt", + # CentOS/RHEL 7 + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + # OpenBSD, FreeBSD (optional symlink) + "/etc/ssl/cert.pem", + # Fedora/RHEL + "/etc/pki/tls/certs", + # Android + "/system/etc/security/cacerts", + # FreeBSD + "/usr/local/share/certs", + # NetBSD + "/etc/openssl/certs", + ] + +when defined(haiku): + const + B_FIND_PATH_EXISTING_ONLY = 0x4 + B_FIND_PATH_DATA_DIRECTORY = 6 + + proc find_paths_etc(architecture: cstring, baseDirectory: cint, + subPath: cstring, flags: uint32, + paths: var ptr UncheckedArray[cstring], + pathCount: var csize_t): int32 + {.importc, header: "<FindDirectory.h>".} + proc free(p: pointer) {.importc, header: "<stdlib.h>".} + +iterator scanSSLCertificates*(useEnvVars = false): string = + ## Scan for SSL/TLS CA certificates on disk. + ## + ## if `useEnvVars` is true, the SSL_CERT_FILE and SSL_CERT_DIR + ## environment variables can be used to override the certificate + ## directories to scan or specify a CA certificate file. + if useEnvVars and existsEnv("SSL_CERT_FILE"): + yield getEnv("SSL_CERT_FILE") + + elif useEnvVars and existsEnv("SSL_CERT_DIR"): + let p = getEnv("SSL_CERT_DIR") + for fn in joinPath(p, "*").walkFiles(): + yield fn + + else: + when defined(windows): + const cacert = "cacert.pem" + let pem = getAppDir() / cacert + if fileExists(pem): + yield pem + else: + let path = getEnv("PATH") + for candidate in split(path, PathSep): + if candidate.len != 0: + let x = (if candidate[0] == '"' and candidate[^1] == '"': + substr(candidate, 1, candidate.len-2) else: candidate) / cacert + if fileExists(x): + yield x + elif not defined(haiku): + for p in certificatePaths: + if p.endsWith(".pem") or p.endsWith(".crt"): + if fileExists(p): + yield p + elif dirExists(p): + # check if it's a dir where each cert is one file + # named by it's hasg + for fn in joinPath(p, "*.0").walkFiles: + yield p.normalizePathEnd(true) + break + for fn in joinPath(p, "*").walkFiles(): + + yield fn + else: + var + paths: ptr UncheckedArray[cstring] + size: csize_t + let err = find_paths_etc( + nil, B_FIND_PATH_DATA_DIRECTORY, "ssl/CARootCertificates.pem", + B_FIND_PATH_EXISTING_ONLY, paths, size + ) + if err == 0: + defer: free(paths) + for i in 0 ..< size: + yield $paths[i] + +# Certificates management on windows +# when defined(windows) or defined(nimdoc): +# +# import std/openssl +# +# type +# PCCertContext {.final, pure.} = pointer +# X509 {.final, pure.} = pointer +# CertStore {.final, pure.} = pointer +# +# # OpenSSL cert store +# +# {.push stdcall, dynlib: "kernel32", importc.} +# +# proc CertOpenSystemStore*(hprov: pointer=nil, szSubsystemProtocol: cstring): CertStore +# +# proc CertEnumCertificatesInStore*(hCertStore: CertStore, pPrevCertContext: PCCertContext): pointer +# +# proc CertFreeCertificateContext*(pContext: PCCertContext): bool +# +# proc CertCloseStore*(hCertStore:CertStore, flags:cint): bool +# +# {.pop.} diff --git a/lib/pure/ssl_config.nim b/lib/pure/ssl_config.nim new file mode 100644 index 000000000..14f66ede4 --- /dev/null +++ b/lib/pure/ssl_config.nim @@ -0,0 +1,51 @@ +# This file was automatically generated by tools/ssl_config_parser on 2020-06-03T22:02:05Z. DO NOT EDIT. + +## This module contains SSL configuration parameters obtained from +## `Mozilla OpSec <https://wiki.mozilla.org/Security/Server_Side_TLS>`_. +## +## The configuration file used to generate this module: https://ssl-config.mozilla.org/guidelines/5.4.json + +const CiphersModern* = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" + ## An OpenSSL-compatible list of secure ciphers for ``modern`` compatibility + ## per Mozilla's recommendations. + ## + ## Oldest clients supported by this list: + ## * Firefox 63 + ## * Android 10.0 + ## * Chrome 70 + ## * Edge 75 + ## * Java 11 + ## * OpenSSL 1.1.1 + ## * Opera 57 + ## * Safari 12.1 + +const CiphersIntermediate* = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" + ## An OpenSSL-compatible list of secure ciphers for ``intermediate`` compatibility + ## per Mozilla's recommendations. + ## + ## Oldest clients supported by this list: + ## * Firefox 27 + ## * Android 4.4.2 + ## * Chrome 31 + ## * Edge + ## * IE 11 on Windows 7 + ## * Java 8u31 + ## * OpenSSL 1.0.1 + ## * Opera 20 + ## * Safari 9 + +const CiphersOld* = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" + ## An OpenSSL-compatible list of secure ciphers for ``old`` compatibility + ## per Mozilla's recommendations. + ## + ## Oldest clients supported by this list: + ## * Firefox 1 + ## * Android 2.3 + ## * Chrome 1 + ## * Edge 12 + ## * IE8 on Windows XP + ## * Java 6 + ## * OpenSSL 0.9.8 + ## * Opera 5 + ## * Safari 1 + diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim new file mode 100644 index 000000000..6a4fd8f01 --- /dev/null +++ b/lib/pure/stats.nim @@ -0,0 +1,335 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Statistical analysis framework for performing +## basic statistical analysis of data. +## The data is analysed in a single pass, when it +## is pushed to a `RunningStat` or `RunningRegress` object. +## +## `RunningStat` calculates for a single data set +## - n (data count) +## - min (smallest value) +## - max (largest value) +## - sum +## - mean +## - variance +## - varianceS (sample variance) +## - standardDeviation +## - standardDeviationS (sample standard deviation) +## - skewness (the third statistical moment) +## - kurtosis (the fourth statistical moment) +## +## `RunningRegress` calculates for two sets of data +## - n (data count) +## - slope +## - intercept +## - correlation +## +## Procs are provided to calculate statistics on `openArray`s. +## +## However, if more than a single statistical calculation is required, it is more +## efficient to push the data once to a `RunningStat` object and then +## call the numerous statistical procs for the `RunningStat` object: + +runnableExamples: + from std/math import almostEqual + + template `~=`(a, b: float): bool = almostEqual(a, b) + + var statistics: RunningStat # must be var + statistics.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) + doAssert statistics.n == 8 + doAssert statistics.mean() ~= 2.0 + doAssert statistics.variance() ~= 1.5 + doAssert statistics.varianceS() ~= 1.714285714285715 + doAssert statistics.skewness() ~= 0.8164965809277261 + doAssert statistics.skewnessS() ~= 1.018350154434631 + doAssert statistics.kurtosis() ~= -1.0 + doAssert statistics.kurtosisS() ~= -0.7000000000000008 + +from std/math import FloatClass, sqrt, pow, round + +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + +{.push debugger: off.} # the user does not want to trace a part + # of the standard library! +{.push checks: off, line_dir: off, stack_trace: off.} + +type + RunningStat* = object ## An accumulator for statistical data. + n*: int ## amount of pushed data + min*, max*, sum*: float ## self-explaining + mom1, mom2, mom3, mom4: float ## statistical moments, mom1 is mean + + RunningRegress* = object ## An accumulator for regression calculations. + n*: int ## amount of pushed data + x_stats*: RunningStat ## stats for the first set of data + y_stats*: RunningStat ## stats for the second set of data + s_xy: float ## accumulated data for combined xy + +# ----------- RunningStat -------------------------- + +proc clear*(s: var RunningStat) = + ## Resets `s`. + s.n = 0 + s.min = 0.0 + s.max = 0.0 + s.sum = 0.0 + s.mom1 = 0.0 + s.mom2 = 0.0 + s.mom3 = 0.0 + s.mom4 = 0.0 + +proc push*(s: var RunningStat, x: float) = + ## Pushes a value `x` for processing. + if s.n == 0: + s.min = x + s.max = x + else: + if s.min > x: s.min = x + if s.max < x: s.max = x + inc(s.n) + # See Knuth TAOCP vol 2, 3rd edition, page 232 + s.sum += x + let n = toFloat(s.n) + let delta = x - s.mom1 + let delta_n = delta / toFloat(s.n) + let delta_n2 = delta_n * delta_n + let term1 = delta * delta_n * toFloat(s.n - 1) + s.mom4 += term1 * delta_n2 * (n*n - 3*n + 3) + + 6*delta_n2*s.mom2 - 4*delta_n*s.mom3 + s.mom3 += term1 * delta_n * (n - 2) - 3*delta_n*s.mom2 + s.mom2 += term1 + s.mom1 += delta_n + +proc push*(s: var RunningStat, x: int) = + ## Pushes a value `x` for processing. + ## + ## `x` is simply converted to `float` + ## and the other push operation is called. + s.push(toFloat(x)) + +proc push*(s: var RunningStat, x: openArray[float|int]) = + ## Pushes all values of `x` for processing. + ## + ## Int values of `x` are simply converted to `float` and + ## the other push operation is called. + for val in x: + s.push(val) + +proc mean*(s: RunningStat): float = + ## Computes the current mean of `s`. + result = s.mom1 + +proc variance*(s: RunningStat): float = + ## Computes the current population variance of `s`. + result = s.mom2 / toFloat(s.n) + +proc varianceS*(s: RunningStat): float = + ## Computes the current sample variance of `s`. + if s.n > 1: result = s.mom2 / toFloat(s.n - 1) + +proc standardDeviation*(s: RunningStat): float = + ## Computes the current population standard deviation of `s`. + result = sqrt(variance(s)) + +proc standardDeviationS*(s: RunningStat): float = + ## Computes the current sample standard deviation of `s`. + result = sqrt(varianceS(s)) + +proc skewness*(s: RunningStat): float = + ## Computes the current population skewness of `s`. + result = sqrt(toFloat(s.n)) * s.mom3 / pow(s.mom2, 1.5) + +proc skewnessS*(s: RunningStat): float = + ## Computes the current sample skewness of `s`. + let s2 = skewness(s) + result = sqrt(toFloat(s.n*(s.n-1)))*s2 / toFloat(s.n-2) + +proc kurtosis*(s: RunningStat): float = + ## Computes the current population kurtosis of `s`. + result = toFloat(s.n) * s.mom4 / (s.mom2 * s.mom2) - 3.0 + +proc kurtosisS*(s: RunningStat): float = + ## Computes the current sample kurtosis of `s`. + result = toFloat(s.n-1) / toFloat((s.n-2)*(s.n-3)) * + (toFloat(s.n+1)*kurtosis(s) + 6) + +proc `+`*(a, b: RunningStat): RunningStat = + ## Combines two `RunningStat`s. + ## + ## Useful when performing parallel analysis of data series + ## and needing to re-combine parallel result sets. + result.clear() + result.n = a.n + b.n + + let delta = b.mom1 - a.mom1 + let delta2 = delta*delta + let delta3 = delta*delta2 + let delta4 = delta2*delta2 + let n = toFloat(result.n) + + result.mom1 = (a.n.float*a.mom1 + b.n.float*b.mom1) / n + result.mom2 = a.mom2 + b.mom2 + delta2 * a.n.float * b.n.float / n + result.mom3 = a.mom3 + b.mom3 + + delta3 * a.n.float * b.n.float * (a.n.float - b.n.float)/(n*n); + result.mom3 += 3.0*delta * (a.n.float*b.mom2 - b.n.float*a.mom2) / n + result.mom4 = a.mom4 + b.mom4 + + delta4*a.n.float*b.n.float * toFloat(a.n*a.n - a.n*b.n + b.n*b.n) / + (n*n*n) + result.mom4 += 6.0*delta2 * (a.n.float*a.n.float*b.mom2 + b.n.float*b.n.float*a.mom2) / + (n*n) + + 4.0*delta*(a.n.float*b.mom3 - b.n.float*a.mom3) / n + result.max = max(a.max, b.max) + result.min = min(a.min, b.min) + +proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = + ## Adds the `RunningStat` `b` to `a`. + a = a + b + +proc `$`*(a: RunningStat): string = + ## Produces a string representation of the `RunningStat`. The exact + ## format is currently unspecified and subject to change. Currently + ## it contains: + ## + ## - the number of probes + ## - min, max values + ## - sum, mean and standard deviation. + result = "RunningStat(\n" + result.add " number of probes: " & $a.n & "\n" + result.add " max: " & $a.max & "\n" + result.add " min: " & $a.min & "\n" + result.add " sum: " & $a.sum & "\n" + result.add " mean: " & $a.mean & "\n" + result.add " std deviation: " & $a.standardDeviation & "\n" + result.add ")" + +# ---------------------- standalone array/seq stats --------------------- + +proc mean*[T](x: openArray[T]): float = + ## Computes the mean of `x`. + var rs: RunningStat + rs.push(x) + result = rs.mean() + +proc variance*[T](x: openArray[T]): float = + ## Computes the population variance of `x`. + var rs: RunningStat + rs.push(x) + result = rs.variance() + +proc varianceS*[T](x: openArray[T]): float = + ## Computes the sample variance of `x`. + var rs: RunningStat + rs.push(x) + result = rs.varianceS() + +proc standardDeviation*[T](x: openArray[T]): float = + ## Computes the population standard deviation of `x`. + var rs: RunningStat + rs.push(x) + result = rs.standardDeviation() + +proc standardDeviationS*[T](x: openArray[T]): float = + ## Computes the sample standard deviation of `x`. + var rs: RunningStat + rs.push(x) + result = rs.standardDeviationS() + +proc skewness*[T](x: openArray[T]): float = + ## Computes the population skewness of `x`. + var rs: RunningStat + rs.push(x) + result = rs.skewness() + +proc skewnessS*[T](x: openArray[T]): float = + ## Computes the sample skewness of `x`. + var rs: RunningStat + rs.push(x) + result = rs.skewnessS() + +proc kurtosis*[T](x: openArray[T]): float = + ## Computes the population kurtosis of `x`. + var rs: RunningStat + rs.push(x) + result = rs.kurtosis() + +proc kurtosisS*[T](x: openArray[T]): float = + ## Computes the sample kurtosis of `x`. + var rs: RunningStat + rs.push(x) + result = rs.kurtosisS() + +# ---------------------- Running Regression ----------------------------- + +proc clear*(r: var RunningRegress) = + ## Resets `r`. + r.x_stats.clear() + r.y_stats.clear() + r.s_xy = 0.0 + r.n = 0 + +proc push*(r: var RunningRegress, x, y: float) = + ## Pushes two values `x` and `y` for processing. + r.s_xy += (r.x_stats.mean() - x)*(r.y_stats.mean() - y) * + toFloat(r.n) / toFloat(r.n + 1) + r.x_stats.push(x) + r.y_stats.push(y) + inc(r.n) + +proc push*(r: var RunningRegress, x, y: int) {.inline.} = + ## Pushes two values `x` and `y` for processing. + ## + ## `x` and `y` are converted to `float` + ## and the other push operation is called. + r.push(toFloat(x), toFloat(y)) + +proc push*(r: var RunningRegress, x, y: openArray[float|int]) = + ## Pushes two sets of values `x` and `y` for processing. + assert(x.len == y.len) + for i in 0..<x.len: + r.push(x[i], y[i]) + +proc slope*(r: RunningRegress): float = + ## Computes the current slope of `r`. + let s_xx = r.x_stats.varianceS()*toFloat(r.n - 1) + result = r.s_xy / s_xx + +proc intercept*(r: RunningRegress): float = + ## Computes the current intercept of `r`. + result = r.y_stats.mean() - r.slope()*r.x_stats.mean() + +proc correlation*(r: RunningRegress): float = + ## Computes the current correlation of the two data + ## sets pushed into `r`. + let t = r.x_stats.standardDeviation() * r.y_stats.standardDeviation() + result = r.s_xy / (toFloat(r.n) * t) + +proc `+`*(a, b: RunningRegress): RunningRegress = + ## Combines two `RunningRegress` objects. + ## + ## Useful when performing parallel analysis of data series + ## and needing to re-combine parallel result sets + result.clear() + result.x_stats = a.x_stats + b.x_stats + result.y_stats = a.y_stats + b.y_stats + result.n = a.n + b.n + + let delta_x = b.x_stats.mean() - a.x_stats.mean() + let delta_y = b.y_stats.mean() - a.y_stats.mean() + result.s_xy = a.s_xy + b.s_xy + + toFloat(a.n*b.n)*delta_x*delta_y/toFloat(result.n) + +proc `+=`*(a: var RunningRegress, b: RunningRegress) = + ## Adds the `RunningRegress` `b` to `a`. + a = a + b + +{.pop.} +{.pop.} diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 38e91fee4..56f49d7b1 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -8,284 +8,1090 @@ # ## This module provides a stream interface and two implementations thereof: -## the `FileStream` and the `StringStream` which implement the stream -## interface for Nim file objects (`File`) and strings. Other modules -## may provide other implementations for this standard stream interface. +## the `FileStream <#FileStream>`_ and the `StringStream <#StringStream>`_ +## which implement the stream interface for Nim file objects (`File`) and +## strings. ## -## Examples: +## Other modules may provide other implementations for this standard +## stream interface. ## -## .. code-block:: Nim +## .. warning:: Due to the use of `pointer`, the `readData`, `peekData` and +## `writeData` interfaces are not available on the compile-time VM, and must +## be cast from a `ptr string` on the JS backend. However, `readDataStr` is +## available generally in place of `readData`. ## -## import streams -## var -## ss = newStringStream("""The first line -## the second line -## the third line""") -## line = "" -## while ss.readLine(line): -## echo line -## ss.close() +## Basic usage +## =========== ## -## var fs = newFileStream("somefile.txt", fmRead) -## if not isNil(fs): -## while fs.readLine(line): -## echo line -## fs.close() +## The basic flow of using this module is: +## +## 1. Open input stream +## 2. Read or write stream +## 3. Close stream +## +## StringStream example +## -------------------- +## +## ```Nim +## import std/streams +## +## var strm = newStringStream("""The first line +## the second line +## the third line""") +## +## var line = "" +## +## while strm.readLine(line): +## echo line +## +## # Output: +## # The first line +## # the second line +## # the third line +## +## strm.close() +## ``` +## +## FileStream example +## ------------------ +## +## Write file stream example: +## +## ```Nim +## import std/streams +## +## var strm = newFileStream("somefile.txt", fmWrite) +## var line = "" +## +## if not isNil(strm): +## strm.writeLine("The first line") +## strm.writeLine("the second line") +## strm.writeLine("the third line") +## strm.close() +## +## # Output (somefile.txt): +## # The first line +## # the second line +## # the third line +## ``` +## +## Read file stream example: +## +## ```Nim +## import std/streams +## +## var strm = newFileStream("somefile.txt", fmRead) +## var line = "" +## +## if not isNil(strm): +## while strm.readLine(line): +## echo line +## strm.close() +## +## # Output: +## # The first line +## # the second line +## # the third line +## ``` +## +## See also +## ======== +## * `asyncstreams module <asyncstreams.html>`_ +## * `io module <syncio.html>`_ for `FileMode enum <syncio.html#FileMode>`_ + +import std/private/since -include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/syncio + export FileMode -proc newEIO(msg: string): ref IOError = +proc newEIO(msg: string): owned(ref IOError) = new(result) result.msg = msg type Stream* = ref StreamObj - StreamObj* = object of RootObj ## Stream interface that supports - ## writing or reading. Note that these fields - ## here shouldn't be used directly. They are - ## accessible so that a stream implementation - ## can override them. - closeImpl*: proc (s: Stream) {.nimcall, tags: [], gcsafe.} - atEndImpl*: proc (s: Stream): bool {.nimcall, tags: [], gcsafe.} - setPositionImpl*: proc (s: Stream, pos: int) {.nimcall, tags: [], gcsafe.} - getPositionImpl*: proc (s: Stream): int {.nimcall, tags: [], gcsafe.} - readDataImpl*: proc (s: Stream, buffer: pointer, - bufLen: int): int {.nimcall, tags: [ReadIOEffect], gcsafe.} - peekDataImpl*: proc (s: Stream, buffer: pointer, - bufLen: int): int {.nimcall, tags: [ReadIOEffect], gcsafe.} - writeDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int) {.nimcall, - tags: [WriteIOEffect], gcsafe.} - flushImpl*: proc (s: Stream) {.nimcall, tags: [WriteIOEffect], gcsafe.} - -{.deprecated: [PStream: Stream, TStream: StreamObj].} + ## All procedures of this module use this type. + ## Procedures don't directly use `StreamObj <#StreamObj>`_. + StreamObj* = object of RootObj + ## Stream interface that supports writing or reading. + ## + ## **Note:** + ## * That these fields here shouldn't be used directly. + ## They are accessible so that a stream implementation can override them. + closeImpl*: proc (s: Stream) + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} + atEndImpl*: proc (s: Stream): bool + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + setPositionImpl*: proc (s: Stream, pos: int) + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + getPositionImpl*: proc (s: Stream): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} + + readDataStrImpl*: proc (s: Stream, buffer: var string, slice: Slice[int]): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} + + readLineImpl*: proc(s: Stream, line: var string): bool + {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} + + readDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} + peekDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int + {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} + writeDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int) + {.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + + flushImpl*: proc (s: Stream) + {.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.} proc flush*(s: Stream) = - ## flushes the buffers that the stream `s` might use. + ## Flushes the buffers that the stream `s` might use. + ## + ## This procedure causes any unwritten data for that stream to be delivered + ## to the host environment to be written to the file. + ## + ## See also: + ## * `close proc <#close,Stream>`_ + runnableExamples: + from std/os import removeFile + + var strm = newFileStream("somefile.txt", fmWrite) + + doAssert "Before write:" & readFile("somefile.txt") == "Before write:" + strm.write("hello") + doAssert "After write:" & readFile("somefile.txt") == "After write:" + + strm.flush() + doAssert "After flush:" & readFile("somefile.txt") == "After flush:hello" + strm.write("HELLO") + strm.flush() + doAssert "After flush:" & readFile("somefile.txt") == "After flush:helloHELLO" + + strm.close() + doAssert "After close:" & readFile("somefile.txt") == "After close:helloHELLO" + removeFile("somefile.txt") + if not isNil(s.flushImpl): s.flushImpl(s) proc close*(s: Stream) = - ## closes the stream `s`. - if not isNil(s.closeImpl): s.closeImpl(s) - -proc close*(s, unused: Stream) {.deprecated.} = - ## closes the stream `s`. - s.closeImpl(s) + ## Closes the stream `s`. + ## + ## See also: + ## * `flush proc <#flush,Stream>`_ + runnableExamples: + block: + let strm = newStringStream("The first line\nthe second line\nthe third line") + ## do something... + strm.close() + + block: + let strm = newFileStream("amissingfile.txt") + # deferring works even if newFileStream fails + defer: strm.close() + if not isNil(strm): + ## do something... + + if not isNil(s) and not isNil(s.closeImpl): + s.closeImpl(s) proc atEnd*(s: Stream): bool = - ## checks if more data can be read from `f`. Returns true if all data has + ## Checks if more data can be read from `s`. Returns ``true`` if all data has ## been read. - result = s.atEndImpl(s) + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + var line = "" + doAssert strm.atEnd() == false + while strm.readLine(line): + discard + doAssert strm.atEnd() == true + strm.close() -proc atEnd*(s, unused: Stream): bool {.deprecated.} = - ## checks if more data can be read from `f`. Returns true if all data has - ## been read. result = s.atEndImpl(s) proc setPosition*(s: Stream, pos: int) = - ## sets the position `pos` of the stream `s`. - s.setPositionImpl(s, pos) + ## Sets the position `pos` of the stream `s`. + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + strm.setPosition(4) + doAssert strm.readLine() == "first line" + strm.setPosition(0) + doAssert strm.readLine() == "The first line" + strm.close() -proc setPosition*(s, unused: Stream, pos: int) {.deprecated.} = - ## sets the position `pos` of the stream `s`. s.setPositionImpl(s, pos) proc getPosition*(s: Stream): int = - ## retrieves the current position in the stream `s`. - result = s.getPositionImpl(s) + ## Retrieves the current position in the stream `s`. + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + doAssert strm.getPosition() == 0 + discard strm.readLine() + doAssert strm.getPosition() == 15 + strm.close() -proc getPosition*(s, unused: Stream): int {.deprecated.} = - ## retrieves the current position in the stream `s`. result = s.getPositionImpl(s) proc readData*(s: Stream, buffer: pointer, bufLen: int): int = - ## low level proc that reads data into an untyped `buffer` of `bufLen` size. - result = s.readDataImpl(s, buffer, bufLen) - -proc readAll*(s: Stream): string = - ## Reads all available data. - const bufferSize = 1000 - result = newString(bufferSize) - var r = 0 - while true: - let readBytes = readData(s, addr(result[r]), bufferSize) - if readBytes < bufferSize: - setLen(result, r+readBytes) - break - inc r, bufferSize - setLen(result, r+bufferSize) + ## Low level proc that reads data into an untyped `buffer` of `bufLen` size. + ## + ## **JS note:** `buffer` is treated as a ``ptr string`` and written to between + ## ``0..<bufLen``. + runnableExamples: + var strm = newStringStream("abcde") + var buffer: array[6, char] + doAssert strm.readData(addr(buffer), 1024) == 5 + doAssert buffer == ['a', 'b', 'c', 'd', 'e', '\x00'] + doAssert strm.atEnd() == true + strm.close() -proc readData*(s, unused: Stream, buffer: pointer, - bufLen: int): int {.deprecated.} = - ## low level proc that reads data into an untyped `buffer` of `bufLen` size. result = s.readDataImpl(s, buffer, bufLen) +proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = + ## Low level proc that reads data into a string ``buffer`` at ``slice``. + runnableExamples: + var strm = newStringStream("abcde") + var buffer = "12345" + doAssert strm.readDataStr(buffer, 0..3) == 4 + doAssert buffer == "abcd5" + strm.close() + + if s.readDataStrImpl != nil: + result = s.readDataStrImpl(s, buffer, slice) + else: + # fallback + when declared(prepareMutation): + # buffer might potentially be a CoW literal with ARC + prepareMutation(buffer) + result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) + +template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = + when nimvm: + block: + caseJsOrVm + else: + block: + when defined(js) or defined(nimscript): + # nimscript has to be here to avoid semantic checking of caseElse + caseJsOrVm + else: + caseElse + +when (NimMajor, NimMinor) >= (1, 3) or not defined(js): + proc readAll*(s: Stream): string = + ## Reads all available data. + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + doAssert strm.readAll() == "The first line\nthe second line\nthe third line" + doAssert strm.atEnd() == true + strm.close() + + const bufferSize = 1024 + jsOrVmBlock: + var buffer2: string + buffer2.setLen(bufferSize) + while true: + let readBytes = readDataStr(s, buffer2, 0..<bufferSize) + if readBytes == 0: + break + let prevLen = result.len + result.setLen(prevLen + readBytes) + result[prevLen..<prevLen+readBytes] = buffer2[0..<readBytes] + if readBytes < bufferSize: + break + do: # not JS or VM + var buffer {.noinit.}: array[bufferSize, char] + while true: + let readBytes = readData(s, addr(buffer[0]), bufferSize) + if readBytes == 0: + break + let prevLen = result.len + result.setLen(prevLen + readBytes) + copyMem(addr(result[prevLen]), addr(buffer[0]), readBytes) + if readBytes < bufferSize: + break + proc peekData*(s: Stream, buffer: pointer, bufLen: int): int = - ## low level proc that reads data into an untyped `buffer` of `bufLen` size - ## without moving stream position + ## Low level proc that reads data into an untyped `buffer` of `bufLen` size + ## without moving stream position. + ## + ## **JS note:** `buffer` is treated as a ``ptr string`` and written to between + ## ``0..<bufLen``. + runnableExamples: + var strm = newStringStream("abcde") + var buffer: array[6, char] + doAssert strm.peekData(addr(buffer), 1024) == 5 + doAssert buffer == ['a', 'b', 'c', 'd', 'e', '\x00'] + doAssert strm.atEnd() == false + strm.close() + result = s.peekDataImpl(s, buffer, bufLen) proc writeData*(s: Stream, buffer: pointer, bufLen: int) = - ## low level proc that writes an untyped `buffer` of `bufLen` size + ## Low level proc that writes an untyped `buffer` of `bufLen` size ## to the stream `s`. - s.writeDataImpl(s, buffer, bufLen) + ## + ## **JS note:** `buffer` is treated as a ``ptr string`` and read between + ## ``0..<bufLen``. + runnableExamples: + ## writeData + var strm = newStringStream("") + var buffer = ['a', 'b', 'c', 'd', 'e'] + strm.writeData(addr(buffer), sizeof(buffer)) + doAssert strm.atEnd() == true + ## readData + strm.setPosition(0) + var buffer2: array[6, char] + doAssert strm.readData(addr(buffer2), sizeof(buffer2)) == 5 + doAssert buffer2 == ['a', 'b', 'c', 'd', 'e', '\x00'] + strm.close() -proc writeData*(s, unused: Stream, buffer: pointer, - bufLen: int) {.deprecated.} = - ## low level proc that writes an untyped `buffer` of `bufLen` size - ## to the stream `s`. s.writeDataImpl(s, buffer, bufLen) proc write*[T](s: Stream, x: T) = - ## generic write procedure. Writes `x` to the stream `s`. Implementation: + ## Generic write procedure. Writes `x` to the stream `s`. Implementation: ## - ## .. code-block:: Nim + ## **Note:** Not available for JS backend. Use `write(Stream, string) + ## <#write,Stream,string>`_ for now. ## - ## s.writeData(s, addr(x), sizeof(x)) - var y: T - shallowCopy(y, x) - writeData(s, addr(y), sizeof(y)) + ## ```Nim + ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ``` + runnableExamples: + var strm = newStringStream("") + strm.write("abcde") + strm.setPosition(0) + doAssert strm.readAll() == "abcde" + strm.close() + + writeData(s, unsafeAddr(x), sizeof(x)) proc write*(s: Stream, x: string) = - ## writes the string `x` to the the stream `s`. No length field or + ## Writes the string `x` to the stream `s`. No length field or ## terminating zero is written. - writeData(s, cstring(x), x.len) + runnableExamples: + var strm = newStringStream("") + strm.write("THE FIRST LINE") + strm.setPosition(0) + doAssert strm.readLine() == "THE FIRST LINE" + strm.close() + + when nimvm: + writeData(s, cstring(x), x.len) + else: + if x.len > 0: + when defined(js): + var x = x + writeData(s, addr(x), x.len) + else: + writeData(s, cstring(x), x.len) + +proc write*(s: Stream, args: varargs[string, `$`]) = + ## Writes one or more strings to the the stream. No length fields or + ## terminating zeros are written. + runnableExamples: + var strm = newStringStream("") + strm.write(1, 2, 3, 4) + strm.setPosition(0) + doAssert strm.readLine() == "1234" + strm.close() -proc writeLn*(s: Stream, args: varargs[string, `$`]) {.deprecated.} = - ## **Deprecated since version 0.11.4:** Use **writeLine** instead. for str in args: write(s, str) - write(s, "\n") proc writeLine*(s: Stream, args: varargs[string, `$`]) = - ## writes one or more strings to the the stream `s` followed + ## Writes one or more strings to the the stream `s` followed ## by a new line. No length field or terminating zero is written. + runnableExamples: + var strm = newStringStream("") + strm.writeLine(1, 2) + strm.writeLine(3, 4) + strm.setPosition(0) + doAssert strm.readAll() == "12\n34\n" + strm.close() + for str in args: write(s, str) write(s, "\n") -proc read[T](s: Stream, result: var T) = - ## generic read procedure. Reads `result` from the stream `s`. +proc read*[T](s: Stream, result: var T) = + ## Generic read procedure. Reads `result` from the stream `s`. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream("012") + ## readInt + var i: int8 + strm.read(i) + doAssert i == 48 + ## readData + var buffer: array[2, char] + strm.read(buffer) + doAssert buffer == ['1', '2'] + strm.close() + if readData(s, addr(result), sizeof(T)) != sizeof(T): raise newEIO("cannot read from stream") -proc peek[T](s: Stream, result: var T) = - ## generic peek procedure. Peeks `result` from the stream `s`. +proc peek*[T](s: Stream, result: var T) = + ## Generic peek procedure. Peeks `result` from the stream `s`. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream("012") + ## peekInt + var i: int8 + strm.peek(i) + doAssert i == 48 + ## peekData + var buffer: array[2, char] + strm.peek(buffer) + doAssert buffer == ['0', '1'] + strm.close() + if peekData(s, addr(result), sizeof(T)) != sizeof(T): raise newEIO("cannot read from stream") proc readChar*(s: Stream): char = - ## reads a char from the stream `s`. Raises `EIO` if an error occurred. - ## Returns '\0' as an EOF marker. - if readData(s, addr(result), sizeof(result)) != 1: result = '\0' + ## Reads a char from the stream `s`. + ## + ## Raises `IOError` if an error occurred. + ## Returns '\\0' as an EOF marker. + runnableExamples: + var strm = newStringStream("12\n3") + doAssert strm.readChar() == '1' + doAssert strm.readChar() == '2' + doAssert strm.readChar() == '\n' + doAssert strm.readChar() == '3' + doAssert strm.readChar() == '\x00' + strm.close() + + jsOrVmBlock: + var str = " " + if readDataStr(s, str, 0..0) != 1: result = '\0' + else: result = str[0] + do: + if readData(s, addr(result), sizeof(result)) != 1: result = '\0' proc peekChar*(s: Stream): char = - ## peeks a char from the stream `s`. Raises `EIO` if an error occurred. - ## Returns '\0' as an EOF marker. - if peekData(s, addr(result), sizeof(result)) != 1: result = '\0' + ## Peeks a char from the stream `s`. Raises `IOError` if an error occurred. + ## Returns '\\0' as an EOF marker. + runnableExamples: + var strm = newStringStream("12\n3") + doAssert strm.peekChar() == '1' + doAssert strm.peekChar() == '1' + discard strm.readAll() + doAssert strm.peekChar() == '\x00' + strm.close() + + when defined(js): + var str = " " + if peekData(s, addr(str), sizeof(result)) != 1: result = '\0' + else: result = str[0] + else: + if peekData(s, addr(result), sizeof(result)) != 1: result = '\0' proc readBool*(s: Stream): bool = - ## reads a bool from the stream `s`. Raises `EIO` if an error occurred. - read(s, result) + ## Reads a bool from the stream `s`. + ## + ## A bool is one byte long and it is `true` for every non-zero + ## (`0000_0000`) value. + ## Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(true) + strm.write(false) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readBool() == true + doAssert strm.readBool() == false + doAssertRaises(IOError): discard strm.readBool() + strm.close() + + var t: byte + read(s, t) + result = t != 0.byte proc peekBool*(s: Stream): bool = - ## peeks a bool from the stream `s`. Raises `EIO` if an error occured. - peek(s, result) + ## Peeks a bool from the stream `s`. + ## + ## A bool is one byte long and it is `true` for every non-zero + ## (`0000_0000`) value. + ## Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(true) + strm.write(false) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekBool() == true + ## not false + doAssert strm.peekBool() == true + doAssert strm.readBool() == true + doAssert strm.peekBool() == false + strm.close() + + var t: byte + peek(s, t) + result = t != 0.byte proc readInt8*(s: Stream): int8 = - ## reads an int8 from the stream `s`. Raises `EIO` if an error occurred. + ## Reads an int8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i8) + strm.write(2'i8) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readInt8() == 1'i8 + doAssert strm.readInt8() == 2'i8 + doAssertRaises(IOError): discard strm.readInt8() + strm.close() + read(s, result) proc peekInt8*(s: Stream): int8 = - ## peeks an int8 from the stream `s`. Raises `EIO` if an error occurred. + ## Peeks an int8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i8) + strm.write(2'i8) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekInt8() == 1'i8 + ## not 2'i8 + doAssert strm.peekInt8() == 1'i8 + doAssert strm.readInt8() == 1'i8 + doAssert strm.peekInt8() == 2'i8 + strm.close() + peek(s, result) proc readInt16*(s: Stream): int16 = - ## reads an int16 from the stream `s`. Raises `EIO` if an error occurred. + ## Reads an int16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i16) + strm.write(2'i16) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readInt16() == 1'i16 + doAssert strm.readInt16() == 2'i16 + doAssertRaises(IOError): discard strm.readInt16() + strm.close() + read(s, result) proc peekInt16*(s: Stream): int16 = - ## peeks an int16 from the stream `s`. Raises `EIO` if an error occurred. + ## Peeks an int16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i16) + strm.write(2'i16) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekInt16() == 1'i16 + ## not 2'i16 + doAssert strm.peekInt16() == 1'i16 + doAssert strm.readInt16() == 1'i16 + doAssert strm.peekInt16() == 2'i16 + strm.close() + peek(s, result) proc readInt32*(s: Stream): int32 = - ## reads an int32 from the stream `s`. Raises `EIO` if an error occurred. + ## Reads an int32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i32) + strm.write(2'i32) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readInt32() == 1'i32 + doAssert strm.readInt32() == 2'i32 + doAssertRaises(IOError): discard strm.readInt32() + strm.close() + read(s, result) proc peekInt32*(s: Stream): int32 = - ## peeks an int32 from the stream `s`. Raises `EIO` if an error occurred. + ## Peeks an int32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i32) + strm.write(2'i32) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekInt32() == 1'i32 + ## not 2'i32 + doAssert strm.peekInt32() == 1'i32 + doAssert strm.readInt32() == 1'i32 + doAssert strm.peekInt32() == 2'i32 + strm.close() + peek(s, result) proc readInt64*(s: Stream): int64 = - ## reads an int64 from the stream `s`. Raises `EIO` if an error occurred. + ## Reads an int64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i64) + strm.write(2'i64) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readInt64() == 1'i64 + doAssert strm.readInt64() == 2'i64 + doAssertRaises(IOError): discard strm.readInt64() + strm.close() + read(s, result) proc peekInt64*(s: Stream): int64 = - ## peeks an int64 from the stream `s`. Raises `EIO` if an error occurred. + ## Peeks an int64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'i64) + strm.write(2'i64) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekInt64() == 1'i64 + ## not 2'i64 + doAssert strm.peekInt64() == 1'i64 + doAssert strm.readInt64() == 1'i64 + doAssert strm.peekInt64() == 2'i64 + strm.close() + + peek(s, result) + +proc readUint8*(s: Stream): uint8 = + ## Reads an uint8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u8) + strm.write(2'u8) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readUint8() == 1'u8 + doAssert strm.readUint8() == 2'u8 + doAssertRaises(IOError): discard strm.readUint8() + strm.close() + + read(s, result) + +proc peekUint8*(s: Stream): uint8 = + ## Peeks an uint8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u8) + strm.write(2'u8) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekUint8() == 1'u8 + ## not 2'u8 + doAssert strm.peekUint8() == 1'u8 + doAssert strm.readUint8() == 1'u8 + doAssert strm.peekUint8() == 2'u8 + strm.close() + + peek(s, result) + +proc readUint16*(s: Stream): uint16 = + ## Reads an uint16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u16) + strm.write(2'u16) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readUint16() == 1'u16 + doAssert strm.readUint16() == 2'u16 + doAssertRaises(IOError): discard strm.readUint16() + strm.close() + + read(s, result) + +proc peekUint16*(s: Stream): uint16 = + ## Peeks an uint16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u16) + strm.write(2'u16) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekUint16() == 1'u16 + ## not 2'u16 + doAssert strm.peekUint16() == 1'u16 + doAssert strm.readUint16() == 1'u16 + doAssert strm.peekUint16() == 2'u16 + strm.close() + + peek(s, result) + +proc readUint32*(s: Stream): uint32 = + ## Reads an uint32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u32) + strm.write(2'u32) + strm.flush() + strm.setPosition(0) + + ## get data + doAssert strm.readUint32() == 1'u32 + doAssert strm.readUint32() == 2'u32 + doAssertRaises(IOError): discard strm.readUint32() + strm.close() + + read(s, result) + +proc peekUint32*(s: Stream): uint32 = + ## Peeks an uint32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u32) + strm.write(2'u32) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekUint32() == 1'u32 + ## not 2'u32 + doAssert strm.peekUint32() == 1'u32 + doAssert strm.readUint32() == 1'u32 + doAssert strm.peekUint32() == 2'u32 + strm.close() + + peek(s, result) + +proc readUint64*(s: Stream): uint64 = + ## Reads an uint64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u64) + strm.write(2'u64) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readUint64() == 1'u64 + doAssert strm.readUint64() == 2'u64 + doAssertRaises(IOError): discard strm.readUint64() + strm.close() + + read(s, result) + +proc peekUint64*(s: Stream): uint64 = + ## Peeks an uint64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'u64) + strm.write(2'u64) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekUint64() == 1'u64 + ## not 2'u64 + doAssert strm.peekUint64() == 1'u64 + doAssert strm.readUint64() == 1'u64 + doAssert strm.peekUint64() == 2'u64 + strm.close() + peek(s, result) proc readFloat32*(s: Stream): float32 = - ## reads a float32 from the stream `s`. Raises `EIO` if an error occurred. + ## Reads a float32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'f32) + strm.write(2'f32) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readFloat32() == 1'f32 + doAssert strm.readFloat32() == 2'f32 + doAssertRaises(IOError): discard strm.readFloat32() + strm.close() + read(s, result) proc peekFloat32*(s: Stream): float32 = - ## peeks a float32 from the stream `s`. Raises `EIO` if an error occurred. + ## Peeks a float32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'f32) + strm.write(2'f32) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekFloat32() == 1'f32 + ## not 2'f32 + doAssert strm.peekFloat32() == 1'f32 + doAssert strm.readFloat32() == 1'f32 + doAssert strm.peekFloat32() == 2'f32 + strm.close() + peek(s, result) proc readFloat64*(s: Stream): float64 = - ## reads a float64 from the stream `s`. Raises `EIO` if an error occurred. + ## Reads a float64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'f64) + strm.write(2'f64) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.readFloat64() == 1'f64 + doAssert strm.readFloat64() == 2'f64 + doAssertRaises(IOError): discard strm.readFloat64() + strm.close() + read(s, result) proc peekFloat64*(s: Stream): float64 = - ## peeks a float64 from the stream `s`. Raises `EIO` if an error occurred. + ## Peeks a float64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. + runnableExamples: + var strm = newStringStream() + ## setup for reading data + strm.write(1'f64) + strm.write(2'f64) + strm.flush() + strm.setPosition(0) + ## get data + doAssert strm.peekFloat64() == 1'f64 + ## not 2'f64 + doAssert strm.peekFloat64() == 1'f64 + doAssert strm.readFloat64() == 1'f64 + doAssert strm.peekFloat64() == 2'f64 + strm.close() + peek(s, result) -proc readStr*(s: Stream, length: int): TaintedString = - ## reads a string of length `length` from the stream `s`. Raises `EIO` if +proc readStrPrivate(s: Stream, length: int, str: var string) = + if length > len(str): setLen(str, length) + var L: int + when nimvm: + L = readDataStr(s, str, 0..length-1) + else: + when defined(js): + L = readData(s, addr(str), length) + else: + L = readData(s, cstring(str), length) + if L != len(str): setLen(str, L) + +proc readStr*(s: Stream, length: int, str: var string) {.since: (1, 3).} = + ## Reads a string of length `length` from the stream `s`. Raises `IOError` if + ## an error occurred. + readStrPrivate(s, length, str) + +proc readStr*(s: Stream, length: int): string = + ## Reads a string of length `length` from the stream `s`. Raises `IOError` if ## an error occurred. - result = newString(length).TaintedString - var L = readData(s, addr(string(result)[0]), length) - if L != length: setLen(result.string, L) + runnableExamples: + var strm = newStringStream("abcde") + doAssert strm.readStr(2) == "ab" + doAssert strm.readStr(2) == "cd" + doAssert strm.readStr(2) == "e" + doAssert strm.readStr(2) == "" + strm.close() + result = newString(length) + readStrPrivate(s, length, result) + +proc peekStrPrivate(s: Stream, length: int, str: var string) = + if length > len(str): setLen(str, length) + when defined(js): + let L = peekData(s, addr(str), length) + else: + let L = peekData(s, cstring(str), length) + if L != len(str): setLen(str, L) -proc peekStr*(s: Stream, length: int): TaintedString = - ## peeks a string of length `length` from the stream `s`. Raises `EIO` if +proc peekStr*(s: Stream, length: int, str: var string) {.since: (1, 3).} = + ## Peeks a string of length `length` from the stream `s`. Raises `IOError` if ## an error occurred. - result = newString(length).TaintedString - var L = peekData(s, addr(string(result)[0]), length) - if L != length: setLen(result.string, L) + peekStrPrivate(s, length, str) -proc readLine*(s: Stream, line: var TaintedString): bool = - ## reads a line of text from the stream `s` into `line`. `line` must not be +proc peekStr*(s: Stream, length: int): string = + ## Peeks a string of length `length` from the stream `s`. Raises `IOError` if + ## an error occurred. + runnableExamples: + var strm = newStringStream("abcde") + doAssert strm.peekStr(2) == "ab" + ## not "cd + doAssert strm.peekStr(2) == "ab" + doAssert strm.readStr(2) == "ab" + doAssert strm.peekStr(2) == "cd" + strm.close() + result = newString(length) + peekStrPrivate(s, length, result) + +proc readLine*(s: Stream, line: var string): bool = + ## Reads a line of text from the stream `s` into `line`. `line` must not be ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ``CR``, ``LF`` or - ## ``CRLF``. The newline character(s) are not part of the returned string. + ## + ## A line of text may be delimited by ``LF`` or ``CRLF``. + ## The newline character(s) are not part of the returned string. ## Returns ``false`` if the end of the file has been reached, ``true`` ## otherwise. If ``false`` is returned `line` contains no new data. - line.string.setLen(0) - while true: - var c = readChar(s) - if c == '\c': - c = readChar(s) - break - elif c == '\L': break - elif c == '\0': - if line.len > 0: break - else: return false - line.string.add(c) - result = true - -proc peekLine*(s: Stream, line: var TaintedString): bool = - ## peeks a line of text from the stream `s` into `line`. `line` must not be + ## + ## See also: + ## * `readLine(Stream) proc <#readLine,Stream>`_ + ## * `peekLine(Stream) proc <#peekLine,Stream>`_ + ## * `peekLine(Stream, string) proc <#peekLine,Stream,string>`_ + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + var line = "" + doAssert strm.readLine(line) == true + doAssert line == "The first line" + doAssert strm.readLine(line) == true + doAssert line == "the second line" + doAssert strm.readLine(line) == true + doAssert line == "the third line" + doAssert strm.readLine(line) == false + doAssert line == "" + strm.close() + + if s.readLineImpl != nil: + result = s.readLineImpl(s, line) + else: + # fallback + line.setLen(0) + while true: + var c = readChar(s) + if c == '\c': + c = readChar(s) + break + elif c == '\L': break + elif c == '\0': + if line.len > 0: break + else: return false + line.add(c) + result = true + +proc peekLine*(s: Stream, line: var string): bool = + ## Peeks a line of text from the stream `s` into `line`. `line` must not be ## ``nil``! May throw an IO exception. + ## ## A line of text may be delimited by ``CR``, ``LF`` or ## ``CRLF``. The newline character(s) are not part of the returned string. ## Returns ``false`` if the end of the file has been reached, ``true`` ## otherwise. If ``false`` is returned `line` contains no new data. + ## + ## See also: + ## * `readLine(Stream) proc <#readLine,Stream>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ + ## * `peekLine(Stream) proc <#peekLine,Stream>`_ + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + var line = "" + doAssert strm.peekLine(line) == true + doAssert line == "The first line" + doAssert strm.peekLine(line) == true + ## not "the second line" + doAssert line == "The first line" + doAssert strm.readLine(line) == true + doAssert line == "The first line" + doAssert strm.peekLine(line) == true + doAssert line == "the second line" + strm.close() + let pos = getPosition(s) defer: setPosition(s, pos) result = readLine(s, line) -proc readLine*(s: Stream): TaintedString = - ## Reads a line from a stream `s`. Note: This is not very efficient. Raises - ## `EIO` if an error occurred. - result = TaintedString"" +proc readLine*(s: Stream): string = + ## Reads a line from a stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** This is not very efficient. + ## + ## See also: + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ + ## * `peekLine(Stream) proc <#peekLine,Stream>`_ + ## * `peekLine(Stream, string) proc <#peekLine,Stream,string>`_ + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + doAssert strm.readLine() == "The first line" + doAssert strm.readLine() == "the second line" + doAssert strm.readLine() == "the third line" + doAssertRaises(IOError): discard strm.readLine() + strm.close() + + result = "" + if s.atEnd: + raise newEIO("cannot read from stream") while true: var c = readChar(s) if c == '\c': @@ -294,136 +1100,386 @@ proc readLine*(s: Stream): TaintedString = if c == '\L' or c == '\0': break else: - result.string.add(c) + result.add(c) + +proc peekLine*(s: Stream): string = + ## Peeks a line from a stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** This is not very efficient. + ## + ## See also: + ## * `readLine(Stream) proc <#readLine,Stream>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ + ## * `peekLine(Stream, string) proc <#peekLine,Stream,string>`_ + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + doAssert strm.peekLine() == "The first line" + ## not "the second line" + doAssert strm.peekLine() == "The first line" + doAssert strm.readLine() == "The first line" + doAssert strm.peekLine() == "the second line" + strm.close() -proc peekLine*(s: Stream): TaintedString = - ## Peeks a line from a stream `s`. Note: This is not very efficient. Raises - ## `EIO` if an error occurred. let pos = getPosition(s) defer: setPosition(s, pos) result = readLine(s) +iterator lines*(s: Stream): string = + ## Iterates over every line in the stream. + ## The iteration is based on ``readLine``. + ## + ## See also: + ## * `readLine(Stream) proc <#readLine,Stream>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + var lines: seq[string] + for line in strm.lines(): + lines.add line + doAssert lines == @["The first line", "the second line", "the third line"] + strm.close() + + var line: string + while s.readLine(line): + yield line + type - StringStream* = ref StringStreamObj ## a stream that encapsulates a string + StringStream* = ref StringStreamObj + ## A stream that encapsulates a string. StringStreamObj* = object of StreamObj - data*: string + ## A string stream object. + data*: string ## A string data. + ## This is updated when called `writeLine` etc. pos: int -{.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} - -proc ssAtEnd(s: Stream): bool = - var s = StringStream(s) - return s.pos >= s.data.len - -proc ssSetPosition(s: Stream, pos: int) = - var s = StringStream(s) - s.pos = clamp(pos, 0, s.data.len) - -proc ssGetPosition(s: Stream): int = - var s = StringStream(s) - return s.pos - -proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = - var s = StringStream(s) - result = min(bufLen, s.data.len - s.pos) - if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) - inc(s.pos, result) - -proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int = - var s = StringStream(s) - result = min(bufLen, s.data.len - s.pos) - if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) - -proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = - var s = StringStream(s) - if bufLen <= 0: - return - if s.pos + bufLen > s.data.len: - setLen(s.data, s.pos + bufLen) - copyMem(addr(s.data[s.pos]), buffer, bufLen) - inc(s.pos, bufLen) - -proc ssClose(s: Stream) = - var s = StringStream(s) - s.data = nil - -proc newStringStream*(s: string = ""): StringStream = - ## creates a new stream from the string `s`. - new(result) - result.data = s - result.pos = 0 - result.closeImpl = ssClose - result.atEndImpl = ssAtEnd - result.setPositionImpl = ssSetPosition - result.getPositionImpl = ssGetPosition - result.readDataImpl = ssReadData - result.peekDataImpl = ssPeekData - result.writeDataImpl = ssWriteData - -when not defined(js): +when (NimMajor, NimMinor) < (1, 3) and defined(js): + proc ssAtEnd(s: Stream): bool {.compileTime.} = + var s = StringStream(s) + return s.pos >= s.data.len + + proc ssSetPosition(s: Stream, pos: int) {.compileTime.} = + var s = StringStream(s) + s.pos = clamp(pos, 0, s.data.len) + + proc ssGetPosition(s: Stream): int {.compileTime.} = + var s = StringStream(s) + return s.pos + + proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int {.compileTime.} = + var s = StringStream(s) + result = min(slice.b + 1 - slice.a, s.data.len - s.pos) + if result > 0: + buffer[slice.a..<slice.a+result] = s.data[s.pos..<s.pos+result] + inc(s.pos, result) + else: + result = 0 + + proc ssClose(s: Stream) {.compileTime.} = + var s = StringStream(s) + s.data = "" + + proc newStringStream*(s: string = ""): owned StringStream {.compileTime.} = + new(result) + result.data = s + result.pos = 0 + result.closeImpl = ssClose + result.atEndImpl = ssAtEnd + result.setPositionImpl = ssSetPosition + result.getPositionImpl = ssGetPosition + result.readDataStrImpl = ssReadDataStr + + proc readAll*(s: Stream): string {.compileTime.} = + const bufferSize = 1024 + var bufferr: string + bufferr.setLen(bufferSize) + while true: + let readBytes = readDataStr(s, bufferr, 0..<bufferSize) + if readBytes == 0: + break + let prevLen = result.len + result.setLen(prevLen + readBytes) + result[prevLen..<prevLen+readBytes] = bufferr[0..<readBytes] + if readBytes < bufferSize: + break + +else: # after 1.3 or JS not defined + proc ssAtEnd(s: Stream): bool = + var s = StringStream(s) + return s.pos >= s.data.len + + proc ssSetPosition(s: Stream, pos: int) = + var s = StringStream(s) + s.pos = clamp(pos, 0, s.data.len) + + proc ssGetPosition(s: Stream): int = + var s = StringStream(s) + return s.pos + + proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = + var s = StringStream(s) + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(buffer) # buffer might potentially be a CoW literal with ARC + result = min(slice.b + 1 - slice.a, s.data.len - s.pos) + if result > 0: + jsOrVmBlock: + buffer[slice.a..<slice.a+result] = s.data[s.pos..<s.pos+result] + do: + copyMem(unsafeAddr buffer[slice.a], addr s.data[s.pos], result) + inc(s.pos, result) + else: + result = 0 + + proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = + var s = StringStream(s) + result = min(bufLen, s.data.len - s.pos) + if result > 0: + when defined(js): + try: + cast[ptr string](buffer)[][0..<result] = s.data[s.pos..<s.pos+result] + except: + raise newException(Defect, "could not read string stream, " & + "did you use a non-string buffer pointer?", getCurrentException()) + elif not defined(nimscript): + copyMem(buffer, addr(s.data[s.pos]), result) + inc(s.pos, result) + else: + result = 0 + + proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int = + var s = StringStream(s) + result = min(bufLen, s.data.len - s.pos) + if result > 0: + when defined(js): + try: + cast[ptr string](buffer)[][0..<result] = s.data[s.pos..<s.pos+result] + except: + raise newException(Defect, "could not peek string stream, " & + "did you use a non-string buffer pointer?", getCurrentException()) + elif not defined(nimscript): + copyMem(buffer, addr(s.data[s.pos]), result) + else: + result = 0 + + proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = + var s = StringStream(s) + if bufLen <= 0: + return + if s.pos + bufLen > s.data.len: + setLen(s.data, s.pos + bufLen) + when defined(js): + try: + s.data[s.pos..<s.pos+bufLen] = cast[ptr string](buffer)[][0..<bufLen] + except: + raise newException(Defect, "could not write to string stream, " & + "did you use a non-string buffer pointer?", getCurrentException()) + elif not defined(nimscript): + copyMem(addr(s.data[s.pos]), buffer, bufLen) + inc(s.pos, bufLen) + + proc ssClose(s: Stream) = + var s = StringStream(s) + s.data = "" + + proc newStringStream*(s: sink string = ""): owned StringStream = + ## Creates a new stream from the string `s`. + ## + ## See also: + ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from + ## opened File. + ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + var strm = newStringStream("The first line\nthe second line\nthe third line") + doAssert strm.readLine() == "The first line" + doAssert strm.readLine() == "the second line" + doAssert strm.readLine() == "the third line" + strm.close() - type - FileStream* = ref FileStreamObj ## a stream that encapsulates a `File` - FileStreamObj* = object of Stream - f: File - {.deprecated: [PFileStream: FileStream, TFileStream: FileStreamObj].} - - proc fsClose(s: Stream) = - if FileStream(s).f != nil: - close(FileStream(s).f) - FileStream(s).f = nil - proc fsFlush(s: Stream) = flushFile(FileStream(s).f) - proc fsAtEnd(s: Stream): bool = return endOfFile(FileStream(s).f) - proc fsSetPosition(s: Stream, pos: int) = setFilePos(FileStream(s).f, pos) - proc fsGetPosition(s: Stream): int = return int(getFilePos(FileStream(s).f)) - - proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int = - result = readBuffer(FileStream(s).f, buffer, bufLen) - - proc fsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let pos = fsGetPosition(s) - defer: fsSetPosition(s, pos) - result = readBuffer(FileStream(s).f, buffer, bufLen) - - proc fsWriteData(s: Stream, buffer: pointer, bufLen: int) = - if writeBuffer(FileStream(s).f, buffer, bufLen) != bufLen: - raise newEIO("cannot write to stream") - - proc newFileStream*(f: File): FileStream = - ## creates a new stream from the file `f`. new(result) - result.f = f - result.closeImpl = fsClose - result.atEndImpl = fsAtEnd - result.setPositionImpl = fsSetPosition - result.getPositionImpl = fsGetPosition - result.readDataImpl = fsReadData - result.peekDataImpl = fsPeekData - result.writeDataImpl = fsWriteData - result.flushImpl = fsFlush - - proc newFileStream*(filename: string, mode: FileMode = fmRead): FileStream = - ## creates a new stream from the file named `filename` with the mode `mode`. - ## If the file cannot be opened, nil is returned. See the `system - ## <system.html>`_ module for a list of available FileMode enums. + result.data = s + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. + result.pos = 0 + result.closeImpl = ssClose + result.atEndImpl = ssAtEnd + result.setPositionImpl = ssSetPosition + result.getPositionImpl = ssGetPosition + result.readDataStrImpl = ssReadDataStr + when nimvm: + discard + else: + result.readDataImpl = ssReadData + result.peekDataImpl = ssPeekData + result.writeDataImpl = ssWriteData + +type + FileStream* = ref FileStreamObj + ## A stream that encapsulates a `File`. + ## + ## **Note:** Not available for JS backend. + FileStreamObj* = object of Stream + ## A file stream object. + ## + ## **Note:** Not available for JS backend. + f: File + +proc fsClose(s: Stream) = + if FileStream(s).f != nil: + close(FileStream(s).f) + FileStream(s).f = nil +proc fsFlush(s: Stream) = flushFile(FileStream(s).f) +proc fsAtEnd(s: Stream): bool = return endOfFile(FileStream(s).f) +proc fsSetPosition(s: Stream, pos: int) = setFilePos(FileStream(s).f, pos) +proc fsGetPosition(s: Stream): int = return int(getFilePos(FileStream(s).f)) + +proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = readBuffer(FileStream(s).f, buffer, bufLen) + +proc fsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = + result = readBuffer(FileStream(s).f, addr buffer[slice.a], slice.b + 1 - slice.a) + +proc fsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let pos = fsGetPosition(s) + defer: fsSetPosition(s, pos) + result = readBuffer(FileStream(s).f, buffer, bufLen) + +proc fsWriteData(s: Stream, buffer: pointer, bufLen: int) = + if writeBuffer(FileStream(s).f, buffer, bufLen) != bufLen: + raise newEIO("cannot write to stream") + +proc fsReadLine(s: Stream, line: var string): bool = + result = readLine(FileStream(s).f, line) + +proc newFileStream*(f: File): owned FileStream = + ## Creates a new stream from the file `f`. + ## + ## **Note:** Not available for JS backend. + ## + ## See also: + ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream + ## from string. + ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same + ## as using `open proc <syncio.html#open,File,string,FileMode,int>`_ + ## on Examples. + ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + ## Input (somefile.txt): + ## The first line + ## the second line + ## the third line var f: File - if open(f, filename, mode): result = newFileStream(f) + if open(f, "somefile.txt", fmRead, -1): + var strm = newFileStream(f) + var line = "" + while strm.readLine(line): + echo line + ## Output: + ## The first line + ## the second line + ## the third line + strm.close() + new(result) + result.f = f + result.closeImpl = fsClose + result.atEndImpl = fsAtEnd + result.setPositionImpl = fsSetPosition + result.getPositionImpl = fsGetPosition + result.readDataStrImpl = fsReadDataStr + result.readDataImpl = fsReadData + result.readLineImpl = fsReadLine + result.peekDataImpl = fsPeekData + result.writeDataImpl = fsWriteData + result.flushImpl = fsFlush + +proc newFileStream*(filename: string, mode: FileMode = fmRead, + bufSize: int = -1): owned FileStream = + ## Creates a new stream from the file named `filename` with the mode `mode`. + ## + ## If the file cannot be opened, `nil` is returned. See the `io module + ## <syncio.html>`_ for a list of available `FileMode enums <syncio.html#FileMode>`_. + ## + ## **Note:** + ## * **This function returns nil in case of failure.** + ## To prevent unexpected behavior and ensure proper error handling, + ## use `openFileStream proc <#openFileStream,string,FileMode,int>`_ + ## instead. + ## * Not available for JS backend. + ## + ## See also: + ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream + ## from string. + ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from + ## opened File. + ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + from std/os import removeFile + var strm = newFileStream("somefile.txt", fmWrite) + if not isNil(strm): + strm.writeLine("The first line") + strm.writeLine("the second line") + strm.writeLine("the third line") + strm.close() + ## Output (somefile.txt) + ## The first line + ## the second line + ## the third line + removeFile("somefile.txt") + + var f: File + if open(f, filename, mode, bufSize): result = newFileStream(f) + +proc openFileStream*(filename: string, mode: FileMode = fmRead, + bufSize: int = -1): owned FileStream = + ## Creates a new stream from the file named `filename` with the mode `mode`. + ## If the file cannot be opened, an IO exception is raised. + ## + ## **Note:** Not available for JS backend. + ## + ## See also: + ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream + ## from string. + ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from + ## opened File. + ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + try: + ## Input (somefile.txt): + ## The first line + ## the second line + ## the third line + var strm = openFileStream("somefile.txt") + echo strm.readLine() + ## Output: + ## The first line + strm.close() + except: + stderr.write getCurrentExceptionMsg() + + var f: File + if open(f, filename, mode, bufSize): + return newFileStream(f) + else: + raise newEIO("cannot open file stream: " & filename) -when true: - discard -else: +when false: type FileHandleStream* = ref FileHandleStreamObj FileHandleStreamObj* = object of Stream handle*: FileHandle pos: int - {.deprecated: [PFileHandleStream: FileHandleStream, - TFileHandleStream: FileHandleStreamObj].} - proc newEOS(msg: string): ref OSError = new(result) result.msg = msg @@ -435,7 +1491,7 @@ else: # do not import windows as this increases compile times: discard else: - import posix + import std/posix proc hsSetPosition(s: FileHandleStream, pos: int) = discard lseek(s.handle, pos, SEEK_SET) @@ -459,7 +1515,7 @@ else: raise newEIO("cannot write to stream") inc(s.pos, bufLen) - proc newFileHandleStream*(handle: FileHandle): FileHandleStream = + proc newFileHandleStream*(handle: FileHandle): owned FileHandleStream = new(result) result.handle = handle result.pos = 0 @@ -472,31 +1528,18 @@ else: result.writeData = hsWriteData proc newFileHandleStream*(filename: string, - mode: FileMode): FileHandleStream = + mode: FileMode): owned FileHandleStream = when defined(windows): discard else: var flags: cint case mode - of fmRead: flags = posix.O_RDONLY - of fmWrite: flags = O_WRONLY or int(O_CREAT) - of fmReadWrite: flags = O_RDWR or int(O_CREAT) + of fmRead: flags = posix.O_RDONLY + of fmWrite: flags = O_WRONLY or int(O_CREAT) + of fmReadWrite: flags = O_RDWR or int(O_CREAT) of fmReadWriteExisting: flags = O_RDWR - of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND + of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND + static: raiseAssert "unreachable" # handle bug #17888 var handle = open(filename, flags) if handle < 0: raise newEOS("posix.open() call failed") result = newFileHandleStream(handle) - -when isMainModule and defined(testing): - var ss = newStringStream("The quick brown fox jumped over the lazy dog.\nThe lazy dog ran") - assert(ss.getPosition == 0) - assert(ss.peekStr(5) == "The q") - assert(ss.getPosition == 0) # haven't moved - assert(ss.readStr(5) == "The q") - assert(ss.getPosition == 5) # did move - assert(ss.peekLine() == "uick brown fox jumped over the lazy dog.") - assert(ss.getPosition == 5) # haven't moved - var str = newString(100) - assert(ss.peekLine(str)) - assert(str == "uick brown fox jumped over the lazy dog.") - assert(ss.getPosition == 5) # haven't moved diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim new file mode 100644 index 000000000..99752a9ab --- /dev/null +++ b/lib/pure/streamwrapper.nim @@ -0,0 +1,121 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements stream wrapper. +## +## **Since** version 1.2. + +import std/[deques, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions + + +type + PipeOutStream*[T] = ref object of T + # When stream peek operation is called, it reads from base stream + # type using `baseReadDataImpl` and stores the content to this buffer. + # Next stream read operation returns data in the buffer so that previus peek + # operation looks like didn't changed read positon. + # When stream read operation that returns N byte data is called and the size is smaller than buffer size, + # first N elements are removed from buffer. + # Deque type can do such operation more efficiently than seq type. + buffer: Deque[char] + baseReadLineImpl: typeof(StreamObj.readLineImpl) + baseReadDataImpl: typeof(StreamObj.readDataImpl) + +proc posReadLine[T](s: Stream, line: var string): bool = + var s = PipeOutStream[T](s) + assert s.baseReadLineImpl != nil + + let n = s.buffer.len + line.setLen(0) + for i in 0..<n: + var c = s.buffer.popFirst + if c == '\c': + c = readChar(s) + return true + elif c == '\L': return true + elif c == '\0': + return line.len > 0 + line.add(c) + + var line2: string + result = s.baseReadLineImpl(s, line2) + line.add line2 + +proc posReadData[T](s: Stream, buffer: pointer, bufLen: int): int = + var s = PipeOutStream[T](s) + assert s.baseReadDataImpl != nil + + let + dest = cast[ptr UncheckedArray[char]](buffer) + n = min(s.buffer.len, bufLen) + result = n + for i in 0..<n: + dest[i] = s.buffer.popFirst + if bufLen > n: + result += s.baseReadDataImpl(s, addr dest[n], bufLen - n) + +proc posReadDataStr[T](s: Stream, buffer: var string, slice: Slice[int]): int = + posReadData[T](s, addr buffer[slice.a], slice.len) + +proc posPeekData[T](s: Stream, buffer: pointer, bufLen: int): int = + var s = PipeOutStream[T](s) + assert s.baseReadDataImpl != nil + + let + dest = cast[ptr UncheckedArray[char]](buffer) + n = min(s.buffer.len, bufLen) + + result = n + for i in 0..<n: + dest[i] = s.buffer[i] + + if bufLen > n: + let + newDataNeeded = bufLen - n + numRead = s.baseReadDataImpl(s, addr dest[n], newDataNeeded) + result += numRead + for i in 0..<numRead: + s.buffer.addLast dest[n + i] + +proc newPipeOutStream*[T](s: sink (ref T)): owned PipeOutStream[T] = + ## Wrap pipe for reading with PipeOutStream so that you can use peek* procs and generate runtime error + ## when setPosition/getPosition is called or write operation is performed. + ## + ## Example: + ## ```Nim + ## import std/[osproc, streamwrapper] + ## var + ## p = startProcess(exePath) + ## outStream = p.outputStream().newPipeOutStream() + ## echo outStream.peekChar + ## p.close() + ## ``` + + assert s.readDataImpl != nil + + new(result) + for dest, src in fields((ref T)(result)[], s[]): + dest = src + wasMoved(s[]) + if result.readLineImpl != nil: + result.baseReadLineImpl = result.readLineImpl + result.readLineImpl = posReadLine[T] + result.baseReadDataImpl = result.readDataImpl + result.readDataImpl = posReadData[T] + result.readDataStrImpl = posReadDataStr[T] + result.peekDataImpl = posPeekData[T] + + # Set nil to anything you may not call. + result.setPositionImpl = nil + result.getPositionImpl = nil + result.writeDataImpl = nil + result.flushImpl = nil diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim new file mode 100644 index 000000000..7d093ebb3 --- /dev/null +++ b/lib/pure/strformat.nim @@ -0,0 +1,790 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +String `interpolation`:idx: / `format`:idx: inspired by +Python's f-strings. + +# `fmt` vs. `&` + +You can use either `fmt` or the unary `&` operator for formatting. The +difference between them is subtle but important. + +The `fmt"{expr}"` syntax is more aesthetically pleasing, but it hides a small +gotcha. The string is a +`generalized raw string literal <manual.html#lexical-analysis-generalized-raw-string-literals>`_. +This has some surprising effects: +]## + +runnableExamples: + let msg = "hello" + assert fmt"{msg}\n" == "hello\\n" + +##[ +Because the literal is a raw string literal, the `\n` is not interpreted as +an escape sequence. + +There are multiple ways to get around this, including the use of the `&` operator: +]## + +runnableExamples: + let msg = "hello" + + assert &"{msg}\n" == "hello\n" + + assert fmt"{msg}{'\n'}" == "hello\n" + assert fmt("{msg}\n") == "hello\n" + assert "{msg}\n".fmt == "hello\n" + +##[ +The choice of style is up to you. + +# Formatting strings +]## + +runnableExamples: + assert &"""{"abc":>4}""" == " abc" + assert &"""{"abc":<4}""" == "abc " + +##[ +# Formatting floats +]## + +runnableExamples: + assert fmt"{-12345:08}" == "-0012345" + assert fmt"{-1:3}" == " -1" + assert fmt"{-1:03}" == "-01" + assert fmt"{16:#X}" == "0x10" + + assert fmt"{123.456}" == "123.456" + assert fmt"{123.456:>9.3f}" == " 123.456" + assert fmt"{123.456:9.3f}" == " 123.456" + assert fmt"{123.456:9.4f}" == " 123.4560" + assert fmt"{123.456:>9.0f}" == " 123." + assert fmt"{123.456:<9.4f}" == "123.4560 " + + assert fmt"{123.456:e}" == "1.234560e+02" + assert fmt"{123.456:>13e}" == " 1.234560e+02" + assert fmt"{123.456:13e}" == " 1.234560e+02" + +##[ +# Expressions +]## +runnableExamples: + let x = 3.14 + assert fmt"{(if x!=0: 1.0/x else: 0):.5}" == "0.31847" + assert fmt"""{(block: + var res: string + for i in 1..15: + res.add (if i mod 15 == 0: "FizzBuzz" + elif i mod 5 == 0: "Buzz" + elif i mod 3 == 0: "Fizz" + else: $i) & " " + res)}""" == "1 2 Fizz 4 Buzz Fizz 7 8 Fizz Buzz 11 Fizz 13 14 FizzBuzz " +##[ +# Debugging strings + +`fmt"{expr=}"` expands to `fmt"expr={expr}"` namely the text of the expression, +an equal sign and the results of evaluated expression. +]## + +runnableExamples: + assert fmt"{123.456=}" == "123.456=123.456" + assert fmt"{123.456=:>9.3f}" == "123.456= 123.456" + + let x = "hello" + assert fmt"{x=}" == "x=hello" + assert fmt"{x =}" == "x =hello" + + let y = 3.1415926 + assert fmt"{y=:.2f}" == fmt"y={y:.2f}" + assert fmt"{y=}" == fmt"y={y}" + assert fmt"{y = : <8}" == fmt"y = 3.14159 " + + proc hello(a: string, b: float): int = 12 + assert fmt"{hello(x, y) = }" == "hello(x, y) = 12" + assert fmt"{x.hello(y) = }" == "x.hello(y) = 12" + assert fmt"{hello x, y = }" == "hello x, y = 12" + +##[ +Note that it is space sensitive: +]## + +runnableExamples: + let x = "12" + assert fmt"{x=}" == "x=12" + assert fmt"{x =:}" == "x =12" + assert fmt"{x =}" == "x =12" + assert fmt"{x= :}" == "x= 12" + assert fmt"{x= }" == "x= 12" + assert fmt"{x = :}" == "x = 12" + assert fmt"{x = }" == "x = 12" + assert fmt"{x = :}" == "x = 12" + assert fmt"{x = }" == "x = 12" + +##[ +# Implementation details + +An expression like `&"{key} is {value:arg} {{z}}"` is transformed into: + + ```nim + var temp = newStringOfCap(educatedCapGuess) + temp.formatValue(key, "") + temp.add(" is ") + temp.formatValue(value, arg) + temp.add(" {z}") + temp + ``` + +Parts of the string that are enclosed in the curly braces are interpreted +as Nim code. To escape a `{` or `}`, double it. + +Within a curly expression, however, `{`, `}`, must be escaped with a backslash. + +To enable evaluating Nim expressions within curlies, colons inside parentheses +do not need to be escaped. +]## + +runnableExamples: + let x = "hello" + assert fmt"""{ "\{(" & x & ")\}" }""" == "{(hello)}" + assert fmt"""{{({ x })}}""" == "{(hello)}" + assert fmt"""{ $(\{x:1,"world":2\}) }""" == """[("hello", 1), ("world", 2)]""" + +##[ +`&` delegates most of the work to an open overloaded set +of `formatValue` procs. The required signature for a type `T` that supports +formatting is usually `proc formatValue(result: var string; x: T; specifier: string)`. + +The subexpression after the colon +(`arg` in `&"{key} is {value:arg} {{z}}"`) is optional. It will be passed as +the last argument to `formatValue`. When the colon with the subexpression it is +left out, an empty string will be taken instead. + +For strings and numeric types the optional argument is a so-called +"standard format specifier". + +# Standard format specifiers for strings, integers and floats + +The general form of a standard format specifier is: + + [[fill]align][sign][#][0][minimumwidth][.precision][type] + +The square brackets `[]` indicate an optional element. + +The optional `align` flag can be one of the following: + +`<` +: Forces the field to be left-aligned within the available + space. (This is the default for strings.) + +`>` +: Forces the field to be right-aligned within the available space. + (This is the default for numbers.) + +`^` +: Forces the field to be centered within the available space. + +Note that unless a minimum field width is defined, the field width +will always be the same size as the data to fill it, so that the alignment +option has no meaning in this case. + +The optional `fill` character defines the character to be used to pad +the field to the minimum width. The fill character, if present, must be +followed by an alignment flag. + +The `sign` option is only valid for numeric types, and can be one of the following: + +================= ==================================================== + Sign Meaning +================= ==================================================== +`+` Indicates that a sign should be used for both + positive as well as negative numbers. +`-` Indicates that a sign should be used only for + negative numbers (this is the default behavior). +(space) Indicates that a leading space should be used on + positive numbers. +================= ==================================================== + +If the `#` character is present, integers use the 'alternate form' for formatting. +This means that binary, octal and hexadecimal output will be prefixed +with `0b`, `0o` and `0x`, respectively. + +`width` is a decimal integer defining the minimum field width. If not specified, +then the field width will be determined by the content. + +If the width field is preceded by a zero (`0`) character, this enables +zero-padding. + +The `precision` is a decimal number indicating how many digits should be displayed +after the decimal point in a floating point conversion. For non-numeric types the +field indicates the maximum field size - in other words, how many characters will +be used from the field content. The precision is ignored for integer conversions. + +Finally, the `type` determines how the data should be presented. + +The available integer presentation types are: + +================= ==================================================== + Type Result +================= ==================================================== +`b` Binary. Outputs the number in base 2. +`d` Decimal Integer. Outputs the number in base 10. +`o` Octal format. Outputs the number in base 8. +`x` Hex format. Outputs the number in base 16, using + lower-case letters for the digits above 9. +`X` Hex format. Outputs the number in base 16, using + uppercase letters for the digits above 9. +(None) The same as `d`. +================= ==================================================== + +The available floating point presentation types are: + +================= ==================================================== + Type Result +================= ==================================================== +`e` Exponent notation. Prints the number in scientific + notation using the letter `e` to indicate the + exponent. +`E` Exponent notation. Same as `e` except it converts + the number to uppercase. +`f` Fixed point. Displays the number as a fixed-point + number. +`F` Fixed point. Same as `f` except it converts the + number to uppercase. +`g` General format. This prints the number as a + fixed-point number, unless the number is too + large, in which case it switches to `e` + exponent notation. +`G` General format. Same as `g` except it switches to `E` + if the number gets to large. +`i` Complex General format. This is only supported for + complex numbers, which it prints using the mathematical + (RE+IMj) format. The real and imaginary parts are printed + using the general format `g` by default, but it is + possible to combine this format with one of the other + formats (e.g `jf`). +(None) Similar to `g`, except that it prints at least one + digit after the decimal point. +================= ==================================================== + +# Limitations + +Because of the well defined order how templates and macros are +expanded, strformat cannot expand template arguments: + + ```nim + template myTemplate(arg: untyped): untyped = + echo "arg is: ", arg + echo &"--- {arg} ---" + + let x = "abc" + myTemplate(x) + ``` + +First the template `myTemplate` is expanded, where every identifier +`arg` is substituted with its argument. The `arg` inside the +format string is not seen by this process, because it is part of a +quoted string literal. It is not an identifier yet. Then the strformat +macro creates the `arg` identifier from the string literal, an +identifier that cannot be resolved anymore. + +The workaround for this is to bind the template argument to a new local variable. + + ```nim + template myTemplate(arg: untyped): untyped = + block: + let arg1 {.inject.} = arg + echo "arg is: ", arg1 + echo &"--- {arg1} ---" + ``` + +The use of `{.inject.}` here is necessary again because of template +expansion order and hygienic templates. But since we generally want to +keep the hygiene of `myTemplate`, and we do not want `arg1` +to be injected into the context where `myTemplate` is expanded, +everything is wrapped in a `block`. + + +# Future directions + +A curly expression with commas in it like `{x, argA, argB}` could be +transformed to `formatValue(result, x, argA, argB)` in order to support +formatters that do not need to parse a custom language within a custom +language but instead prefer to use Nim's existing syntax. This would also +help with readability, since there is only so much you can cram into +single letter DSLs. +]## + +import std/[macros, parseutils, unicode] +import std/strutils except format + +when defined(nimPreviewSlimSystem): + import std/assertions + + +proc mkDigit(v: int, typ: char): string {.inline.} = + assert(v < 26) + if v < 10: + result = $chr(ord('0') + v) + else: + result = $chr(ord(if typ == 'x': 'a' else: 'A') + v - 10) + +proc alignString*(s: string, minimumWidth: int; align = '\0'; fill = ' '): string = + ## Aligns `s` using the `fill` char. + ## This is only of interest if you want to write a custom `format` proc that + ## should support the standard format specifiers. + if minimumWidth == 0: + result = s + else: + let sRuneLen = if s.validateUtf8 == -1: s.runeLen else: s.len + let toFill = minimumWidth - sRuneLen + if toFill <= 0: + result = s + elif align == '<' or align == '\0': + result = s & repeat(fill, toFill) + elif align == '^': + let half = toFill div 2 + result = repeat(fill, half) & s & repeat(fill, toFill - half) + else: + result = repeat(fill, toFill) & s + +type + StandardFormatSpecifier* = object ## Type that describes "standard format specifiers". + fill*, align*: char ## Desired fill and alignment. + sign*: char ## Desired sign. + alternateForm*: bool ## Whether to prefix binary, octal and hex numbers + ## with `0b`, `0o`, `0x`. + padWithZero*: bool ## Whether to pad with zeros rather than spaces. + minimumWidth*, precision*: int ## Desired minimum width and precision. + typ*: char ## Type like 'f', 'g' or 'd'. + endPosition*: int ## End position in the format specifier after + ## `parseStandardFormatSpecifier` returned. + +proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string = + ## Converts `n` to a string. If `n` is `SomeFloat`, it casts to `int64`. + ## Conversion is done using `radix`. If result's length is less than + ## `minimumWidth`, it aligns result to the right or left (depending on `a`) + ## with the `fill` char. + when n is SomeUnsignedInt: + var v = n.uint64 + let negative = false + else: + let n = n.int64 + let negative = n < 0 + var v = + if negative: + # `uint64(-n)`, but accounts for `n == low(int64)` + uint64(not n) + 1 + else: + uint64(n) + + var xx = "" + if spec.alternateForm: + case spec.typ + of 'X': xx = "0x" + of 'x': xx = "0x" + of 'b': xx = "0b" + of 'o': xx = "0o" + else: discard + + if v == 0: + result = "0" + else: + result = "" + while v > typeof(v)(0): + let d = v mod typeof(v)(radix) + v = v div typeof(v)(radix) + result.add(mkDigit(d.int, spec.typ)) + for idx in 0..<(result.len div 2): + swap result[idx], result[result.len - idx - 1] + if spec.padWithZero: + let sign = negative or spec.sign != '-' + let toFill = spec.minimumWidth - result.len - xx.len - ord(sign) + if toFill > 0: + result = repeat('0', toFill) & result + + if negative: + result = "-" & xx & result + elif spec.sign != '-': + result = spec.sign & xx & result + else: + result = xx & result + + if spec.align == '<': + for i in result.len..<spec.minimumWidth: + result.add(spec.fill) + else: + let toFill = spec.minimumWidth - result.len + if spec.align == '^': + let half = toFill div 2 + result = repeat(spec.fill, half) & result & repeat(spec.fill, toFill - half) + else: + if toFill > 0: + result = repeat(spec.fill, toFill) & result + +proc parseStandardFormatSpecifier*(s: string; start = 0; + ignoreUnknownSuffix = false): StandardFormatSpecifier = + ## An exported helper proc that parses the "standard format specifiers", + ## as specified by the grammar: + ## + ## [[fill]align][sign][#][0][minimumwidth][.precision][type] + ## + ## This is only of interest if you want to write a custom `format` proc that + ## should support the standard format specifiers. If `ignoreUnknownSuffix` is true, + ## an unknown suffix after the `type` field is not an error. + const alignChars = {'<', '>', '^'} + result.fill = ' ' + result.align = '\0' + result.sign = '-' + var i = start + if i + 1 < s.len and s[i+1] in alignChars: + result.fill = s[i] + result.align = s[i+1] + inc i, 2 + elif i < s.len and s[i] in alignChars: + result.align = s[i] + inc i + + if i < s.len and s[i] in {'-', '+', ' '}: + result.sign = s[i] + inc i + + if i < s.len and s[i] == '#': + result.alternateForm = true + inc i + + if i + 1 < s.len and s[i] == '0' and s[i+1] in {'0'..'9'}: + result.padWithZero = true + inc i + + let parsedLength = parseSaturatedNatural(s, result.minimumWidth, i) + inc i, parsedLength + if i < s.len and s[i] == '.': + inc i + let parsedLengthB = parseSaturatedNatural(s, result.precision, i) + inc i, parsedLengthB + else: + result.precision = -1 + + if i < s.len and s[i] in {'A'..'Z', 'a'..'z'}: + result.typ = s[i] + inc i + result.endPosition = i + if i != s.len and not ignoreUnknownSuffix: + raise newException(ValueError, + "invalid format string, cannot parse: " & s[i..^1]) + +proc toRadix(typ: char): int = + case typ + of 'x', 'X': 16 + of 'd', '\0': 10 + of 'o': 8 + of 'b': 2 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & typ) + +proc formatValue*[T: SomeInteger](result: var string; value: T; + specifier: static string) = + ## Standard format implementation for `SomeInteger`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) + + result.add formatInt(value, radix, spec) + +proc formatValue*[T: SomeInteger](result: var string; value: T; + specifier: string) = + ## Standard format implementation for `SomeInteger`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + if specifier.len == 0: + result.add $value + else: + let + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) + + result.add formatInt(value, radix, spec) + +proc formatFloat( + result: var string, value: SomeFloat, fmode: FloatFormatMode, + spec: StandardFormatSpecifier) = + var f = formatBiggestFloat(value, fmode, spec.precision) + var sign = false + if value >= 0.0: + if spec.sign != '-': + sign = true + if value == 0.0: + if 1.0 / value == Inf: + # only insert the sign if value != negZero + f.insert($spec.sign, 0) + else: + f.insert($spec.sign, 0) + else: + sign = true + + if spec.padWithZero: + var signStr = "" + if sign: + signStr = $f[0] + f = f[1..^1] + + let toFill = spec.minimumWidth - f.len - ord(sign) + if toFill > 0: + f = repeat('0', toFill) & f + if sign: + f = signStr & f + + # the default for numbers is right-alignment: + let align = if spec.align == '\0': '>' else: spec.align + let res = alignString(f, spec.minimumWidth, align, spec.fill) + if spec.typ in {'A'..'Z'}: + result.add toUpperAscii(res) + else: + result.add res + +proc toFloatFormatMode(typ: char): FloatFormatMode = + case typ + of 'e', 'E': ffScientific + of 'f', 'F': ffDecimal + of 'g', 'G': ffDefault + of '\0': ffDefault + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & typ) + +proc formatValue*(result: var string; value: SomeFloat; specifier: static string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: SomeFloat; specifier: string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + if specifier.len == 0: + result.add $value + else: + let + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: string; specifier: static string) = + ## Standard format implementation for `string`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + const spec = parseStandardFormatSpecifier(specifier) + var value = + when spec.typ in {'s', '\0'}: value + else: static: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + when spec.precision != -1: + if spec.precision < runeLen(value): + const precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + +proc formatValue*(result: var string; value: string; specifier: string) = + ## Standard format implementation for `string`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + let spec = parseStandardFormatSpecifier(specifier) + var value = + if spec.typ in {'s', '\0'}: value + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + if spec.precision != -1: + if spec.precision < runeLen(value): + let precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: static string) = + mixin `$` + formatValue(result, $value, specifier) + +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: string) = + mixin `$` + formatValue(result, $value, specifier) + +template formatValue(result: var string; value: char; specifier: string) = + result.add value + +template formatValue(result: var string; value: cstring; specifier: string) = + result.add value + +proc strformatImpl(f: string; openChar, closeChar: char, + lineInfoNode: NimNode = nil): NimNode = + template missingCloseChar = + error("invalid format string: missing closing character '" & closeChar & "'") + + if openChar == ':' or closeChar == ':': + error "openChar and closeChar must not be ':'" + var i = 0 + let res = genSym(nskVar, "fmtRes") + result = newNimNode(nnkStmtListExpr, lineInfoNode) + # XXX: https://github.com/nim-lang/Nim/issues/8405 + # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils + # module are not accessible at compile-time: + let expectedGrowth = when defined(useNimRtl): 0 else: count(f, openChar) * 10 + result.add newVarStmt(res, newCall(bindSym"newStringOfCap", + newLit(f.len + expectedGrowth))) + var strlit = "" + while i < f.len: + if f[i] == openChar: + inc i + if f[i] == openChar: + inc i + strlit.add openChar + else: + if strlit.len > 0: + result.add newCall(bindSym"add", res, newLit(strlit)) + strlit = "" + + var subexpr = "" + var inParens = 0 + var inSingleQuotes = false + var inDoubleQuotes = false + template notEscaped:bool = f[i-1]!='\\' + while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens != 0): + case f[i] + of '\\': + if i < f.len-1 and f[i+1] in {openChar,closeChar,':'}: inc i + of '\'': + if not inDoubleQuotes and notEscaped: inSingleQuotes = not inSingleQuotes + of '\"': + if notEscaped: inDoubleQuotes = not inDoubleQuotes + of '(': + if not (inSingleQuotes or inDoubleQuotes): inc inParens + of ')': + if not (inSingleQuotes or inDoubleQuotes): dec inParens + of '=': + let start = i + inc i + i += f.skipWhitespace(i) + if i == f.len: + missingCloseChar + if f[i] == closeChar or f[i] == ':': + result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) + else: + subexpr.add f[start ..< i] + continue + else: discard + subexpr.add f[i] + inc i + + if i == f.len: + missingCloseChar + + var x: NimNode + try: + x = parseExpr(subexpr) + except ValueError as e: + error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) + x.copyLineInfo(lineInfoNode) + let formatSym = bindSym("formatValue", brOpen) + var options = "" + if f[i] == ':': + inc i + while i < f.len and f[i] != closeChar: + options.add f[i] + inc i + if i == f.len: + missingCloseChar + if f[i] == closeChar: + inc i + result.add newCall(formatSym, res, x, newLit(options)) + elif f[i] == closeChar: + if i<f.len-1 and f[i+1] == closeChar: + strlit.add closeChar + inc i, 2 + else: + raiseAssert "invalid format string: '$1' instead of '$1$1'" % $closeChar + else: + strlit.add f[i] + inc i + if strlit.len > 0: + result.add newCall(bindSym"add", res, newLit(strlit)) + result.add res + # workaround for #20381 + var blockExpr = newNimNode(nnkBlockExpr, lineInfoNode) + blockExpr.add(newEmptyNode()) + blockExpr.add(result) + result = blockExpr + when defined(debugFmtDsl): + echo repr result + +macro fmt(pattern: static string; openChar: static char, closeChar: static char, lineInfoNode: untyped): string = + ## version of `fmt` with dummy untyped param for line info + strformatImpl(pattern, openChar, closeChar, lineInfoNode) + +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template fmt*(pattern: static string; openChar: static char, closeChar: static char): string {.callsite.} = + ## Interpolates `pattern` using symbols in scope. + runnableExamples: + let x = 7 + assert "var is {x * 2}".fmt == "var is 14" + assert "var is {{x}}".fmt == "var is {x}" # escape via doubling + const s = "foo: {x}" + assert s.fmt == "foo: 7" # also works with const strings + + assert fmt"\n" == r"\n" # raw string literal + assert "\n".fmt == "\n" # regular literal (likewise with `fmt("\n")` or `fmt "\n"`) + runnableExamples: + # custom `openChar`, `closeChar` + let x = 7 + assert "<x>".fmt('<', '>') == "7" + assert "<<<x>>>".fmt('<', '>') == "<7>" + assert "`x`".fmt('`', '`') == "7" + fmt(pattern, openChar, closeChar, dummyForLineInfo) + +template fmt*(pattern: static string): untyped {.callsite.} = + ## Alias for `fmt(pattern, '{', '}')`. + fmt(pattern, '{', '}', dummyForLineInfo) + +template `&`*(pattern: string{lit}): string {.callsite.} = + ## `&pattern` is the same as `pattern.fmt`. + ## For a specification of the `&` macro, see the module level documentation. + # pending bug #18275, bug #18278, use `pattern: static string` + # consider deprecating this, it's redundant with `fmt` and `fmt` is strictly + # more flexible, readable (no confusion with the binary `&`), self-documenting, + # not to mention #18275, bug #18278. + runnableExamples: + let x = 7 + assert &"{x}\n" == "7\n" # regular string literal + assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal + fmt(pattern, '{', '}', dummyForLineInfo) diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim new file mode 100644 index 000000000..a3e539e7e --- /dev/null +++ b/lib/pure/strmisc.nim @@ -0,0 +1,83 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Joey Payne +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains various string utility routines that are uncommonly +## used in comparison to the ones in `strutils <strutils.html>`_. + +import std/strutils + +func expandTabs*(s: string, tabSize: int = 8): string = + ## Expands tab characters in `s`, replacing them by spaces. + ## + ## The amount of inserted spaces for each tab character is the difference + ## between the current column number and the next tab position. Tab positions + ## occur every `tabSize` characters. + ## The column number starts at 0 and is increased with every single character + ## and inserted space, except for newline, which resets the column number + ## back to 0. + runnableExamples: + doAssert expandTabs("\t", 4) == " " + doAssert expandTabs("\tfoo\t", 4) == " foo " + doAssert expandTabs("a\tb\n\txy\t", 3) == "a b\n xy " + + result = newStringOfCap(s.len + s.len shr 2) + + template addSpaces(n) = + for _ in 1..n: + result.add(' ') + pos += n + + var pos = 0 + let denominator = if tabSize > 0: tabSize else: 1 + for c in s: + if c == '\t': + let numSpaces = tabSize - pos mod denominator + addSpaces(numSpaces) + else: + result.add(c) + pos += 1 + if c == '\l': + pos = 0 + +func partition*(s: string, sep: string, + right: bool = false): (string, string, string) = + ## Splits the string at the first (if `right` is false) + ## or last (if `right` is true) occurrence of `sep` into a 3-tuple. + ## + ## Returns a 3-tuple of strings, `(beforeSep, sep, afterSep)` or + ## `(s, "", "")` if `sep` is not found and `right` is false or + ## `("", "", s)` if `sep` is not found and `right` is true. + ## + ## **See also:** + ## * `rpartition proc <#rpartition,string,string>`_ + runnableExamples: + doAssert partition("foo:bar:baz", ":") == ("foo", ":", "bar:baz") + doAssert partition("foo:bar:baz", ":", right = true) == ("foo:bar", ":", "baz") + doAssert partition("foobar", ":") == ("foobar", "", "") + doAssert partition("foobar", ":", right = true) == ("", "", "foobar") + + let position = if right: s.rfind(sep) else: s.find(sep) + if position != -1: + return (s[0 ..< position], sep, s[position + sep.len ..< s.len]) + return if right: ("", "", s) else: (s, "", "") + +func rpartition*(s: string, sep: string): (string, string, string) = + ## Splits the string at the last occurrence of `sep` into a 3-tuple. + ## + ## Returns a 3-tuple of strings, `(beforeSep, sep, afterSep)` or + ## `("", "", s)` if `sep` is not found. This is the same as + ## `partition(s, sep, right = true)`. + ## + ## **See also:** + ## * `partition proc <#partition,string,string,bool>`_ + runnableExamples: + doAssert rpartition("foo:bar:baz", ":") == ("foo:bar", ":", "baz") + doAssert rpartition("foobar", ":") == ("", "", "foobar") + + partition(s, sep, right = true) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim new file mode 100644 index 000000000..16ef9e642 --- /dev/null +++ b/lib/pure/strscans.nim @@ -0,0 +1,696 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +This module contains a `scanf`:idx: macro that can be used for extracting +substrings from an input string. This is often easier than regular expressions. +Some examples as an appetizer: + + ```nim + # check if input string matches a triple of integers: + const input = "(1,2,4)" + var x, y, z: int + if scanf(input, "($i,$i,$i)", x, y, z): + echo "matches and x is ", x, " y is ", y, " z is ", z + + # check if input string matches an ISO date followed by an identifier followed + # by whitespace and a floating point number: + var year, month, day: int + var identifier: string + var myfloat: float + if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): + echo "yes, we have a match!" + ``` + +As can be seen from the examples, strings are matched verbatim except for +substrings starting with ``$``. These constructions are available: + +================= ======================================================== +``$b`` Matches a binary integer. This uses ``parseutils.parseBin``. +``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. +``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. +``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. +``$f`` Matches a floating-point number. Uses ``parseFloat``. +``$w`` Matches an ASCII identifier: ``[A-Za-z_][A-Za-z_0-9]*``. +``$c`` Matches a single ASCII character. +``$s`` Skips optional whitespace. +``$$`` Matches a single dollar sign. +``$.`` Matches if the end of the input string has been reached. +``$*`` Matches until the token following the ``$*`` was found. + The match is allowed to be of 0 length. +``$+`` Matches until the token following the ``$+`` was found. + The match must consist of at least one char. +``${foo}`` User defined matcher. Uses the proc ``foo`` to perform + the match. See below for more details. +``$[foo]`` Call user defined proc ``foo`` to **skip** some optional + parts in the input string. See below for more details. +================= ======================================================== + +Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*`` +and ``.+``, they work quite differently. There is no non-deterministic +state machine involved and the matches are non-greedy. ``[$*]`` +matches ``[xyz]`` via ``parseutils.parseUntil``. + +Furthermore no backtracking is performed, if parsing fails after a value +has already been bound to a matched subexpression this value is not restored +to its original value. This rarely causes problems in practice and if it does +for you, it's easy enough to bind to a temporary variable first. + + +Startswith vs full match +======================== + +``scanf`` returns true if the input string **starts with** the specified +pattern. If instead it should only return true if there is also nothing +left in the input, append ``$.`` to your pattern. + + +User definable matchers +======================= + +One very nice advantage over regular expressions is that ``scanf`` is +extensible with ordinary Nim procs. The proc is either enclosed in ``${}`` +or in ``$[]``. ``${}`` matches and binds the result +to a variable (that was passed to the ``scanf`` macro) while ``$[]`` merely +matches optional tokens without any result binding. + + +In this example, we define a helper proc ``someSep`` that skips some separators +which we then use in our scanf pattern to help us in the matching process: + + ```nim + proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = + # Note: The parameters and return value must match to what ``scanf`` requires + result = 0 + while start+result < input.len and input[start+result] in seps: inc result + + if scanf(input, "$w$[someSep]$w", key, value): + ... + ``` + +It is also possible to pass arguments to a user definable matcher: + + ```nim + proc ndigits(input: string; intVal: var int; start: int; n: int): int = + # matches exactly ``n`` digits. Matchers need to return 0 if nothing + # matched or otherwise the number of processed chars. + var x = 0 + var i = 0 + while i < n and i+start < input.len and input[i+start] in {'0'..'9'}: + x = x * 10 + input[i+start].ord - '0'.ord + inc i + # only overwrite if we had a match + if i == n: + result = n + intVal = x + + # match an ISO date extracting year, month, day at the same time. + # Also ensure the input ends after the ISO date: + var year, month, day: int + if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): + ... + ``` + + +The scanp macro +=============== + +This module also implements a ``scanp`` macro, which syntax somewhat resembles +an EBNF or PEG grammar, except that it uses Nim's expression syntax and so has +to use prefix instead of postfix operators. + +============== =============================================================== +``(E)`` Grouping +``*E`` Zero or more +``+E`` One or more +``?E`` Zero or One +``E{n,m}`` From ``n`` up to ``m`` times ``E`` +``~E`` Not predicate +``a ^* b`` Shortcut for ``?(a *(b a))``. Usually used for separators. +``a ^+ b`` Shortcut for ``?(a +(b a))``. Usually used for separators. +``'a'`` Matches a single character +``{'a'..'b'}`` Matches a character set +``"s"`` Matches a string +``E -> a`` Bind matching to some action +``$_`` Access the currently matched character +============== =============================================================== + +Note that unordered or ordered choice operators (``/``, ``|``) are +not implemented. + +Simple example that parses the ``/etc/passwd`` file line by line: + + ```nim + const + etc_passwd = """root:x:0:0:root:/root:/bin/bash + daemon:x:1:1:daemon:/usr/sbin:/bin/sh + bin:x:2:2:bin:/bin:/bin/sh + sys:x:3:3:sys:/dev:/bin/sh + nobody:x:65534:65534:nobody:/nonexistent:/bin/sh + messagebus:x:103:107::/var/run/dbus:/bin/false + """ + + proc parsePasswd(content: string): seq[string] = + result = @[] + var idx = 0 + while true: + var entry = "" + if scanp(content, idx, +(~{'\L', '\0'} -> entry.add($_)), '\L'): + result.add entry + else: + break + ``` + +The ``scanp`` maps the grammar code into Nim code that performs the parsing. +The parsing is performed with the help of 3 helper templates that that can be +implemented for a custom type. + +These templates need to be named ``atom`` and ``nxt``. ``atom`` should be +overloaded to handle both `char` and `set[char]`. + + ```nim + import std/streams + + template atom(input: Stream; idx: int; c: char): bool = + ## Used in scanp for the matching of atoms (usually chars). + peekChar(input) == c + + template atom(input: Stream; idx: int; s: set[char]): bool = + peekChar(input) in s + + template nxt(input: Stream; idx, step: int = 1) = + inc(idx, step) + setPosition(input, idx) + + if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'): + result.add entry + ``` + +Calling ordinary Nim procs inside the macro is possible: + + ```nim + proc digits(s: string; intVal: var int; start: int): int = + var x = 0 + while result+start < s.len and s[result+start] in {'0'..'9'} and s[result+start] != ':': + x = x * 10 + s[result+start].ord - '0'.ord + inc result + intVal = x + + proc extractUsers(content: string): seq[string] = + # Extracts the username and home directory + # of each entry (with UID greater than 1000) + const + digits = {'0'..'9'} + result = @[] + var idx = 0 + while true: + var login = "" + var uid = 0 + var homedir = "" + if scanp(content, idx, *(~ {':', '\0'}) -> login.add($_), ':', * ~ ':', ':', + digits($input, uid, $index), ':', *`digits`, ':', * ~ ':', ':', + *('/', * ~{':', '/'}) -> homedir.add($_), ':', *('/', * ~{'\L', '/'}), '\L'): + if uid >= 1000: + result.add login & " " & homedir + else: + break + ``` + +When used for matching, keep in mind that likewise scanf, no backtracking +is performed. + + ```nim + proc skipUntil(s: string; until: string; unless = '\0'; start: int): int = + # Skips all characters until the string `until` is found. Returns 0 + # if the char `unless` is found first or the end is reached. + var i = start + var u = 0 + while true: + if i >= s.len or s[i] == unless: + return 0 + elif s[i] == until[0]: + u = 1 + while i+u < s.len and u < until.len and s[i+u] == until[u]: + inc u + if u >= until.len: break + inc(i) + result = i+u-start + + iterator collectLinks(s: string): string = + const quote = {'\'', '"'} + var idx, old = 0 + var res = "" + while idx < s.len: + old = idx + if scanp(s, idx, "<a", skipUntil($input, "href=", '>', $index), + `quote`, *( ~`quote`) -> res.add($_)): + yield res + res = "" + idx = old + 1 + + for r in collectLinks(body): + echo r + ``` + +In this example both macros are combined seamlessly in order to maximise +efficiency and perform different checks. + + ```nim + iterator parseIps*(soup: string): string = + ## ipv4 only! + const digits = {'0'..'9'} + var a, b, c, d: int + var buf = "" + var idx = 0 + while idx < soup.len: + if scanp(soup, idx, (`digits`{1,3}, '.', `digits`{1,3}, '.', + `digits`{1,3}, '.', `digits`{1,3}) -> buf.add($_)): + discard buf.scanf("$i.$i.$i.$i", a, b, c, d) + if (a >= 0 and a <= 254) and + (b >= 0 and b <= 254) and + (c >= 0 and c <= 254) and + (d >= 0 and d <= 254): + yield buf + buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage + idx.inc + ``` +]## + + +import std/[macros, parseutils] +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + + +proc conditionsToIfChain(n, idx, res: NimNode; start: int): NimNode = + assert n.kind == nnkStmtList + if start >= n.len: return newAssignment(res, newLit true) + var ifs: NimNode = nil + if n[start+1].kind == nnkEmpty: + ifs = conditionsToIfChain(n, idx, res, start+3) + else: + ifs = newIfStmt((n[start+1], + newTree(nnkStmtList, newCall(bindSym"inc", idx, n[start+2]), + conditionsToIfChain(n, idx, res, start+3)))) + result = newTree(nnkStmtList, n[start], ifs) + +proc notZero(x: NimNode): NimNode = newCall(bindSym"!=", x, newLit 0) + +proc buildUserCall(x: string; args: varargs[NimNode]): NimNode = + let y = parseExpr(x) + result = newTree(nnkCall) + if y.kind in nnkCallKinds: result.add y[0] + else: result.add y + for a in args: result.add a + if y.kind in nnkCallKinds: + for i in 1..<y.len: result.add y[i] + +macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): bool = + ## See top level documentation of this module about how ``scanf`` works. + template matchBind(parser) {.dirty.} = + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym(parser), inp, results[i], idx)) + conds.add resLen.notZero + conds.add resLen + + template at(s: string; i: int): char = (if i < s.len: s[i] else: '\0') + template matchError() = + error("type mismatch between pattern '$" & pattern[p] & "' (position: " & $p & + ") and " & $getTypeInst(results[i]) & " var '" & repr(results[i]) & "'") + + var i = 0 + var p = 0 + var idx = genSym(nskVar, "idx") + var res = genSym(nskVar, "res") + let inp = genSym(nskLet, "inp") + result = newTree(nnkStmtListExpr, newLetStmt(inp, input), + newVarStmt(idx, newLit 0), newVarStmt(res, newLit false)) + var conds = newTree(nnkStmtList) + var fullMatch = false + while p < pattern.len: + if pattern[p] == '$': + inc p + case pattern[p] + of '$': + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"skip", inp, + newLit($pattern[p]), idx)) + conds.add resLen.notZero + conds.add resLen + of 'w': + if i < results.len and getType(results[i]).typeKind == ntyString: + matchBind "parseIdent" + else: + matchError + inc i + of 'c': + if i < results.len and getType(results[i]).typeKind == ntyChar: + matchBind "parseChar" + else: + matchError + inc i + of 'b': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseBin" + else: + matchError + inc i + of 'o': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseOct" + else: + matchError + inc i + of 'i': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseInt" + else: + matchError + inc i + of 'h': + if i < results.len and getType(results[i]).typeKind == ntyInt: + matchBind "parseHex" + else: + matchError + inc i + of 'f': + if i < results.len and getType(results[i]).typeKind == ntyFloat: + matchBind "parseFloat" + else: + matchError + inc i + of 's': + conds.add newCall(bindSym"inc", idx, + newCall(bindSym"skipWhitespace", inp, idx)) + conds.add newEmptyNode() + conds.add newEmptyNode() + of '.': + if p == pattern.len-1: + fullMatch = true + else: + error("invalid format string") + of '*', '+': + if i < results.len and getType(results[i]).typeKind == ntyString: + var min = ord(pattern[p] == '+') + var q = p+1 + var token = "" + while q < pattern.len and pattern[q] != '$': + token.add pattern[q] + inc q + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"parseUntil", inp, + results[i], newLit(token), idx)) + conds.add newCall(bindSym">=", resLen, newLit min) + conds.add resLen + else: + matchError + inc i + of '{': + inc p + var nesting = 0 + let start = p + while true: + case pattern.at(p) + of '{': inc nesting + of '}': + if nesting == 0: break + dec nesting + of '\0': error("expected closing '}'") + else: discard + inc p + let expr = pattern.substr(start, p-1) + if i < results.len: + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, buildUserCall(expr, inp, results[i], idx)) + conds.add newCall(bindSym"!=", resLen, newLit 0) + conds.add resLen + else: + error("no var given for $" & expr & " (position: " & $p & ")") + inc i + of '[': + inc p + var nesting = 0 + let start = p + while true: + case pattern.at(p) + of '[': inc nesting + of ']': + if nesting == 0: break + dec nesting + of '\0': error("expected closing ']'") + else: discard + inc p + let expr = pattern.substr(start, p-1) + conds.add newCall(bindSym"inc", idx, buildUserCall(expr, inp, idx)) + conds.add newEmptyNode() + conds.add newEmptyNode() + else: error("invalid format string") + inc p + else: + var token = "" + while p < pattern.len and pattern[p] != '$': + token.add pattern[p] + inc p + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"skip", inp, newLit(token), idx)) + conds.add resLen.notZero + conds.add resLen + result.add conditionsToIfChain(conds, idx, res, 0) + if fullMatch: + result.add newCall(bindSym"and", res, + newCall(bindSym">=", idx, newCall(bindSym"len", inp))) + else: + result.add res + +macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[untyped]): untyped {.since: (1, 5).}= + ## Works identically as scanf, but instead of predeclaring variables it returns a tuple. + ## Tuple is started with a bool which indicates if the scan was successful + ## followed by the requested data. + ## If using a user defined matcher, provide the types in order they appear after pattern: + ## `line.scanTuple("${yourMatcher()}", int)` + runnableExamples: + let (success, year, month, day, time) = scanTuple("1000-01-01 00:00:00", "$i-$i-$i$s$+") + if success: + assert year == 1000 + assert month == 1 + assert day == 1 + assert time == "00:00:00" + var + p = 0 + userMatches = 0 + arguments: seq[NimNode] + result = newStmtList() + template addVar(typ: string) = + let varIdent = ident("temp" & $arguments.len) + result.add(newNimNode(nnkVarSection).add(newIdentDefs(varIdent, ident(typ), newEmptyNode()))) + arguments.add(varIdent) + while p < pattern.len: + if pattern[p] == '$': + inc p + case pattern[p] + of 'w', '*', '+': + addVar("string") + of 'c': + addVar("char") + of 'b', 'o', 'i', 'h': + addVar("int") + of 'f': + addVar("float") + of '{': + if userMatches < matcherTypes.len: + let varIdent = ident("temp" & $arguments.len) + result.add(newNimNode(nnkVarSection).add(newIdentDefs(varIdent, matcherTypes[userMatches], newEmptyNode()))) + arguments.add(varIdent) + inc userMatches + else: discard + inc p + result.add nnkTupleConstr.newTree(newCall(ident("scanf"), input, newStrLitNode(pattern))) + for arg in arguments: + result[^1][0].add arg + result[^1].add arg + result = newBlockStmt(result) + +template atom*(input: string; idx: int; c: char): bool = + ## Used in scanp for the matching of atoms (usually chars). + ## EOF is matched as ``'\0'``. + (idx < input.len and input[idx] == c) or (idx == input.len and c == '\0') + +template atom*(input: string; idx: int; s: set[char]): bool = + (idx < input.len and input[idx] in s) or (idx == input.len and '\0' in s) + +template hasNxt*(input: string; idx: int): bool = idx < input.len + +#template prepare*(input: string): int = 0 +template success*(x: int): bool = x != 0 + +template nxt*(input: string; idx, step: int = 1) = inc(idx, step) + +macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = + ## See top level documentation of this module about how ``scanp`` works. + type StmtTriple = tuple[init, cond, action: NimNode] + + template interf(x): untyped = bindSym(x, brForceOpen) + + proc toIfChain(n: seq[StmtTriple]; idx, res: NimNode; start: int): NimNode = + if start >= n.len: return newAssignment(res, newLit true) + var ifs: NimNode = nil + if n[start].cond.kind == nnkEmpty: + ifs = toIfChain(n, idx, res, start+1) + else: + ifs = newIfStmt((n[start].cond, + newTree(nnkStmtList, n[start].action, + toIfChain(n, idx, res, start+1)))) + result = newTree(nnkStmtList, n[start].init, ifs) + + proc attach(x, attached: NimNode): NimNode = + if attached == nil: x + else: newStmtList(attached, x) + + proc placeholder(n, x, j: NimNode): NimNode = + if n.kind == nnkPrefix and n[0].eqIdent("$"): + let n1 = n[1] + if n1.eqIdent"_" or n1.eqIdent"current": + result = newTree(nnkBracketExpr, x, j) + elif n1.eqIdent"input": + result = x + elif n1.eqIdent"i" or n1.eqIdent"index": + result = j + else: + error("unknown pattern " & repr(n)) + else: + result = copyNimNode(n) + for i in 0 ..< n.len: + result.add placeholder(n[i], x, j) + + proc atm(it, input, idx, attached: NimNode): StmtTriple = + template `!!`(x): untyped = attach(x, attached) + case it.kind + of nnkIdent: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, newCall(it, input, idx)), + newCall(interf"success", resLen), + !!newCall(interf"nxt", input, idx, resLen)) + of nnkCallKinds: + # *{'A'..'Z'} !! s.add(!_) + template buildWhile(input, idx, init, cond, action): untyped = + mixin hasNxt + while hasNxt(input, idx): + init + if not cond: break + action + + # (x) a # bind action a to (x) + if it[0].kind in {nnkPar, nnkTupleConstr} and it.len == 2: + result = atm(it[0], input, idx, placeholder(it[1], input, idx)) + elif it.kind == nnkInfix and it[0].eqIdent"->": + # bind matching to some action: + result = atm(it[1], input, idx, placeholder(it[2], input, idx)) + elif it.kind == nnkInfix and it[0].eqIdent"as": + let cond = if it[1].kind in nnkCallKinds: placeholder(it[1], input, idx) + else: newCall(it[1], input, idx) + result = (newLetStmt(it[2], cond), + newCall(interf"success", it[2]), + !!newCall(interf"nxt", input, idx, it[2])) + elif it.kind == nnkPrefix and it[0].eqIdent"*": + let (init, cond, action) = atm(it[1], input, idx, attached) + result = (getAst(buildWhile(input, idx, init, cond, action)), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkPrefix and it[0].eqIdent"+": + # x+ is the same as xx* + result = atm(newTree(nnkTupleConstr, it[1], newTree(nnkPrefix, ident"*", it[1])), + input, idx, attached) + elif it.kind == nnkPrefix and it[0].eqIdent"?": + # optional. + let (init, cond, action) = atm(it[1], input, idx, attached) + if cond.kind == nnkEmpty: + error("'?' operator applied to a non-condition") + else: + result = (newTree(nnkStmtList, init, newIfStmt((cond, action))), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkPrefix and it[0].eqIdent"~": + # not operator + let (init, cond, action) = atm(it[1], input, idx, attached) + if cond.kind == nnkEmpty: + error("'~' operator applied to a non-condition") + else: + result = (init, newCall(bindSym"not", cond), action) + elif it.kind == nnkInfix and it[0].eqIdent"|": + let a = atm(it[1], input, idx, attached) + let b = atm(it[2], input, idx, attached) + if a.cond.kind == nnkEmpty or b.cond.kind == nnkEmpty: + error("'|' operator applied to a non-condition") + else: + result = (newStmtList(a.init, newIfStmt((a.cond, a.action), + (newTree(nnkStmtListExpr, b.init, b.cond), b.action))), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkInfix and it[0].eqIdent"^*": + # a ^* b is rewritten to: (a *(b a))? + #exprList = expr ^+ comma + template tmp(a, b): untyped = ?(a, *(b, a)) + result = atm(getAst(tmp(it[1], it[2])), input, idx, attached) + + elif it.kind == nnkInfix and it[0].eqIdent"^+": + # a ^* b is rewritten to: (a +(b a))? + template tmp(a, b): untyped = (a, *(b, a)) + result = atm(getAst(tmp(it[1], it[2])), input, idx, attached) + elif it.kind == nnkCommand and it.len == 2 and it[0].eqIdent"pred": + # enforce that the wrapped call is interpreted as a predicate, not a non-terminal: + result = (newEmptyNode(), placeholder(it[1], input, idx), newEmptyNode()) + else: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, placeholder(it, input, idx)), + newCall(interf"success", resLen), + !!newCall(interf"nxt", input, idx, resLen)) + of nnkStrLit..nnkTripleStrLit: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, newCall(interf"skip", input, it, idx)), + newCall(interf"success", resLen), + !!newCall(interf"nxt", input, idx, resLen)) + of nnkCurly, nnkAccQuoted, nnkCharLit: + result = (newEmptyNode(), newCall(interf"atom", input, idx, it), + !!newCall(interf"nxt", input, idx)) + of nnkCurlyExpr: + if it.len == 3 and it[1].kind == nnkIntLit and it[2].kind == nnkIntLit: + var h = newTree(nnkTupleConstr, it[0]) + for count in 2i64 .. it[1].intVal: h.add(it[0]) + for count in it[1].intVal .. it[2].intVal-1: + h.add(newTree(nnkPrefix, ident"?", it[0])) + result = atm(h, input, idx, attached) + elif it.len == 2 and it[1].kind == nnkIntLit: + var h = newTree(nnkTupleConstr, it[0]) + for count in 2i64 .. it[1].intVal: h.add(it[0]) + result = atm(h, input, idx, attached) + else: + error("invalid pattern") + of nnkPar, nnkTupleConstr: + if it.len == 1 and it.kind == nnkPar: + result = atm(it[0], input, idx, attached) + else: + # concatenation: + var conds: seq[StmtTriple] = @[] + for x in it: conds.add atm(x, input, idx, attached) + var res = genSym(nskVar, "res") + result = (newStmtList(newVarStmt(res, newLit false), + toIfChain(conds, idx, res, 0)), res, newEmptyNode()) + else: + error("invalid pattern") + + #var idx = genSym(nskVar, "idx") + var res = genSym(nskVar, "res") + result = newTree(nnkStmtListExpr, #newVarStmt(idx, newCall(interf"prepare", input)), + newVarStmt(res, newLit false)) + var conds: seq[StmtTriple] = @[] + for it in pattern: + conds.add atm(it, input, idx, nil) + result.add toIfChain(conds, idx, res, 0) + result.add res + when defined(debugScanp): + echo repr result diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 1c761cd92..4b07aca5a 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -9,70 +9,110 @@ ## The ``strtabs`` module implements an efficient hash table that is a mapping ## from strings to strings. Supports a case-sensitive, case-insensitive and -## style-insensitive mode. An efficient string substitution operator ``%`` -## for the string table is also provided. +## style-insensitive mode. + +runnableExamples: + var t = newStringTable() + t["name"] = "John" + t["city"] = "Monaco" + doAssert t.len == 2 + doAssert t.hasKey "name" + doAssert "name" in t + +## String tables can be created from a table constructor: +runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + +## When using the style insensitive mode (``modeStyleInsensitive``), +## all letters are compared case insensitively within the ASCII range +## and underscores are ignored. +runnableExamples: + var x = newStringTable(modeStyleInsensitive) + x["first_name"] = "John" + x["LastName"] = "Doe" + + doAssert x["firstName"] == "John" + doAssert x["last_name"] == "Doe" + +## An efficient string substitution operator +## `% <#%25,string,StringTableRef,set[FormatFlag]>`_ for the string table +## is also provided. +runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert "${name} lives in ${city}" % t == "John lives in Monaco" + +## **See also:** +## * `tables module <tables.html>`_ for general hash tables +## * `sharedtables module<sharedtables.html>`_ for shared hash table support +## * `strutils module<strutils.html>`_ for common string functions +## * `json module<json.html>`_ for table-like structure which allows +## heterogeneous members + +import std/private/since import - os, hashes, strutils + std/[hashes, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when defined(js) or defined(nimscript) or defined(Standalone): + {.pragma: rtlFunc.} +else: + {.pragma: rtlFunc, rtl.} + import std/envvars include "system/inclrtl" type - StringTableMode* = enum ## describes the tables operation mode - modeCaseSensitive, ## the table is case sensitive - modeCaseInsensitive, ## the table is case insensitive - modeStyleInsensitive ## the table is style insensitive - KeyValuePair = tuple[key, val: string] + StringTableMode* = enum ## Describes the tables operation mode. + modeCaseSensitive, ## the table is case sensitive + modeCaseInsensitive, ## the table is case insensitive + modeStyleInsensitive ## the table is style insensitive + KeyValuePair = tuple[key, val: string, hasValue: bool] KeyValuePairSeq = seq[KeyValuePair] StringTableObj* = object of RootObj counter: int data: KeyValuePairSeq mode: StringTableMode - StringTableRef* = ref StringTableObj ## use this type to declare string tables + StringTableRef* = ref StringTableObj -{.deprecated: [TStringTableMode: StringTableMode, - TStringTable: StringTableObj, PStringTable: StringTableRef].} + FormatFlag* = enum ## Flags for the `%` operator. + useEnvironment, ## Use environment variable if the ``$key`` + ## is not found in the table. + ## Does nothing when using `js` target. + useEmpty, ## Use the empty string as a default, thus it + ## won't throw an exception if ``$key`` is not + ## in the table. + useKey ## Do not replace ``$key`` if it is not found + ## in the table (or in the environment). -proc len*(t: StringTableRef): int {.rtl, extern: "nst$1".} = - ## returns the number of keys in `t`. - result = t.counter +const + growthFactor = 2 + startSize = 64 + +proc mode*(t: StringTableRef): StringTableMode {.inline.} = t.mode iterator pairs*(t: StringTableRef): tuple[key, value: string] = - ## iterates over every (key, value) pair in the table `t`. + ## Iterates over every `(key, value)` pair in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield (t.data[h].key, t.data[h].val) iterator keys*(t: StringTableRef): string = - ## iterates over every key in the table `t`. + ## Iterates over every key in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield t.data[h].key iterator values*(t: StringTableRef): string = - ## iterates over every value in the table `t`. + ## Iterates over every value in the table `t`. for h in 0..high(t.data): - if not isNil(t.data[h].key): + if t.data[h].hasValue: yield t.data[h].val -type - FormatFlag* = enum ## flags for the `%` operator - useEnvironment, ## use environment variable if the ``$key`` - ## is not found in the table - useEmpty, ## use the empty string as a default, thus it - ## won't throw an exception if ``$key`` is not - ## in the table - useKey ## do not replace ``$key`` if it is not found - ## in the table (or in the environment) - -{.deprecated: [TFormatFlag: FormatFlag].} - -# implementation - -const - growthFactor = 2 - startSize = 64 proc myhash(t: StringTableRef, key: string): Hash = case t.mode @@ -91,62 +131,121 @@ proc mustRehash(length, counter: int): bool = result = (length * 2 < counter * 3) or (length - counter < 4) proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash + result = (h + 1) and maxHash proc rawGet(t: StringTableRef, key: string): int = var h: Hash = myhash(t, key) and high(t.data) # start with real hash value - while not isNil(t.data[h].key): + while t.data[h].hasValue: if myCmp(t, t.data[h].key, key): return h h = nextTry(h, high(t.data)) result = - 1 -template get(t: StringTableRef, key: string): stmt {.immediate.} = +template get(t: StringTableRef, key: string) = var index = rawGet(t, key) if index >= 0: result = t.data[index].val else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raise newException(KeyError, "key not found: " & key) + + +proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = + ## Returns the number of keys in `t`. + result = t.counter proc `[]`*(t: StringTableRef, key: string): var string {. - rtl, extern: "nstTake", deprecatedGet.} = - ## retrieves the location at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether - ## the key exists. + rtlFunc, extern: "nstTake".} = + ## Retrieves the location at ``t[key]``. + ## + ## If `key` is not in `t`, the ``KeyError`` exception is raised. + ## One can check with `hasKey proc <#hasKey,StringTableRef,string>`_ + ## whether the key exists. + ## + ## See also: + ## * `getOrDefault proc <#getOrDefault,StringTableRef,string,string>`_ + ## * `[]= proc <#[]=,StringTableRef,string,string>`_ for inserting a new + ## (key, value) pair in the table + ## * `hasKey proc <#hasKey,StringTableRef,string>`_ for checking if a key + ## is in the table + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert t["name"] == "John" + doAssertRaises(KeyError): + echo t["occupation"] get(t, key) -proc mget*(t: StringTableRef, key: string): var string {.deprecated.} = - ## retrieves the location at ``t[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. Use ```[]``` instead. - get(t, key) +proc getOrDefault*(t: StringTableRef; key: string, + default: string = ""): string = + ## Retrieves the location at ``t[key]``. + ## + ## If `key` is not in `t`, the default value is returned (if not specified, + ## it is an empty string (`""`)). + ## + ## See also: + ## * `[] proc <#[],StringTableRef,string>`_ for retrieving a value of a key + ## * `hasKey proc <#hasKey,StringTableRef,string>`_ for checking if a key + ## is in the table + ## * `[]= proc <#[]=,StringTableRef,string,string>`_ for inserting a new + ## (key, value) pair in the table + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert t.getOrDefault("name") == "John" + doAssert t.getOrDefault("occupation") == "" + doAssert t.getOrDefault("occupation", "teacher") == "teacher" + doAssert t.getOrDefault("name", "Paul") == "John" -proc getOrDefault*(t: StringTableRef; key: string): string = var index = rawGet(t, key) if index >= 0: result = t.data[index].val - else: result = "" - -proc hasKey*(t: StringTableRef, key: string): bool {.rtl, extern: "nst$1".} = - ## returns true iff `key` is in the table `t`. + else: result = default + +proc hasKey*(t: StringTableRef, key: string): bool {.rtlFunc, + extern: "nst$1".} = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * `getOrDefault proc <#getOrDefault,StringTableRef,string,string>`_ + ## * `contains proc <#contains,StringTableRef,string>`_ + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert t.hasKey("name") + doAssert not t.hasKey("occupation") result = rawGet(t, key) >= 0 +proc contains*(t: StringTableRef, key: string): bool = + ## Alias of `hasKey proc <#hasKey,StringTableRef,string>`_ for use with + ## the `in` operator. + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert "name" in t + doAssert "occupation" notin t + return hasKey(t, key) + proc rawInsert(t: StringTableRef, data: var KeyValuePairSeq, key, val: string) = var h: Hash = myhash(t, key) and high(data) - while not isNil(data[h].key): + while data[h].hasValue: h = nextTry(h, high(data)) data[h].key = key data[h].val = val + data[h].hasValue = true proc enlarge(t: StringTableRef) = var n: KeyValuePairSeq newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): - if not isNil(t.data[i].key): rawInsert(t, n, t.data[i].key, t.data[i].val) + if t.data[i].hasValue: rawInsert(t, n, move t.data[i].key, move t.data[i].val) swap(t.data, n) -proc `[]=`*(t: StringTableRef, key, val: string) {.rtl, extern: "nstPut".} = - ## puts a (key, value)-pair into `t`. +proc `[]=`*(t: StringTableRef, key, val: string) {. + rtlFunc, extern: "nstPut".} = + ## Inserts a `(key, value)` pair into `t`. + ## + ## See also: + ## * `[] proc <#[],StringTableRef,string>`_ for retrieving a value of a key + ## * `del proc <#del,StringTableRef,string>`_ for removing a key from the table + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + t["occupation"] = "teacher" + doAssert t.hasKey("occupation") + var index = rawGet(t, key) if index >= 0: t.data[index].val = val @@ -155,64 +254,146 @@ proc `[]=`*(t: StringTableRef, key, val: string) {.rtl, extern: "nstPut".} = rawInsert(t, t.data, key, val) inc(t.counter) +proc newStringTable*(mode: StringTableMode): owned(StringTableRef) {. + rtlFunc, extern: "nst$1", noSideEffect.} = + ## Creates a new empty string table. + ## + ## See also: + ## * `newStringTable(keyValuePairs) proc + ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ + result = StringTableRef(mode: mode, counter: 0, data: newSeq[KeyValuePair](startSize)) + +proc newStringTable*(keyValuePairs: varargs[string], + mode: StringTableMode): owned(StringTableRef) {. + rtlFunc, extern: "nst$1WithPairs", noSideEffect.} = + ## Creates a new string table with given `key, value` string pairs. + ## + ## `StringTableMode` must be specified. + runnableExamples: + var mytab = newStringTable("key1", "val1", "key2", "val2", + modeCaseInsensitive) + + result = newStringTable(mode) + var i = 0 + while i < high(keyValuePairs): + {.noSideEffect.}: + result[keyValuePairs[i]] = keyValuePairs[i + 1] + inc(i, 2) + +proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], + mode: StringTableMode = modeCaseSensitive): owned(StringTableRef) {. + rtlFunc, extern: "nst$1WithTableConstr", noSideEffect.} = + ## Creates a new string table with given `(key, value)` tuple pairs. + ## + ## The default mode is case sensitive. + runnableExamples: + var + mytab1 = newStringTable({"key1": "val1", "key2": "val2"}, modeCaseInsensitive) + mytab2 = newStringTable([("key3", "val3"), ("key4", "val4")]) + + result = newStringTable(mode) + for key, val in items(keyValuePairs): + {.noSideEffect.}: + result[key] = val + proc raiseFormatException(s: string) = - var e: ref ValueError - new(e) - e.msg = "format string: key not found: " & s - raise e + raise newException(ValueError, "format string: key not found: " & s) proc getValue(t: StringTableRef, flags: set[FormatFlag], key: string): string = if hasKey(t, key): return t.getOrDefault(key) - # hm difficult: assume safety in taint mode here. XXX This is dangerous! - if useEnvironment in flags: result = os.getEnv(key).string - else: result = "" + when defined(js) or defined(nimscript) or defined(Standalone): + result = "" + else: + if useEnvironment in flags: result = getEnv(key) + else: result = "" if result.len == 0: if useKey in flags: result = '$' & key elif useEmpty notin flags: raiseFormatException(key) -proc newStringTable*(mode: StringTableMode): StringTableRef {. - rtl, extern: "nst$1".} = - ## creates a new string table that is empty. - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) - -proc clear*(s: StringTableRef, mode: StringTableMode) = - ## resets a string table to be empty again. +proc clear*(s: StringTableRef, mode: StringTableMode) {. + rtlFunc, extern: "nst$1".} = + ## Resets a string table to be empty again, perhaps altering the mode. + ## + ## See also: + ## * `del proc <#del,StringTableRef,string>`_ for removing a key from the table + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + clear(t, modeCaseSensitive) + doAssert len(t) == 0 + doAssert "name" notin t + doAssert "city" notin t s.mode = mode s.counter = 0 s.data.setLen(startSize) for i in 0..<s.data.len: - if not isNil(s.data[i].key): - s.data[i].key = nil - -proc newStringTable*(keyValuePairs: varargs[string], - mode: StringTableMode): StringTableRef {. - rtl, extern: "nst$1WithPairs".} = - ## creates a new string table with given key value pairs. - ## Example:: - ## var mytab = newStringTable("key1", "val1", "key2", "val2", - ## modeCaseInsensitive) - result = newStringTable(mode) - var i = 0 - while i < high(keyValuePairs): - result[keyValuePairs[i]] = keyValuePairs[i + 1] - inc(i, 2) - -proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], - mode: StringTableMode = modeCaseSensitive): StringTableRef {. - rtl, extern: "nst$1WithTableConstr".} = - ## creates a new string table with given key value pairs. - ## Example:: - ## var mytab = newStringTable({"key1": "val1", "key2": "val2"}, - ## modeCaseInsensitive) - result = newStringTable(mode) - for key, val in items(keyValuePairs): result[key] = val + s.data[i].hasValue = false + +proc clear*(s: StringTableRef) {.since: (1, 1).} = + ## Resets a string table to be empty again without changing the mode. + s.clear(s.mode) + +proc del*(t: StringTableRef, key: string) = + ## Removes `key` from `t`. + ## + ## See also: + ## * `clear proc <#clear,StringTableRef,StringTableMode>`_ for resetting a + ## table to be empty + ## * `[]= proc <#[]=,StringTableRef,string,string>`_ for inserting a new + ## (key, value) pair in the table + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + t.del("name") + doAssert len(t) == 1 + doAssert "name" notin t + doAssert "city" in t + + # Impl adapted from `tableimpl.delImplIdx` + var i = rawGet(t, key) + let msk = high(t.data) + if i >= 0: + dec(t.counter) + block outer: + while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 + var j = i # The correctness of this depends on (h+1) in nextTry, + var r = j # though may be adaptable to other simple sequences. + t.data[i].hasValue = false # mark current EMPTY + t.data[i].key = "" + t.data[i].val = "" + while true: + i = (i + 1) and msk # increment mod table size + if not t.data[i].hasValue: # end of collision cluster; So all done + break outer + r = t.myhash(t.data[i].key) and msk # "home" location of key@i + if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + break + when defined(js): + t.data[j] = t.data[i] + elif defined(gcDestructors): + t.data[j] = move t.data[i] + else: + shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop + +proc `$`*(t: StringTableRef): string {.rtlFunc, extern: "nstDollar".} = + ## The `$` operator for string tables. Used internally when calling + ## `echo` on a table. + if t.len == 0: + result = "{:}" + else: + result = "{" + for key, val in pairs(t): + if result.len > 1: result.add(", ") + result.add(key) + result.add(": ") + result.add(val) + result.add("}") proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. - rtl, extern: "nstFormat".} = + rtlFunc, extern: "nstFormat".} = ## The `%` operator for string tables. + runnableExamples: + var t = {"name": "John", "city": "Monaco"}.newStringTable + doAssert "${name} lives in ${city}" % t == "John lives in Monaco" + const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF'} result = "" @@ -239,28 +420,3 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. else: add(result, f[i]) inc(i) - -proc `$`*(t: StringTableRef): string {.rtl, extern: "nstDollar".} = - ## The `$` operator for string tables. - if t.len == 0: - result = "{:}" - else: - result = "{" - for key, val in pairs(t): - if result.len > 1: result.add(", ") - result.add(key) - result.add(": ") - result.add(val) - result.add("}") - -when isMainModule: - var x = {"k": "v", "11": "22", "565": "67"}.newStringTable - assert x["k"] == "v" - assert x["11"] == "22" - assert x["565"] == "67" - x["11"] = "23" - assert x["11"] == "23" - - x.clear(modeCaseInsensitive) - x["11"] = "22" - assert x["11"] == "22" diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index aa29bb073..81be7db17 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -7,250 +7,323 @@ # distribution, for details about the copyright. # -## This module contains various string utility routines. -## See the module `re <re.html>`_ for regular expression support. -## See the module `pegs <pegs.html>`_ for PEG support. -## This module is available for the `JavaScript target -## <backends.html#the-javascript-target>`_. +## The system module defines several common functions for working with strings, +## such as: +## * `$` for converting other data-types to strings +## * `&` for string concatenation +## * `add` for adding a new character or a string to the existing one +## * `in` (alias for `contains`) and `notin` for checking if a character +## is in a string +## +## This module builds upon that, providing additional functionality in form of +## procedures, iterators and templates for strings. + +runnableExamples: + let + numbers = @[867, 5309] + multiLineString = "first line\nsecond line\nthird line" + + let jenny = numbers.join("-") + assert jenny == "867-5309" + + assert splitLines(multiLineString) == + @["first line", "second line", "third line"] + assert split(multiLineString) == @["first", "line", "second", + "line", "third", "line"] + assert indent(multiLineString, 4) == + " first line\n second line\n third line" + assert 'z'.repeat(5) == "zzzzz" + +## The chaining of functions is possible thanks to the +## `method call syntax<manual.html#procedures-method-call-syntax>`_: -import parseutils +runnableExamples: + from std/sequtils import map -{.deadCodeElim: on.} + let jenny = "867-5309" + assert jenny.split('-').map(parseInt) == @[867, 5309] -{.push debugger:off .} # the user does not want to trace a part - # of the standard library! + assert "Beetlejuice".indent(1).repeat(3).strip == + "Beetlejuice Beetlejuice Beetlejuice" + +## This module is available for the `JavaScript target +## <backends.html#backends-the-javascript-target>`_. +## +## ---- +## +## **See also:** +## * `strformat module<strformat.html>`_ for string interpolation and formatting +## * `unicode module<unicode.html>`_ for Unicode UTF-8 handling +## * `sequtils module<sequtils.html>`_ for operations on container +## types (including strings) +## * `parsecsv module<parsecsv.html>`_ for a high-performance CSV parser +## * `parseutils module<parseutils.html>`_ for lower-level parsing of tokens, +## numbers, identifiers, etc. +## * `parseopt module<parseopt.html>`_ for command-line parsing +## * `pegs module<pegs.html>`_ for PEG (Parsing Expression Grammar) support +## * `strtabs module<strtabs.html>`_ for efficient hash tables +## (dictionaries, in some programming languages) mapping from strings to strings +## * `ropes module<ropes.html>`_ for rope data type, which can represent very +## long strings efficiently +## * `re module<re.html>`_ for regular expression (regex) support +## * `strscans<strscans.html>`_ for `scanf` and `scanp` macros, which offer +## easier substring extraction than regular expressions + + +import std/parseutils +from std/math import pow, floor, log10 +from std/algorithm import fill, reverse +import std/enumutils + +from std/unicode import toLower, toUpper +export toLower, toUpper include "system/inclrtl" +import std/private/[since, jsutils] +from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, + startsWithImpl, endsWithImpl -{.pop.} +when defined(nimPreviewSlimSystem): + import std/assertions -type - CharSet* {.deprecated.} = set[char] # for compatibility with Nim -{.deprecated: [TCharSet: CharSet].} const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} - ## All the characters that count as whitespace. + ## All the characters that count as whitespace (space, tab, vertical tab, + ## carriage return, new line, form feed). Letters* = {'A'..'Z', 'a'..'z'} - ## the set of letters + ## The set of letters. + + UppercaseLetters* = {'A'..'Z'} + ## The set of uppercase ASCII letters. + + LowercaseLetters* = {'a'..'z'} + ## The set of lowercase ASCII letters. + + PunctuationChars* = {'!'..'/', ':'..'@', '['..'`', '{'..'~'} + ## The set of all ASCII punctuation characters. Digits* = {'0'..'9'} - ## the set of digits + ## The set of digits. HexDigits* = {'0'..'9', 'A'..'F', 'a'..'f'} - ## the set of hexadecimal digits + ## The set of hexadecimal digits. IdentChars* = {'a'..'z', 'A'..'Z', '0'..'9', '_'} - ## the set of characters an identifier can consist of + ## The set of characters an identifier can consist of. IdentStartChars* = {'a'..'z', 'A'..'Z', '_'} - ## the set of characters an identifier can start with + ## The set of characters an identifier can start with. - NewLines* = {'\13', '\10'} - ## the set of characters a newline terminator can start with + Newlines* = {'\13', '\10'} + ## The set of characters a newline terminator can start with (carriage + ## return, line feed). + + PrintableChars* = Letters + Digits + PunctuationChars + Whitespace + ## The set of all printable ASCII characters (letters, digits, whitespace, and punctuation characters). AllChars* = {'\x00'..'\xFF'} ## A set with all the possible characters. ## ## Not very useful by its own, you can use it to create *inverted* sets to - ## make the `find() proc <#find,string,set[char],int>`_ find **invalid** - ## characters in strings. Example: - ## - ## .. code-block:: nim + ## make the `find func<#find,string,set[char],Natural,int>`_ + ## find **invalid** characters in strings. Example: + ## ```nim ## let invalid = AllChars - Digits ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 + ## ``` -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaChar".}= - ## Checks whether or not `c` is alphabetical. +func isAlphaAscii*(c: char): bool {.rtl, extern: "nsuIsAlphaAsciiChar".} = + ## Checks whether or not character `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. + ## Use `Unicode module<unicode.html>`_ for UTF-8 support. + runnableExamples: + doAssert isAlphaAscii('e') == true + doAssert isAlphaAscii('E') == true + doAssert isAlphaAscii('8') == false return c in Letters -proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericChar".}= +func isAlphaNumeric*(c: char): bool {.rtl, extern: "nsuIsAlphaNumericChar".} = ## Checks whether or not `c` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. - return c in Letters or c in Digits + runnableExamples: + doAssert isAlphaNumeric('n') == true + doAssert isAlphaNumeric('8') == true + doAssert isAlphaNumeric(' ') == false + return c in Letters+Digits -proc isDigit*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitChar".}= +func isDigit*(c: char): bool {.rtl, extern: "nsuIsDigitChar".} = ## Checks whether or not `c` is a number. ## ## This checks 0-9 ASCII characters only. + runnableExamples: + doAssert isDigit('n') == false + doAssert isDigit('8') == true return c in Digits -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceChar".}= +func isSpaceAscii*(c: char): bool {.rtl, extern: "nsuIsSpaceAsciiChar".} = ## Checks whether or not `c` is a whitespace character. + runnableExamples: + doAssert isSpaceAscii('n') == false + doAssert isSpaceAscii(' ') == true + doAssert isSpaceAscii('\t') == true return c in Whitespace -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerChar".}= +func isLowerAscii*(c: char): bool {.rtl, extern: "nsuIsLowerAsciiChar".} = ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. - return c in {'a'..'z'} - -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperChar".}= - ## Checks whether or not `c` is an upper case character. + ## Use `Unicode module<unicode.html>`_ for UTF-8 support. ## - ## This checks ASCII characters only. - return c in {'A'..'Z'} - -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaStr".}= - ## Checks whether or not `s` is alphabetical. - ## - ## This checks a-z, A-Z ASCII characters only. - ## Returns true if all characters in `s` are - ## alphabetic and there is at least one character - ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - result = c.isAlpha() and result - -proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericStr".}= - ## Checks whether or not `s` is alphanumeric. - ## - ## This checks a-z, A-Z, 0-9 ASCII characters only. - ## Returns true if all characters in `s` are - ## alpanumeric and there is at least one character - ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - result = c.isAlphaNumeric() and result - -proc isDigit*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitStr".}= - ## Checks whether or not `s` is a numeric value. - ## - ## This checks 0-9 ASCII characters only. - ## Returns true if all characters in `s` are - ## numeric and there is at least one character - ## in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - result = c.isDigit() and result - -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceStr".}= - ## Checks whether or not `s` is completely whitespace. - ## - ## Returns true if all characters in `s` are whitespace - ## characters and there is at least one character in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - result = c.isSpace() and result + ## See also: + ## * `toLowerAscii func<#toLowerAscii,char>`_ + runnableExamples: + doAssert isLowerAscii('e') == true + doAssert isLowerAscii('E') == false + doAssert isLowerAscii('7') == false + return c in LowercaseLetters -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerStr".}= - ## Checks whether or not `s` contains all lower case characters. +func isUpperAscii*(c: char): bool {.rtl, extern: "nsuIsUpperAsciiChar".} = + ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. - ## Returns true if all characters in `s` are lower case - ## and there is at least one character in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - result = c.isLower() and result - -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperStr".}= - ## Checks whether or not `s` contains all upper case characters. + ## Use `Unicode module<unicode.html>`_ for UTF-8 support. ## - ## This checks ASCII characters only. - ## Returns true if all characters in `s` are upper case - ## and there is at least one character in `s`. - if s.len() == 0: - return false - - result = true - for c in s: - result = c.isUpper() and result + ## See also: + ## * `toUpperAscii func<#toUpperAscii,char>`_ + runnableExamples: + doAssert isUpperAscii('e') == false + doAssert isUpperAscii('E') == true + doAssert isUpperAscii('7') == false + return c in UppercaseLetters -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToLowerChar".} = - ## Converts `c` into lower case. +func toLowerAscii*(c: char): char {.rtl, extern: "nsuToLowerAsciiChar".} = + ## Returns the lower case version of character `c`. ## - ## This works only for the letters ``A-Z``. See `unicode.toLower - ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## This works only for the letters `A-Z`. See `unicode.toLower + ## <unicode.html#toLower,Rune>`_ for a version that works for any Unicode ## character. - if c in {'A'..'Z'}: - result = chr(ord(c) + (ord('a') - ord('A'))) + ## + ## See also: + ## * `isLowerAscii func<#isLowerAscii,char>`_ + ## * `toLowerAscii func<#toLowerAscii,string>`_ for converting a string + runnableExamples: + doAssert toLowerAscii('A') == 'a' + doAssert toLowerAscii('e') == 'e' + if c in UppercaseLetters: + result = char(uint8(c) xor 0b0010_0000'u8) else: result = c -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToLowerStr".} = - ## Converts `s` into lower case. - ## - ## This works only for the letters ``A-Z``. See `unicode.toLower - ## <unicode.html#toLower>`_ for a version that works for any Unicode - ## character. +template toImpl(call) = result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toLower(s[i]) + result[i] = call(s[i]) -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToUpperChar".} = - ## Converts `c` into upper case. +func toLowerAscii*(s: string): string {.rtl, extern: "nsuToLowerAsciiStr".} = + ## Converts string `s` into lower case. ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper - ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## This works only for the letters `A-Z`. See `unicode.toLower + ## <unicode.html#toLower,string>`_ for a version that works for any Unicode ## character. - if c in {'a'..'z'}: - result = chr(ord(c) - (ord('a') - ord('A'))) + ## + ## See also: + ## * `normalize func<#normalize,string>`_ + runnableExamples: + doAssert toLowerAscii("FooBar!") == "foobar!" + toImpl toLowerAscii + +func toUpperAscii*(c: char): char {.rtl, extern: "nsuToUpperAsciiChar".} = + ## Converts character `c` into upper case. + ## + ## This works only for the letters `A-Z`. See `unicode.toUpper + ## <unicode.html#toUpper,Rune>`_ for a version that works for any Unicode + ## character. + ## + ## See also: + ## * `isUpperAscii func<#isUpperAscii,char>`_ + ## * `toUpperAscii func<#toUpperAscii,string>`_ for converting a string + ## * `capitalizeAscii func<#capitalizeAscii,string>`_ + runnableExamples: + doAssert toUpperAscii('a') == 'A' + doAssert toUpperAscii('E') == 'E' + if c in LowercaseLetters: + result = char(uint8(c) xor 0b0010_0000'u8) else: result = c -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToUpperStr".} = - ## Converts `s` into upper case. +func toUpperAscii*(s: string): string {.rtl, extern: "nsuToUpperAsciiStr".} = + ## Converts string `s` into upper case. ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper - ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## This works only for the letters `A-Z`. See `unicode.toUpper + ## <unicode.html#toUpper,string>`_ for a version that works for any Unicode ## character. - result = newString(len(s)) - for i in 0..len(s) - 1: - result[i] = toUpper(s[i]) - -proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuCapitalize".} = - ## Converts the first character of `s` into upper case. ## - ## This works only for the letters ``A-Z``. - result = toUpper(s[0]) & substr(s, 1) + ## See also: + ## * `capitalizeAscii func<#capitalizeAscii,string>`_ + runnableExamples: + doAssert toUpperAscii("FooBar!") == "FOOBAR!" + toImpl toUpperAscii + +func capitalizeAscii*(s: string): string {.rtl, extern: "nsuCapitalizeAscii".} = + ## Converts the first character of string `s` into upper case. + ## + ## This works only for the letters `A-Z`. + ## Use `Unicode module<unicode.html>`_ for UTF-8 support. + ## + ## See also: + ## * `toUpperAscii func<#toUpperAscii,char>`_ + runnableExamples: + doAssert capitalizeAscii("foo") == "Foo" + doAssert capitalizeAscii("-bar") == "-bar" + if s.len == 0: result = "" + else: result = toUpperAscii(s[0]) & substr(s, 1) + +func nimIdentNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## That means to convert to lower case and remove any '_' on all characters + ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## <dochelpers.html#nimIdentBackticksNormalize,string>`_ for + ## an alternative approach. + runnableExamples: + doAssert nimIdentNormalize("Foo_bar") == "Foobar" + result = newString(s.len) + if s.len == 0: + return + result[0] = s[0] + var j = 1 + for i in 1..len(s) - 1: + if s[i] in UppercaseLetters: + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + inc j + elif s[i] != '_': + result[j] = s[i] + inc j + if j != s.len: setLen(result, j) -proc normalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuNormalize".} = +func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = ## Normalizes the string `s`. ## - ## That means to convert it to lower case and remove any '_'. This is needed - ## for Nim identifiers for example. + ## That means to convert it to lower case and remove any '_'. This + ## should NOT be used to normalize Nim identifier names. + ## + ## See also: + ## * `toLowerAscii func<#toLowerAscii,string>`_ + runnableExamples: + doAssert normalize("Foo_bar") == "foobar" + doAssert normalize("Foo Bar") == "foo bar" result = newString(s.len) var j = 0 for i in 0..len(s) - 1: - if s[i] in {'A'..'Z'}: + if s[i] in UppercaseLetters: result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) inc j elif s[i] != '_': @@ -258,190 +331,343 @@ proc normalize*(s: string): string {.noSideEffect, procvar, inc j if j != s.len: setLen(result, j) -proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, - rtl, extern: "nsuCmpIgnoreCase", procvar.} = +func cmpIgnoreCase*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b - var i = 0 - var m = min(a.len, b.len) - while i < m: - result = ord(toLower(a[i])) - ord(toLower(b[i])) - if result != 0: return - inc(i) - result = a.len - b.len + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + runnableExamples: + doAssert cmpIgnoreCase("FooBar", "foobar") == 0 + doAssert cmpIgnoreCase("bar", "Foo") < 0 + doAssert cmpIgnoreCase("Foo5", "foo4") > 0 + cmpIgnoreCaseImpl(a, b) + +{.push checks: off, line_trace: off.} # this is a hot-spot in the compiler! + # thus we compile without checks here + +func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = + ## Semantically the same as `cmp(normalize(a), normalize(b))`. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names. + ## Use `macros.eqIdent<macros.html#eqIdent,string,string>`_ for that. + ## + ## Returns: + ## + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + runnableExamples: + doAssert cmpIgnoreStyle("foo_bar", "FooBar") == 0 + doAssert cmpIgnoreStyle("foo_bar_5", "FooBar4") > 0 + cmpIgnoreStyleImpl(a, b) +{.pop.} -{.push checks: off, line_trace: off .} # this is a hot-spot in the compiler! - # thus we compile without checks here +# --------- Private templates for different split separators ----------- -proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, - rtl, extern: "nsuCmpIgnoreStyle", procvar.} = - ## Compares two strings normalized (i.e. case and - ## underscores do not matter). Returns: - ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b - var i = 0 - var j = 0 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) +func substrEq(s: string, pos: int, substr: string): bool = + # Always returns false for empty `substr` + var length = substr.len + if length > 0: + var i = 0 + while i < length and pos+i < s.len and s[pos+i] == substr[i]: + inc i + i == length + else: false +template stringHasSep(s: string, index: int, seps: set[char]): bool = + s[index] in seps -proc strip*(s: string, leading = true, trailing = true, - chars: set[char] = Whitespace): string - {.noSideEffect, rtl, extern: "nsuStrip".} = - ## Strips `chars` from `s` and returns the resulting string. - ## - ## If `leading` is true, leading `chars` are stripped. - ## If `trailing` is true, trailing `chars` are stripped. - var - first = 0 - last = len(s)-1 - if leading: - while s[first] in chars: inc(first) - if trailing: - while last >= 0 and s[last] in chars: dec(last) - result = substr(s, first, last) +template stringHasSep(s: string, index: int, sep: char): bool = + s[index] == sep -proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = - ## Converts a character `c` to its octal representation. +template stringHasSep(s: string, index: int, sep: string): bool = + s.substrEq(index, sep) + +template splitCommon(s, sep, maxsplit, sepLen) = + ## Common code for split procs + var last = 0 + var splits = maxsplit + + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + +template oldSplit(s, seps, maxsplit) = + var last = 0 + var splits = maxsplit + assert(not ('\0' in seps)) + while last < len(s): + while last < len(s) and s[last] in seps: inc(last) + var first = last + while last < len(s) and s[last] notin seps: inc(last) + if first <= last-1: + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + +template accResult(iter: untyped) = + result = @[] + for x in iter: add(result, x) + + +iterator split*(s: string, sep: char, maxsplit: int = -1): string = + ## Splits the string `s` into substrings using a single separator. ## - ## The resulting string may not have a leading zero. Its length is always - ## exactly 3. - result = newString(3) - var val = ord(c) - for i in countdown(2, 0): - result[i] = chr(val mod 8 + ord('0')) - val = val div 8 + ## Substrings are separated by the character `sep`. + ## The code: + ## ```nim + ## for word in split(";;this;is;an;;example;;;", ';'): + ## writeLine(stdout, word) + ## ``` + ## Results in: + ## ``` + ## "" + ## "" + ## "this" + ## "is" + ## "an" + ## "" + ## "example" + ## "" + ## "" + ## "" + ## ``` + ## + ## See also: + ## * `rsplit iterator<#rsplit.i,string,char,int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `split func<#split,string,char,int>`_ + splitCommon(s, sep, maxsplit, 1) -iterator split*(s: string, seps: set[char] = Whitespace): string = +iterator split*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. ## - ## Substrings are separated by a substring containing only `seps`. Note - ## that whole sequences of characters found in ``seps`` will be counted as - ## a single split point and leading/trailing separators will be ignored. - ## The following example: + ## Substrings are separated by a substring containing only `seps`. ## - ## .. code-block:: nim - ## for word in split(" this is an example "): + ## ```nim + ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) + ## ``` ## ## ...generates this output: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "an" ## "example" + ## ``` ## ## And the following code: ## - ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", {';'}): + ## ```nim + ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) + ## ``` ## ## ...produces the same output as the first example. The code: ## - ## .. code-block:: nim + ## ```nim ## let date = "2012-11-20T22:08:08.398990" ## let separators = {' ', '-', ':', 'T'} ## for number in split(date, separators): ## writeLine(stdout, number) + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## "2012" ## "11" ## "20" ## "22" ## "08" ## "08.398990" + ## ``` ## - var last = 0 - assert(not ('\0' in seps)) - while last < len(s): - while s[last] in seps: inc(last) - var first = last - while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! - if first <= last-1: - yield substr(s, first, last-1) + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `rsplit iterator<#rsplit.i,string,set[char],int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `split func<#split,string,set[char],int>`_ + splitCommon(s, seps, maxsplit, 1) -iterator split*(s: string, sep: char): string = - ## Splits the string `s` into substrings using a single separator. +iterator split*(s: string, sep: string, maxsplit: int = -1): string = + ## Splits the string `s` into substrings using a string separator. ## - ## Substrings are separated by the character `sep`. - ## Unlike the version of the iterator which accepts a set of separator - ## characters, this proc will not coalesce groups of the - ## separator, returning a string for each found character. The code: + ## Substrings are separated by the string `sep`. + ## The code: ## - ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", ';'): + ## ```nim + ## for word in split("thisDATAisDATAcorrupted", "DATA"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: - ## "" - ## "" + ## ``` ## "this" ## "is" - ## "an" - ## "" - ## "example" - ## "" - ## "" - ## "" + ## "corrupted" + ## ``` ## - var last = 0 - assert('\0' != sep) - if len(s) > 0: - # `<=` is correct here for the edge cases! - while last <= len(s): - var first = last - while last < len(s) and s[last] != sep: inc(last) - yield substr(s, first, last-1) - inc(last) + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `rsplit iterator<#rsplit.i,string,string,int,bool>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `split func<#split,string,string,int>`_ + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + splitCommon(s, sep, maxsplit, sepLen) -iterator split*(s: string, sep: string): string = - ## Splits the string `s` into substrings using a string separator. + +template rsplitCommon(s, sep, maxsplit, sepLen) = + ## Common code for rsplit functions + var + last = s.len - 1 + first = last + splits = maxsplit + startPos = 0 + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): + dec(first) + if splits == 0: + # No more splits means set first to the beginning + first = -1 + if first == -1: + startPos = 0 + else: + startPos = first + sepLen + yield substr(s, startPos, last) + if splits == 0: break + dec(splits) + dec(first) + last = first + +iterator rsplit*(s: string, sep: char, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char,int>`_ except in **reverse** order. + ## + ## ```nim + ## for piece in "foo:bar".rsplit(':'): + ## echo piece + ## ``` ## - ## Substrings are separated by the string `sep`. - var last = 0 - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and s.substr(last, last + <sep.len) != sep: - inc(last) - yield substr(s, first, last-1) - inc(last, sep.len) + ## Results in: + ## + ## ``` + ## "bar" + ## "foo" + ## ``` + ## + ## Substrings are separated from the right by the char `sep`. + ## + ## See also: + ## * `split iterator<#split.i,string,char,int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `rsplit func<#rsplit,string,char,int>`_ + rsplitCommon(s, sep, maxsplit, 1) -iterator splitLines*(s: string): string = +iterator rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char,int>`_ except in **reverse** order. + ## + ## ```nim + ## for piece in "foo bar".rsplit(WhiteSpace): + ## echo piece + ## ``` + ## + ## Results in: + ## + ## ``` + ## "bar" + ## "foo" + ## ``` + ## + ## Substrings are separated from the right by the set of chars `seps` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `split iterator<#split.i,string,set[char],int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `rsplit func<#rsplit,string,set[char],int>`_ + rsplitCommon(s, seps, maxsplit, 1) + +iterator rsplit*(s: string, sep: string, maxsplit: int = -1, + keepSeparators: bool = false): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,string,int>`_ except in **reverse** order. + ## + ## ```nim + ## for piece in "foothebar".rsplit("the"): + ## echo piece + ## ``` + ## + ## Results in: + ## + ## ``` + ## "bar" + ## "foo" + ## ``` + ## + ## Substrings are separated from the right by the string `sep` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `split iterator<#split.i,string,string,int>`_ + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `rsplit func<#rsplit,string,string,int>`_ + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + rsplitCommon(s, sep, maxsplit, sepLen) + +iterator splitLines*(s: string, keepEol = false): string = ## Splits the string `s` into its containing lines. ## - ## Every `character literal <manual.html#character-literals>`_ newline - ## combination (CR, LF, CR-LF) is supported. The result strings contain no - ## trailing ``\n``. + ## Every `character literal <manual.html#lexical-analysis-character-literals>`_ + ## newline combination (CR, LF, CR-LF) is supported. The result strings + ## contain no trailing end of line characters unless the parameter `keepEol` + ## is set to `true`. ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): ## writeLine(stdout, line) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "" ## "this" ## "is" @@ -449,242 +675,728 @@ iterator splitLines*(s: string): string = ## "" ## "example" ## "" + ## ``` + ## + ## See also: + ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ + ## * `splitLines func<#splitLines,string>`_ var first = 0 var last = 0 + var eolpos = 0 while true: - while s[last] notin {'\0', '\c', '\l'}: inc(last) - yield substr(s, first, last-1) - # skip newlines: - if s[last] == '\l': inc(last) - elif s[last] == '\c': - inc(last) + while last < s.len and s[last] notin {'\c', '\l'}: inc(last) + + eolpos = last + if last < s.len: if s[last] == '\l': inc(last) - else: break # was '\0' - first = last + elif s[last] == '\c': + inc(last) + if last < s.len and s[last] == '\l': inc(last) -proc splitLines*(s: string): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitLines".} = - ## The same as the `splitLines <#splitLines.i,string>`_ iterator, but is a - ## proc that returns a sequence of substrings. - accumulateResult(splitLines(s)) - -proc countLines*(s: string): int {.noSideEffect, - rtl, extern: "nsuCountLines".} = - ## Returns the number of new line separators in the string `s`. - ## - ## This is the same as ``len(splitLines(s))``, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every - ## `character literal <manual.html#character-literals>`_ newline combination - ## (CR, LF, CR-LF) is supported. - ## - ## Despite its name this proc might not actually return the *number of lines* - ## in `s` because the concept of what a line is can vary. For example, a - ## string like ``Hello world`` is a line of text, but the proc will return a - ## value of zero because there are no newline separators. Also, text editors - ## usually don't count trailing newline characters in a text file as a new - ## empty line, but this proc will. - var i = 0 - while i < s.len: - case s[i] - of '\c': - if s[i+1] == '\l': inc i - inc result - of '\l': inc result - else: discard - inc i + yield substr(s, first, if keepEol: last-1 else: eolpos-1) -proc split*(s: string, seps: set[char] = Whitespace): seq[string] {. - noSideEffect, rtl, extern: "nsuSplitCharSet".} = - ## The same as the `split iterator <#split.i,string,set[char]>`_, but is a - ## proc that returns a sequence of substrings. - accumulateResult(split(s, seps)) + # no eol characters consumed means that the string is over + if eolpos == last: + break -proc split*(s: string, sep: char): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitChar".} = - ## The same as the `split iterator <#split.i,string,char>`_, but is a proc - ## that returns a sequence of substrings. - accumulateResult(split(s, sep)) + first = last -proc split*(s: string, sep: string): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitString".} = +iterator splitWhitespace*(s: string, maxsplit: int = -1): string = + ## Splits the string `s` at whitespace stripping leading and trailing + ## whitespace if necessary. If `maxsplit` is specified and is positive, + ## no more than `maxsplit` splits is made. + ## + ## The following code: + ## + ## ```nim + ## let s = " foo \t bar baz " + ## for ms in [-1, 1, 2, 3]: + ## echo "------ maxsplit = ", ms, ":" + ## for item in s.splitWhitespace(maxsplit=ms): + ## echo '"', item, '"' + ## ``` + ## + ## ...results in: + ## + ## ``` + ## ------ maxsplit = -1: + ## "foo" + ## "bar" + ## "baz" + ## ------ maxsplit = 1: + ## "foo" + ## "bar baz " + ## ------ maxsplit = 2: + ## "foo" + ## "bar" + ## "baz " + ## ------ maxsplit = 3: + ## "foo" + ## "bar" + ## "baz" + ## ``` + ## + ## See also: + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + oldSplit(s, Whitespace, maxsplit) + + + +func split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuSplitChar".} = + ## The same as the `split iterator <#split.i,string,char,int>`_ (see its + ## documentation), but is a func that returns a sequence of substrings. + ## + ## See also: + ## * `split iterator <#split.i,string,char,int>`_ + ## * `rsplit func<#rsplit,string,char,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + runnableExamples: + doAssert "a,b,c".split(',') == @["a", "b", "c"] + doAssert "".split(' ') == @[""] + accResult(split(s, sep, maxsplit)) + +func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ + string] {.rtl, extern: "nsuSplitCharSet".} = + ## The same as the `split iterator <#split.i,string,set[char],int>`_ (see its + ## documentation), but is a func that returns a sequence of substrings. + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `split iterator <#split.i,string,set[char],int>`_ + ## * `rsplit func<#rsplit,string,set[char],int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + runnableExamples: + doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] + doAssert "".split({' '}) == @[""] + doAssert "empty seps return unsplit s".split({}) == @["empty seps return unsplit s"] + accResult(split(s, seps, maxsplit)) + +func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuSplitString".} = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. This is a wrapper around the - ## `split iterator <#split.i,string,string>`_. - accumulateResult(split(s, sep)) + ## `split iterator <#split.i,string,string,int>`_. + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `split iterator <#split.i,string,string,int>`_ + ## * `rsplit func<#rsplit,string,string,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + runnableExamples: + doAssert "a,b,c".split(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".split("Elon Musk") == @[""] + doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", + "", "", "", "spaced", "sentence"] + doAssert "a largely spaced sentence".split(" ", maxsplit = 1) == @["a", " largely spaced sentence"] + doAssert "empty sep returns unsplit s".split("") == @["empty sep returns unsplit s"] + accResult(split(s, sep, maxsplit)) + +func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuRSplitChar".} = + ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a func + ## that returns a sequence of substrings in original order. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## ```nim + ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## ``` + ## + ## Results in `tailSplit` containing: + ## + ## ```nim + ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## See also: + ## * `rsplit iterator <#rsplit.i,string,char,int>`_ + ## * `split func<#split,string,char,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + accResult(rsplit(s, sep, maxsplit)) + result.reverse() + +func rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): seq[string] + {.rtl, extern: "nsuRSplitCharSet".} = + ## The same as the `rsplit iterator <#rsplit.i,string,set[char],int>`_, but is a + ## func that returns a sequence of substrings in original order. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## ```nim + ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## ``` + ## + ## Results in `tailSplit` containing: + ## + ## ```nim + ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `rsplit iterator <#rsplit.i,string,set[char],int>`_ + ## * `split func<#split,string,set[char],int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + accResult(rsplit(s, seps, maxsplit)) + result.reverse() + +func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuRSplitString".} = + ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a func + ## that returns a sequence of substrings in original order. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## ```nim + ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## ``` + ## + ## Results in `tailSplit` containing: + ## + ## ```nim + ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## + ## See also: + ## * `rsplit iterator <#rsplit.i,string,string,int,bool>`_ + ## * `split func<#split,string,string,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + runnableExamples: + doAssert "a largely spaced sentence".rsplit(" ", maxsplit = 1) == @[ + "a largely spaced", "sentence"] + doAssert "a,b,c".rsplit(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", + "plan ", "canal panama"] + doAssert "".rsplit("Elon Musk") == @[""] + doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", + "largely", "", "", "", "spaced", "sentence"] + doAssert "empty sep returns unsplit s".rsplit("") == @["empty sep returns unsplit s"] + accResult(rsplit(s, sep, maxsplit)) + result.reverse() + +func splitLines*(s: string, keepEol = false): seq[string] {.rtl, + extern: "nsuSplitLines".} = + ## The same as the `splitLines iterator<#splitLines.i,string>`_ (see its + ## documentation), but is a func that returns a sequence of substrings. + ## + ## See also: + ## * `splitLines iterator<#splitLines.i,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + ## * `countLines func<#countLines,string>`_ + accResult(splitLines(s, keepEol = keepEol)) + +func splitWhitespace*(s: string, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuSplitWhitespace".} = + ## The same as the `splitWhitespace iterator <#splitWhitespace.i,string,int>`_ + ## (see its documentation), but is a func that returns a sequence of substrings. + ## + ## See also: + ## * `splitWhitespace iterator <#splitWhitespace.i,string,int>`_ + ## * `splitLines func<#splitLines,string>`_ + accResult(splitWhitespace(s, maxsplit)) + +func toBin*(x: BiggestInt, len: Positive): string {.rtl, extern: "nsuToBin".} = + ## Converts `x` into its binary representation. + ## + ## The resulting string is always `len` characters long. No leading `0b` + ## prefix is generated. + runnableExamples: + let + a = 29 + b = 257 + doAssert a.toBin(8) == "00011101" + doAssert b.toBin(8) == "00000001" + doAssert b.toBin(9) == "100000001" + var + mask = BiggestUInt 1 + shift = BiggestUInt 0 + assert(len > 0) + result = newString(len) + for j in countdown(len-1, 0): + result[j] = chr(int((BiggestUInt(x) and mask) shr shift) + ord('0')) + inc shift + mask = mask shl BiggestUInt(1) -proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToHex".} = - ## Converts `x` to its hexadecimal representation. +func toOct*(x: BiggestInt, len: Positive): string {.rtl, extern: "nsuToOct".} = + ## Converts `x` into its octal representation. ## - ## The resulting string will be exactly `len` characters long. No prefix like - ## ``0x`` is generated. `x` is treated as an unsigned value. + ## The resulting string is always `len` characters long. No leading `0o` + ## prefix is generated. + ## + ## Do not confuse it with `toOctal func<#toOctal,char>`_. + runnableExamples: + let + a = 62 + b = 513 + doAssert a.toOct(3) == "076" + doAssert b.toOct(3) == "001" + doAssert b.toOct(5) == "01001" + var + mask = BiggestUInt 7 + shift = BiggestUInt 0 + assert(len > 0) + result = newString(len) + for j in countdown(len-1, 0): + result[j] = chr(int((BiggestUInt(x) and mask) shr shift) + ord('0')) + inc shift, 3 + mask = mask shl BiggestUInt(3) + +func toHexImpl(x: BiggestUInt, len: Positive, handleNegative: bool): string = const HexChars = "0123456789ABCDEF" - var - n = x + var n = x result = newString(len) for j in countdown(len-1, 0): - result[j] = HexChars[n and 0xF] + result[j] = HexChars[int(n and 0xF)] n = n shr 4 # handle negative overflow - if n == 0 and x < 0: n = -1 + if n == 0 and handleNegative: n = not(BiggestUInt 0) + +func toHex*[T: SomeInteger](x: T, len: Positive): string = + ## Converts `x` to its hexadecimal representation. + ## + ## The resulting string will be exactly `len` characters long. No prefix like + ## `0x` is generated. `x` is treated as an unsigned value. + runnableExamples: + let + a = 62'u64 + b = 4097'u64 + doAssert a.toHex(3) == "03E" + doAssert b.toHex(3) == "001" + doAssert b.toHex(4) == "1001" + doAssert toHex(62, 3) == "03E" + doAssert toHex(-8, 6) == "FFFFF8" + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), len, x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) + +func toHex*[T: SomeInteger](x: T): string = + ## Shortcut for `toHex(x, T.sizeof * 2)` + runnableExamples: + doAssert toHex(1984'i64) == "00000000000007C0" + doAssert toHex(1984'i16) == "07C0" + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) + +func toHex*(s: string): string {.rtl.} = + ## Converts a bytes string to its hexadecimal representation. + ## + ## The output is twice the input long. No prefix like + ## `0x` is generated. + ## + ## See also: + ## * `parseHexStr func<#parseHexStr,string>`_ for the reverse operation + runnableExamples: + let + a = "1" + b = "A" + c = "\0\255" + doAssert a.toHex() == "31" + doAssert b.toHex() == "41" + doAssert c.toHex() == "00FF" + + const HexChars = "0123456789ABCDEF" + result = newString(s.len * 2) + for pos, c in s: + var n = ord(c) + result[pos * 2 + 1] = HexChars[n and 0xF] + n = n shr 4 + result[pos * 2] = HexChars[n] -proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, - rtl, extern: "nsuIntToStr".} = +func toOctal*(c: char): string {.rtl, extern: "nsuToOctal".} = + ## Converts a character `c` to its octal representation. + ## + ## The resulting string may not have a leading zero. Its length is always + ## exactly 3. + ## + ## Do not confuse it with `toOct func<#toOct,BiggestInt,Positive>`_. + runnableExamples: + doAssert toOctal('1') == "061" + doAssert toOctal('A') == "101" + doAssert toOctal('a') == "141" + doAssert toOctal('!') == "041" + + result = newString(3) + var val = ord(c) + for i in countdown(2, 0): + result[i] = chr(val mod 8 + ord('0')) + val = val div 8 + +func fromBin*[T: SomeInteger](s: string): T = + ## Parses a binary integer value from a string `s`. + ## + ## If `s` is not a valid binary integer, `ValueError` is raised. `s` can have + ## one of the following optional prefixes: `0b`, `0B`. Underscores within + ## `s` are ignored. + ## + ## Does not check for overflow. If the value represented by `s` + ## is too big to fit into a return type, only the value of the rightmost + ## binary digits of `s` is returned without producing an error. + runnableExamples: + let s = "0b_0100_1000_1000_1000_1110_1110_1001_1001" + doAssert fromBin[int](s) == 1216933529 + doAssert fromBin[int8](s) == 0b1001_1001'i8 + doAssert fromBin[int8](s) == -103'i8 + doAssert fromBin[uint8](s) == 153 + doAssert s.fromBin[:int16] == 0b1110_1110_1001_1001'i16 + doAssert s.fromBin[:uint64] == 1216933529'u64 + + let p = parseutils.parseBin(s, result) + if p != s.len or p == 0: + raise newException(ValueError, "invalid binary integer: " & s) + +func fromOct*[T: SomeInteger](s: string): T = + ## Parses an octal integer value from a string `s`. + ## + ## If `s` is not a valid octal integer, `ValueError` is raised. `s` can have + ## one of the following optional prefixes: `0o`, `0O`. Underscores within + ## `s` are ignored. + ## + ## Does not check for overflow. If the value represented by `s` + ## is too big to fit into a return type, only the value of the rightmost + ## octal digits of `s` is returned without producing an error. + runnableExamples: + let s = "0o_123_456_777" + doAssert fromOct[int](s) == 21913087 + doAssert fromOct[int8](s) == 0o377'i8 + doAssert fromOct[int8](s) == -1'i8 + doAssert fromOct[uint8](s) == 255'u8 + doAssert s.fromOct[:int16] == 24063'i16 + doAssert s.fromOct[:uint64] == 21913087'u64 + + let p = parseutils.parseOct(s, result) + if p != s.len or p == 0: + raise newException(ValueError, "invalid oct integer: " & s) + +func fromHex*[T: SomeInteger](s: string): T = + ## Parses a hex integer value from a string `s`. + ## + ## If `s` is not a valid hex integer, `ValueError` is raised. `s` can have + ## one of the following optional prefixes: `0x`, `0X`, `#`. Underscores within + ## `s` are ignored. + ## + ## Does not check for overflow. If the value represented by `s` + ## is too big to fit into a return type, only the value of the rightmost + ## hex digits of `s` is returned without producing an error. + runnableExamples: + let s = "0x_1235_8df6" + doAssert fromHex[int](s) == 305499638 + doAssert fromHex[int8](s) == 0xf6'i8 + doAssert fromHex[int8](s) == -10'i8 + doAssert fromHex[uint8](s) == 246'u8 + doAssert s.fromHex[:int16] == -29194'i16 + doAssert s.fromHex[:uint64] == 305499638'u64 + + let p = parseutils.parseHex(s, result) + if p != s.len or p == 0: + raise newException(ValueError, "invalid hex integer: " & s) + +func intToStr*(x: int, minchars: Positive = 1): string {.rtl, + extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. ## ## The resulting string will be minimally `minchars` characters long. This is ## achieved by adding leading zeros. + runnableExamples: + doAssert intToStr(1984) == "1984" + doAssert intToStr(1984, 6) == "001984" result = $abs(x) for i in 1 .. minchars - len(result): result = '0' & result if x < 0: result = '-' & result -proc parseInt*(s: string): int {.noSideEffect, procvar, - rtl, extern: "nsuParseInt".} = +func parseInt*(s: string): int {.rtl, extern: "nsuParseInt".} = ## Parses a decimal integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. - var L = parseutils.parseInt(s, result, 0) + runnableExamples: + doAssert parseInt("-0042") == -42 + result = 0 + let L = parseutils.parseInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) -proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, - rtl, extern: "nsuParseBiggestInt".} = +func parseBiggestInt*(s: string): BiggestInt {.rtl, + extern: "nsuParseBiggestInt".} = ## Parses a decimal integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. - var L = parseutils.parseBiggestInt(s, result, 0) + result = BiggestInt(0) + let L = parseutils.parseBiggestInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) -proc parseFloat*(s: string): float {.noSideEffect, procvar, - rtl, extern: "nsuParseFloat".} = - ## Parses a decimal floating point value contained in `s`. If `s` is not - ## a valid floating point number, `ValueError` is raised. ``NAN``, - ## ``INF``, ``-INF`` are also supported (case insensitive comparison). - var L = parseutils.parseFloat(s, result, 0) +func parseUInt*(s: string): uint {.rtl, extern: "nsuParseUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + result = uint(0) + let L = parseutils.parseUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +func parseBiggestUInt*(s: string): BiggestUInt {.rtl, + extern: "nsuParseBiggestUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + result = BiggestUInt(0) + let L = parseutils.parseBiggestUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +func parseFloat*(s: string): float {.rtl, extern: "nsuParseFloat".} = + ## Parses a decimal floating point value contained in `s`. + ## + ## If `s` is not a valid floating point number, `ValueError` is raised. + ##`NAN`, `INF`, `-INF` are also supported (case insensitive comparison). + runnableExamples: + doAssert parseFloat("3.14") == 3.14 + doAssert parseFloat("inf") == 1.0/0 + result = 0.0 + let L = parseutils.parseFloat(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid float: " & s) -proc parseHexInt*(s: string): int {.noSideEffect, procvar, - rtl, extern: "nsuParseHexInt".} = +func parseBinInt*(s: string): int {.rtl, extern: "nsuParseBinInt".} = + ## Parses a binary integer value contained in `s`. + ## + ## If `s` is not a valid binary integer, `ValueError` is raised. `s` can have + ## one of the following optional prefixes: `0b`, `0B`. Underscores within + ## `s` are ignored. + runnableExamples: + let + a = "0b11_0101" + b = "111" + doAssert a.parseBinInt() == 53 + doAssert b.parseBinInt() == 7 + + result = 0 + let L = parseutils.parseBin(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid binary integer: " & s) + +func parseOctInt*(s: string): int {.rtl, extern: "nsuParseOctInt".} = + ## Parses an octal integer value contained in `s`. + ## + ## If `s` is not a valid oct integer, `ValueError` is raised. `s` can have one + ## of the following optional prefixes: `0o`, `0O`. Underscores within + ## `s` are ignored. + result = 0 + let L = parseutils.parseOct(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid oct integer: " & s) + +func parseHexInt*(s: string): int {.rtl, extern: "nsuParseHexInt".} = ## Parses a hexadecimal integer value contained in `s`. ## - ## If `s` is not a valid integer, `ValueError` is raised. `s` can have one - ## of the following optional prefixes: ``0x``, ``0X``, ``#``. Underscores + ## If `s` is not a valid hex integer, `ValueError` is raised. `s` can have one + ## of the following optional prefixes: `0x`, `0X`, `#`. Underscores ## within `s` are ignored. - var i = 0 - if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) - elif s[i] == '#': inc(i) - while true: - case s[i] - of '_': inc(i) - of '0'..'9': - result = result shl 4 or (ord(s[i]) - ord('0')) - inc(i) - of 'a'..'f': - result = result shl 4 or (ord(s[i]) - ord('a') + 10) - inc(i) - of 'A'..'F': - result = result shl 4 or (ord(s[i]) - ord('A') + 10) - inc(i) - of '\0': break - else: raise newException(ValueError, "invalid integer: " & s) + result = 0 + let L = parseutils.parseHex(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid hex integer: " & s) -proc parseBool*(s: string): bool = +func generateHexCharToValueMap(): string = + ## Generates a string to map a hex digit to uint value. + result = "" + for inp in 0..255: + let ch = chr(inp) + let o = + case ch + of '0'..'9': inp - ord('0') + of 'a'..'f': inp - ord('a') + 10 + of 'A'..'F': inp - ord('A') + 10 + else: 17 # indicates an invalid hex char + result.add chr(o) + +const hexCharToValueMap = generateHexCharToValueMap() + +func parseHexStr*(s: string): string {.rtl, extern: "nsuParseHexStr".} = + ## Converts hex-encoded string to byte string, e.g.: + ## + ## Raises `ValueError` for an invalid hex values. The comparison is + ## case-insensitive. + ## + ## See also: + ## * `toHex func<#toHex,string>`_ for the reverse operation + runnableExamples: + let + a = "41" + b = "3161" + c = "00ff" + doAssert parseHexStr(a) == "A" + doAssert parseHexStr(b) == "1a" + doAssert parseHexStr(c) == "\0\255" + + if s.len mod 2 != 0: + raise newException(ValueError, "Incorrect hex string len") + result = newString(s.len div 2) + var buf = 0 + for pos, c in s: + let val = hexCharToValueMap[ord(c)].ord + if val == 17: + raise newException(ValueError, "Invalid hex char `" & + c & "` (ord " & $c.ord & ")") + if pos mod 2 == 0: + buf = val + else: + result[pos div 2] = chr(val + buf shl 4) + +func parseBool*(s: string): bool = ## Parses a value into a `bool`. ## - ## If ``s`` is one of the following values: ``y, yes, true, 1, on``, then - ## returns `true`. If ``s`` is one of the following values: ``n, no, false, - ## 0, off``, then returns `false`. If ``s`` is something else a - ## ``ValueError`` exception is raised. + ## If `s` is one of the following values: `y, yes, true, 1, on`, then + ## returns `true`. If `s` is one of the following values: `n, no, false, + ## 0, off`, then returns `false`. If `s` is something else a + ## `ValueError` exception is raised. + runnableExamples: + let a = "n" + doAssert parseBool(a) == false + case normalize(s) of "y", "yes", "true", "1", "on": result = true of "n", "no", "false", "0", "off": result = false else: raise newException(ValueError, "cannot interpret as a bool: " & s) -proc parseEnum*[T: enum](s: string): T = - ## Parses an enum ``T``. +func parseEnum*[T: enum](s: string): T = + ## Parses an enum `T`. This errors at compile time, if the given enum + ## type contains multiple fields with the same string value. ## - ## Raises ``ValueError`` for an invalid value in `s`. The comparison is - ## done in a style insensitive way. - for e in low(T)..high(T): - if cmpIgnoreStyle(s, $e) == 0: - return e - raise newException(ValueError, "invalid enum value: " & s) + ## Raises `ValueError` for an invalid value in `s`. The comparison is + ## done in a style insensitive way (first letter is still case-sensitive). + runnableExamples: + type + MyEnum = enum + first = "1st", + second, + third = "3rd" + + doAssert parseEnum[MyEnum]("1_st") == first + doAssert parseEnum[MyEnum]("second") == second + doAssertRaises(ValueError): + echo parseEnum[MyEnum]("third") -proc parseEnum*[T: enum](s: string, default: T): T = - ## Parses an enum ``T``. + genEnumCaseStmt(T, s, default = nil, ord(low(T)), ord(high(T)), nimIdentNormalize) + +func parseEnum*[T: enum](s: string, default: T): T = + ## Parses an enum `T`. This errors at compile time, if the given enum + ## type contains multiple fields with the same string value. ## ## Uses `default` for an invalid value in `s`. The comparison is done in a - ## style insensitive way. - for e in low(T)..high(T): - if cmpIgnoreStyle(s, $e) == 0: - return e - result = default - -proc repeat*(c: char, count: Natural): string {.noSideEffect, - rtl, extern: "nsuRepeatChar".} = + ## style insensitive way (first letter is still case-sensitive). + runnableExamples: + type + MyEnum = enum + first = "1st", + second, + third = "3rd" + + doAssert parseEnum[MyEnum]("1_st") == first + doAssert parseEnum[MyEnum]("second") == second + doAssert parseEnum[MyEnum]("last", third) == third + + genEnumCaseStmt(T, s, default, ord(low(T)), ord(high(T)), nimIdentNormalize) + +func repeat*(c: char, count: Natural): string {.rtl, extern: "nsuRepeatChar".} = ## Returns a string of length `count` consisting only of - ## the character `c`. You can use this proc to left align strings. Example: - ## - ## .. code-block:: nim - ## proc tabexpand(indent: int, text: string, tabsize: int = 4) = - ## echo '\t'.repeat(indent div tabsize), ' '.repeat(indent mod tabsize), - ## text - ## - ## tabexpand(4, "At four") - ## tabexpand(5, "At five") - ## tabexpand(6, "At six") + ## the character `c`. + runnableExamples: + let a = 'z' + doAssert a.repeat(5) == "zzzzz" result = newString(count) for i in 0..count-1: result[i] = c -proc repeat*(s: string, n: Natural): string {.noSideEffect, - rtl, extern: "nsuRepeatStr".} = - ## Returns String `s` concatenated `n` times. Example: - ## - ## .. code-block:: nim - ## echo "+++ STOP ".repeat(4), "+++" +func repeat*(s: string, n: Natural): string {.rtl, extern: "nsuRepeatStr".} = + ## Returns string `s` concatenated `n` times. + runnableExamples: + doAssert "+ foo +".repeat(3) == "+ foo ++ foo ++ foo +" + result = newStringOfCap(n * s.len) for i in 1..n: result.add(s) -template spaces*(n: Natural): string = repeat(' ',n) - ## Returns a String with `n` space characters. You can use this proc - ## to left align strings. Example: - ## - ## .. code-block:: nim - ## let - ## width = 15 - ## text1 = "Hello user!" - ## text2 = "This is a very long string" - ## echo text1 & spaces(max(0, width - text1.len)) & "|" - ## echo text2 & spaces(max(0, width - text2.len)) & "|" - -proc repeatChar*(count: Natural, c: char = ' '): string {.deprecated.} = - ## deprecated: use repeat() or spaces() - repeat(c, count) - -proc repeatStr*(count: Natural, s: string): string {.deprecated.} = - ## deprecated: use repeat(string, count) or string.repeat(count) - repeat(s, count) - -proc align*(s: string, count: Natural, padding = ' '): string {. - noSideEffect, rtl, extern: "nsuAlignString".} = +func spaces*(n: Natural): string {.inline.} = + ## Returns a string with `n` space characters. You can use this func + ## to left align strings. + ## + ## See also: + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `indent func<#indent,string,Natural,string>`_ + ## * `center func<#center,string,int,char>`_ + runnableExamples: + let + width = 15 + text1 = "Hello user!" + text2 = "This is a very long string" + doAssert text1 & spaces(max(0, width - text1.len)) & "|" == + "Hello user! |" + doAssert text2 & spaces(max(0, width - text2.len)) & "|" == + "This is a very long string|" + repeat(' ', n) + +func align*(s: string, count: Natural, padding = ' '): string {.rtl, + extern: "nsuAlignString".} = ## Aligns a string `s` with `padding`, so that it is of length `count`. ## ## `padding` characters (by default spaces) are added before `s` resulting in - ## right alignment. If ``s.len >= count``, no spaces are added and `s` is - ## returned unchanged. If you need to left align a string use the `repeatChar - ## proc <#repeatChar>`_. Example: - ## - ## .. code-block:: nim - ## assert align("abc", 4) == " abc" - ## assert align("a", 0) == "a" - ## assert align("1232", 6) == " 1232" - ## assert align("1232", 6, '#') == "##1232" + ## right alignment. If `s.len >= count`, no spaces are added and `s` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## func<#alignLeft,string,Natural,char>`_. + ## + ## See also: + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + ## * `center func<#center,string,int,char>`_ + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#') == "##1232" if s.len < count: result = newString(count) let spaces = count - s.len @@ -693,81 +1405,83 @@ proc align*(s: string, count: Natural, padding = ' '): string {. else: result = s -iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ - token: string, isSep: bool] = - ## Tokenizes the string `s` into substrings. - ## - ## Substrings are separated by a substring containing only `seps`. - ## Examples: - ## - ## .. code-block:: nim - ## for word in tokenize(" this is an example "): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: nim - ## (" ", true) - ## ("this", false) - ## (" ", true) - ## ("is", false) - ## (" ", true) - ## ("an", false) - ## (" ", true) - ## ("example", false) - ## (" ", true) - var i = 0 - while true: - var j = i - var isSep = s[j] in seps - while j < s.len and (s[j] in seps) == isSep: inc(j) - if j > i: - yield (substr(s, i, j-1), isSep) - else: - break - i = j +func alignLeft*(s: string, count: Natural, padding = ' '): string = + ## Left-Aligns a string `s` with `padding`, so that it is of length `count`. + ## + ## `padding` characters (by default spaces) are added after `s` resulting in + ## left alignment. If `s.len >= count`, no spaces are added and `s` is + ## returned unchanged. If you need to right align a string use the `align + ## func<#align,string,Natural,char>`_. + ## + ## See also: + ## * `align func<#align,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + ## * `center func<#center,string,int,char>`_ + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#') == "1232##" + if s.len < count: + result = newString(count) + if s.len > 0: + result[0 .. (s.len - 1)] = s + for i in s.len ..< count: + result[i] = padding + else: + result = s -proc wordWrap*(s: string, maxLineWidth = 80, - splitLongWords = true, - seps: set[char] = Whitespace, - newLine = "\n"): string {. - noSideEffect, rtl, extern: "nsuWordWrap".} = - ## Word wraps `s`. - result = newStringOfCap(s.len + s.len shr 6) - var spaceLeft = maxLineWidth - var lastSep = "" - for word, isSep in tokenize(s, seps): - if isSep: - lastSep = word - spaceLeft = spaceLeft - len(word) - continue - if len(word) > spaceLeft: - if splitLongWords and len(word) > maxLineWidth: - result.add(substr(word, 0, spaceLeft-1)) - var w = spaceLeft+1 - var wordLeft = len(word) - spaceLeft - while wordLeft > 0: - result.add(newLine) - var L = min(maxLineWidth, wordLeft) - spaceLeft = maxLineWidth - L - result.add(substr(word, w, w+L-1)) - inc(w, L) - dec(wordLeft, L) - else: - spaceLeft = maxLineWidth - len(word) - result.add(newLine) - result.add(word) +func center*(s: string, width: int, fillChar: char = ' '): string {.rtl, + extern: "nsuCenterString".} = + ## Return the contents of `s` centered in a string `width` long using + ## `fillChar` (default: space) as padding. + ## + ## The original string is returned if `width` is less than or equal + ## to `s.len`. + ## + ## See also: + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + runnableExamples: + let a = "foo" + doAssert a.center(2) == "foo" + doAssert a.center(5) == " foo " + doAssert a.center(6) == " foo " + if width <= s.len: return s + result = newString(width) + # Left padding will be one fillChar + # smaller if there are an odd number + # of characters + let + charsLeft = (width - s.len) + leftPadding = charsLeft div 2 + for i in 0 ..< width: + if i >= leftPadding and i < leftPadding + s.len: + # we are where the string should be located + result[i] = s[i-leftPadding] else: - spaceLeft = spaceLeft - len(word) - result.add(lastSep & word) - lastSep.setLen(0) - -proc indent*(s: string, count: Natural, padding: string = " "): string - {.noSideEffect, rtl, extern: "nsuIndent".} = - ## Indents each line in ``s`` by ``count`` amount of ``padding``. - ## - ## **Note:** This currently does not preserve the specific new line characters - ## used. + # we are either before or after where + # the string s should go + result[i] = fillChar + +func indent*(s: string, count: Natural, padding: string = " "): string {.rtl, + extern: "nsuIndent".} = + ## Indents each line in `s` by `count` amount of `padding`. + ## + ## **Note:** This does not preserve the new line characters used in `s`. + ## + ## See also: + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `unindent func<#unindent,string,Natural,string>`_ + ## * `dedent func<#dedent,string,Natural>`_ + runnableExamples: + doAssert indent("First line\c\l and second line.", 2) == + " First line\l and second line." result = "" var i = 0 for line in s.splitLines(): @@ -778,99 +1492,361 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, eatAllIndent = false): string {. - noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents `s`. - result = newStringOfCap(s.len) +func unindent*(s: string, count: Natural = int.high, + padding: string = " "): string {.rtl, extern: "nsuUnindent".} = + ## Unindents each line in `s` by `count` amount of `padding`. + ## + ## **Note:** This does not preserve the new line characters used in `s`. + ## + ## See also: + ## * `dedent func<#dedent,string,Natural>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + runnableExamples: + let x = """ + Hello + There + """.unindent() + + doAssert x == "Hello\nThere\n" + result = "" var i = 0 - var pattern = true - var indent = 0 - while s[i] == ' ': inc i - var level = if i == 0: -1 else: i - while i < s.len: - if s[i] == ' ': - if i > 0 and s[i-1] in {'\l', '\c'}: - pattern = true - indent = 0 - if pattern: - inc(indent) - if indent > level and not eatAllIndent: - result.add(s[i]) - if level < 0: level = indent - else: - # a space somewhere: do not delete - result.add(s[i]) - else: - pattern = false - result.add(s[i]) - inc i + for line in s.splitLines(): + if i != 0: + result.add("\n") + var indentCount = 0 + for j in 0..<count.int: + indentCount.inc + if j + padding.len-1 >= line.len or line[j .. j + padding.len-1] != padding: + indentCount = j + break + result.add(line[indentCount*padding.len .. ^1]) + i.inc -proc startsWith*(s, prefix: string): bool {.noSideEffect, - rtl, extern: "nsuStartsWith".} = - ## Returns true iff ``s`` starts with ``prefix``. - ## - ## If ``prefix == ""`` true is returned. +func indentation*(s: string): Natural {.since: (1, 3).} = + ## Returns the amount of indentation all lines of `s` have in common, + ## ignoring lines that consist only of whitespace. + result = int.high + for line in s.splitLines: + for i, c in line: + if i >= result: break + elif c != ' ': + result = i + break + if result == int.high: + result = 0 + +func dedent*(s: string, count: Natural = indentation(s)): string {.rtl, + extern: "nsuDedent", since: (1, 3).} = + ## Unindents each line in `s` by `count` amount of `padding`. + ## The only difference between this and the + ## `unindent func<#unindent,string,Natural,string>`_ is that this by default + ## only cuts off the amount of indentation that all lines of `s` share as + ## opposed to all indentation. It only supports spaces as padding. + ## + ## **Note:** This does not preserve the new line characters used in `s`. + ## + ## See also: + ## * `unindent func<#unindent,string,Natural,string>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + runnableExamples: + let x = """ + Hello + There + """.dedent() + + doAssert x == "Hello\n There\n" + unindent(s, count, " ") + +func delete*(s: var string, slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. + ## + ## This operation moves all elements after `s[slice]` in linear time, and + ## is the string analog to `sequtils.delete`. + runnableExamples: + var a = "abcde" + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == "abcde" + a.delete(4..4) + assert a == "abcd" + a.delete(1..2) + assert a == "ad" + a.delete(1..<1) # empty slice + assert a == "ad" + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + # if j < s.len: moveMem(addr s[i], addr s[j], s.len - j) # pending benchmark + while i < newLen: + s[i] = s[j] + inc(i) + inc(j) + setLen(s, newLen) + +func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete", + deprecated: "use `delete(s, first..last)`".} = + ## Deletes in `s` the characters at positions `first .. last` (both ends included). + runnableExamples("--warning:deprecated:off"): + var a = "abracadabra" + + a.delete(4, 5) + doAssert a == "abradabra" + + a.delete(1, 6) + doAssert a == "ara" + + a.delete(2, 999) + doAssert a == "ar" + + var i = first + var j = min(len(s), last+1) + var newLen = len(s)-j+i + while i < newLen: + s[i] = s[j] + inc(i) + inc(j) + setLen(s, newLen) + +func startsWith*(s: string, prefix: char): bool {.inline.} = + ## Returns true if `s` starts with character `prefix`. + ## + ## See also: + ## * `endsWith func<#endsWith,string,char>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removePrefix func<#removePrefix,string,char>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.startsWith('a') == true + doAssert a.startsWith('b') == false + result = s.len > 0 and s[0] == prefix + +func startsWith*(s, prefix: string): bool {.rtl, extern: "nsuStartsWith".} = + ## Returns true if `s` starts with string `prefix`. + ## + ## If `prefix == ""` true is returned. + ## + ## See also: + ## * `endsWith func<#endsWith,string,string>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removePrefix func<#removePrefix,string,string>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.startsWith("abra") == true + doAssert a.startsWith("bra") == false + startsWithImpl(s, prefix) + +func endsWith*(s: string, suffix: char): bool {.inline.} = + ## Returns true if `s` ends with `suffix`. + ## + ## See also: + ## * `startsWith func<#startsWith,string,char>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removeSuffix func<#removeSuffix,string,char>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.endsWith('a') == true + doAssert a.endsWith('b') == false + result = s.len > 0 and s[s.high] == suffix + +func endsWith*(s, suffix: string): bool {.rtl, extern: "nsuEndsWith".} = + ## Returns true if `s` ends with `suffix`. + ## + ## If `suffix == ""` true is returned. + ## + ## See also: + ## * `startsWith func<#startsWith,string,string>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removeSuffix func<#removeSuffix,string,string>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.endsWith("abra") == true + doAssert a.endsWith("dab") == false + endsWithImpl(s, suffix) + +func continuesWith*(s, substr: string, start: Natural): bool {.rtl, + extern: "nsuContinuesWith".} = + ## Returns true if `s` continues with `substr` at position `start`. + ## + ## If `substr == ""` true is returned. + ## + ## See also: + ## * `startsWith func<#startsWith,string,string>`_ + ## * `endsWith func<#endsWith,string,string>`_ + runnableExamples: + let a = "abracadabra" + doAssert a.continuesWith("ca", 4) == true + doAssert a.continuesWith("ca", 5) == false + doAssert a.continuesWith("dab", 6) == true var i = 0 while true: - if prefix[i] == '\0': return true - if s[i] != prefix[i]: return false + if i >= substr.len: return true + if i+start >= s.len or s[i+start] != substr[i]: return false inc(i) -proc endsWith*(s, suffix: string): bool {.noSideEffect, - rtl, extern: "nsuEndsWith".} = - ## Returns true iff ``s`` ends with ``suffix``. + +func removePrefix*(s: var string, chars: set[char] = Newlines) {.rtl, + extern: "nsuRemovePrefixCharSet".} = + ## Removes all characters from `chars` from the start of the string `s` + ## (in-place). + ## + ## See also: + ## * `removeSuffix func<#removeSuffix,string,set[char]>`_ + runnableExamples: + var userInput = "\r\n*~Hello World!" + userInput.removePrefix + doAssert userInput == "*~Hello World!" + userInput.removePrefix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "?!?Hello!?!" + otherInput.removePrefix({'!', '?'}) + doAssert otherInput == "Hello!?!" + + var start = 0 + while start < s.len and s[start] in chars: start += 1 + if start > 0: s.delete(0..start - 1) + +func removePrefix*(s: var string, c: char) {.rtl, + extern: "nsuRemovePrefixChar".} = + ## Removes all occurrences of a single character (in-place) from the start + ## of a string. + ## + ## See also: + ## * `removeSuffix func<#removeSuffix,string,char>`_ + ## * `startsWith func<#startsWith,string,char>`_ + runnableExamples: + var ident = "pControl" + ident.removePrefix('p') + doAssert ident == "Control" + removePrefix(s, chars = {c}) + +func removePrefix*(s: var string, prefix: string) {.rtl, + extern: "nsuRemovePrefixString".} = + ## Remove the first matching prefix (in-place) from a string. + ## + ## See also: + ## * `removeSuffix func<#removeSuffix,string,string>`_ + ## * `startsWith func<#startsWith,string,string>`_ + runnableExamples: + var answers = "yesyes" + answers.removePrefix("yes") + doAssert answers == "yes" + if s.startsWith(prefix) and prefix.len > 0: + s.delete(0..prefix.len - 1) + +func removeSuffix*(s: var string, chars: set[char] = Newlines) {.rtl, + extern: "nsuRemoveSuffixCharSet".} = + ## Removes all characters from `chars` from the end of the string `s` + ## (in-place). + ## + ## See also: + ## * `removePrefix func<#removePrefix,string,set[char]>`_ + runnableExamples: + var userInput = "Hello World!*~\r\n" + userInput.removeSuffix + doAssert userInput == "Hello World!*~" + userInput.removeSuffix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "Hello!?!" + otherInput.removeSuffix({'!', '?'}) + doAssert otherInput == "Hello" + + if s.len == 0: return + var last = s.high + while last > -1 and s[last] in chars: last -= 1 + s.setLen(last + 1) + +func removeSuffix*(s: var string, c: char) {.rtl, + extern: "nsuRemoveSuffixChar".} = + ## Removes all occurrences of a single character (in-place) from the end + ## of a string. ## - ## If ``suffix == ""`` true is returned. - var i = 0 - var j = len(s) - len(suffix) - while i+j <% s.len: - if s[i+j] != suffix[i]: return false - inc(i) - if suffix[i] == '\0': return true + ## See also: + ## * `removePrefix func<#removePrefix,string,char>`_ + ## * `endsWith func<#endsWith,string,char>`_ + runnableExamples: + var table = "users" + table.removeSuffix('s') + doAssert table == "user" + + var dots = "Trailing dots......." + dots.removeSuffix('.') + doAssert dots == "Trailing dots" + + removeSuffix(s, chars = {c}) -proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, - rtl, extern: "nsuContinuesWith".} = - ## Returns true iff ``s`` continues with ``substr`` at position ``start``. +func removeSuffix*(s: var string, suffix: string) {.rtl, + extern: "nsuRemoveSuffixString".} = + ## Remove the first matching suffix (in-place) from a string. ## - ## If ``substr == ""`` true is returned. - var i = 0 - while true: - if substr[i] == '\0': return true - if s[i+start] != substr[i]: return false - inc(i) + ## See also: + ## * `removePrefix func<#removePrefix,string,string>`_ + ## * `endsWith func<#endsWith,string,string>`_ + runnableExamples: + var answers = "yeses" + answers.removeSuffix("es") + doAssert answers == "yes" + var newLen = s.len + if s.endsWith(suffix): + newLen -= len(suffix) + s.setLen(newLen) + -proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) - {.noSideEffect, inline.} = +func addSep*(dest: var string, sep = ", ", startLen: Natural = 0) {.inline.} = ## Adds a separator to `dest` only if its length is bigger than `startLen`. ## ## A shorthand for: ## - ## .. code-block:: nim + ## ```nim ## if dest.len > startLen: add(dest, sep) + ## ``` ## ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than ## `startLen`. The following example creates a string describing - ## an array of integers: - ## - ## .. code-block:: nim - ## var arr = "[" - ## for x in items([2, 3, 5, 7, 11]): - ## addSep(arr, startLen=len("[")) - ## add(arr, $x) - ## add(arr, "]") + ## an array of integers. + runnableExamples: + var arr = "[" + for x in items([2, 3, 5, 7, 11]): + addSep(arr, startLen = len("[")) + add(arr, $x) + add(arr, "]") + doAssert arr == "[2, 3, 5, 7, 11]" + if dest.len > startLen: add(dest, sep) -proc allCharsInSet*(s: string, theSet: set[char]): bool = - ## Returns true iff each character of `s` is in the set `theSet`. +func allCharsInSet*(s: string, theSet: set[char]): bool = + ## Returns true if every character of `s` is in the set `theSet`. + runnableExamples: + doAssert allCharsInSet("aeea", {'a', 'e'}) == true + doAssert allCharsInSet("", {'a', 'e'}) == true + for c in items(s): if c notin theSet: return false return true -proc abbrev*(s: string, possibilities: openArray[string]): int = - ## Returns the index of the first item in `possibilities` if not ambiguous. +func abbrev*(s: string, possibilities: openArray[string]): int = + ## Returns the index of the first item in `possibilities` which starts + ## with `s`, if not ambiguous. ## ## Returns -1 if no item has been found and -2 if multiple items match. + runnableExamples: + doAssert abbrev("fac", ["college", "faculty", "industry"]) == 1 + doAssert abbrev("foo", ["college", "faculty", "industry"]) == -1 # Not found + doAssert abbrev("fac", ["college", "faculty", "faculties"]) == -2 # Ambiguous + doAssert abbrev("college", ["college", "colleges", "industry"]) == 0 + result = -1 # none found for i in 0..possibilities.len-1: if possibilities[i].startsWith(s): @@ -882,9 +1858,12 @@ proc abbrev*(s: string, possibilities: openArray[string]): int = # --------------------------------------------------------------------------- -proc join*(a: openArray[string], sep: string): string {. - noSideEffect, rtl, extern: "nsuJoinSep".} = - ## Concatenates all strings in `a` separating them with `sep`. +func join*(a: openArray[string], sep: string = ""): string {.rtl, + extern: "nsuJoinSep".} = + ## Concatenates all strings in the container `a`, separating them with `sep`. + runnableExamples: + doAssert join(["A", "B", "Conclusion"], " -> ") == "A -> B -> Conclusion" + if len(a) > 0: var L = sep.len * (a.len-1) for i in 0..high(a): inc(L, a[i].len) @@ -896,74 +1875,233 @@ proc join*(a: openArray[string], sep: string): string {. else: result = "" -proc join*(a: openArray[string]): string {. - noSideEffect, rtl, extern: "nsuJoin".} = - ## Concatenates all strings in `a`. - if len(a) > 0: - var L = 0 - for i in 0..high(a): inc(L, a[i].len) - result = newStringOfCap(L) - for i in 0..high(a): add(result, a[i]) - else: - result = "" +proc join*[T: not string](a: openArray[T], sep: string = ""): string = + ## Converts all elements in the container `a` to strings using `$`, + ## and concatenates them with `sep`. + runnableExamples: + doAssert join([1, 2, 3], " -> ") == "1 -> 2 -> 3" + + result = "" + for i, x in a: + if i > 0: + add(result, sep) + add(result, $x) type - SkipTable = array[char, int] + SkipTable* = array[char, int] ## Character table for efficient substring search. -proc preprocessSub(sub: string, a: var SkipTable) = - var m = len(sub) - for i in 0..0xff: a[chr(i)] = m+1 - for i in 0..m-1: a[sub[i]] = m-i +func initSkipTable*(a: var SkipTable, sub: string) {.rtl, + extern: "nsuInitSkipTable".} = + ## Initializes table `a` for efficient search of substring `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + # TODO: this should be the `default()` initializer for the type. + let m = len(sub) + fill(a, m) -proc findAux(s, sub: string, start: int, a: SkipTable): int = - # Fast "quick search" algorithm: - var - m = len(sub) - n = len(s) - # search: - var j = start - while j <= n - m: - block match: - for k in 0..m-1: - if sub[k] != s[k+j]: break match - return j - inc(j, a[s[j+m]]) - return -1 + for i in 0 ..< m - 1: + a[sub[i]] = m - 1 - i + +func initSkipTable*(sub: string): SkipTable {.noinit, rtl, + extern: "nsuInitNewSkipTable".} = + ## Returns a new table initialized for `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + initSkipTable(result, sub) + +func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. + rtl, extern: "nsuFindStrA".} = + ## Searches for `sub` in `s` inside range `start..last` using preprocessed + ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last + ## element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + let + last = if last < 0: s.high else: last + subLast = sub.len - 1 + + if subLast == -1: + # this was an empty needle string, + # we count this as match in the first possible position: + return start + + # This is an implementation of the Boyer-Moore Horspool algorithms + # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + result = -1 + var skip = start + + while last - skip >= subLast: + var i = subLast + while s[skip + i] == sub[i]: + if i == 0: + return skip + dec i + inc skip, a[s[skip + subLast]] + +when not (defined(js) or defined(nimdoc) or defined(nimscript)): + func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} + const hasCStringBuiltin = true +else: + const hasCStringBuiltin = false + +func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuFindChar".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. + ## + ## See also: + ## * `rfind func<#rfind,string,char,Natural,int>`_ + ## * `replace func<#replace,string,char,char>`_ + result = -1 + let last = if last < 0: s.high else: last + + template findImpl = + for i in int(start)..last: + if s[i] == sub: + return i + + when nimvm: + findImpl() + else: + when hasCStringBuiltin: + let length = last-start+1 + if length > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) + if not found.isNil: + return cast[int](found) -% cast[int](s.cstring) + else: + findImpl() -proc find*(s, sub: string, start: Natural = 0): int {.noSideEffect, - rtl, extern: "nsuFindStr".} = - ## Searches for `sub` in `s` starting at position `start`. +func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. + rtl, extern: "nsuFindCharSet".} = + ## Searches for `chars` in `s` inside range `start..last` (both ends included). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). + ## + ## If `s` contains none of the characters in `chars`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. + ## + ## See also: + ## * `rfind func<#rfind,string,set[char],Natural,int>`_ + ## * `multiReplace func<#multiReplace,string,varargs[]>`_ + result = -1 + let last = if last < 0: s.high else: last + for i in int(start)..last: + if s[i] in chars: + return i + +when defined(linux): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: """#define _GNU_SOURCE +#include <string.h>""".} +elif defined(bsd) or (defined(macosx) and not defined(ios)): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: "#include <string.h>".} + +func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuFindStr".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - var a {.noinit.}: SkipTable - preprocessSub(sub, a) - result = findAux(s, sub, start, a) + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. + ## + ## See also: + ## * `rfind func<#rfind,string,string,Natural,int>`_ + ## * `replace func<#replace,string,string,string>`_ + if sub.len > s.len - start: return -1 + if sub.len == 1: return find(s, sub[0], start, last) + + template useSkipTable = + result = find(initSkipTable(sub), s, sub, start, last) + + when nimvm: + useSkipTable() + else: + when declared(memmem): + let subLen = sub.len + if last < 0 and start < s.len and subLen != 0: + let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) + result = if not found.isNil: + cast[int](found) -% cast[int](s.cstring) + else: + -1 + else: + useSkipTable() + else: + useSkipTable() -proc find*(s: string, sub: char, start: Natural = 0): int {.noSideEffect, - rtl, extern: "nsuFindChar".} = - ## Searches for `sub` in `s` starting at position `start`. +func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuRFindChar".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included) + ## in reverse -- starting at high indexes and moving lower to the first + ## character or `start`. If `last` is unspecified, it defaults to `s.high` + ## (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - for i in start..len(s)-1: + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. + ## + ## See also: + ## * `find func<#find,string,char,Natural,int>`_ + let last = if last == -1: s.high else: last + for i in countdown(last, start): if sub == s[i]: return i return -1 -proc find*(s: string, chars: set[char], start: Natural = 0): int {.noSideEffect, - rtl, extern: "nsuFindCharSet".} = - ## Searches for `chars` in `s` starting at position `start`. +func rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. + rtl, extern: "nsuRFindCharSet".} = + ## Searches for `chars` in `s` inside range `start..last` (both ends + ## included) in reverse -- starting at high indexes and moving lower to the + ## first character or `start`. If `last` is unspecified, it defaults to + ## `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. - for i in start..s.len-1: + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. + ## + ## See also: + ## * `find func<#find,string,set[char],Natural,int>`_ + let last = if last == -1: s.high else: last + for i in countdown(last, start): if s[i] in chars: return i return -1 -proc rfind*(s, sub: string, start: int = -1): int {.noSideEffect.} = - ## Searches for `sub` in `s` in reverse, starting at `start` and going - ## backwards to 0. +func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuRFindStr".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included) + ## included) in reverse -- starting at high indexes and moving lower to the + ## first character or `start`. If `last` is unspecified, it defaults to + ## `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - let realStart = if start == -1: s.len else: start - for i in countdown(realStart-sub.len, 0): + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. + ## + ## See also: + ## * `find func<#find,string,string,Natural,int>`_ + if sub.len == 0: + let rightIndex: Natural = if last < 0: s.len else: last + return max(start, rightIndex) + if sub.len > s.len - start: + return -1 + let last = if last == -1: s.high else: last + result = 0 + for i in countdown(last - sub.len + 1, start): for j in 0..sub.len-1: result = i if sub[j] != s[i+j]: @@ -972,90 +2110,139 @@ proc rfind*(s, sub: string, start: int = -1): int {.noSideEffect.} = if result != -1: return return -1 -proc rfind*(s: string, sub: char, start: int = -1): int {.noSideEffect, - rtl.} = - ## Searches for `sub` in `s` in reverse starting at position `start`. + +func count*(s: string, sub: char): int {.rtl, extern: "nsuCountChar".} = + ## Counts the occurrences of the character `sub` in the string `s`. ## - ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - let realStart = if start == -1: s.len-1 else: start - for i in countdown(realStart, 0): - if sub == s[i]: return i - return -1 + ## See also: + ## * `countLines func<#countLines,string>`_ + result = 0 + for c in s: + if c == sub: inc result -proc count*(s: string, sub: string, overlapping: bool = false): int {. - noSideEffect, rtl, extern: "nsuCountString".} = - ## Count the occurrences of a substring `sub` in the string `s`. +func count*(s: string, subs: set[char]): int {.rtl, + extern: "nsuCountCharSet".} = + ## Counts the occurrences of the group of character `subs` in the string `s`. + ## + ## See also: + ## * `countLines func<#countLines,string>`_ + doAssert card(subs) > 0 + result = 0 + for c in s: + if c in subs: inc result + +func count*(s: string, sub: string, overlapping: bool = false): int {.rtl, + extern: "nsuCountString".} = + ## Counts the occurrences of a substring `sub` in the string `s`. ## Overlapping occurrences of `sub` only count when `overlapping` - ## is set to true. + ## is set to true (default: false). + ## + ## See also: + ## * `countLines func<#countLines,string>`_ + doAssert sub.len > 0 + result = 0 var i = 0 while true: i = s.find(sub, i) - if i < 0: - break - if overlapping: - inc i - else: - i += sub.len + if i < 0: break + if overlapping: inc i + else: i += sub.len inc result -proc count*(s: string, sub: char): int {.noSideEffect, - rtl, extern: "nsuCountChar".} = - ## Count the occurrences of the character `sub` in the string `s`. - for c in s: - if c == sub: +func countLines*(s: string): int {.rtl, extern: "nsuCountLines".} = + ## Returns the number of lines in the string `s`. + ## + ## This is the same as `len(splitLines(s))`, but much more efficient + ## because it doesn't modify the string creating temporary objects. Every + ## `character literal <manual.html#lexical-analysis-character-literals>`_ + ## newline combination (CR, LF, CR-LF) is supported. + ## + ## In this context, a line is any string separated by a newline combination. + ## A line can be an empty string. + ## + ## See also: + ## * `splitLines func<#splitLines,string>`_ + runnableExamples: + doAssert countLines("First line\l and second line.") == 2 + result = 1 + var i = 0 + while i < s.len: + case s[i] + of '\c': + if i+1 < s.len and s[i+1] == '\l': inc i inc result + of '\l': inc result + else: discard + inc i -proc count*(s: string, subs: set[char]): int {.noSideEffect, - rtl, extern: "nsuCountCharSet".} = - ## Count the occurrences of the group of character `subs` in the string `s`. - for c in s: - if c in subs: - inc result -proc quoteIfContainsWhite*(s: string): string {.deprecated.} = - ## Returns ``'"' & s & '"'`` if `s` contains a space and does not - ## start with a quote, else returns `s`. +func contains*(s, sub: string): bool = + ## Same as `find(s, sub) >= 0`. ## - ## **DEPRECATED** as it was confused for shell quoting function. For this - ## application use `osproc.quoteShell <osproc.html#quoteShell>`_. - if find(s, {' ', '\t'}) >= 0 and s[0] != '"': - result = '"' & s & '"' - else: - result = s - -proc contains*(s: string, c: char): bool {.noSideEffect.} = - ## Same as ``find(s, c) >= 0``. - return find(s, c) >= 0 - -proc contains*(s, sub: string): bool {.noSideEffect.} = - ## Same as ``find(s, sub) >= 0``. + ## See also: + ## * `find func<#find,string,string,Natural,int>`_ return find(s, sub) >= 0 -proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} = - ## Same as ``find(s, chars) >= 0``. +func contains*(s: string, chars: set[char]): bool = + ## Same as `find(s, chars) >= 0`. + ## + ## See also: + ## * `find func<#find,string,set[char],Natural,int>`_ return find(s, chars) >= 0 -proc replace*(s, sub: string, by = ""): string {.noSideEffect, - rtl, extern: "nsuReplaceStr".} = - ## Replaces `sub` in `s` by the string `by`. - var a {.noinit.}: SkipTable +func replace*(s, sub: string, by = ""): string {.rtl, + extern: "nsuReplaceStr".} = + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. + ## + ## See also: + ## * `find func<#find,string,string,Natural,int>`_ + ## * `replace func<#replace,string,char,char>`_ for replacing + ## single characters + ## * `replaceWord func<#replaceWord,string,string,string>`_ + ## * `multiReplace func<#multiReplace,string,varargs[]>`_ result = "" - preprocessSub(sub, a) - var i = 0 - while true: - var j = findAux(s, sub, i, a) - if j < 0: break - add result, substr(s, i, j - 1) - add result, by - i = j + len(sub) - # copy the rest: - add result, substr(s, i) - -proc replace*(s: string, sub, by: char): string {.noSideEffect, - rtl, extern: "nsuReplaceChar".} = - ## Replaces `sub` in `s` by the character `by`. - ## - ## Optimized version of `replace <#replace,string,string>`_ for characters. + let subLen = sub.len + if subLen == 0: + result = s + elif subLen == 1: + # when the pattern is a single char, we use a faster + # char-based search that doesn't need a skip table: + let c = sub[0] + let last = s.high + var i = 0 + while true: + let j = find(s, c, i, last) + if j < 0: break + add result, substr(s, i, j - 1) + add result, by + i = j + subLen + # copy the rest: + add result, substr(s, i) + else: + var a = initSkipTable(sub) + let last = s.high + var i = 0 + while true: + let j = find(a, s, sub, i, last) + if j < 0: break + add result, substr(s, i, j - 1) + add result, by + i = j + subLen + # copy the rest: + add result, substr(s, i) + +func replace*(s: string, sub, by: char): string {.rtl, + extern: "nsuReplaceChar".} = + ## Replaces every occurrence of the character `sub` in `s` with the character + ## `by`. + ## + ## Optimized version of `replace <#replace,string,string,string>`_ for + ## characters. + ## + ## See also: + ## * `find func<#find,string,char,Natural,int>`_ + ## * `replaceWord func<#replaceWord,string,string,string>`_ + ## * `multiReplace func<#multiReplace,string,varargs[]>`_ result = newString(s.len) var i = 0 while i < s.len: @@ -1063,135 +2250,143 @@ proc replace*(s: string, sub, by: char): string {.noSideEffect, else: result[i] = s[i] inc(i) -proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, - rtl, extern: "nsuReplaceWord".} = - ## Replaces `sub` in `s` by the string `by`. +func replaceWord*(s, sub: string, by = ""): string {.rtl, + extern: "nsuReplaceWord".} = + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## Each occurrence of `sub` has to be surrounded by word boundaries - ## (comparable to ``\\w`` in regular expressions), otherwise it is not + ## (comparable to `\b` in regular expressions), otherwise it is not ## replaced. + if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} - var a {.noinit.}: SkipTable result = "" - preprocessSub(sub, a) + var a = initSkipTable(sub) var i = 0 - while true: - var j = findAux(s, sub, i, a) - if j < 0: break - # word boundary? - if (j == 0 or s[j-1] notin wordChars) and - (j+sub.len >= s.len or s[j+sub.len] notin wordChars): - add result, substr(s, i, j - 1) - add result, by - i = j + len(sub) - else: - add result, substr(s, i, j) - i = j + 1 - # copy the rest: - add result, substr(s, i) - -proc delete*(s: var string, first, last: int) {.noSideEffect, - rtl, extern: "nsuDelete".} = - ## Deletes in `s` the characters at position `first` .. `last`. - ## - ## This modifies `s` itself, it does not return a copy. - var i = first - var j = last+1 - var newLen = len(s)-j+i - while i < newLen: - s[i] = s[j] - inc(i) - inc(j) - setLen(s, newLen) - -proc parseOctInt*(s: string): int {.noSideEffect, - rtl, extern: "nsuParseOctInt".} = - ## Parses an octal integer value contained in `s`. - ## - ## If `s` is not a valid integer, `ValueError` is raised. `s` can have one - ## of the following optional prefixes: ``0o``, ``0O``. Underscores within - ## `s` are ignored. + let last = s.high + let sublen = sub.len + if sublen > 0: + while true: + var j = find(a, s, sub, i, last) + if j < 0: break + # word boundary? + if (j == 0 or s[j-1] notin wordChars) and + (j+sub.len >= s.len or s[j+sub.len] notin wordChars): + add result, substr(s, i, j - 1) + add result, by + i = j + sublen + else: + add result, substr(s, i, j) + i = j + 1 + # copy the rest: + add result, substr(s, i) + +func multiReplace*(s: string, replacements: varargs[(string, string)]): string = + ## Same as `replace<#replace,string,string,string>`_, but specialized for + ## doing multiple replacements in a single pass through the input string. + ## + ## `multiReplace` scans the input string from left to right and replaces the + ## matching substrings in the same order as passed in the argument list. + ## + ## The implications of the order of scanning the string and matching the + ## replacements: + ## - In case of multiple matches at a given position, the earliest + ## replacement is applied. + ## - Overlaps are not handled. After performing a replacement, the scan + ## continues from the character after the matched substring. If the + ## resulting string then contains a possible match starting in a newly + ## placed substring, the additional replacement is not performed. + ## + ## If the resulting string is not longer than the original input string, + ## only a single memory allocation is required. + ## + runnableExamples: + # Swapping occurrences of 'a' and 'b': + doAssert multireplace("abba", [("a", "b"), ("b", "a")]) == "baab" + + # The second replacement ("ab") is matched and performed first, the scan then + # continues from 'c', so the "bc" replacement is never matched and thus skipped. + doAssert multireplace("abc", [("bc", "x"), ("ab", "_b")]) == "_bc" + result = newStringOfCap(s.len) var i = 0 - if s[i] == '0' and (s[i+1] == 'o' or s[i+1] == 'O'): inc(i, 2) - while true: - case s[i] - of '_': inc(i) - of '0'..'7': - result = result shl 3 or (ord(s[i]) - ord('0')) + var fastChk: set[char] = {} + for sub, by in replacements.items: + if sub.len > 0: + # Include first character of all replacements + fastChk.incl sub[0] + while i < s.len: + block sIteration: + # Assume most chars in s are not candidates for any replacement operation + if s[i] in fastChk: + for sub, by in replacements.items: + if sub.len > 0 and s.continuesWith(sub, i): + add result, by + inc(i, sub.len) + break sIteration + # No matching replacement found + # copy current character from s + add result, s[i] inc(i) - of '\0': break - else: raise newException(ValueError, "invalid integer: " & s) -proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToOct".} = - ## Converts `x` into its octal representation. - ## - ## The resulting string is always `len` characters long. No leading ``0o`` - ## prefix is generated. - var - mask: BiggestInt = 7 - shift: BiggestInt = 0 - assert(len > 0) - result = newString(len) - for j in countdown(len-1, 0): - result[j] = chr(int((x and mask) shr shift) + ord('0')) - shift = shift + 3 - mask = mask shl 3 -proc toBin*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToBin".} = - ## Converts `x` into its binary representation. - ## - ## The resulting string is always `len` characters long. No leading ``0b`` - ## prefix is generated. - var - mask: BiggestInt = 1 - shift: BiggestInt = 0 - assert(len > 0) - result = newString(len) - for j in countdown(len-1, 0): - result[j] = chr(int((x and mask) shr shift) + ord('0')) - shift = shift + 1 - mask = mask shl 1 -proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, - rtl, extern: "nsuInsertSep".} = - ## Inserts the separator `sep` after `digits` digits from right to left. +func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, + extern: "nsuInsertSep".} = + ## Inserts the separator `sep` after `digits` characters (default: 3) + ## from right to left. ## ## Even though the algorithm works with any string `s`, it is only useful ## if `s` contains a number. - ## Example: ``insertSep("1000000") == "1_000_000"`` - var L = (s.len-1) div digits + s.len - result = newString(L) + runnableExamples: + doAssert insertSep("1000000") == "1_000_000" + result = newStringOfCap(s.len) + let hasPrefix = isDigit(s[s.low]) == false + var idx: int + if hasPrefix: + result.add s[s.low] + for i in (s.low + 1)..s.high: + idx = i + if not isDigit(s[i]): + result.add s[i] + else: + break + let partsLen = s.len - idx + var L = (partsLen-1) div digits + partsLen + result.setLen(L + idx) var j = 0 dec(L) - for i in countdown(len(s)-1, 0): + for i in countdown(partsLen-1, 0): if j == digits: - result[L] = sep + result[L + idx] = sep dec(L) j = 0 - result[L] = s[i] + result[L + idx] = s[i + idx] inc(j) dec(L) -proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, - rtl, extern: "nsuEscape".} = +func escape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, + extern: "nsuEscape".} = ## Escapes a string `s`. ## - ## This does these operations (at the same time): - ## * replaces any ``\`` by ``\\`` - ## * replaces any ``'`` by ``\'`` - ## * replaces any ``"`` by ``\"`` - ## * replaces any other character in the set ``{'\0'..'\31', '\128'..'\255'}`` - ## by ``\xHH`` where ``HH`` is its hexadecimal value. - ## The procedure has been designed so that its output is usable for many - ## different common syntaxes. The resulting string is prefixed with - ## `prefix` and suffixed with `suffix`. Both may be empty strings. + ## .. note:: The escaping scheme is different from + ## `system.addEscapedChar`. + ## + ## * replaces `'\0'..'\31'` and `'\127'..'\255'` by `\xHH` where `HH` is its hexadecimal value + ## * replaces ``\`` by `\\` + ## * replaces `'` by `\'` + ## * replaces `"` by `\"` + ## + ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. + ## Both may be empty strings. + ## + ## See also: + ## * `addEscapedChar proc<system.html#addEscapedChar,string,char>`_ + ## * `unescape func<#unescape,string,string,string>`_ for the opposite + ## operation result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): case c - of '\0'..'\31', '\128'..'\255': + of '\0'..'\31', '\127'..'\255': add(result, "\\x") add(result, toHex(ord(c), 2)) of '\\': add(result, "\\\\") @@ -1200,252 +2395,394 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, else: add(result, c) add(result, suffix) -proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, - rtl, extern: "nsuUnescape".} = +func unescape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, + extern: "nsuUnescape".} = ## Unescapes a string `s`. ## - ## This complements `escape <#escape>`_ as it performs the opposite - ## operations. + ## This complements `escape func<#escape,string,string,string>`_ + ## as it performs the opposite operations. ## - ## If `s` does not begin with ``prefix`` and end with ``suffix`` a + ## If `s` does not begin with `prefix` and end with `suffix` a ## ValueError exception will be raised. result = newStringOfCap(s.len) - var i = 0 + var i = prefix.len if not s.startsWith(prefix): raise newException(ValueError, - "String does not start with a prefix of: " & prefix) - inc(i) + "String does not start with: " & prefix) while true: - if i == s.len-suffix.len: break - case s[i] - of '\\': + if i >= s.len-suffix.len: break + if s[i] == '\\': + if i+1 >= s.len: + result.add('\\') + break case s[i+1]: of 'x': - inc i - var c: int - i += parseutils.parseHex(s, c, i) + inc i, 2 + var c = 0 + i += parseutils.parseHex(s, c, i, maxLen = 2) result.add(chr(c)) - inc(i, 2) + dec i, 2 of '\\': result.add('\\') of '\'': result.add('\'') of '\"': result.add('\"') - else: result.add("\\" & s[i+1]) - inc(i) - of '\0': break + else: + result.add("\\" & s[i+1]) + inc(i, 2) else: result.add(s[i]) - inc(i) + inc(i) if not s.endsWith(suffix): raise newException(ValueError, - "String does not end with a suffix of: " & suffix) + "String does not end in: " & suffix) -proc validIdentifier*(s: string): bool {.noSideEffect, - rtl, extern: "nsuValidIdentifier".} = +func validIdentifier*(s: string): bool {.rtl, extern: "nsuValidIdentifier".} = ## Returns true if `s` is a valid identifier. ## ## A valid identifier starts with a character of the set `IdentStartChars` ## and is followed by any number of characters of the set `IdentChars`. - if s[0] in IdentStartChars: + runnableExamples: + doAssert "abc_def08".validIdentifier + + if s.len > 0 and s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false return true -proc editDistance*(a, b: string): int {.noSideEffect, - rtl, extern: "nsuEditDistance".} = - ## Returns the edit distance between `a` and `b`. - ## - ## This uses the `Levenshtein`:idx: distance algorithm with only a linear - ## memory overhead. This implementation is highly optimized! - var len1 = a.len - var len2 = b.len - if len1 > len2: - # make `b` the longer string - return editDistance(b, a) - - # strip common prefix: - var s = 0 - while a[s] == b[s] and a[s] != '\0': - inc(s) - dec(len1) - dec(len2) - # strip common suffix: - while len1 > 0 and len2 > 0 and a[s+len1-1] == b[s+len2-1]: - dec(len1) - dec(len2) - # trivial cases: - if len1 == 0: return len2 - if len2 == 0: return len1 - - # another special case: - if len1 == 1: - for j in s..s+len2-1: - if a[s] == b[j]: return len2 - 1 - return len2 - - inc(len1) - inc(len2) - var half = len1 shr 1 - # initalize first row: - #var row = cast[ptr array[0..high(int) div 8, int]](alloc(len2*sizeof(int))) - var row: seq[int] - newSeq(row, len2) - var e = s + len2 - 1 # end marker - for i in 1..len2 - half - 1: row[i] = i - row[0] = len1 - half - 1 - for i in 1 .. len1 - 1: - var char1 = a[i + s - 1] - var char2p: int - var D, x: int - var p: int - if i >= len1 - half: - # skip the upper triangle: - var offset = i - len1 + half - char2p = offset - p = offset - var c3 = row[p] + ord(char1 != b[s + char2p]) - inc(p) - inc(char2p) - x = row[p] + 1 - D = x - if x > c3: x = c3 - row[p] = x - inc(p) - else: - p = 1 - char2p = 0 - D = i - x = i - if i <= half + 1: - # skip the lower triangle: - e = len2 + i - half - 2 - # main: - while p <= e: - dec(D) - var c3 = D + ord(char1 != b[char2p + s]) - inc(char2p) - inc(x) - if x > c3: x = c3 - D = row[p] + 1 - if x > D: x = D - row[p] = x - inc(p) - # lower triangle sentinel: - if i <= half: - dec(D) - var c3 = D + ord(char1 != b[char2p + s]) - inc(x) - if x > c3: x = c3 - row[p] = x - result = row[e] - #dealloc(row) - - -# floating point formating: + +# floating point formatting: when not defined(js): - proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} + func c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs.} type - FloatFormatMode* = enum ## the different modes of floating point formating - ffDefault, ## use the shorter floating point notation - ffDecimal, ## use decimal floating point notation - ffScientific ## use scientific notation (using ``e`` character) - -{.deprecated: [TFloatFormat: FloatFormatMode].} - -proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, - precision: range[0..32] = 16; - decimalSep = '.'): string {. - noSideEffect, rtl, extern: "nsu$1".} = + FloatFormatMode* = enum + ## The different modes of floating point formatting. + ffDefault, ## use the shorter floating point notation + ffDecimal, ## use decimal floating point notation + ffScientific ## use scientific notation (using `e` character) + +func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, + precision: range[-1..32] = 16; + decimalSep = '.'): string {.rtl, extern: "nsu$1".} = ## Converts a floating point value `f` to a string. ## - ## If ``format == ffDecimal`` then precision is the number of digits to + ## If `format == ffDecimal` then precision is the number of digits to ## be printed after the decimal point. - ## If ``format == ffScientific`` then precision is the maximum number + ## If `format == ffScientific` then precision is the maximum number ## of significant digits to be printed. ## `precision`'s default value is the maximum number of meaningful digits - ## after the decimal point for Nim's ``biggestFloat`` type. - ## - ## If ``precision == 0``, it tries to format it nicely. - when defined(js): - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalsep + ## after the decimal point for Nim's `biggestFloat` type. + ## + ## If `precision == -1`, it tries to format it nicely. + runnableExamples: + let x = 123.456 + doAssert x.formatBiggestFloat() == "123.4560000000000" + doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" + doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision > 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(buf, frmtstr, precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_snprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(buf, frmtstr, f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalsep - else: result[i] = buf[i] - -proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, - precision: range[0..32] = 16; decimalSep = '.'): string {. - noSideEffect, rtl, extern: "nsu$1".} = + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) + +func formatFloat*(f: float, format: FloatFormatMode = ffDefault, + precision: range[-1..32] = 16; decimalSep = '.'): string {. + rtl, extern: "nsu$1".} = ## Converts a floating point value `f` to a string. ## - ## If ``format == ffDecimal`` then precision is the number of digits to + ## If `format == ffDecimal` then precision is the number of digits to ## be printed after the decimal point. - ## If ``format == ffScientific`` then precision is the maximum number + ## If `format == ffScientific` then precision is the maximum number ## of significant digits to be printed. ## `precision`'s default value is the maximum number of meaningful digits - ## after the decimal point for Nim's ``float`` type. + ## after the decimal point for Nim's `float` type. + ## + ## If `precision == -1`, it tries to format it nicely. + runnableExamples: + let x = 123.456 + doAssert x.formatFloat() == "123.4560000000000" + doAssert x.formatFloat(ffDecimal, 4) == "123.4560" + doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" + result = formatBiggestFloat(f, format, precision, decimalSep) -proc formatSize*(bytes: BiggestInt, decimalSep = '.'): string = - ## Rounds and formats `bytes`. Examples: - ## - ## .. code-block:: nim - ## - ## formatSize(1'i64 shl 31 + 300'i64) == "2.204GB" - ## formatSize(4096) == "4KB" - ## - template frmt(a, b, c: expr): expr = - let bs = $b - insertSep($a) & decimalSep & bs.substr(0, 2) & c - let gigabytes = bytes shr 30 - let megabytes = bytes shr 20 - let kilobytes = bytes shr 10 - if gigabytes != 0: - result = frmt(gigabytes, megabytes, "GB") - elif megabytes != 0: - result = frmt(megabytes, kilobytes, "MB") - elif kilobytes != 0: - result = frmt(kilobytes, bytes, "KB") - else: - result = insertSep($bytes) & "B" +func trimZeros*(x: var string; decimalSep = '.') = + ## Trim trailing zeros from a formatted floating point + ## value `x` (must be declared as `var`). + ## + ## This modifies `x` itself, it does not return a copy. + runnableExamples: + var x = "123.456000000" + x.trimZeros() + doAssert x == "123.456" + + let sPos = find(x, decimalSep) + if sPos >= 0: + var last = find(x, 'e', start = sPos) + last = if last >= 0: last - 1 else: high(x) + var pos = last + while pos >= 0 and x[pos] == '0': dec(pos) + if pos > sPos: inc(pos) + if last >= pos: + x.delete(pos..last) -proc findNormalized(x: string, inArray: openArray[string]): int = +type + BinaryPrefixMode* = enum ## The different names for binary prefixes. + bpIEC, # use the IEC/ISO standard prefixes such as kibi + bpColloquial # use the colloquial kilo, mega etc + +func formatSize*(bytes: int64, + decimalSep = '.', + prefix = bpIEC, + includeSpace = false): string = + ## Rounds and formats `bytes`. + ## + ## By default, uses the IEC/ISO standard binary prefixes, so 1024 will be + ## formatted as 1KiB. Set prefix to `bpColloquial` to use the colloquial + ## names from the SI standard (e.g. k for 1000 being reused as 1024). + ## + ## `includeSpace` can be set to true to include the (SI preferred) space + ## between the number and the unit (e.g. 1 KiB). + ## + ## See also: + ## * `strformat module<strformat.html>`_ for string interpolation and formatting + runnableExamples: + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096, includeSpace = true) == "4 KiB" + doAssert formatSize(4096, prefix = bpColloquial, includeSpace = true) == "4 kB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(5_378_934, prefix = bpColloquial, decimalSep = ',') == "5,13MB" + + const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] + const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] + var + xb: int64 = bytes + fbytes: float + lastXb: int64 = bytes + matchedIndex = 0 + prefixes: array[9, string] + if prefix == bpColloquial: + prefixes = collPrefixes + else: + prefixes = iecPrefixes + + # Iterate through prefixes seeing if value will be greater than + # 0 in each case + for index in 1..<prefixes.len: + lastXb = xb + xb = bytes div (1'i64 shl (index*10)) + matchedIndex = index + if xb == 0: + xb = lastXb + matchedIndex = index - 1 + break + # xb has the integer number for the latest value; index should be correct + fbytes = bytes.float / (1'i64 shl (matchedIndex*10)).float + result = formatFloat(fbytes, format = ffDecimal, precision = 3, + decimalSep = decimalSep) + result.trimZeros(decimalSep) + if includeSpace: + result &= " " + result &= prefixes[matchedIndex] + result &= "B" + +func formatEng*(f: BiggestFloat, + precision: range[0..32] = 10, + trim: bool = true, + siPrefix: bool = false, + unit: string = "", + decimalSep = '.', + useUnitSpace = false): string = + ## Converts a floating point value `f` to a string using engineering notation. + ## + ## Numbers in of the range -1000.0<f<1000.0 will be formatted without an + ## exponent. Numbers outside of this range will be formatted as a + ## significand in the range -1000.0<f<1000.0 and an exponent that will always + ## be an integer multiple of 3, corresponding with the SI prefix scale k, M, + ## G, T etc for numbers with an absolute value greater than 1 and m, μ, n, p + ## etc for numbers with an absolute value less than 1. + ## + ## The default configuration (`trim=true` and `precision=10`) shows the + ## **shortest** form that precisely (up to a maximum of 10 decimal places) + ## displays the value. For example, 4.100000 will be displayed as 4.1 (which + ## is mathematically identical) whereas 4.1000003 will be displayed as + ## 4.1000003. + ## + ## If `trim` is set to true, trailing zeros will be removed; if false, the + ## number of digits specified by `precision` will always be shown. + ## + ## `precision` can be used to set the number of digits to be shown after the + ## decimal point or (if `trim` is true) the maximum number of digits to be + ## shown. + ## + ## ```nim + ## formatEng(0, 2, trim=false) == "0.00" + ## formatEng(0, 2) == "0" + ## formatEng(0.053, 0) == "53e-3" + ## formatEng(52731234, 2) == "52.73e6" + ## formatEng(-52731234, 2) == "-52.73e6" + ## ``` + ## + ## If `siPrefix` is set to true, the number will be displayed with the SI + ## prefix corresponding to the exponent. For example 4100 will be displayed + ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place + ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute + ## value outside of the range 1e-18<f<1000e18 (1a<f<1000E) will be displayed + ## with an exponent rather than an SI prefix, regardless of whether + ## `siPrefix` is true. + ## + ## If `useUnitSpace` is true, the provided unit will be appended to the string + ## (with a space as required by the SI standard). This behaviour is slightly + ## different to appending the unit to the result as the location of the space + ## is altered depending on whether there is an exponent. + ## + ## ```nim + ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" + ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + ## formatEng(4100, siPrefix=true) == "4.1 k" + ## formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Space with unit="" + ## formatEng(4100, siPrefix=true, unit="") == "4.1 k" + ## formatEng(4100) == "4.1e3" + ## formatEng(4100, unit="V") == "4.1e3 V" + ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true + ## ``` + ## + ## `decimalSep` is used as the decimal separator. + ## + ## See also: + ## * `strformat module<strformat.html>`_ for string interpolation and formatting + var + absolute: BiggestFloat + significand: BiggestFloat + fexponent: BiggestFloat + exponent: int + splitResult: seq[string] + suffix: string = "" + func getPrefix(exp: int): char = + ## Get the SI prefix for a given exponent + ## + ## Assumes exponent is a multiple of 3; returns ' ' if no prefix found + const siPrefixes = ['a', 'f', 'p', 'n', 'u', 'm', ' ', 'k', 'M', 'G', 'T', + 'P', 'E'] + var index: int = (exp div 3) + 6 + result = ' ' + if index in low(siPrefixes)..high(siPrefixes): + result = siPrefixes[index] + + # Most of the work is done with the sign ignored, so get the absolute value + absolute = abs(f) + significand = f + + if absolute == 0.0: + # Simple case: just format it and force the exponent to 0 + exponent = 0 + result = significand.formatBiggestFloat(ffDecimal, precision, + decimalSep = '.') + else: + # Find the best exponent that's a multiple of 3 + fexponent = floor(log10(absolute)) + fexponent = 3.0 * floor(fexponent / 3.0) + # Adjust the significand for the new exponent + significand /= pow(10.0, fexponent) + + # Adjust the significand and check whether it has affected + # the exponent + absolute = abs(significand) + if absolute >= 1000.0: + significand *= 0.001 + fexponent += 3 + # Components of the result: + result = significand.formatBiggestFloat(ffDecimal, precision, + decimalSep = '.') + exponent = fexponent.int() + + splitResult = result.split('.') + result = splitResult[0] + # result should have at most one decimal character + if splitResult.len() > 1: + # If trim is set, we get rid of trailing zeros. Don't use trimZeros here as + # we can be a bit more efficient through knowledge that there will never be + # an exponent in this part. + if trim: + while splitResult[1].endsWith("0"): + # Trim last character + splitResult[1].setLen(splitResult[1].len-1) + if splitResult[1].len() > 0: + result &= decimalSep & splitResult[1] + else: + result &= decimalSep & splitResult[1] + + # Combine the results accordingly + if siPrefix and exponent != 0: + var p = getPrefix(exponent) + if p != ' ': + suffix = " " & p + exponent = 0 # Exponent replaced by SI prefix + if suffix == "" and useUnitSpace: + suffix = " " + suffix &= unit + if exponent != 0: + result &= "e" & $exponent + result &= suffix + +func findNormalized(x: string, inArray: openArray[string]): int = var i = 0 while i < high(inArray): if cmpIgnoreStyle(x, inArray[i]) == 0: return i @@ -1453,21 +2790,20 @@ proc findNormalized(x: string, inArray: openArray[string]): int = # security hole... return -1 -proc invalidFormatString() {.noinline.} = - raise newException(ValueError, "invalid format string") +func invalidFormatString(formatstr: string) {.noinline.} = + raise newException(ValueError, "invalid format string: " & formatstr) -proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. - noSideEffect, rtl, extern: "nsuAddf".} = - ## The same as ``add(s, formatstr % a)``, but more efficient. +func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, + extern: "nsuAddf".} = + ## The same as `add(s, formatstr % a)`, but more efficient. const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '\128'..'\255', '_'} var i = 0 var num = 0 while i < len(formatstr): - if formatstr[i] == '$': - case formatstr[i+1] # again we use the fact that strings - # are zero-terminated here + if formatstr[i] == '$' and i+1 < len(formatstr): + case formatstr[i+1] of '#': - if num >% a.high: invalidFormatString() + if num > a.high: invalidFormatString(formatstr) add s, a[num] inc i, 2 inc num @@ -1479,34 +2815,49 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. inc(i) # skip $ var negative = formatstr[i] == '-' if negative: inc i - while formatstr[i] in Digits: + while i < formatstr.len and formatstr[i] in Digits: j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) let idx = if not negative: j-1 else: a.len-j - if idx >% a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] of '{': - var j = i+1 - while formatstr[j] notin {'\0', '}'}: inc(j) - var x = findNormalized(substr(formatstr, i+2, j-1), a) - if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + var j = i+2 + var k = 0 + var negative = formatstr[j] == '-' + if negative: inc j + var isNumber = 0 + while j < formatstr.len and formatstr[j] notin {'\0', '}'}: + if formatstr[j] in Digits: + k = k * 10 + ord(formatstr[j]) - ord('0') + if isNumber == 0: isNumber = 1 + else: + isNumber = -1 + inc(j) + if isNumber == 1: + let idx = if not negative: k-1 else: a.len-k + if idx < 0 or idx > a.high: invalidFormatString(formatstr) + add s, a[idx] + else: + var x = findNormalized(substr(formatstr, i+2, j-1), a) + if x >= 0 and x < high(a): add s, a[x+1] + else: invalidFormatString(formatstr) i = j+1 of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 - while formatstr[j] in PatternChars: inc(j) + while j < formatstr.len and formatstr[j] in PatternChars: inc(j) var x = findNormalized(substr(formatstr, i+1, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j else: - invalidFormatString() + invalidFormatString(formatstr) else: add s, formatstr[i] inc(i) -proc `%` *(formatstr: string, a: openArray[string]): string {.noSideEffect, - rtl, extern: "nsuFormatOpenArray".} = +func `%`*(formatstr: string, a: openArray[string]): string {.rtl, + extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. ## ## The `substitution`:idx: operator performs string substitutions in @@ -1515,209 +2866,167 @@ proc `%` *(formatstr: string, a: openArray[string]): string {.noSideEffect, ## ## This is best explained by an example: ## - ## .. code-block:: nim + ## ```nim ## "$1 eats $2." % ["The cat", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## - ## The substitution variables (the thing after the ``$``) are enumerated - ## from 1 to ``a.len``. - ## To produce a verbatim ``$``, use ``$$``. - ## The notation ``$#`` can be used to refer to the next substitution + ## The substitution variables (the thing after the `$`) are enumerated + ## from 1 to `a.len`. + ## To produce a verbatim `$`, use `$$`. + ## The notation `$#` can be used to refer to the next substitution ## variable: ## - ## .. code-block:: nim + ## ```nim ## "$# eats $#." % ["The cat", "fish"] + ## ``` ## ## Substitution variables can also be words (that is - ## ``[A-Za-z_]+[A-Za-z0-9_]*``) in which case the arguments in `a` with even + ## `[A-Za-z_]+[A-Za-z0-9_]*`) in which case the arguments in `a` with even ## indices are keys and with odd indices are the corresponding values. ## An example: ## - ## .. code-block:: nim + ## ```nim ## "$animal eats $food." % ["animal", "The cat", "food", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The variables are compared with `cmpIgnoreStyle`. `ValueError` is ## raised if an ill-formed format string has been passed to the `%` operator. + ## + ## See also: + ## * `strformat module<strformat.html>`_ for string interpolation and formatting result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) -proc `%` *(formatstr, a: string): string {.noSideEffect, - rtl, extern: "nsuFormatSingleElem".} = - ## This is the same as ``formatstr % [a]``. +func `%`*(formatstr, a: string): string {.rtl, + extern: "nsuFormatSingleElem".} = + ## This is the same as `formatstr % [a]` (see + ## `% func<#%25,string,openArray[string]>`_). result = newStringOfCap(formatstr.len + a.len) addf(result, formatstr, [a]) -proc format*(formatstr: string, a: varargs[string, `$`]): string {.noSideEffect, - rtl, extern: "nsuFormatVarargs".} = - ## This is the same as ``formatstr % a`` except that it supports +func format*(formatstr: string, a: varargs[string, `$`]): string {.rtl, + extern: "nsuFormatVarargs".} = + ## This is the same as `formatstr % a` (see + ## `% func<#%25,string,openArray[string]>`_) except that it supports ## auto stringification. + ## + ## See also: + ## * `strformat module<strformat.html>`_ for string interpolation and formatting result = newStringOfCap(formatstr.len + a.len) addf(result, formatstr, a) -{.pop.} - -proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. - rtl, extern: "nsuRemoveSuffixCharSet".} = - ## Removes the first matching character from the string (in-place) given a - ## set of characters. If the set of characters is only equal to `Newlines` - ## then it will remove both the newline and return feed. - ## .. code-block:: nim - ## var - ## userInput = "Hello World!\r\n" - ## otherInput = "Hello!?!" - ## userInput.removeSuffix - ## userInput == "Hello World!" - ## userInput.removeSuffix({'!', '?'}) - ## userInput == "Hello World" - ## otherInput.removeSuffix({'!', '?'}) - ## otherInput == "Hello!?" - - var last = len(s) - 1 - - if chars == Newlines: - if s[last] == '\10': - last -= 1 - - if s[last] == '\13': - last -= 1 - else: - if s[last] in chars: - last -= 1 +func strip*(s: string, leading = true, trailing = true, + chars: set[char] = Whitespace): string {.rtl, extern: "nsuStrip".} = + ## Strips leading or trailing `chars` (default: whitespace characters) + ## from `s` and returns the resulting string. + ## + ## If `leading` is true (default), leading `chars` are stripped. + ## If `trailing` is true (default), trailing `chars` are stripped. + ## If both are false, the string is returned unchanged. + ## + ## See also: + ## * `strip proc<strbasics.html#strip,string,set[char]>`_ Inplace version. + ## * `stripLineEnd func<#stripLineEnd,string>`_ + runnableExamples: + let a = " vhellov " + let b = strip(a) + doAssert b == "vhellov" - s.setLen(last + 1) + doAssert a.strip(leading = false) == " vhellov" + doAssert a.strip(trailing = false) == "vhellov " -proc removeSuffix*(s: var string, c: char) {. - rtl, extern: "nsuRemoveSuffixChar".} = - ## Removes a single character (in-place) from a string. - ## .. code-block:: nim - ## var - ## table = "users" - ## table.removeSuffix('s') - ## table == "user" - removeSuffix(s, chars = {c}) + doAssert b.strip(chars = {'v'}) == "hello" + doAssert b.strip(leading = false, chars = {'v'}) == "vhello" -proc removeSuffix*(s: var string, suffix: string) {. - rtl, extern: "nsuRemoveSuffixString".} = - ## Remove the first matching suffix (in-place) from a string. - ## .. code-block:: nim - ## var - ## answers = "yeses" - ## answers.removeSuffix("es") - ## answers == "yes" + let c = "blaXbla" + doAssert c.strip(chars = {'b', 'a'}) == "laXbl" + doAssert c.strip(chars = {'b', 'a', 'l'}) == "X" - var newLen = s.len + var + first = 0 + last = len(s)-1 + if leading: + while first <= last and s[first] in chars: inc(first) + if trailing: + while last >= first and s[last] in chars: dec(last) + result = substr(s, first, last) - if s.endsWith(suffix): - newLen -= len(suffix) +func stripLineEnd*(s: var string) = + ## Strips one of these suffixes from `s` in-place: + ## `\r, \n, \r\n, \f, \v` (at most once instance). + ## For example, can be useful in conjunction with `osproc.execCmdEx`. + ## aka: `chomp`:idx: + runnableExamples: + var s = "foo\n\n" + s.stripLineEnd + doAssert s == "foo\n" + s = "foo\r\n" + s.stripLineEnd + doAssert s == "foo" + + if s.len > 0: + case s[^1] + of '\n': + if s.len > 1 and s[^2] == '\r': + s.setLen s.len-2 + else: + s.setLen s.len-1 + of '\r', '\v', '\f': + s.setLen s.len-1 + else: + discard - s.setLen(newLen) -when isMainModule: - doAssert align("abc", 4) == " abc" - doAssert align("a", 0) == "a" - doAssert align("1232", 6) == " 1232" - doAssert align("1232", 6, '#') == "##1232" +iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ + token: string, isSep: bool] = + ## Tokenizes the string `s` into substrings. + ## + ## Substrings are separated by a substring containing only `seps`. + ## Example: + ## + ## ```nim + ## for word in tokenize(" this is an example "): + ## writeLine(stdout, word) + ## ``` + ## + ## Results in: + ## + ## ```nim + ## (" ", true) + ## ("this", false) + ## (" ", true) + ## ("is", false) + ## (" ", true) + ## ("an", false) + ## (" ", true) + ## ("example", false) + ## (" ", true) + ## ``` + var i = 0 + while true: + var j = i + var isSep = j < s.len and s[j] in seps + while j < s.len and (s[j] in seps) == isSep: inc(j) + if j > i: + yield (substr(s, i, j-1), isSep) + else: + break + i = j - let - inp = """ this is a long text -- muchlongerthan10chars and here - it goes""" - outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" - doAssert wordWrap(inp, 10, false) == outp - - doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" - doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in - ["1,0e-11", "1,0e-011"] - - doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - when not defined(testing): - echo formatSize(1'i64 shl 31 + 300'i64) # == "4,GB" - echo formatSize(1'i64 shl 31) - - doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == - "The cat eats fish." - - doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " - doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" - - type MyEnum = enum enA, enB, enC, enuD, enE - doAssert parseEnum[MyEnum]("enu_D") == enuD - - doAssert parseEnum("invalid enum value", enC) == enC - - doAssert count("foofoofoo", "foofoo") == 1 - doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 - doAssert count("foofoofoo", 'f') == 3 - doAssert count("foofoofoobar", {'f','b'}) == 4 - - doAssert strip(" foofoofoo ") == "foofoofoo" - doAssert strip("sfoofoofoos", chars = {'s'}) == "foofoofoo" - doAssert strip("barfoofoofoobar", chars = {'b', 'a', 'r'}) == "foofoofoo" - doAssert strip("stripme but don't strip this stripme", - chars = {'s', 't', 'r', 'i', 'p', 'm', 'e'}) == - " but don't strip this " - doAssert strip("sfoofoofoos", leading = false, chars = {'s'}) == "sfoofoofoo" - doAssert strip("sfoofoofoos", trailing = false, chars = {'s'}) == "foofoofoos" - - doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - - doAssert isAlpha('r') - doAssert isAlpha('A') - doAssert(not isAlpha('$')) - - doAssert isAlpha("Rasp") - doAssert isAlpha("Args") - doAssert(not isAlpha("$Tomato")) - - doAssert isAlphaNumeric('3') - doAssert isAlphaNumeric('R') - doAssert(not isAlphaNumeric('!')) - - doAssert isAlphaNumeric("34ABc") - doAssert isAlphaNumeric("Rad") - doAssert isAlphaNumeric("1234") - doAssert(not isAlphaNumeric("@nose")) - - doAssert isDigit('3') - doAssert(not isDigit('a')) - doAssert(not isDigit('%')) - - doAssert isDigit("12533") - doAssert(not isDigit("12.33")) - doAssert(not isDigit("A45b")) - - doAssert isSpace('\t') - doAssert isSpace('\l') - doAssert(not isSpace('A')) - - doAssert isSpace("\t\l \v\r\f") - doAssert isSpace(" ") - doAssert(not isSpace("ABc \td")) - - doAssert isLower('a') - doAssert isLower('z') - doAssert(not isLower('A')) - doAssert(not isLower('5')) - doAssert(not isLower('&')) - - doAssert isLower("abcd") - doAssert(not isLower("abCD")) - doAssert(not isLower("33aa")) - - doAssert isUpper('A') - doAssert(not isUpper('b')) - doAssert(not isUpper('5')) - doAssert(not isUpper('%')) - - doAssert isUpper("ABC") - doAssert(not isUpper("AAcc")) - doAssert(not isUpper("A#$")) +func isEmptyOrWhitespace*(s: string): bool {.rtl, + extern: "nsuIsEmptyOrWhitespace".} = + ## Checks if `s` is empty or consists entirely of whitespace characters. + result = s.allCharsInSet(Whitespace) diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim deleted file mode 100644 index 5824ace81..000000000 --- a/lib/pure/subexes.nim +++ /dev/null @@ -1,411 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nim support for `substitution expressions`:idx: (`subex`:idx:). -## -## .. include:: ../doc/subexes.txt -## - -{.push debugger:off .} # the user does not want to trace a part - # of the standard library! - -from strutils import parseInt, cmpIgnoreStyle, Digits -include "system/inclrtl" - - -proc findNormalized(x: string, inArray: openarray[string]): int = - var i = 0 - while i < high(inArray): - if cmpIgnoreStyle(x, inArray[i]) == 0: return i - inc(i, 2) # incrementing by 1 would probably lead to a - # security hole... - return -1 - -type - SubexError* = object of ValueError ## exception that is raised for - ## an invalid subex - -{.deprecated: [EInvalidSubex: SubexError].} - -proc raiseInvalidFormat(msg: string) {.noinline.} = - raise newException(SubexError, "invalid format string: " & msg) - -type - FormatParser = object {.pure, final.} - when defined(js): - f: string # we rely on the '\0' terminator - # which JS's native string doesn't have - else: - f: cstring - num, i, lineLen: int -{.deprecated: [TFormatParser: FormatParser].} - -template call(x: stmt) {.immediate.} = - p.i = i - x - i = p.i - -template callNoLineLenTracking(x: stmt) {.immediate.} = - let oldLineLen = p.lineLen - p.i = i - x - i = p.i - p.lineLen = oldLineLen - -proc getFormatArg(p: var FormatParser, a: openArray[string]): int = - const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '\128'..'\255', '_'} - var i = p.i - var f = p.f - case f[i] - of '#': - result = p.num - inc i - inc p.num - of '1'..'9', '-': - var j = 0 - var negative = f[i] == '-' - if negative: inc i - while f[i] in Digits: - j = j * 10 + ord(f[i]) - ord('0') - inc i - result = if not negative: j-1 else: a.len-j - of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': - var name = "" - while f[i] in PatternChars: - name.add(f[i]) - inc(i) - result = findNormalized(name, a)+1 - of '$': - inc(i) - call: - result = getFormatArg(p, a) - result = parseInt(a[result])-1 - else: - raiseInvalidFormat("'#', '$', number or identifier expected") - if result >=% a.len: raiseInvalidFormat("index out of bounds: " & $result) - p.i = i - -proc scanDollar(p: var FormatParser, a: openarray[string], s: var string) {. - noSideEffect.} - -proc emitChar(p: var FormatParser, x: var string, ch: char) {.inline.} = - x.add(ch) - if ch == '\L': p.lineLen = 0 - else: inc p.lineLen - -proc emitStrLinear(p: var FormatParser, x: var string, y: string) {.inline.} = - for ch in items(y): emitChar(p, x, ch) - -proc emitStr(p: var FormatParser, x: var string, y: string) {.inline.} = - x.add(y) - inc p.lineLen, y.len - -proc scanQuote(p: var FormatParser, x: var string, toAdd: bool) = - var i = p.i+1 - var f = p.f - while true: - if f[i] == '\'': - inc i - if f[i] != '\'': break - inc i - if toAdd: emitChar(p, x, '\'') - elif f[i] == '\0': raiseInvalidFormat("closing \"'\" expected") - else: - if toAdd: emitChar(p, x, f[i]) - inc i - p.i = i - -proc scanBranch(p: var FormatParser, a: openArray[string], - x: var string, choice: int) = - var i = p.i - var f = p.f - var c = 0 - var elsePart = i - var toAdd = choice == 0 - while true: - case f[i] - of ']': break - of '|': - inc i - elsePart = i - inc c - if toAdd: break - toAdd = choice == c - of '\'': - call: scanQuote(p, x, toAdd) - of '\0': raiseInvalidFormat("closing ']' expected") - else: - if toAdd: - if f[i] == '$': - inc i - call: scanDollar(p, a, x) - else: - emitChar(p, x, f[i]) - inc i - else: - inc i - if not toAdd and choice >= 0: - # evaluate 'else' part: - var last = i - i = elsePart - while true: - case f[i] - of '|', ']': break - of '\'': - call: scanQuote(p, x, true) - of '$': - inc i - call: scanDollar(p, a, x) - else: - emitChar(p, x, f[i]) - inc i - i = last - p.i = i+1 - -proc scanSlice(p: var FormatParser, a: openarray[string]): tuple[x, y: int] = - var slice = false - var i = p.i - var f = p.f - - if f[i] == '{': inc i - else: raiseInvalidFormat("'{' expected") - if f[i] == '.' and f[i+1] == '.': - inc i, 2 - slice = true - else: - call: result.x = getFormatArg(p, a) - if f[i] == '.' and f[i+1] == '.': - inc i, 2 - slice = true - if slice: - if f[i] != '}': - call: result.y = getFormatArg(p, a) - else: - result.y = high(a) - else: - result.y = result.x - if f[i] != '}': raiseInvalidFormat("'}' expected") - inc i - p.i = i - -proc scanDollar(p: var FormatParser, a: openarray[string], s: var string) = - var i = p.i - var f = p.f - case f[i] - of '$': - emitChar p, s, '$' - inc i - of '*': - for j in 0..a.high: emitStr p, s, a[j] - inc i - of '{': - call: - let (x, y) = scanSlice(p, a) - for j in x..y: emitStr p, s, a[j] - of '[': - inc i - var start = i - call: scanBranch(p, a, s, -1) - var x: int - if f[i] == '{': - inc i - call: x = getFormatArg(p, a) - if f[i] != '}': raiseInvalidFormat("'}' expected") - inc i - else: - call: x = getFormatArg(p, a) - var last = i - let choice = parseInt(a[x]) - i = start - call: scanBranch(p, a, s, choice) - i = last - of '\'': - var sep = "" - callNoLineLenTracking: scanQuote(p, sep, true) - if f[i] == '~': - # $' '~{1..3} - # insert space followed by 1..3 if not empty - inc i - call: - let (x, y) = scanSlice(p, a) - var L = 0 - for j in x..y: inc L, a[j].len - if L > 0: - emitStrLinear p, s, sep - for j in x..y: emitStr p, s, a[j] - else: - block StringJoin: - block OptionalLineLengthSpecifier: - var maxLen = 0 - case f[i] - of '0'..'9': - while f[i] in Digits: - maxLen = maxLen * 10 + ord(f[i]) - ord('0') - inc i - of '$': - # do not skip the '$' here for `getFormatArg`! - call: - maxLen = getFormatArg(p, a) - else: break OptionalLineLengthSpecifier - var indent = "" - case f[i] - of 'i': - inc i - callNoLineLenTracking: scanQuote(p, indent, true) - - call: - let (x, y) = scanSlice(p, a) - if maxLen < 1: emitStrLinear(p, s, indent) - var items = 1 - emitStr p, s, a[x] - for j in x+1..y: - emitStr p, s, sep - if items >= maxLen: - emitStrLinear p, s, indent - items = 0 - emitStr p, s, a[j] - inc items - of 'c': - inc i - callNoLineLenTracking: scanQuote(p, indent, true) - - call: - let (x, y) = scanSlice(p, a) - if p.lineLen + a[x].len > maxLen: emitStrLinear(p, s, indent) - emitStr p, s, a[x] - for j in x+1..y: - emitStr p, s, sep - if p.lineLen + a[j].len > maxLen: emitStrLinear(p, s, indent) - emitStr p, s, a[j] - - else: raiseInvalidFormat("unit 'c' (chars) or 'i' (items) expected") - break StringJoin - - call: - let (x, y) = scanSlice(p, a) - emitStr p, s, a[x] - for j in x+1..y: - emitStr p, s, sep - emitStr p, s, a[j] - else: - call: - var x = getFormatArg(p, a) - emitStr p, s, a[x] - p.i = i - - -type - Subex* = distinct string ## string that contains a substitution expression - -{.deprecated: [TSubex: Subex].} - -proc subex*(s: string): Subex = - ## constructs a *substitution expression* from `s`. Currently this performs - ## no syntax checking but this may change in later versions. - result = Subex(s) - -proc addf*(s: var string, formatstr: Subex, a: varargs[string, `$`]) {. - noSideEffect, rtl, extern: "nfrmtAddf".} = - ## The same as ``add(s, formatstr % a)``, but more efficient. - var p: FormatParser - p.f = formatstr.string - var i = 0 - while i < len(formatstr.string): - if p.f[i] == '$': - inc i - call: scanDollar(p, a, s) - else: - emitChar(p, s, p.f[i]) - inc(i) - -proc `%` *(formatstr: Subex, a: openarray[string]): string {.noSideEffect, - rtl, extern: "nfrmtFormatOpenArray".} = - ## The `substitution`:idx: operator performs string substitutions in - ## `formatstr` and returns a modified `formatstr`. This is often called - ## `string interpolation`:idx:. - ## - result = newStringOfCap(formatstr.string.len + a.len shl 4) - addf(result, formatstr, a) - -proc `%` *(formatstr: Subex, a: string): string {.noSideEffect, - rtl, extern: "nfrmtFormatSingleElem".} = - ## This is the same as ``formatstr % [a]``. - result = newStringOfCap(formatstr.string.len + a.len) - addf(result, formatstr, [a]) - -proc format*(formatstr: Subex, a: varargs[string, `$`]): string {.noSideEffect, - rtl, extern: "nfrmtFormatVarargs".} = - ## The `substitution`:idx: operator performs string substitutions in - ## `formatstr` and returns a modified `formatstr`. This is often called - ## `string interpolation`:idx:. - ## - result = newStringOfCap(formatstr.string.len + a.len shl 4) - addf(result, formatstr, a) - -{.pop.} - -when isMainModule: - from strutils import replace - - proc `%`(formatstr: string, a: openarray[string]): string = - result = newStringOfCap(formatstr.len + a.len shl 4) - addf(result, formatstr.Subex, a) - - proc `%`(formatstr: string, a: string): string = - result = newStringOfCap(formatstr.len + a.len) - addf(result, formatstr.Subex, [a]) - - - doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == - "The cat eats fish." - - - doAssert "$[abc|def]# $3 $# $#" % ["17", "b", "c"] == "def c b c" - doAssert "$[abc|def]# $3 $# $#" % ["1", "b", "c"] == "def c b c" - doAssert "$[abc|def]# $3 $# $#" % ["0", "b", "c"] == "abc c b c" - doAssert "$[abc|def|]# $3 $# $#" % ["17", "b", "c"] == " c b c" - - doAssert "$[abc|def|]# $3 $# $#" % ["-9", "b", "c"] == " c b c" - doAssert "$1($', '{2..})" % ["f", "a", "b"] == "f(a, b)" - - doAssert "$[$1($', '{2..})|''''|fg'$3']1" % ["7", "a", "b"] == "fg$3" - - doAssert "$[$#($', '{#..})|''''|$3]1" % ["0", "a", "b"] == "0(a, b)" - doAssert "$' '~{..}" % "" == "" - doAssert "$' '~{..}" % "P0" == " P0" - doAssert "${$1}" % "1" == "1" - doAssert "${$$-1} $$1" % "1" == "1 $1" - - doAssert(("$#($', '10c'\n '{#..})" % ["doAssert", "longishA", "longish"]).replace(" \n", "\n") == - """doAssert( - longishA, - longish)""") - - doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", - "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent """ - type MyEnum* = enum - fieldA, fieldB, - FiledClkad, fieldD, - fieldE, longishFieldName""") - - doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" - - doAssert subex"$1 $[files|file|files]{1} copied" % ["1"] == "1 file copied" - - doAssert subex"$['''|'|''''|']']#" % "0" == "'|" - - doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ - "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent """ - type - Enum = enum - fieldNameA, fieldNameB, fieldNameC, - fieldNameD""") diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim new file mode 100644 index 000000000..90ba20c13 --- /dev/null +++ b/lib/pure/sugar.nim @@ -0,0 +1,429 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements nice syntactic sugar based on Nim's +## macro system. + +import std/private/since +import std/macros + +proc checkPragma(ex, prag: var NimNode) = + since (1, 3): + if ex.kind == nnkPragmaExpr: + prag = ex[1] + ex = ex[0] + +proc createProcType(p, b: NimNode): NimNode = + result = newNimNode(nnkProcTy) + var + formalParams = newNimNode(nnkFormalParams).add(b) + p = p + prag = newEmptyNode() + + checkPragma(p, prag) + + case p.kind + of nnkPar, nnkTupleConstr: + for i in 0 ..< p.len: + let ident = p[i] + var identDefs = newNimNode(nnkIdentDefs) + case ident.kind + of nnkExprColonExpr: + identDefs.add ident[0] + identDefs.add ident[1] + else: + identDefs.add newIdentNode("i" & $i) + identDefs.add(ident) + identDefs.add newEmptyNode() + formalParams.add identDefs + else: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add newIdentNode("i0") + identDefs.add(p) + identDefs.add newEmptyNode() + formalParams.add identDefs + + result.add formalParams + result.add prag + +macro `=>`*(p, b: untyped): untyped = + ## Syntax sugar for anonymous procedures. It also supports pragmas. + ## + ## .. warning:: Semicolons can not be used to separate procedure arguments. + runnableExamples: + proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) + + assert passTwoAndTwo((x, y) => x + y) == 4 + + type + Bot = object + call: (string {.noSideEffect.} -> string) + + var myBot = Bot() + + myBot.call = (name: string) {.noSideEffect.} => "Hello " & name & ", I'm a bot." + assert myBot.call("John") == "Hello John, I'm a bot." + + let f = () => (discard) # simplest proc that returns void + f() + + var + params = @[ident"auto"] + name = newEmptyNode() + kind = nnkLambda + pragma = newEmptyNode() + p = p + + checkPragma(p, pragma) + + if p.kind == nnkInfix and p[0].kind == nnkIdent and p[0].eqIdent"->": + params[0] = p[2] + p = p[1] + + checkPragma(p, pragma) # check again after -> transform + + case p.kind + of nnkPar, nnkTupleConstr: + var untypedBeforeColon = 0 + for i, c in p: + var identDefs = newNimNode(nnkIdentDefs) + case c.kind + of nnkExprColonExpr: + let t = c[1] + since (1, 3): + # + 1 here because of return type in params + for j in (i - untypedBeforeColon + 1) .. i: + params[j][1] = t + untypedBeforeColon = 0 + identDefs.add(c[0]) + identDefs.add(t) + identDefs.add(newEmptyNode()) + of nnkIdent: + identDefs.add(c) + identDefs.add(newIdentNode("auto")) + identDefs.add(newEmptyNode()) + inc untypedBeforeColon + of nnkInfix: + if c[0].kind == nnkIdent and c[0].eqIdent"->": + var procTy = createProcType(c[1], c[2]) + params[0] = procTy[0][0] + for i in 1 ..< procTy[0].len: + params.add(procTy[0][i]) + else: + error("Expected proc type (->) got (" & c[0].strVal & ").", c) + break + else: + error("Incorrect procedure parameter.", c) + params.add(identDefs) + of nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add(ident $p) + identDefs.add(ident"auto") + identDefs.add(newEmptyNode()) + params.add(identDefs) + else: + error("Incorrect procedure parameter list.", p) + result = newProc(body = b, params = params, + pragmas = pragma, name = name, + procType = kind) + +macro `->`*(p, b: untyped): untyped = + ## Syntax sugar for procedure types. It also supports pragmas. + ## + ## .. warning:: Semicolons can not be used to separate procedure arguments. + runnableExamples: + proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) + # is the same as: + # proc passTwoAndTwo(f: proc (x, y: int): int): int = f(2, 2) + + assert passTwoAndTwo((x, y) => x + y) == 4 + + proc passOne(f: (int {.noSideEffect.} -> int)): int = f(1) + # is the same as: + # proc passOne(f: proc (x: int): int {.noSideEffect.}): int = f(1) + + assert passOne(x {.noSideEffect.} => x + 1) == 2 + + result = createProcType(p, b) + +macro dump*(x: untyped): untyped = + ## Dumps the content of an expression, useful for debugging. + ## It accepts any expression and prints a textual representation + ## of the tree representing the expression - as it would appear in + ## source code - together with the value of the expression. + ## + ## See also: `dumpToString` which is more convenient and useful since + ## it expands intermediate templates/macros, returns a string instead of + ## calling `echo`, and works with statements and expressions. + runnableExamples("-r:off"): + let + x = 10 + y = 20 + dump(x + y) # prints: `x + y = 30` + + let s = x.toStrLit + result = quote do: + debugEcho `s`, " = ", `x` + +macro dumpToStringImpl(s: static string, x: typed): string = + let s2 = x.toStrLit + if x.typeKind == ntyVoid: + result = quote do: + `s` & ": " & `s2` + else: + result = quote do: + `s` & ": " & `s2` & " = " & $`x` + +macro dumpToString*(x: untyped): string = + ## Returns the content of a statement or expression `x` after semantic analysis, + ## useful for debugging. + runnableExamples: + const a = 1 + let x = 10 + assert dumpToString(a + 2) == "a + 2: 3 = 3" + assert dumpToString(a + x) == "a + x: 1 + x = 11" + template square(x): untyped = x * x + assert dumpToString(square(x)) == "square(x): x * x = 100" + assert not compiles dumpToString(1 + nonexistent) + import std/strutils + assert "failedAssertImpl" in dumpToString(assert true) # example with a statement + result = newCall(bindSym"dumpToStringImpl") + result.add newLit repr(x) + result.add x + +# TODO: consider exporting this in macros.nim +proc freshIdentNodes(ast: NimNode): NimNode = + # Replace NimIdent and NimSym by a fresh ident node + # see also https://github.com/nim-lang/Nim/pull/8531#issuecomment-410436458 + proc inspect(node: NimNode): NimNode = + case node.kind: + of nnkIdent, nnkSym, nnkOpenSymChoice, nnkClosedSymChoice, nnkOpenSym: + result = ident($node) + of nnkEmpty, nnkLiterals: + result = node + else: + result = node.kind.newTree() + for child in node: + result.add inspect(child) + result = inspect(ast) + +macro capture*(locals: varargs[typed], body: untyped): untyped {.since: (1, 1).} = + ## Useful when creating a closure in a loop to capture some local loop variables + ## by their current iteration values. + runnableExamples: + import std/strformat + + var myClosure: () -> string + for i in 5..7: + for j in 7..9: + if i * j == 42: + capture i, j: + myClosure = () => fmt"{i} * {j} = 42" + assert myClosure() == "6 * 7 = 42" + + var params = @[newIdentNode("auto")] + let locals = if locals.len == 1 and locals[0].kind == nnkBracket: locals[0] + else: locals + for arg in locals: + proc getIdent(n: NimNode): NimNode = + case n.kind + of nnkIdent, nnkSym: + let nStr = n.strVal + if nStr == "result": + error("The variable name cannot be `result`!", n) + result = ident(nStr) + of nnkHiddenDeref: result = n[0].getIdent() + else: + error("The argument to be captured `" & n.repr & "` is not a pure identifier. " & + "It is an unsupported `" & $n.kind & "` node.", n) + let argName = getIdent(arg) + params.add(newIdentDefs(argName, freshIdentNodes getTypeInst arg)) + result = newNimNode(nnkCall) + result.add(newProc(newEmptyNode(), params, body, nnkLambda)) + for arg in locals: result.add(arg) + +since (1, 1): + import std/private/underscored_calls + + macro dup*[T](arg: T, calls: varargs[untyped]): T = + ## Turns an `in-place`:idx: algorithm into one that works on + ## a copy and returns this copy, without modifying its input. + ## + ## This macro also allows for (otherwise in-place) function chaining. + ## + ## **Since:** Version 1.2. + runnableExamples: + import std/algorithm + + let a = @[1, 2, 3, 4, 5, 6, 7, 8, 9] + assert a.dup(sort) == sorted(a) + + # Chaining: + var aCopy = a + aCopy.insert(10) + assert a.dup(insert(10), sort) == sorted(aCopy) + + let s1 = "abc" + let s2 = "xyz" + assert s1 & s2 == s1.dup(&= s2) + + # An underscore (_) can be used to denote the place of the argument you're passing: + assert "".dup(addQuoted(_, "foo")) == "\"foo\"" + # but `_` is optional here since the substitution is in 1st position: + assert "".dup(addQuoted("foo")) == "\"foo\"" + + proc makePalindrome(s: var string) = + for i in countdown(s.len-2, 0): + s.add(s[i]) + + let c = "xyz" + + # chaining: + let d = dup c: + makePalindrome # xyzyx + sort(_, SortOrder.Descending) # zyyxx + makePalindrome # zyyxxxyyz + assert d == "zyyxxxyyz" + + result = newNimNode(nnkStmtListExpr, arg) + let tmp = genSym(nskVar, "dupResult") + result.add newVarStmt(tmp, arg) + underscoredCalls(result, calls, tmp) + result.add tmp + +proc trans(n, res, bracketExpr: NimNode): (NimNode, NimNode, NimNode) {.since: (1, 1).} = + # Looks for the last statement of the last statement, etc... + case n.kind + of nnkIfExpr, nnkIfStmt, nnkTryStmt, nnkCaseStmt, nnkWhenStmt: + result[0] = copyNimTree(n) + result[1] = copyNimTree(n) + result[2] = copyNimTree(n) + for i in ord(n.kind == nnkCaseStmt) ..< n.len: + (result[0][i], result[1][^1], result[2][^1]) = trans(n[i], res, bracketExpr) + of nnkStmtList, nnkStmtListExpr, nnkBlockStmt, nnkBlockExpr, nnkWhileStmt, + nnkForStmt, nnkElifBranch, nnkElse, nnkElifExpr, nnkOfBranch, nnkExceptBranch: + result[0] = copyNimTree(n) + result[1] = copyNimTree(n) + result[2] = copyNimTree(n) + if n.len >= 1: + (result[0][^1], result[1][^1], result[2][^1]) = trans(n[^1], + res, bracketExpr) + of nnkTableConstr: + result[1] = n[0][0] + result[2] = n[0][1] + if bracketExpr.len == 0: + bracketExpr.add(ident"initTable") # don't import tables + if bracketExpr.len == 1: + bracketExpr.add([newCall(bindSym"typeof", + newEmptyNode()), newCall(bindSym"typeof", newEmptyNode())]) + template adder(res, k, v) = res[k] = v + result[0] = getAst(adder(res, n[0][0], n[0][1])) + of nnkCurly: + result[2] = n[0] + if bracketExpr.len == 0: + bracketExpr.add(ident"initHashSet") + if bracketExpr.len == 1: + bracketExpr.add(newCall(bindSym"typeof", newEmptyNode())) + template adder(res, v) = res.incl(v) + result[0] = getAst(adder(res, n[0])) + else: + result[2] = n + if bracketExpr.len == 0: + bracketExpr.add(bindSym"newSeq") + if bracketExpr.len == 1: + bracketExpr.add(newCall(bindSym"typeof", newEmptyNode())) + template adder(res, v) = res.add(v) + result[0] = getAst(adder(res, n)) + +proc collectImpl(init, body: NimNode): NimNode {.since: (1, 1).} = + let res = genSym(nskVar, "collectResult") + var bracketExpr: NimNode + if init != nil: + expectKind init, {nnkCall, nnkIdent, nnkSym, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym} + bracketExpr = newTree(nnkBracketExpr, + if init.kind in {nnkCall, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym}: + freshIdentNodes(init[0]) else: freshIdentNodes(init)) + else: + bracketExpr = newTree(nnkBracketExpr) + let (resBody, keyType, valueType) = trans(body, res, bracketExpr) + if bracketExpr.len == 3: + bracketExpr[1][1] = keyType + bracketExpr[2][1] = valueType + else: + bracketExpr[1][1] = valueType + let call = newTree(nnkCall, bracketExpr) + if init != nil and init.kind == nnkCall: + for i in 1 ..< init.len: + call.add init[i] + result = newTree(nnkStmtListExpr, newVarStmt(res, call), resBody, res) + +macro collect*(init, body: untyped): untyped {.since: (1, 1).} = + ## Comprehension for seqs/sets/tables. + ## + ## The last expression of `body` has special syntax that specifies + ## the collection's add operation. Use `{e}` for set's `incl`, + ## `{k: v}` for table's `[]=` and `e` for seq's `add`. + # analyse the body, find the deepest expression 'it' and replace it via + # 'result.add it' + runnableExamples: + import std/[sets, tables] + + let data = @["bird", "word"] + + ## seq: + let k = collect(newSeq): + for i, d in data.pairs: + if i mod 2 == 0: d + assert k == @["bird"] + + ## seq with initialSize: + let x = collect(newSeqOfCap(4)): + for i, d in data.pairs: + if i mod 2 == 0: d + assert x == @["bird"] + + ## HashSet: + let y = collect(initHashSet()): + for d in data.items: {d} + assert y == data.toHashSet + + ## Table: + let z = collect(initTable(2)): + for i, d in data.pairs: {i: d} + assert z == {0: "bird", 1: "word"}.toTable + + result = collectImpl(init, body) + +macro collect*(body: untyped): untyped {.since: (1, 5).} = + ## Same as `collect` but without an `init` parameter. + ## + ## **See also:** + ## * `sequtils.toSeq proc<sequtils.html#toSeq.t%2Cuntyped>`_ + ## * `sequtils.mapIt template<sequtils.html#mapIt.t%2Ctyped%2Cuntyped>`_ + runnableExamples: + import std/[sets, tables] + let data = @["bird", "word"] + + # seq: + let k = collect: + for i, d in data.pairs: + if i mod 2 == 0: d + assert k == @["bird"] + + ## HashSet: + let n = collect: + for d in data.items: {d} + assert n == data.toHashSet + + ## Table: + let m = collect: + for i, d in data.pairs: {i: d} + assert m == {0: "bird", 1: "word"}.toTable + + result = collectImpl(nil, body) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 60f064e7c..53b3d61da 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -12,12 +12,95 @@ ## sequences and does not depend on any other module, on Windows it uses the ## Windows API. ## Changing the style is permanent even after program termination! Use the -## code ``system.addQuitProc(resetAttributes)`` to restore the defaults. +## code `exitprocs.addExitProc(resetAttributes)` to restore the defaults. +## Similarly, if you hide the cursor, make sure to unhide it with +## `showCursor` before quitting. +## +## Progress bar +## ============ +## +## Basic progress bar example: +runnableExamples("-r:off"): + import std/[os, strutils] -import macros + for i in 0..100: + stdout.styledWriteLine(fgRed, "0% ", fgWhite, '#'.repeat i, if i > 50: fgGreen else: fgYellow, "\t", $i , "%") + sleep 42 + cursorUp 1 + eraseLine() + + stdout.resetAttributes() + +##[ +## Playing with colorful and styled text +]## + +## Procs like `styledWriteLine`, `styledEcho` etc. have a temporary effect on +## text parameters. Style parameters only affect the text parameter right after them. +## After being called, these procs will reset the default style of the terminal. +## While `setBackGroundColor`, `setForeGroundColor` etc. have a lasting +## influence on the terminal, you can use `resetAttributes` to +## reset the default style of the terminal. +runnableExamples("-r:off"): + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text without style ") + + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + + # You can specify multiple text parameters. Style parameters + # only affect the text parameter right after them. + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + +import std/macros +import std/strformat +from std/strutils import toLowerAscii, `%`, parseInt +import std/colors when defined(windows): - import winlean, os + import std/winlean + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + +type + PTerminal = ref object + trueColorIsSupported: bool + trueColorIsEnabled: bool + fgSetColor: bool + when defined(windows): + hStdout: Handle + hStderr: Handle + oldStdoutAttr: int16 + oldStderrAttr: int16 + +var gTerm {.threadvar.}: owned(PTerminal) + +when defined(windows) and defined(consoleapp): + proc newTerminal(): owned(PTerminal) {.gcsafe, raises: [OSError].} +else: + proc newTerminal(): owned(PTerminal) {.gcsafe, raises: [].} + +proc getTerminal(): PTerminal {.inline.} = + if isNil(gTerm): + gTerm = newTerminal() + result = gTerm + +const + fgPrefix = "\e[38;2;" + bgPrefix = "\e[48;2;" + ansiResetCode* = "\e[0m" + getPos = "\e[6n" + stylePrefix = "\e[" + +when defined(windows): + import std/[winlean, os] const DUPLICATE_SAME_ACCESS = 2 @@ -29,18 +112,22 @@ when defined(windows): BACKGROUND_GREEN = 32 BACKGROUND_RED = 64 BACKGROUND_INTENSITY = 128 + FOREGROUND_RGB = FOREGROUND_RED or FOREGROUND_GREEN or FOREGROUND_BLUE + BACKGROUND_RGB = BACKGROUND_RED or BACKGROUND_GREEN or BACKGROUND_BLUE + + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 type SHORT = int16 COORD = object - X: SHORT - Y: SHORT + x: SHORT + y: SHORT SMALL_RECT = object - Left: SHORT - Top: SHORT - Right: SHORT - Bottom: SHORT + left: SHORT + top: SHORT + right: SHORT + bottom: SHORT CONSOLE_SCREEN_BUFFER_INFO = object dwSize: COORD @@ -49,85 +136,132 @@ when defined(windows): srWindow: SMALL_RECT dwMaximumWindowSize: COORD - proc duplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, - hTargetProcessHandle: HANDLE, lpTargetHandle: ptr HANDLE, + CONSOLE_CURSOR_INFO = object + dwSize: DWORD + bVisible: WINBOOL + + proc duplicateHandle(hSourceProcessHandle: Handle, hSourceHandle: Handle, + hTargetProcessHandle: Handle, lpTargetHandle: ptr Handle, dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwOptions: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "DuplicateHandle".} - proc getCurrentProcess(): HANDLE{.stdcall, dynlib: "kernel32", + proc getCurrentProcess(): Handle{.stdcall, dynlib: "kernel32", importc: "GetCurrentProcess".} - proc getConsoleScreenBufferInfo(hConsoleOutput: HANDLE, + proc getConsoleScreenBufferInfo(hConsoleOutput: Handle, lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".} - proc setConsoleCursorPosition(hConsoleOutput: HANDLE, + proc getConsoleCursorInfo(hConsoleOutput: Handle, + lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "GetConsoleCursorInfo".} + + proc setConsoleCursorInfo(hConsoleOutput: Handle, + lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleCursorInfo".} + + proc terminalWidthIoctl*(handles: openArray[Handle]): int = + var csbi: CONSOLE_SCREEN_BUFFER_INFO + for h in handles: + if getConsoleScreenBufferInfo(h, addr csbi) != 0: + return int(csbi.srWindow.right - csbi.srWindow.left + 1) + return 0 + + proc terminalHeightIoctl*(handles: openArray[Handle]): int = + var csbi: CONSOLE_SCREEN_BUFFER_INFO + for h in handles: + if getConsoleScreenBufferInfo(h, addr csbi) != 0: + return int(csbi.srWindow.bottom - csbi.srWindow.top + 1) + return 0 + + proc terminalWidth*(): int = + ## Returns the terminal width in columns. + var w: int = 0 + w = terminalWidthIoctl([getStdHandle(STD_INPUT_HANDLE), + getStdHandle(STD_OUTPUT_HANDLE), + getStdHandle(STD_ERROR_HANDLE)]) + if w > 0: return w + return 80 + + proc terminalHeight*(): int = + ## Returns the terminal height in rows. + var h: int = 0 + h = terminalHeightIoctl([getStdHandle(STD_INPUT_HANDLE), + getStdHandle(STD_OUTPUT_HANDLE), + getStdHandle(STD_ERROR_HANDLE)]) + if h > 0: return h + return 0 + + proc setConsoleCursorPosition(hConsoleOutput: Handle, dwCursorPosition: COORD): WINBOOL{. stdcall, dynlib: "kernel32", importc: "SetConsoleCursorPosition".} proc fillConsoleOutputCharacter(hConsoleOutput: Handle, cCharacter: char, - nLength: DWORD, dwWriteCoord: Coord, + nLength: DWORD, dwWriteCoord: COORD, lpNumberOfCharsWritten: ptr DWORD): WINBOOL{. stdcall, dynlib: "kernel32", importc: "FillConsoleOutputCharacterA".} - proc fillConsoleOutputAttribute(hConsoleOutput: HANDLE, wAttribute: int16, + proc fillConsoleOutputAttribute(hConsoleOutput: Handle, wAttribute: int16, nLength: DWORD, dwWriteCoord: COORD, lpNumberOfAttrsWritten: ptr DWORD): WINBOOL{. stdcall, dynlib: "kernel32", importc: "FillConsoleOutputAttribute".} - proc setConsoleTextAttribute(hConsoleOutput: HANDLE, + proc setConsoleTextAttribute(hConsoleOutput: Handle, wAttributes: int16): WINBOOL{. stdcall, dynlib: "kernel32", importc: "SetConsoleTextAttribute".} - var - hStdout: Handle # = createFile("CONOUT$", GENERIC_WRITE, 0, nil, - # OPEN_ALWAYS, 0, 0) - hStderr: Handle + proc getConsoleMode(hConsoleHandle: Handle, dwMode: ptr DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "GetConsoleMode".} - block: - var hStdoutTemp = getStdHandle(STD_OUTPUT_HANDLE) - if duplicateHandle(getCurrentProcess(), hStdoutTemp, getCurrentProcess(), - addr(hStdout), 0, 1, DUPLICATE_SAME_ACCESS) == 0: - when defined(consoleapp): - raiseOSError(osLastError()) - var hStderrTemp = getStdHandle(STD_ERROR_HANDLE) - if duplicateHandle(getCurrentProcess(), hStderrTemp, getCurrentProcess(), - addr(hStderr), 0, 1, DUPLICATE_SAME_ACCESS) == 0: - when defined(consoleapp): - raiseOSError(osLastError()) + proc setConsoleMode(hConsoleHandle: Handle, dwMode: DWORD): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "SetConsoleMode".} - proc getCursorPos(h: Handle): tuple [x,y: int] = - var c: CONSOLESCREENBUFFERINFO + proc getCursorPos(h: Handle): tuple [x, y: int] = + var c: CONSOLE_SCREEN_BUFFER_INFO if getConsoleScreenBufferInfo(h, addr(c)) == 0: raiseOSError(osLastError()) - return (int(c.dwCursorPosition.X), int(c.dwCursorPosition.Y)) + return (int(c.dwCursorPosition.x), int(c.dwCursorPosition.y)) + + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError, OSError].} = + return getCursorPos(getStdHandle(STD_OUTPUT_HANDLE)) proc setCursorPos(h: Handle, x, y: int) = var c: COORD - c.X = int16(x) - c.Y = int16(y) + c.x = int16(x) + c.y = int16(y) if setConsoleCursorPosition(h, c) == 0: raiseOSError(osLastError()) proc getAttributes(h: Handle): int16 = - var c: CONSOLESCREENBUFFERINFO + var c: CONSOLE_SCREEN_BUFFER_INFO # workaround Windows bugs: try several times if getConsoleScreenBufferInfo(h, addr(c)) != 0: return c.wAttributes return 0x70'i16 # ERROR: return white background, black text - var - oldStdoutAttr = getAttributes(hStdout) - oldStderrAttr = getAttributes(hStderr) + proc initTerminal(term: PTerminal) = + var hStdoutTemp = getStdHandle(STD_OUTPUT_HANDLE) + if duplicateHandle(getCurrentProcess(), hStdoutTemp, getCurrentProcess(), + addr(term.hStdout), 0, 1, DUPLICATE_SAME_ACCESS) == 0: + when defined(consoleapp): + raiseOSError(osLastError()) + var hStderrTemp = getStdHandle(STD_ERROR_HANDLE) + if duplicateHandle(getCurrentProcess(), hStderrTemp, getCurrentProcess(), + addr(term.hStderr), 0, 1, DUPLICATE_SAME_ACCESS) == 0: + when defined(consoleapp): + raiseOSError(osLastError()) + term.oldStdoutAttr = getAttributes(term.hStdout) + term.oldStderrAttr = getAttributes(term.hStderr) template conHandle(f: File): Handle = - if f == stderr: hStderr else: hStdout + let term = getTerminal() + if f == stderr: term.hStderr else: term.hStdout else: - import termios + import std/[termios, posix, os, parseutils] proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = var mode: Termios - discard fd.tcgetattr(addr mode) + discard fd.tcGetAttr(addr mode) mode.c_iflag = mode.c_iflag and not Cflag(BRKINT or ICRNL or INPCK or ISTRIP or IXON) mode.c_oflag = mode.c_oflag and not Cflag(OPOST) @@ -135,7 +269,162 @@ else: mode.c_lflag = mode.c_lflag and not Cflag(ECHO or ICANON or IEXTEN or ISIG) mode.c_cc[VMIN] = 1.cuchar mode.c_cc[VTIME] = 0.cuchar - discard fd.tcsetattr(time, addr mode) + discard fd.tcSetAttr(time, addr mode) + + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError].} = + ## Returns cursor position (x, y) + ## writes to stdout and expects the terminal to respond via stdin + var + xStr = "" + yStr = "" + ch: char + ct: int + readX = false + + # use raw mode to ask terminal for cursor position + let fd = getFileHandle(stdin) + var oldMode: Termios + discard fd.tcGetAttr(addr oldMode) + fd.setRaw() + stdout.write(getPos) + flushFile(stdout) + + try: + # parse response format: [yyy;xxxR + while true: + let n = readBuffer(stdin, addr ch, 1) + if n == 0 or ch == 'R': + if xStr == "" or yStr == "": + raise newException(ValueError, "Got character position message that was missing data") + break + ct += 1 + if ct > 16: + raise newException(ValueError, "Got unterminated character position message from terminal") + if ch == ';': + readX = true + elif ch in {'0'..'9'}: + if readX: + xStr.add(ch) + else: + yStr.add(ch) + finally: + # restore previous terminal mode + discard fd.tcSetAttr(TCSADRAIN, addr oldMode) + + return (parseInt(xStr), parseInt(yStr)) + + proc terminalWidthIoctl*(fds: openArray[int]): int = + ## Returns terminal width from first fd that supports the ioctl. + + var win: IOctl_WinSize + for fd in fds: + if ioctl(cint(fd), TIOCGWINSZ, addr win) != -1: + return int(win.ws_col) + return 0 + + proc terminalHeightIoctl*(fds: openArray[int]): int = + ## Returns terminal height from first fd that supports the ioctl. + + var win: IOctl_WinSize + for fd in fds: + if ioctl(cint(fd), TIOCGWINSZ, addr win) != -1: + return int(win.ws_row) + return 0 + + var L_ctermid{.importc, header: "<stdio.h>".}: cint + + proc terminalWidth*(): int = + ## Returns some reasonable terminal width from either standard file + ## descriptors, controlling terminal, environment variables or tradition. + + # POSIX environment variable takes precendence. + # _COLUMNS_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred width in column positions for + # the terminal screen or window. If this variable is unset or null, + # the implementation determines the number of columns, appropriate + # for the terminal or window, in an unspecified manner. + # When COLUMNS is set, any terminal-width information implied by TERM + # is overridden. Users and conforming applications should not set COLUMNS + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var w: int + var s = getEnv("COLUMNS") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, w) > 0 and w > 0: + return w + w = terminalWidthIoctl([0, 1, 2]) # Try standard file descriptors + if w > 0: return w + var cterm = newString(L_ctermid) # Try controlling tty + var fd = open(ctermid(cstring(cterm)), O_RDONLY) + if fd != -1: + w = terminalWidthIoctl([int(fd)]) + discard close(fd) + if w > 0: return w + return 80 # Finally default to venerable value + + proc terminalHeight*(): int = + ## Returns some reasonable terminal height from either standard file + ## descriptors, controlling terminal, environment variables or tradition. + ## Zero is returned if the height could not be determined. + + # POSIX environment variable takes precendence. + # _LINES_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred number of lines on a page or + # the vertical screen or window size in lines. A line in this case + # is a vertical measure large enough to hold the tallest character + # in the character set being displayed. If this variable is unset or null, + # the implementation determines the number of lines, appropriate + # for the terminal or window (size, terminal baud rate, and so on), + # in an unspecified manner. + # When LINES is set, any terminal-height information implied by TERM + # is overridden. Users and conforming applications should not set LINES + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var h: int + var s = getEnv("LINES") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, h) > 0 and h > 0: + return h + h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors + if h > 0: return h + var cterm = newString(L_ctermid) # Try controlling tty + var fd = open(ctermid(cstring(cterm)), O_RDONLY) + if fd != -1: + h = terminalHeightIoctl([int(fd)]) + discard close(fd) + if h > 0: return h + return 0 # Could not determine height + +proc terminalSize*(): tuple[w, h: int] = + ## Returns the terminal width and height as a tuple. Internally calls + ## `terminalWidth` and `terminalHeight`, so the same assumptions apply. + result = (terminalWidth(), terminalHeight()) + +when defined(windows): + proc setCursorVisibility(f: File, visible: bool) = + var ccsi: CONSOLE_CURSOR_INFO + let h = conHandle(f) + if getConsoleCursorInfo(h, addr(ccsi)) == 0: + raiseOSError(osLastError()) + ccsi.bVisible = if visible: 1 else: 0 + if setConsoleCursorInfo(h, addr(ccsi)) == 0: + raiseOSError(osLastError()) + +proc hideCursor*(f: File) = + ## Hides the cursor. + when defined(windows): + setCursorVisibility(f, false) + else: + f.write("\e[?25l") + +proc showCursor*(f: File) = + ## Shows the cursor. + when defined(windows): + setCursorVisibility(f, true) + else: + f.write("\e[?25h") proc setCursorPos*(f: File, x, y: int) = ## Sets the terminal's cursor to the (x,y) position. @@ -144,42 +433,45 @@ proc setCursorPos*(f: File, x, y: int) = let h = conHandle(f) setCursorPos(h, x, y) else: - f.write("\e[" & $y & ';' & $x & 'f') + f.write(fmt"{stylePrefix}{y+1};{x+1}f") proc setCursorXPos*(f: File, x: int) = ## Sets the terminal's cursor to the x position. ## The y position is not changed. when defined(windows): let h = conHandle(f) - var scrbuf: CONSOLESCREENBUFFERINFO + var scrbuf: CONSOLE_SCREEN_BUFFER_INFO if getConsoleScreenBufferInfo(h, addr(scrbuf)) == 0: raiseOSError(osLastError()) var origin = scrbuf.dwCursorPosition - origin.X = int16(x) + origin.x = int16(x) if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) else: - f.write("\e[" & $x & 'G') + f.write(fmt"{stylePrefix}{x+1}G") when defined(windows): proc setCursorYPos*(f: File, y: int) = ## Sets the terminal's cursor to the y position. ## The x position is not changed. - ## **Warning**: This is not supported on UNIX! + ## .. warning:: This is not supported on UNIX! when defined(windows): let h = conHandle(f) - var scrbuf: CONSOLESCREENBUFFERINFO + var scrbuf: CONSOLE_SCREEN_BUFFER_INFO if getConsoleScreenBufferInfo(h, addr(scrbuf)) == 0: raiseOSError(osLastError()) var origin = scrbuf.dwCursorPosition - origin.Y = int16(y) + origin.y = int16(y) if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) else: discard -proc cursorUp*(f: File, count=1) = +proc cursorUp*(f: File, count = 1) = ## Moves the cursor up by `count` rows. + runnableExamples("-r:off"): + stdout.cursorUp(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -188,35 +480,44 @@ proc cursorUp*(f: File, count=1) = else: f.write("\e[" & $count & 'A') -proc cursorDown*(f: File, count=1) = +proc cursorDown*(f: File, count = 1) = ## Moves the cursor down by `count` rows. + runnableExamples("-r:off"): + stdout.cursorDown(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) inc(p.y, count) setCursorPos(h, p.x, p.y) else: - f.write("\e[" & $count & 'B') + f.write(fmt"{stylePrefix}{count}B") -proc cursorForward*(f: File, count=1) = +proc cursorForward*(f: File, count = 1) = ## Moves the cursor forward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorForward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) inc(p.x, count) setCursorPos(h, p.x, p.y) else: - f.write("\e[" & $count & 'C') + f.write(fmt"{stylePrefix}{count}C") -proc cursorBackward*(f: File, count=1) = +proc cursorBackward*(f: File, count = 1) = ## Moves the cursor backward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorBackward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) dec(p.x, count) setCursorPos(h, p.x, p.y) else: - f.write("\e[" & $count & 'D') + f.write(fmt"{stylePrefix}{count}D") when true: discard @@ -251,22 +552,24 @@ else: proc eraseLine*(f: File) = ## Erases the entire current line. + runnableExamples("-r:off"): + write(stdout, "never mind") + stdout.eraseLine() # nothing will be printed on the screen when defined(windows): let h = conHandle(f) - var scrbuf: CONSOLESCREENBUFFERINFO + var scrbuf: CONSOLE_SCREEN_BUFFER_INFO var numwrote: DWORD if getConsoleScreenBufferInfo(h, addr(scrbuf)) == 0: raiseOSError(osLastError()) var origin = scrbuf.dwCursorPosition - origin.X = 0'i16 + origin.x = 0'i16 if setConsoleCursorPosition(h, origin) == 0: raiseOSError(osLastError()) - var ht = scrbuf.dwSize.Y - origin.Y - var wt = scrbuf.dwSize.X - origin.X - if fillConsoleOutputCharacter(h, ' ', ht*wt, + var wt: DWORD = scrbuf.dwSize.x - origin.x + if fillConsoleOutputCharacter(h, ' ', wt, origin, addr(numwrote)) == 0: raiseOSError(osLastError()) - if fillConsoleOutputAttribute(h, scrbuf.wAttributes, ht * wt, + if fillConsoleOutputAttribute(h, scrbuf.wAttributes, wt, scrbuf.dwCursorPosition, addr(numwrote)) == 0: raiseOSError(osLastError()) else: @@ -277,13 +580,13 @@ proc eraseScreen*(f: File) = ## Erases the screen with the background colour and moves the cursor to home. when defined(windows): let h = conHandle(f) - var scrbuf: CONSOLESCREENBUFFERINFO + var scrbuf: CONSOLE_SCREEN_BUFFER_INFO var numwrote: DWORD var origin: COORD # is inititalized to 0, 0 if getConsoleScreenBufferInfo(h, addr(scrbuf)) == 0: raiseOSError(osLastError()) - let numChars = int32(scrbuf.dwSize.X)*int32(scrbuf.dwSize.Y) + let numChars = int32(scrbuf.dwSize.x)*int32(scrbuf.dwSize.y) if fillConsoleOutputCharacter(h, ' ', numChars, origin, addr(numwrote)) == 0: @@ -295,131 +598,207 @@ proc eraseScreen*(f: File) = else: f.write("\e[2J") +when not defined(windows): + var + gFG {.threadvar.}: int + gBG {.threadvar.}: int + proc resetAttributes*(f: File) = ## Resets all attributes. when defined(windows): + let term = getTerminal() if f == stderr: - discard setConsoleTextAttribute(hStderr, oldStderrAttr) + discard setConsoleTextAttribute(term.hStderr, term.oldStderrAttr) else: - discard setConsoleTextAttribute(hStdout, oldStdoutAttr) + discard setConsoleTextAttribute(term.hStdout, term.oldStdoutAttr) else: - f.write("\e[0m") + f.write(ansiResetCode) + gFG = 0 + gBG = 0 type - Style* = enum ## different styles for text output - styleBright = 1, ## bright text - styleDim, ## dim text - styleUnknown, ## unknown - styleUnderscore = 4, ## underscored text - styleBlink, ## blinking/bold text - styleReverse = 7, ## unknown - styleHidden ## hidden text + Style* = enum ## Different styles for text output. + styleBright = 1, ## bright text + styleDim, ## dim text + styleItalic, ## italic (or reverse on terminals not supporting) + styleUnderscore, ## underscored text + styleBlink, ## blinking/bold text + styleBlinkRapid, ## rapid blinking/bold text (not widely supported) + styleReverse, ## reverse + styleHidden, ## hidden text + styleStrikethrough ## strikethrough -{.deprecated: [TStyle: Style].} +proc ansiStyleCode*(style: int): string = + result = fmt"{stylePrefix}{style}m" -when not defined(windows): - var - # XXX: These better be thread-local - gFG = 0 - gBG = 0 +template ansiStyleCode*(style: Style): string = + ansiStyleCode(style.int) + +# The styleCache can be skipped when `style` is known at compile-time +template ansiStyleCode*(style: static[Style]): string = + (static(stylePrefix & $style.int & "m")) proc setStyle*(f: File, style: set[Style]) = ## Sets the terminal style. when defined(windows): let h = conHandle(f) + var old = getAttributes(h) and (FOREGROUND_RGB or BACKGROUND_RGB) var a = 0'i16 if styleBright in style: a = a or int16(FOREGROUND_INTENSITY) if styleBlink in style: a = a or int16(BACKGROUND_INTENSITY) if styleReverse in style: a = a or 0x4000'i16 # COMMON_LVB_REVERSE_VIDEO if styleUnderscore in style: a = a or 0x8000'i16 # COMMON_LVB_UNDERSCORE - discard setConsoleTextAttribute(h, a) + discard setConsoleTextAttribute(h, old or a) else: for s in items(style): - f.write("\e[" & $ord(s) & 'm') + f.write(ansiStyleCode(s)) proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = ## Writes the text `txt` in a given `style` to stdout. when defined(windows): - var old = getAttributes(hStdout) + let term = getTerminal() + var old = getAttributes(term.hStdout) stdout.setStyle(style) stdout.write(txt) - discard setConsoleTextAttribute(hStdout, old) + discard setConsoleTextAttribute(term.hStdout, old) else: stdout.setStyle(style) stdout.write(txt) stdout.resetAttributes() if gFG != 0: - stdout.write("\e[" & $ord(gFG) & 'm') + stdout.write(ansiStyleCode(gFG)) if gBG != 0: - stdout.write("\e[" & $ord(gBG) & 'm') + stdout.write(ansiStyleCode(gBG)) type - ForegroundColor* = enum ## terminal's foreground colors - fgBlack = 30, ## black - fgRed, ## red - fgGreen, ## green - fgYellow, ## yellow - fgBlue, ## blue - fgMagenta, ## magenta - fgCyan, ## cyan - fgWhite ## white - - BackgroundColor* = enum ## terminal's background colors - bgBlack = 40, ## black - bgRed, ## red - bgGreen, ## green - bgYellow, ## yellow - bgBlue, ## blue - bgMagenta, ## magenta - bgCyan, ## cyan - bgWhite ## white - -{.deprecated: [TForegroundColor: ForegroundColor, - TBackgroundColor: BackgroundColor].} - -proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = + ForegroundColor* = enum ## Terminal's foreground colors. + fgBlack = 30, ## black + fgRed, ## red + fgGreen, ## green + fgYellow, ## yellow + fgBlue, ## blue + fgMagenta, ## magenta + fgCyan, ## cyan + fgWhite, ## white + fg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) + fgDefault ## default terminal foreground color + + BackgroundColor* = enum ## Terminal's background colors. + bgBlack = 40, ## black + bgRed, ## red + bgGreen, ## green + bgYellow, ## yellow + bgBlue, ## blue + bgMagenta, ## magenta + bgCyan, ## cyan + bgWhite, ## white + bg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) + bgDefault ## default terminal background color + +when defined(windows): + var defaultForegroundColor, defaultBackgroundColor: int16 = 0xFFFF'i16 # Default to an invalid value 0xFFFF + +proc setForegroundColor*(f: File, fg: ForegroundColor, bright = false) = ## Sets the terminal's foreground color. when defined(windows): let h = conHandle(f) - var old = getAttributes(h) and not 0x0007 - if bright: - old = old or FOREGROUND_INTENSITY - const lookup: array [ForegroundColor, int] = [ - 0, + var old = getAttributes(h) and not FOREGROUND_RGB + if defaultForegroundColor == 0xFFFF'i16: + defaultForegroundColor = old + old = if bright: old or FOREGROUND_INTENSITY + else: old and not(FOREGROUND_INTENSITY) + const lookup: array[ForegroundColor, int] = [ + 0, # ForegroundColor enum with ordinal 30 (FOREGROUND_RED), (FOREGROUND_GREEN), (FOREGROUND_RED or FOREGROUND_GREEN), (FOREGROUND_BLUE), (FOREGROUND_RED or FOREGROUND_BLUE), (FOREGROUND_BLUE or FOREGROUND_GREEN), - (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED)] - discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) + (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED), + 0, # fg8Bit not supported, see `enableTrueColors` instead. + 0] # unused + if fg == fgDefault: + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultForegroundColor))) + else: + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[fg]))) else: gFG = ord(fg) if bright: inc(gFG, 60) - f.write("\e[" & $gFG & 'm') + f.write(ansiStyleCode(gFG)) -proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = +proc setBackgroundColor*(f: File, bg: BackgroundColor, bright = false) = ## Sets the terminal's background color. when defined(windows): let h = conHandle(f) - var old = getAttributes(h) and not 0x0070 - if bright: - old = old or BACKGROUND_INTENSITY - const lookup: array [BackgroundColor, int] = [ - 0, + var old = getAttributes(h) and not BACKGROUND_RGB + if defaultBackgroundColor == 0xFFFF'i16: + defaultBackgroundColor = old + old = if bright: old or BACKGROUND_INTENSITY + else: old and not(BACKGROUND_INTENSITY) + const lookup: array[BackgroundColor, int] = [ + 0, # BackgroundColor enum with ordinal 40 (BACKGROUND_RED), (BACKGROUND_GREEN), (BACKGROUND_RED or BACKGROUND_GREEN), (BACKGROUND_BLUE), (BACKGROUND_RED or BACKGROUND_BLUE), (BACKGROUND_BLUE or BACKGROUND_GREEN), - (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED)] - discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) + (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED), + 0, # bg8Bit not supported, see `enableTrueColors` instead. + 0] # unused + if bg == bgDefault: + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultBackgroundColor))) + else: + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[bg]))) else: gBG = ord(bg) if bright: inc(gBG, 60) - f.write("\e[" & $gBG & 'm') + f.write(ansiStyleCode(gBG)) + +proc ansiForegroundColorCode*(fg: ForegroundColor, bright = false): string = + var style = ord(fg) + if bright: inc(style, 60) + return ansiStyleCode(style) + +template ansiForegroundColorCode*(fg: static[ForegroundColor], + bright: static[bool] = false): string = + ansiStyleCode(fg.int + bright.int * 60) + +proc ansiForegroundColorCode*(color: Color): string = + let rgb = extractRGB(color) + result = fmt"{fgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + +template ansiForegroundColorCode*(color: static[Color]): string = + const rgb = extractRGB(color) + # no usage of `fmt`, see issue #7632 + (static("$1$2;$3;$4m" % [$fgPrefix, $(rgb.r), $(rgb.g), $(rgb.b)])) + +proc ansiBackgroundColorCode*(color: Color): string = + let rgb = extractRGB(color) + result = fmt"{bgPrefix}{rgb.r};{rgb.g};{rgb.b}m" + +template ansiBackgroundColorCode*(color: static[Color]): string = + const rgb = extractRGB(color) + # no usage of `fmt`, see issue #7632 + (static("$1$2;$3;$4m" % [$bgPrefix, $(rgb.r), $(rgb.g), $(rgb.b)])) + +proc setForegroundColor*(f: File, color: Color) = + ## Sets the terminal's foreground true color. + if getTerminal().trueColorIsEnabled: + f.write(ansiForegroundColorCode(color)) + +proc setBackgroundColor*(f: File, color: Color) = + ## Sets the terminal's background true color. + if getTerminal().trueColorIsEnabled: + f.write(ansiBackgroundColorCode(color)) + +proc setTrueColor(f: File, color: Color) = + let term = getTerminal() + if term.fgSetColor: + setForegroundColor(f, color) + else: + setBackgroundColor(f, color) proc isatty*(f: File): bool = ## Returns true if `f` is associated with a terminal device. @@ -433,8 +812,10 @@ proc isatty*(f: File): bool = result = isatty(getFileHandle(f)) != 0'i32 type - TerminalCmd* = enum ## commands that can be expressed as arguments - resetStyle ## reset attributes + TerminalCmd* = enum ## commands that can be expressed as arguments + resetStyle, ## reset attributes + fgColor, ## set foreground's true color + bgColor ## set background's true color template styledEchoProcessArg(f: File, s: string) = write f, s template styledEchoProcessArg(f: File, style: Style) = setStyle(f, {style}) @@ -443,34 +824,34 @@ template styledEchoProcessArg(f: File, color: ForegroundColor) = setForegroundColor f, color template styledEchoProcessArg(f: File, color: BackgroundColor) = setBackgroundColor f, color +template styledEchoProcessArg(f: File, color: Color) = + setTrueColor f, color template styledEchoProcessArg(f: File, cmd: TerminalCmd) = when cmd == resetStyle: resetAttributes(f) + elif cmd in {fgColor, bgColor}: + let term = getTerminal() + term.fgSetColor = cmd == fgColor + +macro styledWrite*(f: File, m: varargs[typed]): untyped = + ## Similar to `write`, but treating terminal style arguments specially. + ## When some argument is `Style`, `set[Style]`, `ForegroundColor`, + ## `BackgroundColor` or `TerminalCmd` then it is not sent directly to + ## `f`, but instead corresponding terminal style proc is called. + runnableExamples("-r:off"): + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgGreen, "green text") -macro styledWriteLine*(f: File, m: varargs[expr]): stmt = - ## Similar to ``writeLine``, but treating terminal style arguments specially. - ## When some argument is ``Style``, ``set[Style]``, ``ForegroundColor``, - ## ``BackgroundColor`` or ``TerminalCmd`` then it is not sent directly to - ## ``f``, but instead corresponding terminal style proc is called. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## proc error(msg: string) = - ## styleWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) - ## - let m = callsite() var reset = false result = newNimNode(nnkStmtList) - for i in countup(2, m.len - 1): + for i in countup(0, m.len - 1): let item = m[i] case item.kind of nnkStrLit..nnkTripleStrLit: if i == m.len - 1: - # optimize if string literal is last, just call writeLine - result.add(newCall(bindSym"writeLine", f, item)) + # optimize if string literal is last, just call write + result.add(newCall(bindSym"write", f, item)) if reset: result.add(newCall(bindSym"resetAttributes", f)) return else: @@ -479,67 +860,174 @@ macro styledWriteLine*(f: File, m: varargs[expr]): stmt = else: result.add(newCall(bindSym"styledEchoProcessArg", f, item)) reset = true - - result.add(newCall(bindSym"write", f, newStrLitNode("\n"))) if reset: result.add(newCall(bindSym"resetAttributes", f)) -macro callStyledEcho(args: varargs[expr]): stmt = - result = newCall(bindSym"styledWriteLine") - result.add(bindSym"stdout") - for arg in children(args[0][1]): - result.add(arg) - -template styledEcho*(args: varargs[expr]): expr = - ## Echoes styles arguments to stdout using ``styledWriteLine``. - callStyledEcho(args) - -when defined(nimdoc): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. - discard -elif not defined(windows): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. +template styledWriteLine*(f: File, args: varargs[untyped]) = + ## Calls `styledWrite` and appends a newline at the end. + runnableExamples: + proc error(msg: string) = + styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + + styledWrite(f, args) + write(f, "\n") + +template styledEcho*(args: varargs[untyped]) = + ## Echoes styles arguments to stdout using `styledWriteLine`. + stdout.styledWriteLine(args) + +proc getch*(): char = + ## Reads a single character from the terminal, blocking until it is entered. + ## The character is not printed to the terminal. + when defined(windows): + let fd = getStdHandle(STD_INPUT_HANDLE) + var keyEvent = KEY_EVENT_RECORD() + var numRead: cint + while true: + # Block until character is entered + doAssert(waitForSingleObject(fd, INFINITE) == WAIT_OBJECT_0) + doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) + if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: + continue + return char(keyEvent.uChar) + else: let fd = getFileHandle(stdin) var oldMode: Termios - discard fd.tcgetattr(addr oldMode) + discard fd.tcGetAttr(addr oldMode) fd.setRaw() result = stdin.readChar() - discard fd.tcsetattr(TCSADRAIN, addr oldMode) + discard fd.tcSetAttr(TCSADRAIN, addr oldMode) + +when defined(windows): + proc readPasswordFromStdin*(prompt: string, password: var string): + bool {.tags: [ReadIOEffect, WriteIOEffect].} = + ## Reads a `password` from stdin without printing it. `password` must not + ## be `nil`! Returns `false` if the end of the file has been reached, + ## `true` otherwise. + password.setLen(0) + stdout.write(prompt) + let hi = createFileA("CONIN$", + GENERIC_READ or GENERIC_WRITE, 0, nil, OPEN_EXISTING, 0, 0) + var mode = DWORD 0 + discard getConsoleMode(hi, addr mode) + let origMode = mode + const + ENABLE_PROCESSED_INPUT = 1 + ENABLE_ECHO_INPUT = 4 + mode = (mode or ENABLE_PROCESSED_INPUT) and not ENABLE_ECHO_INPUT + + discard setConsoleMode(hi, mode) + result = readLine(stdin, password) + discard setConsoleMode(hi, origMode) + discard closeHandle(hi) + stdout.write "\n" + +else: + import std/termios + + proc readPasswordFromStdin*(prompt: string, password: var string): + bool {.tags: [ReadIOEffect, WriteIOEffect].} = + password.setLen(0) + let fd = stdin.getFileHandle() + var cur, old: Termios + discard fd.tcGetAttr(cur.addr) + old = cur + cur.c_lflag = cur.c_lflag and not Cflag(ECHO) + discard fd.tcSetAttr(TCSADRAIN, cur.addr) + stdout.write prompt + result = stdin.readLine(password) + stdout.write "\n" + discard fd.tcSetAttr(TCSADRAIN, old.addr) + +proc readPasswordFromStdin*(prompt = "password: "): string = + ## Reads a password from stdin without printing it. + result = "" + discard readPasswordFromStdin(prompt, result) + # Wrappers assuming output to stdout: +template hideCursor*() = hideCursor(stdout) +template showCursor*() = showCursor(stdout) template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y) -template setCursorXPos*(x: int) = setCursorXPos(stdout, x) +template setCursorXPos*(x: int) = setCursorXPos(stdout, x) when defined(windows): - template setCursorYPos(x: int) = setCursorYPos(stdout, x) -template cursorUp*(count=1) = cursorUp(stdout, f) -template cursorDown*(count=1) = cursorDown(stdout, f) -template cursorForward*(count=1) = cursorForward(stdout, f) -template cursorBackward*(count=1) = cursorBackward(stdout, f) -template eraseLine*() = eraseLine(stdout) -template eraseScreen*() = eraseScreen(stdout) + template setCursorYPos*(x: int) = setCursorYPos(stdout, x) +template cursorUp*(count = 1) = cursorUp(stdout, count) +template cursorDown*(count = 1) = cursorDown(stdout, count) +template cursorForward*(count = 1) = cursorForward(stdout, count) +template cursorBackward*(count = 1) = cursorBackward(stdout, count) +template eraseLine*() = eraseLine(stdout) +template eraseScreen*() = eraseScreen(stdout) template setStyle*(style: set[Style]) = setStyle(stdout, style) -template setForegroundColor*(fg: ForegroundColor, bright=false) = +template setForegroundColor*(fg: ForegroundColor, bright = false) = setForegroundColor(stdout, fg, bright) -template setBackgroundColor*(bg: BackgroundColor, bright=false) = +template setBackgroundColor*(bg: BackgroundColor, bright = false) = setBackgroundColor(stdout, bg, bright) +template setForegroundColor*(color: Color) = + setForegroundColor(stdout, color) +template setBackgroundColor*(color: Color) = + setBackgroundColor(stdout, color) proc resetAttributes*() {.noconv.} = ## Resets all attributes on stdout. ## It is advisable to register this as a quit proc with - ## ``system.addQuitProc(resetAttributes)``. + ## `exitprocs.addExitProc(resetAttributes)`. resetAttributes(stdout) -when not defined(testing) and isMainModule: - #system.addQuitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine("styled text ", {styleBright, styleBlink, styleUnderscore}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.writeLine("ordinary text") - stdout.resetAttributes() +proc isTrueColorSupported*(): bool = + ## Returns true if a terminal supports true color. + return getTerminal().trueColorIsSupported + +when defined(windows): + import std/os + +proc enableTrueColors*() = + ## Enables true color. + var term = getTerminal() + when defined(windows): + var + ver: OSVERSIONINFO + ver.dwOSVersionInfoSize = sizeof(ver).DWORD + let res = getVersionExW(addr ver) + if res == 0: + term.trueColorIsSupported = false + else: + term.trueColorIsSupported = ver.dwMajorVersion > 10 or + (ver.dwMajorVersion == 10 and (ver.dwMinorVersion > 0 or + (ver.dwMinorVersion == 0 and ver.dwBuildNumber >= 10586))) + if not term.trueColorIsSupported: + term.trueColorIsSupported = getEnv("ANSICON_DEF").len > 0 + + if term.trueColorIsSupported: + if getEnv("ANSICON_DEF").len == 0: + var mode: DWORD = 0 + if getConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), addr(mode)) != 0: + mode = mode or ENABLE_VIRTUAL_TERMINAL_PROCESSING + if setConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), mode) != 0: + term.trueColorIsEnabled = true + else: + term.trueColorIsEnabled = false + else: + term.trueColorIsEnabled = true + else: + term.trueColorIsSupported = getEnv("COLORTERM").toLowerAscii() in [ + "truecolor", "24bit"] + term.trueColorIsEnabled = term.trueColorIsSupported + +proc disableTrueColors*() = + ## Disables true color. + var term = getTerminal() + when defined(windows): + if term.trueColorIsSupported: + if getEnv("ANSICON_DEF").len == 0: + var mode: DWORD = 0 + if getConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), addr(mode)) != 0: + mode = mode and not ENABLE_VIRTUAL_TERMINAL_PROCESSING + discard setConsoleMode(getStdHandle(STD_OUTPUT_HANDLE), mode) + term.trueColorIsEnabled = false + else: + term.trueColorIsEnabled = false + +proc newTerminal(): owned(PTerminal) = + new result + when defined(windows): + initTerminal(result) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index c9854a650..e59153455 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1,1209 +1,2889 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2018 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # - -## This module contains routines and types for dealing with time. -## This module is available for the `JavaScript target -## <backends.html#the-javascript-target>`_. -## -## Examples: -## -## .. code-block:: nim -## -## import times, os -## var -## t = cpuTime() -## -## sleep(100) # replace this with something to be timed -## echo "Time taken: ",cpuTime() - t -## -## echo "My formatted time: ", format(getLocalTime(getTime()), "d MMMM yyyy HH:mm") -## echo "Using predefined formats: ", getClockStr(), " ", getDateStr() -## -## echo "epochTime() float value: ", epochTime() -## echo "getTime() float value: ", toSeconds(getTime()) -## echo "cpuTime() float value: ", cpuTime() -## echo "An hour from now : ", getLocalTime(getTime()) + initInterval(0,0,0,1) -## echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1) - -{.push debugger:off.} # the user does not want to trace a part - # of the standard library! - -import - strutils, parseutils - +##[ + The `times` module contains routines and types for dealing with time using + the `proleptic Gregorian calendar<https://en.wikipedia.org/wiki/Proleptic_Gregorian_calendar>`_. + It's also available for the + `JavaScript target <backends.html#backends-the-javascript-target>`_. + + Although the `times` module supports nanosecond time resolution, the + resolution used by `getTime()` depends on the platform and backend + (JS is limited to millisecond precision). + + Examples + ======== + + ```nim + import std/[times, os] + # Simple benchmarking + let time = cpuTime() + sleep(100) # Replace this with something to be timed + echo "Time taken: ", cpuTime() - time + + # Current date & time + let now1 = now() # Current timestamp as a DateTime in local time + let now2 = now().utc # Current timestamp as a DateTime in UTC + let now3 = getTime() # Current timestamp as a Time + + # Arithmetic using Duration + echo "One hour from now : ", now() + initDuration(hours = 1) + # Arithmetic using TimeInterval + echo "One year from now : ", now() + 1.years + echo "One month from now : ", now() + 1.months + ``` + + Parsing and Formatting Dates + ============================ + + The `DateTime` type can be parsed and formatted using the different + `parse` and `format` procedures. + + ```nim + let dt = parse("2000-01-01", "yyyy-MM-dd") + echo dt.format("yyyy-MM-dd") + ``` + + The different format patterns that are supported are documented below. + + =========== ================================================================================= ============================================== + Pattern Description Example + =========== ================================================================================= ============================================== + `d` Numeric value representing the day of the month, | `1/04/2012 -> 1` + it will be either one or two digits long. | `21/04/2012 -> 21` + `dd` Same as above, but is always two digits. | `1/04/2012 -> 01` + | `21/04/2012 -> 21` + `ddd` Three letter string which indicates the day of the week. | `Saturday -> Sat` + | `Monday -> Mon` + `dddd` Full string for the day of the week. | `Saturday -> Saturday` + | `Monday -> Monday` + `GG` The last two digits of the Iso Week-Year | `30/12/2012 -> 13` + `GGGG` The Iso week-calendar year padded to four digits | `30/12/2012 -> 2013` + `h` The hours in one digit if possible. Ranging from 1-12. | `5pm -> 5` + | `2am -> 2` + `hh` The hours in two digits always. If the hour is one digit, 0 is prepended. | `5pm -> 05` + | `11am -> 11` + `H` The hours in one digit if possible, ranging from 0-23. | `5pm -> 17` + | `2am -> 2` + `HH` The hours in two digits always. 0 is prepended if the hour is one digit. | `5pm -> 17` + | `2am -> 02` + `m` The minutes in one digit if possible. | `5:30 -> 30` + | `2:01 -> 1` + `mm` Same as above but always two digits, 0 is prepended if the minute is one digit. | `5:30 -> 30` + | `2:01 -> 01` + `M` The month in one digit if possible. | `September -> 9` + | `December -> 12` + `MM` The month in two digits always. 0 is prepended if the month value is one digit. | `September -> 09` + | `December -> 12` + `MMM` Abbreviated three-letter form of the month. | `September -> Sep` + | `December -> Dec` + `MMMM` Full month string, properly capitalized. | `September -> September` + `s` Seconds as one digit if possible. | `00:00:06 -> 6` + `ss` Same as above but always two digits. 0 is prepended if the second is one digit. | `00:00:06 -> 06` + `t` `A` when time is in the AM. `P` when time is in the PM. | `5pm -> P` + | `2am -> A` + `tt` Same as above, but `AM` and `PM` instead of `A` and `P` respectively. | `5pm -> PM` + | `2am -> AM` + `yy` The last two digits of the year. When parsing, the current century is assumed. | `2012 AD -> 12` + `yyyy` The year, padded to at least four digits. | `2012 AD -> 2012` + Is always positive, even when the year is BC. | `24 AD -> 0024` + When the year is more than four digits, '+' is prepended. | `24 BC -> 00024` + | `12345 AD -> +12345` + `YYYY` The year without any padding. | `2012 AD -> 2012` + Is always positive, even when the year is BC. | `24 AD -> 24` + | `24 BC -> 24` + | `12345 AD -> 12345` + `uuuu` The year, padded to at least four digits. Will be negative when the year is BC. | `2012 AD -> 2012` + When the year is more than four digits, '+' is prepended unless the year is BC. | `24 AD -> 0024` + | `24 BC -> -0023` + | `12345 AD -> +12345` + `UUUU` The year without any padding. Will be negative when the year is BC. | `2012 AD -> 2012` + | `24 AD -> 24` + | `24 BC -> -23` + | `12345 AD -> 12345` + `V` The Iso Week-Number as one or two digits | `3/2/2012 -> 5` + | `1/4/2012 -> 13` + `VV` The Iso Week-Number as two digits always. 0 is prepended if one digit. | `3/2/2012 -> 05` + | `1/4/2012 -> 13` + `z` Displays the timezone offset from UTC. | `UTC+7 -> +7` + | `UTC-5 -> -5` + `zz` Same as above but with leading 0. | `UTC+7 -> +07` + | `UTC-5 -> -05` + `zzz` Same as above but with `:mm` where *mm* represents minutes. | `UTC+7 -> +07:00` + | `UTC-5 -> -05:00` + `ZZZ` Same as above but with `mm` where *mm* represents minutes. | `UTC+7 -> +0700` + | `UTC-5 -> -0500` + `zzzz` Same as above but with `:ss` where *ss* represents seconds. | `UTC+7 -> +07:00:00` + | `UTC-5 -> -05:00:00` + `ZZZZ` Same as above but with `ss` where *ss* represents seconds. | `UTC+7 -> +070000` + | `UTC-5 -> -050000` + `g` Era: AD or BC | `300 AD -> AD` + | `300 BC -> BC` + `fff` Milliseconds display | `1000000 nanoseconds -> 1` + `ffffff` Microseconds display | `1000000 nanoseconds -> 1000` + `fffffffff` Nanoseconds display | `1000000 nanoseconds -> 1000000` + =========== ================================================================================= ============================================== + + Other strings can be inserted by putting them in `''`. For example + `hh'->'mm` will give `01->56`. In addition to spaces, + the following characters can be inserted without quoting them: + `:` `-` `,` `.` `(` `)` `/` `[` `]`. + A literal `'` can be specified with `''`. + + However you don't need to necessarily separate format patterns, as an + unambiguous format string like `yyyyMMddhhmmss` is also valid (although + only for years in the range 1..9999). + + Duration vs TimeInterval + ============================ + The `times` module exports two similar types that are both used to + represent some amount of time: `Duration <#Duration>`_ and + `TimeInterval <#TimeInterval>`_. + This section explains how they differ and when one should be preferred over the + other (short answer: use `Duration` unless support for months and years is + needed). + + Duration + ---------------------------- + A `Duration` represents a duration of time stored as seconds and + nanoseconds. A `Duration` is always fully normalized, so + `initDuration(hours = 1)` and `initDuration(minutes = 60)` are equivalent. + + Arithmetic with a `Duration` is very fast, especially when used with the + `Time` type, since it only involves basic arithmetic. Because `Duration` + is more performant and easier to understand it should generally preferred. + + TimeInterval + ---------------------------- + A `TimeInterval` represents an amount of time expressed in calendar + units, for example "1 year and 2 days". Since some units cannot be + normalized (the length of a year is different for leap years for example), + the `TimeInterval` type uses separate fields for every unit. The + `TimeInterval`'s returned from this module generally don't normalize + **anything**, so even units that could be normalized (like seconds, + milliseconds and so on) are left untouched. + + Arithmetic with a `TimeInterval` can be very slow, because it requires + timezone information. + + Since it's slower and more complex, the `TimeInterval` type should be + avoided unless the program explicitly needs the features it offers that + `Duration` doesn't have. + + How long is a day? + ---------------------------- + It should be especially noted that the handling of days differs between + `TimeInterval` and `Duration`. The `Duration` type always treats a day + as exactly 86400 seconds. For `TimeInterval`, it's more complex. + + As an example, consider the amount of time between these two timestamps, both + in the same timezone: + + - 2018-03-25T12:00+02:00 + - 2018-03-26T12:00+01:00 + + If only the date & time is considered, it appears that exactly one day has + passed. However, the UTC offsets are different, which means that the + UTC offset was changed somewhere in between. This happens twice each year for + timezones that use daylight savings time. Because of this change, the amount + of time that has passed is actually 25 hours. + + The `TimeInterval` type uses calendar units, and will say that exactly one + day has passed. The `Duration` type on the other hand normalizes everything + to seconds, and will therefore say that 90000 seconds has passed, which is + the same as 25 hours. + + See also + ======== + * `monotimes module <monotimes.html>`_ +]## + +import std/[strutils, math, options] + +import std/private/since include "system/inclrtl" -type - Month* = enum ## represents a month - mJan, mFeb, mMar, mApr, mMay, mJun, mJul, mAug, mSep, mOct, mNov, mDec - WeekDay* = enum ## represents a weekday - dMon, dTue, dWed, dThu, dFri, dSat, dSun - -when defined(posix) and not defined(JS): - type - TimeImpl {.importc: "time_t", header: "<time.h>".} = int - Time* = distinct TimeImpl ## distinct type that represents a time - ## measured as number of seconds since the epoch - - Timeval {.importc: "struct timeval", - header: "<sys/select.h>".} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. - - # we cannot import posix.nim here, because posix.nim depends on times.nim. - # Ok, we could, but I don't want circular dependencies. - # And gettimeofday() is not defined in the posix module anyway. Sigh. - - proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. - importc: "gettimeofday", header: "<sys/time.h>".} - - var - timezone {.importc, header: "<time.h>".}: int - tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] - # we also need tzset() to make sure that tzname is initialized - proc tzset() {.importc, header: "<time.h>".} - # calling tzset() implicitly to initialize tzname data. - tzset() +when defined(nimPreviewSlimSystem): + import std/assertions + + +when defined(js): + import std/jscore + + # This is really bad, but overflow checks are broken badly for + # ints on the JS backend. See #6752. + {.push overflowChecks: off.} + proc `*`(a, b: int64): int64 = + system.`*`(a, b) + proc `*`(a, b: int): int = + system.`*`(a, b) + proc `+`(a, b: int64): int64 = + system.`+`(a, b) + proc `+`(a, b: int): int = + system.`+`(a, b) + proc `-`(a, b: int64): int64 = + system.`-`(a, b) + proc `-`(a, b: int): int = + system.`-`(a, b) + proc inc(a: var int, b: int) = + system.inc(a, b) + proc inc(a: var int64, b: int) = + system.inc(a, b) + {.pop.} + +elif defined(posix): + import std/posix + + type CTime = posix.Time + + when defined(macosx): + proc gettimeofday(tp: var Timeval, unused: pointer = nil) + {.importc: "gettimeofday", header: "<sys/time.h>", sideEffect.} elif defined(windows): - import winlean - - when defined(vcc): - # newest version of Visual C++ defines time_t to be of 64 bits - type TimeImpl {.importc: "time_t", header: "<time.h>".} = int64 - # visual c's c runtime exposes these under a different name - var - timezone {.importc: "_timezone", header: "<time.h>".}: int - tzname {.importc: "_tzname", header: "<time.h>"}: array[0..1, cstring] - else: - type TimeImpl {.importc: "time_t", header: "<time.h>".} = int32 - var - timezone {.importc, header: "<time.h>".}: int - tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] + import std/winlean, std/time_t type - Time* = distinct TimeImpl + CTime = time_t.Time + Tm {.importc: "struct tm", header: "<time.h>", final, pure.} = object + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + + proc localtime(a1: var CTime): ptr Tm {.importc, header: "<time.h>", sideEffect.} - -elif defined(JS): - type - Time* {.importc.} = object - getDay: proc (): int {.tags: [], raises: [], benign.} - getFullYear: proc (): int {.tags: [], raises: [], benign.} - getHours: proc (): int {.tags: [], raises: [], benign.} - getMilliseconds: proc (): int {.tags: [], raises: [], benign.} - getMinutes: proc (): int {.tags: [], raises: [], benign.} - getMonth: proc (): int {.tags: [], raises: [], benign.} - getSeconds: proc (): int {.tags: [], raises: [], benign.} - getTime: proc (): int {.tags: [], raises: [], benign.} - getTimezoneOffset: proc (): int {.tags: [], raises: [], benign.} - getDate: proc (): int {.tags: [], raises: [], benign.} - getUTCDate: proc (): int {.tags: [], raises: [], benign.} - getUTCFullYear: proc (): int {.tags: [], raises: [], benign.} - getUTCHours: proc (): int {.tags: [], raises: [], benign.} - getUTCMilliseconds: proc (): int {.tags: [], raises: [], benign.} - getUTCMinutes: proc (): int {.tags: [], raises: [], benign.} - getUTCMonth: proc (): int {.tags: [], raises: [], benign.} - getUTCSeconds: proc (): int {.tags: [], raises: [], benign.} - getUTCDay: proc (): int {.tags: [], raises: [], benign.} - getYear: proc (): int {.tags: [], raises: [], benign.} - parse: proc (s: cstring): Time {.tags: [], raises: [], benign.} - setDate: proc (x: int) {.tags: [], raises: [], benign.} - setFullYear: proc (x: int) {.tags: [], raises: [], benign.} - setHours: proc (x: int) {.tags: [], raises: [], benign.} - setMilliseconds: proc (x: int) {.tags: [], raises: [], benign.} - setMinutes: proc (x: int) {.tags: [], raises: [], benign.} - setMonth: proc (x: int) {.tags: [], raises: [], benign.} - setSeconds: proc (x: int) {.tags: [], raises: [], benign.} - setTime: proc (x: int) {.tags: [], raises: [], benign.} - setUTCDate: proc (x: int) {.tags: [], raises: [], benign.} - setUTCFullYear: proc (x: int) {.tags: [], raises: [], benign.} - setUTCHours: proc (x: int) {.tags: [], raises: [], benign.} - setUTCMilliseconds: proc (x: int) {.tags: [], raises: [], benign.} - setUTCMinutes: proc (x: int) {.tags: [], raises: [], benign.} - setUTCMonth: proc (x: int) {.tags: [], raises: [], benign.} - setUTCSeconds: proc (x: int) {.tags: [], raises: [], benign.} - setYear: proc (x: int) {.tags: [], raises: [], benign.} - toGMTString: proc (): cstring {.tags: [], raises: [], benign.} - toLocaleString: proc (): cstring {.tags: [], raises: [], benign.} +type + Month* = enum ## Represents a month. Note that the enum starts at `1`, + ## so `ord(month)` will give the month number in the + ## range `1..12`. + mJan = (1, "January") + mFeb = "February" + mMar = "March" + mApr = "April" + mMay = "May" + mJun = "June" + mJul = "July" + mAug = "August" + mSep = "September" + mOct = "October" + mNov = "November" + mDec = "December" + + WeekDay* = enum ## Represents a weekday. + dMon = "Monday" + dTue = "Tuesday" + dWed = "Wednesday" + dThu = "Thursday" + dFri = "Friday" + dSat = "Saturday" + dSun = "Sunday" type - TimeInfo* = object of RootObj ## represents a time in different parts - second*: range[0..61] ## The number of seconds after the minute, - ## normally in the range 0 to 59, but can - ## be up to 61 to allow for leap seconds. - minute*: range[0..59] ## The number of minutes after the hour, - ## in the range 0 to 59. - hour*: range[0..23] ## The number of hours past midnight, - ## in the range 0 to 23. - monthday*: range[1..31] ## The day of the month, in the range 1 to 31. - month*: Month ## The current month. - year*: int ## The current year. - weekday*: WeekDay ## The current day of the week. - yearday*: range[0..365] ## The number of days since January 1, - ## in the range 0 to 365. - ## Always 0 if the target is JS. - isDST*: bool ## Determines whether DST is in effect. Always - ## ``False`` if time is UTC. - tzname*: string ## The timezone this time is in. E.g. GMT - timezone*: int ## The offset of the (non-DST) timezone in seconds - ## west of UTC. - - ## I make some assumptions about the data in here. Either - ## everything should be positive or everything negative. Zero is - ## fine too. Mixed signs will lead to unexpected results. - TimeInterval* = object ## a time interval - milliseconds*: int ## The number of milliseconds - seconds*: int ## The number of seconds - minutes*: int ## The number of minutes - hours*: int ## The number of hours - days*: int ## The number of days - months*: int ## The number of months - years*: int ## The number of years - -{.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, - TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].} - -proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds - -proc `miliseconds=`*(t:var TimeInterval, milliseconds: int) {.deprecated.} = - t.milliseconds = milliseconds - -proc getTime*(): Time {.tags: [TimeEffect], benign.} - ## gets the current calendar time as a UNIX epoch value (number of seconds - ## elapsed since 1970) with integer precission. Use epochTime for higher - ## resolution. -proc getLocalTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} - ## converts the calendar time `t` to broken-time representation, - ## expressed relative to the user's specified time zone. -proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} - ## converts the calendar time `t` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). - -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} - ## converts a broken-down time structure to - ## calendar time representation. The function ignores the specified - ## contents of the structure members `weekday` and `yearday` and recomputes - ## them from the other information in the broken-down time structure. - -proc fromSeconds*(since1970: float): Time {.tags: [], raises: [], benign.} - ## Takes a float which contains the number of seconds since the unix epoch and - ## returns a time object. - -proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign.} = - ## Takes an int which contains the number of seconds since the unix epoch and - ## returns a time object. - fromSeconds(float(since1970)) - -proc toSeconds*(time: Time): float {.tags: [], raises: [], benign.} - ## Returns the time in seconds since the unix epoch. - -proc `$` *(timeInfo: TimeInfo): string {.tags: [], raises: [], benign.} - ## converts a `TimeInfo` object to a string representation. -proc `$` *(time: Time): string {.tags: [], raises: [], benign.} - ## converts a calendar time to a string representation. - -proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [], benign.} - ## computes the difference of two calendar times. Result is in seconds. - -proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [].} = - ## returns true iff ``a < b``, that is iff a happened before b. - result = a - b < 0 - -proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [].}= - ## returns true iff ``a <= b``. - result = a - b <= 0 - -proc `==`*(a, b: Time): bool {. - rtl, extern: "ntEqTime", tags: [], raises: [].} = - ## returns true if ``a == b``, that is if both times represent the same value - result = a - b == 0 - -when not defined(JS): - proc getTzname*(): tuple[nonDST, DST: string] {.tags: [TimeEffect], raises: [], - benign.} - ## returns the local timezone; ``nonDST`` is the name of the local non-DST - ## timezone, ``DST`` is the name of the local DST timezone. - -proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign.} - ## returns the offset of the local (non-DST) timezone in seconds west of UTC. - -proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} - ## get the milliseconds from the start of the program. **Deprecated since - ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. - -proc initInterval*(milliseconds, seconds, minutes, hours, days, months, - years: int = 0): TimeInterval = - ## creates a new ``TimeInterval``. - result.milliseconds = milliseconds - result.seconds = seconds - result.minutes = minutes - result.hours = hours - result.days = days - result.months = months - result.years = years + MonthdayRange* = range[1..31] + HourRange* = range[0..23] + MinuteRange* = range[0..59] + SecondRange* = range[0..60] ## \ + ## Includes the value 60 to allow for a leap second. Note however + ## that the `second` of a `DateTime` will never be a leap second. + YeardayRange* = range[0..365] + NanosecondRange* = range[0..999_999_999] + + IsoWeekRange* = range[1 .. 53] + ## An ISO 8601 calendar week number. + IsoYear* = distinct int + ## An ISO 8601 calendar year number. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + + Time* = object ## Represents a point in time. + seconds: int64 + nanosecond: NanosecondRange + + DateTime* = object of RootObj ## \ + ## Represents a time in different parts. Although this type can represent + ## leap seconds, they are generally not supported in this module. They are + ## not ignored, but the `DateTime`'s returned by procedures in this + ## module will never have a leap second. + nanosecond: NanosecondRange + second: SecondRange + minute: MinuteRange + hour: HourRange + monthdayZero: int + monthZero: int + year: int + weekday: WeekDay + yearday: YeardayRange + isDst: bool + timezone: Timezone + utcOffset: int + + Duration* = object ## Represents a fixed duration of time, meaning a duration + ## that has constant length independent of the context. + ## + ## To create a new `Duration`, use `initDuration + ## <#initDuration,int64,int64,int64,int64,int64,int64,int64,int64>`_. + ## Instead of trying to access the private attributes, use + ## `inSeconds <#inSeconds,Duration>`_ for converting to seconds and + ## `inNanoseconds <#inNanoseconds,Duration>`_ for converting to nanoseconds. + seconds: int64 + nanosecond: NanosecondRange + + TimeUnit* = enum ## Different units of time. + Nanoseconds, Microseconds, Milliseconds, Seconds, Minutes, Hours, Days, + Weeks, Months, Years + + FixedTimeUnit* = range[Nanoseconds..Weeks] ## \ + ## Subrange of `TimeUnit` that only includes units of fixed duration. + ## These are the units that can be represented by a `Duration`. + + TimeInterval* = object ## \ + ## Represents a non-fixed duration of time. Can be used to add and + ## subtract non-fixed time units from a `DateTime <#DateTime>`_ or + ## `Time <#Time>`_. + ## + ## Create a new `TimeInterval` with `initTimeInterval proc + ## <#initTimeInterval,int,int,int,int,int,int,int,int,int,int>`_. + ## + ## Note that `TimeInterval` doesn't represent a fixed duration of time, + ## since the duration of some units depend on the context (e.g a year + ## can be either 365 or 366 days long). The non-fixed time units are + ## years, months, days and week. + ## + ## Note that `TimeInterval`'s returned from the `times` module are + ## never normalized. If you want to normalize a time unit, + ## `Duration <#Duration>`_ should be used instead. + nanoseconds*: int ## The number of nanoseconds + microseconds*: int ## The number of microseconds + milliseconds*: int ## The number of milliseconds + seconds*: int ## The number of seconds + minutes*: int ## The number of minutes + hours*: int ## The number of hours + days*: int ## The number of days + weeks*: int ## The number of weeks + months*: int ## The number of months + years*: int ## The number of years + + Timezone* = ref object ## \ + ## Timezone interface for supporting `DateTime <#DateTime>`_\s of arbitrary + ## timezones. The `times` module only supplies implementations for the + ## system's local time and UTC. + zonedTimeFromTimeImpl: proc (x: Time): ZonedTime + {.tags: [], raises: [], benign.} + zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime + {.tags: [], raises: [], benign.} + name: string + + ZonedTime* = object ## Represents a point in time with an associated + ## UTC offset and DST flag. This type is only used for + ## implementing timezones. + time*: Time ## The point in time being represented. + utcOffset*: int ## The offset in seconds west of UTC, + ## including any offset due to DST. + isDst*: bool ## Determines whether DST is in effect. + + DurationParts* = array[FixedTimeUnit, int64] # Array of Duration parts starts + TimeIntervalParts* = array[TimeUnit, int] # Array of Duration parts starts -proc isLeapYear*(year: int): bool = - ## returns true if ``year`` is a leap year +const + secondsInMin = 60 + secondsInHour = 60*60 + secondsInDay = 60*60*24 + rateDiff = 10000000'i64 # 100 nsecs + # The number of hectonanoseconds between 1601/01/01 (windows epoch) + # and 1970/01/01 (unix epoch). + epochDiff = 116444736000000000'i64 + +const unitWeights: array[FixedTimeUnit, int64] = [ + 1'i64, + 1000, + 1_000_000, + 1e9.int64, + secondsInMin * 1e9.int64, + secondsInHour * 1e9.int64, + secondsInDay * 1e9.int64, + 7 * secondsInDay * 1e9.int64, +] + +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} - if year mod 400 == 0: - return true - elif year mod 100 == 0: - return false - elif year mod 4 == 0: - return true + +# +# Helper procs +# + +{.pragma: operator, rtl, noSideEffect, benign.} + +proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T + {.inline.} = + ## Convert a quantity of some duration unit to another duration unit. + ## This proc only deals with integers, so the result might be truncated. + runnableExamples: + doAssert convert(Days, Hours, 2) == 48 + doAssert convert(Days, Weeks, 13) == 1 # Truncated + doAssert convert(Seconds, Milliseconds, -1) == -1000 + if unitFrom < unitTo: + (quantity div (unitWeights[unitTo] div unitWeights[unitFrom])).T else: - return false + ((unitWeights[unitFrom] div unitWeights[unitTo]) * quantity).T + +proc normalize[T: Duration|Time](seconds, nanoseconds: int64): T = + ## Normalize a (seconds, nanoseconds) pair and return it as either + ## a `Duration` or `Time`. A normalized `Duration|Time` has a + ## positive nanosecond part in the range `NanosecondRange`. + result.seconds = seconds + convert(Nanoseconds, Seconds, nanoseconds) + var nanosecond = nanoseconds mod convert(Seconds, Nanoseconds, 1) + if nanosecond < 0: + nanosecond += convert(Seconds, Nanoseconds, 1) + result.seconds -= 1 + result.nanosecond = nanosecond.int -proc getDaysInMonth*(month: Month, year: int): int = - ## Get the number of days in a ``month`` of a ``year`` +proc isLeapYear*(year: int): bool = + ## Returns true if `year` is a leap year. + runnableExamples: + doAssert isLeapYear(2000) + doAssert not isLeapYear(1900) + year mod 4 == 0 and (year mod 100 != 0 or year mod 400 == 0) +proc getDaysInMonth*(month: Month, year: int): int = + ## Get the number of days in `month` of `year`. # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month + runnableExamples: + doAssert getDaysInMonth(mFeb, 2000) == 29 + doAssert getDaysInMonth(mFeb, 2001) == 28 case month of mFeb: result = if isLeapYear(year): 29 else: 28 of mApr, mJun, mSep, mNov: result = 30 else: result = 31 +proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) + {.inline.} = + assert monthday <= getDaysInMonth(month, year), + $year & "-" & intToStr(ord(month), 2) & "-" & $monthday & + " is not a valid date" + +proc toEpochDay(monthday: MonthdayRange, month: Month, year: int): int64 = + ## Get the epoch day from a year/month/day date. + ## The epoch day is the number of days since 1970/01/01 + ## (it might be negative). + # Based on http://howardhinnant.github.io/date_algorithms.html + assertValidDate monthday, month, year + var (y, m, d) = (year, ord(month), monthday.int) + if m <= 2: + y.dec + + let era = (if y >= 0: y else: y-399) div 400 + let yoe = y - era * 400 + let doy = (153 * (m + (if m > 2: -3 else: 9)) + 2) div 5 + d-1 + let doe = yoe * 365 + yoe div 4 - yoe div 100 + doy + return era * 146097 + doe - 719468 + +proc fromEpochDay(epochday: int64): + tuple[monthday: MonthdayRange, month: Month, year: int] = + ## Get the year/month/day date from a epoch day. + ## The epoch day is the number of days since 1970/01/01 + ## (it might be negative). + # Based on http://howardhinnant.github.io/date_algorithms.html + var z = epochday + z.inc 719468 + let era = (if z >= 0: z else: z - 146096) div 146097 + let doe = z - era * 146097 + let yoe = (doe - doe div 1460 + doe div 36524 - doe div 146096) div 365 + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe div 4 - yoe div 100) + let mp = (5 * doy + 2) div 153 + let d = doy - (153 * mp + 2) div 5 + 1 + let m = mp + (if mp < 10: 3 else: -9) + return (d.MonthdayRange, m.Month, (y + ord(m <= 2)).int) + +proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): + YeardayRange {.tags: [], raises: [], benign.} = + ## Returns the day of the year. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).yearday`. + runnableExamples: + doAssert getDayOfYear(1, mJan, 2000) == 0 + doAssert getDayOfYear(10, mJan, 2000) == 9 + doAssert getDayOfYear(10, mFeb, 2000) == 40 + + assertValidDate monthday, month, year + const daysUntilMonth: array[Month, int] = + [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334] + const daysUntilMonthLeap: array[Month, int] = + [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335] + + if isLeapYear(year): + result = daysUntilMonthLeap[month] + monthday - 1 + else: + result = daysUntilMonth[month] + monthday - 1 + +proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay + {.tags: [], raises: [], benign.} = + ## Returns the day of the week enum from day, month and year. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).weekday`. + runnableExamples: + doAssert getDayOfWeek(13, mJun, 1990) == dWed + doAssert $getDayOfWeek(13, mJun, 1990) == "Wednesday" + + assertValidDate monthday, month, year + # 1970-01-01 is a Thursday, we adjust to the previous Monday + let days = toEpochDay(monthday, month, year) - 3 + let weeks = floorDiv(days, 7'i64) + let wd = days - weeks * 7 + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. + result = if wd == 0: dSun else: WeekDay(wd - 1) + proc getDaysInYear*(year: int): int = - ## Get the number of days in a ``year`` + ## Get the number of days in a `year` + runnableExamples: + doAssert getDaysInYear(2000) == 366 + doAssert getDaysInYear(2001) == 365 result = 365 + (if isLeapYear(year): 1 else: 0) -proc toSeconds(a: TimeInfo, interval: TimeInterval): float = - ## Calculates how many seconds the interval is worth by adding up - ## all the fields - - var anew = a - var newinterv = interval - result = 0 - - newinterv.months += interval.years * 12 - var curMonth = anew.month - for mth in 1 .. newinterv.months: - result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) - if curMonth == mDec: - curMonth = mJan - anew.year.inc() - else: - curMonth.inc() - result += float(newinterv.days * 24 * 60 * 60) - result += float(newinterv.hours * 60 * 60) - result += float(newinterv.minutes * 60) - result += float(newinterv.seconds) - result += newinterv.milliseconds / 1000 - -proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## adds ``interval`` time. - ## - ## **Note:** This has been only briefly tested and it may not be - ## very accurate. - let t = toSeconds(timeInfoToTime(a)) - let secs = toSeconds(a, interval) - #if a.tzname == "UTC": - # result = getGMTime(fromSeconds(t + secs)) - #else: - result = getLocalTime(fromSeconds(t + secs)) - -proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## subtracts ``interval`` time. - ## - ## **Note:** This has been only briefly tested, it is inaccurate especially - ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(timeInfoToTime(a)) - let secs = toSeconds(a, interval) - #if a.tzname == "UTC": - # result = getGMTime(fromSeconds(t - secs)) - #else: - result = getLocalTime(fromSeconds(t - secs)) - -when not defined(JS): - proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time after the UNIX epoch (1970) in seconds. It is a float - ## because sub-second resolution is likely to be supported (depending - ## on the hardware/OS). - - proc cpuTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} - ## gets time spent that the CPU spent to run the current process in - ## seconds. This may be more useful for benchmarking than ``epochTime``. - ## However, it may measure the real time instead (depending on the OS). - ## The value of the result has no meaning. - ## To generate useful timing values, take the difference between - ## the results of two ``cpuTime`` calls: - ## - ## .. code-block:: nim - ## var t0 = cpuTime() - ## doWork() - ## echo "CPU time [s] ", cpuTime() - t0 +proc `==`*(a, b: IsoYear): bool {.borrow.} +proc `$`*(p: IsoYear): string {.borrow.} -when not defined(JS): - # C wrapper: - type - StructTM {.importc: "struct tm", final.} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - - TimeInfoPtr = ptr StructTM - Clock {.importc: "clock_t".} = distinct int +proc getWeeksInIsoYear*(y: IsoYear): IsoWeekRange {.since: (1, 5).} = + ## Returns the number of weeks in the specified ISO 8601 week-based year, which can be + ## either 53 or 52. + runnableExamples: + assert getWeeksInIsoYear(IsoYear(2019)) == 52 + assert getWeeksInIsoYear(IsoYear(2020)) == 53 - proc localtime(timer: ptr Time): TimeInfoPtr {. - importc: "localtime", header: "<time.h>", tags: [].} - proc gmtime(timer: ptr Time): TimeInfoPtr {. - importc: "gmtime", header: "<time.h>", tags: [].} - proc timec(timer: ptr Time): Time {. - importc: "time", header: "<time.h>", tags: [].} - proc mktime(t: StructTM): Time {. - importc: "mktime", header: "<time.h>", tags: [].} - proc asctime(tblock: StructTM): cstring {. - importc: "asctime", header: "<time.h>", tags: [].} - proc ctime(time: ptr Time): cstring {. - importc: "ctime", header: "<time.h>", tags: [].} - # strftime(s: CString, maxsize: int, fmt: CString, t: tm): int {. - # importc: "strftime", header: "<time.h>".} - proc getClock(): Clock {.importc: "clock", header: "<time.h>", tags: [TimeEffect].} - proc difftime(a, b: Time): float {.importc: "difftime", header: "<time.h>", - tags: [].} + var y = int(y) - var - clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl.}: int - - # our own procs on top of that: - proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = - const - weekDays: array [0..6, WeekDay] = [ - dSun, dMon, dTue, dWed, dThu, dFri, dSat] - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - tzname: if local: - if tm.isdst > 0: - getTzname().DST - else: - getTzname().nonDST - else: - "UTC", - timezone: if local: getTimezone() else: 0 - ) + # support negative years + y = if y < 0: 400 + y mod 400 else: y - proc timeInfoToTM(t: TimeInfo): StructTM = - const - weekDays: array [WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] - result.second = t.second - result.minute = t.minute - result.hour = t.hour - result.monthday = t.monthday - result.month = ord(t.month) - result.year = cint(t.year - 1900) - result.weekday = weekDays[t.weekday] - result.yearday = t.yearday - result.isdst = if t.isDST: 1 else: 0 - - when not defined(useNimRtl): - proc `-` (a, b: Time): int64 = - return toBiggestInt(difftime(a, b)) - - proc getStartMilsecs(): int = - #echo "clocks per sec: ", clocksPerSec, "clock: ", int(getClock()) - #return getClock() div (clocksPerSec div 1000) - when defined(macosx): - result = toInt(toFloat(int(getClock())) / (toFloat(clocksPerSec) / 1000.0)) - else: - result = int(getClock()) div (clocksPerSec div 1000) - when false: - var a: Timeval - posix_gettimeofday(a) - result = a.tv_sec * 1000'i64 + a.tv_usec div 1000'i64 - #echo "result: ", result - - proc getTime(): Time = return timec(nil) - proc getLocalTime(t: Time): TimeInfo = - var a = t - let lt = localtime(addr(a)) - assert(not lt.isNil) - result = tmToTimeInfo(lt[], true) - # copying is needed anyway to provide reentrancity; thus - # the conversion is not expensive - - proc getGMTime(t: Time): TimeInfo = - var a = t - result = tmToTimeInfo(gmtime(addr(a))[], false) - # copying is needed anyway to provide reentrancity; thus - # the conversion is not expensive - - proc timeInfoToTime(timeInfo: TimeInfo): Time = - var cTimeInfo = timeInfo # for C++ we have to make a copy, - # because the header of mktime is broken in my version of libc - return mktime(timeInfoToTM(cTimeInfo)) - - proc toStringTillNL(p: cstring): string = - result = "" - var i = 0 - while p[i] != '\0' and p[i] != '\10' and p[i] != '\13': - add(result, p[i]) - inc(i) + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let p = (y + (y div 4) - (y div 100) + (y div 400)) mod 7 + let y1 = y - 1 + let p1 = (y1 + (y1 div 4) - (y1 div 100) + (y1 div 400)) mod 7 + if p == 4 or p1 == 3: 53 else: 52 - proc `$`(timeInfo: TimeInfo): string = - # BUGFIX: asctime returns a newline at the end! - var p = asctime(timeInfoToTM(timeInfo)) - result = toStringTillNL(p) - - proc `$`(time: Time): string = - # BUGFIX: ctime returns a newline at the end! - var a = time - return toStringTillNL(ctime(addr(a))) - - const - epochDiff = 116444736000000000'i64 - rateDiff = 10000000'i64 # 100 nsecs - - proc unixTimeToWinTime*(t: Time): int64 = - ## converts a UNIX `Time` (``time_t``) to a Windows file time - result = int64(t) * rateDiff + epochDiff - - proc winTimeToUnixTime*(t: int64): Time = - ## converts a Windows time to a UNIX `Time` (``time_t``) - result = Time((t - epochDiff) div rateDiff) - - proc getTzname(): tuple[nonDST, DST: string] = - return ($tzname[0], $tzname[1]) - - proc getTimezone(): int = - return timezone - - proc fromSeconds(since1970: float): Time = Time(since1970) - - proc toSeconds(time: Time): float = float(time) - - when not defined(useNimRtl): - proc epochTime(): float = - when defined(posix): - var a: Timeval - posix_gettimeofday(a) - result = toFloat(a.tv_sec) + toFloat(a.tv_usec)*0.00_0001 - elif defined(windows): - var f: winlean.FILETIME - getSystemTimeAsFileTime(f) - var i64 = rdFileTime(f) - epochDiff - var secs = i64 div rateDiff - var subsecs = i64 mod rateDiff - result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 - else: - {.error: "unknown OS".} +proc getIsoWeekAndYear*(dt: DateTime): + tuple[isoweek: IsoWeekRange, isoyear: IsoYear] {.since: (1, 5).} = + ## Returns the ISO 8601 week and year. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert getIsoWeekAndYear(initDateTime(21, mApr, 2018, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2018.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) + assert w == 01.IsoWeekRange + assert y == 2020.IsoYear + assert getIsoWeekAndYear(initDateTime(13, mSep, 2020, 00, 00, 00)) == (isoweek: 37.IsoWeekRange, isoyear: 2020.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(2, mJan, 2021, 00, 00, 00)) + assert w.int > 52 + assert w.int < 54 + assert y.int mod 100 == 20 + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + var w = (dt.yearday.int - dt.weekday.int + 10) div 7 + if w < 1: + (isoweek: getWeeksInIsoYear(IsoYear(dt.year - 1)), isoyear: IsoYear(dt.year - 1)) + elif (w > getWeeksInIsoYear(IsoYear(dt.year))): + (isoweek: IsoWeekRange(1), isoyear: IsoYear(dt.year + 1)) + else: + (isoweek: IsoWeekRange(w), isoyear: IsoYear(dt.year)) - proc cpuTime(): float = - result = toFloat(int(getClock())) / toFloat(clocksPerSec) +proc stringifyUnit(value: int | int64, unit: TimeUnit): string = + ## Stringify time unit with it's name, lowercased + let strUnit = $unit + result = "" + result.addInt value + result.add ' ' + if abs(value) != 1: + result.add(strUnit.toLowerAscii()) + else: + result.add(strUnit[0..^2].toLowerAscii()) -elif defined(JS): - proc newDate(): Time {.importc: "new Date".} - proc internGetTime(): Time {.importc: "new Date", tags: [].} - - proc newDate(value: float): Time {.importc: "new Date".} - proc newDate(value: string): Time {.importc: "new Date".} - proc getTime(): Time = - # Warning: This is something different in JS. - return newDate() - - const - weekDays: array [0..6, WeekDay] = [ - dSun, dMon, dTue, dWed, dThu, dFri, dSat] - - proc getLocalTime(t: Time): TimeInfo = - result.second = t.getSeconds() - result.minute = t.getMinutes() - result.hour = t.getHours() - result.monthday = t.getDate() - result.month = Month(t.getMonth()) - result.year = t.getFullYear() - result.weekday = weekDays[t.getDay()] - result.yearday = 0 - - proc getGMTime(t: Time): TimeInfo = - result.second = t.getUTCSeconds() - result.minute = t.getUTCMinutes() - result.hour = t.getUTCHours() - result.monthday = t.getUTCDate() - result.month = Month(t.getUTCMonth()) - result.year = t.getUTCFullYear() - result.weekday = weekDays[t.getUTCDay()] - result.yearday = 0 - - proc timeInfoToTime*(timeInfo: TimeInfo): Time = - result = internGetTime() - result.setSeconds(timeInfo.second) - result.setMinutes(timeInfo.minute) - result.setHours(timeInfo.hour) - result.setMonth(ord(timeInfo.month)) - result.setFullYear(timeInfo.year) - result.setDate(timeInfo.monthday) - - proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) - proc `$`(time: Time): string = return $time.toLocaleString() - - proc `-` (a, b: Time): int64 = - return a.getTime() - b.getTime() +proc humanizeParts(parts: seq[string]): string = + ## Make date string parts human-readable + result = "" + if parts.len == 0: + result.add "0 nanoseconds" + elif parts.len == 1: + result = parts[0] + elif parts.len == 2: + result = parts[0] & " and " & parts[1] + else: + for i in 0..high(parts)-1: + result.add parts[i] & ", " + result.add "and " & parts[high(parts)] - var - startMilsecs = getTime() +template subImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = + normalize[T](a.seconds - b.seconds, a.nanosecond - b.nanosecond) - proc getStartMilsecs(): int = - ## get the milliseconds from the start of the program - return int(getTime() - startMilsecs) +template addImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = + normalize[T](a.seconds + b.seconds, a.nanosecond + b.nanosecond) - proc fromSeconds(since1970: float): Time = result = newDate(since1970 * 1000) +template ltImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds < b.seconds or ( + a.seconds == b.seconds and a.nanosecond < b.nanosecond) - proc toSeconds(time: Time): float = result = time.getTime() / 1000 +template lqImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds < b.seconds or ( + a.seconds == b.seconds and a.nanosecond <= b.nanosecond) - proc getTimezone(): int = result = newDate().getTimezoneOffset() +template eqImpl(a: Duration|Time, b: Duration|Time): bool = + a.seconds == b.seconds and a.nanosecond == b.nanosecond - proc epochTime*(): float {.tags: [TimeEffect].} = newDate().toSeconds() +# +# Duration +# -proc getDateStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## gets the current date as a string of the format ``YYYY-MM-DD``. - var ti = getLocalTime(getTime()) - result = $ti.year & '-' & intToStr(ord(ti.month)+1, 2) & - '-' & intToStr(ti.monthday, 2) +const DurationZero* = Duration() ## \ + ## Zero value for durations. Useful for comparisons. + ## ```nim + ## doAssert initDuration(seconds = 1) > DurationZero + ## doAssert initDuration(seconds = 0) == DurationZero + ## ``` + +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration = + ## Create a new `Duration <#Duration>`_. + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.inMilliseconds == 1001 + doAssert dur.inSeconds == 1 + + let seconds = convert(Weeks, Seconds, weeks) + + convert(Days, Seconds, days) + + convert(Minutes, Seconds, minutes) + + convert(Hours, Seconds, hours) + + convert(Seconds, Seconds, seconds) + + convert(Milliseconds, Seconds, milliseconds) + + convert(Microseconds, Seconds, microseconds) + + convert(Nanoseconds, Seconds, nanoseconds) + let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + + convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + + nanoseconds mod 1_000_000_000).int + # Nanoseconds might be negative so we must normalize. + result = normalize[Duration](seconds, nanoseconds) + +template convert(dur: Duration, unit: static[FixedTimeUnit]): int64 = + # The correction is required due to how durations are normalized. + # For example,` initDuration(nanoseconds = -1)` is stored as + # { seconds = -1, nanoseconds = 999999999 }. + when unit == Nanoseconds: + dur.seconds * 1_000_000_000 + dur.nanosecond + else: + let correction = dur.seconds < 0 and dur.nanosecond > 0 + when unit >= Seconds: + convert(Seconds, unit, dur.seconds + ord(correction)) + else: + if correction: + convert(Seconds, unit, dur.seconds + 1) - + convert(Nanoseconds, unit, + convert(Seconds, Nanoseconds, 1) - dur.nanosecond) + else: + convert(Seconds, unit, dur.seconds) + + convert(Nanoseconds, unit, dur.nanosecond) + +proc inWeeks*(dur: Duration): int64 = + ## Converts the duration to the number of whole weeks. + runnableExamples: + let dur = initDuration(days = 8) + doAssert dur.inWeeks == 1 + dur.convert(Weeks) + +proc inDays*(dur: Duration): int64 = + ## Converts the duration to the number of whole days. + runnableExamples: + let dur = initDuration(hours = -50) + doAssert dur.inDays == -2 + dur.convert(Days) + +proc inHours*(dur: Duration): int64 = + ## Converts the duration to the number of whole hours. + runnableExamples: + let dur = initDuration(minutes = 60, days = 2) + doAssert dur.inHours == 49 + dur.convert(Hours) + +proc inMinutes*(dur: Duration): int64 = + ## Converts the duration to the number of whole minutes. + runnableExamples: + let dur = initDuration(hours = 2, seconds = 10) + doAssert dur.inMinutes == 120 + dur.convert(Minutes) + +proc inSeconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole seconds. + runnableExamples: + let dur = initDuration(hours = 2, milliseconds = 10) + doAssert dur.inSeconds == 2 * 60 * 60 + dur.convert(Seconds) + +proc inMilliseconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole milliseconds. + runnableExamples: + let dur = initDuration(seconds = -2) + doAssert dur.inMilliseconds == -2000 + dur.convert(Milliseconds) + +proc inMicroseconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole microseconds. + runnableExamples: + let dur = initDuration(seconds = -2) + doAssert dur.inMicroseconds == -2000000 + dur.convert(Microseconds) + +proc inNanoseconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole nanoseconds. + runnableExamples: + let dur = initDuration(seconds = -2) + doAssert dur.inNanoseconds == -2000000000 + dur.convert(Nanoseconds) + +proc toParts*(dur: Duration): DurationParts = + ## Converts a duration into an array consisting of fixed time units. + ## + ## Each value in the array gives information about a specific unit of + ## time, for example `result[Days]` gives a count of days. + ## + ## This procedure is useful for converting `Duration` values to strings. + runnableExamples: + var dp = toParts(initDuration(weeks = 2, days = 1)) + doAssert dp[Days] == 1 + doAssert dp[Weeks] == 2 + doAssert dp[Minutes] == 0 + dp = toParts(initDuration(days = -1)) + doAssert dp[Days] == -1 + + var remS = dur.seconds + var remNs = dur.nanosecond.int + + # Ensure the same sign for seconds and nanoseconds + if remS < 0 and remNs != 0: + remNs -= convert(Seconds, Nanoseconds, 1) + remS.inc 1 + + for unit in countdown(Weeks, Seconds): + let quantity = convert(Seconds, unit, remS) + remS = remS mod convert(unit, Seconds, 1) + + result[unit] = quantity + + for unit in countdown(Milliseconds, Nanoseconds): + let quantity = convert(Nanoseconds, unit, remNs) + remNs = remNs mod convert(unit, Nanoseconds, 1) + + result[unit] = quantity + +proc `$`*(dur: Duration): string = + ## Human friendly string representation of a `Duration`. + runnableExamples: + doAssert $initDuration(seconds = 2) == "2 seconds" + doAssert $initDuration(weeks = 1, days = 2) == "1 week and 2 days" + doAssert $initDuration(hours = 1, minutes = 2, seconds = 3) == + "1 hour, 2 minutes, and 3 seconds" + doAssert $initDuration(milliseconds = -1500) == + "-1 second and -500 milliseconds" + var parts = newSeq[string]() + var numParts = toParts(dur) + + for unit in countdown(Weeks, Nanoseconds): + let quantity = numParts[unit] + if quantity != 0.int64: + parts.add(stringifyUnit(quantity, unit)) + + result = humanizeParts(parts) + +proc `+`*(a, b: Duration): Duration {.operator, extern: "ntAddDuration".} = + ## Add two durations together. + runnableExamples: + doAssert initDuration(seconds = 1) + initDuration(days = 1) == + initDuration(seconds = 1, days = 1) + addImpl[Duration](a, b) + +proc `-`*(a, b: Duration): Duration {.operator, extern: "ntSubDuration".} = + ## Subtract a duration from another. + runnableExamples: + doAssert initDuration(seconds = 1, days = 1) - initDuration(seconds = 1) == + initDuration(days = 1) + subImpl[Duration](a, b) + +proc `-`*(a: Duration): Duration {.operator, extern: "ntReverseDuration".} = + ## Reverse a duration. + runnableExamples: + doAssert -initDuration(seconds = 1) == initDuration(seconds = -1) + normalize[Duration](-a.seconds, -a.nanosecond) + +proc `<`*(a, b: Duration): bool {.operator, extern: "ntLtDuration".} = + ## Note that a duration can be negative, + ## so even if `a < b` is true `a` might + ## represent a larger absolute duration. + ## Use `abs(a) < abs(b)` to compare the absolute + ## duration. + runnableExamples: + doAssert initDuration(seconds = 1) < initDuration(seconds = 2) + doAssert initDuration(seconds = -2) < initDuration(seconds = 1) + doAssert initDuration(seconds = -2).abs < initDuration(seconds = 1).abs == false + ltImpl(a, b) + +proc `<=`*(a, b: Duration): bool {.operator, extern: "ntLeDuration".} = + lqImpl(a, b) + +proc `==`*(a, b: Duration): bool {.operator, extern: "ntEqDuration".} = + runnableExamples: + let + d1 = initDuration(weeks = 1) + d2 = initDuration(days = 7) + doAssert d1 == d2 + eqImpl(a, b) + +proc `*`*(a: int64, b: Duration): Duration {.operator, + extern: "ntMulInt64Duration".} = + ## Multiply a duration by some scalar. + runnableExamples: + doAssert 5 * initDuration(seconds = 1) == initDuration(seconds = 5) + doAssert 3 * initDuration(minutes = 45) == initDuration(hours = 2, minutes = 15) + normalize[Duration](a * b.seconds, a * b.nanosecond) + +proc `*`*(a: Duration, b: int64): Duration {.operator, + extern: "ntMulDuration".} = + ## Multiply a duration by some scalar. + runnableExamples: + doAssert initDuration(seconds = 1) * 5 == initDuration(seconds = 5) + doAssert initDuration(minutes = 45) * 3 == initDuration(hours = 2, minutes = 15) + b * a + +proc `+=`*(d1: var Duration, d2: Duration) = + d1 = d1 + d2 + +proc `-=`*(dt: var Duration, ti: Duration) = + dt = dt - ti + +proc `*=`*(a: var Duration, b: int) = + a = a * b + +proc `div`*(a: Duration, b: int64): Duration {.operator, + extern: "ntDivDuration".} = + ## Integer division for durations. + runnableExamples: + doAssert initDuration(seconds = 3) div 2 == + initDuration(milliseconds = 1500) + doAssert initDuration(minutes = 45) div 30 == + initDuration(minutes = 1, seconds = 30) + doAssert initDuration(nanoseconds = 3) div 2 == + initDuration(nanoseconds = 1) + let carryOver = convert(Seconds, Nanoseconds, a.seconds mod b) + normalize[Duration](a.seconds div b, (a.nanosecond + carryOver) div b) + +proc high*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration. + initDuration(seconds = high(int64), nanoseconds = high(NanosecondRange)) + +proc low*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration of negative direction. + initDuration(seconds = low(int64)) + +proc abs*(a: Duration): Duration = + runnableExamples: + doAssert initDuration(milliseconds = -1500).abs == + initDuration(milliseconds = 1500) + initDuration(seconds = abs(a.seconds), nanoseconds = -a.nanosecond) -proc getClockStr*(): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## gets the current clock time as a string of the format ``HH:MM:SS``. - var ti = getLocalTime(getTime()) - result = intToStr(ti.hour, 2) & ':' & intToStr(ti.minute, 2) & - ':' & intToStr(ti.second, 2) +# +# Time +# -proc `$`*(day: WeekDay): string = - ## stingify operator for ``WeekDay``. - const lookup: array[WeekDay, string] = ["Monday", "Tuesday", "Wednesday", - "Thursday", "Friday", "Saturday", "Sunday"] - return lookup[day] +proc initTime*(unix: int64, nanosecond: NanosecondRange): Time = + ## Create a `Time <#Time>`_ from a unix timestamp and a nanosecond part. + result.seconds = unix + result.nanosecond = nanosecond + +proc nanosecond*(time: Time): NanosecondRange = + ## Get the fractional part of a `Time` as the number + ## of nanoseconds of the second. + time.nanosecond + +proc fromUnix*(unix: int64): Time + {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp (seconds since `1970-01-01T00:00:00Z`) + ## to a `Time`. + runnableExamples: + doAssert $fromUnix(0).utc == "1970-01-01T00:00:00Z" + initTime(unix, 0) + +proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert `t` to a unix timestamp (seconds since `1970-01-01T00:00:00Z`). + ## See also `toUnixFloat` for subsecond resolution. + runnableExamples: + doAssert fromUnix(0).toUnix() == 0 + t.seconds + +proc fromUnixFloat(seconds: float): Time {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp in seconds to a `Time`; same as `fromUnix` + ## but with subsecond resolution. + runnableExamples: + doAssert fromUnixFloat(123456.0) == fromUnixFloat(123456) + doAssert fromUnixFloat(-123456.0) == fromUnixFloat(-123456) + let secs = seconds.floor + let nsecs = (seconds - secs) * 1e9 + initTime(secs.int64, nsecs.NanosecondRange) + +proc toUnixFloat(t: Time): float {.benign, tags: [], raises: [].} = + ## Same as `toUnix` but using subsecond resolution. + runnableExamples: + let t = getTime() + # `<` because of rounding errors + doAssert abs(t.toUnixFloat().fromUnixFloat - t) < initDuration(nanoseconds = 1000) + t.seconds.float + t.nanosecond / convert(Seconds, Nanoseconds, 1) + +since((1, 1)): + export fromUnixFloat + export toUnixFloat + + +proc fromWinTime*(win: int64): Time = + ## Convert a Windows file time (100-nanosecond intervals since + ## `1601-01-01T00:00:00Z`) to a `Time`. + const hnsecsPerSec = convert(Seconds, Nanoseconds, 1) div 100 + let nanos = floorMod(win, hnsecsPerSec) * 100 + let seconds = floorDiv(win - epochDiff, hnsecsPerSec) + result = initTime(seconds, nanos) + +proc toWinTime*(t: Time): int64 = + ## Convert `t` to a Windows file time (100-nanosecond intervals + ## since `1601-01-01T00:00:00Z`). + result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 + +proc getTimeImpl(typ: typedesc[Time]): Time = + discard "implemented in the vm" + +proc getTime*(): Time {.tags: [TimeEffect], benign.} = + ## Gets the current time as a `Time` with up to nanosecond resolution. + when nimvm: + result = getTimeImpl(Time) + else: + when defined(js): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, + convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f {.noinit.}: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) + +proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = + ## Computes the duration between two points in time. + runnableExamples: + doAssert initTime(1000, 100) - initTime(500, 20) == + initDuration(minutes = 8, seconds = 20, nanoseconds = 80) + subImpl[Duration](a, b) + +proc `+`*(a: Time, b: Duration): Time {.operator, extern: "ntAddTime".} = + ## Add a duration of time to a `Time`. + runnableExamples: + doAssert (fromUnix(0) + initDuration(seconds = 1)) == fromUnix(1) + addImpl[Time](a, b) + +proc `-`*(a: Time, b: Duration): Time {.operator, extern: "ntSubTime".} = + ## Subtracts a duration of time from a `Time`. + runnableExamples: + doAssert (fromUnix(0) - initDuration(seconds = 1)) == fromUnix(-1) + subImpl[Time](a, b) + +proc `<`*(a, b: Time): bool {.operator, extern: "ntLtTime".} = + ## Returns true if `a < b`, that is if `a` happened before `b`. + runnableExamples: + doAssert initTime(50, 0) < initTime(99, 0) + ltImpl(a, b) + +proc `<=`*(a, b: Time): bool {.operator, extern: "ntLeTime".} = + ## Returns true if `a <= b`. + lqImpl(a, b) + +proc `==`*(a, b: Time): bool {.operator, extern: "ntEqTime".} = + ## Returns true if `a == b`, that is if both times represent the same point in time. + eqImpl(a, b) + +proc `+=`*(t: var Time, b: Duration) = + t = t + b + +proc `-=`*(t: var Time, b: Duration) = + t = t - b + +proc high*(typ: typedesc[Time]): Time = + initTime(high(int64), high(NanosecondRange)) + +proc low*(typ: typedesc[Time]): Time = + initTime(0, 0) -proc `$`*(m: Month): string = - ## stingify operator for ``Month``. - const lookup: array[Month, string] = ["January", "February", "March", - "April", "May", "June", "July", "August", "September", "October", - "November", "December"] - return lookup[m] +# +# DateTime & Timezone +# -proc formatToken(info: TimeInfo, token: string, buf: var string) = - ## Helper of the format proc to parse individual tokens. +template assertDateTimeInitialized(dt: DateTime) = + assert dt.monthdayZero != 0, "Uninitialized datetime" + +proc nanosecond*(dt: DateTime): NanosecondRange {.inline.} = + ## The number of nanoseconds after the second, + ## in the range 0 to 999_999_999. + assertDateTimeInitialized(dt) + dt.nanosecond + +proc second*(dt: DateTime): SecondRange {.inline.} = + ## The number of seconds after the minute, + ## in the range 0 to 59. + assertDateTimeInitialized(dt) + dt.second + +proc minute*(dt: DateTime): MinuteRange {.inline.} = + ## The number of minutes after the hour, + ## in the range 0 to 59. + assertDateTimeInitialized(dt) + dt.minute + +proc hour*(dt: DateTime): HourRange {.inline.} = + ## The number of hours past midnight, + ## in the range 0 to 23. + assertDateTimeInitialized(dt) + dt.hour + +proc monthday*(dt: DateTime): MonthdayRange {.inline.} = + ## The day of the month, in the range 1 to 31. + assertDateTimeInitialized(dt) + # 'cast' to avoid extra range check + cast[MonthdayRange](dt.monthdayZero) + +proc month*(dt: DateTime): Month = + ## The month as an enum, the ordinal value + ## is in the range 1 to 12. + assertDateTimeInitialized(dt) + # 'cast' to avoid extra range check + cast[Month](dt.monthZero) + +proc year*(dt: DateTime): int {.inline.} = + ## The year, using astronomical year numbering + ## (meaning that before year 1 is year 0, + ## then year -1 and so on). + assertDateTimeInitialized(dt) + dt.year + +proc weekday*(dt: DateTime): WeekDay {.inline.} = + ## The day of the week as an enum, the ordinal + ## value is in the range 0 (monday) to 6 (sunday). + assertDateTimeInitialized(dt) + dt.weekday + +proc yearday*(dt: DateTime): YeardayRange {.inline.} = + ## The number of days since January 1, + ## in the range 0 to 365. + assertDateTimeInitialized(dt) + dt.yearday + +proc isDst*(dt: DateTime): bool {.inline.} = + ## Determines whether DST is in effect. + ## Always false for the JavaScript backend. + assertDateTimeInitialized(dt) + dt.isDst + +proc timezone*(dt: DateTime): Timezone {.inline.} = + ## The timezone represented as an implementation + ## of `Timezone`. + assertDateTimeInitialized(dt) + dt.timezone + +proc utcOffset*(dt: DateTime): int {.inline.} = + ## The offset in seconds west of UTC, including + ## any offset due to DST. Note that the sign of + ## this number is the opposite of the one in a + ## formatted offset string like `+01:00` (which + ## would be equivalent to the UTC offset + ## `-3600`). + assertDateTimeInitialized(dt) + dt.utcOffset + +proc isInitialized(dt: DateTime): bool = + # Returns true if `dt` is not the (invalid) default value for `DateTime`. + runnableExamples: + doAssert now().isInitialized + doAssert not default(DateTime).isInitialized + dt.monthZero != 0 + +since((1, 3)): + export isInitialized + +proc isLeapDay*(dt: DateTime): bool {.since: (1, 1).} = + ## Returns whether `t` is a leap day, i.e. Feb 29 in a leap year. This matters + ## as it affects time offset calculations. + runnableExamples: + let dt = dateTime(2020, mFeb, 29, 00, 00, 00, 00, utc()) + doAssert dt.isLeapDay + doAssert dt+1.years-1.years != dt + let dt2 = dateTime(2020, mFeb, 28, 00, 00, 00, 00, utc()) + doAssert not dt2.isLeapDay + doAssert dt2+1.years-1.years == dt2 + doAssertRaises(Exception): discard dateTime(2021, mFeb, 29, 00, 00, 00, 00, utc()) + assertDateTimeInitialized dt + dt.year.isLeapYear and dt.month == mFeb and dt.monthday == 29 + +proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = + ## Converts a `DateTime` to a `Time` representing the same point in time. + assertDateTimeInitialized dt + let epochDay = toEpochDay(dt.monthday, dt.month, dt.year) + var seconds = epochDay * secondsInDay + seconds.inc dt.hour * secondsInHour + seconds.inc dt.minute * 60 + seconds.inc dt.second + seconds.inc dt.utcOffset + result = initTime(seconds, dt.nanosecond) + +proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = + ## Create a new `DateTime` using `ZonedTime` in the specified timezone. + let adjTime = zt.time - initDuration(seconds = zt.utcOffset) + let s = adjTime.seconds + let epochday = floorDiv(s, secondsInDay) + var rem = s - epochday * secondsInDay + let hour = rem div secondsInHour + rem = rem - hour * secondsInHour + let minute = rem div secondsInMin + rem = rem - minute * secondsInMin + let second = rem + + let (d, m, y) = fromEpochDay(epochday) + + DateTime( + year: y, + monthZero: m.int, + monthdayZero: d, + hour: hour, + minute: minute, + second: second, + nanosecond: zt.time.nanosecond, + weekday: getDayOfWeek(d, m, y), + yearday: getDayOfYear(d, m, y), + isDst: zt.isDst, + timezone: zone, + utcOffset: zt.utcOffset + ) + +proc newTimezone*( + name: string, + zonedTimeFromTimeImpl: proc (time: Time): ZonedTime + {.tags: [], raises: [], benign.}, + zonedTimeFromAdjTimeImpl: proc (adjTime: Time): ZonedTime + {.tags: [], raises: [], benign.} + ): owned Timezone = + ## Create a new `Timezone`. ## - ## Pass the found token in the user input string, and the buffer where the - ## final string is being built. This has to be a var value because certain - ## formatting tokens require modifying the previous characters. - case token - of "d": - buf.add($info.monthday) - of "dd": - if info.monthday < 10: - buf.add("0") - buf.add($info.monthday) - of "ddd": - buf.add(($info.weekday)[0 .. 2]) - of "dddd": - buf.add($info.weekday) - of "h": - buf.add($(if info.hour > 12: info.hour - 12 else: info.hour)) - of "hh": - let amerHour = if info.hour > 12: info.hour - 12 else: info.hour - if amerHour < 10: - buf.add('0') - buf.add($amerHour) - of "H": - buf.add($info.hour) - of "HH": - if info.hour < 10: - buf.add('0') - buf.add($info.hour) - of "m": - buf.add($info.minute) - of "mm": - if info.minute < 10: - buf.add('0') - buf.add($info.minute) - of "M": - buf.add($(int(info.month)+1)) - of "MM": - if info.month < mOct: - buf.add('0') - buf.add($(int(info.month)+1)) - of "MMM": - buf.add(($info.month)[0..2]) - of "MMMM": - buf.add($info.month) - of "s": - buf.add($info.second) - of "ss": - if info.second < 10: - buf.add('0') - buf.add($info.second) - of "t": - if info.hour >= 12: - buf.add('P') - else: buf.add('A') - of "tt": - if info.hour >= 12: - buf.add("PM") - else: buf.add("AM") - of "y": - var fr = ($info.year).len()-1 - if fr < 0: fr = 0 - buf.add(($info.year)[fr .. ($info.year).len()-1]) - of "yy": - var fr = ($info.year).len()-2 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 2: fyear = repeat('0', 2-fyear.len()) & fyear - buf.add(fyear) - of "yyy": - var fr = ($info.year).len()-3 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 3: fyear = repeat('0', 3-fyear.len()) & fyear - buf.add(fyear) - of "yyyy": - var fr = ($info.year).len()-4 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 4: fyear = repeat('0', 4-fyear.len()) & fyear - buf.add(fyear) - of "yyyyy": - var fr = ($info.year).len()-5 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear - buf.add(fyear) - of "z": - let hrs = (info.timezone div 60) div 60 - buf.add($hrs) - of "zz": - let hrs = (info.timezone div 60) div 60 - - buf.add($hrs) - if hrs.abs < 10: - var atIndex = buf.len-(($hrs).len-(if hrs < 0: 1 else: 0)) - buf.insert("0", atIndex) - of "zzz": - let hrs = (info.timezone div 60) div 60 - - buf.add($hrs & ":00") - if hrs.abs < 10: - var atIndex = buf.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0)) - buf.insert("0", atIndex) - of "ZZZ": - buf.add(info.tzname) - of "": - discard - else: - raise newException(ValueError, "Invalid format string: " & token) + ## `zonedTimeFromTimeImpl` and `zonedTimeFromAdjTimeImpl` is used + ## as the underlying implementations for `zonedTimeFromTime` and + ## `zonedTimeFromAdjTime`. + ## + ## If possible, the name parameter should match the name used in the + ## tz database. If the timezone doesn't exist in the tz database, or if the + ## timezone name is unknown, then any string that describes the timezone + ## unambiguously can be used. Note that the timezones name is used for + ## checking equality! + runnableExamples: + proc utcTzInfo(time: Time): ZonedTime = + ZonedTime(utcOffset: 0, isDst: false, time: time) + let utc = newTimezone("Etc/UTC", utcTzInfo, utcTzInfo) + Timezone( + name: name, + zonedTimeFromTimeImpl: zonedTimeFromTimeImpl, + zonedTimeFromAdjTimeImpl: zonedTimeFromAdjTimeImpl + ) + +proc name*(zone: Timezone): string = + ## The name of the timezone. + ## + ## If possible, the name will be the name used in the tz database. + ## If the timezone doesn't exist in the tz database, or if the timezone + ## name is unknown, then any string that describes the timezone + ## unambiguously might be used. For example, the string "LOCAL" is used + ## for the system's local timezone. + ## + ## See also: https://en.wikipedia.org/wiki/Tz_database + zone.name +proc zonedTimeFromTime*(zone: Timezone, time: Time): ZonedTime = + ## Returns the `ZonedTime` for some point in time. + zone.zonedTimeFromTimeImpl(time) -proc format*(info: TimeInfo, f: string): string = - ## This function formats `info` as specified by `f`. The following format - ## specifiers are available: +proc zonedTimeFromAdjTime*(zone: Timezone, adjTime: Time): ZonedTime = + ## Returns the `ZonedTime` for some local time. ## - ## ========== ================================================================================= ================================================ - ## Specifier Description Example - ## ========== ================================================================================= ================================================ - ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` - ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` - ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` - ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` - ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` - ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` - ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` - ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` - ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` - ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` - ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` - ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` - ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` - ## MMMM Full month string, properly capitalized. ``September -> September`` - ## s Seconds as one digit if possible. ``00:00:06 -> 6`` - ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` - ## t ``A`` when time is in the AM. ``P`` when time is in the PM. - ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. - ## y(yyyy) This displays the year to different digits. You most likely only want 2 or 4 'y's - ## yy Displays the year to two digits. ``2012 -> 12`` - ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` - ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:00``. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ZZZ Displays the name of the timezone. ``GMT -> GMT``, ``EST -> EST`` - ## ========== ================================================================================= ================================================ + ## Note that the `Time` argument does not represent a point in time, it + ## represent a local time! E.g if `adjTime` is `fromUnix(0)`, it should be + ## interpreted as 1970-01-01T00:00:00 in the `zone` timezone, not in UTC. + zone.zonedTimeFromAdjTimeImpl(adjTime) + +proc `$`*(zone: Timezone): string = + ## Returns the name of the timezone. + if zone != nil: result = zone.name + +proc `==`*(zone1, zone2: Timezone): bool = + ## Two `Timezone`'s are considered equal if their name is equal. + runnableExamples: + doAssert local() == local() + doAssert local() != utc() + if system.`==`(zone1, zone2): + return true + if zone1.isNil or zone2.isNil: + return false + zone1.name == zone2.name + +proc inZone*(time: Time, zone: Timezone): DateTime + {.tags: [], raises: [], benign.} = + ## Convert `time` into a `DateTime` using `zone` as the timezone. + result = initDateTime(zone.zonedTimeFromTime(time), zone) + +proc inZone*(dt: DateTime, zone: Timezone): DateTime + {.tags: [], raises: [], benign.} = + ## Returns a `DateTime` representing the same point in time as `dt` but + ## using `zone` as the timezone. + assertDateTimeInitialized dt + dt.toTime.inZone(zone) + +proc toAdjTime(dt: DateTime): Time = + let epochDay = toEpochDay(dt.monthday, dt.month, dt.year) + var seconds = epochDay * secondsInDay + seconds.inc dt.hour * secondsInHour + seconds.inc dt.minute * secondsInMin + seconds.inc dt.second + result = initTime(seconds, dt.nanosecond) + +when defined(js): + proc localZonedTimeFromTime(time: Time): ZonedTime {.benign.} = + let jsDate = newDate(time.seconds * 1000) + let offset = jsDate.getTimezoneOffset() * secondsInMin + result.time = time + result.utcOffset = offset + result.isDst = false + + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime {.benign.} = + let utcDate = newDate(adjTime.seconds * 1000) + let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), + utcDate.getUTCDate(), utcDate.getUTCHours(), utcDate.getUTCMinutes(), + utcDate.getUTCSeconds(), 0) + + # This is as dumb as it looks - JS doesn't support years in the range + # 0-99 in the constructor because they are assumed to be 19xx... + # Because JS doesn't support timezone history, + # it doesn't really matter in practice. + if utcDate.getUTCFullYear() in 0 .. 99: + localDate.setFullYear(utcDate.getUTCFullYear()) + + result.utcOffset = localDate.getTimezoneOffset() * secondsInMin + result.time = adjTime + initDuration(seconds = result.utcOffset) + result.isDst = false + +else: + proc toAdjUnix(tm: Tm): int64 = + let epochDay = toEpochDay(tm.tm_mday, (tm.tm_mon + 1).Month, + tm.tm_year.int + 1900) + result = epochDay * secondsInDay + result.inc tm.tm_hour * secondsInHour + result.inc tm.tm_min * 60 + result.inc tm.tm_sec + + proc getLocalOffsetAndDst(unix: int64): tuple[offset: int, dst: bool] = + # Windows can't handle unix < 0, so we fall back to unix = 0. + # FIXME: This should be improved by falling back to the WinAPI instead. + when defined(windows): + if unix < 0: + var a = 0.CTime + let tmPtr = localtime(a) + if not tmPtr.isNil: + let tm = tmPtr[] + return ((0 - tm.toAdjUnix).int, false) + return (0, false) + + # In case of a 32-bit time_t, we fallback to the closest available + # timezone information. + var a = clamp(unix, low(CTime).int64, high(CTime).int64).CTime + let tmPtr = localtime(a) + if not tmPtr.isNil: + let tm = tmPtr[] + return ((a.int64 - tm.toAdjUnix).int, tm.tm_isdst > 0) + return (0, false) + + proc localZonedTimeFromTime(time: Time): ZonedTime {.benign.} = + let (offset, dst) = getLocalOffsetAndDst(time.seconds) + result.time = time + result.utcOffset = offset + result.isDst = dst + + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime {.benign.} = + var adjUnix = adjTime.seconds + let past = adjUnix - secondsInDay + let (pastOffset, _) = getLocalOffsetAndDst(past) + + let future = adjUnix + secondsInDay + let (futureOffset, _) = getLocalOffsetAndDst(future) + + var utcOffset: int + if pastOffset == futureOffset: + utcOffset = pastOffset.int + else: + if pastOffset > futureOffset: + adjUnix -= secondsInHour + + adjUnix += pastOffset + utcOffset = getLocalOffsetAndDst(adjUnix).offset + + # This extra roundtrip is needed to normalize any impossible datetimes + # as a result of offset changes (normally due to dst) + let utcUnix = adjTime.seconds + utcOffset + let (finalOffset, dst) = getLocalOffsetAndDst(utcUnix) + result.time = initTime(utcUnix, adjTime.nanosecond) + result.utcOffset = finalOffset + result.isDst = dst + +proc utcTzInfo(time: Time): ZonedTime = + ZonedTime(utcOffset: 0, isDst: false, time: time) + +var utcInstance {.threadvar.}: Timezone +var localInstance {.threadvar.}: Timezone + +proc utc*(): Timezone = + ## Get the `Timezone` implementation for the UTC timezone. + runnableExamples: + doAssert now().utc.timezone == utc() + doAssert utc().name == "Etc/UTC" + if utcInstance.isNil: + utcInstance = newTimezone("Etc/UTC", utcTzInfo, utcTzInfo) + result = utcInstance + +proc local*(): Timezone = + ## Get the `Timezone` implementation for the local timezone. + runnableExamples: + doAssert now().timezone == local() + doAssert local().name == "LOCAL" + if localInstance.isNil: + localInstance = newTimezone("LOCAL", localZonedTimeFromTime, + localZonedTimeFromAdjTime) + result = localInstance + +proc utc*(dt: DateTime): DateTime = + ## Shorthand for `dt.inZone(utc())`. + dt.inZone(utc()) + +proc local*(dt: DateTime): DateTime = + ## Shorthand for `dt.inZone(local())`. + dt.inZone(local()) + +proc utc*(t: Time): DateTime = + ## Shorthand for `t.inZone(utc())`. + t.inZone(utc()) + +proc local*(t: Time): DateTime = + ## Shorthand for `t.inZone(local())`. + t.inZone(local()) + +proc now*(): DateTime {.tags: [TimeEffect], benign.} = + ## Get the current time as a `DateTime` in the local timezone. + ## Shorthand for `getTime().local`. ## - ## Other strings can be inserted by putting them in ``''``. For example - ## ``hh'->'mm`` will give ``01->56``. The following characters can be - ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` - ## ``,``. However you don't need to necessarily separate format specifiers, a - ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. + ## .. warning:: Unsuitable for benchmarking, use `monotimes.getMonoTime` or + ## `cpuTime` instead, depending on the use case. + getTime().local + +proc dateTime*(year: int, month: Month, monthday: MonthdayRange, + hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0, + nanosecond: NanosecondRange = 0, + zone: Timezone = local()): DateTime = + ## Create a new `DateTime <#DateTime>`_ in the specified timezone. + runnableExamples: + assert $dateTime(2017, mMar, 30, zone = utc()) == "2017-03-30T00:00:00Z" + + assertValidDate monthday, month, year + let dt = DateTime( + monthdayZero: monthday, + year: year, + monthZero: month.int, + hour: hour, + minute: minute, + second: second, + nanosecond: nanosecond + ) + result = initDateTime(zone.zonedTimeFromAdjTime(dt.toAdjTime), zone) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = + ## Create a new `DateTime <#DateTime>`_ in the specified timezone. + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) + +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = + ## Create a new `DateTime <#DateTime>`_ in the specified timezone. + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, 0, zone) + +proc `+`*(dt: DateTime, dur: Duration): DateTime = + runnableExamples: + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dur = initDuration(hours = 5) + doAssert $(dt + dur) == "2017-03-30T05:00:00Z" + + (dt.toTime + dur).inZone(dt.timezone) + +proc `-`*(dt: DateTime, dur: Duration): DateTime = + runnableExamples: + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dur = initDuration(days = 5) + doAssert $(dt - dur) == "2017-03-25T00:00:00Z" + + (dt.toTime - dur).inZone(dt.timezone) + +proc `-`*(dt1, dt2: DateTime): Duration = + ## Compute the duration between `dt1` and `dt2`. + runnableExamples: + let dt1 = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dt2 = dateTime(2017, mMar, 25, 00, 00, 00, 00, utc()) + + doAssert dt1 - dt2 == initDuration(days = 5) + + dt1.toTime - dt2.toTime + +proc `<`*(a, b: DateTime): bool = + ## Returns true if `a` happened before `b`. + return a.toTime < b.toTime + +proc `<=`*(a, b: DateTime): bool = + ## Returns true if `a` happened before or at the same time as `b`. + return a.toTime <= b.toTime + +proc `==`*(a, b: DateTime): bool = + ## Returns true if `a` and `b` represent the same point in time. + if not a.isInitialized: not b.isInitialized + elif not b.isInitialized: false + else: a.toTime == b.toTime + +proc `+=`*(a: var DateTime, b: Duration) = + a = a + b + +proc `-=`*(a: var DateTime, b: Duration) = + a = a - b + +proc getDateStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## Gets the current local date as a string of the format `YYYY-MM-dd`. + runnableExamples: + echo getDateStr(now() - 1.months) + assertDateTimeInitialized dt + result = newStringOfCap(10) # len("YYYY-MM-dd") == 10 + result.addInt dt.year + result.add '-' + result.add intToStr(dt.monthZero, 2) + result.add '-' + result.add intToStr(dt.monthday, 2) + +proc getClockStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## Gets the current local clock time as a string of the format `HH:mm:ss`. + runnableExamples: + echo getClockStr(now() - 1.hours) + assertDateTimeInitialized dt + result = newStringOfCap(8) # len("HH:mm:ss") == 8 + result.add intToStr(dt.hour, 2) + result.add ':' + result.add intToStr(dt.minute, 2) + result.add ':' + result.add intToStr(dt.second, 2) - result = "" + +# +# Iso week forward declarations +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} + +# +# TimeFormat +# + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +type + DateTimeLocale* = object + MMM*: array[mJan..mDec, string] + MMMM*: array[mJan..mDec, string] + ddd*: array[dMon..dSun, string] + dddd*: array[dMon..dSun, string] + +when defined(nimHasStyleChecks): + {.pop.} + +type + AmPm = enum + apUnknown, apAm, apPm + + Era = enum + eraUnknown, eraAd, eraBc + + ParsedTime = object + amPm: AmPm + era: Era + year: Option[int] + month: Option[int] + monthday: Option[int] + isoyear: Option[int] + yearweek: Option[int] + weekday: Option[WeekDay] + utcOffset: Option[int] + + # '0' as default for these work fine + # so no need for `Option`. + hour: int + minute: int + second: int + nanosecond: int + + FormatTokenKind = enum + tkPattern, tkLiteral + + FormatPattern {.pure.} = enum + d, dd, ddd, dddd + GG, GGGG + h, hh, H, HH + m, mm, M, MM, MMM, MMMM + s, ss + fff, ffffff, fffffffff + t, tt + yy, yyyy + YYYY + uuuu + UUUU + V, VV + z, zz, zzz, zzzz + ZZZ, ZZZZ + g + + # This is a special value used to mark literal format values. + # See the doc comment for `TimeFormat.patterns`. + Lit + + TimeFormat* = object ## Represents a format for parsing and printing + ## time types. + ## + ## To create a new `TimeFormat` use `initTimeFormat proc + ## <#initTimeFormat,string>`_. + patterns: seq[byte] ## \ + ## Contains the patterns encoded as bytes. + ## Literal values are encoded in a special way. + ## They start with `Lit.byte`, then the length of the literal, then the + ## raw char values of the literal. For example, the literal `foo` would + ## be encoded as `@[Lit.byte, 3.byte, 'f'.byte, 'o'.byte, 'o'.byte]`. + formatStr: string + + TimeParseError* = object of ValueError ## \ + ## Raised when parsing input using a `TimeFormat` fails. + + TimeFormatParseError* = object of ValueError ## \ + ## Raised when parsing a `TimeFormat` string fails. + +const + DefaultLocale* = DateTimeLocale( + MMM: ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", + "Nov", "Dec"], + MMMM: ["January", "February", "March", "April", "May", "June", "July", + "August", "September", "October", "November", "December"], + ddd: ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], + dddd: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", + "Sunday"], + ) + + FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ',', '.'} + +proc `$`*(f: TimeFormat): string = + ## Returns the format string that was used to construct `f`. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + doAssert $f == "yyyy-MM-dd" + f.formatStr + +proc raiseParseException(f: TimeFormat, input: string, msg: string) = + raise newException(TimeParseError, + "Failed to parse '" & input & "' with format '" & $f & + "'. " & msg) + +proc parseInt(s: string, b: var int, start = 0, maxLen = int.high, + allowSign = false): int = + var sign = -1 + var i = start + let stop = start + min(s.high - start + 1, maxLen) - 1 + if allowSign and i <= stop: + if s[i] == '+': + inc(i) + elif s[i] == '-': + inc(i) + sign = 1 + if i <= stop and s[i] in {'0'..'9'}: + b = 0 + while i <= stop and s[i] in {'0'..'9'}: + let c = ord(s[i]) - ord('0') + if b >= (low(int) + c) div 10: + b = b * 10 - c + else: + return 0 + inc(i) + if sign == -1 and b == low(int): + return 0 + b = b * sign + result = i - start + +iterator tokens(f: string): tuple[kind: FormatTokenKind, token: string] = var i = 0 - var currentF = "" - while true: - case f[i] - of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - formatToken(info, currentF, result) + var currToken = "" - currentF = "" - if f[i] == '\0': break + template yieldCurrToken() = + if currToken.len != 0: + yield (tkPattern, currToken) + currToken = "" - if f[i] == '\'': + while i < f.len: + case f[i] + of '\'': + yieldCurrToken() + if i.succ < f.len and f[i.succ] == '\'': + yield (tkLiteral, "'") + i.inc 2 + else: + var token = "" inc(i) # Skip ' - while f[i] != '\'' and f.len-1 > i: - result.add(f[i]) - inc(i) - else: result.add(f[i]) - + while i < f.len and f[i] != '\'': + token.add f[i] + i.inc + + if i > f.high: + raise newException(TimeFormatParseError, + "Unclosed ' in time format string. " & + "For a literal ', use ''.") + i.inc + yield (tkLiteral, token) + of FormatLiterals: + yieldCurrToken() + yield (tkLiteral, $f[i]) + i.inc else: # Check if the letter being added matches previous accumulated buffer. - if currentF.len < 1 or currentF[high(currentF)] == f[i]: - currentF.add(f[i]) + if currToken.len == 0 or currToken[0] == f[i]: + currToken.add(f[i]) + i.inc + else: + yield (tkPattern, currToken) + currToken = $f[i] + i.inc + + yieldCurrToken() + +proc stringToPattern(str: string): FormatPattern = + case str + of "d": result = d + of "dd": result = dd + of "ddd": result = ddd + of "dddd": result = dddd + of "GG": result = GG + of "GGGG": result = GGGG + of "h": result = h + of "hh": result = hh + of "H": result = H + of "HH": result = HH + of "m": result = m + of "mm": result = mm + of "M": result = M + of "MM": result = MM + of "MMM": result = MMM + of "MMMM": result = MMMM + of "s": result = s + of "ss": result = ss + of "fff": result = fff + of "ffffff": result = ffffff + of "fffffffff": result = fffffffff + of "t": result = t + of "tt": result = tt + of "yy": result = yy + of "yyyy": result = yyyy + of "YYYY": result = YYYY + of "uuuu": result = uuuu + of "UUUU": result = UUUU + of "V": result = V + of "VV": result = VV + of "z": result = z + of "zz": result = zz + of "zzz": result = zzz + of "zzzz": result = zzzz + of "ZZZ": result = ZZZ + of "ZZZZ": result = ZZZZ + of "g": result = g + else: raise newException(TimeFormatParseError, + "'" & str & "' is not a valid pattern") + +proc initTimeFormat*(format: string): TimeFormat = + ## Construct a new time format for parsing & formatting time types. + ## + ## See `Parsing and formatting dates`_ for documentation of the + ## `format` argument. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + doAssert "2000-01-01" == "2000-01-01".parse(f).format(f) + result.formatStr = format + result.patterns = @[] + for kind, token in format.tokens: + case kind + of tkLiteral: + case token else: - formatToken(info, currentF, result) - dec(i) # Move position back to re-process the character separately. - currentF = "" - - inc(i) - -{.pop.} - -proc parseToken(info: var TimeInfo; token, value: string; j: var int) = - ## Helper of the parse proc to parse individual tokens. - var sv: int - case token - of "d": - var pd = parseInt(value[j..j+1], sv) - info.monthday = sv - j += pd - of "dd": - info.monthday = value[j..j+1].parseInt() - j += 2 - of "ddd": - case value[j..j+2].toLower() - of "sun": - info.weekday = dSun - of "mon": - info.weekday = dMon - of "tue": - info.weekday = dTue - of "wed": - info.weekday = dWed - of "thu": - info.weekday = dThu - of "fri": - info.weekday = dFri - of "sat": - info.weekday = dSat + result.patterns.add(FormatPattern.Lit.byte) + if token.len > 255: + raise newException(TimeFormatParseError, + "Format literal is to long:" & token) + result.patterns.add(token.len.byte) + for c in token: + result.patterns.add(c.byte) + of tkPattern: + result.patterns.add(stringToPattern(token).byte) + +proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, + loc: DateTimeLocale) = + template yearOfEra(dt: DateTime): int = + if dt.year <= 0: abs(dt.year) + 1 else: dt.year + + case pattern + of d: + result.add $dt.monthday + of dd: + result.add dt.monthday.intToStr(2) + of ddd: + result.add loc.ddd[dt.weekday] + of dddd: + result.add loc.dddd[dt.weekday] + of GG: + result.add (dt.getIsoWeekAndYear.isoyear.int mod 100).intToStr(2) + of GGGG: + result.add $dt.getIsoWeekAndYear.isoyear + of h: + result.add( + if dt.hour == 0: "12" + elif dt.hour > 12: $(dt.hour - 12) + else: $dt.hour + ) + of hh: + result.add( + if dt.hour == 0: "12" + elif dt.hour > 12: (dt.hour - 12).intToStr(2) + else: dt.hour.intToStr(2) + ) + of H: + result.add $dt.hour + of HH: + result.add dt.hour.intToStr(2) + of m: + result.add $dt.minute + of mm: + result.add dt.minute.intToStr(2) + of M: + result.add $ord(dt.month) + of MM: + result.add ord(dt.month).intToStr(2) + of MMM: + result.add loc.MMM[dt.month] + of MMMM: + result.add loc.MMMM[dt.month] + of s: + result.add $dt.second + of ss: + result.add dt.second.intToStr(2) + of fff: + result.add(intToStr(convert(Nanoseconds, Milliseconds, dt.nanosecond), 3)) + of ffffff: + result.add(intToStr(convert(Nanoseconds, Microseconds, dt.nanosecond), 6)) + of fffffffff: + result.add(intToStr(dt.nanosecond, 9)) + of t: + result.add if dt.hour >= 12: "P" else: "A" + of tt: + result.add if dt.hour >= 12: "PM" else: "AM" + of yy: + result.add (dt.yearOfEra mod 100).intToStr(2) + of yyyy: + let year = dt.yearOfEra + if year < 10000: + result.add year.intToStr(4) else: - raise newException(ValueError, - "Couldn't parse day of week (ddd), got: " & value[j..j+2]) - j += 3 - of "dddd": - if value.len >= j+6 and value[j..j+5].cmpIgnoreCase("sunday") == 0: - info.weekday = dSun - j += 6 - elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("monday") == 0: - info.weekday = dMon - j += 6 - elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("tuesday") == 0: - info.weekday = dTue - j += 7 - elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("wednesday") == 0: - info.weekday = dWed - j += 9 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("thursday") == 0: - info.weekday = dThu - j += 8 - elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("friday") == 0: - info.weekday = dFri - j += 6 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("saturday") == 0: - info.weekday = dSat - j += 8 + result.add '+' & $year + of YYYY: + if dt.year < 1: + result.add $(abs(dt.year) + 1) else: - raise newException(ValueError, - "Couldn't parse day of week (dddd), got: " & value) - of "h", "H": - var pd = parseInt(value[j..j+1], sv) - info.hour = sv - j += pd - of "hh", "HH": - info.hour = value[j..j+1].parseInt() - j += 2 - of "m": - var pd = parseInt(value[j..j+1], sv) - info.minute = sv - j += pd - of "mm": - info.minute = value[j..j+1].parseInt() - j += 2 - of "M": - var pd = parseInt(value[j..j+1], sv) - info.month = Month(sv-1) - info.monthday = sv - j += pd - of "MM": - var month = value[j..j+1].parseInt() - j += 2 - info.month = Month(month-1) - of "MMM": - case value[j..j+2].toLower(): - of "jan": - info.month = mJan - of "feb": - info.month = mFeb - of "mar": - info.month = mMar - of "apr": - info.month = mApr - of "may": - info.month = mMay - of "jun": - info.month = mJun - of "jul": - info.month = mJul - of "aug": - info.month = mAug - of "sep": - info.month = mSep - of "oct": - info.month = mOct - of "nov": - info.month = mNov - of "dec": - info.month = mDec + result.add $dt.year + of uuuu: + let year = dt.year + if year < 10000 or year < 0: + result.add year.intToStr(4) else: - raise newException(ValueError, - "Couldn't parse month (MMM), got: " & value) - j += 3 - of "MMMM": - if value.len >= j+7 and value[j..j+6].cmpIgnoreCase("january") == 0: - info.month = mJan - j += 7 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("february") == 0: - info.month = mFeb - j += 8 - elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("march") == 0: - info.month = mMar - j += 5 - elif value.len >= j+5 and value[j..j+4].cmpIgnoreCase("april") == 0: - info.month = mApr - j += 5 - elif value.len >= j+3 and value[j..j+2].cmpIgnoreCase("may") == 0: - info.month = mMay - j += 3 - elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("june") == 0: - info.month = mJun - j += 4 - elif value.len >= j+4 and value[j..j+3].cmpIgnoreCase("july") == 0: - info.month = mJul - j += 4 - elif value.len >= j+6 and value[j..j+5].cmpIgnoreCase("august") == 0: - info.month = mAug - j += 6 - elif value.len >= j+9 and value[j..j+8].cmpIgnoreCase("september") == 0: - info.month = mSep - j += 9 - elif value.len >= j+7 and value[j..j+6].cmpIgnoreCase("october") == 0: - info.month = mOct - j += 7 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("november") == 0: - info.month = mNov - j += 8 - elif value.len >= j+8 and value[j..j+7].cmpIgnoreCase("december") == 0: - info.month = mDec - j += 8 + result.add '+' & $year + of UUUU: + result.add $dt.year + of V: + result.add $dt.getIsoWeekAndYear.isoweek + of VV: + result.add dt.getIsoWeekAndYear.isoweek.intToStr(2) + of z, zz, zzz, zzzz, ZZZ, ZZZZ: + if dt.timezone != nil and dt.timezone.name == "Etc/UTC": + result.add 'Z' else: - raise newException(ValueError, - "Couldn't parse month (MMMM), got: " & value) - of "s": - var pd = parseInt(value[j..j+1], sv) - info.second = sv - j += pd - of "ss": - info.second = value[j..j+1].parseInt() - j += 2 - of "t": - if value[j] == 'P' and info.hour > 0 and info.hour < 12: - info.hour += 12 - j += 1 - of "tt": - if value[j..j+1] == "PM" and info.hour > 0 and info.hour < 12: - info.hour += 12 - j += 2 - of "yy": + result.add if -dt.utcOffset >= 0: '+' else: '-' + let absOffset = abs(dt.utcOffset) + case pattern: + of z: + result.add $(absOffset div 3600) + of zz: + result.add (absOffset div 3600).intToStr(2) + of zzz, ZZZ: + let h = (absOffset div 3600).intToStr(2) + let m = ((absOffset div 60) mod 60).intToStr(2) + let sep = if pattern == zzz: ":" else: "" + result.add h & sep & m + of zzzz, ZZZZ: + let absOffset = abs(dt.utcOffset) + let h = (absOffset div 3600).intToStr(2) + let m = ((absOffset div 60) mod 60).intToStr(2) + let s = (absOffset mod 60).intToStr(2) + let sep = if pattern == zzzz: ":" else: "" + result.add h & sep & m & sep & s + else: assert false + of g: + result.add if dt.year < 1: "BC" else: "AD" + of Lit: assert false # Can't happen + +proc parsePattern(input: string, pattern: FormatPattern, i: var int, + parsed: var ParsedTime, loc: DateTimeLocale): bool = + template takeInt(allowedWidth: Slice[int], allowSign = false): int = + var sv = 0 + var pd = parseInt(input, sv, i, allowedWidth.b, allowSign) + if pd < allowedWidth.a: + return false + i.inc pd + sv + + template contains[T](t: typedesc[T], i: int): bool = + i in low(t)..high(t) + + result = true + + case pattern + of d: + let monthday = takeInt(1..2) + parsed.monthday = some(monthday) + result = monthday in MonthdayRange + of dd: + let monthday = takeInt(2..2) + parsed.monthday = some(monthday) + result = monthday in MonthdayRange + of ddd: + result = false + for d, v in loc.ddd: + if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) + result = true + i.inc v.len + break + of dddd: + result = false + for d, v in loc.dddd: + if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) + result = true + i.inc v.len + break + of GG: # Assumes current century - var year = value[j..j+1].parseInt() - var thisCen = getLocalTime(getTime()).year div 100 - info.year = thisCen*100 + year - j += 2 - of "yyyy": - info.year = value[j..j+3].parseInt() - j += 4 - of "z": - if value[j] == '+': - info.timezone = parseInt($value[j+1]) - elif value[j] == '-': - info.timezone = 0-parseInt($value[j+1]) + var isoyear = takeInt(2..2) + var thisCen = now().year div 100 + parsed.isoyear = some(thisCen*100 + isoyear) + result = isoyear > 0 + of GGGG: + let isoyear = takeInt(1..high(int)) + parsed.isoyear = some(isoyear) + result = isoyear > 0 + of h, H: + parsed.hour = takeInt(1..2) + result = parsed.hour in HourRange + of hh, HH: + parsed.hour = takeInt(2..2) + result = parsed.hour in HourRange + of m: + parsed.minute = takeInt(1..2) + result = parsed.hour in MinuteRange + of mm: + parsed.minute = takeInt(2..2) + result = parsed.hour in MinuteRange + of M: + let month = takeInt(1..2) + result = month in 1..12 + parsed.month = some(month) + of MM: + let month = takeInt(2..2) + result = month in 1..12 + parsed.month = some(month) + of MMM: + result = false + for n, v in loc.MMM: + if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + result = true + i.inc v.len + parsed.month = some(n.int) + break + of MMMM: + result = false + for n, v in loc.MMMM: + if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + result = true + i.inc v.len + parsed.month = some(n.int) + break + of s: + parsed.second = takeInt(1..2) + of ss: + parsed.second = takeInt(2..2) + of fff, ffffff, fffffffff: + let len = ($pattern).len + let v = takeInt(len..len) + parsed.nanosecond = v * 10^(9 - len) + result = parsed.nanosecond in NanosecondRange + of t: + case input[i]: + of 'P': + parsed.amPm = apPm + of 'A': + parsed.amPm = apAm else: - raise newException(ValueError, - "Couldn't parse timezone offset (z), got: " & value[j]) - j += 2 - of "zz": - if value[j] == '+': - info.timezone = value[j+1..j+2].parseInt() - elif value[j] == '-': - info.timezone = 0-value[j+1..j+2].parseInt() + result = false + i.inc 1 + of tt: + if input.substr(i, i+1).cmpIgnoreCase("AM") == 0: + parsed.amPm = apAm + i.inc 2 + elif input.substr(i, i+1).cmpIgnoreCase("PM") == 0: + parsed.amPm = apPm + i.inc 2 else: - raise newException(ValueError, - "Couldn't parse timezone offset (zz), got: " & value[j]) - j += 3 - of "zzz": - if value[j] == '+': - info.timezone = value[j+1..j+2].parseInt() - elif value[j] == '-': - info.timezone = 0-value[j+1..j+2].parseInt() + result = false + of yy: + # Assumes current century + var year = takeInt(2..2) + var thisCen = now().year div 100 + parsed.year = some(thisCen*100 + year) + result = year > 0 + of yyyy: + let year = + if input[i] in {'+', '-'}: + takeInt(4..high(int), allowSign = true) + else: + takeInt(4..4) + result = year > 0 + parsed.year = some(year) + of YYYY: + let year = takeInt(1..high(int)) + parsed.year = some(year) + result = year > 0 + of uuuu: + let year = + if input[i] in {'+', '-'}: + takeInt(4..high(int), allowSign = true) + else: + takeInt(4..4) + parsed.year = some(year) + of UUUU: + parsed.year = some(takeInt(1..high(int), allowSign = true)) + of V: + let yearweek = takeInt(1..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of VV: + let yearweek = takeInt(2..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of z, zz, zzz, zzzz, ZZZ, ZZZZ: + case input[i] + of '+', '-': + let sign = if input[i] == '-': 1 else: -1 + i.inc + var offset = 0 + case pattern + of z: + offset = takeInt(1..2) * 3600 + of zz: + offset = takeInt(2..2) * 3600 + of zzz, ZZZ: + offset.inc takeInt(2..2) * 3600 + if pattern == zzz: + if input[i] != ':': + return false + i.inc + offset.inc takeInt(2..2) * 60 + of zzzz, ZZZZ: + offset.inc takeInt(2..2) * 3600 + if pattern == zzzz: + if input[i] != ':': + return false + i.inc + offset.inc takeInt(2..2) * 60 + if pattern == zzzz: + if input[i] != ':': + return false + i.inc + offset.inc takeInt(2..2) + else: assert false + parsed.utcOffset = some(offset * sign) + of 'Z': + parsed.utcOffset = some(0) + i.inc + else: + result = false + of g: + if input.substr(i, i+1).cmpIgnoreCase("BC") == 0: + parsed.era = eraBc + i.inc 2 + elif input.substr(i, i+1).cmpIgnoreCase("AD") == 0: + parsed.era = eraAd + i.inc 2 else: - raise newException(ValueError, - "Couldn't parse timezone offset (zzz), got: " & value[j]) - j += 6 - of "ZZZ": - info.tzname = value[j..j+2].toUpper() - j += 3 + result = false + of Lit: raiseAssert "Can't happen" + +proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var year = p.year.get(0) + var month = p.month.get(1).Month + var monthday = p.monthday.get(1) + year = + case p.era + of eraUnknown: + year + of eraBc: + if year < 1: + raiseParseException(f, input, + "Expected year to be positive " & + "(use 'UUUU' or 'uuuu' for negative years).") + -year + 1 + of eraAd: + if year < 1: + raiseParseException(f, input, + "Expected year to be positive " & + "(use 'UUUU' or 'uuuu' for negative years).") + year + + let hour = + case p.amPm + of apUnknown: + p.hour + of apAm: + if p.hour notin 1..12: + raiseParseException(f, input, + "AM/PM time must be in the interval 1..12") + if p.hour == 12: 0 else: p.hour + of apPm: + if p.hour notin 1..12: + raiseParseException(f, input, + "AM/PM time must be in the interval 1..12") + if p.hour == 12: p.hour else: p.hour + 12 + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if monthday > getDaysInMonth(month, year): + raiseParseException(f, input, + $year & "-" & ord(month).intToStr(2) & + "-" & $monthday & " is not a valid date") + + if p.utcOffset.isNone: + # No timezone parsed - assume timezone is `zone` + result = dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) + else: + # Otherwise convert to `zone` + result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) + +proc toDateTimeByWeek(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var isoyear = p.isoyear.get(0) + var yearweek = p.yearweek.get(1) + var weekday = p.weekday.get(dMon) + + if p.amPm != apUnknown: + raiseParseException(f, input, "Parsing iso weekyear dates does not support am/pm") + + if p.year.isSome: + raiseParseException(f, input, "Use iso-year GG or GGGG as year with iso week number") + + if p.month.isSome: + raiseParseException(f, input, "Use either iso week number V or VV or month") + + if p.monthday.isSome: + raiseParseException(f, input, "Use weekday ddd or dddd as day with with iso week number") + + if p.isoyear.isNone: + raiseParseException(f, input, "Need iso-year with week number") + + let hour = p.hour + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if p.utcOffset.isNone: + result = initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone) else: - # Ignore the token and move forward in the value string by the same length - j += token.len + result = (initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) + +proc format*(dt: DateTime, f: TimeFormat, + loc: DateTimeLocale = DefaultLocale): string {.raises: [].} = + ## Format `dt` using the format specified by `f`. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) + doAssert "2000-01-01" == dt.format(f) + assertDateTimeInitialized dt + result = "" + var idx = 0 + while idx <= f.patterns.high: + case f.patterns[idx].FormatPattern + of Lit: + idx.inc + let len = f.patterns[idx] + for i in 1'u8..len: + idx.inc + result.add f.patterns[idx].char + idx.inc + else: + formatPattern(dt, f.patterns[idx].FormatPattern, result = result, loc = loc) + idx.inc -proc parse*(value, layout: string): TimeInfo = - ## This function parses a date/time string using the standard format identifiers (below) - ## The function defaults information not provided in the format string from the running program (timezone, month, year, etc) +proc format*(dt: DateTime, f: string, loc: DateTimeLocale = DefaultLocale): string + {.raises: [TimeFormatParseError].} = + ## Shorthand for constructing a `TimeFormat` and using it to format `dt`. ## - ## ========== ================================================================================= ================================================ - ## Specifier Description Example - ## ========== ================================================================================= ================================================ - ## d Numeric value of the day of the month, it will be one or two digits long. ``1/04/2012 -> 1``, ``21/04/2012 -> 21`` - ## dd Same as above, but always two digits. ``1/04/2012 -> 01``, ``21/04/2012 -> 21`` - ## ddd Three letter string which indicates the day of the week. ``Saturday -> Sat``, ``Monday -> Mon`` - ## dddd Full string for the day of the week. ``Saturday -> Saturday``, ``Monday -> Monday`` - ## h The hours in one digit if possible. Ranging from 0-12. ``5pm -> 5``, ``2am -> 2`` - ## hh The hours in two digits always. If the hour is one digit 0 is prepended. ``5pm -> 05``, ``11am -> 11`` - ## H The hours in one digit if possible, randing from 0-24. ``5pm -> 17``, ``2am -> 2`` - ## HH The hours in two digits always. 0 is prepended if the hour is one digit. ``5pm -> 17``, ``2am -> 02`` - ## m The minutes in 1 digit if possible. ``5:30 -> 30``, ``2:01 -> 1`` - ## mm Same as above but always 2 digits, 0 is prepended if the minute is one digit. ``5:30 -> 30``, ``2:01 -> 01`` - ## M The month in one digit if possible. ``September -> 9``, ``December -> 12`` - ## MM The month in two digits always. 0 is prepended. ``September -> 09``, ``December -> 12`` - ## MMM Abbreviated three-letter form of the month. ``September -> Sep``, ``December -> Dec`` - ## MMMM Full month string, properly capitalized. ``September -> September`` - ## s Seconds as one digit if possible. ``00:00:06 -> 6`` - ## ss Same as above but always two digits. 0 is prepended. ``00:00:06 -> 06`` - ## t ``A`` when time is in the AM. ``P`` when time is in the PM. - ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. - ## yy Displays the year to two digits. ``2012 -> 12`` - ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` - ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:00``. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ZZZ Displays the name of the timezone. ``GMT -> GMT``, ``EST -> EST`` - ## ========== ================================================================================= ================================================ + ## See `Parsing and formatting dates`_ for documentation of the + ## `format` argument. + runnableExamples: + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) + doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") + let dtFormat = initTimeFormat(f) + result = dt.format(dtFormat, loc) + +proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = + ## Overload that validates `format` at compile time. + const f2 = initTimeFormat(f) + result = dt.format(f2) + +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = + ## adapter for strformat. Not intended to be called directly. + result.add format(value, + if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) + +proc format*(time: Time, f: string, zone: Timezone = local()): string + {.raises: [TimeFormatParseError].} = + ## Shorthand for constructing a `TimeFormat` and using it to format + ## `time`. Will use the timezone specified by `zone`. ## - ## Other strings can be inserted by putting them in ``''``. For example - ## ``hh'->'mm`` will give ``01->56``. The following characters can be - ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` - ## ``,``. However you don't need to necessarily separate format specifiers, a - ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. - var i = 0 # pointer for format string - var j = 0 # pointer for value string - var token = "" - # Assumes current day of month, month and year, but time is reset to 00:00:00. Weekday will be reset after parsing. - var info = getLocalTime(getTime()) - info.hour = 0 - info.minute = 0 - info.second = 0 - while true: - case layout[i] - of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - if token.len > 0: - parseToken(info, token, value, j) - # Reset token - token = "" - # Break if at end of line - if layout[i] == '\0': break - # Skip separator and everything between single quotes - # These are literals in both the layout and the value string - if layout[i] == '\'': - inc(i) - while layout[i] != '\'' and layout.len-1 > i: - inc(i) - inc(j) - inc(i) - else: - inc(i) - inc(j) + ## See `Parsing and formatting dates`_ for documentation of the + ## `f` argument. + runnableExamples: + var dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, utc()) + var tm = dt.toTime() + doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" + time.inZone(zone).format(f) + +proc format*(time: Time, f: static[string], zone: Timezone = local()): string + {.raises: [].} = + ## Overload that validates `f` at compile time. + const f2 = initTimeFormat(f) + result = time.inZone(zone).format(f2) + +proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = + ## Parses `input` as a `DateTime` using the format specified by `f`. + ## If no UTC offset was parsed, then `input` is assumed to be specified in + ## the `zone` timezone. If a UTC offset was parsed, the result will be + ## converted to the `zone` timezone. + ## + ## Month and day names from the passed in `loc` are used. + runnableExamples: + let f = initTimeFormat("yyyy-MM-dd") + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) + doAssert dt == "2000-01-01".parse(f, utc()) + var inpIdx = 0 # Input index + var patIdx = 0 # Pattern index + var parsed: ParsedTime + while inpIdx <= input.high and patIdx <= f.patterns.high: + let pattern = f.patterns[patIdx].FormatPattern + case pattern + of Lit: + patIdx.inc + let len = f.patterns[patIdx] + patIdx.inc + for _ in 1'u8..len: + if input[inpIdx] != f.patterns[patIdx].char: + raiseParseException(f, input, + "Unexpected character: " & input[inpIdx]) + inpIdx.inc + patIdx.inc else: - # Check if the letter being added matches previous accumulated buffer. - if token.len < 1 or token[high(token)] == layout[i]: - token.add(layout[i]) - inc(i) + if not parsePattern(input, pattern, inpIdx, parsed, loc): + raiseParseException(f, input, "Failed on pattern '" & $pattern & "'") + patIdx.inc + + if inpIdx <= input.high: + raiseParseException(f, input, + "Parsing ended but there was still input remaining") + + if patIdx <= f.patterns.high: + raiseParseException(f, input, + "Parsing ended but there was still patterns remaining") + + if parsed.yearweek.isSome: + result = toDateTimeByWeek(parsed, zone, f, input) + elif parsed.isoyear.isSome: + raiseParseException(f, input, "Iso year GG or GGGG require iso week V or VV") + else: + result = toDateTime(parsed, zone, f, input) + +proc parse*(input, f: string, tz: Timezone = local(), + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = + ## Shorthand for constructing a `TimeFormat` and using it to parse + ## `input` as a `DateTime`. + ## + ## See `Parsing and formatting dates`_ for documentation of the + ## `f` argument. + runnableExamples: + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) + doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) + let dtFormat = initTimeFormat(f) + result = input.parse(dtFormat, tz, loc = loc) + +proc parse*(input: string, f: static[string], zone: Timezone = local(), + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = + ## Overload that validates `f` at compile time. + const f2 = initTimeFormat(f) + result = input.parse(f2, zone, loc = loc) + +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = + ## Shorthand for constructing a `TimeFormat` and using it to parse + ## `input` as a `DateTime`, then converting it a `Time`. + ## + ## See `Parsing and formatting dates`_ for documentation of the + ## `format` argument. + runnableExamples: + let tStr = "1970-01-01T00:00:00+00:00" + doAssert parseTime(tStr, "yyyy-MM-dd'T'HH:mm:sszzz", utc()) == fromUnix(0) + parse(input, f, zone).toTime() + +proc parseTime*(input: string, f: static[string], zone: Timezone): Time + {.parseRaises.} = + ## Overload that validates `format` at compile time. + const f2 = initTimeFormat(f) + result = input.parse(f2, zone).toTime() + +proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = + ## Converts a `DateTime` object to a string representation. + ## It uses the format `yyyy-MM-dd'T'HH:mm:sszzz`. + runnableExamples: + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) + doAssert $dt == "2000-01-01T12:00:00Z" + doAssert $default(DateTime) == "Uninitialized DateTime" + if not dt.isInitialized: + result = "Uninitialized DateTime" + else: + result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") + +proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = + ## Converts a `Time` value to a string representation. It will use the local + ## time zone and use the format `yyyy-MM-dd'T'HH:mm:sszzz`. + runnableExamples: + let dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, local()) + let tm = dt.toTime() + doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") + $time.local + +# +# TimeInterval +# + +proc initTimeInterval*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, + days, weeks, months, years: int = 0): TimeInterval = + ## Creates a new `TimeInterval <#TimeInterval>`_. + ## + ## This proc doesn't perform any normalization! For example, + ## `initTimeInterval(hours = 24)` and `initTimeInterval(days = 1)` are + ## not equal. + ## + ## You can also use the convenience procedures called `milliseconds`, + ## `seconds`, `minutes`, `hours`, `days`, `months`, and `years`. + runnableExamples: + let day = initTimeInterval(hours = 24) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) + doAssert $(dt + day) == "2000-01-02T12:00:00Z" + doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) + result.nanoseconds = nanoseconds + result.microseconds = microseconds + result.milliseconds = milliseconds + result.seconds = seconds + result.minutes = minutes + result.hours = hours + result.days = days + result.weeks = weeks + result.months = months + result.years = years + +proc `+`*(ti1, ti2: TimeInterval): TimeInterval = + ## Adds two `TimeInterval` objects together. + result.nanoseconds = ti1.nanoseconds + ti2.nanoseconds + result.microseconds = ti1.microseconds + ti2.microseconds + result.milliseconds = ti1.milliseconds + ti2.milliseconds + result.seconds = ti1.seconds + ti2.seconds + result.minutes = ti1.minutes + ti2.minutes + result.hours = ti1.hours + ti2.hours + result.days = ti1.days + ti2.days + result.weeks = ti1.weeks + ti2.weeks + result.months = ti1.months + ti2.months + result.years = ti1.years + ti2.years + +proc `-`*(ti: TimeInterval): TimeInterval = + ## Reverses a time interval + runnableExamples: + let day = -initTimeInterval(hours = 24) + doAssert day.hours == -24 + + result = TimeInterval( + nanoseconds: -ti.nanoseconds, + microseconds: -ti.microseconds, + milliseconds: -ti.milliseconds, + seconds: -ti.seconds, + minutes: -ti.minutes, + hours: -ti.hours, + days: -ti.days, + weeks: -ti.weeks, + months: -ti.months, + years: -ti.years + ) + +proc `-`*(ti1, ti2: TimeInterval): TimeInterval = + ## Subtracts TimeInterval `ti1` from `ti2`. + ## + ## Time components are subtracted one-by-one, see output: + runnableExamples: + let ti1 = initTimeInterval(hours = 24) + let ti2 = initTimeInterval(hours = 4) + doAssert (ti1 - ti2) == initTimeInterval(hours = 20) + + result = ti1 + (-ti2) + +proc `+=`*(a: var TimeInterval, b: TimeInterval) = + a = a + b + +proc `-=`*(a: var TimeInterval, b: TimeInterval) = + a = a - b + +proc isStaticInterval(interval: TimeInterval): bool = + interval.years == 0 and interval.months == 0 and + interval.days == 0 and interval.weeks == 0 + +proc evaluateStaticInterval(interval: TimeInterval): Duration = + assert interval.isStaticInterval + initDuration(nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + +proc between*(startDt, endDt: DateTime): TimeInterval = + ## Gives the difference between `startDt` and `endDt` as a + ## `TimeInterval`. The following guarantees about the result is given: + ## + ## - All fields will have the same sign. + ## - If `startDt.timezone == endDt.timezone`, it is guaranteed that + ## `startDt + between(startDt, endDt) == endDt`. + ## - If `startDt.timezone != endDt.timezone`, then the result will be + ## equivalent to `between(startDt.utc, endDt.utc)`. + runnableExamples: + var a = dateTime(2015, mMar, 25, 12, 0, 0, 00, utc()) + var b = dateTime(2017, mApr, 1, 15, 0, 15, 00, utc()) + var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) + doAssert between(a, b) == ti + doAssert between(a, b) == -between(b, a) + + if startDt.timezone != endDt.timezone: + return between(startDt.utc, endDt.utc) + elif endDt < startDt: + return -between(endDt, startDt) + + type Date = tuple[year, month, monthday: int] + var startDate: Date = (startDt.year, startDt.month.ord, startDt.monthday) + var endDate: Date = (endDt.year, endDt.month.ord, endDt.monthday) + + # Subtract one day from endDate if time of day is earlier than startDay + # The subtracted day will be counted by fixed units (hour and lower) + # at the end of this proc + if (endDt.hour, endDt.minute, endDt.second, endDt.nanosecond) < + (startDt.hour, startDt.minute, startDt.second, startDt.nanosecond): + if endDate.month == 1 and endDate.monthday == 1: + endDate.year.dec + endDate.monthday = 31 + endDate.month = 12 + elif endDate.monthday == 1: + endDate.month.dec + endDate.monthday = getDaysInMonth(endDate.month.Month, endDate.year) + else: + endDate.monthday.dec + + # Years + result.years = endDate.year - startDate.year - 1 + if (startDate.month, startDate.monthday) <= (endDate.month, endDate.monthday): + result.years.inc + startDate.year.inc result.years + + # Months + if startDate.year < endDate.year: + result.months.inc 12 - startDate.month # Move to dec + if endDate.month != 1 or (startDate.monthday <= endDate.monthday): + result.months.inc + startDate.year = endDate.year + startDate.month = 1 + else: + startDate.month = 12 + if startDate.year == endDate.year: + if (startDate.monthday <= endDate.monthday): + result.months.inc endDate.month - startDate.month + startDate.month = endDate.month + elif endDate.month != 1: + let month = endDate.month - 1 + let daysInMonth = getDaysInMonth(month.Month, startDate.year) + if daysInMonth < startDate.monthday: + if startDate.monthday - daysInMonth < endDate.monthday: + result.months.inc endDate.month - startDate.month - 1 + startDate.month = endDate.month + startDate.monthday = startDate.monthday - daysInMonth + else: + result.months.inc endDate.month - startDate.month - 2 + startDate.month = endDate.month - 2 else: - parseToken(info, token, value, j) - token = "" - # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(timeInfoToTime(info)).weekday - return info - -# Leap year calculations are adapted from: -# http://www.codeproject.com/Articles/7358/Ultra-fast-Algorithms-for-Working-with-Leap-Years -# The dayOfTheWeek procs are adapated from: -# http://stason.org/TULARC/society/calendars/2-5-What-day-of-the-week-was-2-August-1953.html - -proc countLeapYears*(yearSpan: int): int = - ## Returns the number of leap years spanned by a given number of years. + result.months.inc endDate.month - startDate.month - 1 + startDate.month = endDate.month - 1 + + # Days + # This means that start = dec and end = jan + if startDate.year < endDate.year: + result.days.inc 31 - startDate.monthday + endDate.monthday + startDate = endDate + else: + while startDate.month < endDate.month: + let daysInMonth = getDaysInMonth(startDate.month.Month, startDate.year) + result.days.inc daysInMonth - startDate.monthday + 1 + startDate.month.inc + startDate.monthday = 1 + result.days.inc endDate.monthday - startDate.monthday + result.weeks = result.days div 7 + result.days = result.days mod 7 + startDate = endDate + + # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds + let newStartDt = dateTime(startDate.year, startDate.month.Month, + startDate.monthday, startDt.hour, startDt.minute, startDt.second, + startDt.nanosecond, startDt.timezone) + let dur = endDt - newStartDt + let parts = toParts(dur) + # There can still be a full day in `parts` since `Duration` and `TimeInterval` + # models days differently. + result.hours = parts[Hours].int + parts[Days].int * 24 + result.minutes = parts[Minutes].int + result.seconds = parts[Seconds].int + result.milliseconds = parts[Milliseconds].int + result.microseconds = parts[Microseconds].int + result.nanoseconds = parts[Nanoseconds].int + +proc toParts*(ti: TimeInterval): TimeIntervalParts = + ## Converts a `TimeInterval` into an array consisting of its time units, + ## starting with nanoseconds and ending with years. ## - ## Note: for leap years, start date is assumed to be 1 AD. - ## counts the number of leap years up to January 1st of a given year. - ## Keep in mind that if specified year is a leap year, the leap day - ## has not happened before January 1st of that year. - (((yearSpan - 1) / 4) - ((yearSpan - 1) / 100) + ((yearSpan - 1) / 400)).int - -proc countDays*(yearSpan: int): int = - ## Returns the number of days spanned by a given number of years. - (yearSpan - 1) * 365 + countLeapYears(yearSpan) - -proc countYears*(daySpan: int): int = - ## Returns the number of years spanned by a given number of days. - ((daySpan - countLeapYears(daySpan div 365)) div 365) - -proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] = - ## Returns the number of years spanned by a given number of days and the - ## remainder as days. - let days = daySpan - countLeapYears(daySpan div 365) - result.years = days div 365 - result.days = days mod 365 + ## This procedure is useful for converting `TimeInterval` values to strings. + ## E.g. then you need to implement custom interval printing + runnableExamples: + var tp = toParts(initTimeInterval(years = 1, nanoseconds = 123)) + doAssert tp[Years] == 1 + doAssert tp[Nanoseconds] == 123 + + var index = 0 + for name, value in fieldPairs(ti): + result[index.TimeUnit()] = value + index += 1 + +proc `$`*(ti: TimeInterval): string = + ## Get string representation of `TimeInterval`. + runnableExamples: + doAssert $initTimeInterval(years = 1, nanoseconds = 123) == + "1 year and 123 nanoseconds" + doAssert $initTimeInterval() == "0 nanoseconds" + + var parts: seq[string] = @[] + var tiParts = toParts(ti) + for unit in countdown(Years, Nanoseconds): + if tiParts[unit] != 0: + parts.add(stringifyUnit(tiParts[unit], unit)) + + result = humanizeParts(parts) + +proc nanoseconds*(nanos: int): TimeInterval {.inline.} = + ## TimeInterval of `nanos` nanoseconds. + initTimeInterval(nanoseconds = nanos) + +proc microseconds*(micros: int): TimeInterval {.inline.} = + ## TimeInterval of `micros` microseconds. + initTimeInterval(microseconds = micros) + +proc milliseconds*(ms: int): TimeInterval {.inline.} = + ## TimeInterval of `ms` milliseconds. + initTimeInterval(milliseconds = ms) + +proc seconds*(s: int): TimeInterval {.inline.} = + ## TimeInterval of `s` seconds. + ## + ## `echo getTime() + 5.seconds` + initTimeInterval(seconds = s) -const - secondsInMin = 60 - secondsInHour = 60*60 - secondsInDay = 60*60*24 - epochStartYear = 1970 +proc minutes*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` minutes. + ## + ## `echo getTime() + 5.minutes` + initTimeInterval(minutes = m) -proc getDayOfWeek*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year. - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct - # for the WeekDay type. - if d == 0: return dSun - result = (d-1).WeekDay - -proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, according to the Julian calender. - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 - result = d.WeekDay - -proc timeToTimeInfo*(t: Time): TimeInfo = - ## Converts a Time to TimeInfo. - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - y = yearsSinceEpoch + epochStartYear +proc hours*(h: int): TimeInterval {.inline.} = + ## TimeInterval of `h` hours. + ## + ## `echo getTime() + 2.hours` + initTimeInterval(hours = h) - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, y) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, y) - - let - yd = daysRemaining - m = mon # month is zero indexed enum - md = days - # NB: month is zero indexed but dayOfWeek expects 1 indexed. - wd = getDayOfWeek(days, mon.int + 1, y).Weekday - h = daySeconds div secondsInHour + 1 - mi = (daySeconds mod secondsInHour) div secondsInMin - s = daySeconds mod secondsInMin - result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) - -proc timeToTimeInterval*(t: Time): TimeInterval = - ## Converts a Time to a TimeInterval. - - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - result.years = yearsSinceEpoch + epochStartYear +proc days*(d: int): TimeInterval {.inline.} = + ## TimeInterval of `d` days. + ## + ## `echo getTime() + 2.days` + initTimeInterval(days = d) - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, result.years) +proc weeks*(w: int): TimeInterval {.inline.} = + ## TimeInterval of `w` weeks. + ## + ## `echo getTime() + 2.weeks` + initTimeInterval(weeks = w) - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, result.years) +proc months*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` months. + ## + ## `echo getTime() + 2.months` + initTimeInterval(months = m) - result.months = mon.int + 1 # month is 1 indexed int - result.days = days - result.hours = daySeconds div secondsInHour + 1 - result.minutes = (daySeconds mod secondsInHour) div secondsInMin - result.seconds = daySeconds mod secondsInMin - # Milliseconds not available from Time +proc years*(y: int): TimeInterval {.inline.} = + ## TimeInterval of `y` years. + ## + ## `echo getTime() + 2.years` + initTimeInterval(years = y) + +proc evaluateInterval(dt: DateTime, interval: TimeInterval): + tuple[adjDur, absDur: Duration] = + ## Evaluates how many nanoseconds the interval is worth + ## in the context of `dt`. + ## The result in split into an adjusted diff and an absolute diff. + var months = interval.years * 12 + interval.months + var curYear = dt.year + var curMonth = dt.month + result = default(tuple[adjDur, absDur: Duration]) + # Subtracting + if months < 0: + for mth in countdown(-1 * months, 1): + if curMonth == mJan: + curMonth = mDec + curYear.dec + else: + curMonth.dec() + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur - initDuration(days = days) + # Adding + else: + for mth in 1 .. months: + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur + initDuration(days = days) + if curMonth == mDec: + curMonth = mJan + curYear.inc + else: + curMonth.inc() + + result.adjDur = result.adjDur + initDuration( + days = interval.days, + weeks = interval.weeks) + result.absDur = initDuration( + nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + +proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Adds `interval` to `dt`. Components from `interval` are added + ## in the order of their size, i.e. first the `years` component, then the + ## `months` component and so on. The returned `DateTime` will have the + ## same timezone as the input. + ## + ## Note that when adding months, monthday overflow is allowed. This means that + ## if the resulting month doesn't have enough days it, the month will be + ## incremented and the monthday will be set to the number of days overflowed. + ## So adding one month to `31 October` will result in `31 November`, which + ## will overflow and result in `1 December`. + runnableExamples: + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" + # This is correct and happens due to monthday overflow. + doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" + let (adjDur, absDur) = evaluateInterval(dt, interval) + + if adjDur != DurationZero: + var zt = dt.timezone.zonedTimeFromAdjTime(dt.toAdjTime + adjDur) + if absDur != DurationZero: + zt = dt.timezone.zonedTimeFromTime(zt.time + absDur) + result = initDateTime(zt, dt.timezone) + else: + result = initDateTime(zt, dt.timezone) + else: + var zt = dt.timezone.zonedTimeFromTime(dt.toTime + absDur) + result = initDateTime(zt, dt.timezone) + +proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Subtract `interval` from `dt`. Components from `interval` are + ## subtracted in the order of their size, i.e. first the `years` component, + ## then the `months` component and so on. The returned `DateTime` will + ## have the same timezone as the input. + runnableExamples: + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" + + dt + (-interval) + +proc `+`*(time: Time, interval: TimeInterval): Time = + ## Adds `interval` to `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(0) + doAssert tm + 5.seconds == fromUnix(5) + + if interval.isStaticInterval: + time + evaluateStaticInterval(interval) + else: + toTime(time.local + interval) + +proc `-`*(time: Time, interval: TimeInterval): Time = + ## Subtracts `interval` from Time `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(5) + doAssert tm - 5.seconds == fromUnix(0) + + if interval.isStaticInterval: + time - evaluateStaticInterval(interval) + else: + toTime(time.local - interval) + +proc `+=`*(a: var DateTime, b: TimeInterval) = + a = a + b + +proc `-=`*(a: var DateTime, b: TimeInterval) = + a = a - b + +proc `+=`*(t: var Time, b: TimeInterval) = + t = t + b + +proc `-=`*(t: var Time, b: TimeInterval) = + t = t - b + +# +# Iso week +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year + ## in the specified timezone. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert initDateTime(21, mApr, 2018, 00, 00, 00) == initDateTime(dSat, 16, 2018.IsoYear, 00, 00, 00) + assert initDateTime(30, mDec, 2019, 00, 00, 00) == initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) + assert initDateTime(13, mSep, 2020, 00, 00, 00) == initDateTime(dSun, 37, 2020.IsoYear, 00, 00, 00) + assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) + + # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00, zone).weekday.int - 4 + initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) + +# +# Other +# + +proc epochTime*(): float {.tags: [TimeEffect].} = + ## Gets time after the UNIX epoch (1970) in seconds. It is a float + ## because sub-second resolution is likely to be supported (depending + ## on the hardware/OS). + ## + ## `getTime` should generally be preferred over this proc. + ## + ## .. warning:: Unsuitable for benchmarking (but still better than `now`), + ## use `monotimes.getMonoTime` or `cpuTime` instead, depending on the use case. + when defined(js): + result = newDate().getTime() / 1000 + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = toBiggestFloat(a.tv_sec.int64) + toBiggestFloat( + a.tv_usec)*0.00_0001 + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = toBiggestFloat(ts.tv_sec.int64) + + toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 + elif defined(windows): + var f {.noinit.}: winlean.FILETIME + getSystemTimeAsFileTime(f) + var i64 = rdFileTime(f) - epochDiff + var secs = i64 div rateDiff + var subsecs = i64 mod rateDiff + result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 + else: + {.error: "unknown OS".} + +when not defined(js): + type + Clock {.importc: "clock_t".} = distinct int + + proc getClock(): Clock + {.importc: "clock", header: "<time.h>", tags: [TimeEffect], used, sideEffect.} -when isMainModule: - # this is testing non-exported function var - t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 - t4L = getLocalTime(fromSeconds(876124714)) - assert toSeconds(t4, initInterval(seconds=0)) == 0.0 - assert toSeconds(t4L, initInterval(milliseconds=1)) == toSeconds(t4, initInterval(milliseconds=1)) - assert toSeconds(t4L, initInterval(seconds=1)) == toSeconds(t4, initInterval(seconds=1)) - assert toSeconds(t4L, initInterval(minutes=1)) == toSeconds(t4, initInterval(minutes=1)) - assert toSeconds(t4L, initInterval(hours=1)) == toSeconds(t4, initInterval(hours=1)) - assert toSeconds(t4L, initInterval(days=1)) == toSeconds(t4, initInterval(days=1)) - assert toSeconds(t4L, initInterval(months=1)) == toSeconds(t4, initInterval(months=1)) - assert toSeconds(t4L, initInterval(years=1)) == toSeconds(t4, initInterval(years=1)) - - # Further tests are in tests/stdlib/ttime.nim - # koch test c stdlib + clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl, used.}: int + + proc cpuTime*(): float {.tags: [TimeEffect].} = + ## Gets time spent that the CPU spent to run the current process in + ## seconds. This may be more useful for benchmarking than `epochTime`. + ## However, it may measure the real time instead (depending on the OS). + ## The value of the result has no meaning. + ## To generate useful timing values, take the difference between + ## the results of two `cpuTime` calls: + runnableExamples: + var t0 = cpuTime() + # some useless work here (calculate fibonacci) + var fib = @[0, 1, 1] + for i in 1..10: + fib.add(fib[^1] + fib[^2]) + echo "CPU time [s] ", cpuTime() - t0 + echo "Fib is [s] ", fib + ## When the flag `--benchmarkVM` is passed to the compiler, this proc is + ## also available at compile time + when defined(posix) and not defined(osx) and declared(CLOCK_THREAD_CPUTIME_ID): + # 'clocksPerSec' is a compile-time constant, possibly a + # rather awful one, so use clock_gettime instead + var ts: Timespec + discard clock_gettime(CLOCK_THREAD_CPUTIME_ID, ts) + result = toFloat(ts.tv_sec.int) + + toFloat(ts.tv_nsec.int) / 1_000_000_000 + else: + result = toFloat(int(getClock())) / toFloat(clocksPerSec) + + +# +# Deprecations +# + +proc `nanosecond=`*(dt: var DateTime, value: NanosecondRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.nanosecond = value + +proc `second=`*(dt: var DateTime, value: SecondRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.second = value + +proc `minute=`*(dt: var DateTime, value: MinuteRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.minute = value + +proc `hour=`*(dt: var DateTime, value: HourRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.hour = value + +proc `monthdayZero=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.monthdayZero = value + +proc `monthZero=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.monthZero = value + +proc `year=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.year = value + +proc `weekday=`*(dt: var DateTime, value: WeekDay) {.deprecated: "Deprecated since v1.3.1".} = + dt.weekday = value + +proc `yearday=`*(dt: var DateTime, value: YeardayRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.yearday = value + +proc `isDst=`*(dt: var DateTime, value: bool) {.deprecated: "Deprecated since v1.3.1".} = + dt.isDst = value + +proc `timezone=`*(dt: var DateTime, value: Timezone) {.deprecated: "Deprecated since v1.3.1".} = + dt.timezone = value + +proc `utcOffset=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.utcOffset = value diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 2c3d872df..78af84fdd 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -8,29 +8,369 @@ # ## This module defines compile-time reflection procs for -## working with types +## working with types. +## +## Unstable API. -proc name*(t: typedesc): string {.magic: "TypeTrait".} - ## Returns the name of the given type. - ## - ## Example: +import std/private/since +export system.`$` # for backward compatibility + +when defined(nimPreviewSlimSystem): + import std/assertions + + +type HoleyEnum* = (not Ordinal) and enum ## Enum with holes. +type OrdinalEnum* = Ordinal and enum ## Enum without holes. + +runnableExamples: + type A = enum a0 = 2, a1 = 4, a2 + type B = enum b0 = 2, b1, b2 + assert A is enum + assert A is HoleyEnum + assert A isnot OrdinalEnum + assert B isnot HoleyEnum + assert B is OrdinalEnum + assert int isnot HoleyEnum + type C[T] = enum h0 = 2, h1 = 4 + assert C[float] is HoleyEnum + +proc name*(t: typedesc): string {.magic: "TypeTrait".} = + ## Returns the name of `t`. ## - ## .. code-block:: + ## Alias for `system.\`$\`(t) <dollars.html#$,typedesc>`_ since Nim v0.20. + runnableExamples: + doAssert name(int) == "int" + doAssert name(seq[string]) == "seq[string]" + +proc arity*(t: typedesc): int {.magic: "TypeTrait".} = + ## Returns the arity of `t`. This is the number of "type" + ## components or the number of generic parameters a given type `t` has. + runnableExamples: + doAssert arity(int) == 0 + doAssert arity(seq[string]) == 1 + doAssert arity(array[3, int]) == 2 + doAssert arity((int, int, float, string)) == 4 + +proc genericHead*(t: typedesc): typedesc {.magic: "TypeTrait".} = + ## Accepts an instantiated generic type and returns its + ## uninstantiated form. + ## A compile-time error will be produced if the supplied type + ## is not generic. ## - ## import typetraits + ## **See also:** + ## * `stripGenericParams proc <#stripGenericParams,typedesc>`_ + runnableExamples: + type + Foo[T] = object + FooInst = Foo[int] + Foo2 = genericHead(FooInst) + + doAssert Foo2 is Foo and Foo is Foo2 + doAssert genericHead(Foo[seq[string]]) is Foo + doAssert not compiles(genericHead(int)) + + type Generic = concept f + type _ = genericHead(typeof(f)) + + proc bar(a: Generic): typeof(a) = a + + doAssert bar(Foo[string].default) == Foo[string]() + doAssert not compiles bar(string.default) + + when false: # these don't work yet + doAssert genericHead(Foo[int])[float] is Foo[float] + doAssert seq[int].genericHead is seq + +proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} = + ## This trait is similar to `genericHead <#genericHead,typedesc>`_, but + ## instead of producing an error for non-generic types, it will just return + ## them unmodified. + runnableExamples: + type Foo[T] = object + + doAssert stripGenericParams(Foo[string]) is Foo + doAssert stripGenericParams(int) is int + +proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} + ## Returns true if `t` is safe to use for `copyMem`:idx:. ## - ## proc `$`*[T](some:typedesc[T]): string = name(T) + ## Other languages name a type like these `blob`:idx:. + +proc hasDefaultValue*(t: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true if `t` has a valid default value. + runnableExamples: + {.experimental: "strictNotNil".} + type + NilableObject = ref object + a: int + Object = NilableObject not nil + RequiresInit[T] = object + a {.requiresInit.}: T + + assert hasDefaultValue(NilableObject) + assert not hasDefaultValue(Object) + assert hasDefaultValue(string) + assert not hasDefaultValue(var string) + assert not hasDefaultValue(RequiresInit[int]) + +proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true for named tuples, false for any other type. + runnableExamples: + doAssert not isNamedTuple(int) + doAssert not isNamedTuple((string, int)) + doAssert isNamedTuple(tuple[name: string, age: int]) + +template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = + ## Returns `T` for `ref T | ptr T`. + runnableExamples: + assert (ref int).pointerBase is int + type A = ptr seq[float] + assert A.pointerBase is seq[float] + assert (ref A).pointerBase is A # not seq[float] + assert (var s = "abc"; s[0].addr).typeof.pointerBase is char + T + +proc rangeBase*(T: typedesc[range]): typedesc {.magic: "TypeTrait".} = + ## Returns the base type for range types, or the type itself otherwise. ## - ## template test(x): stmt = - ## echo "type: ", type(x), ", value: ", x + ## **See also:** + ## * `rangeBase template <#rangeBase.t,T>`_ + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + doAssert rangeBase(MyRange) is int + doAssert rangeBase(MyEnumRange) is MyEnum + doAssert rangeBase(range['a'..'z']) is char + +template rangeBase*[T: range](a: T): untyped = + ## Overload of `rangeBase <#rangeBase,typedesc,static[bool]>`_ for values. + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + let x = MyRange(3) + doAssert rangeBase(x) is int + doAssert $typeof(rangeBase(x)) == "int" + doAssert rangeBase(x) == 3 + let y: set[MyEnumRange] = {c} + for e in y: + doAssert rangeBase(e) is MyEnum + doAssert $typeof(rangeBase(e)) == "MyEnum" + doAssert rangeBase(e) == c + let z: seq[range['a'..'z']] = @['c'] + doAssert rangeBase(z[0]) is char + doAssert $typeof(rangeBase(z[0])) == "char" + doAssert rangeBase(z[0]) == 'c' + rangeBase(typeof(T))(a) + +proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = + ## Returns the base type for distinct types, or the type itself otherwise. + ## If `recursive` is false, only the immediate distinct base will be returned. ## - ## test 42 - ## # --> type: int, value: 42 - ## test "Foo" - ## # --> type: string, value: Foo - ## test(@['A','B']) - ## # --> type: seq[char], value: @[A, B] + ## **See also:** + ## * `distinctBase template <#distinctBase.t,T,static[bool]>`_ + runnableExamples: + type MyInt = distinct int + type MyOtherInt = distinct MyInt + doAssert distinctBase(MyInt) is int + doAssert distinctBase(MyOtherInt) is int + doAssert distinctBase(MyOtherInt, false) is MyInt + doAssert distinctBase(int) is int + +since (1, 1): + template distinctBase*[T](a: T, recursive: static bool = true): untyped = + ## Overload of `distinctBase <#distinctBase,typedesc,static[bool]>`_ for values. + runnableExamples: + type MyInt = distinct int + type MyOtherInt = distinct MyInt + doAssert 12.MyInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase(false) is MyInt + doAssert 12.distinctBase == 12 + when T is distinct: + distinctBase(typeof(a), recursive)(a) + else: # avoids hint ConvFromXtoItselfNotNeeded + a + + proc tupleLen*(T: typedesc[tuple]): int {.magic: "TypeTrait".} = + ## Returns the number of elements of the tuple type `T`. + ## + ## **See also:** + ## * `tupleLen template <#tupleLen.t>`_ + runnableExamples: + doAssert tupleLen((int, int, float, string)) == 4 + doAssert tupleLen(tuple[name: string, age: int]) == 2 + + template tupleLen*(t: tuple): int = + ## Returns the number of elements of the tuple `t`. + ## + ## **See also:** + ## * `tupleLen proc <#tupleLen,typedesc>`_ + runnableExamples: + doAssert tupleLen((1, 2)) == 2 + + tupleLen(typeof(t)) + + template get*(T: typedesc[tuple], i: static int): untyped = + ## Returns the `i`-th element of `T`. + # Note: `[]` currently gives: `Error: no generic parameters allowed for ...` + runnableExamples: + doAssert get((int, int, float, string), 2) is float + + typeof(default(T)[i]) + + type StaticParam*[value: static type] = object + ## Used to wrap a static value in `genericParams <#genericParams.t,typedesc>`_. + +since (1, 3, 5): + template elementType*(a: untyped): typedesc = + ## Returns the element type of `a`, which can be any iterable (over which you + ## can iterate). + runnableExamples: + iterator myiter(n: int): auto = + for i in 0 ..< n: + yield i + + doAssert elementType(@[1,2]) is int + doAssert elementType("asdf") is char + doAssert elementType(myiter(3)) is int + + typeof(block: (for ai in a: ai)) + +import std/macros + +macro enumLen*(T: typedesc[enum]): int = + ## Returns the number of items in the enum `T`. + runnableExamples: + type Foo = enum + fooItem1 + fooItem2 + + doAssert Foo.enumLen == 2 + + let bracketExpr = getType(T) + expectKind(bracketExpr, nnkBracketExpr) + let enumTy = bracketExpr[1] + expectKind(enumTy, nnkEnumTy) + result = newLit(enumTy.len - 1) + +macro genericParamsImpl(T: typedesc): untyped = + # auxiliary macro needed, can't do it directly in `genericParams` + result = newNimNode(nnkTupleConstr) + var impl = getTypeImpl(T) + expectKind(impl, nnkBracketExpr) + impl = impl[1] + while true: + case impl.kind + of nnkSym: + impl = impl.getImpl + of nnkTypeDef: + impl = impl[2] + of nnkTypeOfExpr: + impl = getTypeInst(impl[0]) + of nnkBracketExpr: + for i in 1..<impl.len: + let ai = impl[i] + var ret: NimNode = nil + case ai.typeKind + of ntyTypeDesc: + ret = ai + of ntyStatic: raiseAssert "unreachable" + else: + # getType from a resolved symbol might return a typedesc symbol. + # If so, use it directly instead of wrapping it in StaticParam. + if (ai.kind == nnkSym and ai.symKind == nskType) or + (ai.kind == nnkBracketExpr and ai[0].kind == nnkSym and + ai[0].symKind == nskType) or ai.kind in {nnkRefTy, nnkVarTy, nnkPtrTy, nnkProcTy}: + ret = ai + elif ai.kind == nnkInfix and ai[0].kind == nnkIdent and + ai[0].strVal == "..": + # For built-in array types, the "2" is translated to "0..1" then + # automagically translated to "range[0..1]". However this is not + # reflected in the AST, thus requiring manual transformation here. + # + # We will also be losing some context here: + # var a: array[10, int] + # will be translated to: + # var a: array[0..9, int] + # after typecheck. This means that we can't get the exact + # definition as typed by the user, which will cause confusion for + # users expecting: + # genericParams(typeof(a)) is (StaticParam(10), int) + # to be true while in fact the result will be: + # genericParams(typeof(a)) is (range[0..9], int) + ret = newTree(nnkBracketExpr, @[bindSym"range", ai]) + else: + since (1, 1): + ret = newTree(nnkBracketExpr, @[bindSym"StaticParam", ai]) + result.add ret + break + else: + error "wrong kind: " & $impl.kind, impl + +since (1, 1): + template genericParams*(T: typedesc): untyped = + ## Returns the tuple of generic parameters for the generic type `T`. + ## + ## **Note:** For the builtin array type, the index generic parameter will + ## **always** become a range type after it's bound to a variable. + runnableExamples: + type Foo[T1, T2] = object + + doAssert genericParams(Foo[float, string]) is (float, string) + + type Bar[N: static float, T] = object + + doAssert genericParams(Bar[1.0, string]) is (StaticParam[1.0], string) + doAssert genericParams(Bar[1.0, string]).get(0).value == 1.0 + doAssert genericParams(seq[Bar[2.0, string]]).get(0) is Bar[2.0, string] + var s: seq[Bar[3.0, string]] + doAssert genericParams(typeof(s)) is (Bar[3.0, string],) + + doAssert genericParams(array[10, int]) is (StaticParam[10], int) + var a: array[10, int] + doAssert genericParams(typeof(a)) is (range[0..9], int) + + type T2 = T + genericParamsImpl(T2) + + +proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = + ## Returns true if the func/proc/etc `fn` has `closure`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasClosureImpl(fn) +template toUnsigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns an unsigned type with same bit size as `T`. + runnableExamples: + assert int8.toUnsigned is uint8 + assert uint.toUnsigned is uint + assert int.toUnsigned is uint + # range types are currently unsupported: + assert not compiles(toUnsigned(range[0..7])) + when T is int8: uint8 + elif T is int16: uint16 + elif T is int32: uint32 + elif T is int64: uint64 + elif T is int: uint + else: T -proc arity*(t: typedesc): int {.magic: "TypeTrait".} - ## Returns the arity of the given type +template toSigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns a signed type with same bit size as `T`. + runnableExamples: + assert int8.toSigned is int8 + assert uint16.toSigned is int16 + # range types are currently unsupported: + assert not compiles(toSigned(range[0..7])) + when T is uint8: int8 + elif T is uint16: int16 + elif T is uint32: int32 + elif T is uint64: int64 + elif T is uint: int + else: T diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 45f52eb7f..8cbe117bb 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -8,1147 +8,463 @@ # ## This module provides support to handle the Unicode UTF-8 encoding. - -{.deadCodeElim: on.} +## +## There are no specialized ``insert``, ``delete``, ``add`` and ``contains`` +## procedures for ``seq[Rune]`` in this module because the generic variants +## of these procedures in the system module already work with it. +## +## The current version is compatible with Unicode v12.0.0. +## +## **See also:** +## * `strutils module <strutils.html>`_ +## * `unidecode module <unidecode.html>`_ +## * `encodings module <encodings.html>`_ include "system/inclrtl" +import std/strbasics +template toOa(s: string): auto = s.toOpenArray(0, s.high) -type - RuneImpl = int # underlying type of Rune - Rune* = distinct RuneImpl ## type that can hold any Unicode character - Rune16* = distinct int16 ## 16 bit Unicode character +proc substr(s: openArray[char] , first, last: int): string = + # Copied substr from system + let first = max(first, 0) + let L = max(min(last, high(s)) - first + 1, 0) + result = newString(L) + for i in 0 .. L-1: + result[i] = s[i+first] -{.deprecated: [TRune: Rune, TRune16: Rune16].} +type + RuneImpl = int32 # underlying type of Rune + Rune* = distinct RuneImpl ## \ + ## Type that can hold a single Unicode code point. + ## + ## A Rune may be composed with other Runes to a character on the screen. + ## `RuneImpl` is the underlying type used to store Runes, currently `int32`. -proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b) -proc `<%`*(a, b: Rune): bool = return int(a) <% int(b) -proc `==`*(a, b: Rune): bool = return int(a) == int(b) +template ones(n: untyped): untyped = ((1 shl n)-1) -template ones(n: expr): expr = ((1 shl n)-1) +proc runeLen*(s: openArray[char]): int {.rtl, extern: "nuc$1".} = + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 -proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = - ## Returns the number of Unicode characters of the string ``s`` + result = 0 var i = 0 while i < len(s): - if ord(s[i]) <=% 127: inc(i) - elif ord(s[i]) shr 5 == 0b110: inc(i, 2) - elif ord(s[i]) shr 4 == 0b1110: inc(i, 3) - elif ord(s[i]) shr 3 == 0b11110: inc(i, 4) - elif ord(s[i]) shr 2 == 0b111110: inc(i, 5) - elif ord(s[i]) shr 1 == 0b1111110: inc(i, 6) + if uint(s[i]) <= 127: inc(i) + elif uint(s[i]) shr 5 == 0b110: inc(i, 2) + elif uint(s[i]) shr 4 == 0b1110: inc(i, 3) + elif uint(s[i]) shr 3 == 0b11110: inc(i, 4) + elif uint(s[i]) shr 2 == 0b111110: inc(i, 5) + elif uint(s[i]) shr 1 == 0b1111110: inc(i, 6) else: inc i inc(result) -proc runeLenAt*(s: string, i: Natural): int = - ## Returns the number of bytes the rune starting at ``s[i]`` takes - if ord(s[i]) <=% 127: result = 1 - elif ord(s[i]) shr 5 == 0b110: result = 2 - elif ord(s[i]) shr 4 == 0b1110: result = 3 - elif ord(s[i]) shr 3 == 0b11110: result = 4 - elif ord(s[i]) shr 2 == 0b111110: result = 5 - elif ord(s[i]) shr 1 == 0b1111110: result = 6 +proc runeLenAt*(s: openArray[char], i: Natural): int = + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + ## + ## See also: + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + + if uint(s[i]) <= 127: result = 1 + elif uint(s[i]) shr 5 == 0b110: result = 2 + elif uint(s[i]) shr 4 == 0b1110: result = 3 + elif uint(s[i]) shr 3 == 0b11110: result = 4 + elif uint(s[i]) shr 2 == 0b111110: result = 5 + elif uint(s[i]) shr 1 == 0b1111110: result = 6 else: result = 1 -template fastRuneAt*(s: string, i: int, result: expr, doInc = true) = - ## Returns the Unicode character ``s[i]`` in ``result``. If ``doInc == true`` - ## ``i`` is incremented by the number of bytes that have been processed. +const replRune = Rune(0xFFFD) + +template fastRuneAt*(s: openArray[char] or string, i: int, result: untyped, doInc = true) = + ## Returns the rune ``s[i]`` in ``result``. + ## + ## If ``doInc == true`` (default), ``i`` is incremented by the number + ## of bytes that have been processed. bind ones - if ord(s[i]) <=% 127: - result = Rune(ord(s[i])) + if uint(s[i]) <= 127: + result = Rune(uint(s[i])) when doInc: inc(i) - elif ord(s[i]) shr 5 == 0b110: - # assert(ord(s[i+1]) shr 6 == 0b10) - result = Rune((ord(s[i]) and (ones(5))) shl 6 or - (ord(s[i+1]) and ones(6))) - when doInc: inc(i, 2) - elif ord(s[i]) shr 4 == 0b1110: - # assert(ord(s[i+1]) shr 6 == 0b10) - # assert(ord(s[i+2]) shr 6 == 0b10) - result = Rune((ord(s[i]) and ones(4)) shl 12 or - (ord(s[i+1]) and ones(6)) shl 6 or - (ord(s[i+2]) and ones(6))) - when doInc: inc(i, 3) - elif ord(s[i]) shr 3 == 0b11110: - # assert(ord(s[i+1]) shr 6 == 0b10) - # assert(ord(s[i+2]) shr 6 == 0b10) - # assert(ord(s[i+3]) shr 6 == 0b10) - result = Rune((ord(s[i]) and ones(3)) shl 18 or - (ord(s[i+1]) and ones(6)) shl 12 or - (ord(s[i+2]) and ones(6)) shl 6 or - (ord(s[i+3]) and ones(6))) - when doInc: inc(i, 4) - elif ord(s[i]) shr 2 == 0b111110: - # assert(ord(s[i+1]) shr 6 == 0b10) - # assert(ord(s[i+2]) shr 6 == 0b10) - # assert(ord(s[i+3]) shr 6 == 0b10) - # assert(ord(s[i+4]) shr 6 == 0b10) - result = Rune((ord(s[i]) and ones(2)) shl 24 or - (ord(s[i+1]) and ones(6)) shl 18 or - (ord(s[i+2]) and ones(6)) shl 12 or - (ord(s[i+3]) and ones(6)) shl 6 or - (ord(s[i+4]) and ones(6))) - when doInc: inc(i, 5) - elif ord(s[i]) shr 1 == 0b1111110: - # assert(ord(s[i+1]) shr 6 == 0b10) - # assert(ord(s[i+2]) shr 6 == 0b10) - # assert(ord(s[i+3]) shr 6 == 0b10) - # assert(ord(s[i+4]) shr 6 == 0b10) - # assert(ord(s[i+5]) shr 6 == 0b10) - result = Rune((ord(s[i]) and ones(1)) shl 30 or - (ord(s[i+1]) and ones(6)) shl 24 or - (ord(s[i+2]) and ones(6)) shl 18 or - (ord(s[i+3]) and ones(6)) shl 12 or - (ord(s[i+4]) and ones(6)) shl 6 or - (ord(s[i+5]) and ones(6))) - when doInc: inc(i, 6) + elif uint(s[i]) shr 5 == 0b110: + # assert(uint(s[i+1]) shr 6 == 0b10) + if i <= s.len - 2: + result = Rune((uint(s[i]) and (ones(5))) shl 6 or + (uint(s[i+1]) and ones(6))) + when doInc: inc(i, 2) + else: + result = replRune + when doInc: inc(i) + elif uint(s[i]) shr 4 == 0b1110: + # assert(uint(s[i+1]) shr 6 == 0b10) + # assert(uint(s[i+2]) shr 6 == 0b10) + if i <= s.len - 3: + result = Rune((uint(s[i]) and ones(4)) shl 12 or + (uint(s[i+1]) and ones(6)) shl 6 or + (uint(s[i+2]) and ones(6))) + when doInc: inc(i, 3) + else: + result = replRune + when doInc: inc(i) + elif uint(s[i]) shr 3 == 0b11110: + # assert(uint(s[i+1]) shr 6 == 0b10) + # assert(uint(s[i+2]) shr 6 == 0b10) + # assert(uint(s[i+3]) shr 6 == 0b10) + if i <= s.len - 4: + result = Rune((uint(s[i]) and ones(3)) shl 18 or + (uint(s[i+1]) and ones(6)) shl 12 or + (uint(s[i+2]) and ones(6)) shl 6 or + (uint(s[i+3]) and ones(6))) + when doInc: inc(i, 4) + else: + result = replRune + when doInc: inc(i) + elif uint(s[i]) shr 2 == 0b111110: + # assert(uint(s[i+1]) shr 6 == 0b10) + # assert(uint(s[i+2]) shr 6 == 0b10) + # assert(uint(s[i+3]) shr 6 == 0b10) + # assert(uint(s[i+4]) shr 6 == 0b10) + if i <= s.len - 5: + result = Rune((uint(s[i]) and ones(2)) shl 24 or + (uint(s[i+1]) and ones(6)) shl 18 or + (uint(s[i+2]) and ones(6)) shl 12 or + (uint(s[i+3]) and ones(6)) shl 6 or + (uint(s[i+4]) and ones(6))) + when doInc: inc(i, 5) + else: + result = replRune + when doInc: inc(i) + elif uint(s[i]) shr 1 == 0b1111110: + # assert(uint(s[i+1]) shr 6 == 0b10) + # assert(uint(s[i+2]) shr 6 == 0b10) + # assert(uint(s[i+3]) shr 6 == 0b10) + # assert(uint(s[i+4]) shr 6 == 0b10) + # assert(uint(s[i+5]) shr 6 == 0b10) + if i <= s.len - 6: + result = Rune((uint(s[i]) and ones(1)) shl 30 or + (uint(s[i+1]) and ones(6)) shl 24 or + (uint(s[i+2]) and ones(6)) shl 18 or + (uint(s[i+3]) and ones(6)) shl 12 or + (uint(s[i+4]) and ones(6)) shl 6 or + (uint(s[i+5]) and ones(6))) + when doInc: inc(i, 6) + else: + result = replRune + when doInc: inc(i) else: - result = Rune(ord(s[i])) + result = Rune(uint(s[i])) when doInc: inc(i) -proc validateUtf8*(s: string): int = +proc runeAt*(s: openArray[char], i: Natural): Rune = + ## Returns the rune in ``s`` at **byte index** ``i``. + ## + ## See also: + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) + fastRuneAt(s, i, result, false) + +proc validateUtf8*(s: openArray[char]): int = ## Returns the position of the invalid byte in ``s`` if the string ``s`` does ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. + ## + ## See also: + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ var i = 0 let L = s.len while i < L: - if ord(s[i]) <=% 127: + if uint(s[i]) <= 127: inc(i) - elif ord(s[i]) shr 5 == 0b110: - if ord(s[i]) < 0xc2: return i # Catch overlong ascii representations. - if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2) + elif uint(s[i]) shr 5 == 0b110: + if uint(s[i]) < 0xc2: return i # Catch overlong ascii representations. + if i+1 < L and uint(s[i+1]) shr 6 == 0b10: inc(i, 2) else: return i - elif ord(s[i]) shr 4 == 0b1110: - if i+2 < L and ord(s[i+1]) shr 6 == 0b10 and ord(s[i+2]) shr 6 == 0b10: + elif uint(s[i]) shr 4 == 0b1110: + if i+2 < L and uint(s[i+1]) shr 6 == 0b10 and uint(s[i+2]) shr 6 == 0b10: inc i, 3 else: return i - elif ord(s[i]) shr 3 == 0b11110: - if i+3 < L and ord(s[i+1]) shr 6 == 0b10 and - ord(s[i+2]) shr 6 == 0b10 and - ord(s[i+3]) shr 6 == 0b10: + elif uint(s[i]) shr 3 == 0b11110: + if i+3 < L and uint(s[i+1]) shr 6 == 0b10 and + uint(s[i+2]) shr 6 == 0b10 and + uint(s[i+3]) shr 6 == 0b10: inc i, 4 else: return i else: return i return -1 -proc runeAt*(s: string, i: Natural): Rune = - ## Returns the unicode character in ``s`` at byte index ``i`` - fastRuneAt(s, i, result, false) - -proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = - ## Converts a rune into its UTF-8 representation +template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) = + ## Copies UTF-8 representation of ``c`` into the preallocated string ``s`` + ## starting at position ``pos``. + ## + ## If ``doInc == true`` (default), ``pos`` is incremented + ## by the number of bytes that have been processed. + ## + ## To be the most efficient, make sure ``s`` is preallocated + ## with an additional amount equal to the byte length of ``c``. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` var i = RuneImpl(c) if i <=% 127: - result = newString(1) - result[0] = chr(i) + s.setLen(pos+1) + s[pos+0] = chr(i) + when doInc: inc(pos) elif i <=% 0x07FF: - result = newString(2) - result[0] = chr((i shr 6) or 0b110_00000) - result[1] = chr((i and ones(6)) or 0b10_0000_00) + s.setLen(pos+2) + s[pos+0] = chr((i shr 6) or 0b110_00000) + s[pos+1] = chr((i and ones(6)) or 0b10_0000_00) + when doInc: inc(pos, 2) elif i <=% 0xFFFF: - result = newString(3) - result[0] = chr(i shr 12 or 0b1110_0000) - result[1] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[2] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+3) + s[pos+0] = chr(i shr 12 or 0b1110_0000) + s[pos+1] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 3) elif i <=% 0x001FFFFF: - result = newString(4) - result[0] = chr(i shr 18 or 0b1111_0000) - result[1] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[3] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+4) + s[pos+0] = chr(i shr 18 or 0b1111_0000) + s[pos+1] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 4) elif i <=% 0x03FFFFFF: - result = newString(5) - result[0] = chr(i shr 24 or 0b111110_00) - result[1] = chr(i shr 18 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[3] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[4] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+5) + s[pos+0] = chr(i shr 24 or 0b111110_00) + s[pos+1] = chr(i shr 18 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+4] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 5) elif i <=% 0x7FFFFFFF: - result = newString(6) - result[0] = chr(i shr 30 or 0b1111110_0) - result[1] = chr(i shr 24 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 18 and ones(6) or 0b10_0000_00) - result[3] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[4] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[5] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+6) + s[pos+0] = chr(i shr 30 or 0b1111110_0) + s[pos+1] = chr(i shr 24 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 18 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+5] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 6) else: discard # error, exception? +proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = + ## Converts a rune into its UTF-8 representation. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `utf8 iterator <#utf8.i,string>`_ + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1).toUTF8 == "ñ" + + result = "" + fastToUTF8Copy(c, result, 0, false) + +proc add*(s: var string; c: Rune) = + ## Adds a rune ``c`` to a string ``s``. + runnableExamples: + var s = "abc" + let c = "ä".runeAt(0) + s.add(c) + doAssert s == "abcä" + + let pos = s.len + fastToUTF8Copy(c, s, pos, false) + proc `$`*(rune: Rune): string = - ## Converts a Rune to a string + ## An alias for `toUTF8 <#toUTF8,Rune>`_. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ rune.toUTF8 proc `$`*(runes: seq[Rune]): string = - ## Converts a sequence of Runes to a string + ## Converts a sequence of Runes to a string. + ## + ## See also: + ## * `toRunes <#toRunes,string>`_ for a reverse operation + runnableExamples: + let + someString = "öÑ" + someRunes = toRunes(someString) + doAssert $someRunes == someString + result = "" - for rune in runes: result.add(rune.toUTF8) - -const - alphaRanges = [ - 0x00d8, 0x00f6, # - - 0x00f8, 0x01f5, # - - 0x0250, 0x02a8, # - - 0x038e, 0x03a1, # - - 0x03a3, 0x03ce, # - - 0x03d0, 0x03d6, # - - 0x03e2, 0x03f3, # - - 0x0490, 0x04c4, # - - 0x0561, 0x0587, # - - 0x05d0, 0x05ea, # - - 0x05f0, 0x05f2, # - - 0x0621, 0x063a, # - - 0x0640, 0x064a, # - - 0x0671, 0x06b7, # - - 0x06ba, 0x06be, # - - 0x06c0, 0x06ce, # - - 0x06d0, 0x06d3, # - - 0x0905, 0x0939, # - - 0x0958, 0x0961, # - - 0x0985, 0x098c, # - - 0x098f, 0x0990, # - - 0x0993, 0x09a8, # - - 0x09aa, 0x09b0, # - - 0x09b6, 0x09b9, # - - 0x09dc, 0x09dd, # - - 0x09df, 0x09e1, # - - 0x09f0, 0x09f1, # - - 0x0a05, 0x0a0a, # - - 0x0a0f, 0x0a10, # - - 0x0a13, 0x0a28, # - - 0x0a2a, 0x0a30, # - - 0x0a32, 0x0a33, # - - 0x0a35, 0x0a36, # - - 0x0a38, 0x0a39, # - - 0x0a59, 0x0a5c, # - - 0x0a85, 0x0a8b, # - - 0x0a8f, 0x0a91, # - - 0x0a93, 0x0aa8, # - - 0x0aaa, 0x0ab0, # - - 0x0ab2, 0x0ab3, # - - 0x0ab5, 0x0ab9, # - - 0x0b05, 0x0b0c, # - - 0x0b0f, 0x0b10, # - - 0x0b13, 0x0b28, # - - 0x0b2a, 0x0b30, # - - 0x0b32, 0x0b33, # - - 0x0b36, 0x0b39, # - - 0x0b5c, 0x0b5d, # - - 0x0b5f, 0x0b61, # - - 0x0b85, 0x0b8a, # - - 0x0b8e, 0x0b90, # - - 0x0b92, 0x0b95, # - - 0x0b99, 0x0b9a, # - - 0x0b9e, 0x0b9f, # - - 0x0ba3, 0x0ba4, # - - 0x0ba8, 0x0baa, # - - 0x0bae, 0x0bb5, # - - 0x0bb7, 0x0bb9, # - - 0x0c05, 0x0c0c, # - - 0x0c0e, 0x0c10, # - - 0x0c12, 0x0c28, # - - 0x0c2a, 0x0c33, # - - 0x0c35, 0x0c39, # - - 0x0c60, 0x0c61, # - - 0x0c85, 0x0c8c, # - - 0x0c8e, 0x0c90, # - - 0x0c92, 0x0ca8, # - - 0x0caa, 0x0cb3, # - - 0x0cb5, 0x0cb9, # - - 0x0ce0, 0x0ce1, # - - 0x0d05, 0x0d0c, # - - 0x0d0e, 0x0d10, # - - 0x0d12, 0x0d28, # - - 0x0d2a, 0x0d39, # - - 0x0d60, 0x0d61, # - - 0x0e01, 0x0e30, # - - 0x0e32, 0x0e33, # - - 0x0e40, 0x0e46, # - - 0x0e5a, 0x0e5b, # - - 0x0e81, 0x0e82, # - - 0x0e87, 0x0e88, # - - 0x0e94, 0x0e97, # - - 0x0e99, 0x0e9f, # - - 0x0ea1, 0x0ea3, # - - 0x0eaa, 0x0eab, # - - 0x0ead, 0x0eae, # - - 0x0eb2, 0x0eb3, # - - 0x0ec0, 0x0ec4, # - - 0x0edc, 0x0edd, # - - 0x0f18, 0x0f19, # - - 0x0f40, 0x0f47, # - - 0x0f49, 0x0f69, # - - 0x10d0, 0x10f6, # - - 0x1100, 0x1159, # - - 0x115f, 0x11a2, # - - 0x11a8, 0x11f9, # - - 0x1e00, 0x1e9b, # - - 0x1f50, 0x1f57, # - - 0x1f80, 0x1fb4, # - - 0x1fb6, 0x1fbc, # - - 0x1fc2, 0x1fc4, # - - 0x1fc6, 0x1fcc, # - - 0x1fd0, 0x1fd3, # - - 0x1fd6, 0x1fdb, # - - 0x1fe0, 0x1fec, # - - 0x1ff2, 0x1ff4, # - - 0x1ff6, 0x1ffc, # - - 0x210a, 0x2113, # - - 0x2115, 0x211d, # - - 0x2120, 0x2122, # - - 0x212a, 0x2131, # - - 0x2133, 0x2138, # - - 0x3041, 0x3094, # - - 0x30a1, 0x30fa, # - - 0x3105, 0x312c, # - - 0x3131, 0x318e, # - - 0x3192, 0x319f, # - - 0x3260, 0x327b, # - - 0x328a, 0x32b0, # - - 0x32d0, 0x32fe, # - - 0x3300, 0x3357, # - - 0x3371, 0x3376, # - - 0x337b, 0x3394, # - - 0x3399, 0x339e, # - - 0x33a9, 0x33ad, # - - 0x33b0, 0x33c1, # - - 0x33c3, 0x33c5, # - - 0x33c7, 0x33d7, # - - 0x33d9, 0x33dd, # - - 0x4e00, 0x9fff, # - - 0xac00, 0xd7a3, # - - 0xf900, 0xfb06, # - - 0xfb13, 0xfb17, # - - 0xfb1f, 0xfb28, # - - 0xfb2a, 0xfb36, # - - 0xfb38, 0xfb3c, # - - 0xfb40, 0xfb41, # - - 0xfb43, 0xfb44, # - - 0xfb46, 0xfbb1, # - - 0xfbd3, 0xfd3d, # - - 0xfd50, 0xfd8f, # - - 0xfd92, 0xfdc7, # - - 0xfdf0, 0xfdf9, # - - 0xfe70, 0xfe72, # - - 0xfe76, 0xfefc, # - - 0xff66, 0xff6f, # - - 0xff71, 0xff9d, # - - 0xffa0, 0xffbe, # - - 0xffc2, 0xffc7, # - - 0xffca, 0xffcf, # - - 0xffd2, 0xffd7, # - - 0xffda, 0xffdc] # - - - alphaSinglets = [ - 0x00aa, # - 0x00b5, # - 0x00ba, # - 0x03da, # - 0x03dc, # - 0x03de, # - 0x03e0, # - 0x06d5, # - 0x09b2, # - 0x0a5e, # - 0x0a8d, # - 0x0ae0, # - 0x0b9c, # - 0x0cde, # - 0x0e4f, # - 0x0e84, # - 0x0e8a, # - 0x0e8d, # - 0x0ea5, # - 0x0ea7, # - 0x0eb0, # - 0x0ebd, # - 0x1fbe, # - 0x207f, # - 0x20a8, # - 0x2102, # - 0x2107, # - 0x2124, # - 0x2126, # - 0x2128, # - 0xfb3e, # - 0xfe74] # - - spaceRanges = [ - 0x0009, 0x000d, # tab and newline - 0x0020, 0x0020, # space - 0x0085, 0x0085, # next line - 0x00a0, 0x00a0, # - 0x1680, 0x1680, # Ogham space mark - 0x2000, 0x200b, # en dash .. zero-width space - 0x200e, 0x200f, # LTR mark .. RTL mark (pattern whitespace) - 0x2028, 0x2029, # - 0x3000, 0x3000, # - 0x202f, 0x202f, # narrow no-break space - 0x205f, 0x205f, # medium mathematical space - 0x3000, 0x3000, # ideographic space - 0xfeff, 0xfeff] # - - toupperRanges = [ - 0x0061, 0x007a, 468, # a-z A-Z - 0x00e0, 0x00f6, 468, # - - - 0x00f8, 0x00fe, 468, # - - - 0x0256, 0x0257, 295, # - - - 0x0258, 0x0259, 298, # - - - 0x028a, 0x028b, 283, # - - - 0x03ad, 0x03af, 463, # - - - 0x03b1, 0x03c1, 468, # - - - 0x03c3, 0x03cb, 468, # - - - 0x03cd, 0x03ce, 437, # - - - 0x0430, 0x044f, 468, # - - - 0x0451, 0x045c, 420, # - - - 0x045e, 0x045f, 420, # - - - 0x0561, 0x0586, 452, # - - - 0x1f00, 0x1f07, 508, # - - - 0x1f10, 0x1f15, 508, # - - - 0x1f20, 0x1f27, 508, # - - - 0x1f30, 0x1f37, 508, # - - - 0x1f40, 0x1f45, 508, # - - - 0x1f60, 0x1f67, 508, # - - - 0x1f70, 0x1f71, 574, # - - - 0x1f72, 0x1f75, 586, # - - - 0x1f76, 0x1f77, 600, # - - - 0x1f78, 0x1f79, 628, # - - - 0x1f7a, 0x1f7b, 612, # - - - 0x1f7c, 0x1f7d, 626, # - - - 0x1f80, 0x1f87, 508, # - - - 0x1f90, 0x1f97, 508, # - - - 0x1fa0, 0x1fa7, 508, # - - - 0x1fb0, 0x1fb1, 508, # - - - 0x1fd0, 0x1fd1, 508, # - - - 0x1fe0, 0x1fe1, 508, # - - - 0x2170, 0x217f, 484, # - - - 0x24d0, 0x24e9, 474, # - - - 0xff41, 0xff5a, 468] # - - - - toupperSinglets = [ - 0x00ff, 621, # - 0x0101, 499, # - 0x0103, 499, # - 0x0105, 499, # - 0x0107, 499, # - 0x0109, 499, # - 0x010b, 499, # - 0x010d, 499, # - 0x010f, 499, # - 0x0111, 499, # - 0x0113, 499, # - 0x0115, 499, # - 0x0117, 499, # - 0x0119, 499, # - 0x011b, 499, # - 0x011d, 499, # - 0x011f, 499, # - 0x0121, 499, # - 0x0123, 499, # - 0x0125, 499, # - 0x0127, 499, # - 0x0129, 499, # - 0x012b, 499, # - 0x012d, 499, # - 0x012f, 499, # - 0x0131, 268, # I - 0x0133, 499, # - 0x0135, 499, # - 0x0137, 499, # - 0x013a, 499, # - 0x013c, 499, # - 0x013e, 499, # - 0x0140, 499, # - 0x0142, 499, # - 0x0144, 499, # - 0x0146, 499, # - 0x0148, 499, # - 0x014b, 499, # - 0x014d, 499, # - 0x014f, 499, # - 0x0151, 499, # - 0x0153, 499, # - 0x0155, 499, # - 0x0157, 499, # - 0x0159, 499, # - 0x015b, 499, # - 0x015d, 499, # - 0x015f, 499, # - 0x0161, 499, # - 0x0163, 499, # - 0x0165, 499, # - 0x0167, 499, # - 0x0169, 499, # - 0x016b, 499, # - 0x016d, 499, # - 0x016f, 499, # - 0x0171, 499, # - 0x0173, 499, # - 0x0175, 499, # - 0x0177, 499, # - 0x017a, 499, # - 0x017c, 499, # - 0x017e, 499, # - 0x017f, 200, # S - 0x0183, 499, # - 0x0185, 499, # - 0x0188, 499, # - 0x018c, 499, # - 0x0192, 499, # - 0x0199, 499, # - 0x01a1, 499, # - 0x01a3, 499, # - 0x01a5, 499, # - 0x01a8, 499, # - 0x01ad, 499, # - 0x01b0, 499, # - 0x01b4, 499, # - 0x01b6, 499, # - 0x01b9, 499, # - 0x01bd, 499, # - 0x01c5, 499, # - 0x01c6, 498, # - 0x01c8, 499, # - 0x01c9, 498, # - 0x01cb, 499, # - 0x01cc, 498, # - 0x01ce, 499, # - 0x01d0, 499, # - 0x01d2, 499, # - 0x01d4, 499, # - 0x01d6, 499, # - 0x01d8, 499, # - 0x01da, 499, # - 0x01dc, 499, # - 0x01df, 499, # - 0x01e1, 499, # - 0x01e3, 499, # - 0x01e5, 499, # - 0x01e7, 499, # - 0x01e9, 499, # - 0x01eb, 499, # - 0x01ed, 499, # - 0x01ef, 499, # - 0x01f2, 499, # - 0x01f3, 498, # - 0x01f5, 499, # - 0x01fb, 499, # - 0x01fd, 499, # - 0x01ff, 499, # - 0x0201, 499, # - 0x0203, 499, # - 0x0205, 499, # - 0x0207, 499, # - 0x0209, 499, # - 0x020b, 499, # - 0x020d, 499, # - 0x020f, 499, # - 0x0211, 499, # - 0x0213, 499, # - 0x0215, 499, # - 0x0217, 499, # - 0x0253, 290, # - 0x0254, 294, # - 0x025b, 297, # - 0x0260, 295, # - 0x0263, 293, # - 0x0268, 291, # - 0x0269, 289, # - 0x026f, 289, # - 0x0272, 287, # - 0x0283, 282, # - 0x0288, 282, # - 0x0292, 281, # - 0x03ac, 462, # - 0x03cc, 436, # - 0x03d0, 438, # - 0x03d1, 443, # - 0x03d5, 453, # - 0x03d6, 446, # - 0x03e3, 499, # - 0x03e5, 499, # - 0x03e7, 499, # - 0x03e9, 499, # - 0x03eb, 499, # - 0x03ed, 499, # - 0x03ef, 499, # - 0x03f0, 414, # - 0x03f1, 420, # - 0x0461, 499, # - 0x0463, 499, # - 0x0465, 499, # - 0x0467, 499, # - 0x0469, 499, # - 0x046b, 499, # - 0x046d, 499, # - 0x046f, 499, # - 0x0471, 499, # - 0x0473, 499, # - 0x0475, 499, # - 0x0477, 499, # - 0x0479, 499, # - 0x047b, 499, # - 0x047d, 499, # - 0x047f, 499, # - 0x0481, 499, # - 0x0491, 499, # - 0x0493, 499, # - 0x0495, 499, # - 0x0497, 499, # - 0x0499, 499, # - 0x049b, 499, # - 0x049d, 499, # - 0x049f, 499, # - 0x04a1, 499, # - 0x04a3, 499, # - 0x04a5, 499, # - 0x04a7, 499, # - 0x04a9, 499, # - 0x04ab, 499, # - 0x04ad, 499, # - 0x04af, 499, # - 0x04b1, 499, # - 0x04b3, 499, # - 0x04b5, 499, # - 0x04b7, 499, # - 0x04b9, 499, # - 0x04bb, 499, # - 0x04bd, 499, # - 0x04bf, 499, # - 0x04c2, 499, # - 0x04c4, 499, # - 0x04c8, 499, # - 0x04cc, 499, # - 0x04d1, 499, # - 0x04d3, 499, # - 0x04d5, 499, # - 0x04d7, 499, # - 0x04d9, 499, # - 0x04db, 499, # - 0x04dd, 499, # - 0x04df, 499, # - 0x04e1, 499, # - 0x04e3, 499, # - 0x04e5, 499, # - 0x04e7, 499, # - 0x04e9, 499, # - 0x04eb, 499, # - 0x04ef, 499, # - 0x04f1, 499, # - 0x04f3, 499, # - 0x04f5, 499, # - 0x04f9, 499, # - 0x1e01, 499, # - 0x1e03, 499, # - 0x1e05, 499, # - 0x1e07, 499, # - 0x1e09, 499, # - 0x1e0b, 499, # - 0x1e0d, 499, # - 0x1e0f, 499, # - 0x1e11, 499, # - 0x1e13, 499, # - 0x1e15, 499, # - 0x1e17, 499, # - 0x1e19, 499, # - 0x1e1b, 499, # - 0x1e1d, 499, # - 0x1e1f, 499, # - 0x1e21, 499, # - 0x1e23, 499, # - 0x1e25, 499, # - 0x1e27, 499, # - 0x1e29, 499, # - 0x1e2b, 499, # - 0x1e2d, 499, # - 0x1e2f, 499, # - 0x1e31, 499, # - 0x1e33, 499, # - 0x1e35, 499, # - 0x1e37, 499, # - 0x1e39, 499, # - 0x1e3b, 499, # - 0x1e3d, 499, # - 0x1e3f, 499, # - 0x1e41, 499, # - 0x1e43, 499, # - 0x1e45, 499, # - 0x1e47, 499, # - 0x1e49, 499, # - 0x1e4b, 499, # - 0x1e4d, 499, # - 0x1e4f, 499, # - 0x1e51, 499, # - 0x1e53, 499, # - 0x1e55, 499, # - 0x1e57, 499, # - 0x1e59, 499, # - 0x1e5b, 499, # - 0x1e5d, 499, # - 0x1e5f, 499, # - 0x1e61, 499, # - 0x1e63, 499, # - 0x1e65, 499, # - 0x1e67, 499, # - 0x1e69, 499, # - 0x1e6b, 499, # - 0x1e6d, 499, # - 0x1e6f, 499, # - 0x1e71, 499, # - 0x1e73, 499, # - 0x1e75, 499, # - 0x1e77, 499, # - 0x1e79, 499, # - 0x1e7b, 499, # - 0x1e7d, 499, # - 0x1e7f, 499, # - 0x1e81, 499, # - 0x1e83, 499, # - 0x1e85, 499, # - 0x1e87, 499, # - 0x1e89, 499, # - 0x1e8b, 499, # - 0x1e8d, 499, # - 0x1e8f, 499, # - 0x1e91, 499, # - 0x1e93, 499, # - 0x1e95, 499, # - 0x1ea1, 499, # - 0x1ea3, 499, # - 0x1ea5, 499, # - 0x1ea7, 499, # - 0x1ea9, 499, # - 0x1eab, 499, # - 0x1ead, 499, # - 0x1eaf, 499, # - 0x1eb1, 499, # - 0x1eb3, 499, # - 0x1eb5, 499, # - 0x1eb7, 499, # - 0x1eb9, 499, # - 0x1ebb, 499, # - 0x1ebd, 499, # - 0x1ebf, 499, # - 0x1ec1, 499, # - 0x1ec3, 499, # - 0x1ec5, 499, # - 0x1ec7, 499, # - 0x1ec9, 499, # - 0x1ecb, 499, # - 0x1ecd, 499, # - 0x1ecf, 499, # - 0x1ed1, 499, # - 0x1ed3, 499, # - 0x1ed5, 499, # - 0x1ed7, 499, # - 0x1ed9, 499, # - 0x1edb, 499, # - 0x1edd, 499, # - 0x1edf, 499, # - 0x1ee1, 499, # - 0x1ee3, 499, # - 0x1ee5, 499, # - 0x1ee7, 499, # - 0x1ee9, 499, # - 0x1eeb, 499, # - 0x1eed, 499, # - 0x1eef, 499, # - 0x1ef1, 499, # - 0x1ef3, 499, # - 0x1ef5, 499, # - 0x1ef7, 499, # - 0x1ef9, 499, # - 0x1f51, 508, # - 0x1f53, 508, # - 0x1f55, 508, # - 0x1f57, 508, # - 0x1fb3, 509, # - 0x1fc3, 509, # - 0x1fe5, 507, # - 0x1ff3, 509] # - - tolowerRanges = [ - 0x0041, 0x005a, 532, # A-Z a-z - 0x00c0, 0x00d6, 532, # - - - 0x00d8, 0x00de, 532, # - - - 0x0189, 0x018a, 705, # - - - 0x018e, 0x018f, 702, # - - - 0x01b1, 0x01b2, 717, # - - - 0x0388, 0x038a, 537, # - - - 0x038e, 0x038f, 563, # - - - 0x0391, 0x03a1, 532, # - - - 0x03a3, 0x03ab, 532, # - - - 0x0401, 0x040c, 580, # - - - 0x040e, 0x040f, 580, # - - - 0x0410, 0x042f, 532, # - - - 0x0531, 0x0556, 548, # - - - 0x10a0, 0x10c5, 548, # - - - 0x1f08, 0x1f0f, 492, # - - - 0x1f18, 0x1f1d, 492, # - - - 0x1f28, 0x1f2f, 492, # - - - 0x1f38, 0x1f3f, 492, # - - - 0x1f48, 0x1f4d, 492, # - - - 0x1f68, 0x1f6f, 492, # - - - 0x1f88, 0x1f8f, 492, # - - - 0x1f98, 0x1f9f, 492, # - - - 0x1fa8, 0x1faf, 492, # - - - 0x1fb8, 0x1fb9, 492, # - - - 0x1fba, 0x1fbb, 426, # - - - 0x1fc8, 0x1fcb, 414, # - - - 0x1fd8, 0x1fd9, 492, # - - - 0x1fda, 0x1fdb, 400, # - - - 0x1fe8, 0x1fe9, 492, # - - - 0x1fea, 0x1feb, 388, # - - - 0x1ff8, 0x1ff9, 372, # - - - 0x1ffa, 0x1ffb, 374, # - - - 0x2160, 0x216f, 516, # - - - 0x24b6, 0x24cf, 526, # - - - 0xff21, 0xff3a, 532] # - - - - tolowerSinglets = [ - 0x0100, 501, # - 0x0102, 501, # - 0x0104, 501, # - 0x0106, 501, # - 0x0108, 501, # - 0x010a, 501, # - 0x010c, 501, # - 0x010e, 501, # - 0x0110, 501, # - 0x0112, 501, # - 0x0114, 501, # - 0x0116, 501, # - 0x0118, 501, # - 0x011a, 501, # - 0x011c, 501, # - 0x011e, 501, # - 0x0120, 501, # - 0x0122, 501, # - 0x0124, 501, # - 0x0126, 501, # - 0x0128, 501, # - 0x012a, 501, # - 0x012c, 501, # - 0x012e, 501, # - 0x0130, 301, # i - 0x0132, 501, # - 0x0134, 501, # - 0x0136, 501, # - 0x0139, 501, # - 0x013b, 501, # - 0x013d, 501, # - 0x013f, 501, # - 0x0141, 501, # - 0x0143, 501, # - 0x0145, 501, # - 0x0147, 501, # - 0x014a, 501, # - 0x014c, 501, # - 0x014e, 501, # - 0x0150, 501, # - 0x0152, 501, # - 0x0154, 501, # - 0x0156, 501, # - 0x0158, 501, # - 0x015a, 501, # - 0x015c, 501, # - 0x015e, 501, # - 0x0160, 501, # - 0x0162, 501, # - 0x0164, 501, # - 0x0166, 501, # - 0x0168, 501, # - 0x016a, 501, # - 0x016c, 501, # - 0x016e, 501, # - 0x0170, 501, # - 0x0172, 501, # - 0x0174, 501, # - 0x0176, 501, # - 0x0178, 379, # - 0x0179, 501, # - 0x017b, 501, # - 0x017d, 501, # - 0x0181, 710, # - 0x0182, 501, # - 0x0184, 501, # - 0x0186, 706, # - 0x0187, 501, # - 0x018b, 501, # - 0x0190, 703, # - 0x0191, 501, # - 0x0193, 705, # - 0x0194, 707, # - 0x0196, 711, # - 0x0197, 709, # - 0x0198, 501, # - 0x019c, 711, # - 0x019d, 713, # - 0x01a0, 501, # - 0x01a2, 501, # - 0x01a4, 501, # - 0x01a7, 501, # - 0x01a9, 718, # - 0x01ac, 501, # - 0x01ae, 718, # - 0x01af, 501, # - 0x01b3, 501, # - 0x01b5, 501, # - 0x01b7, 719, # - 0x01b8, 501, # - 0x01bc, 501, # - 0x01c4, 502, # - 0x01c5, 501, # - 0x01c7, 502, # - 0x01c8, 501, # - 0x01ca, 502, # - 0x01cb, 501, # - 0x01cd, 501, # - 0x01cf, 501, # - 0x01d1, 501, # - 0x01d3, 501, # - 0x01d5, 501, # - 0x01d7, 501, # - 0x01d9, 501, # - 0x01db, 501, # - 0x01de, 501, # - 0x01e0, 501, # - 0x01e2, 501, # - 0x01e4, 501, # - 0x01e6, 501, # - 0x01e8, 501, # - 0x01ea, 501, # - 0x01ec, 501, # - 0x01ee, 501, # - 0x01f1, 502, # - 0x01f2, 501, # - 0x01f4, 501, # - 0x01fa, 501, # - 0x01fc, 501, # - 0x01fe, 501, # - 0x0200, 501, # - 0x0202, 501, # - 0x0204, 501, # - 0x0206, 501, # - 0x0208, 501, # - 0x020a, 501, # - 0x020c, 501, # - 0x020e, 501, # - 0x0210, 501, # - 0x0212, 501, # - 0x0214, 501, # - 0x0216, 501, # - 0x0386, 538, # - 0x038c, 564, # - 0x03e2, 501, # - 0x03e4, 501, # - 0x03e6, 501, # - 0x03e8, 501, # - 0x03ea, 501, # - 0x03ec, 501, # - 0x03ee, 501, # - 0x0460, 501, # - 0x0462, 501, # - 0x0464, 501, # - 0x0466, 501, # - 0x0468, 501, # - 0x046a, 501, # - 0x046c, 501, # - 0x046e, 501, # - 0x0470, 501, # - 0x0472, 501, # - 0x0474, 501, # - 0x0476, 501, # - 0x0478, 501, # - 0x047a, 501, # - 0x047c, 501, # - 0x047e, 501, # - 0x0480, 501, # - 0x0490, 501, # - 0x0492, 501, # - 0x0494, 501, # - 0x0496, 501, # - 0x0498, 501, # - 0x049a, 501, # - 0x049c, 501, # - 0x049e, 501, # - 0x04a0, 501, # - 0x04a2, 501, # - 0x04a4, 501, # - 0x04a6, 501, # - 0x04a8, 501, # - 0x04aa, 501, # - 0x04ac, 501, # - 0x04ae, 501, # - 0x04b0, 501, # - 0x04b2, 501, # - 0x04b4, 501, # - 0x04b6, 501, # - 0x04b8, 501, # - 0x04ba, 501, # - 0x04bc, 501, # - 0x04be, 501, # - 0x04c1, 501, # - 0x04c3, 501, # - 0x04c7, 501, # - 0x04cb, 501, # - 0x04d0, 501, # - 0x04d2, 501, # - 0x04d4, 501, # - 0x04d6, 501, # - 0x04d8, 501, # - 0x04da, 501, # - 0x04dc, 501, # - 0x04de, 501, # - 0x04e0, 501, # - 0x04e2, 501, # - 0x04e4, 501, # - 0x04e6, 501, # - 0x04e8, 501, # - 0x04ea, 501, # - 0x04ee, 501, # - 0x04f0, 501, # - 0x04f2, 501, # - 0x04f4, 501, # - 0x04f8, 501, # - 0x1e00, 501, # - 0x1e02, 501, # - 0x1e04, 501, # - 0x1e06, 501, # - 0x1e08, 501, # - 0x1e0a, 501, # - 0x1e0c, 501, # - 0x1e0e, 501, # - 0x1e10, 501, # - 0x1e12, 501, # - 0x1e14, 501, # - 0x1e16, 501, # - 0x1e18, 501, # - 0x1e1a, 501, # - 0x1e1c, 501, # - 0x1e1e, 501, # - 0x1e20, 501, # - 0x1e22, 501, # - 0x1e24, 501, # - 0x1e26, 501, # - 0x1e28, 501, # - 0x1e2a, 501, # - 0x1e2c, 501, # - 0x1e2e, 501, # - 0x1e30, 501, # - 0x1e32, 501, # - 0x1e34, 501, # - 0x1e36, 501, # - 0x1e38, 501, # - 0x1e3a, 501, # - 0x1e3c, 501, # - 0x1e3e, 501, # - 0x1e40, 501, # - 0x1e42, 501, # - 0x1e44, 501, # - 0x1e46, 501, # - 0x1e48, 501, # - 0x1e4a, 501, # - 0x1e4c, 501, # - 0x1e4e, 501, # - 0x1e50, 501, # - 0x1e52, 501, # - 0x1e54, 501, # - 0x1e56, 501, # - 0x1e58, 501, # - 0x1e5a, 501, # - 0x1e5c, 501, # - 0x1e5e, 501, # - 0x1e60, 501, # - 0x1e62, 501, # - 0x1e64, 501, # - 0x1e66, 501, # - 0x1e68, 501, # - 0x1e6a, 501, # - 0x1e6c, 501, # - 0x1e6e, 501, # - 0x1e70, 501, # - 0x1e72, 501, # - 0x1e74, 501, # - 0x1e76, 501, # - 0x1e78, 501, # - 0x1e7a, 501, # - 0x1e7c, 501, # - 0x1e7e, 501, # - 0x1e80, 501, # - 0x1e82, 501, # - 0x1e84, 501, # - 0x1e86, 501, # - 0x1e88, 501, # - 0x1e8a, 501, # - 0x1e8c, 501, # - 0x1e8e, 501, # - 0x1e90, 501, # - 0x1e92, 501, # - 0x1e94, 501, # - 0x1ea0, 501, # - 0x1ea2, 501, # - 0x1ea4, 501, # - 0x1ea6, 501, # - 0x1ea8, 501, # - 0x1eaa, 501, # - 0x1eac, 501, # - 0x1eae, 501, # - 0x1eb0, 501, # - 0x1eb2, 501, # - 0x1eb4, 501, # - 0x1eb6, 501, # - 0x1eb8, 501, # - 0x1eba, 501, # - 0x1ebc, 501, # - 0x1ebe, 501, # - 0x1ec0, 501, # - 0x1ec2, 501, # - 0x1ec4, 501, # - 0x1ec6, 501, # - 0x1ec8, 501, # - 0x1eca, 501, # - 0x1ecc, 501, # - 0x1ece, 501, # - 0x1ed0, 501, # - 0x1ed2, 501, # - 0x1ed4, 501, # - 0x1ed6, 501, # - 0x1ed8, 501, # - 0x1eda, 501, # - 0x1edc, 501, # - 0x1ede, 501, # - 0x1ee0, 501, # - 0x1ee2, 501, # - 0x1ee4, 501, # - 0x1ee6, 501, # - 0x1ee8, 501, # - 0x1eea, 501, # - 0x1eec, 501, # - 0x1eee, 501, # - 0x1ef0, 501, # - 0x1ef2, 501, # - 0x1ef4, 501, # - 0x1ef6, 501, # - 0x1ef8, 501, # - 0x1f59, 492, # - 0x1f5b, 492, # - 0x1f5d, 492, # - 0x1f5f, 492, # - 0x1fbc, 491, # - 0x1fcc, 491, # - 0x1fec, 493, # - 0x1ffc, 491] # - - toTitleSinglets = [ - 0x01c4, 501, # - 0x01c6, 499, # - 0x01c7, 501, # - 0x01c9, 499, # - 0x01ca, 501, # - 0x01cc, 499, # - 0x01f1, 501, # - 0x01f3, 499] # - -proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int = + for rune in runes: + result.add rune + +proc runeOffset*(s: openArray[char], pos: Natural, start: Natural = 0): int = + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeReverseOffset proc <#runeReverseOffset,string,Positive>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeOffset(1) == 1 + doAssert a.runeOffset(3) == 4 + doAssert a.runeOffset(4) == 6 + + var + i = 0 + o = start + while i < pos: + o += runeLenAt(s, o) + if o >= s.len: + return -1 + inc i + return o + +proc runeReverseOffset*(s: openArray[char], rev: Positive): (int, int) = + ## Returns a tuple with the byte offset of the + ## rune at position ``rev`` in ``s``, counting + ## from the end (starting with 1) and the total + ## number of runes in the string. + ## + ## Returns a negative value for offset if there are too few runes in + ## the string to satisfy the request. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeOffset proc <#runeOffset,string,Natural,Natural>`_ + var + a = rev.int + o = 0 + x = 0 + let times = 2*rev.int-s.runeLen # transformed from rev.int - a < s.runeLen - rev.int + while o < s.len: + let r = runeLenAt(s, o) + o += r + if a > times: + x += r + dec a + result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) + +proc runeAtPos*(s: openArray[char], pos: int): Rune = + ## Returns the rune at position ``pos``. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + fastRuneAt(s, runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: openArray[char], pos: Natural): string = + ## Returns the rune at position ``pos`` as UTF8 String. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + let o = runeOffset(s, pos) + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) + +proc runeSubStr*(s: openArray[char], pos: int, len: int = int.high): string = + ## Returns the UTF-8 substring starting at code point ``pos`` + ## with ``len`` code points. + ## + ## If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest + ## possible string. + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + + if pos < 0: + let (o, rl) = runeReverseOffset(s, -pos) + if len >= rl: + result = s.substr(o, s.high) + elif len < 0: + let e = rl + len + if e < 0: + result = "" + else: + result = s.substr(o, runeOffset(s, e-(rl+pos), o)-1) + else: + result = s.substr(o, runeOffset(s, len, o)-1) + else: + let o = runeOffset(s, pos) + if o < 0: + result = "" + elif len == int.high: + result = s.substr(o, s.len-1) + elif len < 0: + let (e, rl) = runeReverseOffset(s, -len) + discard rl + if e <= 0: + result = "" + else: + result = s.substr(o, e-1) + else: + var e = runeOffset(s, len, o) + if e < 0: + e = s.len + result = s.substr(o, e-1) + +proc `<=%`*(a, b: Rune): bool = + ## Checks if code point of `a` is smaller or equal to code point of `b`. + runnableExamples: + let + a = "ú".runeAt(0) + b = "ü".runeAt(0) + doAssert a <=% b + return int(a) <=% int(b) + +proc `<%`*(a, b: Rune): bool = + ## Checks if code point of `a` is smaller than code point of `b`. + runnableExamples: + let + a = "ú".runeAt(0) + b = "ü".runeAt(0) + doAssert a <% b + return int(a) <% int(b) + +proc `==`*(a, b: Rune): bool = + ## Checks if two runes are equal. + return int(a) == int(b) + + +include "includes/unicode_ranges" + +proc binarySearch(c: RuneImpl, tab: openArray[int32], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -1163,64 +479,102 @@ proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int return t return -1 -proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = - ## Converts ``c`` into lower case. This works for any Unicode character. +proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1".} = + ## Converts ``c`` into lower case. This works for any rune. + ## ## If possible, prefer ``toLower`` over ``toUpper``. + ## + ## See also: + ## * `toUpper proc <#toUpper,Rune>`_ + ## * `toTitle proc <#toTitle,Rune>`_ + ## * `isLower proc <#isLower,Rune>`_ var c = RuneImpl(c) - var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3) - if p >= 0 and c >= tolowerRanges[p] and c <= tolowerRanges[p+1]: - return Rune(c + tolowerRanges[p+2] - 500) - p = binarySearch(c, tolowerSinglets, len(tolowerSinglets) div 2, 2) - if p >= 0 and c == tolowerSinglets[p]: - return Rune(c + tolowerSinglets[p+1] - 500) + var p = binarySearch(c, toLowerRanges, len(toLowerRanges) div 3, 3) + if p >= 0 and c >= toLowerRanges[p] and c <= toLowerRanges[p+1]: + return Rune(c + toLowerRanges[p+2] - 500) + p = binarySearch(c, toLowerSinglets, len(toLowerSinglets) div 2, 2) + if p >= 0 and c == toLowerSinglets[p]: + return Rune(c + toLowerSinglets[p+1] - 500) return Rune(c) -proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = - ## Converts ``c`` into upper case. This works for any Unicode character. +proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1".} = + ## Converts ``c`` into upper case. This works for any rune. + ## ## If possible, prefer ``toLower`` over ``toUpper``. + ## + ## See also: + ## * `toLower proc <#toLower,Rune>`_ + ## * `toTitle proc <#toTitle,Rune>`_ + ## * `isUpper proc <#isUpper,Rune>`_ var c = RuneImpl(c) - var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3) - if p >= 0 and c >= toupperRanges[p] and c <= toupperRanges[p+1]: - return Rune(c + toupperRanges[p+2] - 500) - p = binarySearch(c, toupperSinglets, len(toupperSinglets) div 2, 2) - if p >= 0 and c == toupperSinglets[p]: - return Rune(c + toupperSinglets[p+1] - 500) + var p = binarySearch(c, toUpperRanges, len(toUpperRanges) div 3, 3) + if p >= 0 and c >= toUpperRanges[p] and c <= toUpperRanges[p+1]: + return Rune(c + toUpperRanges[p+2] - 500) + p = binarySearch(c, toUpperSinglets, len(toUpperSinglets) div 2, 2) + if p >= 0 and c == toUpperSinglets[p]: + return Rune(c + toUpperSinglets[p+1] - 500) return Rune(c) -proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = - ## Converts ``c`` to title case +proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1".} = + ## Converts ``c`` to title case. + ## + ## See also: + ## * `toLower proc <#toLower,Rune>`_ + ## * `toUpper proc <#toUpper,Rune>`_ + ## * `isTitle proc <#isTitle,Rune>`_ var c = RuneImpl(c) var p = binarySearch(c, toTitleSinglets, len(toTitleSinglets) div 2, 2) if p >= 0 and c == toTitleSinglets[p]: return Rune(c + toTitleSinglets[p+1] - 500) return Rune(c) -proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a lower case Unicode character. +proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1".} = + ## Returns true if ``c`` is a lower case rune. + ## ## If possible, prefer ``isLower`` over ``isUpper``. + ## + ## See also: + ## * `toLower proc <#toLower,Rune>`_ + ## * `isUpper proc <#isUpper,Rune>`_ + ## * `isTitle proc <#isTitle,Rune>`_ var c = RuneImpl(c) # Note: toUpperRanges is correct here! - var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3) - if p >= 0 and c >= toupperRanges[p] and c <= toupperRanges[p+1]: + var p = binarySearch(c, toUpperRanges, len(toUpperRanges) div 3, 3) + if p >= 0 and c >= toUpperRanges[p] and c <= toUpperRanges[p+1]: return true - p = binarySearch(c, toupperSinglets, len(toupperSinglets) div 2, 2) - if p >= 0 and c == toupperSinglets[p]: + p = binarySearch(c, toUpperSinglets, len(toUpperSinglets) div 2, 2) + if p >= 0 and c == toUpperSinglets[p]: return true -proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a upper case Unicode character. +proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1".} = + ## Returns true if ``c`` is a upper case rune. + ## ## If possible, prefer ``isLower`` over ``isUpper``. + ## + ## See also: + ## * `toUpper proc <#toUpper,Rune>`_ + ## * `isLower proc <#isLower,Rune>`_ + ## * `isTitle proc <#isTitle,Rune>`_ + ## * `isAlpha proc <#isAlpha,Rune>`_ + ## * `isWhiteSpace proc <#isWhiteSpace,Rune>`_ var c = RuneImpl(c) # Note: toLowerRanges is correct here! - var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3) - if p >= 0 and c >= tolowerRanges[p] and c <= tolowerRanges[p+1]: + var p = binarySearch(c, toLowerRanges, len(toLowerRanges) div 3, 3) + if p >= 0 and c >= toLowerRanges[p] and c <= toLowerRanges[p+1]: return true - p = binarySearch(c, tolowerSinglets, len(tolowerSinglets) div 2, 2) - if p >= 0 and c == tolowerSinglets[p]: + p = binarySearch(c, toLowerSinglets, len(toLowerSinglets) div 2, 2) + if p >= 0 and c == toLowerSinglets[p]: return true -proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is an *alpha* Unicode character (i.e., a letter) +proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1".} = + ## Returns true if ``c`` is an *alpha* rune (i.e., a letter). + ## + ## See also: + ## * `isLower proc <#isLower,Rune>`_ + ## * `isTitle proc <#isTitle,Rune>`_ + ## * `isAlpha proc <#isAlpha,Rune>`_ + ## * `isWhiteSpace proc <#isWhiteSpace,Rune>`_ + ## * `isCombining proc <#isCombining,Rune>`_ if isUpper(c) or isLower(c): return true var c = RuneImpl(c) @@ -1231,19 +585,38 @@ proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c == alphaSinglets[p]: return true -proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a Unicode titlecase character +proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1".} = + ## Returns true if ``c`` is a Unicode titlecase code point. + ## + ## See also: + ## * `toTitle proc <#toTitle,Rune>`_ + ## * `isLower proc <#isLower,Rune>`_ + ## * `isUpper proc <#isUpper,Rune>`_ + ## * `isAlpha proc <#isAlpha,Rune>`_ + ## * `isWhiteSpace proc <#isWhiteSpace,Rune>`_ return isUpper(c) and isLower(c) -proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a Unicode whitespace character +proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1".} = + ## Returns true if ``c`` is a Unicode whitespace code point. + ## + ## See also: + ## * `isLower proc <#isLower,Rune>`_ + ## * `isUpper proc <#isUpper,Rune>`_ + ## * `isTitle proc <#isTitle,Rune>`_ + ## * `isAlpha proc <#isAlpha,Rune>`_ var c = RuneImpl(c) var p = binarySearch(c, spaceRanges, len(spaceRanges) div 2, 2) if p >= 0 and c >= spaceRanges[p] and c <= spaceRanges[p+1]: return true -proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = - ## Returns true iff ``c`` is a Unicode combining character +proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1".} = + ## Returns true if ``c`` is a Unicode combining code unit. + ## + ## See also: + ## * `isLower proc <#isLower,Rune>`_ + ## * `isUpper proc <#isUpper,Rune>`_ + ## * `isTitle proc <#isTitle,Rune>`_ + ## * `isAlpha proc <#isAlpha,Rune>`_ var c = RuneImpl(c) # Optimized to return false immediately for ASCII @@ -1253,8 +626,179 @@ proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = (c >= 0x20d0 and c <= 0x20ff) or (c >= 0xfe20 and c <= 0xfe2f)) -iterator runes*(s: string): Rune = - ## Iterates over any unicode character of the string ``s`` +template runeCheck(s, runeProc) = + ## Common code for isAlpha and isSpace. + result = if len(s) == 0: false else: true + var + i = 0 + rune: Rune + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc = true) + result = runeProc(rune) and result + +proc isAlpha*(s: openArray[char]): bool {.noSideEffect, + rtl, extern: "nuc$1Str".} = + ## Returns true if ``s`` contains all alphabetic runes. + runnableExamples: + let a = "añyóng" + doAssert a.isAlpha + runeCheck(s, isAlpha) + +proc isSpace*(s: openArray[char]): bool {.noSideEffect, + rtl, extern: "nuc$1Str".} = + ## Returns true if ``s`` contains all whitespace runes. + runnableExamples: + let a = "\t\l \v\r\f" + doAssert a.isSpace + runeCheck(s, isWhiteSpace) + + +template convertRune(s, runeProc) = + ## Convert runes in ``s`` using ``runeProc`` as the converter. + result = newString(len(s)) + var + i = 0 + resultIndex = 0 + rune: Rune + while i < len(s): + fastRuneAt(s, i, rune, doInc = true) + rune = runeProc(rune) + fastToUTF8Copy(rune, result, resultIndex, doInc = true) + +proc toUpper*(s: openArray[char]): string {.noSideEffect, + rtl, extern: "nuc$1Str".} = + ## Converts ``s`` into upper-case runes. + runnableExamples: + doAssert toUpper("abγ") == "ABΓ" + convertRune(s, toUpper) + +proc toLower*(s: openArray[char]): string {.noSideEffect, + rtl, extern: "nuc$1Str".} = + ## Converts ``s`` into lower-case runes. + runnableExamples: + doAssert toLower("ABΓ") == "abγ" + convertRune(s, toLower) + +proc swapCase*(s: openArray[char]): string {.noSideEffect, + rtl, extern: "nuc$1".} = + ## Swaps the case of runes in ``s``. + ## + ## Returns a new string such that the cases of all runes + ## are swapped if possible. + runnableExamples: + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + + var + i = 0 + resultIndex = 0 + rune: Rune + result = newString(len(s)) + while i < len(s): + fastRuneAt(s, i, rune) + if rune.isUpper(): + rune = rune.toLower() + elif rune.isLower(): + rune = rune.toUpper() + fastToUTF8Copy(rune, result, resultIndex, doInc = true) + +proc capitalize*(s: openArray[char]): string {.noSideEffect, + rtl, extern: "nuc$1".} = + ## Converts the first character of ``s`` into an upper-case rune. + runnableExamples: + doAssert capitalize("βeta") == "Βeta" + + if len(s) == 0: + return "" + var + rune: Rune + i = 0 + fastRuneAt(s, i, rune, doInc = true) + result = $toUpper(rune) & substr(s.toOpenArray(i, s.high)) + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +proc translate*(s: openArray[char], replacements: proc(key: string): string): string {. + rtl, extern: "nuc$1", effectsOf: replacements.} = + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. + ## + ## ``replacements`` is any proc that takes a word and returns + ## a new word to fill it's place. + runnableExamples: + proc wordToNumber(s: string): string = + case s + of "one": "1" + of "two": "2" + else: s + let a = "one two three four" + doAssert a.translate(wordToNumber) == "1 2 three four" + + # Allocate memory for the new string based on the old one. + # If the new string length is less than the old, no allocations + # will be needed. If the new string length is greater than the + # old, then maybe only one allocation is needed + result = newStringOfCap(s.len) + var + index = 0 + lastIndex = 0 + wordStart = 0 + inWord = false + rune: Rune + + while index < len(s): + lastIndex = index + fastRuneAt(s, index, rune) + let whiteSpace = rune.isWhiteSpace() + + if whiteSpace and inWord: + # If we've reached the end of a word + let word = substr(s.toOpenArray(wordStart, lastIndex - 1)) + result.add(replacements(word)) + result.add($rune) + inWord = false + elif not whiteSpace and not inWord: + # If we've hit a non space character and + # are not currently in a word, track + # the starting index of the word + inWord = true + wordStart = lastIndex + elif whiteSpace: + result.add($rune) + + if wordStart < len(s) and inWord: + # Get the trailing word at the end + let word = substr(s.toOpenArray(wordStart, s.high)) + result.add(replacements(word)) + +proc title*(s: openArray[char]): string {.noSideEffect, + rtl, extern: "nuc$1".} = + ## Converts ``s`` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside ``s`` is capitalized. + runnableExamples: + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + + var + i = 0 + resultIndex = 0 + rune: Rune + result = newString(len(s)) + var firstRune = true + + while i < len(s): + fastRuneAt(s, i, rune) + if not rune.isWhiteSpace() and firstRune: + rune = rune.toUpper() + firstRune = false + elif rune.isWhiteSpace(): + firstRune = true + fastToUTF8Copy(rune, result, resultIndex, doInc = true) + + +iterator runes*(s: openArray[char]): Rune = + ## Iterates over any rune of the string ``s`` returning runes. var i = 0 result: Rune @@ -1262,18 +806,39 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result -proc toRunes*(s: string): seq[Rune] = - ## Obtains a sequence containing the Runes in ``s`` +iterator utf8*(s: openArray[char]): string = + ## Iterates over any rune of the string ``s`` returning utf8 values. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + var o = 0 + while o < s.len: + let n = runeLenAt(s, o) + yield substr(s.toOpenArray(o, (o+n-1))) + o += n + +proc toRunes*(s: openArray[char]): seq[Rune] = + ## Obtains a sequence containing the Runes in ``s``. + ## + ## See also: + ## * `$ proc <#$,Rune>`_ for a reverse operation + runnableExamples: + let a = toRunes("aáä") + doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] + result = newSeq[Rune]() for r in s.runes: result.add(r) -proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = +proc cmpRunesIgnoreCase*(a, b: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Compares two UTF-8 strings and ignores the case. Returns: ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b var i = 0 var j = 0 var ar, br: Rune @@ -1281,20 +846,25 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = # slow path: fastRuneAt(a, i, ar) fastRuneAt(b, j, br) - result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) + when sizeof(int) < 4: + const lo = low(int).int32 + const hi = high(int).int32 + result = clamp(RuneImpl(toLower(ar)) - RuneImpl(toLower(br)), lo, hi).int + else: + result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) if result != 0: return result = a.len - b.len -proc reversed*(s: string): string = - ## Returns the reverse of ``s``, interpreting it as Unicode characters. - ## Unicode combining characters are correctly interpreted as well: +proc reversed*(s: openArray[char]): string = + ## Returns the reverse of ``s``, interpreting it as runes. ## - ## .. code-block:: nim - ## - ## assert reversed("Reverse this!") == "!siht esreveR" - ## assert reversed("先秦兩漢") == "漢兩秦先" - ## assert reversed("as⃝df̅") == "f̅ds⃝a" - ## assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + var i = 0 lastI = 0 @@ -1302,7 +872,7 @@ proc reversed*(s: string): string = blockPos = 0 r: Rune - template reverseUntil(pos): stmt = + template reverseUntil(pos) = var j = pos - 1 while j > blockPos: result[newPos] = s[j] @@ -1320,9 +890,15 @@ proc reversed*(s: string): string = reverseUntil(len(s)) -proc graphemeLen*(s: string; i: Natural): Natural = - ## The number of bytes belonging to 's[i]' including following combining - ## characters. +proc graphemeLen*(s: openArray[char]; i: Natural): Natural = + ## The number of bytes belonging to byte index ``s[i]``, + ## including following combining code units. + runnableExamples: + let a = "añyóng" + doAssert a.graphemeLen(1) == 2 ## ñ + doAssert a.graphemeLen(2) == 1 + doAssert a.graphemeLen(4) == 2 ## ó + var j = i.int var r, r2: Rune if j < s.len: @@ -1333,30 +909,607 @@ proc graphemeLen*(s: string; i: Natural): Natural = if not isCombining(r2): break result = j-i -proc lastRune*(s: string; last: int): (Rune, int) = - ## length of the last rune in 's[0..last]'. Returns the rune and its length +proc lastRune*(s: openArray[char]; last: int): (Rune, int) = + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length ## in bytes. if s[last] <= chr(127): result = (Rune(s[last]), 1) else: var L = 0 - while last-L >= 0 and ord(s[last-L]) shr 6 == 0b10: inc(L) + while last-L >= 0 and uint(s[last-L]) shr 6 == 0b10: inc(L) var r: Rune fastRuneAt(s, last-L, r, false) result = (r, L+1) -when isMainModule: +proc size*(r: Rune): int {.noSideEffect.} = + ## Returns the number of bytes the rune ``r`` takes. + runnableExamples: + let a = toRunes "aá" + doAssert size(a[0]) == 1 + doAssert size(a[1]) == 2 + + let v = r.uint32 + if v <= 0x007F'u32: result = 1 + elif v <= 0x07FF'u32: result = 2 + elif v <= 0xFFFF'u32: result = 3 + elif v <= 0x1FFFFF'u32: result = 4 + elif v <= 0x3FFFFFF'u32: result = 5 + elif v <= 0x7FFFFFFF'u32: result = 6 + else: result = 1 + +# --------- Private templates for different split separators ----------- +proc stringHasSep(s: openArray[char], index: int, seps: openArray[Rune]): bool = + var rune: Rune + fastRuneAt(s, index, rune, false) + return seps.contains(rune) + +proc stringHasSep(s: openArray[char], index: int, sep: Rune): bool = + var rune: Rune + fastRuneAt(s, index, rune, false) + return sep == rune + +template splitCommon(s, sep, maxsplit: untyped) = + ## Common code for split procedures. let - someString = "öÑ" - someRunes = @[runeAt(someString, 0), runeAt(someString, 2)] - compared = (someString == $someRunes) - doAssert compared == true - - doAssert reversed("Reverse this!") == "!siht esreveR" - doAssert reversed("先秦兩漢") == "漢兩秦先" - doAssert reversed("as⃝df̅") == "f̅ds⃝a" - doAssert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" - doAssert len(toRunes("as⃝df̅")) == runeLen("as⃝df̅") - const test = "as⃝" - doAssert lastRune(test, test.len-1)[1] == 3 - doAssert graphemeLen("è", 0) == 2 + sLen = len(s) + var + last = 0 + splits = maxsplit + if sLen > 0: + while last <= sLen: + var first = last + while last < sLen and not stringHasSep(s, last, sep): + inc(last, runeLenAt(s, last)) + if splits == 0: last = sLen + yield substr(s.toOpenArray(first, (last - 1))) + if splits == 0: break + dec(splits) + inc(last, if last < sLen: runeLenAt(s, last) else: 1) + +iterator split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, + maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a group of separators. + ## + ## Substrings are separated by a substring containing only ``seps``. + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + + splitCommon(s, seps, maxsplit) + +iterator splitWhitespace*(s: openArray[char]): string = + ## Splits a unicode string at whitespace runes. + splitCommon(s, unicodeSpaces, -1) + +template accResult(iter: untyped) = + result = @[] + for x in iter: add(result, x) + +proc splitWhitespace*(s: openArray[char]): seq[string] {.noSideEffect, + rtl, extern: "ncuSplitWhitespace".} = + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accResult(splitWhitespace(s)) + +iterator split*(s: openArray[char], sep: Rune, maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a single separator. + ## Substrings are separated by the rune ``sep``. + runnableExamples: + import std/sequtils + + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(s, sep, maxsplit) + +proc split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + seq[string] {.noSideEffect, rtl, extern: "nucSplitRunes".} = + ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, + ## but is a proc that returns a sequence of substrings. + accResult(split(s, seps, maxsplit)) + +proc split*(s: openArray[char], sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, + rtl, extern: "nucSplitRune".} = + ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc + ## that returns a sequence of substrings. + accResult(split(s, sep, maxsplit)) + +proc strip*(s: openArray[char], leading = true, trailing = true, + runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, + rtl, extern: "nucStrip".} = + ## Strips leading or trailing ``runes`` from ``s`` and returns + ## the resulting string. + ## + ## If ``leading`` is true (default), leading ``runes`` are stripped. + ## If ``trailing`` is true (default), trailing ``runes`` are stripped. + ## If both are false, the string is returned unchanged. + runnableExamples: + let a = "\táñyóng " + doAssert a.strip == "áñyóng" + doAssert a.strip(leading = false) == "\táñyóng" + doAssert a.strip(trailing = false) == "áñyóng " + + var + sI = 0 ## starting index into string ``s`` + eI = len(s) - 1 ## ending index into ``s``, where the last ``Rune`` starts + if leading: + var + i = 0 + xI: int ## value of ``sI`` at the beginning of the iteration + rune: Rune + while i < len(s): + xI = i + fastRuneAt(s, i, rune) + sI = i # Assume to start from next rune + if not runes.contains(rune): + sI = xI # Go back to where the current rune starts + break + if trailing: + var + i = eI + xI: int + rune: Rune + while i >= 0: + xI = i + fastRuneAt(s, xI, rune) + var yI = i - 1 + while yI >= 0: + var + yIend = yI + pRune: Rune + fastRuneAt(s, yIend, pRune) + if yIend < xI: break + i = yI + rune = pRune + dec(yI) + if not runes.contains(rune): + eI = xI - 1 + break + dec(i) + let newLen = eI - sI + 1 + result = newStringOfCap(newLen) + if newLen > 0: + result.add substr(s.toOpenArray(sI, eI)) + +proc repeat*(c: Rune, count: Natural): string {.noSideEffect, + rtl, extern: "nucRepeatRune".} = + ## Returns a string of ``count`` Runes ``c``. + ## + ## The returned string will have a rune-length of ``count``. + runnableExamples: + let a = "ñ".runeAt(0) + doAssert a.repeat(5) == "ñññññ" + + let s = $c + result = newStringOfCap(count * s.len) + for i in 0 ..< count: + result.add s + +proc align*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. + noSideEffect, rtl, extern: "nucAlignString".} = + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. + ## + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft,string,Natural>`_. + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#'.Rune) == "##1232" + assert align("Åge", 5) == " Åge" + assert align("×", 4, '_'.Rune) == "___×" + + let sLen = s.runeLen + if sLen < count: + let padStr = $padding + result = newStringOfCap(padStr.len * count) + let spaces = count - sLen + for i in 0 ..< spaces: result.add padStr + result.add s + else: + result = s.substr + +proc alignLeft*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. + noSideEffect.} = + ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. + ## + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align,string,Natural>`_. + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#'.Rune) == "1232##" + assert alignLeft("Åge", 5) == "Åge " + assert alignLeft("×", 4, '_'.Rune) == "×___" + let sLen = s.runeLen + if sLen < count: + let padStr = $padding + result = newStringOfCap(s.len + (count - sLen) * padStr.len) + result.add s + for i in sLen ..< count: + result.add padStr + else: + result = s.substr + + +proc runeLen*(s: string): int {.inline.} = + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 + runeLen(toOa(s)) + +proc runeLenAt*(s: string, i: Natural): int {.inline.} = + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + ## + ## See also: + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + runeLenAt(toOa(s), i) + +proc runeAt*(s: string, i: Natural): Rune {.inline.} = + ## Returns the rune in ``s`` at **byte index** ``i``. + ## + ## See also: + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) + fastRuneAt(s, i, result, false) + +proc validateUtf8*(s: string): int {.inline.} = + ## Returns the position of the invalid byte in ``s`` if the string ``s`` does + ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. + ## + ## See also: + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + validateUtf8(toOa(s)) + +proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int {.inline.} = + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeReverseOffset proc <#runeReverseOffset,string,Positive>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeOffset(1) == 1 + doAssert a.runeOffset(3) == 4 + doAssert a.runeOffset(4) == 6 + runeOffset(toOa(s), pos, start) + +proc runeReverseOffset*(s: string, rev: Positive): (int, int) {.inline.} = + ## Returns a tuple with the byte offset of the + ## rune at position ``rev`` in ``s``, counting + ## from the end (starting with 1) and the total + ## number of runes in the string. + ## + ## Returns a negative value for offset if there are too few runes in + ## the string to satisfy the request. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeOffset proc <#runeOffset,string,Natural,Natural>`_ + runeReverseOffset(toOa(s), rev) + +proc runeAtPos*(s: string, pos: int): Rune {.inline.} = + ## Returns the rune at position ``pos``. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + fastRuneAt(toOa(s), runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: string, pos: Natural): string {.inline.} = + ## Returns the rune at position ``pos`` as UTF8 String. + ## + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. + ## + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + let o = runeOffset(s, pos) + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) + +proc runeSubStr*(s: string, pos: int, len: int = int.high): string {.inline.} = + ## Returns the UTF-8 substring starting at code point ``pos`` + ## with ``len`` code points. + ## + ## If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest + ## possible string. + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + runeSubStr(toOa(s), pos, len) + + +proc isAlpha*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all alphabetic runes. + runnableExamples: + let a = "añyóng" + doAssert a.isAlpha + isAlpha(toOa(s)) + +proc isSpace*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all whitespace runes. + runnableExamples: + let a = "\t\l \v\r\f" + doAssert a.isSpace + isSpace(toOa(s)) + + +proc toUpper*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into upper-case runes. + runnableExamples: + doAssert toUpper("abγ") == "ABΓ" + toUpper(toOa(s)) + +proc toLower*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into lower-case runes. + runnableExamples: + doAssert toLower("ABΓ") == "abγ" + toLower(toOa(s)) + +proc swapCase*(s: string): string {.noSideEffect, inline.} = + ## Swaps the case of runes in ``s``. + ## + ## Returns a new string such that the cases of all runes + ## are swapped if possible. + runnableExamples: + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + swapCase(toOa(s)) + +proc capitalize*(s: string): string {.noSideEffect.} = + ## Converts the first character of ``s`` into an upper-case rune. + runnableExamples: + doAssert capitalize("βeta") == "Βeta" + capitalize(toOa(s)) + + +proc translate*(s: string, replacements: proc(key: string): string): string {.effectsOf: replacements, inline.} = + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. + ## + ## ``replacements`` is any proc that takes a word and returns + ## a new word to fill it's place. + runnableExamples: + proc wordToNumber(s: string): string = + case s + of "one": "1" + of "two": "2" + else: s + let a = "one two three four" + doAssert a.translate(wordToNumber) == "1 2 three four" + translate(toOa(s), replacements) + +proc title*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside ``s`` is capitalized. + runnableExamples: + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + title(toOa(s)) + + +iterator runes*(s: string): Rune = + ## Iterates over any rune of the string ``s`` returning runes. + for rune in runes(toOa(s)): + yield rune + +iterator utf8*(s: string): string = + ## Iterates over any rune of the string ``s`` returning utf8 values. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + for str in utf8(toOa(s)): + yield str + +proc toRunes*(s: string): seq[Rune] {.inline.} = + ## Obtains a sequence containing the Runes in ``s``. + ## + ## See also: + ## * `$ proc <#$,Rune>`_ for a reverse operation + runnableExamples: + let a = toRunes("aáä") + doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] + toRunes(toOa(s)) + +proc cmpRunesIgnoreCase*(a, b: string): int {.inline.} = + ## Compares two UTF-8 strings and ignores the case. Returns: + ## + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + cmpRunesIgnoreCase(a.toOa(), b.toOa()) + +proc reversed*(s: string): string {.inline.} = + ## Returns the reverse of ``s``, interpreting it as runes. + ## + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + reversed(toOa(s)) + +proc graphemeLen*(s: string; i: Natural): Natural {.inline.} = + ## The number of bytes belonging to byte index ``s[i]``, + ## including following combining code unit. + runnableExamples: + let a = "añyóng" + doAssert a.graphemeLen(1) == 2 ## ñ + doAssert a.graphemeLen(2) == 1 + doAssert a.graphemeLen(4) == 2 ## ó + graphemeLen(toOa(s), i) + +proc lastRune*(s: string; last: int): (Rune, int) {.inline.} = + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length + ## in bytes. + lastRune(toOa(s), last) + +iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, + maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a group of separators. + ## + ## Substrings are separated by a substring containing only ``seps``. + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + + splitCommon(toOa(s), seps, maxsplit) + +iterator splitWhitespace*(s: string): string = + ## Splits a unicode string at whitespace runes. + splitCommon(s.toOa(), unicodeSpaces, -1) + + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, inline.}= + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accResult(splitWhitespace(toOa(s))) + +iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a single separator. + ## Substrings are separated by the rune ``sep``. + runnableExamples: + import std/sequtils + + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(toOa(s), sep, maxsplit) + +proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, + ## but is a proc that returns a sequence of substrings. + accResult(split(toOa(s), seps, maxsplit)) + +proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc + ## that returns a sequence of substrings. + accResult(split(toOa(s), sep, maxsplit)) + +proc strip*(s: string, leading = true, trailing = true, + runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, inline.} = + ## Strips leading or trailing ``runes`` from ``s`` and returns + ## the resulting string. + ## + ## If ``leading`` is true (default), leading ``runes`` are stripped. + ## If ``trailing`` is true (default), trailing ``runes`` are stripped. + ## If both are false, the string is returned unchanged. + runnableExamples: + let a = "\táñyóng " + doAssert a.strip == "áñyóng" + doAssert a.strip(leading = false) == "\táñyóng" + doAssert a.strip(trailing = false) == "áñyóng " + strip(toOa(s), leading, trailing, runes) + + +proc align*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. + ## + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft,string,Natural>`_. + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#'.Rune) == "##1232" + assert align("Åge", 5) == " Åge" + assert align("×", 4, '_'.Rune) == "___×" + align(toOa(s), count, padding) + +proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. + ## + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align,string,Natural>`_. + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#'.Rune) == "1232##" + assert alignLeft("Åge", 5) == "Åge " + assert alignLeft("×", 4, '_'.Rune) == "×___" + alignLeft(toOa(s), count, padding) diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index 8da0136ff..2fb69f7b2 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,26 +1,30 @@ -#! usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module # (c) 2010 Andreas Rumpf from unidecode import unidecode +try: + import warnings + warnings.simplefilter("ignore") +except ImportError: + pass -def main2(): - data = [] - for x in xrange(128, 0xffff + 1): - u = eval("u'\u%04x'" % x) - - val = unidecode(u) - data.append(val) - - - f = open("unidecode.dat", "wb+") - for d in data: - f.write("%s\n" % d) - f.close() +def main(): + f = open("unidecode.dat", "wb+") + for x in range(128, 0xffff + 1): + u = eval("u'\\u%04x'" % x) + val = unidecode(u) -main2() + # f.write("%x | " % x) + if x == 0x2028: # U+2028 = LINE SEPARATOR + val = "" + elif x == 0x2029: # U+2029 = PARAGRAPH SEPARATOR + val = "" + f.write("%s\n" % val) + f.close() +main() diff --git a/lib/pure/unidecode/unidecode.dat b/lib/pure/unidecode/unidecode.dat index 9dff0a4a9..5f4c075d8 100644 --- a/lib/pure/unidecode/unidecode.dat +++ b/lib/pure/unidecode/unidecode.dat @@ -58,9 +58,9 @@ P 1 o >> -1/4 -1/2 -3/4 + 1/4 + 1/2 + 3/4 ? A A @@ -91,7 +91,7 @@ U U U U -U +Y Th ss a @@ -177,7 +177,7 @@ i I i IJ - +ij J j K @@ -368,7 +368,7 @@ ZH zh j DZ -D +Dz dz G g @@ -414,8 +414,8 @@ Y y H h -[?] -[?] +N +d OU ou Z @@ -434,34 +434,34 @@ O o Y y +l +n +t +j +db +qp +A +C +c +L +T +s +z [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +B +U +^ +E +e +J +j +q +q +R +r +Y +y a a a @@ -503,13 +503,13 @@ o OE O F -R -R -R -R r r -R +r +r +r +r +r R R s @@ -519,12 +519,12 @@ S S t t -U +u U v ^ -W -Y +w +y Y z z @@ -556,9 +556,9 @@ ls lz WW ]] -[?] -[?] -k +h +h +h h j r @@ -737,19 +737,19 @@ V -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +a +e +i +o +u +c +d +h +m +r +t +v +x [?] [?] [?] @@ -1287,7 +1287,7 @@ o f ew [?] -. +: - [?] [?] @@ -1340,9 +1340,9 @@ o u ' +- - - +| : @@ -7402,41 +7402,41 @@ bh +b +d +f +m +n +p +r +r +s +t +z +g +p +b +d +f +g +k +l +m +n +p +r +s - - - - - - - - - - - - - - - - - - - - - - - - - - - +v +x +z @@ -7708,7 +7708,7 @@ a S [?] [?] -[?] +Ss [?] A a @@ -8109,9 +8109,6 @@ _ - - - %0 %00 @@ -8136,19 +8133,23 @@ _ / -[ ]- -[?] +?? ?! !? 7 PP (] [) +* [?] [?] [?] +% +~ [?] [?] [?] +'''' [?] [?] [?] @@ -8156,12 +8157,8 @@ PP [?] [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] + + [?] [?] [?] @@ -8178,7 +8175,7 @@ PP 0 - +i 4 @@ -8209,19 +8206,19 @@ n ( ) [?] +a +e +o +x [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +h +k +l +m +n +p +s +t [?] [?] [?] @@ -8237,26 +8234,26 @@ Rs W NS D -EU +EUR K T Dr -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +Pf +P +G +A +UAH +C| +L +Sm +T +Rs +L +M +m +R +l +BTC [?] [?] [?] @@ -8294,6 +8291,7 @@ Dr [?] + [?] [?] [?] @@ -8319,63 +8317,67 @@ Dr [?] [?] [?] -[?] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + a/c + a/s +C + c/o + c/u +g +H +H +H +h +I +I +L +l +N +No. +P +Q +R +R +R +(sm) +TEL +(tm) +Z +Z +K +A +B +C +e +e +E +F +F +M +o +i +FAX @@ -8385,25 +8387,20 @@ Dr [?] [?] [?] +D +d +e +i +j [?] [?] [?] [?] +F [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] + 1/7 + 1/9 + 1/10 1/3 2/3 1/5 @@ -8458,7 +8455,7 @@ D) [?] [?] [?] -[?] + 0/3 [?] [?] [?] @@ -8595,8 +8592,12 @@ V [?] [?] [?] +- [?] [?] +/ +\ +* [?] [?] [?] @@ -8608,6 +8609,7 @@ V [?] [?] [?] +| [?] [?] [?] @@ -8626,11 +8628,13 @@ V [?] [?] [?] +: [?] [?] [?] [?] [?] +~ [?] [?] [?] @@ -8670,17 +8674,10 @@ V [?] [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +<= +>= +<= +>= [?] [?] [?] @@ -8836,6 +8833,7 @@ V [?] [?] [?] +^ [?] [?] [?] @@ -8873,9 +8871,8 @@ V [?] [?] [?] -[?] -[?] -[?] +< +> [?] [?] [?] @@ -9185,166 +9182,166 @@ V [?] [?] [?] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] - +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +(1) +(2) +(3) +(4) +(5) +(6) +(7) +(8) +(9) +(10) +(11) +(12) +(13) +(14) +(15) +(16) +(17) +(18) +(19) +(20) +1. +2. +3. +4. +5. +6. +7. +8. +9. +10. +11. +12. +13. +14. +15. +16. +17. +18. +19. +20. +(a) +(b) +(c) +(d) +(e) +(f) +(g) +(h) +(i) +(j) +(k) +(l) +(m) +(n) +(o) +(p) +(q) +(r) +(s) +(t) +(u) +(v) +(w) +(x) +(y) +(z) +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +0 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +0 - - | @@ -9712,7 +9709,7 @@ O - +# [?] @@ -9906,6 +9903,7 @@ O +* @@ -9944,8 +9942,7 @@ O - - +| @@ -9955,7 +9952,7 @@ O [?] [?] - +! @@ -10087,10 +10084,10 @@ O [?] [?] [?] +[ [?] -[?] -[?] -[?] +< +> [?] [?] [?] @@ -10500,6 +10497,8 @@ y +{ +} @@ -10739,6 +10738,9 @@ y +::= +== +=== @@ -11228,27 +11230,22 @@ y +L +l +L +P +R +a +t +H +h +K +k +Z +z - - - - - - - - - - - - - - - - - - - - +M +A @@ -12754,21 +12751,21 @@ H [?] [?] [?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 (g) (n) (d) @@ -12850,21 +12847,21 @@ KIS (Zi) (Xie) (Ye) -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] -[?] +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 1M 2M 3M @@ -12877,10 +12874,10 @@ KIS 10M 11M 12M -[?] -[?] -[?] -[?] +Hg +erg +eV +LTD a i u @@ -13042,16 +13039,16 @@ watt 22h 23h 24h -HPA +hPa da AU bar oV pc -[?] -[?] -[?] -[?] +dm +dm^2 +dm^3 +IU Heisei Syouwa Taisyou @@ -13092,7 +13089,7 @@ mm^2 cm^2 m^2 km^2 -mm^4 +mm^3 cm^3 m^3 km^3 @@ -13184,7 +13181,7 @@ Wb 29d 30d 31d - +gal @@ -19841,7 +19838,7 @@ Wb [?] [?] -[?] +Yi Ding Kao Qi diff --git a/lib/pure/unidecode/unidecode.nim b/lib/pure/unidecode/unidecode.nim index 9d8843f06..9affc53f6 100644 --- a/lib/pure/unidecode/unidecode.nim +++ b/lib/pure/unidecode/unidecode.nim @@ -7,30 +7,31 @@ # distribution, for details about the copyright. # -## This module is based on Python's Unidecode module by Tomaz Solc, -## which in turn is based on the ``Text::Unidecode`` Perl module by -## Sean M. Burke -## (http://search.cpan.org/~sburke/Text-Unidecode-0.04/lib/Text/Unidecode.pm ). +## This module is based on Python's [Unidecode](https://pypi.org/project/Unidecode/) +## module by Tomaz Solc, which in turn is based on the +## [Text::Unidecode](https://metacpan.org/pod/Text::Unidecode) +## Perl module by Sean M. Burke. ## -## It provides a single proc that does Unicode to ASCII transliterations: -## It finds the sequence of ASCII characters that is the closest approximation -## to the Unicode string. +## It provides a `unidecode proc <#unidecode,string>`_ that does +## Unicode to ASCII transliterations: It finds the sequence of ASCII characters +## that is the closest approximation to the Unicode string. ## ## For example, the closest to string "Äußerst" in ASCII is "Ausserst". Some ## information is lost in this transformation, of course, since several Unicode -## strings can be transformed in the same ASCII representation. So this is a -## strictly one-way transformation. However a human reader will probably -## still be able to guess what original string was meant from the context. +## strings can be transformed to the same ASCII representation. So this is a +## strictly one-way transformation. However, a human reader will probably +## still be able to guess from the context, what the original string was. ## -## This module needs the data file "unidecode.dat" to work: You can either -## ship this file with your application and initialize this module with the -## `loadUnidecodeTable` proc or you can define the ``embedUnidecodeTable`` -## symbol to embed the file as a resource into your application. +## This module needs the data file `unidecode.dat` to work: This file is +## embedded as a resource into your application by default. You can also +## define the symbol `--define:noUnidecodeTable` during compile time and +## use the `loadUnidecodeTable proc <#loadUnidecodeTable>`_ to initialize +## this module. -import unicode +import std/unicode -when defined(embedUnidecodeTable): - import strutils +when not defined(noUnidecodeTable): + import std/strutils const translationTable = splitLines(slurp"unidecode/unidecode.dat") else: @@ -38,37 +39,26 @@ else: var translationTable: seq[string] proc loadUnidecodeTable*(datafile = "unidecode.dat") = - ## loads the datafile that `unidecode` to work. Unless this module is - ## compiled with the ``embedUnidecodeTable`` symbol defined, this needs - ## to be called by the main thread before any thread can make a call - ## to `unidecode`. - when not defined(embedUnidecodeTable): + ## Loads the datafile that `unidecode <#unidecode,string>`_ needs to work. + ## This is only required if the module was compiled with the + ## `--define:noUnidecodeTable` switch. This needs to be called by the + ## main thread before any thread can make a call to `unidecode`. + when defined(noUnidecodeTable): newSeq(translationTable, 0xffff) var i = 0 for line in lines(datafile): - translationTable[i] = line.string + translationTable[i] = line inc(i) proc unidecode*(s: string): string = ## Finds the sequence of ASCII characters that is the closest approximation ## to the UTF-8 string `s`. - ## - ## Example: - ## - ## ..code-block:: nim - ## - ## unidecode("\x53\x17\x4E\xB0") - ## - ## Results in: "Bei Jing" - ## - assert(not isNil(translationTable)) + runnableExamples: + doAssert unidecode("北京") == "Bei Jing " + doAssert unidecode("Äußerst") == "Ausserst" + result = "" for r in runes(s): var c = int(r) if c <=% 127: add(result, chr(c)) - elif c <% translationTable.len: add(result, translationTable[c-128]) - -when isMainModule: - loadUnidecodeTable("lib/pure/unidecode/unidecode.dat") - assert unidecode("Äußerst") == "Ausserst" - + elif c <% translationTable.len: add(result, translationTable[c - 128]) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index aca9d51e2..cfb762258 100755..100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -11,11 +11,74 @@ ## ## This module implements boilerplate to make unit testing easy. ## -## Example: +## The test status and name is printed after any output or traceback. ## -## .. code:: nim +## Tests can be nested, however failure of a nested test will not mark the +## parent test as failed. Setup and teardown are inherited. Setup can be +## overridden locally. ## +## Compiled test files as well as `nim c -r <testfile.nim>` +## exit with 0 for success (no failed tests) or 1 for failure. +## +## Testament +## ========= +## +## Instead of `unittest`, please consider using +## `the Testament tool <testament.html>`_ which offers process isolation for your tests. +## +## Alternatively using `when isMainModule: doAssert conditionHere` is usually a +## much simpler solution for testing purposes. +## +## Running a single test +## ===================== +## +## Specify the test name as a command line argument. +## +## ```cmd +## nim c -r test "my test name" "another test" +## ``` +## +## Multiple arguments can be used. +## +## Running a single test suite +## =========================== +## +## Specify the suite name delimited by `"::"`. +## +## ```cmd +## nim c -r test "my test name::" +## ``` +## +## Selecting tests by pattern +## ========================== +## +## A single ``"*"`` can be used for globbing. +## +## Delimit the end of a suite name with `"::"`. +## +## Tests matching **any** of the arguments are executed. +## +## ```cmd +## nim c -r test fast_suite::mytest1 fast_suite::mytest2 +## nim c -r test "fast_suite::mytest*" +## nim c -r test "auth*::" "crypto::hashing*" +## # Run suites starting with 'bug #' and standalone tests starting with '#' +## nim c -r test 'bug #*::' '::#*' +## ``` +## +## Examples +## ======== +## +## ```nim ## suite "description for this stuff": +## echo "suite setup: run once before the tests" +## +## setup: +## echo "run before each test" +## +## teardown: +## echo "run after each test" +## ## test "essential truths": ## # give up and stop if this fails ## require(true) @@ -28,52 +91,381 @@ ## ## test "out of bounds error is thrown on bad access": ## let v = @[1, 2, 3] # you can do initialization here -## expect(IndexError): +## expect(IndexDefect): ## discard v[4] +## +## echo "suite teardown: run once after the tests" +## ``` +## +## Limitations/Bugs +## ================ +## Since `check` will rewrite some expressions for supporting checkpoints +## (namely assigns expressions to variables), some type conversions are not supported. +## For example `check 4.0 == 2 + 2` won't work. But `doAssert 4.0 == 2 + 2` works. +## Make sure both sides of the operator (such as `==`, `>=` and so on) have the same type. +## -import - macros +import std/private/since +import std/exitprocs -when declared(stdout): - import os +when defined(nimPreviewSlimSystem): + import std/assertions -when not defined(ECMAScript): - import terminal +import std/[macros, strutils, streams, times, sets, sequtils] -type - TestStatus* = enum OK, FAILED ## The status of a test when it is done. - OutputLevel* = enum ## The output verbosity of the tests. - PRINT_ALL, ## Print as much as possible. - PRINT_FAILURES, ## Print only the failed tests. - PRINT_NONE ## Print nothing. +when declared(stdout): + import std/os -{.deprecated: [TTestStatus: TestStatus, TOutputLevel: OutputLevel]} +const useTerminal = not defined(js) -var ## Global unittest settings! +when useTerminal: + import std/terminal +type + TestStatus* = enum ## The status of a test when it is done. + OK, + FAILED, + SKIPPED + + OutputLevel* = enum ## The output verbosity of the tests. + PRINT_ALL, ## Print as much as possible. + PRINT_FAILURES, ## Print only the failed tests. + PRINT_NONE ## Print nothing. + + TestResult* = object + suiteName*: string + ## Name of the test suite that contains this test case. + ## Can be ``nil`` if the test case is not in a suite. + testName*: string + ## Name of the test case + status*: TestStatus + + OutputFormatter* = ref object of RootObj + + ConsoleOutputFormatter* = ref object of OutputFormatter + colorOutput: bool + ## Have test results printed in color. + ## Default is `auto` depending on `isatty(stdout)`, or override it with + ## `-d:nimUnittestColor:auto|on|off`. + ## + ## Deprecated: Setting the environment variable `NIMTEST_COLOR` to `always` + ## or `never` changes the default for the non-js target to true or false respectively. + ## Deprecated: the environment variable `NIMTEST_NO_COLOR`, when set, changes the + ## default to true, if `NIMTEST_COLOR` is undefined. + outputLevel: OutputLevel + ## Set the verbosity of test results. + ## Default is `PRINT_ALL`, or override with: + ## `-d:nimUnittestOutputLevel:PRINT_ALL|PRINT_FAILURES|PRINT_NONE`. + ## + ## Deprecated: the `NIMTEST_OUTPUT_LVL` environment variable is set for the non-js target. + isInSuite: bool + isInTest: bool + + JUnitOutputFormatter* = ref object of OutputFormatter + stream: Stream + testErrors: seq[string] + testStartTime: float + testStackTrace: string + +var abortOnError* {.threadvar.}: bool ## Set to true in order to quit ## immediately on fail. Default is false, - ## unless the ``NIMTEST_ABORT_ON_ERROR`` - ## environment variable is set for - ## the non-js target. - outputLevel* {.threadvar.}: OutputLevel ## Set the verbosity of test results. - ## Default is ``PRINT_ALL``, unless - ## the ``NIMTEST_OUTPUT_LVL`` environment - ## variable is set for the non-js target. - - colorOutput* {.threadvar.}: bool ## Have test results printed in color. - ## Default is true for the non-js target - ## unless, the environment variable - ## ``NIMTEST_NO_COLOR`` is set. + ## or override with `-d:nimUnittestAbortOnError:on|off`. + ## + ## Deprecated: can also override depending on whether + ## `NIMTEST_ABORT_ON_ERROR` environment variable is set. checkpoints {.threadvar.}: seq[string] + formatters {.threadvar.}: seq[OutputFormatter] + testsFilters {.threadvar.}: HashSet[string] + disabledParamFiltering {.threadvar.}: bool + +const + outputLevelDefault = PRINT_ALL + nimUnittestOutputLevel {.strdefine.} = $outputLevelDefault + nimUnittestColor {.strdefine.} = "auto" ## auto|on|off + nimUnittestAbortOnError {.booldefine.} = false -checkpoints = @[] +template deprecateEnvVarHere() = + # xxx issue a runtime warning to deprecate this envvar. + discard + +abortOnError = nimUnittestAbortOnError +when declared(stdout): + if existsEnv("NIMTEST_ABORT_ON_ERROR"): + deprecateEnvVarHere() + abortOnError = true -proc shouldRun(testName: string): bool = - result = true +method suiteStarted*(formatter: OutputFormatter, suiteName: string) {.base, gcsafe.} = + discard +method testStarted*(formatter: OutputFormatter, testName: string) {.base, gcsafe.} = + discard +method failureOccurred*(formatter: OutputFormatter, checkpoints: seq[string], + stackTrace: string) {.base, gcsafe.} = + ## ``stackTrace`` is provided only if the failure occurred due to an exception. + ## ``checkpoints`` is never ``nil``. + discard +method testEnded*(formatter: OutputFormatter, testResult: TestResult) {.base, gcsafe.} = + discard +method suiteEnded*(formatter: OutputFormatter) {.base, gcsafe.} = + discard + +proc addOutputFormatter*(formatter: OutputFormatter) = + formatters.add(formatter) + +proc delOutputFormatter*(formatter: OutputFormatter) = + keepIf(formatters, proc (x: OutputFormatter): bool = + x != formatter) + +proc resetOutputFormatters* {.since: (1, 1).} = + formatters = @[] + +proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault, + colorOutput = true): ConsoleOutputFormatter = + ConsoleOutputFormatter( + outputLevel: outputLevel, + colorOutput: colorOutput + ) + +proc colorOutput(): bool = + let color = nimUnittestColor + case color + of "auto": + when declared(stdout): result = isatty(stdout) + else: result = false + of "on": result = true + of "off": result = false + else: raiseAssert $color + + when declared(stdout): + if existsEnv("NIMTEST_COLOR"): + deprecateEnvVarHere() + let colorEnv = getEnv("NIMTEST_COLOR") + if colorEnv == "never": + result = false + elif colorEnv == "always": + result = true + elif existsEnv("NIMTEST_NO_COLOR"): + deprecateEnvVarHere() + result = false + +proc defaultConsoleFormatter*(): ConsoleOutputFormatter = + var colorOutput = colorOutput() + var outputLevel = nimUnittestOutputLevel.parseEnum[:OutputLevel] + when declared(stdout): + const a = "NIMTEST_OUTPUT_LVL" + if existsEnv(a): + # xxx issue a warning to deprecate this envvar. + outputLevel = getEnv(a).parseEnum[:OutputLevel] + result = newConsoleOutputFormatter(outputLevel, colorOutput) + +method suiteStarted*(formatter: ConsoleOutputFormatter, suiteName: string) = + template rawPrint() = echo("\n[Suite] ", suiteName) + when useTerminal: + if formatter.colorOutput: + styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, suiteName + else: rawPrint() + else: rawPrint() + formatter.isInSuite = true + +method testStarted*(formatter: ConsoleOutputFormatter, testName: string) = + formatter.isInTest = true + +method failureOccurred*(formatter: ConsoleOutputFormatter, + checkpoints: seq[string], stackTrace: string) = + if stackTrace.len > 0: + echo stackTrace + let prefix = if formatter.isInSuite: " " else: "" + for msg in items(checkpoints): + echo prefix, msg + +method testEnded*(formatter: ConsoleOutputFormatter, testResult: TestResult) = + formatter.isInTest = false + + if formatter.outputLevel != OutputLevel.PRINT_NONE and + (formatter.outputLevel == OutputLevel.PRINT_ALL or testResult.status == TestStatus.FAILED): + let prefix = if testResult.suiteName.len > 0: " " else: "" + template rawPrint() = echo(prefix, "[", $testResult.status, "] ", + testResult.testName) + when useTerminal: + if formatter.colorOutput: + var color = case testResult.status + of TestStatus.OK: fgGreen + of TestStatus.FAILED: fgRed + of TestStatus.SKIPPED: fgYellow + styledEcho styleBright, color, prefix, "[", $testResult.status, "] ", + resetStyle, testResult.testName + else: + rawPrint() + else: + rawPrint() -template suite*(name: expr, body: stmt): stmt {.immediate, dirty.} = +method suiteEnded*(formatter: ConsoleOutputFormatter) = + formatter.isInSuite = false + +proc xmlEscape(s: string): string = + result = newStringOfCap(s.len) + for c in items(s): + case c: + of '<': result.add("<") + of '>': result.add(">") + of '&': result.add("&") + of '"': result.add(""") + of '\'': result.add("'") + else: + if ord(c) < 32: + result.add("&#" & $ord(c) & ';') + else: + result.add(c) + +proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = + ## Creates a formatter that writes report to the specified stream in + ## JUnit format. + ## The ``stream`` is NOT closed automatically when the test are finished, + ## because the formatter has no way to know when all tests are finished. + ## You should invoke formatter.close() to finalize the report. + result = JUnitOutputFormatter( + stream: stream, + testErrors: @[], + testStackTrace: "", + testStartTime: 0.0 + ) + stream.writeLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>") + stream.writeLine("<testsuites>") + +proc close*(formatter: JUnitOutputFormatter) = + ## Completes the report and closes the underlying stream. + formatter.stream.writeLine("</testsuites>") + formatter.stream.close() + +method suiteStarted*(formatter: JUnitOutputFormatter, suiteName: string) = + formatter.stream.writeLine("\t<testsuite name=\"$1\">" % xmlEscape(suiteName)) + +method testStarted*(formatter: JUnitOutputFormatter, testName: string) = + formatter.testErrors.setLen(0) + formatter.testStackTrace.setLen(0) + formatter.testStartTime = epochTime() + +method failureOccurred*(formatter: JUnitOutputFormatter, + checkpoints: seq[string], stackTrace: string) = + ## ``stackTrace`` is provided only if the failure occurred due to an exception. + ## ``checkpoints`` is never ``nil``. + formatter.testErrors.add(checkpoints) + if stackTrace.len > 0: + formatter.testStackTrace = stackTrace + +method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) = + let time = epochTime() - formatter.testStartTime + let timeStr = time.formatFloat(ffDecimal, precision = 8) + formatter.stream.writeLine("\t\t<testcase name=\"$#\" time=\"$#\">" % [ + xmlEscape(testResult.testName), timeStr]) + case testResult.status + of TestStatus.OK: + discard + of TestStatus.SKIPPED: + formatter.stream.writeLine("<skipped />") + of TestStatus.FAILED: + let failureMsg = if formatter.testStackTrace.len > 0 and + formatter.testErrors.len > 0: + xmlEscape(formatter.testErrors[^1]) + elif formatter.testErrors.len > 0: + xmlEscape(formatter.testErrors[0]) + else: "The test failed without outputting an error" + + var errs = "" + if formatter.testErrors.len > 1: + var startIdx = if formatter.testStackTrace.len > 0: 0 else: 1 + var endIdx = if formatter.testStackTrace.len > 0: + formatter.testErrors.len - 2 + else: formatter.testErrors.len - 1 + + for errIdx in startIdx..endIdx: + if errs.len > 0: + errs.add("\n") + errs.add(xmlEscape(formatter.testErrors[errIdx])) + + if formatter.testStackTrace.len > 0: + formatter.stream.writeLine("\t\t\t<error message=\"$#\">$#</error>" % [ + failureMsg, xmlEscape(formatter.testStackTrace)]) + if errs.len > 0: + formatter.stream.writeLine("\t\t\t<system-err>$#</system-err>" % errs) + else: + formatter.stream.writeLine("\t\t\t<failure message=\"$#\">$#</failure>" % + [failureMsg, errs]) + + formatter.stream.writeLine("\t\t</testcase>") + +method suiteEnded*(formatter: JUnitOutputFormatter) = + formatter.stream.writeLine("\t</testsuite>") + +proc glob(matcher, filter: string): bool = + ## Globbing using a single `*`. Empty `filter` matches everything. + if filter.len == 0: + return true + + if not filter.contains('*'): + return matcher == filter + + let beforeAndAfter = filter.split('*', maxsplit = 1) + if beforeAndAfter.len == 1: + # "foo*" + return matcher.startsWith(beforeAndAfter[0]) + + if matcher.len < filter.len - 1: + return false # "12345" should not match "123*345" + + return matcher.startsWith(beforeAndAfter[0]) and matcher.endsWith( + beforeAndAfter[1]) + +proc matchFilter(suiteName, testName, filter: string): bool = + if filter == "": + return true + if testName == filter: + # corner case for tests containing "::" in their name + return true + let suiteAndTestFilters = filter.split("::", maxsplit = 1) + + if suiteAndTestFilters.len == 1: + # no suite specified + let testFilter = suiteAndTestFilters[0] + return glob(testName, testFilter) + + return glob(suiteName, suiteAndTestFilters[0]) and + glob(testName, suiteAndTestFilters[1]) + +proc shouldRun(currentSuiteName, testName: string): bool = + ## Check if a test should be run by matching suiteName and testName against + ## test filters. + if testsFilters.len == 0: + return true + + for f in testsFilters: + if matchFilter(currentSuiteName, testName, f): + return true + + return false + +proc ensureInitialized() = + if formatters.len == 0: + formatters = @[OutputFormatter(defaultConsoleFormatter())] + + if not disabledParamFiltering: + when declared(paramCount): + # Read tests to run from the command line. + for i in 1 .. paramCount(): + testsFilters.incl(paramStr(i)) + +# These two procs are added as workarounds for +# https://github.com/nim-lang/Nim/issues/5549 +proc suiteEnded() = + for formatter in formatters: + formatter.suiteEnded() + +proc testEnded(testResult: TestResult) = + for formatter in formatters: + formatter.testEnded(testResult) + +template suite*(name, body) {.dirty.} = ## Declare a test suite identified by `name` with optional ``setup`` ## and/or ``teardown`` section. ## @@ -81,96 +473,120 @@ template suite*(name: expr, body: stmt): stmt {.immediate, dirty.} = ## common fixture (``setup``, ``teardown``). The fixture is executed ## for EACH test. ## - ## .. code-block:: nim - ## suite "test suite for addition": - ## setup: - ## let result = 4 + ## ```nim + ## suite "test suite for addition": + ## setup: + ## let result = 4 ## - ## test "2 + 2 = 4": - ## check(2+2 == result) + ## test "2 + 2 = 4": + ## check(2+2 == result) ## - ## test "(2 + -2) != 4": - ## check(2 + -2 != result) + ## test "(2 + -2) != 4": + ## check(2 + -2 != result) ## - ## # No teardown needed + ## # No teardown needed + ## ``` ## ## The suite will run the individual test cases in the order in which ## they were listed. With default global settings the above code prints: ## - ## .. code-block:: - ## - ## [OK] 2 + 2 = 4 - ## [OK] (2 + -2) != 4 + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 + bind formatters, ensureInitialized, suiteEnded + block: - template setup(setupBody: stmt): stmt {.immediate, dirty.} = - var testSetupIMPLFlag = true - template testSetupIMPL: stmt {.immediate, dirty.} = setupBody - - template teardown(teardownBody: stmt): stmt {.immediate, dirty.} = - var testTeardownIMPLFlag = true - template testTeardownIMPL: stmt {.immediate, dirty.} = teardownBody - - body - -proc testDone(name: string, s: TestStatus) = - if s == FAILED: - programResult += 1 - - if outputLevel != PRINT_NONE and (outputLevel == PRINT_ALL or s == FAILED): - template rawPrint() = echo("[", $s, "] ", name) - when not defined(ECMAScript): - if colorOutput and not defined(ECMAScript): - var color = (if s == OK: fgGreen else: fgRed) - styledEcho styleBright, color, "[", $s, "] ", fgWhite, name - else: - rawPrint() - else: - rawPrint() + template setup(setupBody: untyped) {.dirty, used.} = + var testSetupIMPLFlag {.used.} = true + template testSetupIMPL: untyped {.dirty.} = setupBody + + template teardown(teardownBody: untyped) {.dirty, used.} = + var testTeardownIMPLFlag {.used.} = true + template testTeardownIMPL: untyped {.dirty.} = teardownBody -template test*(name: expr, body: stmt): stmt {.immediate, dirty.} = + let testSuiteName {.used.} = name + + ensureInitialized() + try: + for formatter in formatters: + formatter.suiteStarted(name) + body + finally: + suiteEnded() + +proc exceptionTypeName(e: ref Exception): string {.inline.} = + if e == nil: "<foreign exception>" + else: $e.name + +when not declared(setProgramResult): + {.warning: "setProgramResult not available on platform, unittest will not" & + " give failing exit code on test failure".} + template setProgramResult(a: int) = + discard + +template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## - ## .. code-block:: nim - ## - ## test "roses are red": - ## let roses = "red" - ## check(roses == "red") + ## ```nim + ## test "roses are red": + ## let roses = "red" + ## check(roses == "red") + ## ``` ## ## The above code outputs: ## - ## .. code-block:: - ## - ## [OK] roses are red - bind shouldRun, checkpoints, testDone + ## [OK] roses are red + bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName, setProgramResult - if shouldRun(name): + ensureInitialized() + + if shouldRun(when declared(testSuiteName): testSuiteName else: "", name): checkpoints = @[] - var testStatusIMPL {.inject.} = OK + var testStatusIMPL {.inject.} = TestStatus.OK + + for formatter in formatters: + formatter.testStarted(name) + {.push warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() - body when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.push warning[BareExcept]:on.} + body + {.pop.} except: - when not defined(js): - checkpoint("Unhandled exception: " & getCurrentExceptionMsg()) - echo getCurrentException().getStackTrace() - fail() + let e = getCurrentException() + let eTypeDesc = "[" & exceptionTypeName(e) & "]" + checkpoint("Unhandled exception: " & getCurrentExceptionMsg() & " " & eTypeDesc) + if e == nil: # foreign + fail() + else: + var stackTrace {.inject.} = e.getStackTrace() + fail() finally: - testDone name, testStatusIMPL + if testStatusIMPL == TestStatus.FAILED: + setProgramResult 1 + let testResult = TestResult( + suiteName: when declared(testSuiteName): testSuiteName else: "", + testName: name, + status: testStatusIMPL + ) + testEnded(testResult) + checkpoints = @[] + {.pop.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all ## checkpoints encountered so far are printed out. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## check((42, "the Answer to life and everything") == (1, "a")) - ## checkpoint("Checkpoint B") + ## ```nim + ## checkpoint("Checkpoint A") + ## check((42, "the Answer to life and everything") == (1, "a")) + ## checkpoint("Checkpoint B") + ## ``` ## ## outputs "Checkpoint A" once it fails. checkpoints.add(msg) @@ -182,117 +598,144 @@ template fail* = ## failed (change exit code and test status). This template is useful ## for debugging, but is otherwise mostly used internally. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## complicatedProcInThread() - ## fail() + ## ```nim + ## checkpoint("Checkpoint A") + ## complicatedProcInThread() + ## fail() + ## ``` ## ## outputs "Checkpoint A" before quitting. - bind checkpoints - for msg in items(checkpoints): - echo msg - - when not defined(ECMAScript): - if abortOnError: quit(1) - + bind ensureInitialized, setProgramResult when declared(testStatusIMPL): - testStatusIMPL = FAILED + testStatusIMPL = TestStatus.FAILED else: - programResult += 1 + setProgramResult 1 + + ensureInitialized() + + # var stackTrace: string = nil + for formatter in formatters: + when declared(stackTrace): + formatter.failureOccurred(checkpoints, stackTrace) + else: + formatter.failureOccurred(checkpoints, "") + + if abortOnError: quit(1) checkpoints = @[] -macro check*(conditions: stmt): stmt {.immediate.} = +template skip* = + ## Mark the test as skipped. Should be used directly + ## in case when it is not possible to perform test + ## for reasons depending on outer environment, + ## or certain application logic conditions or configurations. + ## The test code is still executed. + ## ```nim + ## if not isGLContextCreated(): + ## skip() + ## ``` + bind checkpoints + + testStatusIMPL = TestStatus.SKIPPED + checkpoints = @[] + +macro check*(conditions: untyped): untyped = ## Verify if a statement or a list of statements is true. ## A helpful error message and set checkpoints are printed out on ## failure (if ``outputLevel`` is not ``PRINT_NONE``). - ## Example: - ## - ## .. code-block:: nim - ## - ## import strutils - ## - ## check("AKB48".toLower() == "akb48") - ## - ## let teams = {'A', 'K', 'B', '4', '8'} - ## - ## check: - ## "AKB48".toLower() == "akb48" - ## 'C' in teams + runnableExamples: + import std/strutils + + check("AKB48".toLowerAscii() == "akb48") + + let teams = {'A', 'K', 'B', '4', '8'} + + check: + "AKB48".toLowerAscii() == "akb48" + 'C' notin teams + let checked = callsite()[1] - var - argsAsgns = newNimNode(nnkStmtList) - argsPrintOuts = newNimNode(nnkStmtList) - counter = 0 - template asgn(a, value: expr): stmt = + template asgn(a: untyped, value: typed) = var a = value # XXX: we need "var: var" here in order to # preserve the semantics of var params - template print(name, value: expr): stmt = + template print(name: untyped, value: typed) = when compiles(string($value)): checkpoint(name & " was " & $value) - proc inspectArgs(exp: NimNode): NimNode = - result = copyNimTree(exp) - for i in countup(1, exp.len - 1): - if exp[i].kind notin nnkLiterals: - inc counter - var arg = newIdentNode(":p" & $counter) - var argStr = exp[i].toStrLit - var paramAst = exp[i] - if exp[i].kind == nnkIdent: - argsPrintOuts.add getAst(print(argStr, paramAst)) - if exp[i].kind in nnkCallKinds: - var callVar = newIdentNode(":c" & $counter) - argsAsgns.add getAst(asgn(callVar, paramAst)) - result[i] = callVar - argsPrintOuts.add getAst(print(argStr, callVar)) - if exp[i].kind == nnkExprEqExpr: - # ExprEqExpr - # Ident !"v" - # IntLit 2 - result[i] = exp[i][1] - if exp[i].typekind notin {ntyTypeDesc}: - argsAsgns.add getAst(asgn(arg, paramAst)) - argsPrintOuts.add getAst(print(argStr, arg)) - if exp[i].kind != nnkExprEqExpr: - result[i] = arg - else: - result[i][1] = arg + proc inspectArgs(exp: NimNode): tuple[assigns, check, printOuts: NimNode] = + result.check = copyNimTree(exp) + result.assigns = newNimNode(nnkStmtList) + result.printOuts = newNimNode(nnkStmtList) + + var counter = 0 + + if exp[0].kind in {nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym} and + $exp[0] in ["not", "in", "notin", "==", "<=", + ">=", "<", ">", "!=", "is", "isnot"]: + + for i in 1 ..< exp.len: + if exp[i].kind notin nnkLiterals: + inc counter + let argStr = exp[i].toStrLit + let paramAst = exp[i] + if exp[i].kind == nnkIdent: + result.printOuts.add getAst(print(argStr, paramAst)) + if exp[i].kind in nnkCallKinds + {nnkDotExpr, nnkBracketExpr, nnkPar} and + (exp[i].typeKind notin {ntyTypeDesc} or $exp[0] notin ["is", "isnot"]): + let callVar = newIdentNode(":c" & $counter) + result.assigns.add getAst(asgn(callVar, paramAst)) + result.check[i] = callVar + result.printOuts.add getAst(print(argStr, callVar)) + if exp[i].kind == nnkExprEqExpr: + # ExprEqExpr + # Ident "v" + # IntLit 2 + result.check[i] = exp[i][1] + if exp[i].typeKind notin {ntyTypeDesc}: + let arg = newIdentNode(":p" & $counter) + result.assigns.add getAst(asgn(arg, paramAst)) + result.printOuts.add getAst(print(argStr, arg)) + if exp[i].kind != nnkExprEqExpr: + result.check[i] = arg + else: + result.check[i][1] = arg case checked.kind of nnkCallKinds: - template rewrite(call, lineInfoLit: expr, callLit: string, - argAssgs, argPrintOuts: stmt): stmt = + + let (assigns, check, printOuts) = inspectArgs(checked) + let lineinfo = newStrLitNode(checked.lineInfo) + let callLit = checked.toStrLit + result = quote do: block: - argAssgs #all callables (and assignments) are run here - if not call: - checkpoint(lineInfoLit & ": Check failed: " & callLit) - argPrintOuts + `assigns` + if `check`: + discard + else: + checkpoint(`lineinfo` & ": Check failed: " & `callLit`) + `printOuts` fail() - var checkedStr = checked.toStrLit - let parameterizedCheck = inspectArgs(checked) - result = getAst(rewrite(parameterizedCheck, checked.lineinfo, checkedStr, - argsAsgns, argsPrintOuts)) - of nnkStmtList: result = newNimNode(nnkStmtList) - for i in countup(0, checked.len - 1): - if checked[i].kind != nnkCommentStmt: - result.add(newCall(!"check", checked[i])) + for node in checked: + if node.kind != nnkCommentStmt: + result.add(newCall(newIdentNode("check"), node)) else: - template rewrite(Exp, lineInfoLit: expr, expLit: string): stmt = - if not Exp: - checkpoint(lineInfoLit & ": Check failed: " & expLit) - fail() + let lineinfo = newStrLitNode(checked.lineInfo) + let callLit = checked.toStrLit - result = getAst(rewrite(checked, checked.lineinfo, checked.toStrLit)) + result = quote do: + if `checked`: + discard + else: + checkpoint(`lineinfo` & ": Check failed: " & `callLit`) + fail() -template require*(conditions: stmt): stmt {.immediate.} = +template require*(conditions: untyped) = ## Same as `check` except any failed test causes the program to quit ## immediately. Any teardown statements are not executed and the failed ## test output is not generated. @@ -302,60 +745,45 @@ template require*(conditions: stmt): stmt {.immediate.} = check conditions abortOnError = savedAbortOnError -macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = +macro expect*(exceptions: varargs[typed], body: untyped): untyped = ## Test if `body` raises an exception found in the passed `exceptions`. ## The test passes if the raised exception is part of the acceptable ## exceptions. Otherwise, it fails. - ## Example: - ## - ## .. code-block:: nim - ## - ## import math - ## proc defectiveRobot() = - ## randomize() - ## case random(1..4) - ## of 1: raise newException(OSError, "CANNOT COMPUTE!") - ## of 2: discard parseInt("Hello World!") - ## of 3: raise newException(IOError, "I can't do that Dave.") - ## else: assert 2 + 2 == 5 - ## - ## expect IOError, OSError, ValueError, AssertionError: - ## defectiveRobot() - let exp = callsite() - template expectBody(errorTypes, lineInfoLit: expr, - body: stmt): NimNode {.dirty.} = + runnableExamples: + import std/[math, random, strutils] + proc defectiveRobot() = + randomize() + case rand(1..4) + of 1: raise newException(OSError, "CANNOT COMPUTE!") + of 2: discard parseInt("Hello World!") + of 3: raise newException(IOError, "I can't do that Dave.") + else: assert 2 + 2 == 5 + + expect IOError, OSError, ValueError, AssertionDefect: + defectiveRobot() + + template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} = + {.push warning[BareExcept]:off.} try: + {.push warning[BareExcept]:on.} body + {.pop.} checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") fail() except errorTypes: discard except: - checkpoint(lineInfoLit & ": Expect Failed, unexpected exception was thrown.") + let err = getCurrentException() + checkpoint(lineInfoLit & ": Expect Failed, " & $err.name & " was thrown.") fail() - - var body = exp[exp.len - 1] + {.pop.} var errorTypes = newNimNode(nnkBracket) - for i in countup(1, exp.len - 2): - errorTypes.add(exp[i]) - - result = getAst(expectBody(errorTypes, exp.lineinfo, body)) + for exp in exceptions: + errorTypes.add(exp) + result = getAst(expectBody(errorTypes, errorTypes.lineInfo, body)) -when declared(stdout): - # Reading settings - # On a terminal this branch is executed - var envOutLvl = os.getEnv("NIMTEST_OUTPUT_LVL").string - abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR") - colorOutput = not existsEnv("NIMTEST_NO_COLOR") - -else: - var envOutLvl = "" # TODO - colorOutput = false - -if envOutLvl.len > 0: - for opt in countup(low(OutputLevel), high(OutputLevel)): - if $opt == envOutLvl: - outputLevel = opt - break +proc disableParamFiltering* = + ## disables filtering tests with the command line params + disabledParamFiltering = true diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index abb1a462d..725d5bbd9 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -8,8 +8,43 @@ # ## This module implements URI parsing as specified by RFC 3986. +## +## A Uniform Resource Identifier (URI) provides a simple and extensible +## means for identifying a resource. A URI can be further classified +## as a locator, a name, or both. The term "Uniform Resource Locator" +## (URL) refers to the subset of URIs. +## +## .. warning:: URI parsers in this module do not perform security validation. +## +## # Basic usage + + +## ## Combine URIs +runnableExamples: + let host = parseUri("https://nim-lang.org") + assert $host == "https://nim-lang.org" + assert $(host / "/blog.html") == "https://nim-lang.org/blog.html" + assert $(host / "blog2.html") == "https://nim-lang.org/blog2.html" + +## ## Access URI item +runnableExamples: + let res = parseUri("sftp://127.0.0.1:4343") + assert isAbsolute(res) + assert res.port == "4343" + +## ## Data URI Base64 +runnableExamples: + assert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" + assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + + +import std/[strutils, parseutils, base64] +import std/private/[since, decode_helpers] + +when defined(nimPreviewSlimSystem): + import std/assertions + -import strutils, parseutils type Url* = distinct string @@ -17,40 +52,160 @@ type scheme*, username*, password*: string hostname*, port*, path*, query*, anchor*: string opaque*: bool + isIpv6*: bool + + UriParseError* = object of ValueError -{.deprecated: [TUrl: Url, TUri: Uri].} -{.push warning[deprecated]: off.} -proc `$`*(url: Url): string {.deprecated.} = - ## **Deprecated since 0.9.6**: Use ``Uri`` instead. - return string(url) +proc uriParseError*(msg: string) {.noreturn.} = + ## Raises a `UriParseError` exception with message `msg`. + raise newException(UriParseError, msg) -proc `/`*(a, b: Url): Url {.deprecated.} = - ## Joins two URLs together, separating them with / if needed. +func encodeUrl*(s: string, usePlus = true): string = + ## Encodes a URL according to RFC3986. ## - ## **Deprecated since 0.9.6**: Use ``Uri`` instead. - var urlS = $a - var bS = $b - if urlS == "": return b - if urlS[urlS.len-1] != '/': - urlS.add('/') - if bS[0] == '/': - urlS.add(bS.substr(1)) - else: - urlS.add(bs) - result = Url(urlS) + ## This means that characters in the set + ## `{'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~'}` are + ## carried over to the result. + ## All other characters are encoded as `%xx` where `xx` + ## denotes its hexadecimal value. + ## + ## As a special rule, when the value of `usePlus` is true, + ## spaces are encoded as `+` instead of `%20`. + ## + ## **See also:** + ## * `decodeUrl func<#decodeUrl,string>`_ + runnableExamples: + assert encodeUrl("https://nim-lang.org") == "https%3A%2F%2Fnim-lang.org" + assert encodeUrl("https://nim-lang.org/this is a test") == "https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test" + assert encodeUrl("https://nim-lang.org/this is a test", false) == "https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test" + result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars + let fromSpace = if usePlus: "+" else: "%20" + for c in s: + case c + # https://tools.ietf.org/html/rfc3986#section-2.3 + of 'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~': add(result, c) + of ' ': add(result, fromSpace) + else: + add(result, '%') + add(result, toHex(ord(c), 2)) -proc add*(url: var Url, a: Url) {.deprecated.} = - ## Appends url to url. +func decodeUrl*(s: string, decodePlus = true): string = + ## Decodes a URL according to RFC3986. + ## + ## This means that any `%xx` (where `xx` denotes a hexadecimal + ## value) are converted to the character with ordinal number `xx`, + ## and every other character is carried over. + ## If `xx` is not a valid hexadecimal value, it is left intact. + ## + ## As a special rule, when the value of `decodePlus` is true, `+` + ## characters are converted to a space. + ## + ## **See also:** + ## * `encodeUrl func<#encodeUrl,string>`_ + runnableExamples: + assert decodeUrl("https%3A%2F%2Fnim-lang.org") == "https://nim-lang.org" + assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test") == "https://nim-lang.org/this is a test" + assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test", + false) == "https://nim-lang.org/this is a test" + assert decodeUrl("abc%xyz") == "abc%xyz" + + result = newString(s.len) + var i = 0 + var j = 0 + while i < s.len: + case s[i] + of '%': + result[j] = decodePercent(s, i) + of '+': + if decodePlus: + result[j] = ' ' + else: + result[j] = s[i] + else: result[j] = s[i] + inc(i) + inc(j) + setLen(result, j) + +func encodeQuery*(query: openArray[(string, string)], usePlus = true, + omitEq = true, sep = '&'): string = + ## Encodes a set of (key, value) parameters into a URL query string. ## - ## **Deprecated since 0.9.6**: Use ``Uri`` instead. - url = url / a -{.pop.} + ## Every (key, value) pair is URL-encoded and written as `key=value`. If the + ## value is an empty string then the `=` is omitted, unless `omitEq` is + ## false. + ## The pairs are joined together by the `sep` character. + ## + ## The `usePlus` parameter is passed down to the `encodeUrl` function that + ## is used for the URL encoding of the string values. + ## + ## **See also:** + ## * `encodeUrl func<#encodeUrl,string>`_ + runnableExamples: + assert encodeQuery({: }) == "" + assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" + assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + assert encodeQuery({"a": "1", "b": ""}, omitEq = false, sep = ';') == "a=1;b=" + for elem in query: + # Encode the `key = value` pairs and separate them with 'sep' + if result.len > 0: result.add(sep) + let (key, val) = elem + result.add(encodeUrl(key, usePlus)) + # Omit the '=' if the value string is empty + if not omitEq or val.len > 0: + result.add('=') + result.add(encodeUrl(val, usePlus)) + +iterator decodeQuery*(data: string, sep = '&'): tuple[key, value: string] = + ## Reads and decodes the query string `data` and yields the `(key, value)` pairs + ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, + ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded + ## value, which was the behavior in Nim < 1.5.1. + runnableExamples: + import std/sequtils + assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("foo=1;bar=2=3", ';')) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] + + proc parseData(data: string, i: int, field: var string, sep: char): int = + result = i + while result < data.len: + let c = data[result] + case c + of '%': add(field, decodePercent(data, result)) + of '+': add(field, ' ') + of '&': break + else: + if c == sep: break + else: add(field, data[result]) + inc(result) -proc parseAuthority(authority: string, result: var Uri) = + var i = 0 + var name = "" + var value = "" + # decode everything in one pass: + while i < data.len: + setLen(name, 0) # reuse memory + i = parseData(data, i, name, '=') + setLen(value, 0) # reuse memory + if i < data.len and data[i] == '=': + inc(i) # skip '=' + when defined(nimLegacyParseQueryStrict): + i = parseData(data, i, value, '=') + else: + i = parseData(data, i, value, sep) + yield (name, value) + if i < data.len: + when defined(nimLegacyParseQueryStrict): + if data[i] != '&': + uriParseError("'&' expected at index '$#' for '$#'" % [$i, data]) + inc(i) + +func parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false - while true: + var inIPv6 = false + while i < authority.len: case authority[i] of '@': swap result.password, result.port @@ -59,8 +214,15 @@ proc parseAuthority(authority: string, result: var Uri) = result.hostname.setLen(0) inPort = false of ':': - inPort = true - of '\0': break + if inIPv6: + result.hostname.add(authority[i]) + else: + inPort = true + of '[': + inIPv6 = true + result.isIpv6 = true + of ']': + inIPv6 = false else: if inPort: result.port.add(authority[i]) @@ -68,52 +230,71 @@ proc parseAuthority(authority: string, result: var Uri) = result.hostname.add(authority[i]) i.inc -proc parsePath(uri: string, i: var int, result: var Uri) = - +func parsePath(uri: string, i: var int, result: var Uri) = i.inc parseUntil(uri, result.path, {'?', '#'}, i) # The 'mailto' scheme's PATH actually contains the hostname/username - if result.scheme.toLower == "mailto": + if cmpIgnoreCase(result.scheme, "mailto") == 0: parseAuthority(result.path, result) result.path.setLen(0) - if uri[i] == '?': + if i < uri.len and uri[i] == '?': i.inc # Skip '?' i.inc parseUntil(uri, result.query, {'#'}, i) - if uri[i] == '#': + if i < uri.len and uri[i] == '#': i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) -proc initUri*(): Uri = - ## Initializes a URI. +func initUri*(isIpv6 = false): Uri = + ## Initializes a URI with `scheme`, `username`, `password`, + ## `hostname`, `port`, `path`, `query`, `anchor` and `isIpv6`. + ## + ## **See also:** + ## * `Uri type <#Uri>`_ for available fields in the URI type + runnableExamples: + var uri2 = initUri(isIpv6 = true) + uri2.scheme = "tcp" + uri2.hostname = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + uri2.port = "8080" + assert $uri2 == "tcp://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8080" result = Uri(scheme: "", username: "", password: "", hostname: "", port: "", - path: "", query: "", anchor: "") + path: "", query: "", anchor: "", isIpv6: isIpv6) -proc resetUri(uri: var Uri) = +func resetUri(uri: var Uri) = for f in uri.fields: when f is string: f.setLen(0) else: f = false -proc parseUri*(uri: string, result: var Uri) = +func parseUri*(uri: string, result: var Uri) = ## Parses a URI. The `result` variable will be cleared before. + ## + ## **See also:** + ## * `Uri type <#Uri>`_ for available fields in the URI type + ## * `initUri func <#initUri>`_ for initializing a URI + runnableExamples: + var res = initUri() + parseUri("https://nim-lang.org/docs/manual.html", res) + assert res.scheme == "https" + assert res.hostname == "nim-lang.org" + assert res.path == "/docs/manual.html" resetUri(result) var i = 0 # Check if this is a reference URI (relative URI) - let doubleSlash = uri.len > 1 and uri[1] == '/' - if uri[i] == '/': - # Make sure ``uri`` doesn't begin with '//'. + let doubleSlash = uri.len > 1 and uri[0] == '/' and uri[1] == '/' + if i < uri.len and uri[i] == '/': + # Make sure `uri` doesn't begin with '//'. if not doubleSlash: parsePath(uri, i, result) return # Scheme i.inc parseWhile(uri, result.scheme, Letters + Digits + {'+', '-', '.'}, i) - if uri[i] != ':' and not doubleSlash: + if (i >= uri.len or uri[i] != ':') and not doubleSlash: # Assume this is a reference URI (relative URI) i = 0 result.scheme.setLen(0) @@ -123,57 +304,70 @@ proc parseUri*(uri: string, result: var Uri) = i.inc # Skip ':' # Authority - if uri[i] == '/' and uri[i+1] == '/': + if i+1 < uri.len and uri[i] == '/' and uri[i+1] == '/': i.inc(2) # Skip // var authority = "" i.inc parseUntil(uri, authority, {'/', '?', '#'}, i) - if authority == "": - raise newException(ValueError, "Expected authority got nothing.") - parseAuthority(authority, result) + if authority.len > 0: + parseAuthority(authority, result) else: result.opaque = true # Path parsePath(uri, i, result) -proc parseUri*(uri: string): Uri = +func parseUri*(uri: string): Uri = ## Parses a URI and returns it. + ## + ## **See also:** + ## * `Uri type <#Uri>`_ for available fields in the URI type + runnableExamples: + let res = parseUri("ftp://Username:Password@Hostname") + assert res.username == "Username" + assert res.password == "Password" + assert res.scheme == "ftp" result = initUri() parseUri(uri, result) -proc removeDotSegments(path: string): string = +func removeDotSegments(path: string): string = + ## Collapses `..` and `.` in `path` in a similar way as done in `os.normalizedPath` + ## Caution: this is buggy. + runnableExamples: + assert removeDotSegments("a1/a2/../a3/a4/a5/./a6/a7/.//./") == "a1/a3/a4/a5/a6/a7/" + assert removeDotSegments("http://www.ai.") == "http://www.ai." + # xxx adapt or reuse `pathnorm.normalizePath(path, '/')` to make this more reliable, but + # taking into account url specificities such as not collapsing leading `//` in scheme + # `https://`. see `turi` for failing tests. if path.len == 0: return "" var collection: seq[string] = @[] - let endsWithSlash = path[path.len-1] == '/' + let endsWithSlash = path.endsWith '/' var i = 0 var currentSegment = "" - while true: + while i < path.len: case path[i] of '/': collection.add(currentSegment) currentSegment = "" of '.': - if path[i+1] == '.' and path[i+2] == '/': + if i+2 < path.len and path[i+1] == '.' and path[i+2] == '/': if collection.len > 0: discard collection.pop() i.inc 3 continue - elif path[i+1] == '/': + elif i + 1 < path.len and path[i+1] == '/': i.inc 2 continue currentSegment.add path[i] - of '\0': - if currentSegment != "": - collection.add currentSegment - break else: currentSegment.add path[i] i.inc + if currentSegment != "": + collection.add currentSegment result = collection.join("/") if endsWithSlash: result.add '/' -proc merge(base, reference: Uri): string = +func merge(base, reference: Uri): string = # http://tools.ietf.org/html/rfc3986#section-5.2.3 if base.hostname != "" and base.path == "": '/' & reference.path @@ -184,30 +378,26 @@ proc merge(base, reference: Uri): string = else: base.path[0 .. lastSegment] & reference.path -proc combine*(base: Uri, reference: Uri): Uri = +func combine*(base: Uri, reference: Uri): Uri = ## Combines a base URI with a reference URI. ## ## This uses the algorithm specified in ## `section 5.2.2 of RFC 3986 <http://tools.ietf.org/html/rfc3986#section-5.2.2>`_. ## - ## This means that the slashes inside the base URI's path as well as reference - ## URI's path affect the resulting URI. - ## - ## For building URIs you may wish to use \`/\` instead. + ## This means that the slashes inside the base URIs path as well as reference + ## URIs path affect the resulting URI. ## - ## Examples: - ## - ## .. code-block:: - ## let foo = combine(parseUri("http://example.com/foo/bar"), parseUri("/baz")) - ## assert foo.path == "/baz" - ## - ## let bar = combine(parseUri("http://example.com/foo/bar"), parseUri("baz")) - ## assert bar.path == "/foo/baz" - ## - ## let bar = combine(parseUri("http://example.com/foo/bar/"), parseUri("baz")) - ## assert bar.path == "/foo/bar/baz" - - template setAuthority(dest, src: expr): stmt = + ## **See also:** + ## * `/ func <#/,Uri,string>`_ for building URIs + runnableExamples: + let foo = combine(parseUri("https://nim-lang.org/foo/bar"), parseUri("/baz")) + assert foo.path == "/baz" + let bar = combine(parseUri("https://nim-lang.org/foo/bar"), parseUri("baz")) + assert bar.path == "/foo/baz" + let qux = combine(parseUri("https://nim-lang.org/foo/bar/"), parseUri("baz")) + assert qux.path == "/foo/bar/baz" + + template setAuthority(dest, src): untyped = dest.hostname = src.hostname dest.username = src.username dest.port = src.port @@ -239,206 +429,144 @@ proc combine*(base: Uri, reference: Uri): Uri = result.scheme = base.scheme result.anchor = reference.anchor -proc combine*(uris: varargs[Uri]): Uri = +func combine*(uris: varargs[Uri]): Uri = ## Combines multiple URIs together. + ## + ## **See also:** + ## * `/ func <#/,Uri,string>`_ for building URIs + runnableExamples: + let foo = combine(parseUri("https://nim-lang.org/"), parseUri("docs/"), + parseUri("manual.html")) + assert foo.hostname == "nim-lang.org" + assert foo.path == "/docs/manual.html" result = uris[0] - for i in 1 .. <uris.len: + for i in 1 ..< uris.len: result = combine(result, uris[i]) -proc `/`*(x: Uri, path: string): Uri = - ## Concatenates the path specified to the specified URI's path. +func isAbsolute*(uri: Uri): bool = + ## Returns true if URI is absolute, false otherwise. + runnableExamples: + assert parseUri("https://nim-lang.org").isAbsolute + assert not parseUri("nim-lang").isAbsolute + return uri.scheme != "" and (uri.hostname != "" or uri.path != "") + +func `/`*(x: Uri, path: string): Uri = + ## Concatenates the path specified to the specified URIs path. ## - ## Contrary to the ``combine`` procedure you do not have to worry about - ## the slashes at the beginning and end of the path and URI's path + ## Contrary to the `combine func <#combine,Uri,Uri>`_ you do not have to worry about + ## the slashes at the beginning and end of the path and URIs path ## respectively. ## - ## Examples: - ## - ## .. code-block:: - ## let foo = parseUri("http://example.com/foo/bar") / parseUri("/baz") - ## assert foo.path == "/foo/bar/baz" - ## - ## let bar = parseUri("http://example.com/foo/bar") / parseUri("baz") - ## assert bar.path == "/foo/bar/baz" - ## - ## let bar = parseUri("http://example.com/foo/bar/") / parseUri("baz") - ## assert bar.path == "/foo/bar/baz" + ## **See also:** + ## * `combine func <#combine,Uri,Uri>`_ + runnableExamples: + let foo = parseUri("https://nim-lang.org/foo/bar") / "/baz" + assert foo.path == "/foo/bar/baz" + let bar = parseUri("https://nim-lang.org/foo/bar") / "baz" + assert bar.path == "/foo/bar/baz" + let qux = parseUri("https://nim-lang.org/foo/bar/") / "baz" + assert qux.path == "/foo/bar/baz" result = x - if result.path[result.path.len-1] == '/': - if path[0] == '/': + + if result.path.len == 0: + if path.len == 0 or path[0] != '/': + result.path = "/" + result.path.add(path) + return + + if result.path.len > 0 and result.path[result.path.len-1] == '/': + if path.len > 0 and path[0] == '/': result.path.add(path[1 .. path.len-1]) else: result.path.add(path) else: - if path[0] != '/': + if path.len == 0 or path[0] != '/': result.path.add '/' result.path.add(path) -proc `$`*(u: Uri): string = +func `?`*(u: Uri, query: openArray[(string, string)]): Uri = + ## Concatenates the query parameters to the specified URI object. + runnableExamples: + let foo = parseUri("https://example.com") / "foo" ? {"bar": "qux"} + assert $foo == "https://example.com/foo?bar=qux" + result = u + result.query = encodeQuery(query) + +func `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - if u.opaque: - result.add(":") + runnableExamples: + assert $parseUri("https://nim-lang.org") == "https://nim-lang.org" + # Get the len of all the parts. + let schemeLen = u.scheme.len + let usernameLen = u.username.len + let passwordLen = u.password.len + let hostnameLen = u.hostname.len + let portLen = u.port.len + let pathLen = u.path.len + let queryLen = u.query.len + let anchorLen = u.anchor.len + # Prepare a string that fits all the parts and all punctuation chars. + # 12 is the max len required by all possible punctuation chars. + result = newStringOfCap( + schemeLen + usernameLen + passwordLen + hostnameLen + portLen + pathLen + queryLen + anchorLen + 12 + ) + # Insert to result. + if schemeLen > 0: + result.add u.scheme + result.add ':' + if not u.opaque: + result.add '/' + result.add '/' + if usernameLen > 0: + result.add u.username + if passwordLen > 0: + result.add ':' + result.add u.password + result.add '@' + if u.hostname.endsWith('/'): + if u.isIpv6: + result.add '[' + result.add u.hostname[0 .. ^2] + result.add ']' else: - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - result.add(u.path) - if u.query.len > 0: - result.add("?") - result.add(u.query) - if u.anchor.len > 0: - result.add("#") - result.add(u.anchor) - -when isMainModule: - block: - let str = "http://localhost" - let test = parseUri(str) - doAssert test.path == "" - - block: - let str = "http://localhost/" - let test = parseUri(str) - doAssert test.path == "/" - - block: - let str = "http://localhost:8080/test" - let test = parseUri(str) - doAssert test.scheme == "http" - doAssert test.port == "8080" - doAssert test.path == "/test" - doAssert test.hostname == "localhost" - doAssert($test == str) - - block: - let str = "foo://username:password@example.com:8042/over/there" & - "/index.dtb?type=animal&name=narwhal#nose" - let test = parseUri(str) - doAssert test.scheme == "foo" - doAssert test.username == "username" - doAssert test.password == "password" - doAssert test.hostname == "example.com" - doAssert test.port == "8042" - doAssert test.path == "/over/there/index.dtb" - doAssert test.query == "type=animal&name=narwhal" - doAssert test.anchor == "nose" - doAssert($test == str) - - block: - let str = "urn:example:animal:ferret:nose" - let test = parseUri(str) - doAssert test.scheme == "urn" - doAssert test.path == "example:animal:ferret:nose" - doAssert($test == str) - - block: - let str = "mailto:username@example.com?subject=Topic" - let test = parseUri(str) - doAssert test.scheme == "mailto" - doAssert test.username == "username" - doAssert test.hostname == "example.com" - doAssert test.query == "subject=Topic" - doAssert($test == str) - - block: - let str = "magnet:?xt=urn:sha1:72hsga62ba515sbd62&dn=foobar" - let test = parseUri(str) - doAssert test.scheme == "magnet" - doAssert test.query == "xt=urn:sha1:72hsga62ba515sbd62&dn=foobar" - doAssert($test == str) - - block: - let str = "/test/foo/bar?q=2#asdf" - let test = parseUri(str) - doAssert test.scheme == "" - doAssert test.path == "/test/foo/bar" - doAssert test.query == "q=2" - doAssert test.anchor == "asdf" - doAssert($test == str) - - block: - let str = "test/no/slash" - let test = parseUri(str) - doAssert test.path == "test/no/slash" - doAssert($test == str) - - block: - let str = "//git@github.com:dom96/packages" - let test = parseUri(str) - doAssert test.scheme == "" - doAssert test.username == "git" - doAssert test.hostname == "github.com" - doAssert test.port == "dom96" - doAssert test.path == "/packages" - - # Remove dot segments tests - block: - doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" - - # Combine tests - block: - let concat = combine(parseUri("http://google.com/foo/bar/"), parseUri("baz")) - doAssert concat.path == "/foo/bar/baz" - doAssert concat.hostname == "google.com" - doAssert concat.scheme == "http" - - block: - let concat = combine(parseUri("http://google.com/foo"), parseUri("/baz")) - doAssert concat.path == "/baz" - doAssert concat.hostname == "google.com" - doAssert concat.scheme == "http" - - block: - let concat = combine(parseUri("http://google.com/foo/test"), parseUri("bar")) - doAssert concat.path == "/foo/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test"), parseUri("/bar")) - doAssert concat.path == "/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test"), parseUri("bar")) - doAssert concat.path == "/foo/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test/"), parseUri("bar")) - doAssert concat.path == "/foo/test/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test/"), parseUri("bar/")) - doAssert concat.path == "/foo/test/bar/" - - block: - let concat = combine(parseUri("http://google.com/foo/test/"), parseUri("bar/"), - parseUri("baz")) - doAssert concat.path == "/foo/test/bar/baz" - - # `/` tests - block: - let test = parseUri("http://example.com/foo") / "bar/asd" - doAssert test.path == "/foo/bar/asd" - - block: - let test = parseUri("http://example.com/foo/") / "/bar/asd" - doAssert test.path == "/foo/bar/asd" - - # removeDotSegments tests - block: - # empty test - doAssert removeDotSegments("") == "" - - # bug #3207 - block: - doAssert parseUri("http://qq/1").combine(parseUri("https://qqq")).`$` == "https://qqq" + result.add u.hostname[0 .. ^2] + else: + if u.isIpv6: + result.add '[' + result.add u.hostname + result.add ']' + else: + result.add u.hostname + if portLen > 0: + result.add ':' + result.add u.port + if pathLen > 0: + if hostnameLen > 0 and u.path[0] != '/': + result.add '/' + result.add u.path + if queryLen > 0: + result.add '?' + result.add u.query + if anchorLen > 0: + result.add '#' + result.add u.anchor + + +proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3).} = + ## Convenience proc for `base64.encode` returns a standard Base64 Data URI (RFC-2397) + ## + ## **See also:** + ## * `mimetypes <mimetypes.html>`_ for `mime` argument + ## * https://tools.ietf.org/html/rfc2397 + ## * https://en.wikipedia.org/wiki/Data_URI_scheme + runnableExamples: static: assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + assert encoding.len > 0 and mime.len > 0 # Must *not* be URL-Safe, see RFC-2397 + let base64encoded: string = base64.encode(data) + # ("data:".len + ";charset=".len + ";base64,".len) == 22 + result = newStringOfCap(22 + mime.len + encoding.len + base64encoded.len) + result.add "data:" + result.add mime + result.add ";charset=" + result.add encoding + result.add ";base64," + result.add base64encoded diff --git a/lib/pure/volatile.nim b/lib/pure/volatile.nim new file mode 100644 index 000000000..a38247c7d --- /dev/null +++ b/lib/pure/volatile.nim @@ -0,0 +1,34 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Jeff Ciesielski +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains code for generating volatile loads and stores, +## which are useful in embedded and systems programming. + +proc volatileLoad*[T](src: ptr T): T {.inline, noinit.} = + ## Generates a volatile load of the value stored in the container `src`. + ## Note that this only effects code generation on `C` like backends. + when nimvm: + result = src[] + else: + when defined(js): + result = src[] + else: + {.emit: [result, " = (*(", typeof(src[]), " volatile*)", src, ");"].} + +proc volatileStore*[T](dest: ptr T, val: T) {.inline.} = + ## Generates a volatile store into the container `dest` of the value + ## `val`. Note that this only effects code generation on `C` like + ## backends. + when nimvm: + dest[] = val + else: + when defined(js): + dest[] = val + else: + {.emit: ["*((", typeof(dest[]), " volatile*)(", dest, ")) = ", val, ";"].} diff --git a/lib/pure/xmldom.nim b/lib/pure/xmldom.nim deleted file mode 100644 index 6cf837f25..000000000 --- a/lib/pure/xmldom.nim +++ /dev/null @@ -1,1120 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -import strutils -## This module implements XML DOM Level 2 Core -## specification (http://www.w3.org/TR/2000/REC-DOM-Level-2-Core-20001113/core.html) - - -#http://www.w3.org/TR/2000/REC-DOM-Level-2-Core-20001113/core.html - -#Exceptions -type - EDOMException* = object of ValueError ## Base exception object for all DOM Exceptions - EDOMStringSizeErr* = object of EDOMException ## If the specified range of text does not fit into a DOMString - ## Currently not used(Since DOMString is just string) - EHierarchyRequestErr* = object of EDOMException ## If any node is inserted somewhere it doesn't belong - EIndexSizeErr* = object of EDOMException ## If index or size is negative, or greater than the allowed value - EInuseAttributeErr* = object of EDOMException ## If an attempt is made to add an attribute that is already in use elsewhere - EInvalidAccessErr* = object of EDOMException ## If a parameter or an operation is not supported by the underlying object. - EInvalidCharacterErr* = object of EDOMException ## This exception is raised when a string parameter contains an illegal character - EInvalidModificationErr* = object of EDOMException ## If an attempt is made to modify the type of the underlying object. - EInvalidStateErr* = object of EDOMException ## If an attempt is made to use an object that is not, or is no longer, usable. - ENamespaceErr* = object of EDOMException ## If an attempt is made to create or change an object in a way which is incorrect with regard to namespaces. - ENotFoundErr* = object of EDOMException ## If an attempt is made to reference a node in a context where it does not exist - ENotSupportedErr* = object of EDOMException ## If the implementation does not support the requested type of object or operation. - ENoDataAllowedErr* = object of EDOMException ## If data is specified for a node which does not support data - ENoModificationAllowedErr* = object of EDOMException ## If an attempt is made to modify an object where modifications are not allowed - ESyntaxErr* = object of EDOMException ## If an invalid or illegal string is specified. - EWrongDocumentErr* = object of EDOMException ## If a node is used in a different document than the one that created it (that doesn't support it) - -const - ElementNode* = 1 - AttributeNode* = 2 - TextNode* = 3 - CDataSectionNode* = 4 - ProcessingInstructionNode* = 7 - CommentNode* = 8 - DocumentNode* = 9 - DocumentFragmentNode* = 11 - - # Nodes which are childless - Not sure about AttributeNode - childlessObjects = {DocumentNode, AttributeNode, TextNode, - CDataSectionNode, ProcessingInstructionNode, CommentNode} - # Illegal characters - illegalChars = {'>', '<', '&', '"'} - - -type - Feature = tuple[name: string, version: string] - PDOMImplementation* = ref DOMImplementation - DOMImplementation = object - features: seq[Feature] # Read-Only - - PNode* = ref Node - Node = object of RootObj - attributes*: seq[PAttr] - childNodes*: seq[PNode] - fLocalName: string # Read-only - fNamespaceURI: string # Read-only - fNodeName: string # Read-only - nodeValue*: string - fNodeType: int # Read-only - fOwnerDocument: PDocument # Read-Only - fParentNode: PNode # Read-Only - prefix*: string # Setting this should change some values... TODO! - - PElement* = ref Element - Element = object of Node - fTagName: string # Read-only - - PCharacterData* = ref CharacterData - CharacterData = object of Node - data*: string - - PDocument* = ref Document - Document = object of Node - fImplementation: PDOMImplementation # Read-only - fDocumentElement: PElement # Read-only - - PAttr* = ref Attr - Attr = object of Node - fName: string # Read-only - fSpecified: bool # Read-only - value*: string - fOwnerElement: PElement # Read-only - - PDocumentFragment* = ref DocumentFragment - DocumentFragment = object of Node - - PText* = ref Text - Text = object of CharacterData - - PComment* = ref Comment - Comment = object of CharacterData - - PCDataSection* = ref CDataSection - CDataSection = object of Text - - PProcessingInstruction* = ref ProcessingInstruction - ProcessingInstruction = object of Node - data*: string - fTarget: string # Read-only - -# DOMImplementation -proc getDOM*(): PDOMImplementation = - ## Returns a DOMImplementation - new(result) - result.features = @[(name: "core", version: "2.0"), - (name: "core", version: "1.0"), - (name: "XML", version: "2.0")] - -proc createDocument*(dom: PDOMImplementation, namespaceURI: string, qualifiedName: string): PDocument = - ## Creates an XML Document object of the specified type with its document element. - var doc: PDocument - new(doc) - doc.fNamespaceURI = namespaceURI - doc.fImplementation = dom - - var elTag: PElement - new(elTag) - elTag.fTagName = qualifiedName - elTag.fNodeName = qualifiedName - doc.fDocumentElement = elTag - doc.fNodeType = DocumentNode - - return doc - -proc createDocument*(dom: PDOMImplementation, n: PElement): PDocument = - ## Creates an XML Document object of the specified type with its document element. - - # This procedure is not in the specification, it's provided for the parser. - var doc: PDocument - new(doc) - doc.fDocumentElement = n - doc.fImplementation = dom - doc.fNodeType = DocumentNode - - return doc - -proc hasFeature*(dom: PDOMImplementation, feature: string, version: string = ""): bool = - ## Returns ``true`` if this ``version`` of the DomImplementation implements ``feature``, otherwise ``false`` - for iName, iVersion in items(dom.features): - if iName == feature: - if version == "": - return true - else: - if iVersion == version: - return true - return false - - -# Document -# Attributes - -proc implementation*(doc: PDocument): PDOMImplementation = - return doc.fImplementation - -proc documentElement*(doc: PDocument): PElement = - return doc.fDocumentElement - -# Internal procedures -proc findNodes(nl: PNode, name: string): seq[PNode] = - # Made for getElementsByTagName - var r: seq[PNode] = @[] - if isNil(nl.childNodes): return @[] - if nl.childNodes.len() == 0: return @[] - - for i in items(nl.childNodes): - if i.fNodeType == ElementNode: - if i.fNodeName == name or name == "*": - r.add(i) - - if not isNil(i.childNodes): - if i.childNodes.len() != 0: - r.add(findNodes(i, name)) - - return r - -proc findNodesNS(nl: PNode, namespaceURI: string, localName: string): seq[PNode] = - # Made for getElementsByTagNameNS - var r: seq[PNode] = @[] - if isNil(nl.childNodes): return @[] - if nl.childNodes.len() == 0: return @[] - - for i in items(nl.childNodes): - if i.fNodeType == ElementNode: - if (i.fNamespaceURI == namespaceURI or namespaceURI == "*") and (i.fLocalName == localName or localName == "*"): - r.add(i) - - if not isNil(i.childNodes): - if i.childNodes.len() != 0: - r.add(findNodesNS(i, namespaceURI, localName)) - - return r - - -#Procedures -proc createAttribute*(doc: PDocument, name: string): PAttr = - ## Creates an Attr of the given name. Note that the Attr instance can then be set on an Element using the setAttributeNode method. - ## To create an attribute with a qualified name and namespace URI, use the createAttributeNS method. - - # Check if name contains illegal characters - if illegalChars in name: - raise newException(EInvalidCharacterErr, "Invalid character") - - var attrNode: PAttr - new(attrNode) - attrNode.fName = name - attrNode.fNodeName = name - attrNode.fLocalName = nil - attrNode.prefix = nil - attrNode.fNamespaceURI = nil - attrNode.value = "" - attrNode.fSpecified = false - return attrNode - -proc createAttributeNS*(doc: PDocument, namespaceURI: string, qualifiedName: string): PAttr = - ## Creates an attribute of the given qualified name and namespace URI - - # Check if name contains illegal characters - if illegalChars in namespaceURI or illegalChars in qualifiedName: - raise newException(EInvalidCharacterErr, "Invalid character") - # Exceptions - if qualifiedName.contains(':'): - if isNil(namespaceURI): - raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qualifiedName.split(':')[0].toLower() == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace": - raise newException(ENamespaceErr, - "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - elif qualifiedName.split(':')[1].toLower() == "xmlns" and namespaceURI != "http://www.w3.org/2000/xmlns/": - raise newException(ENamespaceErr, - "When the namespace prefix is \"xmlns\" namespaceURI has to be \"http://www.w3.org/2000/xmlns/\"") - - var attrNode: PAttr - new(attrNode) - attrNode.fName = qualifiedName - attrNode.fNodeName = qualifiedName - attrNode.fSpecified = false - attrNode.fNamespaceURI = namespaceURI - if qualifiedName.contains(':'): - attrNode.prefix = qualifiedName.split(':')[0] - attrNode.fLocalName = qualifiedName.split(':')[1] - else: - attrNode.prefix = nil - attrNode.fLocalName = qualifiedName - attrNode.value = "" - - attrNode.fNodeType = AttributeNode - return attrNode - -proc createCDATASection*(doc: PDocument, data: string): PCDataSection = - ## Creates a CDATASection node whose value is the specified string. - var cData: PCDataSection - new(cData) - cData.data = data - cData.nodeValue = data - cData.fNodeName = "#text" # Not sure about this, but this is technically a TextNode - cData.fNodeType = CDataSectionNode - return cData - -proc createComment*(doc: PDocument, data: string): PComment = - ## Creates a Comment node given the specified string. - var comm: PComment - new(comm) - comm.data = data - comm.nodeValue = data - - comm.fNodeType = CommentNode - return comm - -proc createDocumentFragment*(doc: PDocument): PDocumentFragment = - ## Creates an empty DocumentFragment object. - var df: PDocumentFragment - new(df) - return df - -proc createElement*(doc: PDocument, tagName: string): PElement = - ## Creates an element of the type specified. - - # Check if name contains illegal characters - if illegalChars in tagName: - raise newException(EInvalidCharacterErr, "Invalid character") - - var elNode: PElement - new(elNode) - elNode.fTagName = tagName - elNode.fNodeName = tagName - elNode.fLocalName = nil - elNode.prefix = nil - elNode.fNamespaceURI = nil - elNode.childNodes = @[] - elNode.attributes = @[] - - elNode.fNodeType = ElementNode - - return elNode - -proc createElementNS*(doc: PDocument, namespaceURI: string, qualifiedName: string): PElement = - ## Creates an element of the given qualified name and namespace URI. - if qualifiedName.contains(':'): - if isNil(namespaceURI): - raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qualifiedName.split(':')[0].toLower() == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace": - raise newException(ENamespaceErr, - "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - - # Check if name contains illegal characters - if illegalChars in namespaceURI or illegalChars in qualifiedName: - raise newException(EInvalidCharacterErr, "Invalid character") - - var elNode: PElement - new(elNode) - elNode.fTagName = qualifiedName - elNode.fNodeName = qualifiedName - if qualifiedName.contains(':'): - elNode.prefix = qualifiedName.split(':')[0] - elNode.fLocalName = qualifiedName.split(':')[1] - else: - elNode.prefix = nil - elNode.fLocalName = qualifiedName - elNode.fNamespaceURI = namespaceURI - elNode.childNodes = @[] - elNode.attributes = @[] - - elNode.fNodeType = ElementNode - - return elNode - -proc createProcessingInstruction*(doc: PDocument, target: string, data: string): PProcessingInstruction = - ## Creates a ProcessingInstruction node given the specified name and data strings. - - #Check if name contains illegal characters - if illegalChars in target: - raise newException(EInvalidCharacterErr, "Invalid character") - - var pi: PProcessingInstruction - new(pi) - pi.fTarget = target - pi.data = data - pi.fNodeType = ProcessingInstructionNode - return pi - -proc createTextNode*(doc: PDocument, data: string): PText = #Propably TextNode - ## Creates a Text node given the specified string. - var txtNode: PText - new(txtNode) - txtNode.data = data - txtNode.nodeValue = data - txtNode.fNodeName = "#text" - - txtNode.fNodeType = TextNode - return txtNode - -discard """proc getElementById*(doc: PDocument, elementId: string): PElement = - ##Returns the ``Element`` whose ID is given by ``elementId``. If no such element exists, returns ``nil`` - #TODO""" - -proc getElementsByTagName*(doc: PDocument, tagName: string): seq[PNode] = - ## Returns a NodeList of all the Elements with a given tag name in - ## the order in which they are encountered in a preorder traversal of the Document tree. - result = @[] - if doc.fDocumentElement.fNodeName == tagName or tagName == "*": - result.add(doc.fDocumentElement) - - result.add(doc.fDocumentElement.findNodes(tagName)) - -proc getElementsByTagNameNS*(doc: PDocument, namespaceURI: string, localName: string): seq[PNode] = - ## Returns a NodeList of all the Elements with a given localName and namespaceURI - ## in the order in which they are encountered in a preorder traversal of the Document tree. - result = @[] - if doc.fDocumentElement.fLocalName == localName or localName == "*": - if doc.fDocumentElement.fNamespaceURI == namespaceURI or namespaceURI == "*": - result.add(doc.fDocumentElement) - - result.add(doc.fDocumentElement.findNodesNS(namespaceURI, localName)) - -proc importNode*(doc: PDocument, importedNode: PNode, deep: bool): PNode = - ## Imports a node from another document to this document - case importedNode.fNodeType - of AttributeNode: - var nAttr: PAttr = PAttr(importedNode) - nAttr.fOwnerDocument = doc - nAttr.fParentNode = nil - nAttr.fOwnerElement = nil - nAttr.fSpecified = true - return nAttr - of DocumentFragmentNode: - var n: PNode - new(n) - n = importedNode - n.fOwnerDocument = doc - n.fParentNode = nil - - n.fOwnerDocument = doc - n.fParentNode = nil - var tmp: seq[PNode] = n.childNodes - n.childNodes = @[] - if deep: - for i in low(tmp.len())..high(tmp.len()): - n.childNodes.add(importNode(doc, tmp[i], deep)) - - return n - of ElementNode: - var n: PNode - new(n) - n = importedNode - n.fOwnerDocument = doc - n.fParentNode = nil - - var tmpA: seq[PAttr] = n.attributes - n.attributes = @[] - # Import the Element node's attributes - for i in low(tmpA.len())..high(tmpA.len()): - n.attributes.add(PAttr(importNode(doc, tmpA[i], deep))) - # Import the childNodes - var tmp: seq[PNode] = n.childNodes - n.childNodes = @[] - if deep: - for i in low(tmp.len())..high(tmp.len()): - n.childNodes.add(importNode(doc, tmp[i], deep)) - - return n - of ProcessingInstructionNode, TextNode, CDataSectionNode, CommentNode: - var n: PNode - new(n) - n = importedNode - n.fOwnerDocument = doc - n.fParentNode = nil - return n - else: - raise newException(ENotSupportedErr, "The type of node being imported is not supported") - - -# Node -# Attributes - -proc firstChild*(n: PNode): PNode = - ## Returns this node's first child - - if not isNil(n.childNodes) and n.childNodes.len() > 0: - return n.childNodes[0] - else: - return nil - -proc lastChild*(n: PNode): PNode = - ## Returns this node's last child - - if not isNil(n.childNodes) and n.childNodes.len() > 0: - return n.childNodes[n.childNodes.len() - 1] - else: - return nil - -proc localName*(n: PNode): string = - ## Returns this nodes local name - - return n.fLocalName - -proc namespaceURI*(n: PNode): string = - ## Returns this nodes namespace URI - - return n.fNamespaceURI - -proc `namespaceURI=`*(n: PNode, value: string) = - n.fNamespaceURI = value - -proc nextSibling*(n: PNode): PNode = - ## Returns the next sibling of this node - - if isNil(n.fParentNode) or isNil(n.fParentNode.childNodes): - return nil - var nLow: int = low(n.fParentNode.childNodes) - var nHigh: int = high(n.fParentNode.childNodes) - for i in nLow..nHigh: - if n.fParentNode.childNodes[i] == n: - return n.fParentNode.childNodes[i + 1] - return nil - -proc nodeName*(n: PNode): string = - ## Returns the name of this node - - return n.fNodeName - -proc nodeType*(n: PNode): int = - ## Returns the type of this node - - return n.fNodeType - -proc ownerDocument*(n: PNode): PDocument = - ## Returns the owner document of this node - - return n.fOwnerDocument - -proc parentNode*(n: PNode): PNode = - ## Returns the parent node of this node - - return n.fParentNode - -proc previousSibling*(n: PNode): PNode = - ## Returns the previous sibling of this node - - if isNil(n.fParentNode) or isNil(n.fParentNode.childNodes): - return nil - var nLow: int = low(n.fParentNode.childNodes) - var nHigh: int = high(n.fParentNode.childNodes) - for i in nLow..nHigh: - if n.fParentNode.childNodes[i] == n: - return n.fParentNode.childNodes[i - 1] - return nil - -proc `prefix=`*(n: PNode, value: string) = - ## Modifies the prefix of this node - - # Setter - # Check if name contains illegal characters - if illegalChars in value: - raise newException(EInvalidCharacterErr, "Invalid character") - - if isNil(n.fNamespaceURI): - raise newException(ENamespaceErr, "namespaceURI cannot be nil") - elif value.toLower() == "xml" and n.fNamespaceURI != "http://www.w3.org/XML/1998/namespace": - raise newException(ENamespaceErr, - "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - elif value.toLower() == "xmlns" and n.fNamespaceURI != "http://www.w3.org/2000/xmlns/": - raise newException(ENamespaceErr, - "When the namespace prefix is \"xmlns\" namespaceURI has to be \"http://www.w3.org/2000/xmlns/\"") - elif value.toLower() == "xmlns" and n.fNodeType == AttributeNode: - raise newException(ENamespaceErr, "An AttributeNode cannot have a prefix of \"xmlns\"") - - n.fNodeName = value & ":" & n.fLocalName - if n.nodeType == ElementNode: - var el: PElement = PElement(n) - el.fTagName = value & ":" & n.fLocalName - - elif n.nodeType == AttributeNode: - var attr: PAttr = PAttr(n) - attr.fName = value & ":" & n.fLocalName - -# Procedures -proc appendChild*(n: PNode, newChild: PNode) = - ## Adds the node newChild to the end of the list of children of this node. - ## If the newChild is already in the tree, it is first removed. - - # Check if n contains newChild - if not isNil(n.childNodes): - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == newChild: - raise newException(EHierarchyRequestErr, "The node to append is already in this nodes children.") - - # Check if newChild is from this nodes document - if n.fOwnerDocument != newChild.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - - if n == newChild: - raise newException(EHierarchyRequestErr, "You can't add a node into itself") - - if n.nodeType in childlessObjects: - raise newException(ENoModificationAllowedErr, "Cannot append children to a childless node") - - if isNil(n.childNodes): n.childNodes = @[] - - newChild.fParentNode = n - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == newChild: - n.childNodes[i] = newChild - - n.childNodes.add(newChild) - -proc cloneNode*(n: PNode, deep: bool): PNode = - ## Returns a duplicate of this node, if ``deep`` is `true`, Element node's children are copied - case n.fNodeType - of AttributeNode: - var newNode: PAttr - new(newNode) - newNode = PAttr(n) - newNode.fSpecified = true - newNode.fOwnerElement = nil - return newNode - of ElementNode: - var newNode: PElement - new(newNode) - newNode = PElement(n) - # Import the childNodes - var tmp: seq[PNode] = n.childNodes - n.childNodes = @[] - if deep and not isNil(tmp): - for i in low(tmp.len())..high(tmp.len()): - n.childNodes.add(cloneNode(tmp[i], deep)) - return newNode - else: - var newNode: PNode - new(newNode) - newNode = n - return newNode - -proc hasAttributes*(n: PNode): bool = - ## Returns whether this node (if it is an element) has any attributes. - return not isNil(n.attributes) and n.attributes.len() > 0 - -proc hasChildNodes*(n: PNode): bool = - ## Returns whether this node has any children. - return not isNil(n.childNodes) and n.childNodes.len() > 0 - -proc insertBefore*(n: PNode, newChild: PNode, refChild: PNode): PNode = - ## Inserts the node ``newChild`` before the existing child node ``refChild``. - ## If ``refChild`` is nil, insert ``newChild`` at the end of the list of children. - - # Check if newChild is from this nodes document - if n.fOwnerDocument != newChild.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - - if isNil(n.childNodes): - n.childNodes = @[] - - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == refChild: - n.childNodes.insert(newChild, i - 1) - return - - n.childNodes.add(newChild) - -proc isSupported*(n: PNode, feature: string, version: string): bool = - ## Tests whether the DOM implementation implements a specific - ## feature and that feature is supported by this node. - return n.fOwnerDocument.fImplementation.hasFeature(feature, version) - -proc isEmpty(s: string): bool = - - if isNil(s) or s == "": - return true - for i in items(s): - if i != ' ': - return false - return true - -proc normalize*(n: PNode) = - ## Merges all separated TextNodes together, and removes any empty TextNodes - var curTextNode: PNode = nil - var i: int = 0 - - var newChildNodes: seq[PNode] = @[] - while true: - if isNil(n.childNodes) or i >= n.childNodes.len: - break - if n.childNodes[i].nodeType == TextNode: - - #If the TextNode is empty, remove it - if PText(n.childNodes[i]).data.isEmpty(): - inc(i) - - if isNil(curTextNode): - curTextNode = n.childNodes[i] - else: - PText(curTextNode).data.add(PText(n.childNodes[i]).data) - curTextNode.nodeValue.add(PText(n.childNodes[i]).data) - inc(i) - else: - newChildNodes.add(curTextNode) - newChildNodes.add(n.childNodes[i]) - curTextNode = nil - - inc(i) - n.childNodes = newChildNodes - -proc removeChild*(n: PNode, oldChild: PNode): PNode = - ## Removes the child node indicated by ``oldChild`` from the list of children, and returns it. - if not isNil(n.childNodes): - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == oldChild: - result = n.childNodes[i] - n.childNodes.delete(i) - return - - raise newException(ENotFoundErr, "Node not found") - -proc replaceChild*(n: PNode, newChild: PNode, oldChild: PNode): PNode = - ## Replaces the child node ``oldChild`` with ``newChild`` in the list of children, and returns the ``oldChild`` node. - - # Check if newChild is from this nodes document - if n.fOwnerDocument != newChild.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - - if not isNil(n.childNodes): - for i in low(n.childNodes)..high(n.childNodes): - if n.childNodes[i] == oldChild: - result = n.childNodes[i] - n.childNodes[i] = newChild - return - - raise newException(ENotFoundErr, "Node not found") - -# NamedNodeMap - -proc getNamedItem*(nList: seq[PNode], name: string): PNode = - ## Retrieves a node specified by ``name``. If this node cannot be found returns ``nil`` - for i in items(nList): - if i.nodeName() == name: - return i - return nil - -proc getNamedItem*(nList: seq[PAttr], name: string): PAttr = - ## Retrieves a node specified by ``name``. If this node cannot be found returns ``nil`` - for i in items(nList): - if i.nodeName() == name: - return i - return nil - -proc getNamedItemNS*(nList: seq[PNode], namespaceURI: string, localName: string): PNode = - ## Retrieves a node specified by ``localName`` and ``namespaceURI``. If this node cannot be found returns ``nil`` - for i in items(nList): - if i.namespaceURI() == namespaceURI and i.localName() == localName: - return i - return nil - -proc getNamedItemNS*(nList: seq[PAttr], namespaceURI: string, localName: string): PAttr = - ## Retrieves a node specified by ``localName`` and ``namespaceURI``. If this node cannot be found returns ``nil`` - for i in items(nList): - if i.namespaceURI() == namespaceURI and i.localName() == localName: - return i - return nil - -proc item*(nList: seq[PNode], index: int): PNode = - ## Returns the ``index`` th item in the map. - ## If ``index`` is greater than or equal to the number of nodes in this map, this returns ``nil``. - if index >= nList.len(): return nil - else: return nList[index] - -proc removeNamedItem*(nList: var seq[PNode], name: string): PNode = - ## Removes a node specified by ``name`` - ## Raises the ``ENotFoundErr`` exception, if the node was not found - for i in low(nList)..high(nList): - if nList[i].fNodeName == name: - result = nList[i] - nList.delete(i) - return - - raise newException(ENotFoundErr, "Node not found") - -proc removeNamedItemNS*(nList: var seq[PNode], namespaceURI: string, localName: string): PNode = - ## Removes a node specified by local name and namespace URI - for i in low(nList)..high(nList): - if nList[i].fLocalName == localName and nList[i].fNamespaceURI == namespaceURI: - result = nList[i] - nList.delete(i) - return - - raise newException(ENotFoundErr, "Node not found") - -proc setNamedItem*(nList: var seq[PNode], arg: PNode): PNode = - ## Adds ``arg`` as a ``Node`` to the ``NList`` - ## If a node with the same name is already present in this map, it is replaced by the new one. - if not isNil(nList): - if nList.len() > 0: - #Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - #Exceptions End - - var item: PNode = nList.getNamedItem(arg.nodeName()) - if isNil(item): - nList.add(arg) - return nil - else: - # Node with the same name exists - var index: int = 0 - for i in low(nList)..high(nList): - if nList[i] == item: - index = i - break - nList[index] = arg - return item # Return the replaced node - -proc setNamedItem*(nList: var seq[PAttr], arg: PAttr): PAttr = - ## Adds ``arg`` as a ``Node`` to the ``NList`` - ## If a node with the same name is already present in this map, it is replaced by the new one. - if not isNil(nList): - if nList.len() > 0: - # Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - - if not isNil(arg.fOwnerElement): - raise newException(EInuseAttributeErr, "This attribute is in use by another element, use cloneNode") - - # Exceptions end - var item: PAttr = nList.getNamedItem(arg.nodeName()) - if isNil(item): - nList.add(arg) - return nil - else: - # Node with the same name exists - var index: int = 0 - for i in low(nList)..high(nList): - if nList[i] == item: - index = i - break - nList[index] = arg - return item # Return the replaced node - -proc setNamedItemNS*(nList: var seq[PNode], arg: PNode): PNode = - ## Adds a node using its ``namespaceURI`` and ``localName`` - if not isNil(nList): - if nList.len() > 0: - # Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - #Exceptions end - - var item: PNode = nList.getNamedItemNS(arg.namespaceURI(), arg.localName()) - if isNil(item): - nList.add(arg) - return nil - else: - # Node with the same name exists - var index: int = 0 - for i in low(nList)..high(nList): - if nList[i] == item: - index = i - break - nList[index] = arg - return item # Return the replaced node - -proc setNamedItemNS*(nList: var seq[PAttr], arg: PAttr): PAttr = - ## Adds a node using its ``namespaceURI`` and ``localName`` - if not isNil(nList): - if nList.len() > 0: - # Check if newChild is from this nodes document - if nList[0].fOwnerDocument != arg.fOwnerDocument: - raise newException(EWrongDocumentErr, "This node belongs to a different document, use importNode.") - - if not isNil(arg.fOwnerElement): - raise newException(EInuseAttributeErr, "This attribute is in use by another element, use cloneNode") - - # Exceptions end - var item: PAttr = nList.getNamedItemNS(arg.namespaceURI(), arg.localName()) - if isNil(item): - nList.add(arg) - return nil - else: - # Node with the same name exists - var index: int = 0 - for i in low(nList)..high(nList): - if nList[i] == item: - index = i - break - nList[index] = arg - return item # Return the replaced node - -# CharacterData - Decided to implement this, -# Didn't add the procedures, because you can just edit .data - -# Attr -# Attributes -proc name*(a: PAttr): string = - ## Returns the name of the Attribute - - return a.fName - -proc specified*(a: PAttr): bool = - ## Specifies whether this attribute was specified in the original document - - return a.fSpecified - -proc ownerElement*(a: PAttr): PElement = - ## Returns this Attributes owner element - - return a.fOwnerElement - -# Element -# Attributes - -proc tagName*(el: PElement): string = - ## Returns the Element Tag Name - - return el.fTagName - -# Procedures -proc getAttribute*(el: PElement, name: string): string = - ## Retrieves an attribute value by ``name`` - if isNil(el.attributes): - return nil - var attribute = el.attributes.getNamedItem(name) - if not isNil(attribute): - return attribute.value - else: - return nil - -proc getAttributeNS*(el: PElement, namespaceURI: string, localName: string): string = - ## Retrieves an attribute value by ``localName`` and ``namespaceURI`` - if isNil(el.attributes): - return nil - var attribute = el.attributes.getNamedItemNS(namespaceURI, localName) - if not isNil(attribute): - return attribute.value - else: - return nil - -proc getAttributeNode*(el: PElement, name: string): PAttr = - ## Retrieves an attribute node by ``name`` - ## To retrieve an attribute node by qualified name and namespace URI, use the `getAttributeNodeNS` method - if isNil(el.attributes): - return nil - return el.attributes.getNamedItem(name) - -proc getAttributeNodeNS*(el: PElement, namespaceURI: string, localName: string): PAttr = - ## Retrieves an `Attr` node by ``localName`` and ``namespaceURI`` - if isNil(el.attributes): - return nil - return el.attributes.getNamedItemNS(namespaceURI, localName) - -proc getElementsByTagName*(el: PElement, name: string): seq[PNode] = - ## Returns a `NodeList` of all descendant `Elements` of ``el`` with a given tag ``name``, - ## in the order in which they are encountered in a preorder traversal of this `Element` tree - ## If ``name`` is `*`, returns all descendant of ``el`` - result = el.findNodes(name) - -proc getElementsByTagNameNS*(el: PElement, namespaceURI: string, localName: string): seq[PNode] = - ## Returns a `NodeList` of all the descendant Elements with a given - ## ``localName`` and ``namespaceURI`` in the order in which they are - ## encountered in a preorder traversal of this Element tree - result = el.findNodesNS(namespaceURI, localName) - -proc hasAttribute*(el: PElement, name: string): bool = - ## Returns ``true`` when an attribute with a given ``name`` is specified - ## on this element , ``false`` otherwise. - if isNil(el.attributes): - return false - return not isNil(el.attributes.getNamedItem(name)) - -proc hasAttributeNS*(el: PElement, namespaceURI: string, localName: string): bool = - ## Returns ``true`` when an attribute with a given ``localName`` and - ## ``namespaceURI`` is specified on this element , ``false`` otherwise - if isNil(el.attributes): - return false - return not isNil(el.attributes.getNamedItemNS(namespaceURI, localName)) - -proc removeAttribute*(el: PElement, name: string) = - ## Removes an attribute by ``name`` - if not isNil(el.attributes): - for i in low(el.attributes)..high(el.attributes): - if el.attributes[i].fName == name: - el.attributes.delete(i) - -proc removeAttributeNS*(el: PElement, namespaceURI: string, localName: string) = - ## Removes an attribute by ``localName`` and ``namespaceURI`` - if not isNil(el.attributes): - for i in low(el.attributes)..high(el.attributes): - if el.attributes[i].fNamespaceURI == namespaceURI and - el.attributes[i].fLocalName == localName: - el.attributes.delete(i) - -proc removeAttributeNode*(el: PElement, oldAttr: PAttr): PAttr = - ## Removes the specified attribute node - ## If the attribute node cannot be found raises ``ENotFoundErr`` - if not isNil(el.attributes): - for i in low(el.attributes)..high(el.attributes): - if el.attributes[i] == oldAttr: - result = el.attributes[i] - el.attributes.delete(i) - return - - raise newException(ENotFoundErr, "oldAttr is not a member of el's Attributes") - -proc setAttributeNode*(el: PElement, newAttr: PAttr): PAttr = - ## Adds a new attribute node, if an attribute with the same `nodeName` is - ## present, it is replaced by the new one and the replaced attribute is - ## returned, otherwise ``nil`` is returned. - - # Check if newAttr is from this nodes document - if el.fOwnerDocument != newAttr.fOwnerDocument: - raise newException(EWrongDocumentErr, - "This node belongs to a different document, use importNode.") - - if not isNil(newAttr.fOwnerElement): - raise newException(EInuseAttributeErr, - "This attribute is in use by another element, use cloneNode") - # Exceptions end - - if isNil(el.attributes): el.attributes = @[] - return el.attributes.setNamedItem(newAttr) - -proc setAttributeNodeNS*(el: PElement, newAttr: PAttr): PAttr = - ## Adds a new attribute node, if an attribute with the localName and - ## namespaceURI of ``newAttr`` is present, it is replaced by the new one - ## and the replaced attribute is returned, otherwise ``nil`` is returned. - - # Check if newAttr is from this nodes document - if el.fOwnerDocument != newAttr.fOwnerDocument: - raise newException(EWrongDocumentErr, - "This node belongs to a different document, use importNode.") - - if not isNil(newAttr.fOwnerElement): - raise newException(EInuseAttributeErr, - "This attribute is in use by another element, use cloneNode") - # Exceptions end - - if isNil(el.attributes): el.attributes = @[] - return el.attributes.setNamedItemNS(newAttr) - -proc setAttribute*(el: PElement, name: string, value: string) = - ## Adds a new attribute, as specified by ``name`` and ``value`` - ## If an attribute with that name is already present in the element, its - ## value is changed to be that of the value parameter - ## Raises the EInvalidCharacterErr if the specified ``name`` contains - ## illegal characters - var attrNode = el.fOwnerDocument.createAttribute(name) - # Check if name contains illegal characters - if illegalChars in name: - raise newException(EInvalidCharacterErr, "Invalid character") - - discard el.setAttributeNode(attrNode) - # Set the info later, the setAttributeNode checks - # if FOwnerElement is nil, and if it isn't it raises an exception - attrNode.fOwnerElement = el - attrNode.fSpecified = true - attrNode.value = value - -proc setAttributeNS*(el: PElement, namespaceURI, localName, value: string) = - ## Adds a new attribute, as specified by ``namespaceURI``, ``localName`` - ## and ``value``. - - # Check if name contains illegal characters - if illegalChars in namespaceURI or illegalChars in localName: - raise newException(EInvalidCharacterErr, "Invalid character") - - var attrNode = el.fOwnerDocument.createAttributeNS(namespaceURI, localName) - - discard el.setAttributeNodeNS(attrNode) - # Set the info later, the setAttributeNode checks - # if FOwnerElement is nil, and if it isn't it raises an exception - attrNode.fOwnerElement = el - attrNode.fSpecified = true - attrNode.value = value - -# Text -proc splitData*(textNode: PText, offset: int): PText = - ## Breaks this node into two nodes at the specified offset, - ## keeping both in the tree as siblings. - - if offset > textNode.data.len(): - raise newException(EIndexSizeErr, "Index out of bounds") - - var left: string = textNode.data.substr(0, offset) - textNode.data = left - var right: string = textNode.data.substr(offset, textNode.data.len()) - - if not isNil(textNode.fParentNode) and not isNil(textNode.fParentNode.childNodes): - for i in low(textNode.fParentNode.childNodes)..high(textNode.fParentNode.childNodes): - if textNode.fParentNode.childNodes[i] == textNode: - var newNode: PText = textNode.fOwnerDocument.createTextNode(right) - textNode.fParentNode.childNodes.insert(newNode, i) - return newNode - else: - var newNode: PText = textNode.fOwnerDocument.createTextNode(right) - return newNode - - -# ProcessingInstruction -proc target*(pi: PProcessingInstruction): string = - ## Returns the Processing Instructions target - - return pi.fTarget - - -# --Other stuff-- -# Writer -proc addEscaped(s: string): string = - result = "" - for c in items(s): - case c - of '<': result.add("<") - of '>': result.add(">") - of '&': result.add("&") - of '"': result.add(""") - else: result.add(c) - -proc nodeToXml(n: PNode, indent: int = 0): string = - result = spaces(indent) & "<" & n.nodeName - if not isNil(n.attributes): - for i in items(n.attributes): - result.add(" " & i.name & "=\"" & addEscaped(i.value) & "\"") - - if isNil(n.childNodes) or n.childNodes.len() == 0: - result.add("/>") # No idea why this doesn't need a \n :O - else: - # End the beginning of this tag - result.add(">\n") - for i in items(n.childNodes): - case i.nodeType - of ElementNode: - result.add(nodeToXml(i, indent + 2)) - of TextNode: - result.add(spaces(indent * 2)) - result.add(addEscaped(i.nodeValue)) - of CDataSectionNode: - result.add(spaces(indent * 2)) - result.add("<![CDATA[" & i.nodeValue & "]]>") - of ProcessingInstructionNode: - result.add(spaces(indent * 2)) - result.add("<?" & PProcessingInstruction(i).target & " " & - PProcessingInstruction(i).data & " ?>") - of CommentNode: - result.add(spaces(indent * 2)) - result.add("<!-- " & i.nodeValue & " -->") - else: - continue - result.add("\n") - # Add the ending tag - </tag> - result.add(spaces(indent) & "</" & n.nodeName & ">") - -proc `$`*(doc: PDocument): string = - ## Converts a PDocument object into a string representation of it's XML - result = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" - result.add(nodeToXml(doc.documentElement)) diff --git a/lib/pure/xmldomparser.nim b/lib/pure/xmldomparser.nim deleted file mode 100644 index 7c7f7b99c..000000000 --- a/lib/pure/xmldomparser.nim +++ /dev/null @@ -1,168 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import xmldom, os, streams, parsexml, strutils - -## This module parses a XML Document into a XML DOM Document representation. - -#XMLDom's Parser - Turns XML into a Document - -type - # Parsing errors - EMismatchedTag* = object of ValueError ## Raised when a tag is not properly closed - EParserError* = object of ValueError ## Raised when an unexpected XML Parser event occurs - - # For namespaces - XmlnsAttr = tuple[name, value: string, ownerElement: PElement] - -var nsList: seq[XmlnsAttr] = @[] # Used for storing namespaces - -proc getNS(prefix: string): string = - var defaultNS: seq[string] = @[] - - for key, value, tag in items(nsList): - if ":" in key: - if key.split(':')[1] == prefix: - return value - - if key == "xmlns": - defaultNS.add(value) - - # Don't return the default namespaces - # in the loop, because then they would have a precedence - # over normal namespaces - if defaultNS.len() > 0: - return defaultNS[0] # Return the first found default namespace - # if none are specified for this prefix - - return "" - -proc parseText(x: var XmlParser, doc: var PDocument): PText = - result = doc.createTextNode(x.charData()) - -proc parseElement(x: var XmlParser, doc: var PDocument): PElement = - var n = doc.createElement("") - - while true: - case x.kind() - of xmlEof: - break - of xmlElementStart, xmlElementOpen: - if n.tagName() != "": - n.appendChild(parseElement(x, doc)) - else: - n = doc.createElementNS("", x.elementName) - - of xmlElementEnd: - if x.elementName == n.nodeName: - # n.normalize() # Remove any whitespace etc. - - var ns: string - if x.elementName.contains(':'): - ns = getNS(x.elementName.split(':')[0]) - else: - ns = getNS("") - - n.namespaceURI = ns - - # Remove any namespaces this element declared - var count = 0 # Variable which keeps the index - # We need to edit it.. - for i in low(nsList)..len(nsList)-1: - if nsList[count][2] == n: - nsList.delete(count) - dec(count) - inc(count) - - return n - else: #The wrong element is ended - raise newException(EMismatchedTag, "Mismatched tag at line " & - $x.getLine() & " column " & $x.getColumn) - - of xmlCharData: - n.appendChild(parseText(x, doc)) - of xmlAttribute: - if x.attrKey == "xmlns" or x.attrKey.startsWith("xmlns:"): - nsList.add((x.attrKey, x.attrValue, n)) - - if x.attrKey.contains(':'): - var ns = getNS(x.attrKey) - n.setAttributeNS(ns, x.attrKey, x.attrValue) - else: - n.setAttribute(x.attrKey, x.attrValue) - - of xmlCData: - n.appendChild(doc.createCDATASection(x.charData())) - of xmlComment: - n.appendChild(doc.createComment(x.charData())) - of xmlPI: - n.appendChild(doc.createProcessingInstruction(x.piName(), x.piRest())) - - of xmlWhitespace, xmlElementClose, xmlEntity, xmlSpecial: - discard " Unused \'events\'" - - else: - raise newException(EParserError, "Unexpected XML Parser event") - x.next() - - raise newException(EMismatchedTag, - "Mismatched tag at line " & $x.getLine() & " column " & $x.getColumn) - -proc loadXMLStream*(stream: Stream): PDocument = - ## Loads and parses XML from a stream specified by ``stream``, and returns - ## a ``PDocument`` - - var x: XmlParser - open(x, stream, nil, {reportComments}) - - var xmlDoc: PDocument - var dom: PDOMImplementation = getDOM() - - while true: - x.next() - case x.kind() - of xmlEof: - break - of xmlElementStart, xmlElementOpen: - var el: PElement = parseElement(x, xmlDoc) - xmlDoc = dom.createDocument(el) - of xmlWhitespace, xmlElementClose, xmlEntity, xmlSpecial: - discard " Unused \'events\'" - else: - raise newException(EParserError, "Unexpected XML Parser event") - - return xmlDoc - -proc loadXML*(xml: string): PDocument = - ## Loads and parses XML from a string specified by ``xml``, and returns - ## a ``PDocument`` - var s = newStringStream(xml) - return loadXMLStream(s) - - -proc loadXMLFile*(path: string): PDocument = - ## Loads and parses XML from a file specified by ``path``, and returns - ## a ``PDocument`` - - var s = newFileStream(path, fmRead) - if s == nil: raise newException(IOError, "Unable to read file " & path) - return loadXMLStream(s) - - -when not defined(testing) and isMainModule: - var xml = loadXMLFile("nim/xmldom/test.xml") - #echo(xml.getElementsByTagName("m:test2")[0].namespaceURI) - #echo(xml.getElementsByTagName("bla:test")[0].namespaceURI) - #echo(xml.getElementsByTagName("test")[0].namespaceURI) - for i in items(xml.getElementsByTagName("*")): - if i.namespaceURI != nil: - echo(i.nodeName, "=", i.namespaceURI) - - - echo($xml) diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 2a2c3e1dd..2c1e4e37c 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -9,14 +9,15 @@ ## This module parses an XML document and creates its XML tree representation. -import streams, parsexml, strtabs, xmltree +import std/[streams, parsexml, strtabs, xmltree] -type - XmlError* = object of ValueError ## exception that is raised - ## for invalid XML - errors*: seq[string] ## all detected parsing errors +when defined(nimPreviewSlimSystem): + import std/syncio -{.deprecated: [EInvalidXml: XmlError].} +type + XmlError* = object of ValueError ## Exception that is raised + ## for invalid XML. + errors*: seq[string] ## All detected parsing errors. proc raiseInvalidXml(errors: seq[string]) = var e: ref XmlError @@ -28,7 +29,7 @@ proc raiseInvalidXml(errors: seq[string]) = proc addNode(father, son: XmlNode) = if son != nil: add(father, son) -proc parse(x: var XmlParser, errors: var seq[string]): XmlNode +proc parse(x: var XmlParser, errors: var seq[string]): XmlNode {.gcsafe.} proc untilElementEnd(x: var XmlParser, result: XmlNode, errors: var seq[string]) = @@ -61,7 +62,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = of xmlError: errors.add(errorMsg(x)) next(x) - of xmlElementStart: ## ``<elem>`` + of xmlElementStart: ## ``<elem>`` result = newElement(x.elementName) next(x) untilElementEnd(x, result, errors) @@ -101,11 +102,11 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = of xmlEof: discard proc parseXml*(s: Stream, filename: string, - errors: var seq[string]): XmlNode = - ## parses the XML from stream `s` and returns a ``PXmlNode``. Every - ## occurred parsing error is added to the `errors` sequence. + errors: var seq[string], options: set[XmlParseOption] = {reportComments}): XmlNode = + ## Parses the XML from stream ``s`` and returns a ``XmlNode``. Every + ## occurred parsing error is added to the ``errors`` sequence. var x: XmlParser - open(x, s, filename, {reportComments}) + open(x, s, filename, options) while true: x.next() case x.kind @@ -120,32 +121,37 @@ proc parseXml*(s: Stream, filename: string, break close(x) -proc parseXml*(s: Stream): XmlNode = - ## parses the XTML from stream `s` and returns a ``PXmlNode``. All parsing - ## errors are turned into an ``EInvalidXML`` exception. +proc parseXml*(s: Stream, options: set[XmlParseOption] = {reportComments}): XmlNode = + ## Parses the XML from stream ``s`` and returns a ``XmlNode``. All parsing + ## errors are turned into an ``XmlError`` exception. var errors: seq[string] = @[] - result = parseXml(s, "unknown_html_doc", errors) + result = parseXml(s, "unknown_xml_doc", errors, options) if errors.len > 0: raiseInvalidXml(errors) -proc loadXml*(path: string, errors: var seq[string]): XmlNode = +proc parseXml*(str: string, options: set[XmlParseOption] = {reportComments}): XmlNode = + ## Parses the XML from string ``str`` and returns a ``XmlNode``. All parsing + ## errors are turned into an ``XmlError`` exception. + parseXml(newStringStream(str), options) + +proc loadXml*(path: string, errors: var seq[string], options: set[XmlParseOption] = {reportComments}): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. Every occurred parsing error is added to the `errors` + ## a ``XmlNode``. Every occurred parsing error is added to the ``errors`` ## sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) - result = parseXml(s, path, errors) + result = parseXml(s, path, errors, options) -proc loadXml*(path: string): XmlNode = +proc loadXml*(path: string, options: set[XmlParseOption] = {reportComments}): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns - ## a ``PXmlNode``. All parsing errors are turned into an ``EInvalidXML`` + ## a ``XmlNode``. All parsing errors are turned into an ``XmlError`` ## exception. var errors: seq[string] = @[] - result = loadXml(path, errors) + result = loadXml(path, errors, options) if errors.len > 0: raiseInvalidXml(errors) when isMainModule: when not defined(testing): - import os + import std/os var errors: seq[string] = @[] var x = loadXml(paramStr(1), errors) @@ -164,3 +170,7 @@ when isMainModule: var xml = loadXml(filePath, errors) assert(errors.len == 0, "The file tests/testdata/doc1.xml should be parsed without errors.") + block bug1518: + var err: seq[string] = @[] + assert $parsexml(newStringStream"<tag>One & two</tag>", "temp.xml", + err) == "<tag>One & two</tag>" diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index a9fc8998a..5c0cbc5e4 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -7,228 +7,760 @@ # distribution, for details about the copyright. # -## A simple XML tree. More efficient and simpler than the DOM. +## A simple XML tree generator. +## +runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") -import macros, strtabs + var h = newElement("secondTag") + h.add newEntity("some entity") -type - XmlNode* = ref XmlNodeObj ## an XML tree consists of ``PXmlNode``'s. + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + let k = newXmlTree("treeTag", [g, h], att) + + doAssert $k == """<treeTag key1="first value" key2="second value"> + <myTag>some text<!-- this is comment --></myTag> + <secondTag>&some entity;</secondTag> +</treeTag>""" + +## **See also:** +## * `xmlparser module <xmlparser.html>`_ for high-level XML parsing +## * `parsexml module <parsexml.html>`_ for low-level XML parsing +## * `htmlgen module <htmlgen.html>`_ for html code generator - XmlNodeKind* = enum ## different kinds of ``PXmlNode``'s - xnText, ## a text element - xnElement, ## an element with 0 or more children - xnCData, ## a CDATA node - xnEntity, ## an entity (like ``&thing;``) - xnComment ## an XML comment +import std/private/since +import std/[macros, strtabs, strutils, sequtils] - XmlAttributes* = StringTableRef ## an alias for a string to string mapping +when defined(nimPreviewSlimSystem): + import std/assertions + + +type + XmlNode* = ref XmlNodeObj ## An XML tree consisting of XML nodes. + ## + ## Use `newXmlTree proc <#newXmlTree,string,openArray[XmlNode],XmlAttributes>`_ + ## for creating a new tree. + + XmlNodeKind* = enum ## Different kinds of XML nodes. + xnText, ## a text element + xnVerbatimText, ## + xnElement, ## an element with 0 or more children + xnCData, ## a CDATA node + xnEntity, ## an entity (like ``&thing;``) + xnComment ## an XML comment + + XmlAttributes* = StringTableRef ## An alias for a string to string mapping. + ## + ## Use `toXmlAttributes proc <#toXmlAttributes,varargs[tuple[string,string]]>`_ + ## to create `XmlAttributes`. XmlNodeObj {.acyclic.} = object case k: XmlNodeKind # private, use the kind() proc to read this field. - of xnText, xnComment, xnCData, xnEntity: + of xnText, xnVerbatimText, xnComment, xnCData, xnEntity: fText: string of xnElement: fTag: string s: seq[XmlNode] fAttr: XmlAttributes - fClientData: int ## for other clients + fClientData: int ## for other clients -{.deprecated: [PXmlNode: XmlNode, TXmlNodeKind: XmlNodeKind, PXmlAttributes: - XmlAttributes, TXmlNode: XmlNodeObj].} +const + xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + ## Header to use for complete XML output. + +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k proc newXmlNode(kind: XmlNodeKind): XmlNode = - ## creates a new ``XmlNode``. - new(result) - result.k = kind + ## Creates a new ``XmlNode``. + result = XmlNode(k: kind) + +proc newElement*(tag: sink string): XmlNode = + ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`. + ## + ## See also: + ## * `newXmlTree proc <#newXmlTree,string,openArray[XmlNode],XmlAttributes>`_ + ## * [<> macro](#<>.m,untyped) + runnableExamples: + var a = newElement("firstTag") + a.add newElement("childTag") + assert a.kind == xnElement + assert $a == """<firstTag> + <childTag /> +</firstTag>""" -proc newElement*(tag: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnText`` with the given `tag`. result = newXmlNode(xnElement) result.fTag = tag result.s = @[] - # init attributes lazily to safe memory + # init attributes lazily to save memory + +proc newText*(text: sink string): XmlNode = + ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`. + runnableExamples: + var b = newText("my text") + assert b.kind == xnText + assert $b == "my text" -proc newText*(text: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnText`` with the text `text`. result = newXmlNode(xnText) result.fText = text -proc newComment*(comment: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnComment`` with the text `comment`. +proc newVerbatimText*(text: sink string): XmlNode {.since: (1, 3).} = + ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`. + ## **Since**: Version 1.3. + result = newXmlNode(xnVerbatimText) + result.fText = text + +proc newComment*(comment: sink string): XmlNode = + ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`. + runnableExamples: + var c = newComment("my comment") + assert c.kind == xnComment + assert $c == "<!-- my comment -->" + result = newXmlNode(xnComment) result.fText = comment -proc newCData*(cdata: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnComment`` with the text `cdata`. +proc newCData*(cdata: sink string): XmlNode = + ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`. + runnableExamples: + var d = newCData("my cdata") + assert d.kind == xnCData + assert $d == "<![CDATA[my cdata]]>" + result = newXmlNode(xnCData) result.fText = cdata proc newEntity*(entity: string): XmlNode = - ## creates a new ``PXmlNode`` of kind ``xnEntity`` with the text `entity`. + ## Creates a new ``XmlNode`` of kind ``xnEntity`` with the text `entity`. + runnableExamples: + var e = newEntity("my entity") + assert e.kind == xnEntity + assert $e == "&my entity;" + result = newXmlNode(xnEntity) result.fText = entity -proc text*(n: XmlNode): string {.inline.} = - ## gets the associated text with the node `n`. `n` can be a CDATA, Text, - ## comment, or entity node. - assert n.k in {xnText, xnComment, xnCData, xnEntity} +proc newXmlTree*(tag: sink string, children: openArray[XmlNode], + attributes: XmlAttributes = nil): XmlNode = + ## Creates a new XML tree with `tag`, `children` and `attributes`. + ## + ## See also: + ## * `newElement proc <#newElement,string>`_ + ## * [<> macro](#<>.m,untyped) + + runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + var h = newElement("secondTag") + h.add newEntity("some entity") + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + let k = newXmlTree("treeTag", [g, h], att) + + doAssert $k == """<treeTag key1="first value" key2="second value"> + <myTag>some text<!-- this is comment --></myTag> + <secondTag>&some entity;</secondTag> +</treeTag>""" + + result = newXmlNode(xnElement) + result.fTag = tag + newSeq(result.s, children.len) + for i in 0..children.len-1: result.s[i] = children[i] + result.fAttr = attributes + +proc text*(n: XmlNode): lent string {.inline.} = + ## Gets the associated text with the node `n`. + ## + ## `n` can be a CDATA, Text, comment, or entity node. + ## + ## See also: + ## * `text= proc <#text=,XmlNode,string>`_ for text setter + ## * `tag proc <#tag,XmlNode>`_ for tag getter + ## * `tag= proc <#tag=,XmlNode,string>`_ for tag setter + ## * `innerText proc <#innerText,XmlNode>`_ + runnableExamples: + var c = newComment("my comment") + assert $c == "<!-- my comment -->" + assert c.text == "my comment" + + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} result = n.fText -proc `text=`*(n: XmlNode, text: string){.inline.} = - ## sets the associated text with the node `n`. `n` can be a CDATA, Text, - ## comment, or entity node. - assert n.k in {xnText, xnComment, xnCData, xnEntity} +proc `text=`*(n: XmlNode, text: sink string) {.inline.} = + ## Sets the associated text with the node `n`. + ## + ## `n` can be a CDATA, Text, comment, or entity node. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ for text getter + ## * `tag proc <#tag,XmlNode>`_ for tag getter + ## * `tag= proc <#tag=,XmlNode,string>`_ for tag setter + runnableExamples: + var e = newEntity("my entity") + assert $e == "&my entity;" + e.text = "a new entity text" + assert $e == "&a new entity text;" + + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} n.fText = text +proc tag*(n: XmlNode): lent string {.inline.} = + ## Gets the tag name of `n`. + ## + ## `n` has to be an ``xnElement`` node. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ for text getter + ## * `text= proc <#text=,XmlNode,string>`_ for text setter + ## * `tag= proc <#tag=,XmlNode,string>`_ for tag setter + ## * `innerText proc <#innerText,XmlNode>`_ + runnableExamples: + var a = newElement("firstTag") + a.add newElement("childTag") + assert $a == """<firstTag> + <childTag /> +</firstTag>""" + assert a.tag == "firstTag" + + n.expect xnElement + result = n.fTag + +proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = + ## Sets the tag name of `n`. + ## + ## `n` has to be an ``xnElement`` node. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ for text getter + ## * `text= proc <#text=,XmlNode,string>`_ for text setter + ## * `tag proc <#tag,XmlNode>`_ for tag getter + runnableExamples: + var a = newElement("firstTag") + a.add newElement("childTag") + assert $a == """<firstTag> + <childTag /> +</firstTag>""" + a.tag = "newTag" + assert $a == """<newTag> + <childTag /> +</newTag>""" + + n.expect xnElement + n.fTag = tag + proc rawText*(n: XmlNode): string {.inline.} = - ## returns the underlying 'text' string by reference. + ## Returns the underlying 'text' string by reference. + ## ## This is only used for speed hacks. - shallowCopy(result, n.fText) + when defined(gcDestructors): + result = move(n.fText) + else: + shallowCopy(result, n.fText) proc rawTag*(n: XmlNode): string {.inline.} = - ## returns the underlying 'tag' string by reference. + ## Returns the underlying 'tag' string by reference. + ## ## This is only used for speed hacks. - shallowCopy(result, n.fTag) + when defined(gcDestructors): + result = move(n.fTag) + else: + shallowCopy(result, n.fTag) proc innerText*(n: XmlNode): string = - ## gets the inner text of `n`. `n` has to be an ``xnElement`` node. Only - ## ``xnText`` and ``xnEntity`` nodes are considered part of `n`'s inner text, - ## other child nodes are silently ignored. - result = "" - assert n.k == xnElement - for i in 0 .. n.s.len-1: - if n.s[i].k in {xnText, xnEntity}: result.add(n.s[i].fText) - -proc tag*(n: XmlNode): string {.inline.} = - ## gets the tag name of `n`. `n` has to be an ``xnElement`` node. - assert n.k == xnElement - result = n.fTag + ## Gets the inner text of `n`: + ## + ## - If `n` is `xnText` or `xnEntity`, returns its content. + ## - If `n` is `xnElement`, runs recursively on each child node and + ## concatenates the results. + ## - Otherwise returns an empty string. + ## + ## See also: + ## * `text proc <#text,XmlNode>`_ + runnableExamples: + var f = newElement("myTag") + f.add newText("my text") + f.add newComment("my comment") + f.add newEntity("my entity") + assert $f == "<myTag>my text<!-- my comment -->&my entity;</myTag>" + assert innerText(f) == "my textmy entity" + + proc worker(res: var string, n: XmlNode) = + case n.k + of xnText, xnEntity: + res.add(n.fText) + of xnElement: + for sub in n.s: + worker(res, sub) + else: + discard -proc `tag=`*(n: XmlNode, tag: string) {.inline.} = - ## sets the tag name of `n`. `n` has to be an ``xnElement`` node. - assert n.k == xnElement - n.fTag = tag + result = "" + worker(result, n) proc add*(father, son: XmlNode) {.inline.} = - ## adds the child `son` to `father`. + ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newText("my text") + f.add newElement("sonTag") + f.add newEntity("my entity") + assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = - ## insert the child `son` to a given position in `father`. - assert father.k == xnElement and son.k == xnElement + ## Inserts the child `son` to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + assert $f == """<myTag> + <second /> + <first /> +</myTag>""" + + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """<myTag> + <second /> + <third /> + <first /> +</myTag>""" + + father.expect xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + +proc delete*(n: XmlNode, i: Natural) = + ## Deletes the `i`'th child of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.delete(0) + assert $f == """<myTag> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(i) + +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """<myTag> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) + proc len*(n: XmlNode): int {.inline.} = - ## returns the number `n`'s children. + ## Returns the number of `n`'s children. + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + assert len(f) == 2 if n.k == xnElement: result = len(n.s) proc kind*(n: XmlNode): XmlNodeKind {.inline.} = - ## returns `n`'s kind. + ## Returns `n`'s kind. + runnableExamples: + var a = newElement("firstTag") + assert a.kind == xnElement + var b = newText("my text") + assert b.kind == xnText result = n.k -proc `[]`* (n: XmlNode, i: int): XmlNode {.inline.} = - ## returns the `i`'th child of `n`. - assert n.k == xnElement - result = n.s[i] +proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = + ## Returns the `i`'th child of `n`. + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + assert $f[1] == "<first />" + assert $f[0] == "<second />" -proc delete*(n: XmlNode, i: Natural) {.noSideEffect.} = - ## delete the `i`'th child of `n`. - assert n.k == xnElement - n.s.delete(i) + n.expect xnElement + result = n.s[i] -proc `[]`* (n: var XmlNode, i: int): var XmlNode {.inline.} = - ## returns the `i`'th child of `n` so that it can be modified - assert n.k == xnElement +proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = + ## Returns the `i`'th child of `n` so that it can be modified. + n.expect xnElement result = n.s[i] -proc mget*(n: var XmlNode, i: int): var XmlNode {.inline, deprecated.} = - ## returns the `i`'th child of `n` so that it can be modified. Use ```[]``` - ## instead. - n[i] +proc clear*(n: var XmlNode) = + ## Recursively clears all children of an XmlNode. + ## + runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + + var h = newElement("secondTag") + h.add newEntity("some entity") + + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + var k = newXmlTree("treeTag", [g, h], att) + + doAssert $k == """<treeTag key1="first value" key2="second value"> + <myTag>some text<!-- this is comment --></myTag> + <secondTag>&some entity;</secondTag> +</treeTag>""" + + clear(k) + doAssert $k == """<treeTag key1="first value" key2="second value" />""" + + for i in 0 ..< n.len: + clear(n[i]) + if n.k == xnElement: + n.s.setLen(0) + iterator items*(n: XmlNode): XmlNode {.inline.} = - ## iterates over any child of `n`. - assert n.k == xnElement + ## Iterates over all direct children of `n`. + + runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + + var h = newElement("secondTag") + h.add newEntity("some entity") + g.add h + + assert $g == "<myTag>some text<!-- this is comment --><secondTag>&some entity;</secondTag></myTag>" + + # for x in g: # the same as `for x in items(g):` + # echo x + + # some text + # <!-- this is comment --> + # <secondTag>&some entity;<![CDATA[some cdata]]></secondTag> + + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = - ## iterates over any child of `n`. - assert n.k == xnElement + ## Iterates over all direct children of `n` so that they can be modified. + n.expect xnElement for i in 0 .. n.len-1: yield n[i] +proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, + val: string]]): XmlAttributes = + ## Converts `{key: value}` pairs into `XmlAttributes`. + ## + runnableExamples: + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + var j = newElement("myTag") + j.attrs = att + + doAssert $j == """<myTag key1="first value" key2="second value" />""" + + newStringTable(keyValuePairs) + proc attrs*(n: XmlNode): XmlAttributes {.inline.} = - ## gets the attributes belonging to `n`. + ## Gets the attributes belonging to `n`. + ## ## Returns `nil` if attributes have not been initialised for this node. - assert n.k == xnElement + ## + ## See also: + ## * `attrs= proc <#attrs=,XmlNode,XmlAttributes>`_ for XmlAttributes setter + ## * `attrsLen proc <#attrsLen,XmlNode>`_ for number of attributes + ## * `attr proc <#attr,XmlNode,string>`_ for finding an attribute + runnableExamples: + var j = newElement("myTag") + assert j.attrs == nil + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attrs == att + + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = - ## sets the attributes belonging to `n`. - assert n.k == xnElement + ## Sets the attributes belonging to `n`. + ## + ## See also: + ## * `attrs proc <#attrs,XmlNode>`_ for XmlAttributes getter + ## * `attrsLen proc <#attrsLen,XmlNode>`_ for number of attributes + ## * `attr proc <#attr,XmlNode,string>`_ for finding an attribute + runnableExamples: + var j = newElement("myTag") + assert j.attrs == nil + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attrs == att + + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = - ## returns the number of `n`'s attributes. - assert n.k == xnElement + ## Returns the number of `n`'s attributes. + ## + ## See also: + ## * `attrs proc <#attrs,XmlNode>`_ for XmlAttributes getter + ## * `attrs= proc <#attrs=,XmlNode,XmlAttributes>`_ for XmlAttributes setter + ## * `attr proc <#attr,XmlNode,string>`_ for finding an attribute + runnableExamples: + var j = newElement("myTag") + assert j.attrsLen == 0 + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attrsLen == 2 + + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) +proc attr*(n: XmlNode, name: string): string = + ## Finds the first attribute of `n` with a name of `name`. + ## Returns "" on failure. + ## + ## See also: + ## * `attrs proc <#attrs,XmlNode>`_ for XmlAttributes getter + ## * `attrs= proc <#attrs=,XmlNode,XmlAttributes>`_ for XmlAttributes setter + ## * `attrsLen proc <#attrsLen,XmlNode>`_ for number of attributes + runnableExamples: + var j = newElement("myTag") + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + j.attrs = att + assert j.attr("key1") == "first value" + assert j.attr("key2") == "second value" + + n.expect xnElement + if n.attrs == nil: return "" + return n.attrs.getOrDefault(name) + proc clientData*(n: XmlNode): int {.inline.} = - ## gets the client data of `n`. The client data field is used by the HTML - ## parser and generator. + ## Gets the client data of `n`. + ## + ## The client data field is used by the HTML parser and generator. result = n.fClientData proc `clientData=`*(n: XmlNode, data: int) {.inline.} = - ## sets the client data of `n`. The client data field is used by the HTML - ## parser and generator. + ## Sets the client data of `n`. + ## + ## The client data field is used by the HTML parser and generator. n.fClientData = data proc addEscaped*(result: var string, s: string) = - ## same as ``result.add(escape(s))``, but more efficient. + ## The same as `result.add(escape(s)) <#escape,string>`_, but more efficient. for c in items(s): case c of '<': result.add("<") of '>': result.add(">") of '&': result.add("&") of '"': result.add(""") - of '\'': result.add("'") - of '/': result.add("/") + of '\'': result.add("'") else: result.add(c) proc escape*(s: string): string = - ## escapes `s` for inclusion into an XML document. + ## Escapes `s` for inclusion into an XML document. + ## ## Escapes these characters: ## - ## ------------ ------------------- + ## ============ =================== ## char is converted to - ## ------------ ------------------- + ## ============ =================== ## ``<`` ``<`` ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` - ## ``'`` ``'`` - ## ``/`` ``/`` - ## ------------ ------------------- + ## ``'`` ``'`` + ## ============ =================== + ## + ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) addEscaped(result, s) -proc addIndent(result: var string, indent: int) = - result.add("\n") - for i in 1..indent: result.add(' ') +proc addIndent(result: var string, indent: int, addNewLines: bool) = + if addNewLines: + result.add("\n") + for i in 1 .. indent: + result.add(' ') + +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = + proc noWhitespace(n: XmlNode): bool = + for i in 0 ..< n.len: + if n[i].kind in {xnText, xnVerbatimText, xnEntity}: return true + + proc addEscapedAttr(result: var string, s: string) = + # `addEscaped` alternative with less escaped characters. + # Only to be used for escaping attribute values enclosed in double quotes! + for c in items(s): + case c + of '<': result.add("<") + of '>': result.add(">") + of '&': result.add("&") + of '"': result.add(""") + else: result.add(c) -proc noWhitespace(n: XmlNode): bool = - #for i in 1..n.len-1: - # if n[i].kind != n[0].kind: return true - for i in 0..n.len-1: - if n[i].kind in {xnText, xnEntity}: return true - -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = - ## adds the textual representation of `n` to `result`. if n == nil: return + case n.k of xnElement: + if indent > 0 and not lastNodeIsText: + result.addIndent(indent, addNewLines) + + let + addNewLines = if n.noWhitespace(): + false + else: + addNewLines + result.add('<') result.add(n.fTag) if not isNil(n.fAttr): @@ -236,30 +768,34 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = result.add(' ') result.add(key) result.add("=\"") - result.addEscaped(val) + result.addEscapedAttr(val) result.add('"') - if n.len > 0: - result.add('>') - if n.len > 1: - if noWhitespace(n): - # for mixed leaves, we cannot output whitespace for readability, - # because this would be wrong. For example: ``a<b>b</b>`` is - # different from ``a <b>b</b>``. - for i in 0..n.len-1: result.add(n[i], indent+indWidth, indWidth) - else: - for i in 0..n.len-1: - result.addIndent(indent+indWidth) - result.add(n[i], indent+indWidth, indWidth) - result.addIndent(indent) - else: - result.add(n[0], indent+indWidth, indWidth) - result.add("</") - result.add(n.fTag) - result.add(">") - else: + + if n.len == 0: result.add(" />") + return + + let + indentNext = if n.noWhitespace(): + indent + else: + indent+indWidth + result.add('>') + var lastNodeIsText = false + for i in 0 ..< n.len: + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = (n[i].kind == xnText) or (n[i].kind == xnVerbatimText) + + if not n.noWhitespace(): + result.addIndent(indent, addNewLines) + + result.add("</") + result.add(n.fTag) + result.add(">") of xnText: result.addEscaped(n.fText) + of xnVerbatimText: + result.add(n.fText) of xnComment: result.add("<!-- ") result.addEscaped(n.fText) @@ -273,37 +809,118 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = result.add(n.fText) result.add(';') -const - xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" - ## header to use for complete XML output +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "<!-- my comment --><firstTag />my text" + result.addImpl(n, indent, indWidth, addNewLines) proc `$`*(n: XmlNode): string = - ## converts `n` into its string representation. No ``<$xml ...$>`` declaration - ## is produced, so that the produced XML fragments are composable. + ## Converts `n` into its string representation. + ## + ## No ``<$xml ...$>`` declaration is produced, so that the produced + ## XML fragments are composable. result = "" result.add(n) -proc newXmlTree*(tag: string, children: openArray[XmlNode], - attributes: XmlAttributes = nil): XmlNode = - ## creates a new XML tree with `tag`, `children` and `attributes` - result = newXmlNode(xnElement) - result.fTag = tag - newSeq(result.s, children.len) - for i in 0..children.len-1: result.s[i] = children[i] - result.fAttr = attributes +proc child*(n: XmlNode, name: string): XmlNode = + ## Finds the first child element of `n` with a name of `name`. + ## Returns `nil` on failure. + runnableExamples: + var f = newElement("myTag") + f.add newElement("firstSon") + f.add newElement("secondSon") + f.add newElement("thirdSon") + assert $(f.child("secondSon")) == "<secondSon />" + + n.expect xnElement + for i in items(n): + if i.kind == xnElement: + if i.tag == name: + return i + +proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], + caseInsensitive = false) = + ## Iterates over all the children of `n` returning those matching `tag`. + ## + ## Found nodes satisfying the condition will be appended to the `result` + ## sequence. + runnableExamples: + var + b = newElement("good") + c = newElement("bad") + d = newElement("BAD") + e = newElement("GOOD") + b.add newText("b text") + c.add newText("c text") + d.add newText("d text") + e.add newText("e text") + let a = newXmlTree("father", [b, c, d, e]) + var s = newSeq[XmlNode]() + a.findAll("good", s) + assert $s == "@[<good>b text</good>]" + s.setLen(0) + a.findAll("good", s, caseInsensitive = true) + assert $s == "@[<good>b text</good>, <GOOD>e text</GOOD>]" + s.setLen(0) + a.findAll("BAD", s) + assert $s == "@[<BAD>d text</BAD>]" + s.setLen(0) + a.findAll("BAD", s, caseInsensitive = true) + assert $s == "@[<bad>c text</bad>, <BAD>d text</BAD>]" + + n.expect xnElement + for child in n.items(): + if child.k != xnElement: + continue + if child.tag == tag or + (caseInsensitive and cmpIgnoreCase(child.tag, tag) == 0): + result.add(child) + child.findAll(tag, result) + +proc findAll*(n: XmlNode, tag: string, caseInsensitive = false): seq[XmlNode] = + ## A shortcut version to assign in let blocks. + runnableExamples: + var + b = newElement("good") + c = newElement("bad") + d = newElement("BAD") + e = newElement("GOOD") + b.add newText("b text") + c.add newText("c text") + d.add newText("d text") + e.add newText("e text") + let a = newXmlTree("father", [b, c, d, e]) + assert $(a.findAll("good")) == "@[<good>b text</good>]" + assert $(a.findAll("BAD")) == "@[<BAD>d text</BAD>]" + assert $(a.findAll("good", caseInsensitive = true)) == "@[<good>b text</good>, <GOOD>e text</GOOD>]" + assert $(a.findAll("BAD", caseInsensitive = true)) == "@[<bad>c text</bad>, <BAD>d text</BAD>]" + + newSeq(result, 0) + findAll(n, tag, result, caseInsensitive) -proc xmlConstructor(e: NimNode): NimNode {.compileTime.} = - expectLen(e, 2) - var a = e[1] +proc xmlConstructor(a: NimNode): NimNode = if a.kind == nnkCall: result = newCall("newXmlTree", toStrLit(a[0])) var attrs = newNimNode(nnkBracket, a) - var newStringTabCall = newCall("newStringTable", attrs, - newIdentNode("modeCaseSensitive")) + var newStringTabCall = newCall(bindSym"newStringTable", attrs, + bindSym"modeCaseSensitive") var elements = newNimNode(nnkBracket, a) for i in 1..a.len-1: if a[i].kind == nnkExprEqExpr: - attrs.add(toStrLit(a[i][0])) + # In order to support attributes like `data-lang` we have to + # replace whitespace because `toStrLit` gives `data - lang`. + let attrName = toStrLit(a[i][0]).strVal.replace(" ", "") + attrs.add(newStrLitNode(attrName)) attrs.add(a[i][1]) #echo repr(attrs) else: @@ -315,72 +932,15 @@ proc xmlConstructor(e: NimNode): NimNode {.compileTime.} = else: result = newCall("newXmlTree", toStrLit(a)) -macro `<>`*(x: expr): expr {.immediate.} = +macro `<>`*(x: untyped): untyped = ## Constructor macro for XML. Example usage: ## - ## .. code-block:: nim + ## ```nim ## <>a(href="http://nim-lang.org", newText("Nim rules.")) + ## ``` ## - ## Produces an XML tree for:: + ## Produces an XML tree for: ## - ## <a href="http://nim-lang.org">Nim rules.</a> + ## <a href="http://nim-lang.org">Nim rules.</a> ## - let x = callsite() result = xmlConstructor(x) - -proc child*(n: XmlNode, name: string): XmlNode = - ## Finds the first child element of `n` with a name of `name`. - ## Returns `nil` on failure. - assert n.kind == xnElement - for i in items(n): - if i.kind == xnElement: - if i.tag == name: - return i - -proc attr*(n: XmlNode, name: string): string = - ## Finds the first attribute of `n` with a name of `name`. - ## Returns "" on failure. - assert n.kind == xnElement - if n.attrs == nil: return "" - return n.attrs.getOrDefault(name) - -proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode]) = - ## Iterates over all the children of `n` returning those matching `tag`. - ## - ## Found nodes satisfying the condition will be appended to the `result` - ## sequence, which can't be nil or the proc will crash. Usage example: - ## - ## .. code-block:: - ## var - ## html: XmlNode - ## tags: seq[XmlNode] = @[] - ## - ## html = buildHtml() - ## findAll(html, "img", tags) - ## for imgTag in tags: - ## process(imgTag) - assert isNil(result) == false - assert n.k == xnElement - for child in n.items(): - if child.k != xnElement: - continue - if child.tag == tag: - result.add(child) - child.findAll(tag, result) - -proc findAll*(n: XmlNode, tag: string): seq[XmlNode] = - ## Shortcut version to assign in let blocks. Example: - ## - ## .. code-block:: - ## var html: XmlNode - ## - ## html = buildHtml(html) - ## for imgTag in html.findAll("img"): - ## process(imgTag) - newSeq(result, 0) - findAll(n, tag, result) - -when isMainModule: - let link = "http://nim-lang.org" - assert """<a href="""" & escape(link) & """">Nim rules.</a>""" == - $(<>a(href="http://nim-lang.org", newText("Nim rules."))) diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim new file mode 100644 index 000000000..963451efe --- /dev/null +++ b/lib/std/appdirs.nim @@ -0,0 +1,94 @@ +## This module implements helpers for determining special directories used by apps. + +## .. importdoc:: paths.nim + +from std/private/osappdirs import nil +import std/paths +import std/envvars + +proc getHomeDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getHomeDir()) + +proc getDataDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + result = Path(osappdirs.getDataDir()) + +proc getConfigDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getConfigDir()) + +proc getCacheDir*(): Path {.inline.} = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + # follows https://crates.io/crates/platform-dirs + result = Path(osappdirs.getCacheDir()) + +proc getCacheDir*(app: Path): Path {.inline.} = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + result = Path(osappdirs.getCacheDir(app.string)) + +proc getTempDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## .. Note:: This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + result = Path(osappdirs.getTempDir()) diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim new file mode 100644 index 000000000..56c37d205 --- /dev/null +++ b/lib/std/assertions.nim @@ -0,0 +1,122 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" + include "system/fatal" + +## This module implements assertion handling. + +import std/private/miscdollars +# --------------------------------------------------------------------------- +# helpers + +type InstantiationInfo = tuple[filename: string, line: int, column: int] + +proc `$`(info: InstantiationInfo): string = + # The +1 is needed here + # instead of overriding `$` (and changing its meaning), consider explicit name. + result = "" + result.toLocation(info.filename, info.line, info.column + 1) + +# --------------------------------------------------------------------------- + + +proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = + ## Raises an `AssertionDefect` with `msg`. + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) + +proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = + ## Raises an `AssertionDefect` with `msg`, but this is hidden + ## from the effect system. Called when an assertion failed. + raiseAssert(msg) + +template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = + when enabled: + const + loc = instantiationInfo(fullPaths = compileOption("excessiveStackTrace")) + ploc = $loc + bind instantiationInfo + mixin failedAssertImpl + {.line: loc.}: + if not cond: + failedAssertImpl(ploc & " `" & expr & "` " & msg) + +template assert*(cond: untyped, msg = "") = + ## Raises `AssertionDefect` with `msg` if `cond` is false. Note + ## that `AssertionDefect` is hidden from the effect system, so it doesn't + ## produce `{.raises: [AssertionDefect].}`. This exception is only supposed + ## to be caught by unit testing frameworks. + ## + ## No code will be generated for `assert` when passing `-d:danger` (implied by `--assertions:off`). + ## See `command line switches <nimc.html#compiler-usage-commandminusline-switches>`_. + runnableExamples: assert 1 == 1 + runnableExamples("--assertions:off"): + assert 1 == 2 # no code generated, no failure here + runnableExamples("-d:danger"): assert 1 == 2 # ditto + assertImpl(cond, msg, astToStr(cond), compileOption("assertions")) + +template doAssert*(cond: untyped, msg = "") = + ## Similar to `assert <#assert.t,untyped,string>`_ but is always turned on regardless of `--assertions`. + runnableExamples: + doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off` + assertImpl(cond, msg, astToStr(cond), true) + +template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = + ## Sets an assertion failure handler that will intercept any assert + ## statements following `onFailedAssert` in the current scope. + runnableExamples: + type MyError = object of CatchableError + lineinfo: tuple[filename: string, line: int, column: int] + # block-wide policy to change the failed assert exception type in order to + # include a lineinfo + onFailedAssert(msg): + raise (ref MyError)(msg: msg, lineinfo: instantiationInfo(-2)) + doAssertRaises(MyError): doAssert false + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template failedAssertImpl(msgIMPL: string): untyped {.dirty, redefine.} = + let msg = msgIMPL + code + +template doAssertRaises*(exception: typedesc, code: untyped) = + ## Raises `AssertionDefect` if specified `code` does not raise `exception`. + runnableExamples: + doAssertRaises(ValueError): raise newException(ValueError, "Hello World") + doAssertRaises(CatchableError): raise newException(ValueError, "Hello World") + doAssertRaises(AssertionDefect): doAssert false + var wrong = false + const begin = "expected raising '" & astToStr(exception) & "', instead" + const msgEnd = " by: " & astToStr(code) + template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.push warning[BareExcept]:off.} + when Exception is exception: + try: + if true: + code + wrong = true + except Exception as e: discard + except: raisedForeign() + else: + try: + if true: + code + wrong = true + except exception: + discard + except Exception as e: + mixin `$` # alternatively, we could define $cstring in this module + raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) + except: raisedForeign() + {.pop.} + if wrong: + raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim new file mode 100644 index 000000000..0ba4619e5 --- /dev/null +++ b/lib/std/cmdline.nim @@ -0,0 +1,313 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains system facilities for reading command +## line parameters. + +## **See also:** +## * `parseopt module <parseopt.html>`_ for command-line parser beyond +## `parseCmdLine proc`_ + + +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/widestrs + +when defined(nodejs): + from std/private/oscommon import ReadDirEffect + + +const weirdTarget = defined(nimscript) or defined(js) + + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix +else: + {.error: "The cmdline module has not been implemented for the target platform.".} + + +# Needed by windows in order to obtain the command line for targets +# other than command line targets +when defined(windows) and not weirdTarget: + template getCommandLine*(): untyped = getCommandLineW() + + +proc parseCmdLine*(c: string): seq[string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a `command line`:idx: into several components. + ## + ## **Note**: This proc is only occasionally useful, better use the + ## `parseopt module <parseopt.html>`_. + ## + ## On Windows, it uses the `following parsing rules + ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: + ## + ## * Arguments are delimited by white space, which is either a space or a tab. + ## * The caret character (^) is not recognized as an escape character or + ## delimiter. The character is handled completely by the command-line parser + ## in the operating system before being passed to the argv array in the + ## program. + ## * A string surrounded by double quotation marks ("string") is interpreted + ## as a single argument, regardless of white space contained within. A + ## quoted string can be embedded in an argument. + ## * A double quotation mark preceded by a backslash (\") is interpreted as a + ## literal double quotation mark character ("). + ## * Backslashes are interpreted literally, unless they immediately precede + ## a double quotation mark. + ## * If an even number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is interpreted as a string delimiter. + ## * If an odd number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is "escaped" by the remaining backslash, + ## causing a literal double quotation mark (") to be placed in argv. + ## + ## On Posix systems, it uses the following parsing rules: + ## Components are separated by whitespace unless the whitespace + ## occurs within ``"`` or ``'`` quotes. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break + when defined(windows): + # parse a single argument according to the above rules: + var inQuote = false + while i < c.len: + case c[i] + of '\\': + var j = i + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': + for k in 1..(j-i) div 2: a.add('\\') + if (j-i) mod 2 == 0: + i = j + else: + a.add('"') + i = j+1 + else: + a.add(c[i]) + inc(i) + of '"': + inc(i) + if not inQuote: inQuote = true + elif i < c.len and c[i] == '"': + a.add(c[i]) + inc(i) + else: + inQuote = false + break + of ' ', '\t': + if not inQuote: break + a.add(c[i]) + inc(i) + else: + a.add(c[i]) + inc(i) + else: + case c[i] + of '\'', '\"': + var delim = c[i] + inc(i) # skip ' or " + while i < c.len and c[i] != delim: + add a, c[i] + inc(i) + if i < c.len: inc(i) + else: + while i < c.len and c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, move a) + +when defined(nimdoc): + # Common forward declaration docstring block for parameter retrieval procs. + proc paramCount*(): int {.tags: [ReadIOEffect].} = + ## Returns the number of `command line arguments`:idx: given to the + ## application. + ## + ## Unlike `argc`:idx: in C, if your binary was called without parameters this + ## will return zero. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramCount): + ## # Use paramCount() here + ## else: + ## # Do something else! + ## ``` + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + ## Returns the `i`-th `command line argument`:idx: given to the application. + ## + ## `i` should be in the range `1..paramCount()`, the `IndexDefect` + ## exception will be raised for invalid values. Instead of iterating + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. + ## + ## Similarly to `argv`:idx: in C, + ## it is possible to call `paramStr(0)` but this will return OS specific + ## contents (usually the name of the invoked executable). You should avoid + ## this and call `getAppFilename() <os.html#getAppFilename>`_ instead. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramStr): + ## # Use paramStr() here + ## else: + ## # Do something else! + ## ``` + +elif defined(nimscript): discard +elif defined(nodejs): + type Argv = object of JsRoot + let argv {.importjs: "process.argv".} : Argv + proc len(argv: Argv): int {.importjs: "#.length".} + proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} + + proc paramCount*(): int {.tags: [ReadDirEffect].} = + result = argv.len - 2 + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + let i = i + 1 + if i < argv.len and i >= 0: + result = $argv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) +elif defined(windows): + # Since we support GUI applications with Nim, we sometimes generate + # a WinMain entry proc. But a WinMain proc has no access to the parsed + # command line arguments. The way to get them differs. Thus we parse them + # ourselves. This has the additional benefit that the program's behaviour + # is always the same -- independent of the used C compiler. + var + ownArgv {.threadvar.}: seq[string] + ownParsedArgv {.threadvar.}: bool + + proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + result = ownArgv.len-1 + + proc paramStr*(i: int): string {.rtl, extern: "nos$1", + tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + if i < ownArgv.len and i >= 0: + result = ownArgv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) + +elif defined(genode): + proc paramStr*(i: int): string = + raise newException(OSError, "paramStr is not implemented on Genode") + + proc paramCount*(): int = + raise newException(OSError, "paramCount is not implemented on Genode") +elif weirdTarget or (defined(posix) and appType == "lib"): + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramStr is not implemented on current platform") + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramCount is not implemented on current platform") +elif not defined(createNimRtl) and + not(defined(posix) and appType == "lib"): + # On Posix, there is no portable way to get the command line from a DLL. + var + cmdCount {.importc: "cmdCount".}: cint + cmdLine {.importc: "cmdLine".}: cstringArray + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if i < cmdCount and i >= 0: + result = $cmdLine[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + result = cmdCount-1 + +when declared(paramCount) or defined(nimdoc): + proc commandLineParams*(): seq[string] = + ## Convenience proc which returns the command line parameters. + ## + ## This returns **only** the parameters. If you want to get the application + ## executable filename, call `getAppFilename() <os.html#getAppFilename>`_. + ## + ## **Availability**: On Posix there is no portable way to get the command + ## line from a DLL and thus the proc isn't defined in this environment. You + ## can test for its availability with `declared() + ## <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(commandLineParams): + ## # Use commandLineParams() here + ## else: + ## # Do something else! + ## ``` + result = @[] + for i in 1..paramCount(): + result.add(paramStr(i)) +else: + proc commandLineParams*(): seq[string] {.error: + "commandLineParams() unsupported by dynamic libraries".} = + discard diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim new file mode 100644 index 000000000..6d8bd22f4 --- /dev/null +++ b/lib/std/compilesettings.nim @@ -0,0 +1,66 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows querying the compiler about +## diverse configuration settings. See also `compileOption`. + +# Note: Only add new enum values at the end to ensure binary compatibility with +# other Nim compiler versions! + +type + SingleValueSetting* {.pure.} = enum ## \ + ## settings resulting in a single string value + arguments, ## experimental: the arguments passed after '-r' + outFile, ## experimental: the output file + outDir, ## the output directory + nimcacheDir, ## the location of the 'nimcache' directory + projectName, ## the project's name that is being compiled + projectPath, ## experimental: some path to the project that is being compiled + projectFull, ## the full path to the project that is being compiled + command, ## experimental: the command (e.g. 'c', 'cpp', 'doc') passed to + ## the Nim compiler + commandLine, ## experimental: the command line passed to Nim + linkOptions, ## additional options passed to the linker + compileOptions, ## additional options passed to the C/C++ compiler + ccompilerPath ## the path to the C/C++ compiler + backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` + ## and `nim js` would imply backend=js + libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 + gc {.deprecated.} ## gc selected + mm ## memory management selected + + MultipleValueSetting* {.pure.} = enum ## \ + ## settings resulting in a seq of string values + nimblePaths, ## the nimble path(s) + searchPaths, ## the search path for modules + lazyPaths, ## experimental: even more paths + commandArgs, ## the arguments passed to the Nim compiler + cincludes, ## the #include paths passed to the C/C++ compiler + clibs ## libraries passed to the C/C++ compiler + +proc querySetting*(setting: SingleValueSetting): string {. + compileTime, noSideEffect.} = + ## Can be used to get a string compile-time option. + ## + ## See also: + ## * `compileOption <system.html#compileOption,string>`_ for `on|off` options + ## * `compileOption <system.html#compileOption,string,string>`_ for enum options + ## + runnableExamples: + const nimcache = querySetting(SingleValueSetting.nimcacheDir) + +proc querySettingSeq*(setting: MultipleValueSetting): seq[string] {. + compileTime, noSideEffect.} = + ## Can be used to get a multi-string compile-time option. + ## + ## See also: + ## * `compileOption <system.html#compileOption,string>`_ for `on|off` options + ## * `compileOption <system.html#compileOption,string,string>`_ for enum options + runnableExamples: + const nimblePaths = querySettingSeq(MultipleValueSetting.nimblePaths) diff --git a/lib/std/decls.nim b/lib/std/decls.nim new file mode 100644 index 000000000..bb7ec3593 --- /dev/null +++ b/lib/std/decls.nim @@ -0,0 +1,31 @@ +## This module implements syntax sugar for some declarations. + +import std/macros + +macro byaddr*(sect) = + ## Allows a syntax for l-value references, being an exact analog to + ## `auto& a = ex;` in C++. + ## + ## .. warning:: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. + runnableExamples: + var s = @[10, 11, 12] + var a {.byaddr.} = s[0] + a += 100 + assert s == @[110, 11, 12] + assert a is int + var b {.byaddr.}: int = s[0] + assert a.addr == b.addr + expectLen sect, 1 + let def = sect[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = addr(`ex`) + template `lhs`: untyped = tmp[] + result.copyLineInfo(def) diff --git a/lib/std/dirs.nim b/lib/std/dirs.nim new file mode 100644 index 000000000..380d6d08f --- /dev/null +++ b/lib/std/dirs.nim @@ -0,0 +1,135 @@ +## This module implements directory handling. + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir, + moveDir, walkDir, setCurrentDir, + walkDirRec, PathComponent + +export PathComponent + +proc dirExists*(dir: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + result = dirExists(dir.string) + +proc createDir*(dir: Path) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `moveDir proc`_ + createDir(dir.string) + +proc existsOrCreateDir*(dir: Path): bool {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + result = existsOrCreateDir(dir.string) + +proc removeDir*(dir: Path, checkDir = false + ) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `removeFile proc <files.html#removeFile>`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + removeDir(dir.string, checkDir) + +proc moveDir*(source, dest: Path) {.inline, tags: [ReadIOEffect, WriteIOEffect].} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc <files.html#moveFile>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + moveDir(source.string, dest.string) + +iterator walkDir*(dir: Path; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + for (k, p) in walkDir(dir.string, relative, checkDir, skipSpecial): + yield (k, Path(p)) + +iterator walkDirRec*(dir: Path, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + Path {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkDir iterator`_ + for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, + checkDir, skipSpecial): + yield Path(p) + +proc setCurrentDir*(newDir: Path) {.inline, tags: [].} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getCurrentDir proc <paths.html#getCurrentDir>`_ + osdirs.setCurrentDir(newDir.string) diff --git a/lib/std/editdistance.nim b/lib/std/editdistance.nim new file mode 100644 index 000000000..40c0017ae --- /dev/null +++ b/lib/std/editdistance.nim @@ -0,0 +1,266 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements an algorithm to compute the +## `edit distance`:idx: between two Unicode strings. + +import std/unicode + +proc editDistance*(a, b: string): int {.noSideEffect.} = + ## Returns the **unicode-rune** edit distance between `a` and `b`. + ## + ## This uses the `Levenshtein`:idx: distance algorithm with only a linear + ## memory overhead. + runnableExamples: static: doAssert editdistance("Kitten", "Bitten") == 1 + if runeLen(a) > runeLen(b): + # make `b` the longer string + return editDistance(b, a) + # strip common prefix + var + iStart = 0 ## The character starting index of the first rune in both strings `a` and `b` + iNextA = 0 + iNextB = 0 + runeA, runeB: Rune + lenRunesA = 0 ## The number of relevant runes in string `a`. + lenRunesB = 0 ## The number of relevant runes in string `b`. + block commonPrefix: + # `a` is the shorter string + while iStart < len(a): + iNextA = iStart + a.fastRuneAt(iNextA, runeA, doInc = true) + iNextB = iStart + b.fastRuneAt(iNextB, runeB, doInc = true) + if runeA != runeB: + inc(lenRunesA) + inc(lenRunesB) + break + iStart = iNextA + var + # we know that we are either at the start of the strings + # or that the current value of runeA is not equal to runeB + # => start search for common suffix after the current rune (`i_next_*`) + iEndA = iNextA ## The exclusive upper index bound of string `a`. + iEndB = iNextB ## The exclusive upper index bound of string `b`. + iCurrentA = iNextA + iCurrentB = iNextB + block commonSuffix: + var + addRunesA = 0 + addRunesB = 0 + while iCurrentA < len(a) and iCurrentB < len(b): + iNextA = iCurrentA + a.fastRuneAt(iNextA, runeA) + iNextB = iCurrentB + b.fastRuneAt(iNextB, runeB) + inc(addRunesA) + inc(addRunesB) + if runeA != runeB: + iEndA = iNextA + iEndB = iNextB + inc(lenRunesA, addRunesA) + inc(lenRunesB, addRunesB) + addRunesA = 0 + addRunesB = 0 + iCurrentA = iNextA + iCurrentB = iNextB + if iCurrentA >= len(a): # `a` exhausted + if iCurrentB < len(b): # `b` not exhausted + iEndA = iCurrentA + iEndB = iCurrentB + inc(lenRunesA, addRunesA) + inc(lenRunesB, addRunesB) + while true: + b.fastRuneAt(iEndB, runeB) + inc(lenRunesB) + if iEndB >= len(b): break + elif iCurrentB >= len(b): # `b` exhausted and `a` not exhausted + iEndA = iCurrentA + iEndB = iCurrentB + inc(lenRunesA, addRunesA) + inc(lenRunesB, addRunesB) + while true: + a.fastRuneAt(iEndA, runeA) + inc(lenRunesA) + if iEndA >= len(a): break + block specialCases: + # trivial cases: + if lenRunesA == 0: return lenRunesB + if lenRunesB == 0: return lenRunesA + # another special case: + if lenRunesA == 1: + a.fastRuneAt(iStart, runeA, doInc = false) + var iCurrentB = iStart + while iCurrentB < iEndB: + b.fastRuneAt(iCurrentB, runeB, doInc = true) + if runeA == runeB: return lenRunesB - 1 + return lenRunesB + # common case: + var + len1 = lenRunesA + 1 + len2 = lenRunesB + 1 + row: seq[int] + let half = lenRunesA div 2 + newSeq(row, len2) + var e = iStart + len2 - 1 # end marker + # initialize first row: + for i in 1 .. (len2 - half - 1): row[i] = i + row[0] = len1 - half - 1 + iCurrentA = iStart + var + char2pI = -1 + char2pPrev: int + for i in 1 .. (len1 - 1): + iNextA = iCurrentA + a.fastRuneAt(iNextA, runeA) + var + char2p: int + diff, x: int + p: int + if i >= (len1 - half): + # skip the upper triangle: + let offset = i + half - len1 + if char2pI == i: + b.fastRuneAt(char2pPrev, runeB) + char2p = char2pPrev + char2pI = i + 1 + else: + char2p = iStart + for j in 0 ..< offset: + runeB = b.runeAt(char2p) + inc(char2p, runeB.size) + char2pI = i + 1 + char2pPrev = char2p + p = offset + runeB = b.runeAt(char2p) + var c3 = row[p] + (if runeA != runeB: 1 else: 0) + inc(char2p, runeB.size) + inc(p) + x = row[p] + 1 + diff = x + if x > c3: x = c3 + row[p] = x + inc(p) + else: + p = 1 + char2p = iStart + diff = i + x = i + if i <= (half + 1): + # skip the lower triangle: + e = len2 + i - half - 2 + # main: + while p <= e: + dec(diff) + runeB = b.runeAt(char2p) + var c3 = diff + (if runeA != runeB: 1 else: 0) + inc(char2p, runeB.size) + inc(x) + if x > c3: x = c3 + diff = row[p] + 1 + if x > diff: x = diff + row[p] = x + inc(p) + # lower triangle sentinel: + if i <= half: + dec(diff) + runeB = b.runeAt(char2p) + var c3 = diff + (if runeA != runeB: 1 else: 0) + inc(x) + if x > c3: x = c3 + row[p] = x + iCurrentA = iNextA + result = row[e] + +proc editDistanceAscii*(a, b: string): int {.noSideEffect.} = + ## Returns the edit distance between `a` and `b`. + ## + ## This uses the `Levenshtein`:idx: distance algorithm with only a linear + ## memory overhead. + runnableExamples: static: doAssert editDistanceAscii("Kitten", "Bitten") == 1 + var len1 = a.len + var len2 = b.len + if len1 > len2: + # make `b` the longer string + return editDistanceAscii(b, a) + + # strip common prefix: + var s = 0 + while s < len1 and a[s] == b[s]: + inc(s) + dec(len1) + dec(len2) + # strip common suffix: + while len1 > 0 and len2 > 0 and a[s+len1-1] == b[s+len2-1]: + dec(len1) + dec(len2) + # trivial cases: + if len1 == 0: return len2 + if len2 == 0: return len1 + + # another special case: + if len1 == 1: + for j in s..s+len2-1: + if a[s] == b[j]: return len2 - 1 + return len2 + + inc(len1) + inc(len2) + var half = len1 shr 1 + # initialize first row: + #var row = cast[ptr array[0..high(int) div 8, int]](alloc(len2*sizeof(int))) + var row: seq[int] + newSeq(row, len2) + var e = s + len2 - 1 # end marker + for i in 1..len2 - half - 1: row[i] = i + row[0] = len1 - half - 1 + for i in 1 .. len1 - 1: + var char1 = a[i + s - 1] + var char2p: int + var diff, x: int + var p: int + if i >= len1 - half: + # skip the upper triangle: + var offset = i - len1 + half + char2p = offset + p = offset + var c3 = row[p] + ord(char1 != b[s + char2p]) + inc(p) + inc(char2p) + x = row[p] + 1 + diff = x + if x > c3: x = c3 + row[p] = x + inc(p) + else: + p = 1 + char2p = 0 + diff = i + x = i + if i <= half + 1: + # skip the lower triangle: + e = len2 + i - half - 2 + # main: + while p <= e: + dec(diff) + var c3 = diff + ord(char1 != b[char2p + s]) + inc(char2p) + inc(x) + if x > c3: x = c3 + diff = row[p] + 1 + if x > diff: x = diff + row[p] = x + inc(p) + # lower triangle sentinel: + if i <= half: + dec(diff) + var c3 = diff + ord(char1 != b[char2p + s]) + inc(x) + if x > c3: x = c3 + row[p] = x + result = row[e] diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim new file mode 100644 index 000000000..3d1b4ffd3 --- /dev/null +++ b/lib/std/effecttraits.nim @@ -0,0 +1,63 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides access to the inferred .raises effects +## for Nim's macro system. +## **Since**: Version 1.4. +## +## One can test for the existence of this standard module +## via `defined(nimHasEffectTraitsModule)`. + +import std/macros + +proc getRaisesListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc getTagsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc getForbidsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc isGcSafeImpl(n: NimNode): bool = discard "see compiler/vmops.nim" +proc hasNoSideEffectsImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc getRaisesList*(fn: NimNode): NimNode = + ## Extracts the `.raises` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getRaisesListImpl(fn) + +proc getTagsList*(fn: NimNode): NimNode = + ## Extracts the `.tags` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getTagsListImpl(fn) + +proc getForbidsList*(fn: NimNode): NimNode = + ## Extracts the `.forbids` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getForbidsListImpl(fn) + +proc isGcSafe*(fn: NimNode): bool = + ## Return true if the func/proc/etc `fn` is `gcsafe`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = isGcSafeImpl(fn) + +proc hasNoSideEffects*(fn: NimNode): bool = + ## Return true if the func/proc/etc `fn` has `noSideEffect`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasNoSideEffectsImpl(fn) diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim new file mode 100644 index 000000000..beb65ed30 --- /dev/null +++ b/lib/std/enumerate.nim @@ -0,0 +1,70 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements `enumerate` syntactic sugar based on Nim's +## macro system. + +import std/private/since +import std/macros + + +macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = + ## Enumerating iterator for collections. + ## + ## It yields `(count, value)` tuples (which must be immediately unpacked). + ## The default starting count `0` can be manually overridden if needed. + runnableExamples: + let a = [10, 20, 30] + var b: seq[(int, int)] = @[] + for i, x in enumerate(a): + b.add((i, x)) + assert b == @[(0, 10), (1, 20), (2, 30)] + + let c = "abcd" + var d: seq[(int, char)] + for (i, x) in enumerate(97, c): + d.add((i, x)) + assert d == @[(97, 'a'), (98, 'b'), (99, 'c'), (100, 'd')] + + template genCounter(x): untyped = + # We strip off the first for loop variable and use it as an integer counter. + # We must immediately decrement it by one, because it gets incremented before + # the loop body - to be able to use the final expression in other macros. + newVarStmt(x, infix(countStart, "-", newLit(1))) + + template genInc(x): untyped = + newCall(bindSym"inc", x) + + expectKind x, nnkForStmt + # check if the starting count is specified: + var countStart = if x[^2].len == 2: newLit(0) else: x[^2][1] + result = newStmtList() + var body = x[^1] + if body.kind != nnkStmtList: + body = newTree(nnkStmtList, body) + var newFor = newTree(nnkForStmt) + if x.len == 3: # single iteration variable + if x[0].kind == nnkVarTuple: # for (x, y, ...) in iter + result.add genCounter(x[0][0]) + body.insert(0, genInc(x[0][0])) + for i in 1 .. x[0].len-2: + newFor.add x[0][i] + else: + error("Missing second for loop variable") # for x in iter + else: # for x, y, ... in iter + result.add genCounter(x[0]) + body.insert(0, genInc(x[0])) + for i in 1 .. x.len-3: + newFor.add x[i] + # transform enumerate(X) to 'X' + newFor.add x[^2][^1] + newFor.add body + result.add newFor + # now wrap the whole macro in a block to create a new scope + result = newBlockStmt(result) diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim new file mode 100644 index 000000000..9c338817d --- /dev/null +++ b/lib/std/enumutils.nim @@ -0,0 +1,202 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import std/macros +from std/typetraits import OrdinalEnum, HoleyEnum + +when defined(nimPreviewSlimSystem): + import std/assertions + + +# xxx `genEnumCaseStmt` needs tests and runnableExamples + +macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, + userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = + # Generates a case stmt, which assigns the correct enum field given + # a normalized string comparison to the `argSym` input. + # string normalization is done using passed normalizer. + let typ = typ.getTypeInst[1] + let typSym = typ.getTypeImpl.getTypeInst # skip aliases etc to get type sym + let impl = typSym.getImpl[2] + expectKind impl, nnkEnumTy + let normalizerNode = quote: `normalizer` + expectKind normalizerNode, nnkSym + result = nnkCaseStmt.newTree(newCall(normalizerNode, argSym)) + # stores all processed field strings to give error msg for ambiguous enums + var foundFields: seq[string] = @[] + var fVal = "" + var fStr = "" # string of current field + var fNum = BiggestInt(0) # int value of current field + for f in impl: + case f.kind + of nnkEmpty: continue # skip first node of `enumTy` + of nnkSym, nnkIdent: + fVal = f.strVal + fStr = fVal + of nnkAccQuoted: + fVal = "" + for ch in f: + fVal.add ch.strVal + fStr = fVal + of nnkEnumFieldDef: + fVal = f[0].strVal + case f[1].kind + of nnkStrLit: + fStr = f[1].strVal + of nnkTupleConstr: + fStr = f[1][1].strVal + fNum = f[1][0].intVal + of nnkIntLit: + fStr = f[0].strVal + fNum = f[1].intVal + else: + let fAst = f[0].getImpl + if fAst.kind == nnkStrLit: + fStr = fAst.strVal + else: + error("Invalid tuple syntax!", f[1]) + else: error("Invalid node for enum type `" & $f.kind & "`!", f) + # add field if string not already added + if fNum >= userMin and fNum <= userMax: + fStr = normalizer(fStr) + if fStr notin foundFields: + result.add nnkOfBranch.newTree(newLit fStr, newDotExpr(typ, ident fVal)) + foundFields.add fStr + else: + error("Ambiguous enums cannot be parsed, field " & $fStr & + " appears multiple times!", f) + inc fNum + # finally add else branch to raise or use default + if default == nil: + let raiseStmt = quote do: + raise newException(ValueError, "Invalid enum value: " & $`argSym`) + result.add nnkElse.newTree(raiseStmt) + else: + expectKind(default, nnkSym) + result.add nnkElse.newTree(default) + +macro enumFullRange(a: typed): untyped = + newNimNode(nnkBracket).add(a.getType[1][1..^1]) + +macro enumNames(a: typed): untyped = + # this could be exported too; in particular this could be useful for enum with holes. + result = newNimNode(nnkBracket) + for ai in a.getType[1][1..^1]: + assert ai.kind == nnkSym + result.add newLit ai.strVal + +iterator items*[T: HoleyEnum](E: typedesc[T]): T = + ## Iterates over an enum with holes. + runnableExamples: + type + A = enum + a0 = 2 + a1 = 4 + a2 + B[T] = enum + b0 = 2 + b1 = 4 + from std/sequtils import toSeq + assert A.toSeq == [a0, a1, a2] + assert B[float].toSeq == [B[float].b0, B[float].b1] + for a in enumFullRange(E): yield a + +func span(T: typedesc[HoleyEnum]): int = + (T.high.ord - T.low.ord) + 1 + +const invalidSlot = uint8.high + +proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = + const n = span(T) + var i = 0 + assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] + for ai in mitems(ret): ai = invalidSlot + for ai in items(T): + ret[ai.ord - T.low.ord] = uint8(i) + inc(i) + return ret + +func symbolRankImpl[T](a: T): int {.inline.} = + const n = T.span + const thres = 255 # must be <= `invalidSlot`, but this should be tuned. + when n <= thres: + const lookup = genLookup(T) + let lookup2 {.global.} = lookup # xxx improve pending https://github.com/timotheecour/Nim/issues/553 + #[ + This could be optimized using a hash adapted to `T` (possible since it's known at CT) + to get better key distribution before indexing into the lookup table table. + ]# + {.noSideEffect.}: # because it's immutable + let ret = lookup2[ord(a) - T.low.ord] + if ret != invalidSlot: return ret.int + else: + var i = 0 + # we could also generate a case statement as optimization + for ai in items(T): + if ai == a: return i + inc(i) + raise newException(IndexDefect, $ord(a) & " invalid for " & $T) + +template symbolRank*[T: enum](a: T): int = + ## Returns the index in which `a` is listed in `T`. + ## + ## The cost for a `HoleyEnum` is implementation defined, currently optimized + ## for small enums, otherwise is `O(T.enumLen)`. + runnableExamples: + type + A = enum # HoleyEnum + a0 = -3 + a1 = 10 + a2 + a3 = (20, "f3Alt") + B = enum # OrdinalEnum + b0 + b1 + b2 + C = enum # OrdinalEnum + c0 = 10 + c1 + c2 + assert a2.symbolRank == 2 + assert b2.symbolRank == 2 + assert c2.symbolRank == 2 + assert c2.ord == 12 + assert a2.ord == 11 + var invalid = 7.A + doAssertRaises(IndexDefect): discard invalid.symbolRank + when T is Ordinal: ord(a) - T.low.ord.static + else: symbolRankImpl(a) + +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + +func symbolName*[T: enum](a: T): string = + ## Returns the symbol name of an enum. + ## + ## This uses `symbolRank`. + runnableExamples: + type B = enum + b0 = (10, "kb0") + b1 = "kb1" + b2 + let b = B.low + assert b.symbolName == "b0" + assert $b == "kb0" + static: assert B.high.symbolName == "b2" + type C = enum # HoleyEnum + c0 = -3 + c1 = 4 + c2 = 20 + assert c1.symbolName == "c1" + when T is range: + const names = enumNames(rangeBase T) + else: + const names = enumNames(T) + names[a.symbolRank] diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim new file mode 100644 index 000000000..a955077ea --- /dev/null +++ b/lib/std/envvars.nim @@ -0,0 +1,221 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/envvars` module implements environment variable handling. +import std/oserrors + +type + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. + + +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + var num: int + var keys: RootObj + {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} + for i in 0..<num: + var key, value: cstring + {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".} + yield ($key, $value) + + # commented because it must keep working with js+VM + # elif defined(js): + # {.error: "requires -d:nodejs".} + + else: + + when defined(windows): + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".} + from std/private/win_setenv import setEnvImpl + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t", header: "<stdlib.h>".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", + header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) + else: + proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "<stdlib.h>".} + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): cstring = c_getenv(env) + + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc`_. + ## + ## See also: + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + + let env = getEnvImpl(key) + if env == nil: + result = default + else: + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + result = getEnvImpl(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + else: + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "<string.h>".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "<string.h>".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # <crt_externs.h>, can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "<crt_externs.h>".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim new file mode 100644 index 000000000..f26368f42 --- /dev/null +++ b/lib/std/exitprocs.nim @@ -0,0 +1,87 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows adding hooks to program exit. + +import std/locks +when defined(js) and not defined(nodejs): + import std/assertions + +type + FunKind = enum kClosure, kNoconv # extend as needed + Fun = object + case kind: FunKind + of kClosure: fun1: proc () {.closure.} + of kNoconv: fun2: proc () {.noconv.} + +var + gFunsLock: Lock + gFuns {.cursor.}: seq[Fun] #Intentionally use the cursor to break up the lifetime trace and make it compatible with JS. + +initLock(gFunsLock) + +when defined(js): + proc addAtExit(quitProc: proc() {.noconv.}) = + when defined(nodejs): + {.emit: """ + process.on('exit', `quitProc`); + """.} + elif defined(js): + {.emit: """ + window.onbeforeunload = `quitProc`; + """.} +else: + proc addAtExit(quitProc: proc() {.noconv.}) {. + importc: "atexit", header: "<stdlib.h>".} + +proc callClosures() {.noconv.} = + withLock gFunsLock: + for i in countdown(gFuns.len-1, 0): + let fun = gFuns[i] + case fun.kind + of kClosure: fun.fun1() + of kNoconv: fun.fun2() + gFuns.setLen(0) + +template fun() = + if gFuns.len == 0: + addAtExit(callClosures) + +proc addExitProc*(cl: proc () {.closure.}) = + ## Adds/registers a quit procedure. Each call to `addExitProc` registers + ## another quit procedure. They are executed on a last-in, first-out basis. + # Support for `addExitProc` is done by Ansi C's facilities here. + # In case of an unhandled exception the exit handlers should + # not be called explicitly! The user may decide to do this manually though. + withLock gFunsLock: + fun() + gFuns.add Fun(kind: kClosure, fun1: cl) + +proc addExitProc*(cl: proc() {.noconv.}) = + ## overload for `noconv` procs. + withLock gFunsLock: + fun() + gFuns.add Fun(kind: kNoconv, fun2: cl) + +when not defined(nimscript) and (not defined(js) or defined(nodejs)): + proc getProgramResult*(): int = + when defined(js) and defined(nodejs): + {.emit: """ +`result` = process.exitCode; +""".} + else: + result = programResult + + proc setProgramResult*(a: int) = + when defined(js) and defined(nodejs): + {.emit: """ +process.exitCode = `a`; +""".} + else: + programResult = a diff --git a/lib/std/files.nim b/lib/std/files.nim new file mode 100644 index 000000000..c4e0491c9 --- /dev/null +++ b/lib/std/files.nim @@ -0,0 +1,46 @@ +## This module implements file handling. +## +## **See also:** +## * `paths module <paths.html>`_ for path manipulation + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osfiles import fileExists, removeFile, + moveFile + + +proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + result = fileExists(filename.string) + +proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc <dirs.html#removeDir>`_ + ## * `moveFile proc`_ + removeFile(file.string) + +proc moveFile*(source, dest: Path) {.inline, + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc <dirs.html#moveDir>`_ + ## * `removeFile proc`_ + moveFile(source.string, dest.string) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim new file mode 100644 index 000000000..9258245f6 --- /dev/null +++ b/lib/std/formatfloat.nim @@ -0,0 +1,143 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements formatting floats as strings. + +when defined(nimPreviewSlimSystem): + import std/assertions + +proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "<string.h>", discardable.} + +proc addCstringN(result: var string, buf: cstring; buflen: int) = + # no nimvm support needed, so it doesn't need to be fast here either + let oldLen = result.len + let newLen = oldLen + buflen + result.setLen newLen + c_memcpy(result[oldLen].addr, buf, buflen.csize_t) + +import std/private/[dragonbox, schubfach] + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + result = toChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int = + result = float32ToChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs, noSideEffect.} + +proc writeToBuffer(buf: var array[65, char]; value: cstring) = + var i = 0 + while value[i] != '\0': + buf[i] = value[i] + inc i + +proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + var n = c_snprintf(cast[cstring](addr buf), 65, "%.16g", value).int + var hasDot = false + for i in 0..n-1: + if buf[i] == ',': + buf[i] = '.' + hasDot = true + elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: + hasDot = true + if not hasDot: + buf[n] = '.' + buf[n+1] = '0' + buf[n+2] = '\0' + result = n + 2 + else: + result = n + # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' + # of '-1.#IND' are produced. + # We want to get rid of these here: + if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: + writeToBuffer(buf, "nan") + result = 3 + elif buf[n-1] == 'F': + if buf[0] == '-': + writeToBuffer(buf, "-inf") + result = 4 + else: + writeToBuffer(buf, "inf") + result = 3 + +proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + writeFloatToBufferRoundtrip(buf, value) + else: + writeFloatToBufferSprintf(buf, value) + +proc addFloatRoundtrip*(result: var string; x: float | float32) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferRoundtrip(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +proc addFloatSprintf*(result: var string; x: float) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferSprintf(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +when defined(js): + proc nimFloatToString(a: float): cstring = + ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 + # print `-0.0` properly + {.emit: """ + function nimOnlyDigitsOrMinus(n) { + return n.toString().match(/^-?\d+$/); + } + if (Number.isSafeInteger(`a`)) + `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"; + else { + `result` = `a`+""; + if(nimOnlyDigitsOrMinus(`result`)){ + `result` = `a`+".0"; + } + } + """.} + +proc addFloat*(result: var string; x: float | float32) {.inline.} = + ## Converts float to its string representation and appends it to `result`. + runnableExamples: + var + s = "foo:" + b = 45.67 + s.addFloat(45.67) + assert s == "foo:45.67" + template impl = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + addFloatRoundtrip(result, x) + else: + addFloatSprintf(result, x) + when defined(js): + when nimvm: impl() + else: + result.add nimFloatToString(x) + else: impl() + +when defined(nimPreviewSlimSystem): + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim new file mode 100644 index 000000000..d0f07c527 --- /dev/null +++ b/lib/std/genasts.nim @@ -0,0 +1,89 @@ +## This module implements AST generation using captured variables for macros. + +import std/macros + +type GenAstOpt* = enum + kDirtyTemplate, + # When set, uses a dirty template in implementation of `genAst`. This + # is occasionally useful as workaround for issues such as #8220, see + # `strformat limitations <strformat.html#limitations>`_ for details. + # Default is unset, to avoid hijacking of uncaptured local symbols by + # symbols in caller scope. + kNoNewLit, + # don't call call newLit automatically in `genAst` capture parameters + +macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped = + ## Accepts a list of captured variables `a=b` or `a` and a block and returns the + ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured + ## unless `kDirtyTemplate in options`. + runnableExamples: + # This example shows how one could write a simplified version of `unittest.check`. + import std/[macros, strutils] + macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + raiseAssert "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] + let a = 3 + check2 a*2 == a+3 + if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' + + runnableExamples: + # This example goes in more details about the capture semantics. + macro fun(a: string, b: static bool): untyped = + let c = 'z' + var d = 11 # implicitly {.gensym.} and needs to be captured for use in `genAst`. + proc localFun(): auto = 12 # implicitly {.inject.}, doesn't need to be captured. + genAst(a, b, c = true): + # `a`, `b` are captured explicitly, `c` is a local definition masking `c = 'z'`. + const b2 = b # macro static param `b` is forwarded here as a static param. + # `echo d` would give: `var not init` because `d` is not captured. + (a & a, b, c, localFun()) # localFun can be called without capture. + assert fun("ab", false) == ("abab", false, true, 12) + + let params = newTree(nnkFormalParams, newEmptyNode()) + let pragmas = + if kDirtyTemplate in options: + nnkPragma.newTree(ident"dirty") + else: + newEmptyNode() + + template newLitMaybe(a): untyped = + when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)): + a # `proc` actually also covers template, macro + else: newLit(a) + + # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669 + let name = genSym(nskTemplate, "_fun") + let call = newCall(name) + for a in args[0..^2]: + var varName: NimNode + var varVal: NimNode + case a.kind + of nnkExprEqExpr: + varName = a[0] + varVal = a[1] + of nnkIdent: + varName = a + varVal = a + else: error("invalid argument kind: " & $a.kind, a) + if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal) + + params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode()) + call.add varVal + + result = newStmtList() + result.add nnkTemplateDef.newTree( + name, + newEmptyNode(), + newEmptyNode(), + params, + pragmas, + newEmptyNode(), + args[^1]) + result.add newCall(bindSym"getAst", call) + +template genAst*(args: varargs[untyped]): untyped = + ## Convenience wrapper around `genAstOpt`. + genAstOpt({}, args) diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim new file mode 100644 index 000000000..d2da76ea8 --- /dev/null +++ b/lib/std/importutils.nim @@ -0,0 +1,44 @@ +##[ +Utilities related to import and symbol resolution. + +Experimental API, subject to change. +]## + +#[ +Possible future APIs: +* module symbols (https://github.com/nim-lang/Nim/pull/9560) +* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376) +* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530) +* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527) +]# + +when defined(nimImportutilsExample): + type + Foo = object + f0: int # private + Goo*[T] = object + g0: int # private + proc initFoo*(): auto = Foo() + +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = + ## Enables access to private fields of `t` in current scope. + runnableExamples("-d:nimImportutilsExample"): + # here we're importing a module containing: + # type + # Foo = object + # f0: int # private + # Goo*[T] = object + # g0: int # private + # proc initFoo*(): auto = Foo() + var f = initFoo() + block: + assert not compiles(f.f0) + privateAccess(f.type) + f.f0 = 1 # accessible in this scope + block: + assert f.f0 == 1 # still in scope + assert not compiles(f.f0) + + # this also works with generics + privateAccess(Goo) + assert Goo[float](g0: 1).g0 == 1 diff --git a/lib/std/isolation.nim b/lib/std/isolation.nim new file mode 100644 index 000000000..b03e00651 --- /dev/null +++ b/lib/std/isolation.nim @@ -0,0 +1,49 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the `Isolated[T]` type for +## safe construction of isolated subgraphs that can be +## passed efficiently to different channels and threads. +## +## .. warning:: This module is experimental and its interface may change. +## + +type + Isolated*[T] {.sendable.} = object ## Isolated data can only be moved, not copied. + value: T + +proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} + +proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} = + # delegate to value's sink operation + `=sink`(dest.value, src.value) + +proc `=destroy`*[T](dest: var Isolated[T]) {.inline.} = + # delegate to value's destroy operation + `=destroy`(dest.value) + +func isolate*[T](value: sink T): Isolated[T] {.magic: "Isolate".} = + ## Creates an isolated subgraph from the expression `value`. + ## Isolation is checked at compile time. + ## + ## Please read https://github.com/nim-lang/RFCs/issues/244 + ## for more details. + Isolated[T](value: value) + +func unsafeIsolate*[T](value: sink T): Isolated[T] = + ## Creates an isolated subgraph from the expression `value`. + ## + ## .. warning:: The proc doesn't check whether `value` is isolated. + ## + Isolated[T](value: value) + +func extract*[T](src: var Isolated[T]): T = + ## Returns the internal value of `src`. + ## The value is moved from `src`. + result = move(src.value) diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim new file mode 100644 index 000000000..4e996ea7b --- /dev/null +++ b/lib/std/jsbigints.nim @@ -0,0 +1,228 @@ +## Arbitrary precision integers. +## * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt +when not defined(js): + {.fatal: "Module jsbigints is designed to be used with the JavaScript backend.".} + +when defined(nimPreviewSlimSystem): + import std/assertions + +type JsBigIntImpl {.importjs: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 +type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. + +func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = + ## Constructor for `JsBigInt`. + runnableExamples: + doAssert big(1234567890) == big"1234567890" + doAssert 0b1111100111.big == 0o1747.big and 0o1747.big == 999.big + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Constructor for `JsBigInt`. + runnableExamples: + doAssert -1'big == 1'big - 2'big + # supports decimal, binary, octal, hex: + doAssert -12'big == big"-12" + doAssert 12'big == 12.big + doAssert 0b101'big == 0b101.big + doAssert 0o701'big == 0o701.big + doAssert 0xdeadbeaf'big == 0xdeadbeaf.big + doAssert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big + doAssert not compiles(static(12'big)) + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Alias for `'big` + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func toCstring*(this: JsBigInt; radix: 2..36): cstring {.importjs: "#.toString(#)".} = + ## Converts from `JsBigInt` to `cstring` representation. + ## * `radix` Base to use for representing numeric values. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/toString + runnableExamples: + doAssert big"2147483647".toCstring(2) == "1111111111111111111111111111111".cstring + +func toCstring*(this: JsBigInt): cstring {.importjs: "#.toString()".} + ## Converts from `JsBigInt` to `cstring` representation. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/toString + +func `$`*(this: JsBigInt): string = + ## Returns a `string` representation of `JsBigInt`. + runnableExamples: doAssert $big"1024" == "1024n" + $toCstring(this) & 'n' + +func wrapToInt*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: + "(() => { const i = #, b = #; return BigInt.asIntN(b, i) })()".} = + ## Wraps `this` to a signed `JsBigInt` of `bits` bits in `-2 ^ (bits - 1)` .. `2 ^ (bits - 1) - 1`. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/asIntN + runnableExamples: + doAssert (big("3") + big("2") ** big("66")).wrapToInt(13) == big("3") + +func wrapToUint*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: + "(() => { const i = #, b = #; return BigInt.asUintN(b, i) })()".} = + ## Wraps `this` to an unsigned `JsBigInt` of `bits` bits in 0 .. `2 ^ bits - 1`. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/asUintN + runnableExamples: + doAssert (big("3") + big("2") ** big("66")).wrapToUint(66) == big("3") + +func toNumber*(this: JsBigInt): int {.importjs: "Number(#)".} = + ## Does not do any bounds check and may or may not return an inexact representation. + runnableExamples: + doAssert toNumber(big"2147483647") == 2147483647.int + +func `+`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert (big"9" + big"1") == big"10" + +func `-`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert (big"9" - big"1") == big"8" + +func `*`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert (big"42" * big"9") == big"378" + +func `div`*(x, y: JsBigInt): JsBigInt {.importjs: "(# / #)".} = + ## Same as `div` but for `JsBigInt`(uses JavaScript `BigInt() / BigInt()`). + runnableExamples: + doAssert big"13" div big"3" == big"4" + doAssert big"-13" div big"3" == big"-4" + doAssert big"13" div big"-3" == big"-4" + doAssert big"-13" div big"-3" == big"4" + +func `mod`*(x, y: JsBigInt): JsBigInt {.importjs: "(# % #)".} = + ## Same as `mod` but for `JsBigInt` (uses JavaScript `BigInt() % BigInt()`). + runnableExamples: + doAssert big"13" mod big"3" == big"1" + doAssert big"-13" mod big"3" == big"-1" + doAssert big"13" mod big"-3" == big"1" + doAssert big"-13" mod big"-3" == big"-1" + +func `<`*(x, y: JsBigInt): bool {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert big"2" < big"9" + +func `<=`*(x, y: JsBigInt): bool {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert big"1" <= big"5" + +func `==`*(x, y: JsBigInt): bool {.importjs: "(# == #)".} = + runnableExamples: + doAssert big"42" == big"42" + +func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = + # (#) needed due to unary minus + runnableExamples: + doAssert big"2" ** big"64" == big"18446744073709551616" + doAssert big"-2" ** big"3" == big"-8" + doAssert -big"2" ** big"2" == big"4" # parsed as: (-2n) ** 2n + doAssert big"0" ** big"0" == big"1" # edge case + var ok = false + try: discard big"2" ** big"-1" # raises foreign `RangeError` + except: ok = true + doAssert ok + +func `and`*(x, y: JsBigInt): JsBigInt {.importjs: "(# & #)".} = + runnableExamples: + doAssert (big"555" and big"2") == big"2" + +func `or`*(x, y: JsBigInt): JsBigInt {.importjs: "(# | #)".} = + runnableExamples: + doAssert (big"555" or big"2") == big"555" + +func `xor`*(x, y: JsBigInt): JsBigInt {.importjs: "(# ^ #)".} = + runnableExamples: + doAssert (big"555" xor big"2") == big"553" + +func `shl`*(a, b: JsBigInt): JsBigInt {.importjs: "(# << #)".} = + runnableExamples: + doAssert (big"999" shl big"2") == big"3996" + +func `shr`*(a, b: JsBigInt): JsBigInt {.importjs: "(# >> #)".} = + runnableExamples: + doAssert (big"999" shr big"2") == big"249" + +func `-`*(this: JsBigInt): JsBigInt {.importjs: "($1#)".} = + runnableExamples: + doAssert -(big"10101010101") == big"-10101010101" + +func inc*(this: var JsBigInt) {.importjs: "(++[#][0][0])".} = + runnableExamples: + var big1: JsBigInt = big"1" + inc big1 + doAssert big1 == big"2" + +func dec*(this: var JsBigInt) {.importjs: "(--[#][0][0])".} = + runnableExamples: + var big1: JsBigInt = big"2" + dec big1 + doAssert big1 == big"1" + +func inc*(this: var JsBigInt; amount: JsBigInt) {.importjs: "([#][0][0] += #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + inc big1, big"2" + doAssert big1 == big"3" + +func dec*(this: var JsBigInt; amount: JsBigInt) {.importjs: "([#][0][0] -= #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + dec big1, big"2" + doAssert big1 == big"-1" + +func `+=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + big1 += big"2" + doAssert big1 == big"3" + +func `-=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + big1 -= big"2" + doAssert big1 == big"-1" + +func `*=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + runnableExamples: + var big1: JsBigInt = big"2" + big1 *= big"4" + doAssert big1 == big"8" + +func `/=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + ## Same as `x = x div y`. + runnableExamples: + var big1: JsBigInt = big"11" + big1 /= big"2" + doAssert big1 == big"5" + +proc `+`*(_: JsBigInt): JsBigInt {.error: + "See https://github.com/tc39/proposal-bigint/blob/master/ADVANCED.md#dont-break-asmjs".} # Can not be used by design + ## **Do NOT use.** https://github.com/tc39/proposal-bigint/blob/master/ADVANCED.md#dont-break-asmjs + +proc low*(_: typedesc[JsBigInt]): JsBigInt {.error: + "Arbitrary precision integers do not have a known low.".} ## **Do NOT use.** + +proc high*(_: typedesc[JsBigInt]): JsBigInt {.error: + "Arbitrary precision integers do not have a known high.".} ## **Do NOT use.** + + +runnableExamples: + block: + let big1: JsBigInt = big"2147483647" + let big2: JsBigInt = big"666" + doAssert JsBigInt isnot int + doAssert big1 != big2 + doAssert big1 > big2 + doAssert big1 >= big2 + doAssert big2 < big1 + doAssert big2 <= big1 + doAssert not(big1 == big2) + let z = JsBigInt.default + doAssert $z == "0n" + block: + var a: seq[JsBigInt] + a.setLen 2 + doAssert a == @[big"0", big"0"] + doAssert a[^1] == big"0" + var b: JsBigInt + doAssert b == big"0" + doAssert b == JsBigInt.default diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim new file mode 100644 index 000000000..219594619 --- /dev/null +++ b/lib/std/jsfetch.nim @@ -0,0 +1,202 @@ +## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API +when not defined(js): + {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} + +import std/[asyncjs, jsformdata, jsheaders] +export jsformdata, jsheaders +from std/httpcore import HttpMethod +from std/jsffi import JsObject + +type + FetchOptions* = ref object of JsRoot ## Options for Fetch API. + keepalive*: bool + metod* {.importjs: "method".}: cstring + body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring + headers*: Headers + + FetchModes* = enum ## Mode options. + fmCors = "cors" + fmNoCors = "no-cors" + fmSameOrigin = "same-origin" + + FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials + fcInclude = "include" + fcSameOrigin = "same-origin" + fcOmit = "omit" + + FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache + fchDefault = "default" + fchNoStore = "no-store" + fchReload = "reload" + fchNoCache = "no-cache" + fchForceCache = "force-cache" + + FetchRedirects* = enum ## Redirects options. + frFollow = "follow" + frError = "error" + frManual = "manual" + + FetchReferrerPolicies* = enum ## Referrer Policy options. + frpNoReferrer = "no-referrer" + frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" + frpOrigin = "origin" + frpOriginWhenCrossOrigin = "origin-when-cross-origin" + frpUnsafeUrl = "unsafe-url" + + Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring + + Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring + +func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} + ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. + +func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} + ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. + +func newRequest*(url: cstring; fetchOptions: FetchOptions): Request {.importjs: "(new Request(#, #))".} + ## Constructor for `Request` with `fetchOptions`. Same as `fetch(url, fetchOptions)`. + +func clone*(self: Response | Request): Response {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone + +proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/text + +proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/json + +proc formData*(self: Response): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/formData + +proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring; headers: Headers = newHeaders()): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} + ## .. warning:: Unsafe `newfetchOptions`. + +func newfetchOptions*(metod = HttpGet; body: cstring = nil; + mode = fmCors; credentials = fcSameOrigin; cache = fchDefault; referrerPolicy = frpNoReferrerWhenDowngrade; + keepalive = false; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, + headers: Headers = newHeaders()): FetchOptions = + ## Constructor for `FetchOptions`. + result = FetchOptions( + body: if metod notin {HttpHead, HttpGet}: body else: nil, + mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, headers: headers, + metod: (case metod + of HttpHead: "HEAD".cstring + of HttpGet: "GET".cstring + of HttpPost: "POST".cstring + of HttpPut: "PUT".cstring + of HttpDelete: "DELETE".cstring + of HttpPatch: "PATCH".cstring + else: "GET".cstring + ) + ) + +proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} + ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. + +proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} + ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. + +func toCstring*(self: Request | Response | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} + +func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) + + +runnableExamples("-r:off"): + import std/[asyncjs, jsconsole, jsformdata, jsheaders] + from std/httpcore import HttpMethod + from std/jsffi import JsObject + from std/sugar import `=>` + + block: + let options0: FetchOptions = unsafeNewFetchOptions( + metod = "POST".cstring, + body = """{"key": "value"}""".cstring, + mode = "no-cors".cstring, + credentials = "omit".cstring, + cache = "no-cache".cstring, + referrerPolicy = "no-referrer".cstring, + keepalive = false, + redirect = "follow".cstring, + referrer = "client".cstring, + integrity = "".cstring, + headers = newHeaders() + ) + assert options0.keepalive == false + assert options0.metod == "POST".cstring + assert options0.body == """{"key": "value"}""".cstring + assert options0.mode == "no-cors".cstring + assert options0.credentials == "omit".cstring + assert options0.cache == "no-cache".cstring + assert options0.referrerPolicy == "no-referrer".cstring + assert options0.redirect == "follow".cstring + assert options0.referrer == "client".cstring + assert options0.integrity == "".cstring + assert options0.headers.len == 0 + + block: + let options1: FetchOptions = newFetchOptions( + metod = HttpPost, + body = """{"key": "value"}""".cstring, + mode = fmNoCors, + credentials = fcOmit, + cache = fchNoCache, + referrerPolicy = frpNoReferrer, + keepalive = false, + redirect = frFollow, + referrer = "client".cstring, + integrity = "".cstring, + headers = newHeaders() + ) + assert options1.keepalive == false + assert options1.metod == $HttpPost + assert options1.body == """{"key": "value"}""".cstring + assert options1.mode == $fmNoCors + assert options1.credentials == $fcOmit + assert options1.cache == $fchNoCache + assert options1.referrerPolicy == $frpNoReferrer + assert options1.redirect == $frFollow + assert options1.referrer == "client".cstring + assert options1.integrity == "".cstring + assert options1.headers.len == 0 + + block: + let response: Response = newResponse(body = "-. .. --".cstring) + let request: Request = newRequest(url = "http://nim-lang.org".cstring) + + if not defined(nodejs): + block: + proc doFetch(): Future[Response] {.async.} = + fetch "https://httpbin.org/get".cstring + + proc example() {.async.} = + let response: Response = await doFetch() + assert response.ok + assert response.status == 200.cint + assert response.headers is Headers + assert response.body is cstring + + discard example() + + block: + proc example2 {.async.} = + await fetch("https://api.github.com/users/torvalds".cstring) + .then((response: Response) => response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) + + discard example2() diff --git a/lib/std/jsformdata.nim b/lib/std/jsformdata.nim new file mode 100644 index 000000000..61dcc39a3 --- /dev/null +++ b/lib/std/jsformdata.nim @@ -0,0 +1,69 @@ +## - `FormData` for the JavaScript target: https://developer.mozilla.org/en-US/docs/Web/API/FormData +when not defined(js): + {.fatal: "Module jsformdata is designed to be used with the JavaScript backend.".} + +from std/dom import Blob + +type FormData* = ref object of JsRoot ## FormData API. + +func newFormData*(): FormData {.importjs: "new FormData()".} + +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.append(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. + +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.append(#, #, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. + +func delete*(self: FormData; name: cstring) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/delete + ## + ## .. warning:: Deletes *all items* with the same key name. + +func getAll*(self: FormData; name: cstring): seq[cstring] {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/getAll + +func hasKey*(self: FormData; name: cstring): bool {.importjs: "#.has(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/has + +func keys*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/keys + +func values*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/values + +func pairs*(self: FormData): seq[tuple[key, val: cstring]] {.importjs: "Array.from(#.entries())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/entries + +func put*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.set(#, #, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set + +func `[]=`*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.set(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set + +func `[]`*(self: FormData; name: cstring): cstring {.importjs: "#.get(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/get + +func clear*(self: FormData) {.importjs: + "(() => { const frmdt = #; Array.from(frmdt.keys()).forEach((key) => frmdt.delete(key)) })()".} + ## Convenience func to delete all items from `FormData`. + +func toCstring*(self: FormData): cstring {.importjs: "JSON.stringify(#)".} + +func `$`*(self: FormData): string = $toCstring(self) + +func len*(self: FormData): int {.importjs: "Array.from(#.entries()).length".} + + +runnableExamples("-r:off"): + let data: FormData = newFormData() + data["key0"] = "value0".cstring + data.add("key1".cstring, "value1".cstring) + data.delete("key1") + assert data.hasKey("key0") + assert data["key0"] == "value0".cstring + data.clear() + assert data.len == 0 diff --git a/lib/std/jsheaders.nim b/lib/std/jsheaders.nim new file mode 100644 index 000000000..6fd3b3468 --- /dev/null +++ b/lib/std/jsheaders.nim @@ -0,0 +1,83 @@ +## - HTTP Headers for the JavaScript target: https://developer.mozilla.org/en-US/docs/Web/API/Headers +when not defined(js): + {.fatal: "Module jsheaders is designed to be used with the JavaScript backend.".} + +type Headers* = ref object of JsRoot ## HTTP Headers API. + +func newHeaders*(): Headers {.importjs: "new Headers()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers + +func add*(self: Headers; key: cstring; value: cstring) {.importjs: "#.append(#, #)".} + ## Allows duplicated keys. + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/append + +func delete*(self: Headers; key: cstring) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/delete + ## + ## .. warning:: Delete *all* items with `key` from the headers, including duplicated keys. + +func hasKey*(self: Headers; key: cstring): bool {.importjs: "#.has(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/has + +func keys*(self: Headers): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/keys + +func values*(self: Headers): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/values + +func entries*(self: Headers): seq[tuple[key, value: cstring]] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/entries + +func `[]`*(self: Headers; key: cstring): cstring {.importjs: "#.get(#)".} + ## Get *all* items with `key` from the headers, including duplicated values. + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/get + +func `[]=`*(self: Headers; key: cstring; value: cstring) {.importjs: "#.set(#, #)".} + ## Do *not* allow duplicated keys, overwrites duplicated keys. + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/set + +func clear*(self: Headers) {.importjs: + "(() => { const header = #; Array.from(header.keys()).forEach((key) => header.delete(key)) })()".} + ## Convenience func to delete all items from `Headers`. + +func toCstring*(self: Headers): cstring {.importjs: "JSON.stringify(Array.from(#.entries()))".} + ## Returns a `cstring` representation of `Headers`. + +func `$`*(self: Headers): string = $toCstring(self) + +func len*(self: Headers): int {.importjs: "Array.from(#.entries()).length".} + + +runnableExamples("-r:off"): + + block: + let header: Headers = newHeaders() + header.add("key", "value") + assert header.hasKey("key") + assert header.keys() == @["key".cstring] + assert header.values() == @["value".cstring] + assert header["key"] == "value".cstring + header["other"] = "another".cstring + assert header["other"] == "another".cstring + assert header.entries() == @[("key".cstring, "value".cstring), ("other".cstring, "another".cstring)] + assert header.toCstring() == """[["key","value"],["other","another"]]""".cstring + header.delete("other") + assert header.entries() == @[("key".cstring, "value".cstring)] + header.clear() + assert header.entries() == @[] + assert header.len == 0 + + block: + let header: Headers = newHeaders() + header.add("key", "a") + header.add("key", "b") ## Duplicated. + header.add("key", "c") ## Duplicated. + assert header["key"] == "a, b, c".cstring + header["key"] = "value".cstring + assert header["key"] == "value".cstring + + block: + let header: Headers = newHeaders() + header["key"] = "a" + header["key"] = "b" ## Overwrites. + assert header["key"] == "b".cstring diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim new file mode 100644 index 000000000..2d28748ce --- /dev/null +++ b/lib/std/jsonutils.nim @@ -0,0 +1,493 @@ +##[ +This module implements a hookable (de)serialization for arbitrary types using JSON. +Design goal: avoid importing modules where a custom serialization is needed; +see strtabs.fromJsonHook,toJsonHook for an example. +]## + +runnableExamples: + import std/[strtabs,json] + type Foo = ref object + t: bool + z1: int8 + let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable]) + let j = a.toJson + assert j.jsonTo(typeof(a)).toJson == j + assert $[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert 0.0.toJson.kind == JFloat + assert Inf.toJson.kind == JString + +import std/[json, strutils, tables, sets, strtabs, options, strformat] + +#[ +Future directions: +add a way to customize serialization, for e.g.: +* field renaming +* allow serializing `enum` and `char` as `string` instead of `int` + (enum is more compact/efficient, and robust to enum renamings, but string + is more human readable) +* handle cyclic references, using a cache of already visited addresses +* implement support for serialization and de-serialization of nested variant + objects. +]# + +import std/macros +from std/enumutils import symbolName +from std/typetraits import OrdinalEnum, tupleLen + +when defined(nimPreviewSlimSystem): + import std/assertions + + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +type + Joptions* = object # xxx rename FromJsonOptions + ## Options controlling the behavior of `fromJson`. + allowExtraKeys*: bool + ## If `true` Nim's object to which the JSON is parsed is not required to + ## have a field for every JSON key. + allowMissingKeys*: bool + ## If `true` Nim's object to which JSON is parsed is allowed to have + ## fields without corresponding JSON keys. + # in future work: a key rename could be added + EnumMode* = enum + joptEnumOrd + joptEnumSymbol + joptEnumString + JsonNodeMode* = enum ## controls `toJson` for JsonNode types + joptJsonNodeAsRef ## returns the ref as is + joptJsonNodeAsCopy ## returns a deep copy of the JsonNode + joptJsonNodeAsObject ## treats JsonNode as a regular ref object + ToJsonOptions* = object + enumMode*: EnumMode + jsonNodeMode*: JsonNodeMode + # xxx charMode, etc + +proc initToJsonOptions*(): ToJsonOptions = + ## initializes `ToJsonOptions` with sane options. + ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) + +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a) + +macro getDiscriminants(a: typedesc): seq[string] = + ## return the discriminant keys + # candidate for std/typetraits + var a = a.getTypeImpl + doAssert a.kind == nnkBracketExpr + let sym = a[1] + let t = sym.getTypeImpl + let t2 = t[2] + case t2.kind + of nnkEmpty: # allow empty objects + result = quote do: + seq[string].default + of nnkRecList: + result = newTree(nnkBracket) + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + result.add newLit key.strVal + if result.len > 0: + result = quote do: + @`result` + else: + result = quote do: + seq[string].default + else: + raiseAssert "unexpected kind: " & $t2.kind + +macro initCaseObject(T: typedesc, fun: untyped): untyped = + ## does the minimum to construct a valid case object, only initializing + ## the discriminant fields; see also `getDiscriminants` + # maybe candidate for std/typetraits + var a = T.getTypeImpl + doAssert a.kind == nnkBracketExpr + let sym = a[1] + let t = sym.getTypeImpl + var t2: NimNode + case t.kind + of nnkObjectTy: t2 = t[2] + of nnkRefTy: t2 = t[0].getTypeImpl[2] + else: raiseAssert $t.kind # xxx `nnkPtrTy` could be handled too + doAssert t2.kind == nnkRecList + result = newTree(nnkObjConstr) + result.add sym + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + let typ = ti[0][1] + let key2 = key.strVal + let val = quote do: + `fun`(`key2`, typedesc[`typ`]) + result.add newTree(nnkExprColonExpr, key, val) + +proc raiseJsonException(condStr: string, msg: string) {.noinline.} = + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, condStr & " failed: " & msg) + +template checkJson(cond: untyped, msg = "") = + if not cond: + raiseJsonException(astToStr(cond), msg) + +proc hasField[T](obj: T, field: string): bool = + for k, _ in fieldPairs(obj): + if k == field: + return true + return false + +macro accessField(obj: typed, name: static string): untyped = + newDotExpr(obj, ident(name)) + +template fromJsonFields(newObj, oldObj, json, discKeys, opt) = + type T = typeof(newObj) + # we could customize whether to allow JNull + checkJson json.kind == JObject, $json.kind + var num, numMatched = 0 + for key, val in fieldPairs(newObj): + num.inc + when key notin discKeys: + if json.hasKey key: + numMatched.inc + fromJson(val, json[key], opt) + elif opt.allowMissingKeys: + # if there are no discriminant keys the `oldObj` must always have the + # same keys as the new one. Otherwise we must check, because they could + # be set to different branches. + when typeof(oldObj) isnot typeof(nil): + if discKeys.len == 0 or hasField(oldObj, key): + val = accessField(oldObj, key) + else: + checkJson false, "key '$1' for $2 not in $3" % [key, $T, json.pretty()] + else: + if json.hasKey key: + numMatched.inc + + let ok = + if opt.allowExtraKeys and opt.allowMissingKeys: + true + elif opt.allowExtraKeys: + # This check is redundant because if here missing keys are not allowed, + # and if `num != numMatched` it will fail in the loop above but it is left + # for clarity. + assert num == numMatched + num == numMatched + elif opt.allowMissingKeys: + json.len == numMatched + else: + json.len == num and num == numMatched + + checkJson ok, "There were $1 keys (expecting $2) for $3 with $4" % [$json.len, $num, $T, json.pretty()] + +proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) + +proc discKeyMatch[T](obj: T, json: JsonNode, key: static string): bool = + if not json.hasKey key: + return true + let field = accessField(obj, key) + var jsonVal: typeof(field) + fromJson(jsonVal, json[key]) + if jsonVal != field: + return false + return true + +macro discKeysMatchBodyGen(obj: typed, json: JsonNode, + keys: static seq[string]): untyped = + result = newStmtList() + let r = ident("result") + for key in keys: + let keyLit = newLit key + result.add quote do: + `r` = `r` and discKeyMatch(`obj`, `json`, `keyLit`) + +proc discKeysMatch[T](obj: T, json: JsonNode, keys: static seq[string]): bool = + result = true + discKeysMatchBodyGen(obj, json, keys) + +proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = + ## inplace version of `jsonTo` + #[ + adding "json path" leading to `b` can be added in future work. + ]# + checkJson b != nil, $($T, b) + when compiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt) + elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + elif T is bool: a = to(b,T) + elif T is enum: + case b.kind + of JInt: a = T(b.getBiggestInt()) + of JString: a = parseEnum[T](b.getStr()) + else: checkJson false, fmt"Expecting int/string for {$T} got {b.pretty()}" + elif T is uint|uint64: a = T(to(b, uint64)) + elif T is Ordinal: a = cast[T](to(b, int)) + elif T is pointer: a = cast[pointer](to(b, int)) + elif T is distinct: a.distinctBase.fromJson(b) + elif T is string|SomeNumber: a = to(b,T) + elif T is cstring: + case b.kind + of JNull: a = nil + of JString: a = b.str + else: checkJson false, fmt"Expecting null/string for {$T} got {b.pretty()}" + elif T is JsonNode: a = b + elif T is ref | ptr: + if b.kind == JNull: a = nil + else: + a = T() + fromJson(a[], b, opt) + elif T is array: + checkJson a.len == b.len, fmt"Json array size doesn't match for {$T}" + var i = 0 + for ai in mitems(a): + fromJson(ai, b[i], opt) + i.inc + elif T is set: + type E = typeof(for ai in a: ai) + for val in b.getElems: + incl a, jsonTo(val, E) + elif T is seq: + a.setLen b.len + for i, val in b.getElems: + fromJson(a[i], val, opt) + elif T is object: + template fun(key, typ): untyped {.used.} = + if b.hasKey key: + jsonTo(b[key], typ) + elif hasField(a, key): + accessField(a, key) + else: + default(typ) + const keys = getDiscriminants(T) + when keys.len == 0: + fromJsonFields(a, nil, b, keys, opt) + else: + if discKeysMatch(a, b, keys): + fromJsonFields(a, nil, b, keys, opt) + else: + var newObj = initCaseObject(T, fun) + fromJsonFields(newObj, a, b, keys, opt) + a = newObj + elif T is tuple: + when isNamedTuple(T): + fromJsonFields(a, nil, b, seq[string].default, opt) + else: + checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + + when compiles(tupleLen(T)): + let tupleSize = tupleLen(T) + else: + # Tuple len isn't in csources_v1 so using tupleLen would fail. + # Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency + var tupleSize = 0 + for val in fields(a): + tupleSize.inc + + checkJson b.len == tupleSize, fmt"Json doesn't match expected length of {tupleSize}, got {b.pretty()}" + var i = 0 + for val in fields(a): + fromJson(val, b[i], opt) + i.inc + else: + # checkJson not appropriate here + static: raiseAssert "not yet implemented: " & $T + +proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = + ## reverse of `toJson` + fromJson(result, b, opt) + +proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = + ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to + ## customize serialization, see strtabs.toJsonHook for an example. + ## + ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can + ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. + ## It is expected that this behavior becomes the new default in upcoming versions. + when compiles(toJsonHook(a, opt)): result = toJsonHook(a, opt) + elif compiles(toJsonHook(a)): result = toJsonHook(a) + elif T is object | tuple: + when T is object or isNamedTuple(T): + result = newJObject() + for k, v in a.fieldPairs: result[k] = toJson(v, opt) + else: + result = newJArray() + for v in a.fields: result.add toJson(v, opt) + elif T is ref | ptr: + template impl = + if system.`==`(a, nil): result = newJNull() + else: result = toJson(a[], opt) + when T is JsonNode: + case opt.jsonNodeMode + of joptJsonNodeAsRef: result = a + of joptJsonNodeAsCopy: result = copy(a) + of joptJsonNodeAsObject: impl() + else: impl() + elif T is array | seq | set: + result = newJArray() + for ai in a: result.add toJson(ai, opt) + elif T is pointer: result = toJson(cast[int](a), opt) + # edge case: `a == nil` could've also led to `newJNull()`, but this results + # in simpler code for `toJson` and `fromJson`. + elif T is distinct: result = toJson(a.distinctBase, opt) + elif T is bool: result = %(a) + elif T is SomeInteger: result = %a + elif T is enum: + case opt.enumMode + of joptEnumOrd: + when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord) + else: toJson($a, opt) + of joptEnumSymbol: + when T is OrdinalEnum: + toJson(symbolName(a), opt) + else: + toJson($a, opt) + of joptEnumString: toJson($a, opt) + elif T is Ordinal: result = %(a.ord) + elif T is cstring: (if a == nil: result = newJNull() else: result = % $a) + else: result = %a + +proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]), + jsonNode: JsonNode, opt = Joptions()) = + ## Enables `fromJson` for `Table` and `OrderedTable` types. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook>`_ + runnableExamples: + import std/[tables, json] + var foo: tuple[t: Table[string, int], ot: OrderedTable[string, int]] + fromJson(foo, parseJson(""" + {"t":{"two":2,"one":1},"ot":{"one":1,"three":3}}""")) + assert foo.t == [("one", 1), ("two", 2)].toTable + assert foo.ot == [("one", 1), ("three", 3)].toOrderedTable + + assert jsonNode.kind == JObject, + "The kind of the `jsonNode` must be `JObject`, but its actual " & + "type is `" & $jsonNode.kind & "`." + clear(t) + for k, v in jsonNode: + t[k] = jsonTo(v, V, opt) + +proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V]), opt = initToJsonOptions()): JsonNode = + ## Enables `toJson` for `Table` and `OrderedTable` types. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_ + runnableExamples: + import std/[tables, json, sugar] + let foo = ( + t: [("two", 2)].toTable, + ot: [("one", 1), ("three", 3)].toOrderedTable) + assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}""" + # if keys are not string|cstring, you can use this: + let a = {10: "foo", 11: "bar"}.newOrderedTable + let a2 = collect: (for k,v in a: (k,v)) + assert $toJson(a2) == """[[10,"foo"],[11,"bar"]]""" + + result = newJObject() + for k, v in pairs(t): + # not sure if $k has overhead for string + result[(when K is string: k else: $k)] = toJson(v, opt) + +proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode, opt = Joptions()) = + ## Enables `fromJson` for `HashSet` and `OrderedSet` types. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_ + runnableExamples: + import std/[sets, json] + var foo: tuple[hs: HashSet[string], os: OrderedSet[string]] + fromJson(foo, parseJson(""" + {"hs": ["hash", "set"], "os": ["ordered", "set"]}""")) + assert foo.hs == ["hash", "set"].toHashSet + assert foo.os == ["ordered", "set"].toOrderedSet + + assert jsonNode.kind == JArray, + "The kind of the `jsonNode` must be `JArray`, but its actual " & + "type is `" & $jsonNode.kind & "`." + clear(s) + for v in jsonNode: + incl(s, jsonTo(v, A, opt)) + +proc toJsonHook*[A](s: SomeSet[A], opt = initToJsonOptions()): JsonNode = + ## Enables `toJson` for `HashSet` and `OrderedSet` types. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,SomeSet[A],JsonNode>`_ + runnableExamples: + import std/[sets, json] + let foo = (hs: ["hash"].toHashSet, os: ["ordered", "set"].toOrderedSet) + assert $toJson(foo) == """{"hs":["hash"],"os":["ordered","set"]}""" + + result = newJArray() + for k in s: + add(result, toJson(k, opt)) + +proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode, opt = Joptions()) = + ## Enables `fromJson` for `Option` types. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook,Option[T]>`_ + runnableExamples: + import std/[options, json] + var opt: Option[string] + fromJsonHook(opt, parseJson("\"test\"")) + assert get(opt) == "test" + fromJson(opt, parseJson("null")) + assert isNone(opt) + + if jsonNode.kind != JNull: + self = some(jsonTo(jsonNode, T, opt)) + else: + self = none[T]() + +proc toJsonHook*[T](self: Option[T], opt = initToJsonOptions()): JsonNode = + ## Enables `toJson` for `Option` types. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,Option[T],JsonNode>`_ + runnableExamples: + import std/[options, json] + let optSome = some("test") + assert $toJson(optSome) == "\"test\"" + let optNone = none[string]() + assert $toJson(optNone) == "null" + + if isSome(self): + toJson(get(self), opt) + else: + newJNull() + +proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = + ## Enables `fromJson` for `StringTableRef` type. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook,StringTableRef>`_ + runnableExamples: + import std/[strtabs, json] + var t = newStringTable(modeCaseSensitive) + let jsonStr = """{"mode": 0, "table": {"name": "John", "surname": "Doe"}}""" + fromJsonHook(t, parseJson(jsonStr)) + assert t[] == newStringTable("name", "John", "surname", "Doe", + modeCaseSensitive)[] + + var mode = jsonTo(b["mode"], StringTableMode) + a = newStringTable(mode) + let b2 = b["table"] + for k,v in b2: a[k] = jsonTo(v, string) + +proc toJsonHook*(a: StringTableRef): JsonNode = + ## Enables `toJson` for `StringTableRef` type. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_ + runnableExamples: + import std/[strtabs, json] + let t = newStringTable("name", "John", "surname", "Doe", modeCaseSensitive) + let jsonStr = """{"mode": "modeCaseSensitive", + "table": {"name": "John", "surname": "Doe"}}""" + assert toJson(t) == parseJson(jsonStr) + + result = newJObject() + result["mode"] = toJson($a.mode) + let t = newJObject() + for k,v in a: t[k] = toJson(v) + result["table"] = t diff --git a/lib/std/logic.nim b/lib/std/logic.nim new file mode 100644 index 000000000..84640d380 --- /dev/null +++ b/lib/std/logic.nim @@ -0,0 +1,10 @@ +## This module provides further logic operators like 'forall' and 'exists' +## They are only supported in `.ensures` etc pragmas. + +proc `->`*(a, b: bool): bool {.magic: "Implies".} +proc `<->`*(a, b: bool): bool {.magic: "Iff".} + +proc forall*(args: varargs[untyped]): bool {.magic: "Forall".} +proc exists*(args: varargs[untyped]): bool {.magic: "Exists".} + +proc old*[T](x: T): T {.magic: "Old".} diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim new file mode 100644 index 000000000..bf6dc776b --- /dev/null +++ b/lib/std/monotimes.nim @@ -0,0 +1,160 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +The `std/monotimes` module implements monotonic timestamps. A monotonic +timestamp represents the time that has passed since some system defined +point in time. The monotonic timestamps are guaranteed not to decrease, +meaning that that the following is guaranteed to work: +]## + +runnableExamples: + let a = getMonoTime() + let b = getMonoTime() + assert a <= b + +##[ +This is not guaranteed for the `times.Time` type! This means that the +`MonoTime` should be used when measuring durations of time with +high precision. + +However, since `MonoTime` represents the time that has passed since some +unknown time origin, it cannot be converted to a human readable timestamp. +If this is required, the `times.Time` type should be used instead. + +The `MonoTime` type stores the timestamp in nanosecond resolution, but note +that the actual supported time resolution differs for different systems. + +See also +======== +* `times module <times.html>`_ +]## + +import std/times + +type + MonoTime* = object ## Represents a monotonic timestamp. + ticks: int64 + +when defined(macosx): + type + MachTimebaseInfoData {.pure, final, importc: "mach_timebase_info_data_t", + header: "<mach/mach_time.h>".} = object + numer, denom: int32 + + proc mach_absolute_time(): int64 {.importc, header: "<mach/mach.h>".} + proc mach_timebase_info(info: var MachTimebaseInfoData) {.importc, + header: "<mach/mach_time.h>".} + +when defined(js): + proc getJsTicks: float = + ## Returns ticks in the unit seconds. + when defined(nodejs): + {.emit: """ + let process = require('process'); + let time = process.hrtime(); + `result` = time[0] + time[1] / 1000000000; + """.} + else: + proc jsNow(): float {.importjs: "window.performance.now()".} + result = jsNow() / 1000 + + # Workaround for #6752. + {.push overflowChecks: off.} + proc `-`(a, b: int64): int64 = + system.`-`(a, b) + proc `+`(a, b: int64): int64 = + system.`+`(a, b) + {.pop.} + +elif defined(posix) and not defined(osx): + import std/posix + +when defined(zephyr): + proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "<kernel.h>".} + proc k_ticks_to_ns_floor64(ticks: int64): int64 {.importc: "k_ticks_to_ns_floor64", header: "<kernel.h>".} + +elif defined(windows): + proc QueryPerformanceCounter(res: var uint64) {. + importc: "QueryPerformanceCounter", stdcall, dynlib: "kernel32".} + proc QueryPerformanceFrequency(res: var uint64) {. + importc: "QueryPerformanceFrequency", stdcall, dynlib: "kernel32".} + +proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = + ## Returns the current `MonoTime` timestamp. + ## + ## When compiled with the JS backend and executed in a browser, + ## this proc calls `window.performance.now()`. + ## See [MDN](https://developer.mozilla.org/en-US/docs/Web/API/Performance/now) + ## for more information. + when defined(js): + let ticks = getJsTicks() + result = MonoTime(ticks: (ticks * 1_000_000_000).int64) + elif defined(macosx): + let ticks = mach_absolute_time() + var machAbsoluteTimeFreq: MachTimebaseInfoData + mach_timebase_info(machAbsoluteTimeFreq) + result = MonoTime(ticks: ticks * machAbsoluteTimeFreq.numer div + machAbsoluteTimeFreq.denom) + elif defined(zephyr): + let ticks = k_ticks_to_ns_floor64(k_uptime_ticks()) + result = MonoTime(ticks: ticks) + elif defined(posix): + var ts: Timespec + discard clock_gettime(CLOCK_MONOTONIC, ts) + result = MonoTime(ticks: ts.tv_sec.int64 * 1_000_000_000 + + ts.tv_nsec.int64) + elif defined(windows): + var ticks: uint64 + QueryPerformanceCounter(ticks) + + var freq: uint64 + QueryPerformanceFrequency(freq) + let queryPerformanceCounterFreq = 1_000_000_000'u64 div freq + result = MonoTime(ticks: (ticks * queryPerformanceCounterFreq).int64) + +proc ticks*(t: MonoTime): int64 = + ## Returns the raw ticks value from a `MonoTime`. This value always uses + ## nanosecond time resolution. + t.ticks + +proc `$`*(t: MonoTime): string = + $t.ticks + +proc `-`*(a, b: MonoTime): Duration = + ## Returns the difference between two `MonoTime` timestamps as a `Duration`. + initDuration(nanoseconds = (a.ticks - b.ticks)) + +proc `+`*(a: MonoTime, b: Duration): MonoTime = + ## Increases `a` by `b`. + MonoTime(ticks: a.ticks + b.inNanoseconds) + +proc `-`*(a: MonoTime, b: Duration): MonoTime = + ## Reduces `a` by `b`. + MonoTime(ticks: a.ticks - b.inNanoseconds) + +proc `<`*(a, b: MonoTime): bool = + ## Returns true if `a` happened before `b`. + a.ticks < b.ticks + +proc `<=`*(a, b: MonoTime): bool = + ## Returns true if `a` happened before `b` or if they happened simultaneous. + a.ticks <= b.ticks + +proc `==`*(a, b: MonoTime): bool = + ## Returns true if `a` and `b` happened simultaneous. + a.ticks == b.ticks + +proc high*(typ: typedesc[MonoTime]): MonoTime = + ## Returns the highest representable `MonoTime`. + MonoTime(ticks: high(int64)) + +proc low*(typ: typedesc[MonoTime]): MonoTime = + ## Returns the lowest representable `MonoTime`. + MonoTime(ticks: low(int64)) diff --git a/lib/std/objectdollar.nim b/lib/std/objectdollar.nim new file mode 100644 index 000000000..86ce9afc8 --- /dev/null +++ b/lib/std/objectdollar.nim @@ -0,0 +1,13 @@ +## This module implements a generic `$` operator to convert objects to strings. + +import std/private/miscdollars + +proc `$`*[T: object](x: T): string = + ## Generic `$` operator for objects with similar output to + ## `$` for named tuples. + runnableExamples: + type Foo = object + a, b: int + let x = Foo(a: 23, b: 45) + assert $x == "(a: 23, b: 45)" + tupleObjectDollar(result, x) diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim new file mode 100644 index 000000000..7b11c5e8e --- /dev/null +++ b/lib/std/oserrors.nim @@ -0,0 +1,117 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/oserrors` module implements OS error reporting. + +type + OSErrorCode* = distinct int32 ## Specifies an OS Error Code. + +when not defined(nimscript): + when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + else: + var errno {.importc, header: "<errno.h>".}: cint + + proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "<string.h>".} + +proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} +proc `$`*(err: OSErrorCode): string {.borrow.} + +proc osErrorMsg*(errorCode: OSErrorCode): string = + ## Converts an OS error code into a human readable string. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If conversion fails, or `errorCode` is `0` then `""` will be + ## returned. + ## + ## See also: + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ + runnableExamples: + when defined(linux): + assert osErrorMsg(OSErrorCode(0)) == "" + assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" + assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" + + result = "" + when defined(nimscript): + discard + elif defined(windows): + if errorCode != OSErrorCode(0'i32): + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: + result = $msgbuf + if msgbuf != nil: localFree(cast[pointer](msgbuf)) + else: + if errorCode != OSErrorCode(0'i32): + result = $c_strerror(errorCode.int32) + +proc newOSError*( + errorCode: OSErrorCode, additionalInfo = "" +): owned(ref OSError) {.noinline.} = + ## Creates a new `OSError exception <system.html#OSError>`_. + ## + ## The `errorCode` will determine the + ## message, `osErrorMsg proc`_ will be used + ## to get this message. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If the error code is `0` or an error message could not be retrieved, + ## the message `unknown OS error` will be used. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) + if additionalInfo.len > 0: + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. + if result.msg == "": + result.msg = "unknown OS error" + +proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = + ## Raises an `OSError exception <system.html#OSError>`_. + ## + ## Read the description of the `newOSError proc`_ to learn + ## how the exception object is created. + raise newOSError(errorCode, additionalInfo) + +{.push stackTrace:off.} +proc osLastError*(): OSErrorCode {.sideEffect.} = + ## Retrieves the last operating system error code. + ## + ## This procedure is useful in the event when an OS call fails. In that case + ## this procedure will return the error code describing the reason why the + ## OS call failed. The `OSErrorMsg` procedure can then be used to convert + ## this code into a string. + ## + ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. + ## On Windows some OS calls can reset the error code to `0` causing this + ## procedure to return `0`. It is therefore advised to call this procedure + ## immediately after an OS call fails. On POSIX systems this is not a problem. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ + when defined(nimscript): + discard + elif defined(windows): + result = cast[OSErrorCode](getLastError()) + else: + result = OSErrorCode(errno) +{.pop.} diff --git a/lib/std/outparams.nim b/lib/std/outparams.nim new file mode 100644 index 000000000..a471fbaa7 --- /dev/null +++ b/lib/std/outparams.nim @@ -0,0 +1,38 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## `outParamsAt` macro for easy writing code that works with both 2.0 and 1.x. + +import std/macros + +macro outParamsAt*(positions: static openArray[int]; n: untyped): untyped = + ## Use this macro to annotate `out` parameters in a portable way. + runnableExamples: + proc p(x: var int) {.outParamsAt: [1].} = + discard "x is really an 'out int' if the Nim compiler supports 'out' parameters" + + result = n + when defined(nimHasOutParams): + var p = n.params + for po in positions: + p[po][^2].expectKind nnkVarTy + p[po][^2] = newTree(nnkOutTy, p[po][^2][0]) + +when isMainModule: + {.experimental: "strictDefs".} + + proc main(x: var int) {.outParamsAt: [1].} = + x = 3 + + proc us = + var x: int + main x + echo x + + us() diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim new file mode 100644 index 000000000..3320558f2 --- /dev/null +++ b/lib/std/packedsets.nim @@ -0,0 +1,601 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## The `packedsets` module implements an efficient `Ordinal` set implemented as a +## `sparse bit set`:idx:. +## +## Supports any Ordinal type. +## +## See also +## ======== +## * `sets module <sets.html>`_ for more general hash sets + +import std/private/since +import std/hashes + +when defined(nimPreviewSlimSystem): + import std/assertions + +type + BitScalar = uint + +const + InitIntSetSize = 8 # must be a power of two! + TrunkShift = 9 + BitsPerTrunk = 1 shl TrunkShift # needs to be a power of 2 and + # divisible by 64 + TrunkMask = BitsPerTrunk - 1 + IntsPerTrunk = BitsPerTrunk div (sizeof(BitScalar) * 8) + IntShift = 5 + ord(sizeof(BitScalar) == 8) # 5 or 6, depending on int width + IntMask = 1 shl IntShift - 1 + +type + Trunk {.acyclic.} = ref object + next: Trunk # all nodes are connected with this pointer + key: int # start address at bit 0 + bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector + + TrunkSeq = seq[Trunk] + + PackedSet*[A: Ordinal] = object + ## An efficient set of `Ordinal` types implemented as a sparse bit set. + elems: int # only valid for small numbers + counter, max: int + head: Trunk + data: TrunkSeq + a: array[0..33, int] # profiling shows that 34 elements are enough + +proc mustRehash[T](t: T): bool {.inline.} = + let length = t.max + 1 + assert length > t.counter + result = (length * 2 < t.counter * 3) or (length - t.counter < 4) + +proc nextTry(h, maxHash: Hash, perturb: var Hash): Hash {.inline.} = + const PERTURB_SHIFT = 5 + var perturb2 = cast[uint](perturb) shr PERTURB_SHIFT + perturb = cast[Hash](perturb2) + result = ((5 * h) + 1 + perturb) and maxHash + +proc packedSetGet[A](t: PackedSet[A], key: int): Trunk = + var h = key and t.max + var perturb = key + while t.data[h] != nil: + if t.data[h].key == key: + return t.data[h] + h = nextTry(h, t.max, perturb) + result = nil + +proc intSetRawInsert[A](t: PackedSet[A], data: var TrunkSeq, desc: Trunk) = + var h = desc.key and t.max + var perturb = desc.key + while data[h] != nil: + assert data[h] != desc + h = nextTry(h, t.max, perturb) + assert data[h] == nil + data[h] = desc + +proc intSetEnlarge[A](t: var PackedSet[A]) = + var n: TrunkSeq + var oldMax = t.max + t.max = ((t.max + 1) * 2) - 1 + newSeq(n, t.max + 1) + for i in countup(0, oldMax): + if t.data[i] != nil: intSetRawInsert(t, n, t.data[i]) + swap(t.data, n) + +proc intSetPut[A](t: var PackedSet[A], key: int): Trunk = + var h = key and t.max + var perturb = key + while t.data[h] != nil: + if t.data[h].key == key: + return t.data[h] + h = nextTry(h, t.max, perturb) + if mustRehash(t): intSetEnlarge(t) + inc(t.counter) + h = key and t.max + perturb = key + while t.data[h] != nil: h = nextTry(h, t.max, perturb) + assert t.data[h] == nil + new(result) + result.next = t.head + result.key = key + t.head = result + t.data[h] = result + +proc bitincl[A](s: var PackedSet[A], key: int) {.inline.} = + var t = intSetPut(s, key shr TrunkShift) + var u = key and TrunkMask + t.bits[u shr IntShift] = t.bits[u shr IntShift] or + (BitScalar(1) shl (u and IntMask)) + +proc exclImpl[A](s: var PackedSet[A], key: int) = + if s.elems <= s.a.len: + for i in 0..<s.elems: + if s.a[i] == key: + s.a[i] = s.a[s.elems - 1] + dec(s.elems) + return + else: + var t = packedSetGet(s, key shr TrunkShift) + if t != nil: + var u = key and TrunkMask + t.bits[u shr IntShift] = t.bits[u shr IntShift] and + not(BitScalar(1) shl (u and IntMask)) + +template dollarImpl(): untyped = + result = "{" + for key in items(s): + if result.len > 1: result.add(", ") + result.add $key + result.add("}") + +iterator items*[A](s: PackedSet[A]): A {.inline.} = + ## Iterates over any included element of `s`. + if s.elems <= s.a.len: + for i in 0..<s.elems: + yield A(s.a[i]) + else: + var r = s.head + while r != nil: + var i = 0 + while i <= high(r.bits): + var w: uint = r.bits[i] + # taking a copy of r.bits[i] here is correct, because + # modifying operations are not allowed during traversation + var j = 0 + while w != 0: # test all remaining bits for zero + if (w and 1) != 0: # the bit is set! + yield A((r.key shl TrunkShift) or (i shl IntShift +% j)) + inc(j) + w = w shr 1 + inc(i) + r = r.next + +proc initPackedSet*[A]: PackedSet[A] = + ## Returns an empty `PackedSet[A]`. + ## `A` must be `Ordinal`. + ## + ## **See also:** + ## * `toPackedSet proc <#toPackedSet,openArray[A]>`_ + runnableExamples: + let a = initPackedSet[int]() + assert len(a) == 0 + + type Id = distinct int + var ids = initPackedSet[Id]() + ids.incl(3.Id) + + result = PackedSet[A]( + elems: 0, + counter: 0, + max: 0, + head: nil, + data: @[]) + # a: array[0..33, int] # profiling shows that 34 elements are enough + +proc contains*[A](s: PackedSet[A], key: A): bool = + ## Returns true if `key` is in `s`. + ## + ## This allows the usage of the `in` operator. + runnableExamples: + type ABCD = enum A, B, C, D + + let a = [1, 3, 5].toPackedSet + assert a.contains(3) + assert 3 in a + assert not a.contains(8) + assert 8 notin a + + let letters = [A, C].toPackedSet + assert A in letters + assert C in letters + assert B notin letters + + if s.elems <= s.a.len: + result = false + for i in 0..<s.elems: + if s.a[i] == ord(key): return true + else: + var t = packedSetGet(s, ord(key) shr TrunkShift) + if t != nil: + var u = ord(key) and TrunkMask + result = (t.bits[u shr IntShift] and + (BitScalar(1) shl (u and IntMask))) != 0 + else: + result = false + +proc incl*[A](s: var PackedSet[A], key: A) = + ## Includes an element `key` in `s`. + ## + ## This doesn't do anything if `key` is already in `s`. + ## + ## **See also:** + ## * `excl proc <#excl,PackedSet[A],A>`_ for excluding an element + ## * `incl proc <#incl,PackedSet[A],PackedSet[A]>`_ for including a set + ## * `containsOrIncl proc <#containsOrIncl,PackedSet[A],A>`_ + runnableExamples: + var a = initPackedSet[int]() + a.incl(3) + a.incl(3) + assert len(a) == 1 + + if s.elems <= s.a.len: + for i in 0..<s.elems: + if s.a[i] == ord(key): return + if s.elems < s.a.len: + s.a[s.elems] = ord(key) + inc(s.elems) + return + newSeq(s.data, InitIntSetSize) + s.max = InitIntSetSize - 1 + for i in 0..<s.elems: + bitincl(s, s.a[i]) + s.elems = s.a.len + 1 + # fall through: + bitincl(s, ord(key)) + +proc incl*[A](s: var PackedSet[A], other: PackedSet[A]) = + ## Includes all elements from `other` into `s`. + ## + ## This is the in-place version of `s + other <#+,PackedSet[A],PackedSet[A]>`_. + ## + ## **See also:** + ## * `excl proc <#excl,PackedSet[A],PackedSet[A]>`_ for excluding a set + ## * `incl proc <#incl,PackedSet[A],A>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,PackedSet[A],A>`_ + runnableExamples: + var a = [1].toPackedSet + a.incl([5].toPackedSet) + assert len(a) == 2 + assert 5 in a + + for item in other.items: incl(s, item) + +proc toPackedSet*[A](x: openArray[A]): PackedSet[A] {.since: (1, 3).} = + ## Creates a new `PackedSet[A]` that contains the elements of `x`. + ## + ## Duplicates are removed. + ## + ## **See also:** + ## * `initPackedSet proc <#initPackedSet>`_ + runnableExamples: + let a = [5, 6, 7, 8, 8].toPackedSet + assert len(a) == 4 + assert $a == "{5, 6, 7, 8}" + + result = initPackedSet[A]() + for item in x: + result.incl(item) + +proc containsOrIncl*[A](s: var PackedSet[A], key: A): bool = + ## Includes `key` in the set `s` and tells if `key` was already in `s`. + ## + ## The difference with regards to the `incl proc <#incl,PackedSet[A],A>`_ is + ## that this proc returns true if `s` already contained `key`. The + ## proc will return false if `key` was added as a new value to `s` during + ## this call. + ## + ## **See also:** + ## * `incl proc <#incl,PackedSet[A],A>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,PackedSet[A],A>`_ + runnableExamples: + var a = initPackedSet[int]() + assert a.containsOrIncl(3) == false + assert a.containsOrIncl(3) == true + assert a.containsOrIncl(4) == false + + if s.elems <= s.a.len: + for i in 0..<s.elems: + if s.a[i] == ord(key): + return true + incl(s, key) + result = false + else: + var t = packedSetGet(s, ord(key) shr TrunkShift) + if t != nil: + var u = ord(key) and TrunkMask + result = (t.bits[u shr IntShift] and BitScalar(1) shl (u and IntMask)) != 0 + if not result: + t.bits[u shr IntShift] = t.bits[u shr IntShift] or + (BitScalar(1) shl (u and IntMask)) + else: + incl(s, key) + result = false + +proc excl*[A](s: var PackedSet[A], key: A) = + ## Excludes `key` from the set `s`. + ## + ## This doesn't do anything if `key` is not found in `s`. + ## + ## **See also:** + ## * `incl proc <#incl,PackedSet[A],A>`_ for including an element + ## * `excl proc <#excl,PackedSet[A],PackedSet[A]>`_ for excluding a set + ## * `missingOrExcl proc <#missingOrExcl,PackedSet[A],A>`_ + runnableExamples: + var a = [3].toPackedSet + a.excl(3) + a.excl(3) + a.excl(99) + assert len(a) == 0 + + exclImpl[A](s, ord(key)) + +proc excl*[A](s: var PackedSet[A], other: PackedSet[A]) = + ## Excludes all elements from `other` from `s`. + ## + ## This is the in-place version of `s - other <#-,PackedSet[A],PackedSet[A]>`_. + ## + ## **See also:** + ## * `incl proc <#incl,PackedSet[A],PackedSet[A]>`_ for including a set + ## * `excl proc <#excl,PackedSet[A],A>`_ for excluding an element + ## * `missingOrExcl proc <#missingOrExcl,PackedSet[A],A>`_ + runnableExamples: + var a = [1, 5].toPackedSet + a.excl([5].toPackedSet) + assert len(a) == 1 + assert 5 notin a + + for item in other.items: + excl(s, item) + +proc len*[A](s: PackedSet[A]): int {.inline.} = + ## Returns the number of elements in `s`. + runnableExamples: + let a = [1, 3, 5].toPackedSet + assert len(a) == 3 + + if s.elems < s.a.len: + result = s.elems + else: + result = 0 + for _ in s.items: + # pending bug #11167; when fixed, check each explicit `items` to see if it can be removed + inc(result) + +proc missingOrExcl*[A](s: var PackedSet[A], key: A): bool = + ## Excludes `key` from the set `s` and tells if `key` was already missing from `s`. + ## + ## The difference with regards to the `excl proc <#excl,PackedSet[A],A>`_ is + ## that this proc returns true if `key` was missing from `s`. + ## The proc will return false if `key` was in `s` and it was removed + ## during this call. + ## + ## **See also:** + ## * `excl proc <#excl,PackedSet[A],A>`_ for excluding an element + ## * `excl proc <#excl,PackedSet[A],PackedSet[A]>`_ for excluding a set + ## * `containsOrIncl proc <#containsOrIncl,PackedSet[A],A>`_ + runnableExamples: + var a = [5].toPackedSet + assert a.missingOrExcl(5) == false + assert a.missingOrExcl(5) == true + + var count = s.len + exclImpl(s, ord(key)) + result = count == s.len + +proc clear*[A](result: var PackedSet[A]) = + ## Clears the `PackedSet[A]` back to an empty state. + runnableExamples: + var a = [5, 7].toPackedSet + clear(a) + assert len(a) == 0 + + # setLen(result.data, InitIntSetSize) + # for i in 0..InitIntSetSize - 1: result.data[i] = nil + # result.max = InitIntSetSize - 1 + result.data = @[] + result.max = 0 + result.counter = 0 + result.head = nil + result.elems = 0 + +proc isNil*[A](x: PackedSet[A]): bool {.inline.} = + ## Returns true if `x` is empty, false otherwise. + runnableExamples: + var a = initPackedSet[int]() + assert a.isNil + a.incl(2) + assert not a.isNil + a.excl(2) + assert a.isNil + + x.head.isNil and x.elems == 0 + +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + if src.elems <= src.a.len: + dest.data = @[] + dest.max = 0 + dest.counter = src.counter + dest.head = nil + dest.elems = src.elems + dest.a = src.a + else: + dest.counter = src.counter + dest.max = src.max + dest.elems = src.elems + newSeq(dest.data, src.data.len) + + var it = src.head + while it != nil: + var h = it.key and dest.max + var perturb = it.key + while dest.data[h] != nil: h = nextTry(h, dest.max, perturb) + assert dest.data[h] == nil + var n: Trunk + new(n) + n.next = dest.head + n.key = it.key + n.bits = it.bits + dest.head = n + dest.data[h] = n + it = it.next + +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + +proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the union of the sets `s1` and `s2`. + ## + ## The same as `s1 + s2 <#+,PackedSet[A],PackedSet[A]>`_. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = union(a, b) + assert c.len == 5 + assert c == [1, 2, 3, 4, 5].toPackedSet + + result.assign(s1) + incl(result, s2) + +proc intersection*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the intersection of the sets `s1` and `s2`. + ## + ## The same as `s1 * s2 <#*,PackedSet[A],PackedSet[A]>`_. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = intersection(a, b) + assert c.len == 1 + assert c == [3].toPackedSet + + result = initPackedSet[A]() + for item in s1.items: + if contains(s2, item): + incl(result, item) + +proc difference*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the difference of the sets `s1` and `s2`. + ## + ## The same as `s1 - s2 <#-,PackedSet[A],PackedSet[A]>`_. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = difference(a, b) + assert c.len == 2 + assert c == [1, 2].toPackedSet + + result = initPackedSet[A]() + for item in s1.items: + if not contains(s2, item): + incl(result, item) + +proc symmetricDifference*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the symmetric difference of the sets `s1` and `s2`. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = symmetricDifference(a, b) + assert c.len == 4 + assert c == [1, 2, 4, 5].toPackedSet + + result.assign(s1) + for item in s2.items: + if containsOrIncl(result, item): + excl(result, item) + +proc `+`*[A](s1, s2: PackedSet[A]): PackedSet[A] {.inline.} = + ## Alias for `union(s1, s2) <#union,PackedSet[A],PackedSet[A]>`_. + result = union(s1, s2) + +proc `*`*[A](s1, s2: PackedSet[A]): PackedSet[A] {.inline.} = + ## Alias for `intersection(s1, s2) <#intersection,PackedSet[A],PackedSet[A]>`_. + result = intersection(s1, s2) + +proc `-`*[A](s1, s2: PackedSet[A]): PackedSet[A] {.inline.} = + ## Alias for `difference(s1, s2) <#difference,PackedSet[A],PackedSet[A]>`_. + result = difference(s1, s2) + +proc disjoint*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if the sets `s1` and `s2` have no items in common. + runnableExamples: + let + a = [1, 2].toPackedSet + b = [2, 3].toPackedSet + c = [3, 4].toPackedSet + assert disjoint(a, b) == false + assert disjoint(a, c) == true + + for item in s1.items: + if contains(s2, item): + return false + return true + +proc card*[A](s: PackedSet[A]): int {.inline.} = + ## Alias for `len() <#len,PackedSet[A]>`_. + ## + ## Card stands for the [cardinality](http://en.wikipedia.org/wiki/Cardinality) + ## of a set. + result = s.len() + +proc `<=`*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if `s1` is a subset of `s2`. + ## + ## A subset `s1` has all of its elements in `s2`, but `s2` doesn't necessarily + ## have more elements than `s1`. That is, `s1` can be equal to `s2`. + runnableExamples: + let + a = [1].toPackedSet + b = [1, 2].toPackedSet + c = [1, 3].toPackedSet + assert a <= b + assert b <= b + assert not (c <= b) + + for item in s1.items: + if not s2.contains(item): + return false + return true + +proc `<`*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if `s1` is a proper subset of `s2`. + ## + ## A strict or proper subset `s1` has all of its elements in `s2`, but `s2` has + ## more elements than `s1`. + runnableExamples: + let + a = [1].toPackedSet + b = [1, 2].toPackedSet + c = [1, 3].toPackedSet + assert a < b + assert not (b < b) + assert not (c < b) + + return s1 <= s2 and not (s2 <= s1) + +proc `==`*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if both `s1` and `s2` have the same elements and set size. + runnableExamples: + assert [1, 2].toPackedSet == [2, 1].toPackedSet + assert [1, 2].toPackedSet == [2, 1, 2].toPackedSet + + return s1 <= s2 and s2 <= s1 + +proc `$`*[A](s: PackedSet[A]): string = + ## Converts `s` to a string. + runnableExamples: + let a = [1, 2, 3].toPackedSet + assert $a == "{1, 2, 3}" + + dollarImpl() diff --git a/lib/std/paths.nim b/lib/std/paths.nim new file mode 100644 index 000000000..664dedd31 --- /dev/null +++ b/lib/std/paths.nim @@ -0,0 +1,302 @@ +## This module implements path handling. +## +## **See also:** +## * `files module <files.html>`_ for file access + +import std/private/osseps +export osseps + +import std/envvars +import std/private/osappdirs + +import std/[pathnorm, hashes, sugar, strutils] + +from std/private/ospaths2 import joinPath, splitPath, + ReadDirEffect, WriteDirEffect, + isAbsolute, relativePath, + normalizePathEnd, isRelativeTo, parentDir, + tailDir, isRootDir, parentDirs, `/../`, + extractFilename, lastPathPart, + changeFileExt, addFileExt, cmpPaths, splitFile, + unixToNativePath, absolutePath, normalizeExe, + normalizePath +export ReadDirEffect, WriteDirEffect + +type + Path* = distinct string + +func hash*(x: Path): Hash = + let x = x.string.dup(normalizePath) + if FileSystemCaseSensitive: + result = x.hash + else: + result = x.toLowerAscii.hash + +template `$`*(x: Path): string = + string(x) + +func `==`*(x, y: Path): bool {.inline.} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. + result = cmpPaths(x.string, y.string) == 0 + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +func add(x: var string, tail: string) = + var state = 0 + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and x.endsWith({DirSep, AltSep}) + normalizePathEnd(x, trailingSep=false) + addNormalizePath(tail, x, state, DirSep) + normalizePathEnd(x, trailingSep=trailingSep) + +func add*(x: var Path, y: Path) {.borrow.} + +func `/`*(head, tail: Path): Path {.inline.} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + Path(joinPath(head.string, tail.string)) + +func splitPath*(path: Path): tuple[head, tail: Path] {.inline.} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `add proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + let res = splitPath(path.string) + result = (Path(res.head), Path(res.tail)) + +func splitFile*(path: Path): tuple[dir, name: Path, ext: string] {.inline.} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + let res = splitFile(path.string) + result = (Path(res.dir), Path(res.name), res.ext) + +func isAbsolute*(path: Path): bool {.inline, raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + result = isAbsolute(path.string) + +proc relativePath*(path, base: Path, sep = DirSep): Path {.inline.} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + result = Path(relativePath(path.string, base.string, sep)) + +proc isRelativeTo*(path: Path, base: Path): bool {.inline.} = + ## Returns true if `path` is relative to `base`. + result = isRelativeTo(path.string, base.string) + + +func parentDir*(path: Path): Path {.inline.} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + result = Path(parentDir(path.string)) + +func tailDir*(path: Path): Path {.inline.} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + result = Path(tailDir(path.string)) + +func isRootDir*(path: Path): bool {.inline.} = + ## Checks whether a given `path` is a root directory. + result = isRootDir(path.string) + +iterator parentDirs*(path: Path, fromRoot=false, inclusive=true): Path = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + for p in parentDirs(path.string, fromRoot, inclusive): + yield Path(p) + +func `/../`*(head, tail: Path): Path {.inline.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + Path(`/../`(head.string, tail.string)) + +func extractFilename*(path: Path): Path {.inline.} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(extractFilename(path.string)) + +func lastPathPart*(path: Path): Path {.inline.} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(lastPathPart(path.string)) + +func changeFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + result = Path(changeFileExt(filename.string, ext)) + +func addFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + result = Path(addFileExt(filename.string, ext)) + +func unixToNativePath*(path: Path, drive=Path("")): Path {.inline.} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + result = Path(unixToNativePath(path.string, drive.string)) + +proc getCurrentDir*(): Path {.inline, tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc <appdirs.html#getHomeDir>`_ + ## * `getConfigDir proc <appdirs.html#getConfigDir>`_ + ## * `getTempDir proc <appdirs.html#getTempDir>`_ + ## * `setCurrentDir proc <dirs.html#setCurrentDir>`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + result = Path(ospaths2.getCurrentDir()) + +proc normalizeExe*(file: var Path) {.borrow.} + +proc normalizePath*(path: var Path) {.borrow.} + +proc normalizePathEnd*(path: var Path, trailingSep = false) {.borrow.} + +proc absolutePath*(path: Path, root = getCurrentDir()): Path = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizePath proc`_ + result = Path(absolutePath(path.string, root.string)) + +proc expandTildeImpl(path: string): string {. + tags: [ReadEnvEffect, ReadIOEffect].} = + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): + result = joinPath(getHomeDir(), path.substr(2)) + else: + # TODO: handle `~bob` and `~bob/` which means home of bob + result = path + +proc expandTilde*(path: Path): Path {.inline, + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with `getHomeDir() <appdirs.html#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## + ## Windows: this is still supported despite the Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. + runnableExamples: + import std/appdirs + assert expandTilde(Path("~") / Path("appname.cfg")) == getHomeDir() / Path("appname.cfg") + assert expandTilde(Path("~/foo/bar")) == getHomeDir() / Path("foo/bar") + assert expandTilde(Path("/foo/bar")) == Path("/foo/bar") + result = Path(expandTildeImpl(path.string)) diff --git a/lib/std/private/asciitables.nim b/lib/std/private/asciitables.nim new file mode 100644 index 000000000..cbc595651 --- /dev/null +++ b/lib/std/private/asciitables.nim @@ -0,0 +1,83 @@ +#[ +move to std/asciitables.nim once stable, or to a fusion package +once compiler can depend on fusion +]# + +type Cell* = object + text*: string + width*, row*, col*, ncols*, nrows*: int + +iterator parseTableCells*(s: string, delim = '\t'): Cell = + ## Iterates over all cells in a `delim`-delimited `s`, after a 1st + ## pass that computes number of rows, columns, and width of each column. + var widths: seq[int] + var cell: Cell + template update() = + if widths.len<=cell.col: + widths.setLen cell.col+1 + widths[cell.col] = cell.width + else: + widths[cell.col] = max(widths[cell.col], cell.width) + cell.width = 0 + + for a in s: + case a + of '\n': + update() + cell.col = 0 + cell.row.inc + elif a == delim: + update() + cell.col.inc + else: + # todo: consider multi-width chars when porting to non-ascii implementation + cell.width.inc + if s.len > 0 and s[^1] != '\n': + update() + + cell.ncols = widths.len + cell.nrows = cell.row + 1 + cell.row = 0 + cell.col = 0 + cell.width = 0 + + template update2() = + cell.width = widths[cell.col] + yield cell + cell.text = "" + cell.width = 0 + cell.col.inc + + template finishRow() = + for col in cell.col..<cell.ncols: + cell.col = col + update2() + cell.col = 0 + + for a in s: + case a + of '\n': + finishRow() + cell.row.inc + elif a == delim: + update2() + else: + cell.width.inc + cell.text.add a + + if s.len > 0 and s[^1] != '\n': + finishRow() + +proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string = + ## Formats a `delim`-delimited `s` representing a table; each cell is aligned + ## to a width that's computed for each column; consecutive columns are + ## delimited by `sep`, and alignment space is filled using `fill`. + ## More customized formatting can be done by calling `parseTableCells` directly. + for cell in parseTableCells(s, delim): + result.add cell.text + for i in cell.text.len..<cell.width: + result.add fill + if cell.col < cell.ncols-1: + result.add sep + if cell.col == cell.ncols-1 and cell.row < cell.nrows - 1: + result.add '\n' diff --git a/lib/std/private/bitops_utils.nim b/lib/std/private/bitops_utils.nim new file mode 100644 index 000000000..0b9484416 --- /dev/null +++ b/lib/std/private/bitops_utils.nim @@ -0,0 +1,22 @@ +template forwardImpl*(impl, arg) {.dirty.} = + when sizeof(x) <= 4: + when x is SomeSignedInt: + impl(cast[uint32](x.int32)) + else: + impl(x.uint32) + else: + when x is SomeSignedInt: + impl(cast[uint64](x.int64)) + else: + impl(x.uint64) + +# this could also be implemented via: +# import std/typetraits +# template castToUnsigned*(x: SomeInteger): auto = cast[toUnsigned(typeof(x))](x) + +template castToUnsigned*(x: int8): uint8 = cast[uint8](x) +template castToUnsigned*(x: int16): uint16 = cast[uint16](x) +template castToUnsigned*(x: int32): uint32 = cast[uint32](x) +template castToUnsigned*(x: int64): uint64 = cast[uint64](x) +template castToUnsigned*(x: int): uint = cast[uint](x) +template castToUnsigned*[T: SomeUnsignedInt](x: T): T = x diff --git a/lib/std/private/decode_helpers.nim b/lib/std/private/decode_helpers.nim new file mode 100644 index 000000000..f11e3060a --- /dev/null +++ b/lib/std/private/decode_helpers.nim @@ -0,0 +1,42 @@ +proc handleHexChar*(c: char, x: var int): bool {.inline.} = + ## Converts `%xx` hexadecimal to the ordinal number and adds the result to `x`. + ## Returns `true` if `c` is hexadecimal. + ## + ## When `c` is hexadecimal, the proc is equal to `x = x shl 4 + hex2Int(c)`. + runnableExamples: + var x = 0 + assert handleHexChar('a', x) + assert x == 10 + + assert handleHexChar('B', x) + assert x == 171 # 10 shl 4 + 11 + + assert not handleHexChar('?', x) + assert x == 171 # unchanged + result = true + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: + result = false + +proc handleHexChar*(c: char): int {.inline.} = + case c + of '0'..'9': result = (ord(c) - ord('0')) + of 'a'..'f': result = (ord(c) - ord('a') + 10) + of 'A'..'F': result = (ord(c) - ord('A') + 10) + else: discard + +proc decodePercent*(s: openArray[char], i: var int): char = + ## Converts `%xx` hexadecimal to the character with ordinal number `xx`. + ## + ## If `xx` is not a valid hexadecimal value, it is left intact: only the + ## leading `%` is returned as-is, and `xx` characters will be processed in the + ## next step (e.g. in `uri.decodeUrl`) as regular characters. + result = '%' + if i+2 < s.len: + var x = 0 + if handleHexChar(s[i+1], x) and handleHexChar(s[i+2], x): + result = chr(x) + inc(i, 2) diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim new file mode 100644 index 000000000..f2d0d25cb --- /dev/null +++ b/lib/std/private/digitsutils.nim @@ -0,0 +1,116 @@ +const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + +# Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c +# Generates: +# ```nim +# var res = "" +# for i in 0 .. 99: +# if i < 10: +# res.add "0" & $i +# else: +# res.add $i +# doAssert res == digits100 +# ``` + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int {.inline.} = + trailingZeros100[digits] + +when defined(js): + proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} + +func addChars[T](result: var string, x: T, start: int, n: int) {.inline.} = + let old = result.len + result.setLen old + n + template impl = + for i in 0..<n: result[old + i] = x[start + i] + when nimvm: impl + else: + when defined(js) or defined(nimscript): impl + else: + {.noSideEffect.}: + copyMem result[old].addr, x[start].unsafeAddr, n + +func addChars[T](result: var string, x: T) {.inline.} = + addChars(result, x, 0, x.len) + +func addIntImpl(result: var string, x: uint64) {.inline.} = + var tmp {.noinit.}: array[24, char] + var num = x + var next = tmp.len - 1 + const nbatch = 100 + + while num >= nbatch: + let originNum = num + num = num div nbatch + let index = int16((originNum - num * nbatch) shl 1) + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec(next, 2) + + # process last 1-2 digits + if num < 10: + tmp[next] = chr(ord('0') + num.uint8) + else: + let index = num * 2 + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec next + addChars(result, tmp, next, tmp.len - next) + +when not defined(nimHasEnforceNoRaises): + {.pragma: enforceNoRaises.} + +func addInt*(result: var string, x: uint64) {.enforceNoRaises.} = + when nimvm: addIntImpl(result, x) + else: + when not defined(js): addIntImpl(result, x) + else: + addChars(result, numToString(x)) + +proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = + ## Converts integer to its string representation and appends it to `result`. + runnableExamples: + var s = "foo" + s.addInt(45) + assert s == "foo45" + template impl = + var num: uint64 + if x < 0: + if x == low(int64): + num = cast[uint64](x) + else: + num = uint64(-x) + result.add '-' + else: + num = uint64(x) + addInt(result, num) + when nimvm: impl() + else: + when defined(js): + addChars(result, numToString(x)) + else: impl() + +proc addInt*(result: var string; x: int) {.inline, enforceNoRaises.} = + addInt(result, int64(x)) diff --git a/lib/std/private/dragonbox.nim b/lib/std/private/dragonbox.nim new file mode 100644 index 000000000..85ffea84a --- /dev/null +++ b/lib/std/private/dragonbox.nim @@ -0,0 +1,1325 @@ +## Copyright 2020 Junekey Jeon +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## char* output_end = Dtoa(buffer, value); +## +## Converts the given double-precision number into decimal form and stores the result in the given +## buffer. +## +## The buffer must be large enough, i.e. >= DtoaMinBufferLength. +## The output format is similar to printf("%g"). +## The output is _not_ null-terminted. +## +## The output is optimal, i.e. the output string +## 1. rounds back to the input number when read in (using round-to-nearest-even) +## 2. is as short as possible, +## 3. is as close to the input number as possible. +## +## Note: +## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + +const + dtoaMinBufferLength*: cint = 64 + +## This file contains an implementation of Junekey Jeon's Dragonbox algorithm. +## +## It is a simplified version of the reference implementation found here: +## https://github.com/jk-jeon/dragonbox +## +## The reference implementation also works with single-precision floating-point numbers and +## has options to configure the rounding mode. + +template dragonbox_Assert*(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType* = float + BitsType* = uint64 + +type + Double* = object + bits*: BitsType + +const ## = p (includes the hidden bit) + significandSize*: int32 = 53 + +const ## static constexpr int32_t MaxExponent = 1024 - 1 - (SignificandSize - 1); + ## static constexpr int32_t MinExponent = std::numeric_limits<value_type>::min_exponent - 1 - (SignificandSize - 1); + exponentBias*: int32 = 1024 - 1 + (significandSize - 1) + +const + maxIeeeExponent*: BitsType = BitsType(2 * 1024 - 1) + +const ## = 2^(p-1) + hiddenBit*: BitsType = BitsType(1) shl (significandSize - 1) + +const ## = 2^(p-1) - 1 + significandMask*: BitsType = hiddenBit - 1 + +const + exponentMask*: BitsType = maxIeeeExponent shl (significandSize - 1) + +const + signMask*: BitsType = not (not BitsType(0) shr 1) + +proc constructDouble*(bits: BitsType): Double = + result.bits = bits + +proc constructDouble*(value: ValueType): Double = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent*(this: Double): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero*(this: Double): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit*(this: Double): int {.noSideEffect.} = + return ord((this.bits and signMask) != 0) + + +# ================================================================================================== +# +# ================================================================================================== +## namespace +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2*(x: int32; n: int32): int32 {.inline.} = + return x shr n + +proc floorLog2Pow10*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1233) + dragonbox_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc floorLog10Pow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611, 22) + +proc floorLog10ThreeQuartersPow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611 - 524031, 22) + +# ================================================================================================== +# +# ================================================================================================== + +type + uint64x2* {.bycopy.} = object + hi*: uint64 + lo*: uint64 + + +proc computePow10*(k: int32): uint64x2 {.inline.} = + const + kMin: int32 = -292 + const + kMax: int32 = 326 + const + pow10: array[kMax - kMin + 1, uint64x2] = [ + uint64x2(hi: 0xFF77B1FCBEBCDC4F'u, lo: 0x25E8E89C13BB0F7B'u), + uint64x2(hi: 0x9FAACF3DF73609B1'u, lo: 0x77B191618C54E9AD'u), + uint64x2(hi: 0xC795830D75038C1D'u, lo: 0xD59DF5B9EF6A2418'u), + uint64x2(hi: 0xF97AE3D0D2446F25'u, lo: 0x4B0573286B44AD1E'u), + uint64x2(hi: 0x9BECCE62836AC577'u, lo: 0x4EE367F9430AEC33'u), + uint64x2(hi: 0xC2E801FB244576D5'u, lo: 0x229C41F793CDA740'u), + uint64x2(hi: 0xF3A20279ED56D48A'u, lo: 0x6B43527578C11110'u), + uint64x2(hi: 0x9845418C345644D6'u, lo: 0x830A13896B78AAAA'u), + uint64x2(hi: 0xBE5691EF416BD60C'u, lo: 0x23CC986BC656D554'u), + uint64x2(hi: 0xEDEC366B11C6CB8F'u, lo: 0x2CBFBE86B7EC8AA9'u), + uint64x2(hi: 0x94B3A202EB1C3F39'u, lo: 0x7BF7D71432F3D6AA'u), + uint64x2(hi: 0xB9E08A83A5E34F07'u, lo: 0xDAF5CCD93FB0CC54'u), + uint64x2(hi: 0xE858AD248F5C22C9'u, lo: 0xD1B3400F8F9CFF69'u), + uint64x2(hi: 0x91376C36D99995BE'u, lo: 0x23100809B9C21FA2'u), + uint64x2(hi: 0xB58547448FFFFB2D'u, lo: 0xABD40A0C2832A78B'u), + uint64x2(hi: 0xE2E69915B3FFF9F9'u, lo: 0x16C90C8F323F516D'u), + uint64x2(hi: 0x8DD01FAD907FFC3B'u, lo: 0xAE3DA7D97F6792E4'u), + uint64x2(hi: 0xB1442798F49FFB4A'u, lo: 0x99CD11CFDF41779D'u), + uint64x2(hi: 0xDD95317F31C7FA1D'u, lo: 0x40405643D711D584'u), + uint64x2(hi: 0x8A7D3EEF7F1CFC52'u, lo: 0x482835EA666B2573'u), + uint64x2(hi: 0xAD1C8EAB5EE43B66'u, lo: 0xDA3243650005EED0'u), + uint64x2(hi: 0xD863B256369D4A40'u, lo: 0x90BED43E40076A83'u), + uint64x2(hi: 0x873E4F75E2224E68'u, lo: 0x5A7744A6E804A292'u), + uint64x2(hi: 0xA90DE3535AAAE202'u, lo: 0x711515D0A205CB37'u), + uint64x2(hi: 0xD3515C2831559A83'u, lo: 0x0D5A5B44CA873E04'u), + uint64x2(hi: 0x8412D9991ED58091'u, lo: 0xE858790AFE9486C3'u), + uint64x2(hi: 0xA5178FFF668AE0B6'u, lo: 0x626E974DBE39A873'u), + uint64x2(hi: 0xCE5D73FF402D98E3'u, lo: 0xFB0A3D212DC81290'u), + uint64x2(hi: 0x80FA687F881C7F8E'u, lo: 0x7CE66634BC9D0B9A'u), + uint64x2(hi: 0xA139029F6A239F72'u, lo: 0x1C1FFFC1EBC44E81'u), + uint64x2(hi: 0xC987434744AC874E'u, lo: 0xA327FFB266B56221'u), + uint64x2(hi: 0xFBE9141915D7A922'u, lo: 0x4BF1FF9F0062BAA9'u), + uint64x2(hi: 0x9D71AC8FADA6C9B5'u, lo: 0x6F773FC3603DB4AA'u), + uint64x2(hi: 0xC4CE17B399107C22'u, lo: 0xCB550FB4384D21D4'u), + uint64x2(hi: 0xF6019DA07F549B2B'u, lo: 0x7E2A53A146606A49'u), + uint64x2(hi: 0x99C102844F94E0FB'u, lo: 0x2EDA7444CBFC426E'u), + uint64x2(hi: 0xC0314325637A1939'u, lo: 0xFA911155FEFB5309'u), + uint64x2(hi: 0xF03D93EEBC589F88'u, lo: 0x793555AB7EBA27CB'u), + uint64x2(hi: 0x96267C7535B763B5'u, lo: 0x4BC1558B2F3458DF'u), + uint64x2(hi: 0xBBB01B9283253CA2'u, lo: 0x9EB1AAEDFB016F17'u), + uint64x2(hi: 0xEA9C227723EE8BCB'u, lo: 0x465E15A979C1CADD'u), + uint64x2(hi: 0x92A1958A7675175F'u, lo: 0x0BFACD89EC191ECA'u), + uint64x2(hi: 0xB749FAED14125D36'u, lo: 0xCEF980EC671F667C'u), + uint64x2(hi: 0xE51C79A85916F484'u, lo: 0x82B7E12780E7401B'u), + uint64x2(hi: 0x8F31CC0937AE58D2'u, lo: 0xD1B2ECB8B0908811'u), + uint64x2(hi: 0xB2FE3F0B8599EF07'u, lo: 0x861FA7E6DCB4AA16'u), + uint64x2(hi: 0xDFBDCECE67006AC9'u, lo: 0x67A791E093E1D49B'u), + uint64x2(hi: 0x8BD6A141006042BD'u, lo: 0xE0C8BB2C5C6D24E1'u), + uint64x2(hi: 0xAECC49914078536D'u, lo: 0x58FAE9F773886E19'u), + uint64x2(hi: 0xDA7F5BF590966848'u, lo: 0xAF39A475506A899F'u), + uint64x2(hi: 0x888F99797A5E012D'u, lo: 0x6D8406C952429604'u), + uint64x2(hi: 0xAAB37FD7D8F58178'u, lo: 0xC8E5087BA6D33B84'u), + uint64x2(hi: 0xD5605FCDCF32E1D6'u, lo: 0xFB1E4A9A90880A65'u), + uint64x2(hi: 0x855C3BE0A17FCD26'u, lo: 0x5CF2EEA09A550680'u), + uint64x2(hi: 0xA6B34AD8C9DFC06F'u, lo: 0xF42FAA48C0EA481F'u), + uint64x2(hi: 0xD0601D8EFC57B08B'u, lo: 0xF13B94DAF124DA27'u), + uint64x2(hi: 0x823C12795DB6CE57'u, lo: 0x76C53D08D6B70859'u), + uint64x2(hi: 0xA2CB1717B52481ED'u, lo: 0x54768C4B0C64CA6F'u), + uint64x2(hi: 0xCB7DDCDDA26DA268'u, lo: 0xA9942F5DCF7DFD0A'u), + uint64x2(hi: 0xFE5D54150B090B02'u, lo: 0xD3F93B35435D7C4D'u), + uint64x2(hi: 0x9EFA548D26E5A6E1'u, lo: 0xC47BC5014A1A6DB0'u), + uint64x2(hi: 0xC6B8E9B0709F109A'u, lo: 0x359AB6419CA1091C'u), + uint64x2(hi: 0xF867241C8CC6D4C0'u, lo: 0xC30163D203C94B63'u), + uint64x2(hi: 0x9B407691D7FC44F8'u, lo: 0x79E0DE63425DCF1E'u), + uint64x2(hi: 0xC21094364DFB5636'u, lo: 0x985915FC12F542E5'u), + uint64x2(hi: 0xF294B943E17A2BC4'u, lo: 0x3E6F5B7B17B2939E'u), + uint64x2(hi: 0x979CF3CA6CEC5B5A'u, lo: 0xA705992CEECF9C43'u), + uint64x2(hi: 0xBD8430BD08277231'u, lo: 0x50C6FF782A838354'u), + uint64x2(hi: 0xECE53CEC4A314EBD'u, lo: 0xA4F8BF5635246429'u), + uint64x2(hi: 0x940F4613AE5ED136'u, lo: 0x871B7795E136BE9A'u), + uint64x2(hi: 0xB913179899F68584'u, lo: 0x28E2557B59846E40'u), + uint64x2(hi: 0xE757DD7EC07426E5'u, lo: 0x331AEADA2FE589D0'u), + uint64x2(hi: 0x9096EA6F3848984F'u, lo: 0x3FF0D2C85DEF7622'u), + uint64x2(hi: 0xB4BCA50B065ABE63'u, lo: 0x0FED077A756B53AA'u), + uint64x2(hi: 0xE1EBCE4DC7F16DFB'u, lo: 0xD3E8495912C62895'u), + uint64x2(hi: 0x8D3360F09CF6E4BD'u, lo: 0x64712DD7ABBBD95D'u), + uint64x2(hi: 0xB080392CC4349DEC'u, lo: 0xBD8D794D96AACFB4'u), + uint64x2(hi: 0xDCA04777F541C567'u, lo: 0xECF0D7A0FC5583A1'u), + uint64x2(hi: 0x89E42CAAF9491B60'u, lo: 0xF41686C49DB57245'u), + uint64x2(hi: 0xAC5D37D5B79B6239'u, lo: 0x311C2875C522CED6'u), + uint64x2(hi: 0xD77485CB25823AC7'u, lo: 0x7D633293366B828C'u), + uint64x2(hi: 0x86A8D39EF77164BC'u, lo: 0xAE5DFF9C02033198'u), + uint64x2(hi: 0xA8530886B54DBDEB'u, lo: 0xD9F57F830283FDFD'u), + uint64x2(hi: 0xD267CAA862A12D66'u, lo: 0xD072DF63C324FD7C'u), + uint64x2(hi: 0x8380DEA93DA4BC60'u, lo: 0x4247CB9E59F71E6E'u), + uint64x2(hi: 0xA46116538D0DEB78'u, lo: 0x52D9BE85F074E609'u), + uint64x2(hi: 0xCD795BE870516656'u, lo: 0x67902E276C921F8C'u), + uint64x2(hi: 0x806BD9714632DFF6'u, lo: 0x00BA1CD8A3DB53B7'u), + uint64x2(hi: 0xA086CFCD97BF97F3'u, lo: 0x80E8A40ECCD228A5'u), + uint64x2(hi: 0xC8A883C0FDAF7DF0'u, lo: 0x6122CD128006B2CE'u), + uint64x2(hi: 0xFAD2A4B13D1B5D6C'u, lo: 0x796B805720085F82'u), + uint64x2(hi: 0x9CC3A6EEC6311A63'u, lo: 0xCBE3303674053BB1'u), + uint64x2(hi: 0xC3F490AA77BD60FC'u, lo: 0xBEDBFC4411068A9D'u), + uint64x2(hi: 0xF4F1B4D515ACB93B'u, lo: 0xEE92FB5515482D45'u), + uint64x2(hi: 0x991711052D8BF3C5'u, lo: 0x751BDD152D4D1C4B'u), + uint64x2(hi: 0xBF5CD54678EEF0B6'u, lo: 0xD262D45A78A0635E'u), + uint64x2(hi: 0xEF340A98172AACE4'u, lo: 0x86FB897116C87C35'u), + uint64x2(hi: 0x9580869F0E7AAC0E'u, lo: 0xD45D35E6AE3D4DA1'u), + uint64x2(hi: 0xBAE0A846D2195712'u, lo: 0x8974836059CCA10A'u), + uint64x2(hi: 0xE998D258869FACD7'u, lo: 0x2BD1A438703FC94C'u), + uint64x2(hi: 0x91FF83775423CC06'u, lo: 0x7B6306A34627DDD0'u), + uint64x2(hi: 0xB67F6455292CBF08'u, lo: 0x1A3BC84C17B1D543'u), + uint64x2(hi: 0xE41F3D6A7377EECA'u, lo: 0x20CABA5F1D9E4A94'u), + uint64x2(hi: 0x8E938662882AF53E'u, lo: 0x547EB47B7282EE9D'u), + uint64x2(hi: 0xB23867FB2A35B28D'u, lo: 0xE99E619A4F23AA44'u), + uint64x2(hi: 0xDEC681F9F4C31F31'u, lo: 0x6405FA00E2EC94D5'u), + uint64x2(hi: 0x8B3C113C38F9F37E'u, lo: 0xDE83BC408DD3DD05'u), + uint64x2(hi: 0xAE0B158B4738705E'u, lo: 0x9624AB50B148D446'u), + uint64x2(hi: 0xD98DDAEE19068C76'u, lo: 0x3BADD624DD9B0958'u), + uint64x2(hi: 0x87F8A8D4CFA417C9'u, lo: 0xE54CA5D70A80E5D7'u), + uint64x2(hi: 0xA9F6D30A038D1DBC'u, lo: 0x5E9FCF4CCD211F4D'u), + uint64x2(hi: 0xD47487CC8470652B'u, lo: 0x7647C32000696720'u), + uint64x2(hi: 0x84C8D4DFD2C63F3B'u, lo: 0x29ECD9F40041E074'u), + uint64x2(hi: 0xA5FB0A17C777CF09'u, lo: 0xF468107100525891'u), + uint64x2(hi: 0xCF79CC9DB955C2CC'u, lo: 0x7182148D4066EEB5'u), + uint64x2(hi: 0x81AC1FE293D599BF'u, lo: 0xC6F14CD848405531'u), + uint64x2(hi: 0xA21727DB38CB002F'u, lo: 0xB8ADA00E5A506A7D'u), + uint64x2(hi: 0xCA9CF1D206FDC03B'u, lo: 0xA6D90811F0E4851D'u), + uint64x2(hi: 0xFD442E4688BD304A'u, lo: 0x908F4A166D1DA664'u), + uint64x2(hi: 0x9E4A9CEC15763E2E'u, lo: 0x9A598E4E043287FF'u), + uint64x2(hi: 0xC5DD44271AD3CDBA'u, lo: 0x40EFF1E1853F29FE'u), + uint64x2(hi: 0xF7549530E188C128'u, lo: 0xD12BEE59E68EF47D'u), + uint64x2(hi: 0x9A94DD3E8CF578B9'u, lo: 0x82BB74F8301958CF'u), + uint64x2(hi: 0xC13A148E3032D6E7'u, lo: 0xE36A52363C1FAF02'u), + uint64x2(hi: 0xF18899B1BC3F8CA1'u, lo: 0xDC44E6C3CB279AC2'u), + uint64x2(hi: 0x96F5600F15A7B7E5'u, lo: 0x29AB103A5EF8C0BA'u), + uint64x2(hi: 0xBCB2B812DB11A5DE'u, lo: 0x7415D448F6B6F0E8'u), + uint64x2(hi: 0xEBDF661791D60F56'u, lo: 0x111B495B3464AD22'u), + uint64x2(hi: 0x936B9FCEBB25C995'u, lo: 0xCAB10DD900BEEC35'u), + uint64x2(hi: 0xB84687C269EF3BFB'u, lo: 0x3D5D514F40EEA743'u), + uint64x2(hi: 0xE65829B3046B0AFA'u, lo: 0x0CB4A5A3112A5113'u), + uint64x2(hi: 0x8FF71A0FE2C2E6DC'u, lo: 0x47F0E785EABA72AC'u), + uint64x2(hi: 0xB3F4E093DB73A093'u, lo: 0x59ED216765690F57'u), + uint64x2(hi: 0xE0F218B8D25088B8'u, lo: 0x306869C13EC3532D'u), + uint64x2(hi: 0x8C974F7383725573'u, lo: 0x1E414218C73A13FC'u), + uint64x2(hi: 0xAFBD2350644EEACF'u, lo: 0xE5D1929EF90898FB'u), + uint64x2(hi: 0xDBAC6C247D62A583'u, lo: 0xDF45F746B74ABF3A'u), + uint64x2(hi: 0x894BC396CE5DA772'u, lo: 0x6B8BBA8C328EB784'u), + uint64x2(hi: 0xAB9EB47C81F5114F'u, lo: 0x066EA92F3F326565'u), + uint64x2(hi: 0xD686619BA27255A2'u, lo: 0xC80A537B0EFEFEBE'u), + uint64x2(hi: 0x8613FD0145877585'u, lo: 0xBD06742CE95F5F37'u), + uint64x2(hi: 0xA798FC4196E952E7'u, lo: 0x2C48113823B73705'u), + uint64x2(hi: 0xD17F3B51FCA3A7A0'u, lo: 0xF75A15862CA504C6'u), + uint64x2(hi: 0x82EF85133DE648C4'u, lo: 0x9A984D73DBE722FC'u), + uint64x2(hi: 0xA3AB66580D5FDAF5'u, lo: 0xC13E60D0D2E0EBBB'u), + uint64x2(hi: 0xCC963FEE10B7D1B3'u, lo: 0x318DF905079926A9'u), + uint64x2(hi: 0xFFBBCFE994E5C61F'u, lo: 0xFDF17746497F7053'u), + uint64x2(hi: 0x9FD561F1FD0F9BD3'u, lo: 0xFEB6EA8BEDEFA634'u), + uint64x2(hi: 0xC7CABA6E7C5382C8'u, lo: 0xFE64A52EE96B8FC1'u), + uint64x2(hi: 0xF9BD690A1B68637B'u, lo: 0x3DFDCE7AA3C673B1'u), + uint64x2(hi: 0x9C1661A651213E2D'u, lo: 0x06BEA10CA65C084F'u), + uint64x2(hi: 0xC31BFA0FE5698DB8'u, lo: 0x486E494FCFF30A63'u), + uint64x2(hi: 0xF3E2F893DEC3F126'u, lo: 0x5A89DBA3C3EFCCFB'u), + uint64x2(hi: 0x986DDB5C6B3A76B7'u, lo: 0xF89629465A75E01D'u), + uint64x2(hi: 0xBE89523386091465'u, lo: 0xF6BBB397F1135824'u), + uint64x2(hi: 0xEE2BA6C0678B597F'u, lo: 0x746AA07DED582E2D'u), + uint64x2(hi: 0x94DB483840B717EF'u, lo: 0xA8C2A44EB4571CDD'u), + uint64x2(hi: 0xBA121A4650E4DDEB'u, lo: 0x92F34D62616CE414'u), + uint64x2(hi: 0xE896A0D7E51E1566'u, lo: 0x77B020BAF9C81D18'u), + uint64x2(hi: 0x915E2486EF32CD60'u, lo: 0x0ACE1474DC1D122F'u), + uint64x2(hi: 0xB5B5ADA8AAFF80B8'u, lo: 0x0D819992132456BB'u), + uint64x2(hi: 0xE3231912D5BF60E6'u, lo: 0x10E1FFF697ED6C6A'u), + uint64x2(hi: 0x8DF5EFABC5979C8F'u, lo: 0xCA8D3FFA1EF463C2'u), + uint64x2(hi: 0xB1736B96B6FD83B3'u, lo: 0xBD308FF8A6B17CB3'u), + uint64x2(hi: 0xDDD0467C64BCE4A0'u, lo: 0xAC7CB3F6D05DDBDF'u), + uint64x2(hi: 0x8AA22C0DBEF60EE4'u, lo: 0x6BCDF07A423AA96C'u), + uint64x2(hi: 0xAD4AB7112EB3929D'u, lo: 0x86C16C98D2C953C7'u), + uint64x2(hi: 0xD89D64D57A607744'u, lo: 0xE871C7BF077BA8B8'u), + uint64x2(hi: 0x87625F056C7C4A8B'u, lo: 0x11471CD764AD4973'u), + uint64x2(hi: 0xA93AF6C6C79B5D2D'u, lo: 0xD598E40D3DD89BD0'u), + uint64x2(hi: 0xD389B47879823479'u, lo: 0x4AFF1D108D4EC2C4'u), + uint64x2(hi: 0x843610CB4BF160CB'u, lo: 0xCEDF722A585139BB'u), + uint64x2(hi: 0xA54394FE1EEDB8FE'u, lo: 0xC2974EB4EE658829'u), + uint64x2(hi: 0xCE947A3DA6A9273E'u, lo: 0x733D226229FEEA33'u), + uint64x2(hi: 0x811CCC668829B887'u, lo: 0x0806357D5A3F5260'u), + uint64x2(hi: 0xA163FF802A3426A8'u, lo: 0xCA07C2DCB0CF26F8'u), + uint64x2(hi: 0xC9BCFF6034C13052'u, lo: 0xFC89B393DD02F0B6'u), + uint64x2(hi: 0xFC2C3F3841F17C67'u, lo: 0xBBAC2078D443ACE3'u), + uint64x2(hi: 0x9D9BA7832936EDC0'u, lo: 0xD54B944B84AA4C0E'u), + uint64x2(hi: 0xC5029163F384A931'u, lo: 0x0A9E795E65D4DF12'u), + uint64x2(hi: 0xF64335BCF065D37D'u, lo: 0x4D4617B5FF4A16D6'u), + uint64x2(hi: 0x99EA0196163FA42E'u, lo: 0x504BCED1BF8E4E46'u), + uint64x2(hi: 0xC06481FB9BCF8D39'u, lo: 0xE45EC2862F71E1D7'u), + uint64x2(hi: 0xF07DA27A82C37088'u, lo: 0x5D767327BB4E5A4D'u), + uint64x2(hi: 0x964E858C91BA2655'u, lo: 0x3A6A07F8D510F870'u), + uint64x2(hi: 0xBBE226EFB628AFEA'u, lo: 0x890489F70A55368C'u), + uint64x2(hi: 0xEADAB0ABA3B2DBE5'u, lo: 0x2B45AC74CCEA842F'u), + uint64x2(hi: 0x92C8AE6B464FC96F'u, lo: 0x3B0B8BC90012929E'u), + uint64x2(hi: 0xB77ADA0617E3BBCB'u, lo: 0x09CE6EBB40173745'u), + uint64x2(hi: 0xE55990879DDCAABD'u, lo: 0xCC420A6A101D0516'u), + uint64x2(hi: 0x8F57FA54C2A9EAB6'u, lo: 0x9FA946824A12232E'u), + uint64x2(hi: 0xB32DF8E9F3546564'u, lo: 0x47939822DC96ABFA'u), + uint64x2(hi: 0xDFF9772470297EBD'u, lo: 0x59787E2B93BC56F8'u), + uint64x2(hi: 0x8BFBEA76C619EF36'u, lo: 0x57EB4EDB3C55B65B'u), + uint64x2(hi: 0xAEFAE51477A06B03'u, lo: 0xEDE622920B6B23F2'u), + uint64x2(hi: 0xDAB99E59958885C4'u, lo: 0xE95FAB368E45ECEE'u), + uint64x2(hi: 0x88B402F7FD75539B'u, lo: 0x11DBCB0218EBB415'u), + uint64x2(hi: 0xAAE103B5FCD2A881'u, lo: 0xD652BDC29F26A11A'u), + uint64x2(hi: 0xD59944A37C0752A2'u, lo: 0x4BE76D3346F04960'u), + uint64x2(hi: 0x857FCAE62D8493A5'u, lo: 0x6F70A4400C562DDC'u), + uint64x2(hi: 0xA6DFBD9FB8E5B88E'u, lo: 0xCB4CCD500F6BB953'u), + uint64x2(hi: 0xD097AD07A71F26B2'u, lo: 0x7E2000A41346A7A8'u), + uint64x2(hi: 0x825ECC24C873782F'u, lo: 0x8ED400668C0C28C9'u), + uint64x2(hi: 0xA2F67F2DFA90563B'u, lo: 0x728900802F0F32FB'u), + uint64x2(hi: 0xCBB41EF979346BCA'u, lo: 0x4F2B40A03AD2FFBA'u), + uint64x2(hi: 0xFEA126B7D78186BC'u, lo: 0xE2F610C84987BFA9'u), + uint64x2(hi: 0x9F24B832E6B0F436'u, lo: 0x0DD9CA7D2DF4D7CA'u), + uint64x2(hi: 0xC6EDE63FA05D3143'u, lo: 0x91503D1C79720DBC'u), + uint64x2(hi: 0xF8A95FCF88747D94'u, lo: 0x75A44C6397CE912B'u), + uint64x2(hi: 0x9B69DBE1B548CE7C'u, lo: 0xC986AFBE3EE11ABB'u), + uint64x2(hi: 0xC24452DA229B021B'u, lo: 0xFBE85BADCE996169'u), + uint64x2(hi: 0xF2D56790AB41C2A2'u, lo: 0xFAE27299423FB9C4'u), + uint64x2(hi: 0x97C560BA6B0919A5'u, lo: 0xDCCD879FC967D41B'u), + uint64x2(hi: 0xBDB6B8E905CB600F'u, lo: 0x5400E987BBC1C921'u), + uint64x2(hi: 0xED246723473E3813'u, lo: 0x290123E9AAB23B69'u), + uint64x2(hi: 0x9436C0760C86E30B'u, lo: 0xF9A0B6720AAF6522'u), + uint64x2(hi: 0xB94470938FA89BCE'u, lo: 0xF808E40E8D5B3E6A'u), + uint64x2(hi: 0xE7958CB87392C2C2'u, lo: 0xB60B1D1230B20E05'u), + uint64x2(hi: 0x90BD77F3483BB9B9'u, lo: 0xB1C6F22B5E6F48C3'u), + uint64x2(hi: 0xB4ECD5F01A4AA828'u, lo: 0x1E38AEB6360B1AF4'u), + uint64x2(hi: 0xE2280B6C20DD5232'u, lo: 0x25C6DA63C38DE1B1'u), + uint64x2(hi: 0x8D590723948A535F'u, lo: 0x579C487E5A38AD0F'u), + uint64x2(hi: 0xB0AF48EC79ACE837'u, lo: 0x2D835A9DF0C6D852'u), + uint64x2(hi: 0xDCDB1B2798182244'u, lo: 0xF8E431456CF88E66'u), + uint64x2(hi: 0x8A08F0F8BF0F156B'u, lo: 0x1B8E9ECB641B5900'u), + uint64x2(hi: 0xAC8B2D36EED2DAC5'u, lo: 0xE272467E3D222F40'u), + uint64x2(hi: 0xD7ADF884AA879177'u, lo: 0x5B0ED81DCC6ABB10'u), + uint64x2(hi: 0x86CCBB52EA94BAEA'u, lo: 0x98E947129FC2B4EA'u), + uint64x2(hi: 0xA87FEA27A539E9A5'u, lo: 0x3F2398D747B36225'u), + uint64x2(hi: 0xD29FE4B18E88640E'u, lo: 0x8EEC7F0D19A03AAE'u), + uint64x2(hi: 0x83A3EEEEF9153E89'u, lo: 0x1953CF68300424AD'u), + uint64x2(hi: 0xA48CEAAAB75A8E2B'u, lo: 0x5FA8C3423C052DD8'u), + uint64x2(hi: 0xCDB02555653131B6'u, lo: 0x3792F412CB06794E'u), + uint64x2(hi: 0x808E17555F3EBF11'u, lo: 0xE2BBD88BBEE40BD1'u), + uint64x2(hi: 0xA0B19D2AB70E6ED6'u, lo: 0x5B6ACEAEAE9D0EC5'u), + uint64x2(hi: 0xC8DE047564D20A8B'u, lo: 0xF245825A5A445276'u), + uint64x2(hi: 0xFB158592BE068D2E'u, lo: 0xEED6E2F0F0D56713'u), + uint64x2(hi: 0x9CED737BB6C4183D'u, lo: 0x55464DD69685606C'u), + uint64x2(hi: 0xC428D05AA4751E4C'u, lo: 0xAA97E14C3C26B887'u), + uint64x2(hi: 0xF53304714D9265DF'u, lo: 0xD53DD99F4B3066A9'u), + uint64x2(hi: 0x993FE2C6D07B7FAB'u, lo: 0xE546A8038EFE402A'u), + uint64x2(hi: 0xBF8FDB78849A5F96'u, lo: 0xDE98520472BDD034'u), + uint64x2(hi: 0xEF73D256A5C0F77C'u, lo: 0x963E66858F6D4441'u), + uint64x2(hi: 0x95A8637627989AAD'u, lo: 0xDDE7001379A44AA9'u), + uint64x2(hi: 0xBB127C53B17EC159'u, lo: 0x5560C018580D5D53'u), + uint64x2(hi: 0xE9D71B689DDE71AF'u, lo: 0xAAB8F01E6E10B4A7'u), + uint64x2(hi: 0x9226712162AB070D'u, lo: 0xCAB3961304CA70E9'u), + uint64x2(hi: 0xB6B00D69BB55C8D1'u, lo: 0x3D607B97C5FD0D23'u), + uint64x2(hi: 0xE45C10C42A2B3B05'u, lo: 0x8CB89A7DB77C506B'u), + uint64x2(hi: 0x8EB98A7A9A5B04E3'u, lo: 0x77F3608E92ADB243'u), + uint64x2(hi: 0xB267ED1940F1C61C'u, lo: 0x55F038B237591ED4'u), + uint64x2(hi: 0xDF01E85F912E37A3'u, lo: 0x6B6C46DEC52F6689'u), + uint64x2(hi: 0x8B61313BBABCE2C6'u, lo: 0x2323AC4B3B3DA016'u), + uint64x2(hi: 0xAE397D8AA96C1B77'u, lo: 0xABEC975E0A0D081B'u), + uint64x2(hi: 0xD9C7DCED53C72255'u, lo: 0x96E7BD358C904A22'u), + uint64x2(hi: 0x881CEA14545C7575'u, lo: 0x7E50D64177DA2E55'u), + uint64x2(hi: 0xAA242499697392D2'u, lo: 0xDDE50BD1D5D0B9EA'u), + uint64x2(hi: 0xD4AD2DBFC3D07787'u, lo: 0x955E4EC64B44E865'u), + uint64x2(hi: 0x84EC3C97DA624AB4'u, lo: 0xBD5AF13BEF0B113F'u), + uint64x2(hi: 0xA6274BBDD0FADD61'u, lo: 0xECB1AD8AEACDD58F'u), + uint64x2(hi: 0xCFB11EAD453994BA'u, lo: 0x67DE18EDA5814AF3'u), + uint64x2(hi: 0x81CEB32C4B43FCF4'u, lo: 0x80EACF948770CED8'u), + uint64x2(hi: 0xA2425FF75E14FC31'u, lo: 0xA1258379A94D028E'u), + uint64x2(hi: 0xCAD2F7F5359A3B3E'u, lo: 0x096EE45813A04331'u), + uint64x2(hi: 0xFD87B5F28300CA0D'u, lo: 0x8BCA9D6E188853FD'u), + uint64x2(hi: 0x9E74D1B791E07E48'u, lo: 0x775EA264CF55347E'u), + uint64x2(hi: 0xC612062576589DDA'u, lo: 0x95364AFE032A819E'u), + uint64x2(hi: 0xF79687AED3EEC551'u, lo: 0x3A83DDBD83F52205'u), + uint64x2(hi: 0x9ABE14CD44753B52'u, lo: 0xC4926A9672793543'u), + uint64x2(hi: 0xC16D9A0095928A27'u, lo: 0x75B7053C0F178294'u), + uint64x2(hi: 0xF1C90080BAF72CB1'u, lo: 0x5324C68B12DD6339'u), + uint64x2(hi: 0x971DA05074DA7BEE'u, lo: 0xD3F6FC16EBCA5E04'u), + uint64x2(hi: 0xBCE5086492111AEA'u, lo: 0x88F4BB1CA6BCF585'u), + uint64x2(hi: 0xEC1E4A7DB69561A5'u, lo: 0x2B31E9E3D06C32E6'u), + uint64x2(hi: 0x9392EE8E921D5D07'u, lo: 0x3AFF322E62439FD0'u), + uint64x2(hi: 0xB877AA3236A4B449'u, lo: 0x09BEFEB9FAD487C3'u), + uint64x2(hi: 0xE69594BEC44DE15B'u, lo: 0x4C2EBE687989A9B4'u), + uint64x2(hi: 0x901D7CF73AB0ACD9'u, lo: 0x0F9D37014BF60A11'u), + uint64x2(hi: 0xB424DC35095CD80F'u, lo: 0x538484C19EF38C95'u), + uint64x2(hi: 0xE12E13424BB40E13'u, lo: 0x2865A5F206B06FBA'u), + uint64x2(hi: 0x8CBCCC096F5088CB'u, lo: 0xF93F87B7442E45D4'u), + uint64x2(hi: 0xAFEBFF0BCB24AAFE'u, lo: 0xF78F69A51539D749'u), + uint64x2(hi: 0xDBE6FECEBDEDD5BE'u, lo: 0xB573440E5A884D1C'u), + uint64x2(hi: 0x89705F4136B4A597'u, lo: 0x31680A88F8953031'u), + uint64x2(hi: 0xABCC77118461CEFC'u, lo: 0xFDC20D2B36BA7C3E'u), + uint64x2(hi: 0xD6BF94D5E57A42BC'u, lo: 0x3D32907604691B4D'u), + uint64x2(hi: 0x8637BD05AF6C69B5'u, lo: 0xA63F9A49C2C1B110'u), + uint64x2(hi: 0xA7C5AC471B478423'u, lo: 0x0FCF80DC33721D54'u), + uint64x2(hi: 0xD1B71758E219652B'u, lo: 0xD3C36113404EA4A9'u), + uint64x2(hi: 0x83126E978D4FDF3B'u, lo: 0x645A1CAC083126EA'u), + uint64x2(hi: 0xA3D70A3D70A3D70A'u, lo: 0x3D70A3D70A3D70A4'u), + uint64x2(hi: 0xCCCCCCCCCCCCCCCC'u, lo: 0xCCCCCCCCCCCCCCCD'u), + uint64x2(hi: 0x8000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC800000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xFA00000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9C40000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC350000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xF424000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9896800000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBEBC200000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xEE6B280000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9502F90000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBA43B74000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE8D4A51000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9184E72A00000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB5E620F480000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE35FA931A0000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8E1BC9BF04000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB1A2BC2EC5000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xDE0B6B3A76400000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8AC7230489E80000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xAD78EBC5AC620000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD8D726B7177A8000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x878678326EAC9000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA968163F0A57B400'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD3C21BCECCEDA100'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x84595161401484A0'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA56FA5B99019A5C8'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xCECB8F27F4200F3A'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x813F3978F8940984'u, lo: 0x4000000000000000'u), + uint64x2(hi: 0xA18F07D736B90BE5'u, lo: 0x5000000000000000'u), + uint64x2(hi: 0xC9F2C9CD04674EDE'u, lo: 0xA400000000000000'u), + uint64x2(hi: 0xFC6F7C4045812296'u, lo: 0x4D00000000000000'u), + uint64x2(hi: 0x9DC5ADA82B70B59D'u, lo: 0xF020000000000000'u), + uint64x2(hi: 0xC5371912364CE305'u, lo: 0x6C28000000000000'u), + uint64x2(hi: 0xF684DF56C3E01BC6'u, lo: 0xC732000000000000'u), + uint64x2(hi: 0x9A130B963A6C115C'u, lo: 0x3C7F400000000000'u), + uint64x2(hi: 0xC097CE7BC90715B3'u, lo: 0x4B9F100000000000'u), + uint64x2(hi: 0xF0BDC21ABB48DB20'u, lo: 0x1E86D40000000000'u), + uint64x2(hi: 0x96769950B50D88F4'u, lo: 0x1314448000000000'u), + uint64x2(hi: 0xBC143FA4E250EB31'u, lo: 0x17D955A000000000'u), + uint64x2(hi: 0xEB194F8E1AE525FD'u, lo: 0x5DCFAB0800000000'u), + uint64x2(hi: 0x92EFD1B8D0CF37BE'u, lo: 0x5AA1CAE500000000'u), + uint64x2(hi: 0xB7ABC627050305AD'u, lo: 0xF14A3D9E40000000'u), + uint64x2(hi: 0xE596B7B0C643C719'u, lo: 0x6D9CCD05D0000000'u), + uint64x2(hi: 0x8F7E32CE7BEA5C6F'u, lo: 0xE4820023A2000000'u), + uint64x2(hi: 0xB35DBF821AE4F38B'u, lo: 0xDDA2802C8A800000'u), + uint64x2(hi: 0xE0352F62A19E306E'u, lo: 0xD50B2037AD200000'u), + uint64x2(hi: 0x8C213D9DA502DE45'u, lo: 0x4526F422CC340000'u), + uint64x2(hi: 0xAF298D050E4395D6'u, lo: 0x9670B12B7F410000'u), + uint64x2(hi: 0xDAF3F04651D47B4C'u, lo: 0x3C0CDD765F114000'u), + uint64x2(hi: 0x88D8762BF324CD0F'u, lo: 0xA5880A69FB6AC800'u), + uint64x2(hi: 0xAB0E93B6EFEE0053'u, lo: 0x8EEA0D047A457A00'u), + uint64x2(hi: 0xD5D238A4ABE98068'u, lo: 0x72A4904598D6D880'u), + uint64x2(hi: 0x85A36366EB71F041'u, lo: 0x47A6DA2B7F864750'u), + uint64x2(hi: 0xA70C3C40A64E6C51'u, lo: 0x999090B65F67D924'u), + uint64x2(hi: 0xD0CF4B50CFE20765'u, lo: 0xFFF4B4E3F741CF6D'u), + uint64x2(hi: 0x82818F1281ED449F'u, lo: 0xBFF8F10E7A8921A4'u), + uint64x2(hi: 0xA321F2D7226895C7'u, lo: 0xAFF72D52192B6A0D'u), + uint64x2(hi: 0xCBEA6F8CEB02BB39'u, lo: 0x9BF4F8A69F764490'u), + uint64x2(hi: 0xFEE50B7025C36A08'u, lo: 0x02F236D04753D5B4'u), + uint64x2(hi: 0x9F4F2726179A2245'u, lo: 0x01D762422C946590'u), + uint64x2(hi: 0xC722F0EF9D80AAD6'u, lo: 0x424D3AD2B7B97EF5'u), + uint64x2(hi: 0xF8EBAD2B84E0D58B'u, lo: 0xD2E0898765A7DEB2'u), + uint64x2(hi: 0x9B934C3B330C8577'u, lo: 0x63CC55F49F88EB2F'u), + uint64x2(hi: 0xC2781F49FFCFA6D5'u, lo: 0x3CBF6B71C76B25FB'u), + uint64x2(hi: 0xF316271C7FC3908A'u, lo: 0x8BEF464E3945EF7A'u), + uint64x2(hi: 0x97EDD871CFDA3A56'u, lo: 0x97758BF0E3CBB5AC'u), + uint64x2(hi: 0xBDE94E8E43D0C8EC'u, lo: 0x3D52EEED1CBEA317'u), + uint64x2(hi: 0xED63A231D4C4FB27'u, lo: 0x4CA7AAA863EE4BDD'u), + uint64x2(hi: 0x945E455F24FB1CF8'u, lo: 0x8FE8CAA93E74EF6A'u), + uint64x2(hi: 0xB975D6B6EE39E436'u, lo: 0xB3E2FD538E122B44'u), + uint64x2(hi: 0xE7D34C64A9C85D44'u, lo: 0x60DBBCA87196B616'u), + uint64x2(hi: 0x90E40FBEEA1D3A4A'u, lo: 0xBC8955E946FE31CD'u), + uint64x2(hi: 0xB51D13AEA4A488DD'u, lo: 0x6BABAB6398BDBE41'u), + uint64x2(hi: 0xE264589A4DCDAB14'u, lo: 0xC696963C7EED2DD1'u), + uint64x2(hi: 0x8D7EB76070A08AEC'u, lo: 0xFC1E1DE5CF543CA2'u), + uint64x2(hi: 0xB0DE65388CC8ADA8'u, lo: 0x3B25A55F43294BCB'u), + uint64x2(hi: 0xDD15FE86AFFAD912'u, lo: 0x49EF0EB713F39EBE'u), + uint64x2(hi: 0x8A2DBF142DFCC7AB'u, lo: 0x6E3569326C784337'u), + uint64x2(hi: 0xACB92ED9397BF996'u, lo: 0x49C2C37F07965404'u), + uint64x2(hi: 0xD7E77A8F87DAF7FB'u, lo: 0xDC33745EC97BE906'u), + uint64x2(hi: 0x86F0AC99B4E8DAFD'u, lo: 0x69A028BB3DED71A3'u), + uint64x2(hi: 0xA8ACD7C0222311BC'u, lo: 0xC40832EA0D68CE0C'u), + uint64x2(hi: 0xD2D80DB02AABD62B'u, lo: 0xF50A3FA490C30190'u), + uint64x2(hi: 0x83C7088E1AAB65DB'u, lo: 0x792667C6DA79E0FA'u), + uint64x2(hi: 0xA4B8CAB1A1563F52'u, lo: 0x577001B891185938'u), + uint64x2(hi: 0xCDE6FD5E09ABCF26'u, lo: 0xED4C0226B55E6F86'u), + uint64x2(hi: 0x80B05E5AC60B6178'u, lo: 0x544F8158315B05B4'u), + uint64x2(hi: 0xA0DC75F1778E39D6'u, lo: 0x696361AE3DB1C721'u), + uint64x2(hi: 0xC913936DD571C84C'u, lo: 0x03BC3A19CD1E38E9'u), + uint64x2(hi: 0xFB5878494ACE3A5F'u, lo: 0x04AB48A04065C723'u), + uint64x2(hi: 0x9D174B2DCEC0E47B'u, lo: 0x62EB0D64283F9C76'u), + uint64x2(hi: 0xC45D1DF942711D9A'u, lo: 0x3BA5D0BD324F8394'u), + uint64x2(hi: 0xF5746577930D6500'u, lo: 0xCA8F44EC7EE36479'u), + uint64x2(hi: 0x9968BF6ABBE85F20'u, lo: 0x7E998B13CF4E1ECB'u), + uint64x2(hi: 0xBFC2EF456AE276E8'u, lo: 0x9E3FEDD8C321A67E'u), + uint64x2(hi: 0xEFB3AB16C59B14A2'u, lo: 0xC5CFE94EF3EA101E'u), + uint64x2(hi: 0x95D04AEE3B80ECE5'u, lo: 0xBBA1F1D158724A12'u), + uint64x2(hi: 0xBB445DA9CA61281F'u, lo: 0x2A8A6E45AE8EDC97'u), + uint64x2(hi: 0xEA1575143CF97226'u, lo: 0xF52D09D71A3293BD'u), + uint64x2(hi: 0x924D692CA61BE758'u, lo: 0x593C2626705F9C56'u), + uint64x2(hi: 0xB6E0C377CFA2E12E'u, lo: 0x6F8B2FB00C77836C'u), + uint64x2(hi: 0xE498F455C38B997A'u, lo: 0x0B6DFB9C0F956447'u), + uint64x2(hi: 0x8EDF98B59A373FEC'u, lo: 0x4724BD4189BD5EAC'u), + uint64x2(hi: 0xB2977EE300C50FE7'u, lo: 0x58EDEC91EC2CB657'u), + uint64x2(hi: 0xDF3D5E9BC0F653E1'u, lo: 0x2F2967B66737E3ED'u), + uint64x2(hi: 0x8B865B215899F46C'u, lo: 0xBD79E0D20082EE74'u), + uint64x2(hi: 0xAE67F1E9AEC07187'u, lo: 0xECD8590680A3AA11'u), + uint64x2(hi: 0xDA01EE641A708DE9'u, lo: 0xE80E6F4820CC9495'u), + uint64x2(hi: 0x884134FE908658B2'u, lo: 0x3109058D147FDCDD'u), + uint64x2(hi: 0xAA51823E34A7EEDE'u, lo: 0xBD4B46F0599FD415'u), + uint64x2(hi: 0xD4E5E2CDC1D1EA96'u, lo: 0x6C9E18AC7007C91A'u), + uint64x2(hi: 0x850FADC09923329E'u, lo: 0x03E2CF6BC604DDB0'u), + uint64x2(hi: 0xA6539930BF6BFF45'u, lo: 0x84DB8346B786151C'u), + uint64x2(hi: 0xCFE87F7CEF46FF16'u, lo: 0xE612641865679A63'u), + uint64x2(hi: 0x81F14FAE158C5F6E'u, lo: 0x4FCB7E8F3F60C07E'u), + uint64x2(hi: 0xA26DA3999AEF7749'u, lo: 0xE3BE5E330F38F09D'u), + uint64x2(hi: 0xCB090C8001AB551C'u, lo: 0x5CADF5BFD3072CC5'u), + uint64x2(hi: 0xFDCB4FA002162A63'u, lo: 0x73D9732FC7C8F7F6'u), + uint64x2(hi: 0x9E9F11C4014DDA7E'u, lo: 0x2867E7FDDCDD9AFA'u), + uint64x2(hi: 0xC646D63501A1511D'u, lo: 0xB281E1FD541501B8'u), + uint64x2(hi: 0xF7D88BC24209A565'u, lo: 0x1F225A7CA91A4226'u), + uint64x2(hi: 0x9AE757596946075F'u, lo: 0x3375788DE9B06958'u), + uint64x2(hi: 0xC1A12D2FC3978937'u, lo: 0x0052D6B1641C83AE'u), + uint64x2(hi: 0xF209787BB47D6B84'u, lo: 0xC0678C5DBD23A49A'u), + uint64x2(hi: 0x9745EB4D50CE6332'u, lo: 0xF840B7BA963646E0'u), + uint64x2(hi: 0xBD176620A501FBFF'u, lo: 0xB650E5A93BC3D898'u), + uint64x2(hi: 0xEC5D3FA8CE427AFF'u, lo: 0xA3E51F138AB4CEBE'u), + uint64x2(hi: 0x93BA47C980E98CDF'u, lo: 0xC66F336C36B10137'u), + uint64x2(hi: 0xB8A8D9BBE123F017'u, lo: 0xB80B0047445D4184'u), + uint64x2(hi: 0xE6D3102AD96CEC1D'u, lo: 0xA60DC059157491E5'u), + uint64x2(hi: 0x9043EA1AC7E41392'u, lo: 0x87C89837AD68DB2F'u), + uint64x2(hi: 0xB454E4A179DD1877'u, lo: 0x29BABE4598C311FB'u), + uint64x2(hi: 0xE16A1DC9D8545E94'u, lo: 0xF4296DD6FEF3D67A'u), + uint64x2(hi: 0x8CE2529E2734BB1D'u, lo: 0x1899E4A65F58660C'u), + uint64x2(hi: 0xB01AE745B101E9E4'u, lo: 0x5EC05DCFF72E7F8F'u), + uint64x2(hi: 0xDC21A1171D42645D'u, lo: 0x76707543F4FA1F73'u), + uint64x2(hi: 0x899504AE72497EBA'u, lo: 0x6A06494A791C53A8'u), + uint64x2(hi: 0xABFA45DA0EDBDE69'u, lo: 0x0487DB9D17636892'u), + uint64x2(hi: 0xD6F8D7509292D603'u, lo: 0x45A9D2845D3C42B6'u), + uint64x2(hi: 0x865B86925B9BC5C2'u, lo: 0x0B8A2392BA45A9B2'u), + uint64x2(hi: 0xA7F26836F282B732'u, lo: 0x8E6CAC7768D7141E'u), + uint64x2(hi: 0xD1EF0244AF2364FF'u, lo: 0x3207D795430CD926'u), + uint64x2(hi: 0x8335616AED761F1F'u, lo: 0x7F44E6BD49E807B8'u), + uint64x2(hi: 0xA402B9C5A8D3A6E7'u, lo: 0x5F16206C9C6209A6'u), + uint64x2(hi: 0xCD036837130890A1'u, lo: 0x36DBA887C37A8C0F'u), + uint64x2(hi: 0x802221226BE55A64'u, lo: 0xC2494954DA2C9789'u), + uint64x2(hi: 0xA02AA96B06DEB0FD'u, lo: 0xF2DB9BAA10B7BD6C'u), + uint64x2(hi: 0xC83553C5C8965D3D'u, lo: 0x6F92829494E5ACC7'u), + uint64x2(hi: 0xFA42A8B73ABBF48C'u, lo: 0xCB772339BA1F17F9'u), + uint64x2(hi: 0x9C69A97284B578D7'u, lo: 0xFF2A760414536EFB'u), + uint64x2(hi: 0xC38413CF25E2D70D'u, lo: 0xFEF5138519684ABA'u), + uint64x2(hi: 0xF46518C2EF5B8CD1'u, lo: 0x7EB258665FC25D69'u), + uint64x2(hi: 0x98BF2F79D5993802'u, lo: 0xEF2F773FFBD97A61'u), + uint64x2(hi: 0xBEEEFB584AFF8603'u, lo: 0xAAFB550FFACFD8FA'u), + uint64x2(hi: 0xEEAABA2E5DBF6784'u, lo: 0x95BA2A53F983CF38'u), + uint64x2(hi: 0x952AB45CFA97A0B2'u, lo: 0xDD945A747BF26183'u), + uint64x2(hi: 0xBA756174393D88DF'u, lo: 0x94F971119AEEF9E4'u), + uint64x2(hi: 0xE912B9D1478CEB17'u, lo: 0x7A37CD5601AAB85D'u), + uint64x2(hi: 0x91ABB422CCB812EE'u, lo: 0xAC62E055C10AB33A'u), + uint64x2(hi: 0xB616A12B7FE617AA'u, lo: 0x577B986B314D6009'u), + uint64x2(hi: 0xE39C49765FDF9D94'u, lo: 0xED5A7E85FDA0B80B'u), + uint64x2(hi: 0x8E41ADE9FBEBC27D'u, lo: 0x14588F13BE847307'u), + uint64x2(hi: 0xB1D219647AE6B31C'u, lo: 0x596EB2D8AE258FC8'u), + uint64x2(hi: 0xDE469FBD99A05FE3'u, lo: 0x6FCA5F8ED9AEF3BB'u), + uint64x2(hi: 0x8AEC23D680043BEE'u, lo: 0x25DE7BB9480D5854'u), + uint64x2(hi: 0xADA72CCC20054AE9'u, lo: 0xAF561AA79A10AE6A'u), + uint64x2(hi: 0xD910F7FF28069DA4'u, lo: 0x1B2BA1518094DA04'u), + uint64x2(hi: 0x87AA9AFF79042286'u, lo: 0x90FB44D2F05D0842'u), + uint64x2(hi: 0xA99541BF57452B28'u, lo: 0x353A1607AC744A53'u), + uint64x2(hi: 0xD3FA922F2D1675F2'u, lo: 0x42889B8997915CE8'u), + uint64x2(hi: 0x847C9B5D7C2E09B7'u, lo: 0x69956135FEBADA11'u), + uint64x2(hi: 0xA59BC234DB398C25'u, lo: 0x43FAB9837E699095'u), + uint64x2(hi: 0xCF02B2C21207EF2E'u, lo: 0x94F967E45E03F4BB'u), + uint64x2(hi: 0x8161AFB94B44F57D'u, lo: 0x1D1BE0EEBAC278F5'u), + uint64x2(hi: 0xA1BA1BA79E1632DC'u, lo: 0x6462D92A69731732'u), + uint64x2(hi: 0xCA28A291859BBF93'u, lo: 0x7D7B8F7503CFDCFE'u), + uint64x2(hi: 0xFCB2CB35E702AF78'u, lo: 0x5CDA735244C3D43E'u), + uint64x2(hi: 0x9DEFBF01B061ADAB'u, lo: 0x3A0888136AFA64A7'u), + uint64x2(hi: 0xC56BAEC21C7A1916'u, lo: 0x088AAA1845B8FDD0'u), + uint64x2(hi: 0xF6C69A72A3989F5B'u, lo: 0x8AAD549E57273D45'u), + uint64x2(hi: 0x9A3C2087A63F6399'u, lo: 0x36AC54E2F678864B'u), + uint64x2(hi: 0xC0CB28A98FCF3C7F'u, lo: 0x84576A1BB416A7DD'u), + uint64x2(hi: 0xF0FDF2D3F3C30B9F'u, lo: 0x656D44A2A11C51D5'u), + uint64x2(hi: 0x969EB7C47859E743'u, lo: 0x9F644AE5A4B1B325'u), + uint64x2(hi: 0xBC4665B596706114'u, lo: 0x873D5D9F0DDE1FEE'u), + uint64x2(hi: 0xEB57FF22FC0C7959'u, lo: 0xA90CB506D155A7EA'u), + uint64x2(hi: 0x9316FF75DD87CBD8'u, lo: 0x09A7F12442D588F2'u), + uint64x2(hi: 0xB7DCBF5354E9BECE'u, lo: 0x0C11ED6D538AEB2F'u), + uint64x2(hi: 0xE5D3EF282A242E81'u, lo: 0x8F1668C8A86DA5FA'u), + uint64x2(hi: 0x8FA475791A569D10'u, lo: 0xF96E017D694487BC'u), + uint64x2(hi: 0xB38D92D760EC4455'u, lo: 0x37C981DCC395A9AC'u), + uint64x2(hi: 0xE070F78D3927556A'u, lo: 0x85BBE253F47B1417'u), + uint64x2(hi: 0x8C469AB843B89562'u, lo: 0x93956D7478CCEC8E'u), + uint64x2(hi: 0xAF58416654A6BABB'u, lo: 0x387AC8D1970027B2'u), + uint64x2(hi: 0xDB2E51BFE9D0696A'u, lo: 0x06997B05FCC0319E'u), + uint64x2(hi: 0x88FCF317F22241E2'u, lo: 0x441FECE3BDF81F03'u), + uint64x2(hi: 0xAB3C2FDDEEAAD25A'u, lo: 0xD527E81CAD7626C3'u), + uint64x2(hi: 0xD60B3BD56A5586F1'u, lo: 0x8A71E223D8D3B074'u), + uint64x2(hi: 0x85C7056562757456'u, lo: 0xF6872D5667844E49'u), + uint64x2(hi: 0xA738C6BEBB12D16C'u, lo: 0xB428F8AC016561DB'u), + uint64x2(hi: 0xD106F86E69D785C7'u, lo: 0xE13336D701BEBA52'u), + uint64x2(hi: 0x82A45B450226B39C'u, lo: 0xECC0024661173473'u), + uint64x2(hi: 0xA34D721642B06084'u, lo: 0x27F002D7F95D0190'u), + uint64x2(hi: 0xCC20CE9BD35C78A5'u, lo: 0x31EC038DF7B441F4'u), + uint64x2(hi: 0xFF290242C83396CE'u, lo: 0x7E67047175A15271'u), + uint64x2(hi: 0x9F79A169BD203E41'u, lo: 0x0F0062C6E984D386'u), + uint64x2(hi: 0xC75809C42C684DD1'u, lo: 0x52C07B78A3E60868'u), + uint64x2(hi: 0xF92E0C3537826145'u, lo: 0xA7709A56CCDF8A82'u), + uint64x2(hi: 0x9BBCC7A142B17CCB'u, lo: 0x88A66076400BB691'u), + uint64x2(hi: 0xC2ABF989935DDBFE'u, lo: 0x6ACFF893D00EA435'u), + uint64x2(hi: 0xF356F7EBF83552FE'u, lo: 0x0583F6B8C4124D43'u), + uint64x2(hi: 0x98165AF37B2153DE'u, lo: 0xC3727A337A8B704A'u), + uint64x2(hi: 0xBE1BF1B059E9A8D6'u, lo: 0x744F18C0592E4C5C'u), + uint64x2(hi: 0xEDA2EE1C7064130C'u, lo: 0x1162DEF06F79DF73'u), + uint64x2(hi: 0x9485D4D1C63E8BE7'u, lo: 0x8ADDCB5645AC2BA8'u), + uint64x2(hi: 0xB9A74A0637CE2EE1'u, lo: 0x6D953E2BD7173692'u), + uint64x2(hi: 0xE8111C87C5C1BA99'u, lo: 0xC8FA8DB6CCDD0437'u), + uint64x2(hi: 0x910AB1D4DB9914A0'u, lo: 0x1D9C9892400A22A2'u), + uint64x2(hi: 0xB54D5E4A127F59C8'u, lo: 0x2503BEB6D00CAB4B'u), + uint64x2(hi: 0xE2A0B5DC971F303A'u, lo: 0x2E44AE64840FD61D'u), + uint64x2(hi: 0x8DA471A9DE737E24'u, lo: 0x5CEAECFED289E5D2'u), + uint64x2(hi: 0xB10D8E1456105DAD'u, lo: 0x7425A83E872C5F47'u), + uint64x2(hi: 0xDD50F1996B947518'u, lo: 0xD12F124E28F77719'u), + uint64x2(hi: 0x8A5296FFE33CC92F'u, lo: 0x82BD6B70D99AAA6F'u), + uint64x2(hi: 0xACE73CBFDC0BFB7B'u, lo: 0x636CC64D1001550B'u), + uint64x2(hi: 0xD8210BEFD30EFA5A'u, lo: 0x3C47F7E05401AA4E'u), + uint64x2(hi: 0x8714A775E3E95C78'u, lo: 0x65ACFAEC34810A71'u), + uint64x2(hi: 0xA8D9D1535CE3B396'u, lo: 0x7F1839A741A14D0D'u), + uint64x2(hi: 0xD31045A8341CA07C'u, lo: 0x1EDE48111209A050'u), + uint64x2(hi: 0x83EA2B892091E44D'u, lo: 0x934AED0AAB460432'u), + uint64x2(hi: 0xA4E4B66B68B65D60'u, lo: 0xF81DA84D5617853F'u), + uint64x2(hi: 0xCE1DE40642E3F4B9'u, lo: 0x36251260AB9D668E'u), + uint64x2(hi: 0x80D2AE83E9CE78F3'u, lo: 0xC1D72B7C6B426019'u), + uint64x2(hi: 0xA1075A24E4421730'u, lo: 0xB24CF65B8612F81F'u), + uint64x2(hi: 0xC94930AE1D529CFC'u, lo: 0xDEE033F26797B627'u), + uint64x2(hi: 0xFB9B7CD9A4A7443C'u, lo: 0x169840EF017DA3B1'u), + uint64x2(hi: 0x9D412E0806E88AA5'u, lo: 0x8E1F289560EE864E'u), + uint64x2(hi: 0xC491798A08A2AD4E'u, lo: 0xF1A6F2BAB92A27E2'u), + uint64x2(hi: 0xF5B5D7EC8ACB58A2'u, lo: 0xAE10AF696774B1DB'u), + uint64x2(hi: 0x9991A6F3D6BF1765'u, lo: 0xACCA6DA1E0A8EF29'u), + uint64x2(hi: 0xBFF610B0CC6EDD3F'u, lo: 0x17FD090A58D32AF3'u), + uint64x2(hi: 0xEFF394DCFF8A948E'u, lo: 0xDDFC4B4CEF07F5B0'u), + uint64x2(hi: 0x95F83D0A1FB69CD9'u, lo: 0x4ABDAF101564F98E'u), + uint64x2(hi: 0xBB764C4CA7A4440F'u, lo: 0x9D6D1AD41ABE37F1'u), + uint64x2(hi: 0xEA53DF5FD18D5513'u, lo: 0x84C86189216DC5ED'u), + uint64x2(hi: 0x92746B9BE2F8552C'u, lo: 0x32FD3CF5B4E49BB4'u), + uint64x2(hi: 0xB7118682DBB66A77'u, lo: 0x3FBC8C33221DC2A1'u), + uint64x2(hi: 0xE4D5E82392A40515'u, lo: 0x0FABAF3FEAA5334A'u), + uint64x2(hi: 0x8F05B1163BA6832D'u, lo: 0x29CB4D87F2A7400E'u), + uint64x2(hi: 0xB2C71D5BCA9023F8'u, lo: 0x743E20E9EF511012'u), + uint64x2(hi: 0xDF78E4B2BD342CF6'u, lo: 0x914DA9246B255416'u), + uint64x2(hi: 0x8BAB8EEFB6409C1A'u, lo: 0x1AD089B6C2F7548E'u), + uint64x2(hi: 0xAE9672ABA3D0C320'u, lo: 0xA184AC2473B529B1'u), + uint64x2(hi: 0xDA3C0F568CC4F3E8'u, lo: 0xC9E5D72D90A2741E'u), + uint64x2(hi: 0x8865899617FB1871'u, lo: 0x7E2FA67C7A658892'u), + uint64x2(hi: 0xAA7EEBFB9DF9DE8D'u, lo: 0xDDBB901B98FEEAB7'u), + uint64x2(hi: 0xD51EA6FA85785631'u, lo: 0x552A74227F3EA565'u), + uint64x2(hi: 0x8533285C936B35DE'u, lo: 0xD53A88958F87275F'u), + uint64x2(hi: 0xA67FF273B8460356'u, lo: 0x8A892ABAF368F137'u), + uint64x2(hi: 0xD01FEF10A657842C'u, lo: 0x2D2B7569B0432D85'u), + uint64x2(hi: 0x8213F56A67F6B29B'u, lo: 0x9C3B29620E29FC73'u), + uint64x2(hi: 0xA298F2C501F45F42'u, lo: 0x8349F3BA91B47B8F'u), + uint64x2(hi: 0xCB3F2F7642717713'u, lo: 0x241C70A936219A73'u), + uint64x2(hi: 0xFE0EFB53D30DD4D7'u, lo: 0xED238CD383AA0110'u), + uint64x2(hi: 0x9EC95D1463E8A506'u, lo: 0xF4363804324A40AA'u), + uint64x2(hi: 0xC67BB4597CE2CE48'u, lo: 0xB143C6053EDCD0D5'u), + uint64x2(hi: 0xF81AA16FDC1B81DA'u, lo: 0xDD94B7868E94050A'u), + uint64x2(hi: 0x9B10A4E5E9913128'u, lo: 0xCA7CF2B4191C8326'u), + uint64x2(hi: 0xC1D4CE1F63F57D72'u, lo: 0xFD1C2F611F63A3F0'u), + uint64x2(hi: 0xF24A01A73CF2DCCF'u, lo: 0xBC633B39673C8CEC'u), + uint64x2(hi: 0x976E41088617CA01'u, lo: 0xD5BE0503E085D813'u), + uint64x2(hi: 0xBD49D14AA79DBC82'u, lo: 0x4B2D8644D8A74E18'u), + uint64x2(hi: 0xEC9C459D51852BA2'u, lo: 0xDDF8E7D60ED1219E'u), + uint64x2(hi: 0x93E1AB8252F33B45'u, lo: 0xCABB90E5C942B503'u), + uint64x2(hi: 0xB8DA1662E7B00A17'u, lo: 0x3D6A751F3B936243'u), + uint64x2(hi: 0xE7109BFBA19C0C9D'u, lo: 0x0CC512670A783AD4'u), + uint64x2(hi: 0x906A617D450187E2'u, lo: 0x27FB2B80668B24C5'u), + uint64x2(hi: 0xB484F9DC9641E9DA'u, lo: 0xB1F9F660802DEDF6'u), + uint64x2(hi: 0xE1A63853BBD26451'u, lo: 0x5E7873F8A0396973'u), + uint64x2(hi: 0x8D07E33455637EB2'u, lo: 0xDB0B487B6423E1E8'u), + uint64x2(hi: 0xB049DC016ABC5E5F'u, lo: 0x91CE1A9A3D2CDA62'u), + uint64x2(hi: 0xDC5C5301C56B75F7'u, lo: 0x7641A140CC7810FB'u), + uint64x2(hi: 0x89B9B3E11B6329BA'u, lo: 0xA9E904C87FCB0A9D'u), + uint64x2(hi: 0xAC2820D9623BF429'u, lo: 0x546345FA9FBDCD44'u), + uint64x2(hi: 0xD732290FBACAF133'u, lo: 0xA97C177947AD4095'u), + uint64x2(hi: 0x867F59A9D4BED6C0'u, lo: 0x49ED8EABCCCC485D'u), + uint64x2(hi: 0xA81F301449EE8C70'u, lo: 0x5C68F256BFFF5A74'u), + uint64x2(hi: 0xD226FC195C6A2F8C'u, lo: 0x73832EEC6FFF3111'u), + uint64x2(hi: 0x83585D8FD9C25DB7'u, lo: 0xC831FD53C5FF7EAB'u), + uint64x2(hi: 0xA42E74F3D032F525'u, lo: 0xBA3E7CA8B77F5E55'u), + uint64x2(hi: 0xCD3A1230C43FB26F'u, lo: 0x28CE1BD2E55F35EB'u), + uint64x2(hi: 0x80444B5E7AA7CF85'u, lo: 0x7980D163CF5B81B3'u), + uint64x2(hi: 0xA0555E361951C366'u, lo: 0xD7E105BCC332621F'u), + uint64x2(hi: 0xC86AB5C39FA63440'u, lo: 0x8DD9472BF3FEFAA7'u), + uint64x2(hi: 0xFA856334878FC150'u, lo: 0xB14F98F6F0FEB951'u), + uint64x2(hi: 0x9C935E00D4B9D8D2'u, lo: 0x6ED1BF9A569F33D3'u), + uint64x2(hi: 0xC3B8358109E84F07'u, lo: 0x0A862F80EC4700C8'u), + uint64x2(hi: 0xF4A642E14C6262C8'u, lo: 0xCD27BB612758C0FA'u), + uint64x2(hi: 0x98E7E9CCCFBD7DBD'u, lo: 0x8038D51CB897789C'u), + uint64x2(hi: 0xBF21E44003ACDD2C'u, lo: 0xE0470A63E6BD56C3'u), + uint64x2(hi: 0xEEEA5D5004981478'u, lo: 0x1858CCFCE06CAC74'u), + uint64x2(hi: 0x95527A5202DF0CCB'u, lo: 0x0F37801E0C43EBC8'u), + uint64x2(hi: 0xBAA718E68396CFFD'u, lo: 0xD30560258F54E6BA'u), + uint64x2(hi: 0xE950DF20247C83FD'u, lo: 0x47C6B82EF32A2069'u), + uint64x2(hi: 0x91D28B7416CDD27E'u, lo: 0x4CDC331D57FA5441'u), + uint64x2(hi: 0xB6472E511C81471D'u, lo: 0xE0133FE4ADF8E952'u), + uint64x2(hi: 0xE3D8F9E563A198E5'u, lo: 0x58180FDDD97723A6'u), + uint64x2(hi: 0x8E679C2F5E44FF8F'u, lo: 0x570F09EAA7EA7648'u), + uint64x2(hi: 0xB201833B35D63F73'u, lo: 0x2CD2CC6551E513DA'u), + uint64x2(hi: 0xDE81E40A034BCF4F'u, lo: 0xF8077F7EA65E58D1'u), + uint64x2(hi: 0x8B112E86420F6191'u, lo: 0xFB04AFAF27FAF782'u), + uint64x2(hi: 0xADD57A27D29339F6'u, lo: 0x79C5DB9AF1F9B563'u), + uint64x2(hi: 0xD94AD8B1C7380874'u, lo: 0x18375281AE7822BC'u), + uint64x2(hi: 0x87CEC76F1C830548'u, lo: 0x8F2293910D0B15B5'u), + uint64x2(hi: 0xA9C2794AE3A3C69A'u, lo: 0xB2EB3875504DDB22'u), + uint64x2(hi: 0xD433179D9C8CB841'u, lo: 0x5FA60692A46151EB'u), + uint64x2(hi: 0x849FEEC281D7F328'u, lo: 0xDBC7C41BA6BCD333'u), + uint64x2(hi: 0xA5C7EA73224DEFF3'u, lo: 0x12B9B522906C0800'u), + uint64x2(hi: 0xCF39E50FEAE16BEF'u, lo: 0xD768226B34870A00'u), + uint64x2(hi: 0x81842F29F2CCE375'u, lo: 0xE6A1158300D46640'u), + uint64x2(hi: 0xA1E53AF46F801C53'u, lo: 0x60495AE3C1097FD0'u), + uint64x2(hi: 0xCA5E89B18B602368'u, lo: 0x385BB19CB14BDFC4'u), + uint64x2(hi: 0xFCF62C1DEE382C42'u, lo: 0x46729E03DD9ED7B5'u), + uint64x2(hi: 0x9E19DB92B4E31BA9'u, lo: 0x6C07A2C26A8346D1'u), + uint64x2(hi: 0xC5A05277621BE293'u, lo: 0xC7098B7305241885'u), + uint64x2(hi: 0xF70867153AA2DB38'u, lo: 0xB8CBEE4FC66D1EA7'u)] + dragonbox_Assert(k >= kMin) + dragonbox_Assert(k <= kMax) + return pow10[k - kMin] + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2*(value: uint64; e2: int32): bool {.inline.} = + dragonbox_Assert(e2 >= 0) + return e2 < 64 and (value and ((uint64(1) shl e2) - 1)) == 0 + +## Returns whether value is divisible by 5^e5 + +proc multipleOfPow5*(value: uint64; e5: int32): bool {.inline.} = + type + MulCmp {.bycopy.} = object + mul: uint64 + cmp: uint64 + + const + mod5 = [MulCmp(mul: 0x0000000000000001'u, cmp: 0xFFFFFFFFFFFFFFFF'u), + MulCmp(mul: 0xCCCCCCCCCCCCCCCD'u, cmp: 0x3333333333333333'u), + MulCmp(mul: 0x8F5C28F5C28F5C29'u, cmp: 0x0A3D70A3D70A3D70'u), + MulCmp(mul: 0x1CAC083126E978D5'u, cmp: 0x020C49BA5E353F7C'u), + MulCmp(mul: 0xD288CE703AFB7E91'u, cmp: 0x0068DB8BAC710CB2'u), + MulCmp(mul: 0x5D4E8FB00BCBE61D'u, cmp: 0x0014F8B588E368F0'u), + MulCmp(mul: 0x790FB65668C26139'u, cmp: 0x000431BDE82D7B63'u), + MulCmp(mul: 0xE5032477AE8D46A5'u, cmp: 0x0000D6BF94D5E57A'u), + MulCmp(mul: 0xC767074B22E90E21'u, cmp: 0x00002AF31DC46118'u), + MulCmp(mul: 0x8E47CE423A2E9C6D'u, cmp: 0x0000089705F4136B'u), + MulCmp(mul: 0x4FA7F60D3ED61F49'u, cmp: 0x000001B7CDFD9D7B'u), + MulCmp(mul: 0x0FEE64690C913975'u, cmp: 0x00000057F5FF85E5'u), + MulCmp(mul: 0x3662E0E1CF503EB1'u, cmp: 0x000000119799812D'u), + MulCmp(mul: 0xA47A2CF9F6433FBD'u, cmp: 0x0000000384B84D09'u), + MulCmp(mul: 0x54186F653140A659'u, cmp: 0x00000000B424DC35'u), + MulCmp(mul: 0x7738164770402145'u, cmp: 0x0000000024075F3D'u), + MulCmp(mul: 0xE4A4D1417CD9A041'u, cmp: 0x000000000734ACA5'u), + MulCmp(mul: 0xC75429D9E5C5200D'u, cmp: 0x000000000170EF54'u), + MulCmp(mul: 0xC1773B91FAC10669'u, cmp: 0x000000000049C977'u), + MulCmp(mul: 0x26B172506559CE15'u, cmp: 0x00000000000EC1E4'u), + MulCmp(mul: 0xD489E3A9ADDEC2D1'u, cmp: 0x000000000002F394'u), + MulCmp(mul: 0x90E860BB892C8D5D'u, cmp: 0x000000000000971D'u), + MulCmp(mul: 0x502E79BF1B6F4F79'u, cmp: 0x0000000000001E39'u), + MulCmp(mul: 0xDCD618596BE30FE5'u, cmp: 0x000000000000060B'u), + MulCmp(mul: 0x2C2AD1AB7BFA3661'u, cmp: 0x0000000000000135'u)] + dragonbox_Assert(e5 >= 0) + dragonbox_Assert(e5 <= 24) + let m5: MulCmp = mod5[e5] + return value * m5.mul <= m5.cmp + +type + FloatingDecimal64* {.bycopy.} = object + significand*: uint64 + exponent*: int32 + + +proc toDecimal64AsymmetricInterval*(e2: int32): FloatingDecimal64 {.inline.} = + ## NB: + ## accept_lower_endpoint = true + ## accept_upper_endpoint = true + const + P: int32 = significandSize + ## Compute k and beta + let minusK: int32 = floorLog10ThreeQuartersPow2(e2) + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + ## Compute xi and zi + let pow10: uint64x2 = computePow10(-minusK) + let lowerEndpoint: uint64 = (pow10.hi - (pow10.hi shr (P + 1))) shr + (64 - P - betaMinus1) + let upperEndpoint: uint64 = (pow10.hi + (pow10.hi shr (P + 0))) shr + (64 - P - betaMinus1) + ## If we don't accept the left endpoint (but we do!) or + ## if the left endpoint is not an integer, increase it + let lowerEndpointIsInteger: bool = (2 <= e2 and e2 <= 3) + let xi: uint64 = lowerEndpoint + uint64(not lowerEndpointIsInteger) + let zi: uint64 = upperEndpoint + ## Try bigger divisor + var q: uint64 = zi div 10 + if q * 10 >= xi: + return FloatingDecimal64(significand: q, exponent: minusK + 1) + q = ((pow10.hi shr (64 - (P + 1) - betaMinus1)) + 1) div 2 + ## When tie occurs, choose one of them according to the rule + if e2 == -77: + dec(q, ord(q mod 2 != 0)) + ## Round to even. + else: + inc(q, ord(q < xi)) + return FloatingDecimal64(significand: q, exponent: minusK) + +proc computeDelta*(pow10: uint64x2; betaMinus1: int32): uint32 {.inline.} = + dragonbox_Assert(betaMinus1 >= 0) + dragonbox_Assert(betaMinus1 <= 63) + return cast[uint32](pow10.hi shr (64 - 1 - betaMinus1)) + +when defined(sizeof_Int128): + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + ## 1 mulx + type + uint128T = uint128 + let p: uint128T = uint128T(x) * y + let hi: uint64 = cast[uint64](p shr 64) + let lo: uint64 = cast[uint64](p) + return (hi, lo) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + var hi: uint64 = 0 + var lo: uint64 = umul128(x, y, addr(hi)) + return uint64x2(hi: hi, lo: lo) + +else: + proc lo32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + + proc hi32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + + proc mul128*(a: uint64; b: uint64): uint64x2 {.inline.} = + let b00: uint64 = uint64(lo32(a)) * lo32(b) + let b01: uint64 = uint64(lo32(a)) * hi32(b) + let b10: uint64 = uint64(hi32(a)) * lo32(b) + let b11: uint64 = uint64(hi32(a)) * hi32(b) + let mid1: uint64 = b10 + hi32(b00) + let mid2: uint64 = b01 + lo32(mid1) + let hi: uint64 = b11 + hi32(mid1) + hi32(mid2) + let lo: uint64 = lo32(b00) or uint64(lo32(mid2)) shl 32 + return uint64x2(hi: hi, lo: lo) + +## Returns (x * y) / 2^128 + +proc mulShift*(x: uint64; y: uint64x2): uint64 {.inline.} = + ## 2 mulx + var p1: uint64x2 = mul128(x, y.hi) + var p0: uint64x2 = mul128(x, y.lo) + p1.lo += p0.hi + inc(p1.hi, ord(p1.lo < p0.hi)) + return p1.hi + +proc mulParity*(twoF: uint64; pow10: uint64x2; betaMinus1: int32): bool {.inline.} = + ## 1 mulx, 1 mul + dragonbox_Assert(betaMinus1 >= 1) + dragonbox_Assert(betaMinus1 <= 63) + let p01: uint64 = twoF * pow10.hi + let p10: uint64 = mul128(twoF, pow10.lo).hi + let mid: uint64 = p01 + p10 + return (mid and (uint64(1) shl (64 - betaMinus1))) != 0 + +proc isIntegralEndpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -2: + return false + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc isIntegralMidpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -4: + return multipleOfPow2(twoF, minusK - e2 + 1) + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecimal64 {. + inline.} = + const + kappa: int32 = 2 + const + bigDivisor: uint32 = 1000 + ## 10^(kappa + 1) + const + smallDivisor: uint32 = 100 + ## 10^(kappa) + ## + ## Step 1: + ## integer promotion & Schubfach multiplier calculation. + ## + var m2: uint64 + var e2: int32 + if ieeeExponent != 0: + m2 = hiddenBit or ieeeSignificand + e2 = cast[int32](ieeeExponent) - exponentBias + if 0 <= -e2 and -e2 < significandSize and multipleOfPow2(m2, -e2): + ## Small integer. + return FloatingDecimal64(significand: m2 shr -e2, exponent: 0) + if ieeeSignificand == 0 and ieeeExponent > 1: + ## Shorter interval case; proceed like Schubfach. + return toDecimal64AsymmetricInterval(e2) + else: + ## Subnormal case; interval is always regular. + m2 = ieeeSignificand + e2 = 1 - exponentBias + let isEven: bool = (m2 mod 2 == 0) + let acceptLower: bool = isEven + let acceptUpper: bool = isEven + ## Compute k and beta. + let minusK: int32 = floorLog10Pow2(e2) - kappa + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + dragonbox_Assert(betaMinus1 >= 6) + dragonbox_Assert(betaMinus1 <= 9) + let pow10: uint64x2 = computePow10(-minusK) + ## Compute delta + ## 10^kappa <= delta < 10^(kappa + 1) + ## 100 <= delta < 1000 + let delta: uint32 = computeDelta(pow10, betaMinus1) + dragonbox_Assert(delta >= smallDivisor) + dragonbox_Assert(delta < bigDivisor) + let twoFl: uint64 = 2 * m2 - 1 + let twoFc: uint64 = 2 * m2 + let twoFr: uint64 = 2 * m2 + 1 + ## (54 bits) + ## Compute zi + ## (54 + 9 = 63 bits) + let zi: uint64 = mulShift(twoFr shl betaMinus1, pow10) + ## 2 mulx + ## + ## Step 2: + ## Try larger divisor. + ## + var q: uint64 = zi div bigDivisor + ## uint64_t q = Mul128(zi, 0x83126E978D4FDF3Cu).hi >> 9; // 1 mulx + var r: uint32 = cast[uint32](zi) - bigDivisor * cast[uint32](q) + ## r = zi % BigDivisor + ## 0 <= r < 1000 + if r < delta: ## likely ~50% ?! + ## (r > deltai) + ## Exclude the right endpoint if necessary + if r != 0 or acceptUpper or not isIntegralEndpoint(twoFr, e2, minusK): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + dragonbox_Assert(q != 0) + dec(q) + r = bigDivisor + elif r == delta: ## unlikely + ## Compare fractional parts. + ## Check conditions in the order different from the paper + ## to take advantage of short-circuiting + if (acceptLower and isIntegralEndpoint(twoFl, e2, minusK)) or + mulParity(twoFl, pow10, betaMinus1): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + else: + discard + ## + ## Step 3: + ## Find the significand with the smaller divisor + ## + q = q * 10 + ## 1 hmul + ## 0 <= r <= 1000 + let dist: uint32 = r - (delta div 2) + (smallDivisor div 2) + let distQ: uint32 = dist div 100 + ## 1 mul + ## const uint32_t dist_r = dist % 100; + q += distQ + ## if /*likely*/ (dist_r == 0) + if dist == distQ * 100: + ## const bool approx_y_parity = ((dist ^ (SmallDivisor / 2)) & 1) != 0; + let approxYParity: bool = (dist and 1) != 0 + ## Check z^(f) >= epsilon^(f) + ## We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1, + ## where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f) + ## Since there are only 2 possibilities, we only need to care about the + ## parity. Also, zi and r should have the same parity since the divisor + ## is an even number + if mulParity(twoFc, pow10, betaMinus1) != approxYParity: + dec(q) + elif q mod 2 != 0 and isIntegralMidpoint(twoFc, e2, minusK): + dec(q) + return FloatingDecimal64(significand: q, exponent: minusK + kappa) + +# ================================================================================================== +# ToChars +# ================================================================================================== + +when false: + template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) + + template dec(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) - uint(offset)) + template inc(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) + uint(offset)) + + proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.} + proc memmove(a, b: cstring; L: int) {.importc, nodecl.} + +proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int {.inline.} = + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999'u32) + let q: uint32 = digits div 10000 + let r: uint32 = digits mod 10000 + let qH: uint32 = q div 100 + let qL: uint32 = q mod 100 + utoa2Digits(buf, pos, qH) + utoa2Digits(buf, pos + 2, qL) + if r == 0: + return trailingZeros2Digits(if qL == 0: qH else: qL) + (if qL == 0: 6 else: 4) + else: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos + 4, rH) + utoa2Digits(buf, pos + 6, rL) + return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + +proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int {.inline.} = + var pos = pos + var output64 = output64 + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 17 digits remaining + if output64 >= 100000000'u64: + let q: uint64 = output64 div 100000000'u64 + let r: uint32 = cast[uint32](output64 mod 100000000'u64) + output64 = q + dec(pos, 8) + if r != 0: + tz = utoa8DigitsSkipTrailingZeros(buf, pos, r) + dragonbox_Assert(tz >= 0) + dragonbox_Assert(tz <= 7) + else: + tz = 8 + nd = 8 + dragonbox_Assert(output64 <= high(uint32)) + var output = cast[uint32](output64) + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + if tz == nd: + inc(tz, trailingZeros2Digits(if rL == 0: rH else: rL) + + (if rL == 0: 2 else: 0)) + else: + if tz == nd: + inc(tz, 4) + else: + for i in 0..3: buf[pos+i] = '0' + ## (actually not required...) + inc(nd, 4) + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + dragonbox_Assert(output >= 1) + dragonbox_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + dragonbox_Assert(q >= 1) + dragonbox_Assert(q <= 9) + dec(pos) + buf[pos] = chr(ord('0') + q) + return tz + +proc decimalLength*(v: uint64): int {.inline.} = + dragonbox_Assert(v >= 1) + dragonbox_Assert(v <= 99999999999999999'u64) + if cast[uint32](v shr 32) != 0: + if v >= 10000000000000000'u64: + return 17 + if v >= 1000000000000000'u64: + return 16 + if v >= 100000000000000'u64: + return 15 + if v >= 10000000000000'u64: + return 14 + if v >= 1000000000000'u64: + return 13 + if v >= 100000000000'u64: + return 12 + if v >= 10000000000'u64: + return 11 + return 10 + let v32: uint32 = cast[uint32](v) + if v32 >= 1000000000'u32: + return 10 + if v32 >= 100000000'u32: + return 9 + if v32 >= 10000000'u32: + return 8 + if v32 >= 1000000'u32: + return 7 + if v32 >= 100000'u32: + return 6 + if v32 >= 10000'u32: + return 5 + if v32 >= 1000'u32: + return 4 + if v32 >= 100'u32: + return 3 + if v32 >= 10'u32: + return 2 + return 1 + +proc formatDigits*[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint64; decimalExponent: int; + forceTrailingDotZero = false): int {.inline.} = + const + minFixedDecimalPoint = -6 + const + maxFixedDecimalPoint = 17 + var pos:int = pos.int + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 17, "internal error") + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999999999999'u64) + dragonbox_Assert(decimalExponent >= -999) + dragonbox_Assert(decimalExponent <= 999) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + int(decimalDigitsPosition + numDigits) + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + when true: #defined(vcc) and not defined(clang): + ## VC does not inline the memmove call below. (Even if compiled with /arch:AVX2.) + ## However, memcpy will be inlined. + var tmp: array[16, char] + for i in 0..<16: tmp[i] = buffer[i+pos+decimalPoint] + for i in 0..<16: buffer[i+pos+decimalPoint+1] = tmp[i] + else: + memmove(buffer +! (decimalPoint + 1), buffer +! decimalPoint, 16) + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + ## Copy the first digit one place to the left. + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(ord('0') + k) + inc(pos) + elif k < 100: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + else: + let q: uint32 = k div 100 + let r: uint32 = k mod 100 + buffer[pos] = chr(ord('0') + q) + inc(pos) + utoa2Digits(buffer, pos, r) + inc(pos, 2) + return pos + +proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = false): int {. + inline.} = + var pos = 0 + let significand: uint64 = physicalSignificand(constructDouble(v)) + let exponent: uint64 = physicalExponent(constructDouble(v)) + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec = toDecimal64(significand, exponent) + return formatDigits(buffer, pos, dec.significand, dec.exponent.int, + forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 + +when false: + proc toString*(value: float): string = + var buffer: array[dtoaMinBufferLength, char] + let last = toChars(addr buffer, value) + let L = cast[int](last) - cast[int](addr(buffer)) + result = newString(L) + copyMem(addr result[0], addr buffer[0], L) + diff --git a/lib/std/private/gitutils.nim b/lib/std/private/gitutils.nim new file mode 100644 index 000000000..6dc9c8f3b --- /dev/null +++ b/lib/std/private/gitutils.nim @@ -0,0 +1,74 @@ +##[ +internal API for now, API subject to change +]## + +# xxx move other git utilities here; candidate for stdlib. + +import std/[os, paths, osproc, strutils, tempfiles] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +const commitHead* = "HEAD" + +template retryCall*(maxRetry = 3, backoffDuration = 1.0, call: untyped): bool = + ## Retry `call` up to `maxRetry` times with exponential backoff and initial + ## duraton of `backoffDuration` seconds. + ## This is in particular useful for network commands that can fail. + runnableExamples: + doAssert not retryCall(maxRetry = 2, backoffDuration = 0.1, false) + var i = 0 + doAssert: retryCall(maxRetry = 3, backoffDuration = 0.1, (i.inc; i >= 3)) + doAssert retryCall(call = true) + var result = false + var t = backoffDuration + for i in 0..<maxRetry: + if call: + result = true + break + if i == maxRetry - 1: break + sleep(int(t * 1000)) + t = t * 2 # exponential backoff + result + +proc isGitRepo*(dir: string): bool = + ## Avoid calling git since it depends on /bin/sh existing and fails in Nix. + return fileExists(dir/".git/HEAD") + +proc diffFiles*(path1, path2: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of files `path1`, `path2`, the exact form of + ## which is implementation defined. + # This could be customized, e.g. non-git diff with `diff -uNdr`, or with + # git diff options (e.g. --color-moved, --word-diff). + # in general, `git diff` has more options than `diff`. + var status = 0 + (result.output, status) = execCmdEx("git diff --no-index $1 $2" % [path1.quoteShell, path2.quoteShell]) + doAssert (status == 0) or (status == 1) + result.same = status == 0 + +proc diffStrings*(a, b: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of `a`, `b`, the exact form of which is + ## implementation defined. + ## See also `experimental.diff`. + runnableExamples: + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + let (c, same) = diffStrings(a, b) + doAssert not same + let (c2, same2) = diffStrings(a, a) + doAssert same2 + runnableExamples("-r:off"): + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + echo diffStrings(a, b).output + + template tmpFileImpl(prefix, str): auto = + let path = genTempPath(prefix, "") + writeFile(path, str) + path + let patha = tmpFileImpl("diffStrings_a_", a) + let pathb = tmpFileImpl("diffStrings_b_", b) + defer: + removeFile(patha) + removeFile(pathb) + result = diffFiles(patha, pathb) diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim new file mode 100644 index 000000000..a6d088558 --- /dev/null +++ b/lib/std/private/globs.nim @@ -0,0 +1,70 @@ +##[ +unstable API, internal use only for now. +this can eventually be moved to std/os and `walkDirRec` can be implemented in terms of this +to avoid duplication +]## + +import std/os +when defined(windows): + from std/strutils import replace + +when defined(nimPreviewSlimSystem): + import std/[assertions, objectdollar] + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +type + PathEntry* = object + kind*: PathComponent + path*: string + +iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect], effectsOf: follow.} = + ## Improved `os.walkDirRec`. + #[ + note: a yieldFilter isn't needed because caller can filter at call site, without + loss of generality, unlike `follow`. + + Future work: + * need to document + * add a `sort` option, which can be implemented efficiently only here, not at call site. + * provide a way to do error reporting, which is tricky because iteration cannot be resumed + ]# + var stack = @["."] + var checkDir = checkDir + var entry: PathEntry + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): + let rel = d / p + entry.kind = k + if relative: entry.path = rel + else: entry.path = dir / rel + if k in {pcDir, pcLinkToDir}: + if follow == nil or follow(entry): stack.add rel + yield entry + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + +proc nativeToUnixPath*(path: string): string = + # pending https://github.com/nim-lang/Nim/pull/13265 + result = path + when defined(windows): + if path.len >= 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':': + result[0] = '/' + result[1] = path[0] + if path.len > 2 and path[2] != '\\': + raiseAssert "paths like `C:foo` are currently unsupported, path: " & path + when DirSep == '\\': + result = replace(result, '\\', '/') + +when isMainModule: + import std/sugar + for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]): + echo a diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim new file mode 100644 index 000000000..5f79eab27 --- /dev/null +++ b/lib/std/private/jsutils.nim @@ -0,0 +1,96 @@ +when defined(js): + import std/jsbigints + + type + ArrayBuffer* = ref object of JsRoot + Float64Array* = ref object of JsRoot + Uint32Array* = ref object of JsRoot + Uint8Array* = ref object of JsRoot + BigUint64Array* = ref object of JsRoot + + + func newArrayBuffer*(n: int): ArrayBuffer {.importjs: "new ArrayBuffer(#)".} + func newFloat64Array*(buffer: ArrayBuffer): Float64Array {.importjs: "new Float64Array(#)".} + func newUint32Array*(buffer: ArrayBuffer): Uint32Array {.importjs: "new Uint32Array(#)".} + func newBigUint64Array*(buffer: ArrayBuffer): BigUint64Array {.importjs: "new BigUint64Array(#)".} + + func newUint8Array*(n: int): Uint8Array {.importjs: "new Uint8Array(#)".} + + func `[]`*(arr: Uint32Array, i: int): uint32 {.importjs: "#[#]".} + func `[]`*(arr: Uint8Array, i: int): uint8 {.importjs: "#[#]".} + func `[]`*(arr: BigUint64Array, i: int): JsBigInt {.importjs: "#[#]".} + func `[]=`*(arr: Float64Array, i: int, v: float) {.importjs: "#[#] = #".} + + proc jsTypeOf*[T](x: T): cstring {.importjs: "typeof(#)".} = + ## Returns the name of the JsObject's JavaScript type as a cstring. + # xxx replace jsffi.jsTypeOf with this definition and add tests + runnableExamples: + import std/[jsffi, jsbigints] + assert jsTypeOf(1.toJs) == "number" + assert jsTypeOf(false.toJs) == "boolean" + assert [1].toJs.jsTypeOf == "object" # note the difference with `getProtoName` + assert big"1".toJs.jsTypeOf == "bigint" + + proc jsConstructorName*[T](a: T): cstring = + runnableExamples: + import std/jsffi + let a = array[2, float64].default + assert jsConstructorName(a) == "Float64Array" + assert jsConstructorName(a.toJs) == "Float64Array" + {.emit: """`result` = `a`.constructor.name;""".} + + proc hasJsBigInt*(): bool = + {.emit: """`result` = typeof BigInt != 'undefined';""".} + + proc hasBigUint64Array*(): bool = + {.emit: """`result` = typeof BigUint64Array != 'undefined';""".} + + proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} = + runnableExamples: + import std/[jsffi, jsbigints] + type A = ref object + assert 1.toJs.getProtoName == "[object Number]" + assert "a".toJs.getProtoName == "[object String]" + assert big"1".toJs.getProtoName == "[object BigInt]" + assert false.toJs.getProtoName == "[object Boolean]" + assert (a: 1).toJs.getProtoName == "[object Object]" + assert A.default.toJs.getProtoName == "[object Null]" + assert [1].toJs.getProtoName == "[object Int32Array]" # implementation defined + assert @[1].toJs.getProtoName == "[object Array]" # ditto + + const maxSafeInteger* = 9007199254740991 + ## The same as `Number.MAX_SAFE_INTEGER` or `2^53 - 1`. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + runnableExamples: + let a {.importjs: "Number.MAX_SAFE_INTEGER".}: int64 + assert a == maxSafeInteger + + proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} = + runnableExamples: + import std/jsffi + assert 1.isInteger + assert not 1.5.isInteger + assert 1.toJs.isInteger + assert not 1.5.toJs.isInteger + + proc isSafeInteger*[T](x: T): bool {.importjs: "Number.isSafeInteger(#)".} = + runnableExamples: + import std/jsffi + assert not "123".toJs.isSafeInteger + assert 123.isSafeInteger + assert 123.toJs.isSafeInteger + when false: + assert 9007199254740991.toJs.isSafeInteger + assert not 9007199254740992.toJs.isSafeInteger + +template whenJsNoBigInt64*(no64, yes64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + no64 diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim new file mode 100644 index 000000000..06fda6fa1 --- /dev/null +++ b/lib/std/private/miscdollars.nim @@ -0,0 +1,39 @@ +from std/private/digitsutils import addInt + +template toLocation*(result: var string, file: string | cstring, line: int, col: int) = + ## avoids spurious allocations + # Hopefully this can be re-used everywhere so that if a user needs to customize, + # it can be done in a single place. + result.add file + if line > 0: + result.add "(" + addInt(result, line) + if col > 0: + result.add ", " + addInt(result, col) + result.add ")" + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = + result = "(" + const isNamed = T is object or isNamedTuple(typeof(T)) + var count {.used.} = 0 + for name, value in fieldPairs(x): + if count > 0: result.add(", ") + when isNamed: + result.add(name) + result.add(": ") + count.inc + when compiles($value): + when value isnot string and value isnot seq and compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.addQuoted(value) + else: + result.addQuoted(value) + else: + result.add("...") + when not isNamed: + if count == 1: + result.add(",") # $(1,) should print as the semantically legal (1,) + result.add(")") diff --git a/lib/std/private/ntpath.nim b/lib/std/private/ntpath.nim new file mode 100644 index 000000000..7c8661bb7 --- /dev/null +++ b/lib/std/private/ntpath.nim @@ -0,0 +1,61 @@ +# This module is inspired by Python's `ntpath.py` module. + +import std/[ + strutils, +] + +# Adapted `splitdrive` function from the following commits in Python source +# code: +# 5a607a3ee5e81bdcef3f886f9d20c1376a533df4 (2009): Initial UNC handling (by Mark Hammond) +# 2ba0fd5767577954f331ecbd53596cd8035d7186 (2022): Support for "UNC"-device paths (by Barney Gale) +# +# FAQ: Why use `strip` below? `\\?\UNC` is the start of a "UNC symbolic link", +# which is a special UNC form. Running `strip` differentiates `\\?\UNC\` (a UNC +# symbolic link) from e.g. `\\?\UNCD` (UNCD is the server in the UNC path). +func splitDrive*(p: string): tuple[drive, path: string] = + ## Splits a Windows path into a drive and path part. The drive can be e.g. + ## `C:`. It can also be a UNC path (`\\server\drive\path`). + ## + ## The equivalent `splitDrive` for POSIX systems always returns empty drive. + ## Therefore this proc is only necessary on DOS-like file systems (together + ## with Nim's `doslikeFileSystem` conditional variable). + ## + ## This proc's use case is to extract `path` such that it can be manipulated + ## like a POSIX path. + runnableExamples: + doAssert splitDrive("C:") == ("C:", "") + doAssert splitDrive(r"C:\") == (r"C:", r"\") + doAssert splitDrive(r"\\server\drive\foo\bar") == (r"\\server\drive", r"\foo\bar") + doAssert splitDrive(r"\\?\UNC\server\share\dir") == (r"\\?\UNC\server\share", r"\dir") + + result = ("", p) + if p.len < 2: + return + const sep = '\\' + let normp = p.replace('/', sep) + if p.len > 2 and normp[0] == sep and normp[1] == sep and normp[2] != sep: + + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path + # \\machine\mountpoint\directory\etc\... + # directory ^^^^^^^^^^^^^^^ + let start = block: + const unc = "\\\\?\\UNC" # Length is 7 + let idx = min(8, normp.len) + if unc == normp[0..<idx].strip(chars = {sep}, leading = false).toUpperAscii: + 8 + else: + 2 + let index = normp.find(sep, start) + if index == -1: + return + var index2 = normp.find(sep, index + 1) + + # a UNC path can't have two slashes in a row (after the initial two) + if index2 == index + 1: + return + if index2 == -1: + index2 = p.len + return (p[0..<index2], p[index2..^1]) + if p[1] == ':': + return (p[0..1], p[2..^1]) diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim new file mode 100644 index 000000000..07a6809bb --- /dev/null +++ b/lib/std/private/osappdirs.nim @@ -0,0 +1,176 @@ +## .. importdoc:: paths.nim, dirs.nim + +include system/inclrtl +import std/envvars +import std/private/ospaths2 + +proc getHomeDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getDataDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + runnableExamples: + import std/os + assert getHomeDir() == expandTilde("~") + + when defined(windows): return getEnv("USERPROFILE") & "\\" + else: return getEnv("HOME") & "/" + +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + +proc getConfigDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + else: + result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") + result.normalizePathEnd(trailingSep = true) + +proc getCacheDir*(): string = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ + # follows https://crates.io/crates/platform-dirs + when defined(windows): + result = getEnv("LOCALAPPDATA") + elif defined(osx): + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches") + else: + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache") + result.normalizePathEnd(false) + +proc getCacheDir*(app: string): string = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + when defined(windows): + getCacheDir() / app / "cache" + else: + getCacheDir() / app + + +when defined(windows): + type DWORD = uint32 + + when defined(nimPreviewSlimSystem): + import std/widestrs + + proc getTempPath( + nBufferLength: DWORD, lpBuffer: WideCString + ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = + ## Retrieves the path of the directory designated for temporary files. + +template getEnvImpl(result: var string, tempDirList: openArray[string]) = + for dir in tempDirList: + if existsEnv(dir): + result = getEnv(dir) + break + +template getTempDirImpl(result: var string) = + when defined(windows): + getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) + else: + getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) + +proc getTempDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## **Note:** This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + const tempDirDefault = "/tmp" + when defined(tempDir): + const tempDir {.strdefine.}: string = tempDirDefault + result = tempDir + else: + when nimvm: + getTempDirImpl(result) + else: + when defined(windows): + let size = getTempPath(0, nil) + # If the function fails, the return value is zero. + if size > 0: + let buffer = newWideCString(size.int) + if getTempPath(size, buffer) > 0: + result = $buffer + elif defined(android): result = "/data/local/tmp" + else: + getTempDirImpl(result) + if result.len == 0: + result = tempDirDefault + normalizePathEnd(result, trailingSep=true) diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim new file mode 100644 index 000000000..c49d52ef2 --- /dev/null +++ b/lib/std/private/oscommon.nim @@ -0,0 +1,186 @@ +include system/inclrtl + +import std/[oserrors] + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osdirs.nim, os.nim + +const weirdTarget* = defined(nimscript) or defined(js) + + +type + ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read + ## operation from the directory + ## structure. + WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write + ## operation to + ## the directory structure. + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix + proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +when defined(windows) and not weirdTarget: + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) + + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) + + proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = + # Note - takes advantage of null delimiter in the cstring + const dot = ord('.') + result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or + f.cFileName[1].int == dot and f.cFileName[2].int == 0) + + +type + PathComponent* = enum ## Enumeration specifying a path component. + ## + ## See also: + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ + pcFile, ## path refers to a file + pcLinkToFile, ## path refers to a symbolic link to a file + pcDir, ## path refers to a directory + pcLinkToDir ## path refers to a symbolic link to a directory + + +when defined(posix) and not weirdTarget: + proc getSymlinkFileKind*(path: string): + tuple[pc: PathComponent, isSpecial: bool] = + # Helper function. + var s: Stat + assert(path != "") + result = (pcLinkToFile, false) + if stat(path, s) == 0'i32: + if S_ISDIR(s.st_mode): + result = (pcLinkToDir, false) + elif not S_ISREG(s.st_mode): + result = (pcLinkToFile, true) + +proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. + ## + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). + ## In case of other errors `OSError` is raised. + ## Returns true in case of success. + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + else: + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError + else: + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) + +when not defined(windows): + const maxSymlinkLen* = 1024 + +proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noNimJs, sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + ## + ## See also: + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, filename) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 + else: + var res: Stat + return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) + + +proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], + noNimJs, sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + ## + ## See also: + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, dir) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + else: + var res: Stat + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + + +proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], + noWeirdTarget, sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + ## + ## See also: + ## * `fileExists proc`_ + ## * `dirExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, link) + if a != -1'i32: + # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` + # may also be needed. + result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 + else: + var res: Stat + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + +when defined(windows) and not weirdTarget: + proc openHandle*(path: string, followSymlink=true, writeAccess=false): Handle = + var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + if not followSymlink: + flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 + + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim new file mode 100644 index 000000000..a44cad7d9 --- /dev/null +++ b/lib/std/private/osdirs.nim @@ -0,0 +1,570 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + +include system/inclrtl +import std/oserrors + + +import ospaths2, osfiles +import oscommon +export dirExists, PathComponent + + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/[posix, times] + +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +# Templates for filtering directories and files +when defined(windows) and not weirdTarget: + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool {.dirty.} = + dirExists(f) + template isFile(f: string): bool {.dirty.} = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` + when defined(windows): + var + f: WIN32_FIND_DATA + res: int + res = findFirstFile(pattern, f) + if res != -1: + defer: findClose(res) + let dotPos = searchExtPos(pattern) + while true: + if not skipFindData(f) and filter(f): + # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check + # that the file extensions have the same length ... + let ff = getFilename(f) + let idx = ff.len - pattern.len + dotPos + if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or + (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): + yield splitFile(pattern).dir / extractFilename(ff) + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: # here we use glob + var + f: Glob + res: int + f.gl_offs = 0 + f.gl_pathc = 0 + f.gl_pathv = nil + res = glob(pattern, 0, nil, addr(f)) + defer: globfree(addr(f)) + if res == 0: + for i in 0.. f.gl_pathc - 1: + assert(f.gl_pathv[i] != nil) + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files and directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + assert "lib/pure/os.nim".unixToNativePath in paths + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + walkCommon(pattern, isDir) + +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + +iterator walkDir*(dir: string; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + ## + ## **Example:** + ## + ## This directory structure: + ## + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + runnableExamples("-r:off"): + import std/[strutils, sugar] + # note: order is not guaranteed + # this also works at compile time + assert collect(for k in walkDir("dirA"): k.path).join(" ") == + "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ + + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + else: + when weirdTarget: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + elif defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h == -1: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: findClose(h) + while true: + var k = pcFile + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: + var d = opendir(dir) + if d == nil: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: discard closedir(d) + while true: + var x = readdir(d) + if x == nil: break + var y = $cast[cstring](addr x.d_name) + if y != "." and y != "..": + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + template resolveSymlink() = + var isSpecial: bool + (k, isSpecial) = getSymlinkFileKind(path) + if skipSpecial and isSpecial: continue + + template kSetGeneric() = # pure Posix component `k` resolution + if lstat(path.cstring, s) < 0'i32: continue # don't yield + elif S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + resolveSymlink() + elif skipSpecial and not S_ISREG(s.st_mode): continue + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + case x.d_type + of DT_DIR: k = pcDir + of DT_LNK: + resolveSymlink() + of DT_UNKNOWN: + kSetGeneric() + else: # DT_REG or special "files" like FIFOs + if skipSpecial and x.d_type != DT_REG: continue + else: discard # leave it as pcFile + else: # assuming that field `d_type` is not present + kSetGeneric() + + yield (k, y) + +iterator walkDirRec*(dir: string, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + + var stack = @[""] + var checkDir = checkDir + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir, + skipSpecial = skipSpecial): + let rel = d / p + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add rel + if k in yieldFilter: + yield if relative: rel else: dir / rel + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + # Future work can provide a way to customize this and do error reporting. + + +proc rawRemoveDir(dir: string) {.noWeirdTarget.} = + when defined(windows): + wrapUnary(res, removeDirectoryW, dir) + let lastError = osLastError() + if res == 0'i32 and lastError.int32 != 3'i32 and + lastError.int32 != 18'i32 and lastError.int32 != 2'i32: + raiseOSError(lastError, dir) + else: + if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) + +proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ + WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + for kind, path in walkDir(dir, checkDir = checkDir): + case kind + of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) + of pcDir: removeDir(path, true) + # for subdirectories there is no benefit in `checkDir = false` + # (unless perhaps for edge case of concurrent processes also deleting + # the same files) + rawRemoveDir(dir) + +proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = + # Try to create one directory (not the whole path). + # returns `true` for success, `false` if the path has previously existed + # + # This is a thin wrapper over mkDir (or alternatives on other systems), + # so in case of a pre-existing path we don't check that it is a directory. + when defined(solaris): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno in {EEXIST, ENOSYS}: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(haiku): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST or errno == EROFS: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(posix): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST: + result = false + else: + #echo res + raiseOSError(osLastError(), dir) + else: + wrapUnary(res, createDirectoryW, dir) + + if res != 0'i32: + result = true + elif getLastError() == 183'i32: + result = false + else: + raiseOSError(osLastError(), dir) + +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + result = not rawCreateDir(dir) + if result: + # path already exists - need to check that it is indeed a directory + if not dirExists(dir): + raise newException(IOError, "Failed to create '" & dir & "'") + +proc createDir*(dir: string) {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + if dir == "": + return + var omitNext = isAbsolute(dir) + for p in parentDirs(dir, fromRoot=true): + if omitNext: + omitNext = false + else: + discard existsOrCreateDir(p) + +proc copyDir*(source, dest: string, skipSpecial = false) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest`. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + createDir(dest) + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDir(path, dest / noSource, skipSpecial = skipSpecial) + else: + copyFile(path, dest / noSource, {cfSymlinkAsIs}) + + +proc copyDirWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + skipSpecial = false) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], + benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + createDir(dest) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + false) + except: + if not ignorePermissionErrors: + raise + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors, skipSpecial = skipSpecial) + else: + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) + +proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) + +proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + when defined(windows): + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) + else: + if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim new file mode 100644 index 000000000..37d8eabca --- /dev/null +++ b/lib/std/private/osfiles.nim @@ -0,0 +1,416 @@ +include system/inclrtl +import std/private/since +import std/oserrors + +import oscommon +export fileExists + +import ospaths2, ossymlinks + +## .. importdoc:: osdirs.nim, os.nim + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/[posix, times] + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +type + FilePermission* = enum ## File access permission, modelled after UNIX. + ## + ## See also: + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ + fpUserExec, ## execute access for the file owner + fpUserWrite, ## write access for the file owner + fpUserRead, ## read access for the file owner + fpGroupExec, ## execute access for the group + fpGroupWrite, ## write access for the group + fpGroupRead, ## read access for the group + fpOthersExec, ## execute access for others + fpOthersWrite, ## write access for others + fpOthersRead ## read access for others + +proc getFilePermissions*(filename: string): set[FilePermission] {. + rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other + ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var a: Stat + if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) + result = {} + if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) + if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) + if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) + + if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) + if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) + if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) + + if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) + if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) + if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: + result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, + fpOthersExec, fpOthersRead} + else: + result = {fpUserExec..fpOthersRead} + +proc setFilePermissions*(filename: string, permissions: set[FilePermission], + followSymlinks = true) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], + noWeirdTarget.} = + ## Sets the file permissions for `filename`. + ## + ## If `followSymlinks` set to true (default) and ``filename`` points to a + ## symlink, permissions are set to the file symlink points to. + ## `followSymlinks` set to false is a noop on Windows and some POSIX + ## systems (including Linux) on which `lchmod` is either unavailable or always + ## fails, given that symlinks permissions there are not observed. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var p = 0.Mode + if fpUserRead in permissions: p = p or S_IRUSR.Mode + if fpUserWrite in permissions: p = p or S_IWUSR.Mode + if fpUserExec in permissions: p = p or S_IXUSR.Mode + + if fpGroupRead in permissions: p = p or S_IRGRP.Mode + if fpGroupWrite in permissions: p = p or S_IWGRP.Mode + if fpGroupExec in permissions: p = p or S_IXGRP.Mode + + if fpOthersRead in permissions: p = p or S_IROTH.Mode + if fpOthersWrite in permissions: p = p or S_IWOTH.Mode + if fpOthersExec in permissions: p = p or S_IXOTH.Mode + + if not followSymlinks and filename.symlinkExists: + when declared(lchmod): + if lchmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + if chmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if fpUserWrite in permissions: + res = res and not FILE_ATTRIBUTE_READONLY + else: + res = res or FILE_ATTRIBUTE_READONLY + wrapBinary(res2, setFileAttributesW, filename, res) + if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) + + +const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) + # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` + +when hasCCopyfile: + # `copyfile` API available since osx 10.5. + {.push nodecl, header: "<copyfile.h>".} + type + copyfile_state_t {.nodecl.} = pointer + copyfile_flags_t = cint + proc copyfile_state_alloc(): copyfile_state_t + proc copyfile_state_free(state: copyfile_state_t): cint + proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} + when (NimMajor, NimMinor) >= (1, 4): + let + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + else: + var + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + {.pop.} + +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks + +const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} + +proc copyFile*(source, dest: string, options = {cfSymlinkFollow}; bufferSize = 16_384) {.rtl, + extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], + noWeirdTarget.} = + ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ + ## procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes + ## will be preserved and the content overwritten. + ## + ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless + ## `-d:nimLegacyCopyFile` is used. + ## + ## `copyFile` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + + doAssert card(copyFlagSymlink * options) == 1, "There should be exactly one cfSymlink* in options" + let isSymlink = source.symlinkExists + if isSymlink and (cfSymlinkIgnore in options or defined(windows)): + return + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) + else: + if isSymlink and cfSymlinkAsIs in options: + createSymlink(expandSymlink(source), dest) + else: + when hasCCopyfile: + let state = copyfile_state_alloc() + # xxx `COPYFILE_STAT` could be used for one-shot + # `copyFileWithPermissions`. + let status = c_copyfile(source.cstring, dest.cstring, state, + COPYFILE_DATA) + if status != 0: + let err = osLastError() + discard copyfile_state_free(state) + raiseOSError(err, $(source, dest)) + let status2 = copyfile_state_free(state) + if status2 != 0: raiseOSError(osLastError(), $(source, dest)) + else: + # generic version of copyFile which works for any platform: + var d, s: File + if not open(s, source): raiseOSError(osLastError(), source) + if not open(d, dest, fmWrite): + close(s) + raiseOSError(osLastError(), dest) + + # Hints for kernel-level aggressive sequential low-fragmentation read-aheads: + # https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_fadvise.html + when defined(linux) or defined(osx): + discard posix_fadvise(getFileHandle(d), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + discard posix_fadvise(getFileHandle(s), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + + var buf = alloc(bufferSize) + while true: + var bytesread = readBuffer(s, buf, bufferSize) + if bytesread > 0: + var byteswritten = writeBuffer(d, buf, bytesread) + if bytesread != byteswritten: + dealloc(buf) + close(s) + close(d) + raiseOSError(osLastError(), dest) + if bytesread != bufferSize: break + dealloc(buf) + close(s) + flushFile(d) + close(d) + +proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}; bufferSize = 16_384) + {.noWeirdTarget, since: (1,3,7).} = + ## Copies a file `source` into directory `dir`, which must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + if dir.len == 0: # treating "" as "." is error prone + raise newException(ValueError, "dest is empty") + copyFile(source, dir / source.lastPathPart, options, bufferSize) + + +proc copyFileWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + options = {cfSymlinkFollow}) {.noWeirdTarget.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ + copyFile(source, dest, options) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + (cfSymlinkFollow in options)) + except: + if not ignorePermissionErrors: + raise + +when not declared(ENOENT) and not defined(windows): + when defined(nimscript): + when not defined(haiku): + const ENOENT = cint(2) # 2 on most systems including Solaris + else: + const ENOENT = cint(-2147459069) + else: + var ENOENT {.importc, header: "<errno.h>".}: cint + +when defined(windows) and not weirdTarget: + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) + +proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + result = true + when defined(windows): + let f = newWideCString(file) + if deleteFile(f) == 0: + result = false + let err = getLastError() + if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: + result = true + elif err == ERROR_ACCESS_DENIED and + setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and + deleteFile(f) != 0: + result = true + else: + if unlink(file) != 0'i32 and errno != ENOENT: + result = false + +proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ + if not tryRemoveFile(file): + raiseOSError(osLastError(), file) + +proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ + + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + raiseAssert "unreachable" + else: + # Fallback to copy & del + copyFileWithPermissions(source, dest, options={cfSymlinkAsIs}) + try: + removeFile(source) + except: + discard tryRemoveFile(dest) + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim new file mode 100644 index 000000000..bc69ff725 --- /dev/null +++ b/lib/std/private/ospaths2.nim @@ -0,0 +1,1030 @@ +include system/inclrtl +import std/private/since + +import std/[strutils, pathnorm] +import std/oserrors + +import oscommon +export ReadDirEffect, WriteDirEffect + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + +const weirdTarget = defined(nimscript) or defined(js) + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix, system/ansi_c +else: + {.error: "OS module not ported to your operating system!".} + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} + + +import std/private/osseps +export osseps + +proc absolutePathInternal(path: string): string {.gcsafe.} + +proc normalizePathEnd*(path: var string, trailingSep = false) = + ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## ``trailingSep``, and taking care of edge cases: it preservers whether + ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, + ## not `AltSep`. Trailing `/.` are compressed, see examples. + if path.len == 0: return + var i = path.len + while i >= 1: + if path[i-1] in {DirSep, AltSep}: dec(i) + elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) + else: break + if trailingSep: + # foo// => foo + path.setLen(i) + # foo => foo/ + path.add DirSep + elif i > 0: + # foo// => foo + path.setLen(i) + else: + # // => / (empty case was already taken care of) + path = $DirSep + +proc normalizePathEnd*(path: string, trailingSep = false): string = + ## outplace overload + runnableExamples: + when defined(posix): + assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" + assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" + assert normalizePathEnd(".//./.", trailingSep = false) == "." + assert normalizePathEnd("", trailingSep = true) == "" # not / ! + assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! + result = path + result.normalizePathEnd(trailingSep) + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +proc joinPathImpl(result: var string, state: var int, tail: string) = + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) + normalizePathEnd(result, trailingSep=false) + addNormalizePath(tail, result, state, DirSep) + normalizePathEnd(result, trailingSep=trailingSep) + +proc joinPath*(head, tail: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("usr", "lib") == "usr/lib" + assert joinPath("usr", "lib/") == "usr/lib/" + assert joinPath("usr", "") == "usr" + assert joinPath("usr/", "") == "usr/" + assert joinPath("", "") == "" + assert joinPath("", "lib") == "lib" + assert joinPath("", "/lib") == "/lib" + assert joinPath("usr/", "/lib") == "usr/lib" + assert joinPath("usr/lib", "../bin") == "usr/bin" + + result = newStringOfCap(head.len + tail.len) + var state = 0 + joinPathImpl(result, state, head) + joinPathImpl(result, state, tail) + when false: + if len(head) == 0: + result = tail + elif head[len(head)-1] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & substr(tail, 1) + else: + result = head & tail + else: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & tail + else: + result = head & DirSep & tail + +proc joinPath*(parts: varargs[string]): string {.noSideEffect, + rtl, extern: "nos$1OpenArray".} = + ## The same as `joinPath(head, tail) proc`_, + ## but works with any number of directory parts. + ## + ## You need to pass at least one element or the proc + ## will assert in debug builds and crash on release builds. + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ + runnableExamples: + when defined(posix): + assert joinPath("a") == "a" + assert joinPath("a", "b", "c") == "a/b/c" + assert joinPath("usr/lib", "../../var", "log") == "var/log" + + var estimatedLen = 0 + for p in parts: estimatedLen += p.len + result = newStringOfCap(estimatedLen) + var state = 0 + for i in 0..high(parts): + joinPathImpl(result, state, parts[i]) + +proc `/`*(head, tail: string): string {.noSideEffect, inline.} = + ## The same as `joinPath(head, tail) proc`_. + ## + ## See also: + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert "usr" / "" == "usr" + assert "" / "lib" == "lib" + assert "" / "/lib" == "/lib" + assert "usr/" / "/lib/" == "usr/lib/" + assert "usr" / "lib" / "../bin" == "usr/bin" + + result = joinPath(head, tail) + +when doslikeFileSystem: + import std/private/ntpath + +proc splitPath*(path: string): tuple[head, tail: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + runnableExamples: + assert splitPath("usr/local/bin") == ("usr/local", "bin") + assert splitPath("usr/local/bin/") == ("usr/local/bin", "") + assert splitPath("/bin/") == ("/bin", "") + when (NimMajor, NimMinor) <= (1, 0): + assert splitPath("/bin") == ("", "bin") + else: + assert splitPath("/bin") == ("/", "bin") + assert splitPath("bin") == ("", "bin") + assert splitPath("") == ("", "") + + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(path) + let stop = drive.len + else: + const stop = 0 + + var sepPos = -1 + for i in countdown(len(path)-1, stop): + if path[i] in {DirSep, AltSep}: + sepPos = i + break + if sepPos >= 0: + result.head = substr(path, 0, + when (NimMajor, NimMinor) <= (1, 0): + sepPos-1 + else: + if likely(sepPos >= 1): sepPos-1 else: 0 + ) + result.tail = substr(path, sepPos+1) + else: + when doslikeFileSystem: + result.head = drive + result.tail = splitpath + else: + result.head = "" + result.tail = path + +proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + runnableExamples: + assert not "".isAbsolute + assert not ".".isAbsolute + when defined(posix): + assert "/".isAbsolute + assert not "a/".isAbsolute + assert "/a/".isAbsolute + + if len(path) == 0: return false + + when doslikeFileSystem: + var len = len(path) + result = (path[0] in {'/', '\\'}) or + (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') + elif defined(macos): + # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path + result = path[0] != ':' + elif defined(RISCOS): + result = path[0] == '$' + elif defined(posix): + result = path[0] == '/' + elif defined(nodejs): + {.emit: [result," = require(\"path\").isAbsolute(",path.cstring,");"].} + else: + raiseAssert "unreachable" # if ever hits here, adapt as needed + +when FileSystemCaseSensitive: + template `!=?`(a, b: char): bool = a != b +else: + template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) + +when doslikeFileSystem: + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = + ## An absolute path from the root of the current drive (e.g. "\foo") + path.len > 0 and + (path[0] == AltSep or + (path[0] == DirSep and + (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) + + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = + ## Return true if path1 and path2 have a same root. + ## + ## Detail of Windows path formats: + ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats + + assert(isAbsolute(path1)) + assert(isAbsolute(path2)) + + if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): + result = true + elif cmpIgnoreCase(splitDrive(path1).drive, splitDrive(path2).drive) == 0: + result = true + else: + result = false + +proc relativePath*(path, base: string, sep = DirSep): string {. + rtl, extern: "nos$1".} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep_) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + runnableExamples: + assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + when not doslikeFileSystem: # On Windows, UNC-paths start with `//` + assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + assert relativePath("", "/users/moo", '/') == "" + assert relativePath("foo", ".", '/') == "foo" + assert relativePath("foo", "foo", '/') == "." + + if path.len == 0: return "" + var base = if base == ".": "" else: base + var path = path + path.normalizePathAux + base.normalizePathAux + let a1 = isAbsolute(path) + let a2 = isAbsolute(base) + if a1 and not a2: + base = absolutePathInternal(base) + elif a2 and not a1: + path = absolutePathInternal(path) + + when doslikeFileSystem: + if isAbsolute(path) and isAbsolute(base): + if not sameRoot(path, base): + return path + + var f = default PathIter + var b = default PathIter + var ff = (0, -1) + var bb = (0, -1) # (int, int) + result = newStringOfCap(path.len) + # skip the common prefix: + while f.hasNext(path) and b.hasNext(base): + ff = next(f, path) + bb = next(b, base) + let diff = ff[1] - ff[0] + if diff != bb[1] - bb[0]: break + var same = true + for i in 0..diff: + if path[i + ff[0]] !=? base[i + bb[0]]: + same = false + break + if not same: break + ff = (0, -1) + bb = (0, -1) + # for i in 0..diff: + # result.add base[i + bb[0]] + + # /foo/bar/xxx/ -- base + # /foo/bar/baz -- path path + # ../baz + # every directory that is in 'base', needs to add '..' + while true: + if bb[1] >= bb[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + result.add ".." + if not b.hasNext(base): break + bb = b.next(base) + + # add the rest of 'path': + while true: + if ff[1] >= ff[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + for i in 0..ff[1] - ff[0]: + result.add path[i + ff[0]] + if not f.hasNext(path): break + ff = f.next(path) + + when not defined(nimOldRelativePathBehavior): + if result.len == 0: result.add "." + +proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = + ## Returns true if `path` is relative to `base`. + runnableExamples: + doAssert isRelativeTo("./foo//bar", "foo") + doAssert isRelativeTo("foo/bar", ".") + doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") + doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") + let path = path.normalizePath + let base = base.normalizePath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc parentDirPos(path: string): int = + var q = 1 + if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 + for i in countdown(len(path)-q, 0): + if path[i] in {DirSep, AltSep}: return i + result = -1 + +proc parentDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + runnableExamples: + assert parentDir("") == "" + when defined(posix): + assert parentDir("/usr/local/bin") == "/usr/local" + assert parentDir("foo/bar//") == "foo" + assert parentDir("//foo//bar//.") == "/foo" + assert parentDir("./foo") == "." + assert parentDir("/./foo//./") == "/" + assert parentDir("a//./") == "." + assert parentDir("a/b/c/..") == "a" + result = pathnorm.normalizePath(path) + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(result) + result = splitpath + var sepPos = parentDirPos(result) + if sepPos >= 0: + result = substr(result, 0, sepPos) + normalizePathEnd(result) + elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: + # `.` => `..` and .. => `../..`(etc) would be a sensible alternative + # `/` => `/` (as done with splitFile) would be a sensible alternative + result = "" + else: + result = "." + when doslikeFileSystem: + if result.len == 0: + discard + elif drive.len > 0 and result.len == 1 and result[0] in {DirSep, AltSep}: + result = drive + else: + result = drive & result + +proc tailDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + runnableExamples: + assert tailDir("/bin") == "bin" + assert tailDir("bin") == "" + assert tailDir("bin/") == "" + assert tailDir("/usr/local/bin") == "usr/local/bin" + assert tailDir("//usr//local//bin//") == "usr//local//bin//" + assert tailDir("./usr/local/bin") == "usr/local/bin" + assert tailDir("usr/local/bin") == "local/bin" + + var i = 0 + when doslikeFileSystem: + let (drive, splitpath) = path.splitDrive + if drive != "": + return splitpath.strip(chars = {DirSep, AltSep}, trailing = false) + while i < len(path): + if path[i] in {DirSep, AltSep}: + while i < len(path) and path[i] in {DirSep, AltSep}: inc i + return substr(path, i) + inc i + result = "" + +proc isRootDir*(path: string): bool {. + noSideEffect, rtl, extern: "nos$1".} = + ## Checks whether a given `path` is a root directory. + runnableExamples: + assert isRootDir("") + assert isRootDir(".") + assert isRootDir("/") + assert isRootDir("a") + assert not isRootDir("/a") + assert not isRootDir("a/b/c") + + when doslikeFileSystem: + if splitDrive(path).path == "": + return true + result = parentDirPos(path) < 0 + +iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a + + if not fromRoot: + var current = path + if inclusive: yield path + while true: + if current.isRootDir: break + current = current.parentDir + yield current + else: + when doslikeFileSystem: + let start = path.splitDrive.drive.len + else: + const start = 0 + for i in countup(start, path.len - 2): # ignore the last / + # deal with non-normalized paths such as /foo//bar//baz + if path[i] in {DirSep, AltSep} and + (i == 0 or path[i-1] notin {DirSep, AltSep}): + yield path.substr(0, i) + + if inclusive: yield path + +proc `/../`*(head, tail: string): string {.noSideEffect.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + runnableExamples: + when defined(posix): + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" + + when doslikeFileSystem: + let (drive, head) = splitDrive(head) + let sepPos = parentDirPos(head) + if sepPos >= 0: + result = substr(head, 0, sepPos-1) / tail + else: + result = head / tail + when doslikeFileSystem: + result = drive / result + +proc normExt(ext: string): string = + if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here + else: result = ExtSep & ext + +proc searchExtPos*(path: string): int = + ## Returns index of the `'.'` char in `path` if it signifies the beginning + ## of the file extension. Returns -1 otherwise. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert searchExtPos("a/b/c") == -1 + assert searchExtPos("c.nim") == 1 + assert searchExtPos("a/b/c.nim") == 5 + assert searchExtPos("a.b.c.nim") == 5 + assert searchExtPos(".nim") == -1 + assert searchExtPos("..nim") == -1 + assert searchExtPos("a..nim") == 2 + + # Unless there is any char that is not `ExtSep` before last `ExtSep` in the file name, + # it is not a file extension. + const DirSeps = when doslikeFileSystem: {DirSep, AltSep, ':'} else: {DirSep, AltSep} + result = -1 + var i = path.high + while i >= 1: + if path[i] == ExtSep: + break + elif path[i] in DirSeps: + return -1 # do not skip over path + dec i + + for j in countdown(i - 1, 0): + if path[j] in DirSeps: + return -1 + elif path[j] != ExtSep: + result = i + break + +proc splitFile*(path: string): tuple[dir, name, ext: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep_ unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + var (dir, name, ext) = splitFile("usr/local/nimc.html") + assert dir == "usr/local" + assert name == "nimc" + assert ext == ".html" + (dir, name, ext) = splitFile("/usr/local/os") + assert dir == "/usr/local" + assert name == "os" + assert ext == "" + (dir, name, ext) = splitFile("/usr/local/") + assert dir == "/usr/local" + assert name == "" + assert ext == "" + (dir, name, ext) = splitFile("/tmp.txt") + assert dir == "/" + assert name == "tmp" + assert ext == ".txt" + + var namePos = 0 + var dotPos = 0 + when doslikeFileSystem: + let (drive, _) = splitDrive(path) + let stop = len(drive) + result.dir = drive + else: + const stop = 0 + for i in countdown(len(path) - 1, stop): + if path[i] in {DirSep, AltSep} or i == 0: + if path[i] in {DirSep, AltSep}: + result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) + namePos = i + 1 + if dotPos > i: + result.name = substr(path, namePos, dotPos - 1) + result.ext = substr(path, dotPos) + else: + result.name = substr(path, namePos) + break + elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and + path[i - 1] notin {DirSep, AltSep} and + path[i + 1] != ExtSep and dotPos == 0: + dotPos = i + +proc extractFilename*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert extractFilename("foo/bar/") == "" + assert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar.baz") == "bar.baz" + + if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: + result = "" + else: + result = splitPath(path).tail + +proc lastPathPart*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar") == "bar" + + let path = path.normalizePathEnd(trailingSep = false) + result = extractFilename(path) + +proc changeFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert changeFileExt("foo.bar", "baz") == "foo.baz" + assert changeFileExt("foo.bar", "") == "foo" + assert changeFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = substr(filename, 0, extPos-1) & normExt(ext) + +proc addFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + runnableExamples: + assert addFileExt("foo.bar", "baz") == "foo.bar" + assert addFileExt("foo.bar", "") == "foo.bar" + assert addFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = filename + +proc cmpPaths*(pathA, pathB: string): int {. + noSideEffect, rtl, extern: "nos$1".} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. Returns: + ## + ## | `0` if pathA == pathB + ## | `< 0` if pathA < pathB + ## | `> 0` if pathA > pathB + runnableExamples: + when defined(macosx): + assert cmpPaths("foo", "Foo") == 0 + elif defined(posix): + assert cmpPaths("foo", "Foo") > 0 + + let a = normalizePath(pathA) + let b = normalizePath(pathB) + if FileSystemCaseSensitive: + result = cmp(a, b) + else: + when defined(nimscript): + result = cmpic(a, b) + elif defined(nimdoc): discard + else: + result = cmpIgnoreCase(a, b) + +proc unixToNativePath*(path: string, drive=""): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + when defined(unix): + result = path + else: + if path.len == 0: return "" + + var start: int + if path[0] == '/': + # an absolute path + when doslikeFileSystem: + if drive != "": + result = drive & ":" & DirSep + else: + result = $DirSep + elif defined(macos): + result = "" # must not start with ':' + else: + result = $DirSep + start = 1 + elif path[0] == '.' and (path.len == 1 or path[1] == '/'): + # current directory + result = $CurDir + start = when doslikeFileSystem: 1 else: 2 + else: + result = "" + start = 0 + + var i = start + while i < len(path): # ../../../ --> :::: + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + # parent directory + when defined(macos): + if result[high(result)] == ':': + add result, ':' + else: + add result, ParDir + else: + add result, ParDir & DirSep + inc(i, 3) + elif path[i] == '/': + add result, DirSep + inc(i) + else: + add result, path[i] + inc(i) + + +when not defined(nimscript): + proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + when defined(nodejs): + var ret: cstring + {.emit: "`ret` = process.cwd();".} + return $ret + elif defined(js): + raiseAssert "use -d:nodejs to have `getCurrentDir` defined" + elif defined(windows): + var bufsize = MAX_PATH.int32 + var res = newWideCString(bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break + else: + var bufsize = 1024 # should be enough + result = newString(bufsize) + while true: + if getcwd(result.cstring, bufsize) != nil: + setLen(result, c_strlen(result.cstring)) + break + else: + let err = osLastError() + if err.int32 == ERANGE: + bufsize = bufsize shl 1 + doAssert(bufsize >= 0) + result = newString(bufsize) + else: + raiseOSError(osLastError()) + +proc absolutePath*(path: string, root = getCurrentDir()): string = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ + runnableExamples: + assert absolutePath("a") == getCurrentDir() / "a" + + if isAbsolute(path): path + else: + if not root.isAbsolute: + raise newException(ValueError, "The specified root is not absolute: " & root) + joinPath(root, path) + +proc absolutePathInternal(path: string): string = + absolutePath(path, getCurrentDir()) + + +proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = + ## Normalize a path. + ## + ## Consecutive directory separators are collapsed, including an initial double slash. + ## + ## On relative paths, double dot (`..`) sequences are collapsed if possible. + ## On absolute paths they are always collapsed. + ## + ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. + ## Triple dot is not handled. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ + runnableExamples: + when defined(posix): + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" + + path = pathnorm.normalizePath(path) + when false: + let isAbs = isAbsolute(path) + var stack: seq[string] = @[] + for p in split(path, {DirSep}): + case p + of "", ".": + continue + of "..": + if stack.len == 0: + if isAbs: + discard # collapse all double dots on absoluta paths + else: + stack.add(p) + elif stack[^1] == "..": + stack.add(p) + else: + discard stack.pop() + else: + stack.add(p) + + if isAbs: + path = DirSep & join(stack, $DirSep) + elif stack.len > 0: + path = join(stack, $DirSep) + else: + path = "." + +proc normalizePathAux(path: var string) = normalizePath(path) + +proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns a normalized path for the current OS. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version + runnableExamples: + when defined(posix): + assert normalizedPath("a///b//..//c///d") == "a/c/d" + result = pathnorm.normalizePath(path) + +proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = + ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. + runnableExamples: + import std/sugar + when defined(posix): + doAssert "foo".dup(normalizeExe) == "./foo" + doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" + doAssert "".dup(normalizeExe) == "" + when defined(posix): + if file.len > 0 and DirSep notin file and file != "." and file != "..": + file = "./" & file + +proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to the same physical + ## file or directory. + ## + ## Raises `OSError` if any of the files does not + ## exist or information about it can not be obtained. + ## + ## This proc will return true if given two alternative hard-linked or + ## sym-linked paths to the same file or directory. + ## + ## See also: + ## * `sameFileContent proc`_ + when defined(windows): + var success = true + var f1 = openHandle(path1) + var f2 = openHandle(path2) + + var lastErr: OSErrorCode + if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: + var fi1, fi2: BY_HANDLE_FILE_INFORMATION + + if getFileInformationByHandle(f1, addr(fi1)) != 0 and + getFileInformationByHandle(f2, addr(fi2)) != 0: + result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and + fi1.nFileIndexHigh == fi2.nFileIndexHigh and + fi1.nFileIndexLow == fi2.nFileIndexLow + else: + lastErr = osLastError() + success = false + else: + lastErr = osLastError() + success = false + + discard closeHandle(f1) + discard closeHandle(f2) + + if not success: raiseOSError(lastErr, $(path1, path2)) + else: + var a, b: Stat + if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: + raiseOSError(osLastError(), $(path1, path2)) + else: + result = a.st_dev == b.st_dev and a.st_ino == b.st_ino diff --git a/lib/std/private/osseps.nim b/lib/std/private/osseps.nim new file mode 100644 index 000000000..f2d49d886 --- /dev/null +++ b/lib/std/private/osseps.nim @@ -0,0 +1,113 @@ +# Include file that implements 'DirSep' and friends. Do not import this when +# you also import `os.nim`! + +# Improved based on info in 'compiler/platform.nim' + +## .. importdoc:: ospaths2.nim + +const + doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) + +const + CurDir* = + when defined(macos): ':' + elif defined(genode): '/' + else: '.' + ## The constant character used by the operating system to refer to the + ## current directory. + ## + ## For example: `'.'` for POSIX or `':'` for the classic Macintosh. + + ParDir* = + when defined(macos): "::" + else: ".." + ## The constant string used by the operating system to refer to the + ## parent directory. + ## + ## For example: `".."` for POSIX or `"::"` for the classic Macintosh. + + DirSep* = + when defined(macos): ':' + elif doslikeFileSystem or defined(vxworks): '\\' + elif defined(RISCOS): '.' + else: '/' + ## The character used by the operating system to separate pathname + ## components, for example: `'/'` for POSIX, `':'` for the classic + ## Macintosh, and `'\\'` on Windows. + + AltSep* = + when doslikeFileSystem: '/' + else: DirSep + ## An alternative character used by the operating system to separate + ## pathname components, or the same as DirSep_ if only one separator + ## character exists. This is set to `'/'` on Windows systems + ## where DirSep_ is a backslash (`'\\'`). + + PathSep* = + when defined(macos) or defined(RISCOS): ',' + elif doslikeFileSystem or defined(vxworks): ';' + elif defined(PalmOS) or defined(MorphOS): ':' # platform has ':' but osseps has ';' + else: ':' + ## The character conventionally used by the operating system to separate + ## search path components (as in PATH), such as `':'` for POSIX + ## or `';'` for Windows. + + FileSystemCaseSensitive* = + when defined(macos) or defined(macosx) or doslikeFileSystem or defined(vxworks) or + defined(PalmOS) or defined(MorphOS): false + else: true + ## True if the file system is case sensitive, false otherwise. Used by + ## `cmpPaths proc`_ to compare filenames properly. + + ExeExt* = + when doslikeFileSystem: "exe" + elif defined(atari): "tpp" + elif defined(netware): "nlm" + elif defined(vxworks): "vxe" + elif defined(nintendoswitch): "elf" + else: "" + ## The file extension of native executables. For example: + ## `""` for POSIX, `"exe"` on Windows (without a dot). + + ScriptExt* = + when doslikeFileSystem: "bat" + else: "" + ## The file extension of a script file. For example: `""` for POSIX, + ## `"bat"` on Windows. + + DynlibFormat* = + when defined(macos): "$1.dylib" # platform has $1Lib + elif defined(macosx): "lib$1.dylib" + elif doslikeFileSystem or defined(atari): "$1.dll" + elif defined(MorphOS): "$1.prc" + elif defined(PalmOS): "$1.prc" # platform has lib$1.so + elif defined(genode): "$1.lib.so" + elif defined(netware): "$1.nlm" + elif defined(amiga): "$1.Library" + else: "lib$1.so" + ## The format string to turn a filename into a `DLL`:idx: file (also + ## called `shared object`:idx: on some operating systems). + + ExtSep* = '.' + ## The character which separates the base filename from the extension; + ## for example, the `'.'` in `os.nim`. + + # MacOS paths + # =========== + # MacOS directory separator is a colon ":" which is the only character not + # allowed in filenames. + # + # A path containing no colon or which begins with a colon is a partial + # path. + # E.g. ":kalle:petter" ":kalle" "kalle" + # + # All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:" + # When generating paths, one is safe if one ensures that all partial paths + # begin with a colon, and all full paths end with a colon. + # In full paths the first name (e g HD above) is the name of a mounted + # volume. + # These names are not unique, because, for instance, two diskettes with the + # same names could be inserted. This means that paths on MacOS are not + # waterproof. In case of equal names the first volume found will do. + # Two colons "::" are the relative path to the parent. Three is to the + # grandparent etc. diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim new file mode 100644 index 000000000..c1760c42e --- /dev/null +++ b/lib/std/private/ossymlinks.nim @@ -0,0 +1,78 @@ +include system/inclrtl +import std/oserrors + +import oscommon +export symlinkExists + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +## .. importdoc:: os.nim + +proc createSymlink*(src, dest: string) {.noWeirdTarget.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + + when defined(windows): + const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 + # allows anyone with developer mode on to create a link + let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) + else: + if symlink(src, dest) != 0: + raiseOSError(osLastError(), $(src, dest)) + +proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + when defined(windows) or defined(nintendoswitch): + result = symlinkPath + else: + var bufLen = 1024 + while true: + result = newString(bufLen) + let len = readlink(symlinkPath.cstring, result.cstring, bufLen) + if len < 0: + raiseOSError(osLastError(), symlinkPath) + if len < bufLen: + result.setLen(len) + break + bufLen = bufLen shl 1 diff --git a/lib/std/private/schubfach.nim b/lib/std/private/schubfach.nim new file mode 100644 index 000000000..b8c85d2bc --- /dev/null +++ b/lib/std/private/schubfach.nim @@ -0,0 +1,436 @@ +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +# -------------------------------------------------------------------------------------------------- +## This file contains an implementation of the Schubfach algorithm as described in +## +## \[1] Raffaello Giulietti, "The Schubfach way to render doubles", +## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN +# -------------------------------------------------------------------------------------------------- + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + + +template sf_Assert(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType = float32 + BitsType = uint32 + Single {.bycopy.} = object + bits: BitsType + +const + significandSize: int32 = 24 + MaxExponent = 128 + exponentBias: int32 = MaxExponent - 1 + (significandSize - 1) + maxIeeeExponent: BitsType = BitsType(2 * MaxExponent - 1) + hiddenBit: BitsType = BitsType(1) shl (significandSize - 1) + significandMask: BitsType = hiddenBit - 1 + exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) + signMask: BitsType = not (not BitsType(0) shr 1) + +proc constructSingle(bits: BitsType): Single = + result.bits = bits + +proc constructSingle(value: ValueType): Single = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent(this: Single): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero(this: Single): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit(this: Single): int {.noSideEffect.} = + return int((this.bits and signMask) != 0) + +# ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2(x: int32; n: int32): int32 {.inline.} = + return x shr n + +## Returns floor(log_10(2^e)) +## ```c +## static inline int32_t FloorLog10Pow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611, 22); +## } +## ``` +## Returns floor(log_10(3/4 2^e)) +## ```c +## static inline int32_t FloorLog10ThreeQuartersPow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611 - 524031, 22); +## } +## ``` +## Returns floor(log_2(10^e)) + +proc floorLog2Pow10(e: int32): int32 {.inline.} = + sf_Assert(e >= -1233) + sf_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + +proc computePow10Single(k: int32): uint64 {.inline.} = + ## There are unique beta and r such that 10^k = beta 2^r and + ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and + ## beta = 2^-r 10^k. + ## Let g = ceil(beta), so (g-1) 2^r < 10^k <= g 2^r, with the latter + ## value being a pretty good overestimate for 10^k. + ## NB: Since for all the required exponents k, we have g < 2^64, + ## all constants can be stored in 128-bit integers. + sf_Assert(k >= kMin) + sf_Assert(k <= kMax) + return g[k - kMin] + +proc lo32(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + +proc hi32(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + +when defined(sizeof_Int128): + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let p: uint128 = uint128(g) * cp + let y1: uint32 = lo32(cast[uint64](p shr 64)) + let y0: uint32 = hi32(cast[uint64](p)) + return y1 or uint32(y0 > 1) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = + var p1: uint64 = 0 + var p0: uint64 = umul128(g, cpHi, addr(p1)) + let y1: uint32 = lo32(p1) + let y0: uint32 = hi32(p0) + return y1 or uint32(y0 > 1) + +else: + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let b01: uint64 = uint64(lo32(g)) * cp + let b11: uint64 = uint64(hi32(g)) * cp + let hi: uint64 = b11 + hi32(b01) + let y1: uint32 = hi32(hi) + let y0: uint32 = lo32(hi) + return y1 or uint32(y0 > 1) + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2(value: uint32; e2: int32): bool {.inline.} = + sf_Assert(e2 >= 0) + sf_Assert(e2 <= 31) + return (value and ((uint32(1) shl e2) - 1)) == 0 + +type + FloatingDecimal32 {.bycopy.} = object + digits: uint32 ## num_digits <= 9 + exponent: int32 + +proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal32 {. + inline.} = + var c: uint32 + var q: int32 + if ieeeExponent != 0: + c = hiddenBit or ieeeSignificand + q = cast[int32](ieeeExponent) - exponentBias + if 0 <= -q and -q < significandSize and multipleOfPow2(c, -q): + return FloatingDecimal32(digits: c shr -q, exponent: 0'i32) + else: + c = ieeeSignificand + q = 1 - exponentBias + let isEven: bool = (c mod 2 == 0) + let lowerBoundaryIsCloser: bool = (ieeeSignificand == 0 and ieeeExponent > 1) + ## const int32_t qb = q - 2; + let cbl: uint32 = 4 * c - 2 + uint32(lowerBoundaryIsCloser) + let cb: uint32 = 4 * c + let cbr: uint32 = 4 * c + 2 + ## (q * 1262611 ) >> 22 == floor(log_10( 2^q)) + ## (q * 1262611 - 524031) >> 22 == floor(log_10(3/4 2^q)) + sf_Assert(q >= -1500) + sf_Assert(q <= 1500) + let k: int32 = floorDivPow2(q * 1262611 - (if lowerBoundaryIsCloser: 524031 else: 0), 22) + let h: int32 = q + floorLog2Pow10(-k) + 1 + sf_Assert(h >= 1) + sf_Assert(h <= 4) + let pow10: uint64 = computePow10Single(-k) + let vbl: uint32 = roundToOdd(pow10, cbl shl h) + let vb: uint32 = roundToOdd(pow10, cb shl h) + let vbr: uint32 = roundToOdd(pow10, cbr shl h) + let lower: uint32 = vbl + uint32(not isEven) + let upper: uint32 = vbr - uint32(not isEven) + ## See Figure 4 in [1]. + ## And the modifications in Figure 6. + let s: uint32 = vb div 4 + ## NB: 4 * s == vb & ~3 == vb & -4 + if s >= 10: + let sp: uint32 = s div 10 + ## = vb / 40 + let upInside: bool = lower <= 40 * sp + let wpInside: bool = 40 * sp + 40 <= upper + ## if (up_inside || wp_inside) // NB: At most one of u' and w' is in R_v. + if upInside != wpInside: + return FloatingDecimal32(digits: sp + uint32(wpInside), exponent: k + 1) + let uInside: bool = lower <= 4 * s + let wInside: bool = 4 * s + 4 <= upper + if uInside != wInside: + return FloatingDecimal32(digits: s + uint32(wInside), exponent: k) + let mid: uint32 = 4 * s + 2 + ## = 2(s + t) + let roundUp: bool = vb > mid or (vb == mid and (s and 1) != 0) + return FloatingDecimal32(digits: s + uint32(roundUp), exponent: k) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc printDecimalDigitsBackwards[T: Ordinal](buf: var openArray[char]; pos: T; output: uint32): int {.inline.} = + var output = output + var pos = pos + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 9 digits remaining + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + tz = trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + else: + tz = 4 + nd = 4 + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + sf_Assert(output >= 1) + sf_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + sf_Assert(q >= 1) + sf_Assert(q <= 9) + dec(pos) + buf[pos] = chr(uint32('0') + q) + return tz + +proc decimalLength(v: uint32): int {.inline.} = + sf_Assert(v >= 1) + sf_Assert(v <= 999999999'u) + if v >= 100000000'u: + return 9 + if v >= 10000000'u: + return 8 + if v >= 1000000'u: + return 7 + if v >= 100000'u: + return 6 + if v >= 10000'u: + return 5 + if v >= 1000'u: + return 4 + if v >= 100'u: + return 3 + if v >= 10'u: + return 2 + return 1 + +proc formatDigits[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint32; decimalExponent: int; + forceTrailingDotZero: bool = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -4 + maxFixedDecimalPoint: int32 = 9 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 1, "internal error") + sf_Assert(digits >= 1) + sf_Assert(digits <= 999999999'u) + sf_Assert(decimalExponent >= -99) + sf_Assert(decimalExponent <= 99) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + ## Avoid calling memset/memcpy with variable arguments below... + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + decimalDigitsPosition + numDigits + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + for i in countdown(7, 0): + buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(uint32('0') + k) + inc pos + else: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + return pos + +proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZero = false): int {. + inline.} = + let significand: uint32 = physicalSignificand(constructSingle(v)) + let exponent: uint32 = physicalExponent(constructSingle(v)) + var pos = 0 + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec: auto = toDecimal32(significand, exponent) + return formatDigits(buffer, pos, dec.digits, dec.exponent.int, forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 diff --git a/lib/std/private/since.nim b/lib/std/private/since.nim new file mode 100644 index 000000000..720120f11 --- /dev/null +++ b/lib/std/private/since.nim @@ -0,0 +1,33 @@ +##[ +`since` is used to emulate older versions of nim stdlib, +see `tuse_version.nim`. + +If a symbol `foo` is added in version `(1,3,5)`, use `{.since: (1.3.5).}`, not +`{.since: (1.4).}`, so that it works in devel in between releases. + +The emulation cannot be 100% faithful and to avoid adding too much complexity, +`since` is not needed in those cases: +* if a new module is added +* if an overload is added +* if an extra parameter to an existing routine is added +]## + +template since*(version: (int, int), body: untyped) {.dirty.} = + ## Evaluates `body` if the ``(NimMajor, NimMinor)`` is greater than + ## or equal to `version`. Usage: + ## ```Nim + ## proc fun*() {.since: (1, 3).} + ## since (1, 3): fun() + ## ``` + when (NimMajor, NimMinor) >= version: + body + +template since*(version: (int, int, int), body: untyped) {.dirty.} = + ## Evaluates `body` if ``(NimMajor, NimMinor, NimPatch)`` is greater than + ## or equal to `version`. Usage: + ## ```Nim + ## proc fun*() {.since: (1, 3, 1).} + ## since (1, 3, 1): fun() + ## ``` + when (NimMajor, NimMinor, NimPatch) >= version: + body diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim new file mode 100644 index 000000000..f8c9236a5 --- /dev/null +++ b/lib/std/private/strimpl.nim @@ -0,0 +1,113 @@ +func toLowerAscii*(c: char): char {.inline.} = + if c in {'A'..'Z'}: + result = chr(ord(c) + (ord('a') - ord('A'))) + else: + result = c + +template firstCharCaseSensitiveImpl[T: string | cstring](a, b: T, aLen, bLen: int) = + if aLen == 0 or bLen == 0: + return aLen - bLen + if a[0] != b[0]: return ord(a[0]) - ord(b[0]) + +template cmpIgnoreStyleImpl*[T: string | cstring](a, b: T, + firstCharCaseSensitive: static bool = false) = + let aLen = a.len + let bLen = b.len + var i = 0 + var j = 0 + when firstCharCaseSensitive: + firstCharCaseSensitiveImpl(a, b, aLen, bLen) + inc i + inc j + while true: + while i < aLen and a[i] == '_': inc i + while j < bLen and b[j] == '_': inc j + let aa = if i < aLen: toLowerAscii(a[i]) else: '\0' + let bb = if j < bLen: toLowerAscii(b[j]) else: '\0' + result = ord(aa) - ord(bb) + if result != 0: return result + # the characters are identical: + if i >= aLen: + # both cursors at the end: + if j >= bLen: return 0 + # not yet at the end of 'b': + return -1 + elif j >= bLen: + return 1 + inc i + inc j + +template cmpIgnoreCaseImpl*[T: string | cstring](a, b: T, + firstCharCaseSensitive: static bool = false) = + let aLen = a.len + let bLen = b.len + var i = 0 + when firstCharCaseSensitive: + firstCharCaseSensitiveImpl(a, b, aLen, bLen) + inc i + var m = min(aLen, bLen) + while i < m: + result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) + if result != 0: return + inc i + result = aLen - bLen + +template startsWithImpl*[T: string | cstring](s, prefix: T) = + let prefixLen = prefix.len + let sLen = s.len + var i = 0 + while true: + if i >= prefixLen: return true + if i >= sLen or s[i] != prefix[i]: return false + inc(i) + +template endsWithImpl*[T: string | cstring](s, suffix: T) = + let suffixLen = suffix.len + let sLen = s.len + var i = 0 + var j = sLen - suffixLen + while i+j >= 0 and i+j < sLen: + if s[i+j] != suffix[i]: return false + inc(i) + if i >= suffixLen: return true + + +func cmpNimIdentifier*[T: string | cstring](a, b: T): int = + cmpIgnoreStyleImpl(a, b, true) + +func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} +func c_strstr(haystack, needle: cstring): cstring {. + importc: "strstr", header: "<string.h>".} + + +func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].rfind` for a `start`-origin index. + let last = if last == 0: s.high else: last + let L = last-start+1 + if L > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + if not found.isNil: + return cast[int](found) -% cast[int](s) + return -1 + +func find*(s, sub: cstring, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].find` for a `start`-origin index. + if sub.len > s.len - start: return -1 + if sub.len == 1: return find(s, sub[0], start, last) + if last == 0 and s.len > start: + let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) + if not found.isNil: + result = cast[int](found) -% cast[int](s) + else: + result = -1 diff --git a/lib/std/private/syslocks.nim b/lib/std/private/syslocks.nim new file mode 100644 index 000000000..e19ec2c04 --- /dev/null +++ b/lib/std/private/syslocks.nim @@ -0,0 +1,234 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level system locks and condition vars. + +{.push stackTrace: off.} + +when defined(windows): + type + Handle = int + + SysLock* {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure, byref.} = object # CRITICAL_SECTION in WinApi + DebugInfo: pointer + LockCount: int32 + RecursionCount: int32 + OwningThread: int + LockSemaphore: int + SpinCount: int + + SysCond* {.importc: "RTL_CONDITION_VARIABLE", header: "<windows.h>", byref.} = object + thePtr {.importc: "Ptr".} : Handle + + proc initSysLock*(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} + ## Initializes the lock `L`. + + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} + ## Tries to acquire the lock `L`. + + proc tryAcquireSys*(L: var SysLock): bool {.inline.} = + result = tryAcquireSysAux(L) != 0'i32 + + proc acquireSys*(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} + ## Acquires the lock `L`. + + proc releaseSys*(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} + ## Releases the lock `L`. + + proc deinitSys*(L: SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} + + proc initializeConditionVariable( + conditionVariable: var SysCond + ) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "InitializeConditionVariable".} + + proc sleepConditionVariableCS( + conditionVariable: var SysCond, + PCRITICAL_SECTION: var SysLock, + dwMilliseconds: int + ): int32 {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SleepConditionVariableCS".} + + + proc signalSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeConditionVariable".} + + proc broadcastSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeAllConditionVariable".} + + proc initSysCond*(cond: var SysCond) {.inline.} = + initializeConditionVariable(cond) + proc deinitSysCond*(cond: SysCond) {.inline.} = + discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard sleepConditionVariableCS(cond, lock, -1'i32) + +elif defined(genode): + const + Header = "genode_cpp/syslocks.h" + type + SysLock* {.importcpp: "Nim::SysLock", pure, final, + header: Header.} = object + SysCond* {.importcpp: "Nim::SysCond", pure, final, + header: Header.} = object + + proc initSysLock*(L: var SysLock) = discard + proc deinitSys*(L: SysLock) = discard + proc acquireSys*(L: var SysLock) {.noSideEffect, importcpp.} + proc tryAcquireSys*(L: var SysLock): bool {.noSideEffect, importcpp.} + proc releaseSys*(L: var SysLock) {.noSideEffect, importcpp.} + + proc initSysCond*(L: var SysCond) = discard + proc deinitSysCond*(L: SysCond) = discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) {. + noSideEffect, importcpp.} + proc signalSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + proc broadcastSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + +else: + type + SysLockObj {.importc: "pthread_mutex_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[40 div sizeof(clong), clong] + + SysLockAttr* {.importc: "pthread_mutexattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysCondObj {.importc: "pthread_cond_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[48 div sizeof(clonglong), clonglong] + + SysCondAttr {.importc: "pthread_condattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysLockType = distinct cint + + proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. + importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysAux(L: SysLockObj) {.noSideEffect, + importc: "pthread_mutex_destroy", header: "<pthread.h>".} + + proc acquireSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_lock", header: "<pthread.h>".} + proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, + importc: "pthread_mutex_trylock", header: "<pthread.h>".} + + proc releaseSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_unlock", header: "<pthread.h>".} + + when defined(ios): + # iOS will behave badly if sync primitives are moved in memory. In order + # to prevent this once and for all, we're doing an extra malloc when + # initializing the primitive. + type + SysLock* = ptr SysLockObj + SysCond* = ptr SysCondObj + + when not declared(c_malloc): + proc c_malloc(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + + proc initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj)))) + initSysLockAux(L[], attr) + + proc deinitSys*(L: SysLock) = + deinitSysAux(L[]) + c_free(L) + + template acquireSys*(L: var SysLock) = + acquireSysAux(L[]) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L[]) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L[]) + else: + type + SysLock* = SysLockObj + SysCond* = SysCondObj + + template initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + initSysLockAux(L, attr) + template deinitSys*(L: SysLock) = + deinitSysAux(L) + template acquireSys*(L: var SysLock) = + acquireSysAux(L) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L) + + # rlocks + var SysLockType_Reentrant* {.importc: "PTHREAD_MUTEX_RECURSIVE", + header: "<pthread.h>".}: SysLockType + proc initSysLockAttr*(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType*(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + + # locks + proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysCondAux(cond: SysCondObj) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj): cint {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc broadcastSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_broadcast", header: "<pthread.h>", noSideEffect.} + + when defined(ios): + proc initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj)))) + initSysCondAux(cond[], cond_attr) + + proc deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond[]) + c_free(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond[], lock[]) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond[]) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond[]) + else: + template initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + initSysCondAux(cond, cond_attr) + template deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond, lock) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond) + +{.pop.} diff --git a/lib/std/private/threadtypes.nim b/lib/std/private/threadtypes.nim new file mode 100644 index 000000000..a1cdf21dc --- /dev/null +++ b/lib/std/private/threadtypes.nim @@ -0,0 +1,176 @@ +include system/inclrtl + +const hasSharedHeap* = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own + +when defined(windows): + type + Handle* = int + SysThread* = Handle + WinThreadProc* = proc (x: pointer): int32 {.stdcall.} + + proc createThread*(lpThreadAttributes: pointer, dwStackSize: int32, + lpStartAddress: WinThreadProc, + lpParameter: pointer, + dwCreationFlags: int32, + lpThreadId: var int32): SysThread {. + stdcall, dynlib: "kernel32", importc: "CreateThread".} + + proc winSuspendThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "SuspendThread".} + + proc winResumeThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "ResumeThread".} + + proc waitForSingleObject*(hHandle: SysThread, dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + + proc waitForMultipleObjects*(nCount: int32, + lpHandles: ptr SysThread, + bWaitAll: int32, + dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} + + proc terminateThread*(hThread: SysThread, dwExitCode: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "TerminateThread".} + + proc setThreadAffinityMask*(hThread: SysThread, dwThreadAffinityMask: uint) {. + importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + +elif defined(genode): + const + GenodeHeader* = "genode_cpp/threads.h" + type + SysThread* {.importcpp: "Nim::SysThread", + header: GenodeHeader, final, pure.} = object + GenodeThreadProc* = proc (x: pointer) {.noconv.} + + proc initThread*(s: var SysThread, + env: GenodeEnv, + stackSize: culonglong, + entry: GenodeThreadProc, + arg: pointer, + affinity: cuint) {. + importcpp: "#.initThread(@)".} + + +else: + when not (defined(macosx) or defined(haiku)): + {.passl: "-pthread".} + + when not defined(haiku): + {.passc: "-pthread".} + + const + schedh = "#define _GNU_SOURCE\n#include <sched.h>" + pthreadh* = "#define _GNU_SOURCE\n#include <pthread.h>" + + when not declared(Time): + when defined(linux): + type Time = clong + else: + type Time = int + + when (defined(linux) or defined(nintendoswitch)) and defined(amd64): + type + SysThread* {.importc: "pthread_t", + header: "<sys/types.h>" .} = distinct culong + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + abi: array[56 div sizeof(clong), clong] + elif defined(openbsd) and defined(amd64): + type + SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object + Pthread_attr* {.importc: "pthread_attr_t", + header: "<pthread.h>".} = object + else: + type + SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + type + Timespec* {.importc: "struct timespec", header: "<time.h>".} = object + tv_sec*: Time + tv_nsec*: clong + + proc pthread_attr_init*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstacksize*(a1: var Pthread_attr, a2: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_destroy*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + + proc pthread_create*(a1: var SysThread, a2: var Pthread_attr, + a3: proc (x: pointer): pointer {.noconv.}, + a4: pointer): cint {.importc: "pthread_create", + header: pthreadh.} + proc pthread_join*(a1: SysThread, a2: ptr pointer): cint {. + importc, header: pthreadh.} + + proc pthread_cancel*(a1: SysThread): cint {. + importc: "pthread_cancel", header: pthreadh.} + + type CpuSet* {.importc: "cpu_set_t", header: schedh.} = object + when defined(linux) and defined(amd64): + abi: array[1024 div (8 * sizeof(culong)), culong] + + proc cpusetZero*(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} + proc cpusetIncl*(cpu: cint; s: var CpuSet) {. + importc: "CPU_SET", header: schedh.} + + when defined(android): + # libc of android doesn't implement pthread_setaffinity_np, + # it exposes pthread_gettid_np though, so we can use that in combination + # with sched_setaffinity to set the thread affinity. + type Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 # From posix_other.nim + + proc setAffinityTID*(tid: Pid; setsize: csize_t; s: var CpuSet) {. + importc: "sched_setaffinity", header: schedh.} + + proc pthread_gettid_np*(thread: SysThread): Pid {. + importc: "pthread_gettid_np", header: pthreadh.} + + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) = + setAffinityTID(pthread_gettid_np(thread), setsize, s) + else: + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) {. + importc: "pthread_setaffinity_np", header: pthreadh.} + + +const + emulatedThreadVars* = compileOption("tlsEmulation") +# we preallocate a fixed size for thread local storage, so that no heap +# allocations are needed. Currently less than 16K are used on a 64bit machine. +# We use `float` for proper alignment: +const nimTlsSize {.intdefine.} = 16000 +type + ThreadLocalStorage* = array[0..(nimTlsSize div sizeof(float)), float] + PGcThread* = ptr GcThread + GcThread* {.pure, inheritable.} = object + when emulatedThreadVars: + tls*: ThreadLocalStorage + else: + nil + when hasSharedHeap: + next*, prev*: PGcThread + stackBottom*, stackTop*: pointer + stackSize*: int + else: + nil + +const hasAllocStack* = defined(zephyr) # maybe freertos too? + +type + Thread*[TArg] = object + core*: PGcThread + sys*: SysThread + when TArg is void: + dataFn*: proc () {.nimcall, gcsafe.} + else: + dataFn*: proc (m: TArg) {.nimcall, gcsafe.} + data*: TArg + when hasAllocStack: + rawStack*: pointer + +proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} diff --git a/lib/std/private/underscored_calls.nim b/lib/std/private/underscored_calls.nim new file mode 100644 index 000000000..f853572b5 --- /dev/null +++ b/lib/std/private/underscored_calls.nim @@ -0,0 +1,56 @@ + +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This is an internal helper module. Do not use. + +import std/macros + +proc underscoredCalls*(result, calls, arg0: NimNode) + +proc underscoredCall(n, arg0: NimNode): NimNode = + proc underscorePos(n: NimNode): int = + for i in 1 ..< n.len: + if n[i].eqIdent("_"): return i + return 0 + + if n.kind in nnkCallKinds: + if n[0].kind in {nnkIdent, nnkSym} and n[0].eqIdent("with"): + expectKind n[1], {nnkIdent, nnkSym} + + result = newStmtList() + underscoredCalls(result, n[2 .. ^1].newStmtList, newDotExpr(arg0, n[1])) + else: + result = copyNimNode(n) + result.add n[0] + + let u = underscorePos(n) + for i in 1..u-1: result.add n[i] + result.add arg0 + for i in u+1..n.len-1: result.add n[i] + elif n.kind in {nnkAsgn, nnkExprEqExpr}: + var field = n[0] + if n[0].kind == nnkDotExpr and n[0][0].eqIdent("_"): + # handle _.field = ... + field = n[0][1] + result = newDotExpr(arg0, field).newAssignment n[1] + else: + # handle e.g. 'x.dup(sort)' + result = newNimNode(nnkCall, n) + result.add n + result.add arg0 + +proc underscoredCalls*(result, calls, arg0: NimNode) = + expectKind calls, {nnkArgList, nnkStmtList, nnkStmtListExpr} + + for call in calls: + if call.kind in {nnkStmtList, nnkStmtListExpr}: + underscoredCalls(result, call, arg0) + else: + result.add underscoredCall(call, arg0) diff --git a/lib/std/private/win_getsysteminfo.nim b/lib/std/private/win_getsysteminfo.nim new file mode 100644 index 000000000..b98478231 --- /dev/null +++ b/lib/std/private/win_getsysteminfo.nim @@ -0,0 +1,15 @@ +type + SystemInfo* = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors*: uint32 + dwProcessorType: uint32 + dwAllocationGranularity*: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + +proc getSystemInfo*(lpSystemInfo: ptr SystemInfo) {.stdcall, + dynlib: "kernel32", importc: "GetSystemInfo".} diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim new file mode 100644 index 000000000..66e199dfe --- /dev/null +++ b/lib/std/private/win_setenv.nim @@ -0,0 +1,106 @@ +#[ +Copyright (c) Facebook, Inc. and its affiliates. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Adapted `setenv` from https://github.com/facebook/folly/blob/master/folly/portability/Stdlib.cpp +translated from C to nim. +]# + +#[ +Introduced in https://github.com/facebook/folly/commit/5d8ca09a3f96afefb44e35808f03651a096ab9c7 + +TODO: +check errno_t vs cint +]# + +when not defined(windows): discard +else: + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t".} = int16 + + proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} + # same as winlean.setEnvironmentVariableA + + proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "<stdlib.h>".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "<stdlib.h>".} + + var errno {.importc, header: "<errno.h>".}: cint + var genviron {.importc: "_environ".}: ptr ptr char + # xxx `ptr UncheckedArray[WideCString]` did not work + + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "<stdlib.h>".} + # xxx cint vs errno_t? + + proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = + const EINVAL = cint(22) + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: + return 0 + + if value != "": + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) + if e != 0: + errno = EINVAL + return -1 + return 0 + #[ + We are trying to set the value to an empty string, but `_putenv` deletes + entries if the value is an empty string, and just calling + SetEnvironmentVariableA doesn't update `_environ`, + so we have to do these terrible things. + ]# + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: + errno = EINVAL + return -1 + # Here lies the documentation we blatently ignore to make this work. + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[0] = Utf16Char('\0') + #[ + This would result in a double null termination, which normally signifies the + end of the environment variable list, so we stick a completely empty + environment variable into the list instead. + ]# + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[1] = Utf16Char('=') + #[ + If genviron is null, the MBCS environment has not been initialized + yet, and we don't need to try to update it. We have to do this otherwise + we'd be forcing the initialization and maintenance of the MBCS environment + even though it's never actually used in most programs. + ]# + if genviron != nil: + + # wcstombs returns `high(csize_t)` if any characters cannot be represented + # in the current codepage. Skip updating MBCS environment in this case. + # For some reason, second `wcstombs` can find non-convertible characters + # that the first `wcstombs` cannot. + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) + if requiredSizeS != high(csize_t): + let requiredSize = requiredSizeS.int + var buf = newSeq[char](requiredSize + 1) + let buf2 = buf[0].addr + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): + var ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[0] = '\0' + ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[1] = '=' + + # And now, we have to update the outer environment to have a proper empty value. + if setEnvironmentVariableW(wideName, value.newWideCString) == 0: + errno = EINVAL + return -1 + return 0 diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim new file mode 100644 index 000000000..8e7bc6a92 --- /dev/null +++ b/lib/std/setutils.nim @@ -0,0 +1,77 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module adds functionality for the built-in `set` type. +## +## See also +## ======== +## * `std/packedsets <packedsets.html>`_ +## * `std/sets <sets.html>`_ + +import std/[typetraits, macros] + +#[ + type SetElement* = char|byte|bool|int16|uint16|enum|uint8|int8 + ## The allowed types of a built-in set. +]# + +template toSet*(iter: untyped): untyped = + ## Returns a built-in set from the elements of the iterable `iter`. + runnableExamples: + assert "helloWorld".toSet == {'W', 'd', 'e', 'h', 'l', 'o', 'r'} + assert toSet([10u16, 20, 30]) == {10u16, 20, 30} + assert [30u8, 100, 10].toSet == {10u8, 30, 100} + assert toSet(@[1321i16, 321, 90]) == {90i16, 321, 1321} + assert toSet([false]) == {false} + assert toSet(0u8..10) == {0u8..10} + + var result: set[elementType(iter)] + for x in iter: + incl(result, x) + result + +macro enumElementsAsSet(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) + +# func fullSet*(T: typedesc): set[T] {.inline.} = # xxx would give: Error: ordinal type expected +func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = + ## Returns a set containing all elements in `U`. + runnableExamples: + assert bool.fullSet == {true, false} + type A = range[1..3] + assert A.fullSet == {1.A, 2, 3} + assert int8.fullSet.len == 256 + when T is Ordinal: + {T.low..T.high} + else: # Hole filled enum + enumElementsAsSet(T) + +func complement*[T](s: set[T]): set[T] {.inline.} = + ## Returns the set complement of `a`. + runnableExamples: + type Colors = enum + red, green = 3, blue + assert complement({red, blue}) == {green} + assert complement({red, green, blue}).card == 0 + assert complement({range[0..10](0), 1, 2, 3}) == {range[0..10](4), 5, 6, 7, 8, 9, 10} + assert complement({'0'..'9'}) == {0.char..255.char} - {'0'..'9'} + fullSet(T) - s + +func `[]=`*[T](t: var set[T], key: T, val: bool) {.inline.} = + ## Syntax sugar for `if val: t.incl key else: t.excl key` + runnableExamples: + type A = enum + a0, a1, a2, a3 + var s = {a0, a3} + s[a0] = false + s[a1] = false + assert s == {a3} + s[a2] = true + s[a3] = true + assert s == {a2, a3} + if val: t.incl key else: t.excl key diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim new file mode 100644 index 000000000..213af4229 --- /dev/null +++ b/lib/std/sha1.nim @@ -0,0 +1,287 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## [SHA-1 (Secure Hash Algorithm 1)](https://en.wikipedia.org/wiki/SHA-1) +## is a cryptographic hash function which takes an input and produces +## a 160-bit (20-byte) hash value known as a message digest. +## +## See also +## ======== +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder +## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm + +runnableExamples: + let accessName = secureHash("John Doe") + assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" + +runnableExamples("-r:off"): + let + a = secureHashFile("myFile.nim") + b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") + assert a == b, "files don't match" + + +{.deprecated: "use command `nimble install checksums` and import `checksums/sha1` instead".} + +import std/strutils +from std/endians import bigEndian32, bigEndian64 + +when defined(nimPreviewSlimSystem): + import std/syncio + +const Sha1DigestSize = 20 + +type + Sha1Digest* = array[0 .. Sha1DigestSize - 1, uint8] + SecureHash* = distinct Sha1Digest + +type + Sha1State* = object + count: int + state: array[5, uint32] + buf: array[64, byte] + +# This implementation of the SHA-1 algorithm was ported from the Chromium OS one +# with minor modifications that should not affect its functionality. + +proc newSha1State*(): Sha1State = + ## Creates a `Sha1State`. + ## + ## If you use the `secureHash proc <#secureHash,openArray[char]>`_, + ## there's no need to call this function explicitly. + result.count = 0 + result.state[0] = 0x67452301'u32 + result.state[1] = 0xEFCDAB89'u32 + result.state[2] = 0x98BADCFE'u32 + result.state[3] = 0x10325476'u32 + result.state[4] = 0xC3D2E1F0'u32 + +template ror27(val: uint32): uint32 = (val shr 27) or (val shl 5) +template ror2 (val: uint32): uint32 = (val shr 2) or (val shl 30) +template ror31(val: uint32): uint32 = (val shr 31) or (val shl 1) + +proc transform(ctx: var Sha1State) = + var w: array[80, uint32] + var a, b, c, d, e: uint32 + var t = 0 + + a = ctx.state[0] + b = ctx.state[1] + c = ctx.state[2] + d = ctx.state[3] + e = ctx.state[4] + + template shaF1(a, b, c, d, e, t: untyped) = + bigEndian32(addr w[t], addr ctx.buf[t * 4]) + e += ror27(a) + w[t] + (d xor (b and (c xor d))) + 0x5A827999'u32 + b = ror2(b) + + while t < 15: + shaF1(a, b, c, d, e, t + 0) + shaF1(e, a, b, c, d, t + 1) + shaF1(d, e, a, b, c, t + 2) + shaF1(c, d, e, a, b, t + 3) + shaF1(b, c, d, e, a, t + 4) + t += 5 + shaF1(a, b, c, d, e, t + 0) # 16th one, t == 15 + + template shaF11(a, b, c, d, e, t: untyped) = + w[t] = ror31(w[t-3] xor w[t-8] xor w[t-14] xor w[t-16]) + e += ror27(a) + w[t] + (d xor (b and (c xor d))) + 0x5A827999'u32 + b = ror2(b) + + shaF11(e, a, b, c, d, t + 1) + shaF11(d, e, a, b, c, t + 2) + shaF11(c, d, e, a, b, t + 3) + shaF11(b, c, d, e, a, t + 4) + + template shaF2(a, b, c, d, e, t: untyped) = + w[t] = ror31(w[t-3] xor w[t-8] xor w[t-14] xor w[t-16]) + e += ror27(a) + w[t] + (b xor c xor d) + 0x6ED9EBA1'u32 + b = ror2(b) + + t = 20 + while t < 40: + shaF2(a, b, c, d, e, t + 0) + shaF2(e, a, b, c, d, t + 1) + shaF2(d, e, a, b, c, t + 2) + shaF2(c, d, e, a, b, t + 3) + shaF2(b, c, d, e, a, t + 4) + t += 5 + + template shaF3(a, b, c, d, e, t: untyped) = + w[t] = ror31(w[t-3] xor w[t-8] xor w[t-14] xor w[t-16]) + e += ror27(a) + w[t] + ((b and c) or (d and (b or c))) + 0x8F1BBCDC'u32 + b = ror2(b) + + while t < 60: + shaF3(a, b, c, d, e, t + 0) + shaF3(e, a, b, c, d, t + 1) + shaF3(d, e, a, b, c, t + 2) + shaF3(c, d, e, a, b, t + 3) + shaF3(b, c, d, e, a, t + 4) + t += 5 + + template shaF4(a, b, c, d, e, t: untyped) = + w[t] = ror31(w[t-3] xor w[t-8] xor w[t-14] xor w[t-16]) + e += ror27(a) + w[t] + (b xor c xor d) + 0xCA62C1D6'u32 + b = ror2(b) + + while t < 80: + shaF4(a, b, c, d, e, t + 0) + shaF4(e, a, b, c, d, t + 1) + shaF4(d, e, a, b, c, t + 2) + shaF4(c, d, e, a, b, t + 3) + shaF4(b, c, d, e, a, t + 4) + t += 5 + + ctx.state[0] += a + ctx.state[1] += b + ctx.state[2] += c + ctx.state[3] += d + ctx.state[4] += e + +proc update*(ctx: var Sha1State, data: openArray[char]) = + ## Updates the `Sha1State` with `data`. + ## + ## If you use the `secureHash proc <#secureHash,openArray[char]>`_, + ## there's no need to call this function explicitly. + var i = ctx.count mod 64 + var j = 0 + var len = data.len + # Gather 64-bytes worth of data in order to perform a round with the leftover + # data we had stored (but not processed yet) + if len > 64 - i: + copyMem(addr ctx.buf[i], unsafeAddr data[j], 64 - i) + len -= 64 - i + j += 64 - i + transform(ctx) + # Update the index since it's used in the while loop below _and_ we want to + # keep its value if this code path isn't executed + i = 0 + # Process the bulk of the payload + while len >= 64: + copyMem(addr ctx.buf[0], unsafeAddr data[j], 64) + len -= 64 + j += 64 + transform(ctx) + # Process the tail of the payload (len is < 64) + while len > 0: + dec len + ctx.buf[i] = byte(data[j]) + inc i + inc j + if i == 64: + transform(ctx) + i = 0 + ctx.count += data.len + +proc finalize*(ctx: var Sha1State): Sha1Digest = + ## Finalizes the `Sha1State` and returns a `Sha1Digest`. + ## + ## If you use the `secureHash proc <#secureHash,openArray[char]>`_, + ## there's no need to call this function explicitly. + var cnt = uint64(ctx.count * 8) + # a 1 bit + update(ctx, "\x80") + # Add padding until we reach a complexive size of 64 - 8 bytes + while (ctx.count mod 64) != (64 - 8): + update(ctx, "\x00") + # The message length as a 64bit BE number completes the block + var tmp: array[8, char] + bigEndian64(addr tmp[0], addr cnt) + update(ctx, tmp) + # Turn the result into a single 160-bit number + for i in 0 ..< 5: + bigEndian32(addr ctx.state[i], addr ctx.state[i]) + copyMem(addr result[0], addr ctx.state[0], Sha1DigestSize) + +# Public API + +proc secureHash*(str: openArray[char]): SecureHash = + ## Generates a `SecureHash` from `str`. + ## + ## **See also:** + ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a `SecureHash` from a file + ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string `hash` to `SecureHash` + runnableExamples: + let hash = secureHash("Hello World") + assert hash == parseSecureHash("0A4D55A8D778E5022FAB701977C5D840BBC486D0") + + var state = newSha1State() + state.update(str) + SecureHash(state.finalize()) + +proc secureHashFile*(filename: string): SecureHash = + ## Generates a `SecureHash` from a file. + ## + ## **See also:** + ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a `SecureHash` from a string + ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string `hash` to `SecureHash` + const BufferLength = 8192 + + let f = open(filename) + var state = newSha1State() + var buffer = newString(BufferLength) + while true: + let length = readChars(f, buffer) + if length == 0: + break + buffer.setLen(length) + state.update(buffer) + if length != BufferLength: + break + close(f) + + SecureHash(state.finalize()) + +proc `$`*(self: SecureHash): string = + ## Returns the string representation of a `SecureHash`. + ## + ## **See also:** + ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a `SecureHash` from a string + runnableExamples: + let hash = secureHash("Hello World") + assert $hash == "0A4D55A8D778E5022FAB701977C5D840BBC486D0" + + result = "" + for v in Sha1Digest(self): + result.add(toHex(int(v), 2)) + +proc parseSecureHash*(hash: string): SecureHash = + ## Converts a string `hash` to a `SecureHash`. + ## + ## **See also:** + ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a `SecureHash` from a string + ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a `SecureHash` from a file + runnableExamples: + let + hashStr = "0A4D55A8D778E5022FAB701977C5D840BBC486D0" + secureHash = secureHash("Hello World") + assert secureHash == parseSecureHash(hashStr) + + for i in 0 ..< Sha1DigestSize: + Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) + +proc `==`*(a, b: SecureHash): bool = + ## Checks if two `SecureHash` values are identical. + runnableExamples: + let + a = secureHash("Hello World") + b = secureHash("Goodbye World") + c = parseSecureHash("0A4D55A8D778E5022FAB701977C5D840BBC486D0") + assert a != b + assert a == c + + # Not a constant-time comparison, but that's acceptable in this context + Sha1Digest(a) == Sha1Digest(b) + +proc isValidSha1Hash*(s: string): bool = + ## Checks if a string is a valid sha1 hash sum. + s.len == 40 and allCharsInSet(s, HexDigits) \ No newline at end of file diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim new file mode 100644 index 000000000..45e906795 --- /dev/null +++ b/lib/std/socketstreams.nim @@ -0,0 +1,182 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides an implementation of the streams interface for sockets. +## It contains two separate implementations, a +## `ReadSocketStream <#ReadSocketStream>`_ and a +## `WriteSocketStream <#WriteSocketStream>`_. +## +## The `ReadSocketStream` only supports reading, peeking, and seeking. +## It reads into a buffer, so even by +## seeking backwards it will only read the same position a single time from the +## underlying socket. To clear the buffer and free the data read into it you +## can call `resetStream`, this will also reset the position back to 0 but +## won't do anything to the underlying socket. +## +## The `WriteSocketStream` allows both reading and writing, but it performs the +## reads on the internal buffer. So by writing to the buffer you can then read +## back what was written but without receiving anything from the socket. You +## can also set the position and overwrite parts of the buffer, and to send +## anything over the socket you need to call `flush` at which point you can't +## write anything to the buffer before the point of the flush (but it can still +## be read). Again to empty the underlying buffer you need to call +## `resetStream`. +## +## Examples +## ======== +## +## ```Nim +## import std/socketstreams +## +## var +## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## stream = newReadSocketStream(socket) +## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") +## echo stream.readLine() # Will call `recv` +## stream.setPosition(0) +## echo stream.readLine() # Will return the read line from the buffer +## stream.resetStream() # Buffer is now empty, position is 0 +## echo stream.readLine() # Will call `recv` again +## stream.close() # Closes the socket +## ``` +## +## ```Nim +## import std/socketstreams +## +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## socket.connect("127.0.0.1", Port(12345)) +## var sendStream = newWriteSocketStream(socket) +## sendStream.write "NOM" +## sendStream.setPosition(1) +## echo sendStream.peekStr(2) # OM +## sendStream.write "I" +## sendStream.setPosition(0) +## echo sendStream.readStr(3) # NIM +## echo sendStream.getPosition() # 3 +## sendStream.flush() # This actually performs the writing to the socket +## sendStream.setPosition(1) +## sendStream.write "I" # Throws an error as we can't write into an already sent buffer +## ``` + +import std/[net, streams] + +type + ReadSocketStream* = ref ReadSocketStreamObj + ReadSocketStreamObj* = object of StreamObj + data: Socket + pos: int + buf: seq[byte] + WriteSocketStream* = ref WriteSocketStreamObj + WriteSocketStreamObj* = object of ReadSocketStreamObj + lastFlush: int + +proc rsAtEnd(s: Stream): bool = + return false + +proc rsSetPosition(s: Stream, pos: int) = + var s = ReadSocketStream(s) + s.pos = pos + +proc rsGetPosition(s: Stream): int = + var s = ReadSocketStream(s) + return s.pos + +proc rsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let s = ReadSocketStream(s) + if bufLen > 0: + let oldLen = s.buf.len + s.buf.setLen(max(s.pos + bufLen, s.buf.len)) + if s.pos + bufLen > oldLen: + result = s.data.recv(s.buf[oldLen].addr, s.buf.len - oldLen) + if result > 0: + result += oldLen - s.pos + else: + result = bufLen + copyMem(buffer, s.buf[s.pos].addr, result) + +proc rsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = s.rsPeekData(buffer, bufLen) + var s = ReadSocketStream(s) + s.pos += bufLen + +proc rsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = + var s = ReadSocketStream(s) + result = slice.b + 1 - slice.a + if result > 0: + result = s.rsReadData(buffer[slice.a].addr, result) + inc(s.pos, result) + else: + result = 0 + +proc wsWriteData(s: Stream, buffer: pointer, bufLen: int) = + var s = WriteSocketStream(s) + if s.pos < s.lastFlush: + raise newException(IOError, "Unable to write into buffer that has already been sent") + if s.buf.len < s.pos + bufLen: + s.buf.setLen(s.pos + bufLen) + copyMem(s.buf[s.pos].addr, buffer, bufLen) + s.pos += bufLen + +proc wsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + var s = WriteSocketStream(s) + result = bufLen + if result > 0: + if s.pos > s.buf.len or s.pos == s.buf.len or s.pos + bufLen > s.buf.len: + raise newException(IOError, "Unable to read past end of write buffer") + else: + copyMem(buffer, s.buf[s.pos].addr, bufLen) + +proc wsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = s.wsPeekData(buffer, bufLen) + var s = ReadSocketStream(s) + s.pos += bufLen + +proc wsAtEnd(s: Stream): bool = + var s = WriteSocketStream(s) + return s.pos == s.buf.len + +proc wsFlush(s: Stream) = + var s = WriteSocketStream(s) + discard s.data.send(s.buf[s.lastFlush].addr, s.buf.len - s.lastFlush) + s.lastFlush = s.buf.len + +proc rsClose(s: Stream) = + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? + var s = ReadSocketStream(s) + s.data.close() + +proc newReadSocketStream*(s: Socket): owned ReadSocketStream = + result = ReadSocketStream(data: s, pos: 0, + closeImpl: rsClose, + atEndImpl: rsAtEnd, + setPositionImpl: rsSetPosition, + getPositionImpl: rsGetPosition, + readDataImpl: rsReadData, + peekDataImpl: rsPeekData, + readDataStrImpl: rsReadDataStr) + +proc resetStream*(s: ReadSocketStream) = + s.buf = @[] + s.pos = 0 + +proc newWriteSocketStream*(s: Socket): owned WriteSocketStream = + result = WriteSocketStream(data: s, pos: 0, + closeImpl: rsClose, + atEndImpl: wsAtEnd, + setPositionImpl: rsSetPosition, + getPositionImpl: rsGetPosition, + writeDataImpl: wsWriteData, + readDataImpl: wsReadData, + peekDataImpl: wsPeekData, + flushImpl: wsFlush) + +proc resetStream*(s: WriteSocketStream) = + s.buf = @[] + s.pos = 0 + s.lastFlush = 0 diff --git a/lib/std/stackframes.nim b/lib/std/stackframes.nim new file mode 100644 index 000000000..28be7ce11 --- /dev/null +++ b/lib/std/stackframes.nim @@ -0,0 +1,30 @@ +const NimStackTrace = compileOption("stacktrace") +const NimStackTraceMsgs = compileOption("stacktraceMsgs") + +template procName*(): string = + ## returns current C/C++ function name + when defined(c) or defined(cpp): + var name {.inject, noinit.}: cstring + {.emit: "`name` = __func__;".} + $name + +template getPFrame*(): PFrame = + ## avoids a function call (unlike `getFrame()`) + block: + when NimStackTrace: + var framePtr {.inject, noinit.}: PFrame + {.emit: "`framePtr` = &FR_;".} + framePtr + +template setFrameMsg*(msg: string, prefix = " ") = + ## attach a msg to current `PFrame`. This can be called multiple times + ## in a given PFrame. Noop unless passing --stacktraceMsgs and --stacktrace + when NimStackTrace and NimStackTraceMsgs: + block: + var fr {.inject, noinit.}: PFrame + {.emit: "`fr` = &FR_;".} + # consider setting a custom upper limit on size (analog to stack overflow) + frameMsgBuf.setLen fr.frameMsgLen + frameMsgBuf.add prefix + frameMsgBuf.add msg + fr.frameMsgLen += prefix.len + msg.len diff --git a/lib/std/staticos.nim b/lib/std/staticos.nim new file mode 100644 index 000000000..2617c6913 --- /dev/null +++ b/lib/std/staticos.nim @@ -0,0 +1,13 @@ +## This module implements path handling like os module but works at only compile-time. +## This module works even when cross compiling to OS that is not supported by os module. + +proc staticFileExists*(filename: string): bool {.compileTime.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + discard + +proc staticDirExists*(dir: string): bool {.compileTime.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + discard diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim new file mode 100644 index 000000000..b2c36a4be --- /dev/null +++ b/lib/std/strbasics.nim @@ -0,0 +1,119 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides some high performance string operations. +## +## Experimental API, subject to change. + +when defined(nimPreviewSlimSystem): + import std/assertions + + +const whitespaces = {' ', '\t', '\v', '\r', '\l', '\f'} + +proc add*(x: var string, y: openArray[char]) = + ## Concatenates `x` and `y` in place. `y` must not overlap with `x` to + ## allow future `memcpy` optimizations. + # Use `{.noalias.}` ? + let n = x.len + x.setLen n + y.len + # pending #19727 + # setLen unnecessarily zeros memory + var i = 0 + while i < y.len: + x[n + i] = y[i] + i.inc + # xxx use `nimCopyMem(x[n].addr, y[0].addr, y.len)` after some refactoring + +func stripSlice(s: openArray[char], leading = true, trailing = true, chars: set[char] = whitespaces): Slice[int] = + ## Returns the slice range of `s` which is stripped `chars`. + runnableExamples: + assert stripSlice(" abc ") == 1 .. 3 + var + first = 0 + last = high(s) + if leading: + while first <= last and s[first] in chars: inc(first) + if trailing: + while last >= first and s[last] in chars: dec(last) + result = first .. last + +func setSlice*(s: var string, slice: Slice[int]) = + ## Inplace version of `substr`. + runnableExamples: + import std/sugar + + var a = "Hello, Nim!" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(0 .. 0)) == "H" + doAssert a.dup(setSlice(0 .. 1)) == "He" + doAssert a.dup(setSlice(0 .. 10)) == a + doAssert a.dup(setSlice(1 .. 0)).len == 0 + doAssert a.dup(setSlice(20 .. -1)).len == 0 + + + doAssertRaises(AssertionDefect): + discard a.dup(setSlice(-1 .. 1)) + + doAssertRaises(AssertionDefect): + discard a.dup(setSlice(1 .. 11)) + + + let first = slice.a + let last = slice.b + + assert first >= 0 + assert last <= s.high + + if first > last: + s.setLen(0) + return + template impl = + for index in first .. last: + s[index - first] = s[index] + if first > 0: + when nimvm: impl() + else: + # not JS and not Nimscript + when not declared(moveMem): + impl() + else: + when defined(nimSeqsV2): + prepareMutation(s) + moveMem(addr s[0], addr s[first], last - first + 1) + s.setLen(last - first + 1) + +func strip*(a: var string, leading = true, trailing = true, chars: set[char] = whitespaces) {.inline.} = + ## Inplace version of `strip`. Strips leading or + ## trailing `chars` (default: whitespace characters). + ## + ## If `leading` is true (default), leading `chars` are stripped. + ## If `trailing` is true (default), trailing `chars` are stripped. + ## If both are false, the string is unchanged. + runnableExamples: + var a = " vhellov " + strip(a) + assert a == "vhellov" + + a = " vhellov " + a.strip(leading = false) + assert a == " vhellov" + + a = " vhellov " + a.strip(trailing = false) + assert a == "vhellov " + + var c = "blaXbla" + c.strip(chars = {'b', 'a'}) + assert c == "laXbl" + c = "blaXbla" + c.strip(chars = {'b', 'a', 'l'}) + assert c == "X" + + setSlice(a, stripSlice(a, leading, trailing, chars)) diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim new file mode 100644 index 000000000..dbe908612 --- /dev/null +++ b/lib/std/symlinks.nim @@ -0,0 +1,33 @@ +## This module implements symlink (symbolic link) handling. + +## .. importdoc:: os.nim + +from std/paths import Path, ReadDirEffect + +from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink + +proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + result = symlinkExists(link.string) + +proc createSymlink*(src, dest: Path) {.inline.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + createSymlink(src.string, dest.string) + +proc expandSymlink*(symlinkPath: Path): Path {.inline.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + result = Path(expandSymlink(symlinkPath.string)) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim new file mode 100644 index 000000000..c34a025af --- /dev/null +++ b/lib/std/syncio.nim @@ -0,0 +1,942 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements various synchronized I/O operations. + +include system/inclrtl +import std/private/since +import std/formatfloat +when defined(windows): + import std/widestrs + +# ----------------- IO Part ------------------------------------------------ +type + CFile {.importc: "FILE", header: "<stdio.h>", + incompleteStruct.} = object + File* = ptr CFile ## The type representing a file handle. + + FileMode* = enum ## The file mode when opening a file. + fmRead, ## Open the file for read access only. + ## If the file does not exist, it will not + ## be created. + fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWrite, ## Open the file for read and write access. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWriteExisting, ## Open the file for read and write access. + ## If the file does not exist, it will not be + ## created. The existing file will not be cleared. + fmAppend ## Open the file for writing only; append data + ## at the end. If the file does not exist, it + ## will be created. + + FileHandle* = cint ## The type that represents an OS file handle; this is + ## useful for low-level file access. + + FileSeekPos* = enum ## Position relative to which seek should happen. + # The values are ordered so that they match with stdio + # SEEK_SET, SEEK_CUR and SEEK_END respectively. + fspSet ## Seek to absolute value + fspCur ## Seek relative to current position + fspEnd ## Seek relative to end + +# text file handling: +when not defined(nimscript) and not defined(js): + # duplicated between io and ansi_c + const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined( + dragonfly)) and not defined(emscripten) + const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" + const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" + const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" + + var + stdin* {.importc: stdinName, header: "<stdio.h>".}: File + ## The standard input stream. + stdout* {.importc: stdoutName, header: "<stdio.h>".}: File + ## The standard output stream. + stderr* {.importc: stderrName, header: "<stdio.h>".}: File + ## The standard error stream. + +when defined(useStdoutAsStdmsg): + template stdmsg*: File = stdout +else: + template stdmsg*: File = stderr + ## Template which expands to either stdout or stderr depending on + ## `useStdoutAsStdmsg` compile-time switch. + +when defined(windows): + proc c_fileno(f: File): cint {. + importc: "_fileno", header: "<stdio.h>".} +else: + proc c_fileno(f: File): cint {. + importc: "fileno", header: "<fcntl.h>".} + +when defined(windows): + proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "_fdopen", header: "<stdio.h>".} +else: + proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "fdopen", header: "<stdio.h>".} +proc c_fputs(c: cstring, f: File): cint {. + importc: "fputs", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fgets(c: cstring, n: cint, f: File): cstring {. + importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} +proc c_fgetc(stream: File): cint {. + importc: "fgetc", header: "<stdio.h>", tags: [].} +proc c_ungetc(c: cint, f: File): cint {. + importc: "ungetc", header: "<stdio.h>", tags: [].} +proc c_putc(c: cint, stream: File): cint {. + importc: "putc", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fflush(f: File): cint {. + importc: "fflush", header: "<stdio.h>".} +proc c_fclose(f: File): cint {. + importc: "fclose", header: "<stdio.h>".} +proc c_clearerr(f: File) {. + importc: "clearerr", header: "<stdio.h>".} +proc c_feof(f: File): cint {. + importc: "feof", header: "<stdio.h>".} + +when not declared(c_fwrite): + proc c_fwrite(buf: pointer, size, n: csize_t, f: File): csize_t {. + importc: "fwrite", header: "<stdio.h>".} + +# C routine that is used here: +proc c_fread(buf: pointer, size, n: csize_t, f: File): csize_t {. + importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} +when defined(windows): + when not defined(amd64): + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseek", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftell", header: "<stdio.h>", tags: [].} + else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "_fseeki64", header: "<stdio.h>", tags: [].} + when defined(tcc): + proc c_fsetpos(f: File, pos: var int64): int32 {. + importc: "fsetpos", header: "<stdio.h>", tags: [].} + proc c_fgetpos(f: File, pos: var int64): int32 {. + importc: "fgetpos", header: "<stdio.h>", tags: [].} + proc c_telli64(f: cint): int64 {. + importc: "_telli64", header: "<io.h>", tags: [].} + proc c_ftell(f: File): int64 = + # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c + result = -1'i64 + var pos: int64 + if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0: + result = c_telli64(c_fileno(f)) + else: + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "<stdio.h>", tags: [].} +else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseeko", header: "<stdio.h>", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftello", header: "<stdio.h>", tags: [].} +proc c_ferror(f: File): cint {. + importc: "ferror", header: "<stdio.h>", tags: [].} +proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize_t): cint {. + importc: "setvbuf", header: "<stdio.h>", tags: [].} + +proc c_fprintf(f: File, frmt: cstring): cint {. + importc: "fprintf", header: "<stdio.h>", varargs, discardable.} +proc c_fputc(c: char, f: File): cint {. + importc: "fputc", header: "<stdio.h>".} + +proc raiseEIO(msg: string) {.noinline, noreturn.} = + raise newException(IOError, msg) + +proc raiseEOF() {.noinline, noreturn.} = + raise newException(EOFError, "EOF reached") + +proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} + +when not defined(nimscript): + var + errno {.importc, header: "<errno.h>".}: cint ## error variable + EINTR {.importc: "EINTR", header: "<errno.h>".}: cint + +proc checkErr(f: File) = + when not defined(nimscript): + if c_ferror(f) != 0: + let msg = "errno: " & $errno & " `" & $strerror(errno) & "`" + c_clearerr(f) + raiseEIO(msg) + else: + # shouldn't happen + quit(1) + +{.push stackTrace: off, profiler: off.} +proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. + tags: [ReadIOEffect], benign.} = + ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns + ## the actual number of bytes that have been read which may be less than + ## `len` (if not as many bytes are remaining), but not greater. + result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) + if result != len: checkErr(f) + +proc readBytes*(f: File, a: var openArray[int8|uint8], start, + len: Natural): int {. + tags: [ReadIOEffect], benign.} = + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## the actual number of bytes that have been read which may be less than + ## `len` (if not as many bytes are remaining), but not greater. + result = readBuffer(f, addr(a[start]), len) + +proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} = + ## Reads up to `a.len` bytes into the buffer `a`. Returns + ## the actual number of bytes that have been read which may be less than + ## `a.len` (if not as many bytes are remaining), but not greater. + result = readBuffer(f, addr(a[0]), a.len) + +proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. + tags: [ReadIOEffect], benign, deprecated: + "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} = + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## the actual number of bytes that have been read which may be less than + ## `len` (if not as many bytes are remaining), but not greater. + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") + result = readBuffer(f, addr(a[start]), len) + +proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} = + ## Writes a value to the file `f`. May throw an IO exception. + discard c_fputs(c, f) + checkErr(f) + +proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. + tags: [WriteIOEffect], benign.} = + ## Writes the bytes of buffer pointed to by the parameter `buffer` to the + ## file `f`. Returns the number of actual written bytes, which may be less + ## than `len` in case of an error. + result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f)) + checkErr(f) + +proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. + tags: [WriteIOEffect], benign.} = + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## the number of actual written bytes, which may be less than `len` in case + ## of an error. + var x = cast[ptr UncheckedArray[int8]](a) + result = writeBuffer(f, addr(x[int(start)]), len) + +proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. + tags: [WriteIOEffect], benign.} = + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## the number of actual written bytes, which may be less than `len` in case + ## of an error. + var x = cast[ptr UncheckedArray[int8]](a) + result = writeBuffer(f, addr(x[int(start)]), len) + +when defined(windows): + proc writeWindows(f: File; s: string; doRaise = false) = + # Don't ask why but the 'printf' family of function is the only thing + # that writes utf-8 strings reliably on Windows. At least on my Win 10 + # machine. We also enable `setConsoleOutputCP(65001)` now by default. + # But we cannot call printf directly as the string might contain \0. + # So we have to loop over all the sections separated by potential \0s. + var i = int c_fprintf(f, "%s", s) + while i < s.len: + if s[i] == '\0': + let w = c_fputc('\0', f) + if w != 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i + else: + let w = c_fprintf(f, "%s", unsafeAddr s[i]) + if w <= 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i, w + +proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} = + when defined(windows): + writeWindows(f, s, doRaise = true) + else: + if writeBuffer(f, cstring(s), s.len) != s.len: + raiseEIO("cannot write string to file") +{.pop.} + +when defined(nimscript): + when defined(windows): + const + IOFBF = cint(0) + IONBF = cint(4) + else: + # On all systems I could find, including Linux, Mac OS X, and the BSDs + const + IOFBF = cint(0) + IONBF = cint(2) +else: + var + IOFBF {.importc: "_IOFBF", nodecl.}: cint + IONBF {.importc: "_IONBF", nodecl.}: cint + +const SupportIoctlInheritCtl = (defined(linux) or defined(bsd)) and + not defined(nimscript) +when SupportIoctlInheritCtl: + var + FIOCLEX {.importc, header: "<sys/ioctl.h>".}: cint + FIONCLEX {.importc, header: "<sys/ioctl.h>".}: cint + + proc c_ioctl(fd: cint, request: cint): cint {. + importc: "ioctl", header: "<sys/ioctl.h>", varargs.} +elif defined(posix) and not defined(lwip) and not defined(nimscript): + var + F_GETFD {.importc, header: "<fcntl.h>".}: cint + F_SETFD {.importc, header: "<fcntl.h>".}: cint + FD_CLOEXEC {.importc, header: "<fcntl.h>".}: cint + + proc c_fcntl(fd: cint, cmd: cint): cint {. + importc: "fcntl", header: "<fcntl.h>", varargs.} +elif defined(windows): + type + WinDWORD = culong + WinBOOL = cint + + const HANDLE_FLAG_INHERIT = 1.WinDWORD + + proc getOsfhandle(fd: cint): int {. + importc: "_get_osfhandle", header: "<io.h>".} + + type + IoHandle = distinct pointer + ## Windows' HANDLE type. Defined as an untyped pointer but is **not** + ## one. Named like this to avoid collision with other `system` modules. + + proc setHandleInformation(hObject: IoHandle, dwMask, dwFlags: WinDWORD): + WinBOOL {.stdcall, dynlib: "kernel32", + importc: "SetHandleInformation".} + +const + BufSize = 4000 + +proc close*(f: File) {.tags: [], gcsafe, sideEffect.} = + ## Closes the file. + if not f.isNil: + discard c_fclose(f) + +proc readChar*(f: File): char {.tags: [ReadIOEffect].} = + ## Reads a single character from the stream `f`. Should not be used in + ## performance sensitive code. + let x = c_fgetc(f) + if x < 0: + checkErr(f) + raiseEOF() + result = char(x) + +proc flushFile*(f: File) {.tags: [WriteIOEffect].} = + ## Flushes `f`'s buffer. + discard c_fflush(f) + +proc getFileHandle*(f: File): FileHandle = + ## Returns the file handle of the file `f`. This is only useful for + ## platform specific programming. + ## Note that on Windows this doesn't return the Windows-specific handle, + ## but the C library's notion of a handle, whatever that means. + ## Use `getOsFileHandle` instead. + c_fileno(f) + +proc getOsFileHandle*(f: File): FileHandle = + ## Returns the OS file handle of the file `f`. This is only useful for + ## platform specific programming. + when defined(windows): + result = FileHandle getOsfhandle(cint getFileHandle(f)) + else: + result = c_fileno(f) + +when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(windows): + proc setInheritable*(f: FileHandle, inheritable: bool): bool = + ## Controls whether a file handle can be inherited by child processes. Returns + ## `true` on success. This requires the OS file handle, which can be + ## retrieved via `getOsFileHandle <#getOsFileHandle,File>`_. + ## + ## This procedure is not guaranteed to be available for all platforms. Test for + ## availability with `declared() <system.html#declared,untyped>`_. + when SupportIoctlInheritCtl: + result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 + elif defined(freertos) or defined(zephyr): + result = true + elif defined(posix): + var flags = c_fcntl(f, F_GETFD) + if flags == -1: + return false + flags = if inheritable: flags and not FD_CLOEXEC else: flags or FD_CLOEXEC + result = c_fcntl(f, F_SETFD, flags) != -1 + else: + result = setHandleInformation(cast[IoHandle](f), HANDLE_FLAG_INHERIT, + inheritable.WinDWORD) != 0 + +proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], + benign.} = + ## Reads a line of text from the file `f` into `line`. May throw an IO + ## exception. + ## A line of text may be delimited by `LF` or `CRLF`. The newline + ## character(s) are not part of the returned string. Returns `false` + ## if the end of the file has been reached, `true` otherwise. If + ## `false` is returned `line` contains no new data. + proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} + + when defined(windows): + proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, + nNumberOfCharsToRead: int32, + lpNumberOfCharsRead: ptr int32, + pInputControl: pointer): int32 {. + importc: "ReadConsoleW", stdcall, dynlib: "kernel32".} + + proc getLastError(): int32 {. + importc: "GetLastError", stdcall, dynlib: "kernel32", sideEffect.} + + proc formatMessageW(dwFlags: int32, lpSource: pointer, + dwMessageId, dwLanguageId: int32, + lpBuffer: pointer, nSize: int32, + arguments: pointer): int32 {. + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} + + proc localFree(p: pointer) {. + importc: "LocalFree", stdcall, dynlib: "kernel32".} + + proc isatty(f: File): bool = + when defined(posix): + proc isatty(fildes: FileHandle): cint {. + importc: "isatty", header: "<unistd.h>".} + else: + proc isatty(fildes: FileHandle): cint {. + importc: "_isatty", header: "<io.h>".} + result = isatty(getFileHandle(f)) != 0'i32 + + # this implies the file is open + if f.isatty: + const numberOfCharsToRead = 2048 + var numberOfCharsRead = 0'i32 + var buffer = newWideCString(numberOfCharsToRead) + if readConsole(getOsFileHandle(f), addr(buffer[0]), + numberOfCharsToRead, addr(numberOfCharsRead), nil) == 0: + var error = getLastError() + var errorMsg: string + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, error, 0, addr(msgbuf), 0, nil) != 0'i32: + errorMsg = $msgbuf + if msgbuf != nil: + localFree(cast[pointer](msgbuf)) + raiseEIO("error: " & $error & " `" & errorMsg & "`") + # input always ends with "\r\n" + numberOfCharsRead -= 2 + # handle Ctrl+Z as EOF + for i in 0..<numberOfCharsRead: + if buffer[i].uint16 == 26: #Ctrl+Z + close(f) #has the same effect as setting EOF + if i == 0: + line = "" + return false + numberOfCharsRead = i + break + buffer[numberOfCharsRead] = 0.Utf16Char + when defined(nimv2): + line = $toWideCString(buffer) + else: + line = $buffer + return(true) + + var pos = 0 + + # Use the currently reserved space for a first try + var sp = max(line.len, 80) + line.setLen(sp) + + while true: + # memset to \L so that we can tell how far fgets wrote, even on EOF, where + # fgets doesn't append an \L + for i in 0..<sp: line[pos+i] = '\L' + + var fgetsSuccess: bool + while true: + # fixes #9634; this pattern may need to be abstracted as a template if reused; + # likely other io procs need this for correctness. + fgetsSuccess = c_fgets(cast[cstring](addr line[pos]), sp.cint, f) != nil + if fgetsSuccess: break + when not defined(nimscript): + if errno == EINTR: + errno = 0 + c_clearerr(f) + continue + checkErr(f) + break + + let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) + if m != nil: + # \l found: Could be our own or the one by fgets, in any case, we're done + var last = cast[int](m) - cast[int](addr line[0]) + if last > 0 and line[last-1] == '\c': + line.setLen(last-1) + return last > 1 or fgetsSuccess + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: + # \0\l\0 => line ending in a null character. + # \0\l\l => last line without newline, null was put there by fgets. + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 + dec last + line.setLen(last) + return last > 0 or fgetsSuccess + else: + # fgets will have inserted a null byte at the end of the string. + dec sp + # No \l found: Increase buffer and read more + inc pos, sp + sp = 128 # read in 128 bytes at a time + line.setLen(pos+sp) + +proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = + ## Reads a line of text from the file `f`. May throw an IO exception. + ## A line of text may be delimited by `LF` or `CRLF`. The newline + ## character(s) are not part of the returned string. + result = newStringOfCap(80) + if not readLine(f, result): raiseEOF() + +proc write*(f: File, i: int) {.tags: [WriteIOEffect], benign.} = + when sizeof(int) == 8: + if c_fprintf(f, "%lld", i) < 0: checkErr(f) + else: + if c_fprintf(f, "%ld", i) < 0: checkErr(f) + +proc write*(f: File, i: BiggestInt) {.tags: [WriteIOEffect], benign.} = + when sizeof(BiggestInt) == 8: + if c_fprintf(f, "%lld", i) < 0: checkErr(f) + else: + if c_fprintf(f, "%ld", i) < 0: checkErr(f) + +proc write*(f: File, b: bool) {.tags: [WriteIOEffect], benign.} = + if b: write(f, "true") + else: write(f, "false") + +proc write*(f: File, r: float32) {.tags: [WriteIOEffect], benign.} = + var buffer {.noinit.}: array[65, char] + discard writeFloatToBuffer(buffer, r) + if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f) + +proc write*(f: File, r: BiggestFloat) {.tags: [WriteIOEffect], benign.} = + var buffer {.noinit.}: array[65, char] + discard writeFloatToBuffer(buffer, r) + if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f) + +proc write*(f: File, c: char) {.tags: [WriteIOEffect], benign.} = + discard c_putc(cint(c), f) + +proc write*(f: File, a: varargs[string, `$`]) {.tags: [WriteIOEffect], benign.} = + for x in items(a): write(f, x) + +proc readAllBuffer(file: File): string = + # This proc is for File we want to read but don't know how many + # bytes we need to read before the buffer is empty. + result = "" + var buffer = newString(BufSize) + while true: + var bytesRead = readBuffer(file, addr(buffer[0]), BufSize) + if bytesRead == BufSize: + result.add(buffer) + else: + buffer.setLen(bytesRead) + result.add(buffer) + break + +proc rawFileSize(file: File): int64 = + # this does not raise an error opposed to `getFileSize` + var oldPos = c_ftell(file) + discard c_fseek(file, 0, 2) # seek the end of the file + result = c_ftell(file) + discard c_fseek(file, oldPos, 0) + +proc endOfFile*(f: File): bool {.tags: [], benign.} = + ## Returns true if `f` is at the end. + var c = c_fgetc(f) + discard c_ungetc(c, f) + return c < 0'i32 + #result = c_feof(f) != 0 + +proc readAllFile(file: File, len: int64): string = + # We acquire the filesize beforehand and hope it doesn't change. + # Speeds things up. + result = newString(len) + let bytes = readBuffer(file, addr(result[0]), len) + if endOfFile(file): + if bytes.int64 < len: + result.setLen(bytes) + else: + # We read all the bytes but did not reach the EOF + # Try to read it as a buffer + result.add(readAllBuffer(file)) + +proc readAllFile(file: File): string = + var len = rawFileSize(file) + result = readAllFile(file, len) + +proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} = + ## Reads all data from the stream `file`. + ## + ## Raises an IO exception in case of an error. It is an error if the + ## current file position is not at the beginning of the file. + + # Separate handling needed because we need to buffer when we + # don't know the overall length of the File. + when declared(stdin): + let len = if file != stdin: rawFileSize(file) else: -1 + else: + let len = rawFileSize(file) + if len > 0: + result = readAllFile(file, len) + else: + result = readAllBuffer(file) + +proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, + tags: [WriteIOEffect], benign.} = + ## Writes the values `x` to `f` and then writes "\\n". + ## May throw an IO exception. + for i in items(x): + write(f, i) + write(f, "\n") + +# interface to the C procs: + +when defined(windows): + when defined(cpp): + proc wfopen(filename, mode: WideCString): pointer {. + importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} + proc wfreopen(filename, mode: WideCString, stream: File): File {. + importcpp: "_wfreopen((const wchar_t*)#, (const wchar_t*)#, #)", nodecl.} + else: + proc wfopen(filename, mode: WideCString): pointer {. + importc: "_wfopen", nodecl.} + proc wfreopen(filename, mode: WideCString, stream: File): File {. + importc: "_wfreopen", nodecl.} + + proc fopen(filename, mode: cstring): pointer = + var f = newWideCString(filename) + var m = newWideCString(mode) + result = wfopen(f, m) + + proc freopen(filename, mode: cstring, stream: File): File = + var f = newWideCString(filename) + var m = newWideCString(mode) + result = wfreopen(f, m, stream) + +else: + proc fopen(filename, mode: cstring): pointer {.importc: "fopen", nodecl.} + proc freopen(filename, mode: cstring, stream: File): File {. + importc: "freopen", nodecl.} + +const + NoInheritFlag = + # Platform specific flag for creating a File without inheritance. + when not defined(nimInheritHandles): + when defined(windows): + "N" + elif defined(linux) or defined(bsd): + "e" + else: + "" + else: + "" + RawFormatOpen: array[FileMode, cstring] = [ + # used for open by FileHandle, which calls `fdopen` + cstring("rb"), "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, cstring] = [ + cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, + "r+b" & NoInheritFlag, "ab" & NoInheritFlag + ] + #"rt", "wt", "w+t", "r+t", "at" + # we always use binary here as for Nim the OS line ending + # should not be translated. + +when defined(posix) and not defined(nimscript): + when defined(linux) and defined(amd64): + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + + # fillers ensure correct size & offsets + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + filler_1: array[24, char] + st_mode: Mode ## Mode of file + filler_2: array[144 - 24 - 4, char] + + proc modeIsDir(m: Mode): bool = + ## Test for a directory. + (m and 0o170000) == 0o40000 + + else: + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_mode: Mode ## Mode of file + + proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "<sys/stat.h>".} + ## Test for a directory. + + proc c_fstat(a1: cint, a2: var Stat): cint {. + importc: "fstat", header: "<sys/stat.h>".} + + +proc open*(f: var File, filename: string, + mode: FileMode = fmRead, + bufSize: int = -1): bool {.tags: [], raises: [], benign.} = + ## Opens a file named `filename` with given `mode`. + ## + ## Default mode is readonly. Returns true if the file could be opened. + ## This throws no exception if the file could not be opened. + ## + ## The file handle associated with the resulting `File` is not inheritable. + var p = fopen(filename.cstring, FormatOpen[mode]) + if p != nil: + var f2 = cast[File](p) + when defined(posix) and not defined(nimscript): + # How `fopen` handles opening a directory is not specified in ISO C and + # POSIX. We do not want to handle directories as regular files that can + # be opened. + var res {.noinit.}: Stat + if c_fstat(getFileHandle(f2), res) >= 0'i32 and modeIsDir(res.st_mode): + close(f2) + return false + when not defined(nimInheritHandles) and declared(setInheritable) and + NoInheritFlag.len == 0: + if not setInheritable(getOsFileHandle(f2), false): + close(f2) + return false + + result = true + f = cast[File](p) + if bufSize > 0 and bufSize.uint <= high(uint): + discard c_setvbuf(f, nil, IOFBF, cast[csize_t](bufSize)) + elif bufSize == 0: + discard c_setvbuf(f, nil, IONBF, 0) + +proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. + tags: [], benign.} = + ## Reopens the file `f` with given `filename` and `mode`. This + ## is often used to redirect the `stdin`, `stdout` or `stderr` + ## file variables. + ## + ## Default mode is readonly. Returns true if the file could be reopened. + ## + ## The file handle associated with `f` won't be inheritable. + if freopen(filename.cstring, FormatOpen[mode], f) != nil: + when not defined(nimInheritHandles) and declared(setInheritable) and + NoInheritFlag.len == 0: + if not setInheritable(getOsFileHandle(f), false): + close(f) + return false + result = true + +proc open*(f: var File, filehandle: FileHandle, + mode: FileMode = fmRead): bool {.tags: [], raises: [], benign.} = + ## Creates a `File` from a `filehandle` with given `mode`. + ## + ## Default mode is readonly. Returns true if the file could be opened. + ## + ## The passed file handle will no longer be inheritable. + when not defined(nimInheritHandles) and declared(setInheritable): + let oshandle = when defined(windows): FileHandle getOsfhandle( + filehandle) else: filehandle + if not setInheritable(oshandle, false): + return false + f = c_fdopen(filehandle, RawFormatOpen[mode]) + result = f != nil + +proc open*(filename: string, + mode: FileMode = fmRead, bufSize: int = -1): File = + ## Opens a file named `filename` with given `mode`. + ## + ## Default mode is readonly. Raises an `IOError` if the file + ## could not be opened. + ## + ## The file handle associated with the resulting `File` is not inheritable. + if not open(result, filename, mode, bufSize): + raise newException(IOError, "cannot open: " & filename) + +proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign, sideEffect.} = + ## Sets the position of the file pointer that is used for read/write + ## operations. The file's first byte has the index zero. + if c_fseek(f, pos, cint(relativeTo)) != 0: + raiseEIO("cannot set file position") + +proc getFilePos*(f: File): int64 {.benign.} = + ## Retrieves the current position of the file pointer that is used to + ## read from the file `f`. The file's first byte has the index zero. + result = c_ftell(f) + if result < 0: raiseEIO("cannot retrieve file position") + +proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} = + ## Retrieves the file size (in bytes) of `f`. + let oldPos = getFilePos(f) + discard c_fseek(f, 0, 2) # seek the end of the file + result = getFilePos(f) + setFilePos(f, oldPos) + +proc setStdIoUnbuffered*() {.tags: [], benign.} = + ## Configures `stdin`, `stdout` and `stderr` to be unbuffered. + when declared(stdout): + discard c_setvbuf(stdout, nil, IONBF, 0) + when declared(stderr): + discard c_setvbuf(stderr, nil, IONBF, 0) + when declared(stdin): + discard c_setvbuf(stdin, nil, IONBF, 0) + + +when defined(windows) and not defined(nimscript) and not defined(js): + # work-around C's sucking abstraction: + # BUGFIX: stdin and stdout should be binary files! + proc c_setmode(handle, mode: cint) {. + importc: when defined(bcc): "setmode" else: "_setmode", + header: "<io.h>".} + var + O_BINARY {.importc: "_O_BINARY", header: "<fcntl.h>".}: cint + + # we use binary mode on Windows: + c_setmode(c_fileno(stdin), O_BINARY) + c_setmode(c_fileno(stdout), O_BINARY) + c_setmode(c_fileno(stderr), O_BINARY) + +when defined(windows) and appType == "console" and + not defined(nimDontSetUtf8CodePage) and not defined(nimscript): + import std/exitprocs + + proc setConsoleOutputCP(codepage: cuint): int32 {.stdcall, dynlib: "kernel32", + importc: "SetConsoleOutputCP".} + proc setConsoleCP(wCodePageID: cuint): int32 {.stdcall, dynlib: "kernel32", + importc: "SetConsoleCP".} + proc getConsoleOutputCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleOutputCP".} + proc getConsoleCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleCP".} + + const Utf8codepage = 65001'u32 + + let + consoleOutputCP = getConsoleOutputCP() + consoleCP = getConsoleCP() + + proc restoreConsoleOutputCP() = discard setConsoleOutputCP(consoleOutputCP) + proc restoreConsoleCP() = discard setConsoleCP(consoleCP) + + if consoleOutputCP != Utf8codepage: + discard setConsoleOutputCP(Utf8codepage) + addExitProc(restoreConsoleOutputCP) + + if consoleCP != Utf8codepage: + discard setConsoleCP(Utf8codepage) + addExitProc(restoreConsoleCP) + +proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = + ## Opens a file named `filename` for reading, calls `readAll + ## <#readAll,File>`_ and closes the file afterwards. Returns the string. + ## Raises an IO exception in case of an error. If you need to call + ## this inside a compile time macro you can use `staticRead + ## <system.html#staticRead,string>`_. + var f: File = nil + if open(f, filename): + try: + result = readAll(f) + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = + ## Opens a file named `filename` for writing. Then writes the + ## `content` completely to the file and closes the file afterwards. + ## Raises an IO exception in case of an error. + var f: File = nil + if open(f, filename, fmWrite): + try: + f.write(content) + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = + ## Opens a file named `filename` for writing. Then writes the + ## `content` completely to the file and closes the file afterwards. + ## Raises an IO exception in case of an error. + var f: File = nil + if open(f, filename, fmWrite): + try: + discard f.writeBuffer(unsafeAddr content[0], content.len) + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +proc readLines*(filename: string, n: Natural): seq[string] = + ## Reads `n` lines from the file named `filename`. Raises an IO exception + ## in case of an error. Raises EOF if file does not contain at least `n` lines. + ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`. + ## The newline character(s) are not part of the returned strings. + var f: File = nil + if open(f, filename): + try: + result = newSeq[string](n) + for i in 0 .. n - 1: + if not readLine(f, result[i]): + raiseEOF() + finally: + close(f) + else: + raise newException(IOError, "cannot open: " & filename) + +template readLines*(filename: string): seq[ + string] {.deprecated: "use readLines with two arguments".} = + readLines(filename, 1) + +iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = + ## Iterates over any line in the file named `filename`. + ## + ## If the file does not exist `IOError` is raised. The trailing newline + ## character(s) are removed from the iterated lines. Example: + ## + runnableExamples: + import std/strutils + + proc transformLetters(filename: string) = + var buffer = "" + for line in filename.lines: + buffer.add(line.replace("a", "0") & '\n') + writeFile(filename, buffer) + var f = open(filename, bufSize = 8000) + try: + var res = newStringOfCap(80) + while f.readLine(res): yield res + finally: + close(f) + +iterator lines*(f: File): string {.tags: [ReadIOEffect].} = + ## Iterates over any line in the file `f`. + ## + ## The trailing newline character(s) are removed from the iterated lines. + ## + runnableExamples: + proc countZeros(filename: File): tuple[lines, zeros: int] = + for line in filename.lines: + for letter in line: + if letter == '0': + result.zeros += 1 + result.lines += 1 + var res = newStringOfCap(80) + while f.readLine(res): yield res + +template `&=`*(f: File, x: typed) = + ## An alias for `write`. + write(f, x) diff --git a/lib/system/atomics.nim b/lib/std/sysatomics.nim index 158fe91bc..2f203b3eb 100644 --- a/lib/system/atomics.nim +++ b/lib/std/sysatomics.nim @@ -7,13 +7,24 @@ # distribution, for details about the copyright. # +when defined(nimPreviewSlimSystem): + {.deprecated: "use `std/atomics` instead".} + # Atomic operations for Nim. -{.push stackTrace:off.} +{.push stackTrace:off, profiler:off.} + +const + hasThreadSupport = compileOption("threads") and not defined(nimscript) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) +const someVcc = defined(vcc) or defined(clang_cl) -const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) +type + AtomType* = SomeNumber|pointer|ptr|char|bool + ## Type Class representing valid types for use with atomic procs -when someGcc and hasThreadSupport: +when someGcc: type AtomMemModel* = distinct cint + var ATOMIC_RELAXED* {.importc: "__ATOMIC_RELAXED", nodecl.}: AtomMemModel ## No barriers or synchronization. var ATOMIC_CONSUME* {.importc: "__ATOMIC_CONSUME", nodecl.}: AtomMemModel @@ -36,11 +47,6 @@ when someGcc and hasThreadSupport: ## with acquire loads ## and release stores in all threads. - type - AtomType* = SomeNumber|pointer|ptr|char|bool - ## Type Class representing valid types for use with atomic procs - {.deprecated: [TAtomType: AtomType].} - proc atomicLoadN*[T: AtomType](p: ptr T, mem: AtomMemModel): T {. importc: "__atomic_load_n", nodecl.} ## This proc implements an atomic load operation. It returns the contents at p. @@ -72,7 +78,7 @@ when someGcc and hasThreadSupport: proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. - importc: "__atomic_compare_exchange_n ", nodecl.} + importc: "__atomic_compare_exchange_n", nodecl.} ## This proc implements an atomic compare and exchange operation. This compares the ## contents at p with the contents at expected and if equal, writes desired at p. ## If they are not equal, the current contents at p is written into expected. @@ -99,13 +105,13 @@ when someGcc and hasThreadSupport: proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_sub_fetch", nodecl.} proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_or_fetch ", nodecl.} + importc: "__atomic_or_fetch", nodecl.} proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_and_fetch", nodecl.} proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_xor_fetch", nodecl.} proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_nand_fetch ", nodecl.} + importc: "__atomic_nand_fetch", nodecl.} ## Perform the operation return the old value, all memory models are valid proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. @@ -124,8 +130,8 @@ when someGcc and hasThreadSupport: proc atomicTestAndSet*(p: pointer, mem: AtomMemModel): bool {. importc: "__atomic_test_and_set", nodecl.} ## This built-in function performs an atomic test-and-set operation on the byte at p. - ## The byte is set to some implementation defined nonzero “set” value and the return - ## value is true if and only if the previous contents were “set”. + ## The byte is set to some implementation defined nonzero "set" value and the return + ## value is true if and only if the previous contents were "set". ## All memory models are valid. proc atomicClear*(p: pointer, mem: AtomMemModel) {. @@ -163,40 +169,182 @@ when someGcc and hasThreadSupport: ## ignore this parameter. template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) -elif defined(vcc) and hasThreadSupport: - proc addAndFetch*(p: ptr int, val: int): int {. - importc: "NimXadd", nodecl.} +elif someVcc: + type AtomMemModel* = distinct cint + + const + ATOMIC_RELAXED* = 0.AtomMemModel + ATOMIC_CONSUME* = 1.AtomMemModel + ATOMIC_ACQUIRE* = 2.AtomMemModel + ATOMIC_RELEASE* = 3.AtomMemModel + ATOMIC_ACQ_REL* = 4.AtomMemModel + ATOMIC_SEQ_CST* = 5.AtomMemModel + + proc `==`(x1, x2: AtomMemModel): bool {.borrow.} + + proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} proc fence*() {.importc: "_ReadWriteBarrier", header: "<intrin.h>".} + when defined(cpp): + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importcpp: "_InterlockedCompareExchange(static_cast<long volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importcpp: "_InterlockedExchange8(static_cast<NI8 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importcpp: "_InterlockedExchange16(static_cast<NI16 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importcpp: "_InterlockedExchange(static_cast<long volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importcpp: "_InterlockedExchange64(static_cast<NI64 volatile *>(#), #)", header: "<intrin.h>".} + else: + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} + + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64", header: "<intrin.h>".} + + + template barrier(mem: AtomMemModel) = + when mem == ATOMIC_RELAXED: discard + elif mem == ATOMIC_CONSUME: readBarrier() + elif mem == ATOMIC_ACQUIRE: writeBarrier() + elif mem == ATOMIC_RELEASE: fence() + elif mem == ATOMIC_ACQ_REL: fence() + elif mem == ATOMIC_SEQ_CST: fence() + + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: static[AtomMemModel]) = + barrier(mem) + p[] = val + + proc atomicLoadN*[T: AtomType](p: ptr T, mem: static[AtomMemModel]): T = + result = p[] + barrier(mem) + + proc atomicCompareExchangeN*[T: ptr](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](desired), cast[int64](expected)) == + cast[int64](expected) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](desired), cast[int32](expected)) == + cast[int32](expected) + + proc atomicExchangeN*[T: ptr](p: ptr T, val: T, mem: AtomMemModel): T = + when sizeof(T) == 8: + cast[T](interlockedExchange64(p, cast[int64](val))) + elif sizeof(T) == 4: + cast[T](interlockedExchange32(p, cast[int32](val))) + when defined(cpp): + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd64(static_cast<NI volatile *>(#), #)", + header: "<intrin.h>".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd(reinterpret_cast<long volatile *>(#), static_cast<long>(#))", + header: "<intrin.h>".} + else: + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd64", header: "<intrin.h>".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd", header: "<intrin.h>".} + else: proc addAndFetch*(p: ptr int, val: int): int {.inline.} = inc(p[], val) result = p[] -proc atomicInc*(memLoc: var int, x: int = 1): int = + +proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically increments the integer by some `x`. It returns the new value. when someGcc and hasThreadSupport: - result = atomic_add_fetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) + elif someVcc and hasThreadSupport: + result = addAndFetch(memLoc.addr, x) + inc(result, x) else: inc(memLoc, x) result = memLoc -proc atomicDec*(memLoc: var int, x: int = 1): int = +proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically decrements the integer by some `x`. It returns the new value. when someGcc and hasThreadSupport: - when declared(atomic_sub_fetch): - result = atomic_sub_fetch(memLoc.addr, x, ATOMIC_RELAXED) + when declared(atomicSubFetch): + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) else: - result = atomic_add_fetch(memLoc.addr, -x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) + elif someVcc and hasThreadSupport: + result = addAndFetch(memLoc.addr, -x) + dec(result, x) else: dec(memLoc, x) result = memLoc -when defined(windows) and not someGcc: - proc interlockedCompareExchange(p: pointer; exchange, comparand: int): int - {.importc: "InterlockedCompareExchange", header: "<windows.h>", cdecl.} +when someVcc: + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](newValue), cast[int64](oldValue)) == + cast[int64](oldValue) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](newValue), cast[int32](oldValue)) == + cast[int32](oldValue) + elif sizeof(T) == 1: + interlockedCompareExchange8(p, cast[byte](newValue), cast[byte](oldValue)) == + cast[byte](oldValue) + else: + {.error: "invalid CAS instruction".} +elif defined(tcc): + when defined(amd64): + {.emit:""" +static int __tcc_cas(int *ptr, int oldVal, int newVal) +{ + unsigned char ret; + __asm__ __volatile__ ( + " lock\n" + " cmpxchgq %2,%1\n" + " sete %0\n" + : "=q" (ret), "=m" (*ptr) + : "r" (newVal), "m" (*ptr), "a" (oldVal) + : "memory"); + + return ret; +} +""".} + else: + #assert sizeof(int) == 4 + {.emit:""" +static int __tcc_cas(int *ptr, int oldVal, int newVal) +{ + unsigned char ret; + __asm__ __volatile__ ( + " lock\n" + " cmpxchgl %2,%1\n" + " sete %0\n" + : "=q" (ret), "=m" (*ptr) + : "r" (newVal), "m" (*ptr), "a" (oldVal) + : "memory"); + + return ret; +} +""".} + + proc tcc_cas(p: ptr int; oldValue, newValue: int): bool + {.importc: "__tcc_cas", nodecl.} proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = - interlockedCompareExchange(p, cast[int](newValue), cast[int](oldValue)) != 0 - # XXX fix for 64 bit build + tcc_cas(cast[ptr int](p), cast[int](oldValue), cast[int](newValue)) +elif declared(atomicCompareExchangeN): + proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = + atomicCompareExchangeN(p, oldValue.unsafeAddr, newValue, false, ATOMIC_SEQ_CST, ATOMIC_SEQ_CST) else: # this is valid for GCC and Intel C++ proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool @@ -204,18 +352,18 @@ else: # XXX is this valid for 'int'? -when (defined(x86) or defined(amd64)) and someGcc: +when (defined(x86) or defined(amd64)) and someVcc: + proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} +elif (defined(x86) or defined(amd64)) and (someGcc or defined(bcc)): proc cpuRelax* {.inline.} = {.emit: """asm volatile("pause" ::: "memory");""".} -elif someGcc: +elif someGcc or defined(tcc): proc cpuRelax* {.inline.} = {.emit: """asm volatile("" ::: "memory");""".} -elif (defined(x86) or defined(amd64)) and defined(vcc): - proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} elif defined(icl): proc cpuRelax* {.importc: "_mm_pause", header: "xmmintrin.h".} elif false: - from os import sleep + from std/os import sleep proc cpuRelax* {.inline.} = os.sleep(1) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim new file mode 100644 index 000000000..6f2c6b0c1 --- /dev/null +++ b/lib/std/sysrand.nim @@ -0,0 +1,326 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## .. warning:: This module was added in Nim 1.6. If you are using it for cryptographic purposes, +## keep in mind that so far this has not been audited by any security professionals, +## therefore may not be secure. +## +## `std/sysrand` generates random numbers from a secure source provided by the operating system. +## It is a cryptographically secure pseudorandom number generator +## and should be unpredictable enough for cryptographic applications, +## though its exact quality depends on the OS implementation. +## +## | Targets | Implementation | +## | :--- | ----: | +## | Windows | `BCryptGenRandom`_ | +## | Linux | `getrandom`_ | +## | MacOSX | `SecRandomCopyBytes`_ | +## | iOS | `SecRandomCopyBytes`_ | +## | OpenBSD | `getentropy openbsd`_ | +## | FreeBSD | `getrandom freebsd`_ | +## | JS (Web Browser) | `getRandomValues`_ | +## | Node.js | `randomFillSync`_ | +## | Other Unix platforms | `/dev/urandom`_ | +## +## .. _BCryptGenRandom: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom +## .. _getrandom: https://man7.org/linux/man-pages/man2/getrandom.2.html +## .. _getentropy: https://www.unix.com/man-page/mojave/2/getentropy +## .. _SecRandomCopyBytes: https://developer.apple.com/documentation/security/1399291-secrandomcopybytes?language=objc +## .. _getentropy openbsd: https://man.openbsd.org/getentropy.2 +## .. _getrandom freebsd: https://www.freebsd.org/cgi/man.cgi?query=getrandom&manpath=FreeBSD+12.0-stable +## .. _getRandomValues: https://www.w3.org/TR/WebCryptoAPI/#Crypto-method-getRandomValues +## .. _randomFillSync: https://nodejs.org/api/crypto.html#crypto_crypto_randomfillsync_buffer_offset_size +## .. _/dev/urandom: https://en.wikipedia.org/wiki//dev/random +## +## On a Linux target, a call to the `getrandom` syscall can be avoided (e.g. +## for targets running kernel version < 3.17) by passing a compile flag of +## `-d:nimNoGetRandom`. If this flag is passed, sysrand will use `/dev/urandom` +## as with any other POSIX compliant OS. +## + +runnableExamples: + doAssert urandom(0).len == 0 + doAssert urandom(113).len == 113 + doAssert urandom(1234) != urandom(1234) # unlikely to fail in practice + +## +## See also +## ======== +## * `random module <random.html>`_ +## + + +when not defined(js): + import std/oserrors + +when defined(posix): + import std/posix + +when defined(nimPreviewSlimSystem): + import std/assertions + +const + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) + batchSize {.used.} = 256 + +when batchImplOS: + template batchImpl(result: var int, dest: var openArray[byte], getRandomImpl) = + let size = dest.len + if size == 0: + return + + let + chunks = (size - 1) div batchSize + left = size - chunks * batchSize + + for i in 0 ..< chunks: + let readBytes = getRandomImpl(addr dest[result], batchSize) + if readBytes < 0: + return readBytes + inc(result, batchSize) + + result = getRandomImpl(addr dest[result], left) + +when defined(js): + import std/private/jsutils + + when defined(nodejs): + {.emit: "const _nim_nodejs_crypto = require('crypto');".} + + proc randomFillSync(p: Uint8Array) {.importjs: "_nim_nodejs_crypto.randomFillSync(#)".} + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + var src = newUint8Array(size) + randomFillSync(src) + for i in 0 ..< size: + dest[i] = src[i] + + else: + proc getRandomValues(p: Uint8Array) {.importjs: "window.crypto.getRandomValues(#)".} + # The requested length of `p` must not be more than 65536. + + proc assign(dest: var openArray[byte], src: Uint8Array, base: int, size: int) = + getRandomValues(src) + for j in 0 ..< size: + dest[base + j] = src[j] + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + if size <= batchSize: + var src = newUint8Array(size) + assign(dest, src, 0, size) + return + + let + chunks = (size - 1) div batchSize + left = size - chunks * batchSize + + var srcArray = newUint8Array(batchSize) + for i in 0 ..< chunks: + assign(dest, srcArray, result, batchSize) + inc(result, batchSize) + + var leftArray = newUint8Array(left) + assign(dest, leftArray, result, left) + +elif defined(windows): + type + PVOID = pointer + BCRYPT_ALG_HANDLE = PVOID + PUCHAR = ptr uint8 + NTSTATUS = clong + ULONG = culong + + const + STATUS_SUCCESS = 0x00000000 + BCRYPT_USE_SYSTEM_PREFERRED_RNG = 0x00000002 + + proc bCryptGenRandom( + hAlgorithm: BCRYPT_ALG_HANDLE, + pbBuffer: PUCHAR, + cbBuffer: ULONG, + dwFlags: ULONG + ): NTSTATUS {.stdcall, importc: "BCryptGenRandom", dynlib: "Bcrypt.dll".} + + + proc randomBytes(pbBuffer: pointer, cbBuffer: Natural): int {.inline.} = + bCryptGenRandom(nil, cast[PUCHAR](pbBuffer), ULONG(cbBuffer), + BCRYPT_USE_SYSTEM_PREFERRED_RNG) + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + result = randomBytes(addr dest[0], size) + +elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): + when (NimMajor, NimMinor) >= (1, 4): + let SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + else: + var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + const syscallHeader = """#include <unistd.h> +#include <sys/syscall.h>""" + + proc syscall(n: clong): clong {. + importc: "syscall", varargs, header: syscallHeader.} + # When reading from the urandom source (GRND_RANDOM is not set), + # getrandom() will block until the entropy pool has been + # initialized (unless the GRND_NONBLOCK flag was specified). If a + # request is made to read a large number of bytes (more than 256), + # getrandom() will block until those bytes have been generated and + # transferred from kernel memory to buf. + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + while result < size: + let readBytes = syscall(SYS_getrandom, addr dest[result], cint(size - result), 0).int + if readBytes == 0: + raiseAssert "unreachable" + elif readBytes > 0: + inc(result, readBytes) + else: + if osLastError().cint in [EINTR, EAGAIN]: discard + else: + result = -1 + break + +elif defined(openbsd): + proc getentropy(p: pointer, size: cint): cint {.importc: "getentropy", header: "<unistd.h>".} + # Fills a buffer with high-quality entropy, + # which can be used as input for process-context pseudorandom generators like `arc4random`. + # The maximum buffer size permitted is 256 bytes. + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + result = getentropy(p, cint(size)).int + +elif defined(zephyr): + proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "<random/rand32.h>".} + # Fill the destination buffer with cryptographically secure + # random data values + # + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + # 0 if success, -EIO if entropy reseed error + result = sys_csrand_get(p, csize_t(size)).int + +elif defined(freebsd): + type cssize_t {.importc: "ssize_t", header: "<sys/types.h>".} = int + + proc getrandom(p: pointer, size: csize_t, flags: cuint): cssize_t {.importc: "getrandom", header: "<sys/random.h>".} + # Upon successful completion, the number of bytes which were actually read + # is returned. For requests larger than 256 bytes, this can be fewer bytes + # than were requested. Otherwise, -1 is returned and the global variable + # errno is set to indicate the error. + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + result = getrandom(p, csize_t(size), 0) + +elif defined(ios) or defined(macosx): + {.passl: "-framework Security".} + + const errSecSuccess = 0 ## No error. + + type + SecRandom {.importc: "struct __SecRandom".} = object + + SecRandomRef = ptr SecRandom + ## An abstract Core Foundation-type object containing information about a random number generator. + + proc secRandomCopyBytes( + rnd: SecRandomRef, count: csize_t, bytes: pointer + ): cint {.importc: "SecRandomCopyBytes", header: "<Security/SecRandom.h>".} + ## https://developer.apple.com/documentation/security/1399291-secrandomcopybytes + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + result = secRandomCopyBytes(nil, csize_t(size), addr dest[0]) + +else: + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + # see: https://www.2uo.de/myths-about-urandom/ which justifies using urandom instead of random + let fd = posix.open("/dev/urandom", O_RDONLY) + + if fd < 0: + result = -1 + else: + try: + var stat: Stat + if fstat(fd, stat) != -1 and S_ISCHR(stat.st_mode): + let + chunks = (size - 1) div batchSize + left = size - chunks * batchSize + + for i in 0 ..< chunks: + let readBytes = posix.read(fd, addr dest[result], batchSize) + if readBytes < 0: + return readBytes + inc(result, batchSize) + + result = posix.read(fd, addr dest[result], left) + else: + result = -1 + finally: + discard posix.close(fd) + +proc urandomInternalImpl(dest: var openArray[byte]): int {.inline.} = + when batchImplOS: + batchImpl(result, dest, getRandomImpl) + else: + urandomImpl(result, dest) + +proc urandom*(dest: var openArray[byte]): bool = + ## Fills `dest` with random bytes suitable for cryptographic use. + ## If the call succeeds, returns `true`. + ## + ## If `dest` is empty, `urandom` immediately returns success, + ## without calling the underlying operating system API. + ## + ## .. warning:: The code hasn't been audited by cryptography experts and + ## is provided as-is without guarantees. Use at your own risks. For production + ## systems we advise you to request an external audit. + result = true + when defined(js): discard urandomInternalImpl(dest) + else: + let ret = urandomInternalImpl(dest) + when defined(windows): + if ret != STATUS_SUCCESS: + result = false + else: + if ret < 0: + result = false + +proc urandom*(size: Natural): seq[byte] {.inline.} = + ## Returns random bytes suitable for cryptographic use. + ## + ## .. warning:: The code hasn't been audited by cryptography experts and + ## is provided as-is without guarantees. Use at your own risks. For production + ## systems we advise you to request an external audit. + result = newSeq[byte](size) + when defined(js): discard urandomInternalImpl(result) + else: + if not urandom(result): + raiseOSError(osLastError()) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim new file mode 100644 index 000000000..7e59747f5 --- /dev/null +++ b/lib/std/tasks.nim @@ -0,0 +1,312 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides basic primitives for creating parallel programs. +## A `Task` should be only owned by a single Thread, it cannot be shared by threads. + +import std/[macros, isolation, typetraits] + +when defined(nimPreviewSlimSystem): + import std/assertions + +export isolation + + +when compileOption("threads"): + from std/effecttraits import isGcSafe + + +# +# proc hello(a: int, b: string) = +# echo $a & b +# +# let literal = "Nim" +# let t = toTask(hello(521, literal)) +# +# +# is roughly converted to +# +# type +# ScratchObj_369098780 = object +# a: int +# b: string +# +# let scratch_369098762 = cast[ptr ScratchObj_369098780](c_calloc(csize_t 1, +# csize_t sizeof(ScratchObj_369098780))) +# if scratch_369098762.isNil: +# raise newException(OutOfMemDefect, "Could not allocate memory") +# block: +# var isolate_369098776 = isolate(521) +# scratch_369098762.a = extract(isolate_369098776) +# var isolate_369098778 = isolate(literal) +# scratch_369098762.b = extract(isolate_369098778) +# proc hello_369098781(args`gensym3: pointer) {.nimcall.} = +# let objTemp_369098775 = cast[ptr ScratchObj_369098780](args`gensym3) +# let :tmp_369098777 = objTemp_369098775.a +# let :tmp_369098779 = objTemp_369098775.b +# hello(a = :tmp_369098777, b = :tmp_369098779) +# +# proc destroyScratch_369098782(args`gensym3: pointer) {.nimcall.} = +# let obj_369098783 = cast[ptr ScratchObj_369098780](args`gensym3) +# =destroy(obj_369098783[]) +# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782) +# + + +type + Task* = object ## `Task` contains the callback and its arguments. + callback: proc (args, res: pointer) {.nimcall, gcsafe.} + args: pointer + destroy: proc (args: pointer) {.nimcall, gcsafe.} + + +proc `=copy`*(x: var Task, y: Task) {.error.} + +const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) +when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`*(t: Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) +else: + proc `=destroy`*(t: var Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) + +proc invoke*(task: Task; res: pointer = nil) {.inline, gcsafe.} = + ## Invokes the `task`. + assert task.callback != nil + task.callback(task.args, res) + +template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = + # block: + # var isoTempA = isolate(521) + # scratch.a = extract(isolateA) + # var isoTempB = isolate(literal) + # scratch.b = extract(isolateB) + let isolatedTemp = genSym(nskTemp, "isoTemp") + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam)) + scratchAssignList.add newAssignment(scratchDotExpr, + newCall(newIdentNode("extract"), isolatedTemp)) + +template addAllNode(assignParam: NimNode, procParam: NimNode) = + let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) + + checkIsolate(scratchAssignList, procParam, scratchDotExpr) + + let tempNode = genSym(kind = nskTemp, ident = formalParams[i][0].strVal) + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], tempNode) + tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0])) + scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam) + +proc analyseRootSym(s: NimNode): NimNode = + result = s + while true: + case result.kind + of nnkBracketExpr, nnkDerefExpr, nnkHiddenDeref, + nnkAddr, nnkHiddenAddr, + nnkObjDownConv, nnkObjUpConv: + result = result[0] + of nnkDotExpr, nnkCheckedFieldExpr, nnkHiddenStdConv, nnkHiddenSubConv: + result = result[1] + else: + break + +macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task = + ## Converts the call and its arguments to `Task`. + runnableExamples: + proc hello(a: int) = echo a + + let b = toTask hello(13) + assert b is Task + + let retType = getTypeInst(e) + let returnsVoid = retType.typeKind == ntyVoid + + let rootSym = analyseRootSym(e[0]) + expectKind rootSym, nnkSym + + when compileOption("threads"): + if not isGcSafe(rootSym): + error("'toTask' takes a GC safe call expression", e) + + if hasClosure(rootSym): + error("closure call is not allowed", e) + + if e.len > 1: + let scratchIdent = genSym(kind = nskTemp, ident = "scratch") + let impl = e[0].getTypeInst + + when defined(nimTasksDebug): + echo impl.treeRepr + echo e.treeRepr + let formalParams = impl[0] + + var + scratchRecList = newNimNode(nnkRecList) + scratchAssignList: seq[NimNode] + tempAssignList: seq[NimNode] + callNode: seq[NimNode] + + let + objTemp = genSym(nskTemp, ident = "objTemp") + + for i in 1 ..< formalParams.len: + var param = formalParams[i][1] + + if param.kind == nnkBracketExpr and param[0].eqIdent("sink"): + param = param[0] + + if param.typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + + case param.kind + of nnkVarTy: + error("'toTask'ed function cannot have a 'var' parameter", e) + of nnkBracketExpr: + if param[0].typeKind == ntyTypeDesc: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: + if param[1].typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + let + seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) + seqCallNode = newCall("@", e[i]) + addAllNode(seqType, seqCallNode) + else: + addAllNode(param, e[i]) + of nnkBracket, nnkObjConstr: + # passing by static parameters + # so we pass them directly instead of passing by scratchObj + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + of nnkSym, nnkPtrTy, nnkProcTy, nnkTupleConstr: + addAllNode(param, e[i]) + of nnkCharLit..nnkNilLit: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + else: + error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e) + + let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") + let scratchObj = nnkTypeSection.newTree( + nnkTypeDef.newTree( + scratchObjType, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + scratchRecList + ) + ) + ) + + + let scratchObjPtrType = quote do: + cast[ptr `scratchObjType`](allocShared0(sizeof(`scratchObjType`))) + + let scratchLetSection = newLetStmt(scratchIdent, scratchObjPtrType) + + var stmtList = newStmtList() + stmtList.add(scratchObj) + stmtList.add(scratchLetSection) + stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList))) + + var functionStmtList = newStmtList() + let funcCall = newCall(e[0], callNode) + functionStmtList.add tempAssignList + + let funcName = genSym(nskProc, rootSym.strVal) + let destroyName = genSym(nskProc, "destroyScratch") + let objTemp2 = genSym(ident = "obj") + let tempNode = quote("@") do: + `=destroy`(@objTemp2[]) + + var funcDecl: NimNode + if returnsVoid: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + `funcCall` + else: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + cast[ptr `retType`](res)[] = `funcCall` + + result = quote do: + `stmtList` + + `funcDecl` + + proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = + let `objTemp2` = cast[ptr `scratchObjType`](args) + `tempNode` + + Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`) + else: + let funcCall = newCall(e[0]) + let funcName = genSym(nskProc, rootSym.strVal) + + if returnsVoid: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + `funcCall` + + Task(callback: `funcName`, args: nil) + else: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + cast[ptr `retType`](res)[] = `funcCall` + + Task(callback: `funcName`, args: nil) + + + when defined(nimTasksDebug): + echo result.repr + +runnableExamples: + block: + var num = 0 + proc hello(a: int) = inc num, a + + let b = toTask hello(13) + b.invoke() + assert num == 13 + # A task can be invoked multiple times + b.invoke() + assert num == 26 + + block: + type + Runnable = ref object + data: int + + var data: int + proc hello(a: Runnable) {.nimcall.} = + a.data += 2 + data = a.data + + + when false: + # the parameters of call must be isolated. + let x = Runnable(data: 12) + let b = toTask hello(x) # error ----> expression cannot be isolated: x + b.invoke() + + let b = toTask(hello(Runnable(data: 12))) + b.invoke() + assert data == 14 + b.invoke() + assert data == 16 diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim new file mode 100644 index 000000000..539305bde --- /dev/null +++ b/lib/std/tempfiles.nim @@ -0,0 +1,192 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module creates temporary files and directories. +## +## Experimental API, subject to change. + +#[ +See also: +* `GetTempFileName` (on windows), refs https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettempfilenamea +* `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html +]# + +import std / [os, random] + +when defined(nimPreviewSlimSystem): + import std/syncio + +const + maxRetry = 10000 + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nimTempPathLength {.intdefine.} = 8 + + +when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + var O_RDWR {.importc: "_O_RDWR", header: "<fcntl.h>".}: cint + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "_fdopen",header: "<stdio.h>".} + + proc open_osfhandle(osh: Handle, mode: cint): cint {. + importc: "_open_osfhandle", header: "<io.h>".} + + proc close_osfandle(fd: cint): cint {. + importc: "_close", header: "<io.h>".} +else: + import std/posix + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "fdopen",header: "<stdio.h>".} + + +proc safeOpen(filename: string): File = + ## Open files exclusively; returns `nil` if the file already exists. + # xxx this should be clarified; it doesn't in particular prevent other processes + # from opening the file, at least currently. + when defined(windows): + let dwShareMode = FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE + let dwCreation = CREATE_NEW + let dwFlags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + let handle = createFileW(newWideCString(filename), GENERIC_READ or GENERIC_WRITE, dwShareMode, + nil, dwCreation, dwFlags, Handle(0)) + + if handle == INVALID_HANDLE_VALUE: + if getLastError() == ERROR_FILE_EXISTS: + return nil + else: + raiseOSError(osLastError(), filename) + + let fileHandle = open_osfhandle(handle, O_RDWR) + if fileHandle == -1: + discard closeHandle(handle) + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard close_osfandle(fileHandle) + raiseOSError(osLastError(), filename) + else: + # xxx we need a `proc toMode(a: FilePermission): Mode`, possibly by + # exposing fusion/filepermissions.fromFilePermissions to stdlib; then we need + # to expose a `perm` param so users can customize this (e.g. the temp file may + # need execute permissions), and figure out how to make the API cross platform. + let mode = Mode(S_IRUSR or S_IWUSR) + let flags = posix.O_RDWR or posix.O_CREAT or posix.O_EXCL + let fileHandle = posix.open(filename, flags, mode) + if fileHandle == -1: + if errno == EEXIST: + # xxx `getLastError()` should be defined on posix too and resolve to `errno`? + return nil + else: + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard posix.close(fileHandle) # TODO handles failure when closing file + raiseOSError(osLastError(), filename) + + +type + NimTempPathState = object + state: Rand + isInit: bool + +var nimTempPathState {.threadvar.}: NimTempPathState + +template randomPathName(length: Natural): string = + var res = newString(length) + if not nimTempPathState.isInit: + nimTempPathState.isInit = true + nimTempPathState.state = initRand() + + for i in 0 ..< length: + res[i] = nimTempPathState.state.sample(letters) + res + +proc getTempDirImpl(dir: string): string {.inline.} = + result = dir + if result.len == 0: + result = getTempDir() + +proc genTempPath*(prefix, suffix: string, dir = ""): string = + ## Generates a path name in `dir`. + ## + ## The path begins with `prefix` and ends with `suffix`. + ## + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + let dir = getTempDirImpl(dir) + result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + +proc createTempFile*(prefix, suffix: string, dir = ""): tuple[cfile: File, path: string] = + ## Creates a new temporary file in the directory `dir`. + ## + ## This generates a path name using `genTempPath(prefix, suffix, dir)` and + ## returns a file handle to an open file and the path of that file, possibly after + ## retrying to ensure it doesn't already exist. + ## + ## If failing to create a temporary file, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to close `result.cfile` and + ## remove `result.file` when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempFile("", "", "nonexistent") + let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp") + # path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp" + cfile.write "foo" + cfile.setFilePos 0 + assert readAll(cfile) == "foo" + close cfile + assert readFile(path) == "foo" + removeFile(path) + # xxx why does above work without `cfile.flushFile` ? + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result.path = genTempPath(prefix, suffix, dir) + result.cfile = safeOpen(result.path) + if result.cfile != nil: + return + + raise newException(OSError, "Failed to create a temporary file under directory " & dir) + +proc createTempDir*(prefix, suffix: string, dir = ""): string = + ## Creates a new temporary directory in the directory `dir`. + ## + ## This generates a dir name using `genTempPath(prefix, suffix, dir)`, creates + ## the directory and returns it, possibly after retrying to ensure it doesn't + ## already exist. + ## + ## If failing to create a temporary directory, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempDir("", "", "nonexistent") + let dir = createTempDir("tmpprefix_", "_end") + # dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end" + assert dirExists(dir) + removeDir(dir) + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result = genTempPath(prefix, suffix, dir) + if not existsOrCreateDir(result): + return + + raise newException(OSError, "Failed to create a temporary directory under directory " & dir) diff --git a/lib/std/time_t.nim b/lib/std/time_t.nim new file mode 100644 index 000000000..de051b135 --- /dev/null +++ b/lib/std/time_t.nim @@ -0,0 +1,23 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimdoc): + type + Impl = distinct int64 + Time* = Impl ## \ + ## Wrapper for `time_t`. On posix, this is an alias to `posix.Time`. +elif defined(windows): + when defined(i386) and defined(gcc): + type Time* {.importc: "time_t", header: "<time.h>".} = distinct clong + else: + # newest version of Visual C++ defines time_t to be of 64 bits + type Time* {.importc: "time_t", header: "<time.h>".} = distinct int64 +elif defined(posix): + import std/posix + export posix.Time \ No newline at end of file diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim new file mode 100644 index 000000000..7b0b81968 --- /dev/null +++ b/lib/std/typedthreads.nim @@ -0,0 +1,305 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +Thread support for Nim. Threads allow multiple functions to execute concurrently. + +In Nim, threads are a low-level construct and using a library like `malebolgia`, `taskpools` or `weave` is recommended. + +When creating a thread, you can pass arguments to it. As Nim's garbage collector does not use atomic references, sharing +`ref` and other variables managed by the garbage collector between threads is not supported. +Use global variables to do so, or pointers. + +Memory allocated using [`sharedAlloc`](./system.html#allocShared.t%2CNatural) can be used and shared between threads. + +To communicate between threads, consider using [channels](./system.html#Channel) + +Examples +======== + +```Nim +import std/locks + +var + thr: array[0..4, Thread[tuple[a,b: int]]] + L: Lock + +proc threadFunc(interval: tuple[a,b: int]) {.thread.} = + for i in interval.a..interval.b: + acquire(L) # lock stdout + echo i + release(L) + +initLock(L) + +for i in 0..high(thr): + createThread(thr[i], threadFunc, (i*10, i*10+5)) +joinThreads(thr) + +deinitLock(L) +``` + +When using a memory management strategy that supports shared heaps like `arc` or `boehm`, +you can pass pointer to threads and share memory between them, but the memory must outlive the thread. +The default memory management strategy, `orc`, supports this. +The example below is **not valid** for memory management strategies that use local heaps like `refc`! + +```Nim +import locks + +var l: Lock + +proc threadFunc(obj: ptr seq[int]) {.thread.} = + withLock l: + for i in 0..<100: + obj[].add(obj[].len * obj[].len) + +proc threadHandler() = + var thr: array[0..4, Thread[ptr seq[int]]] + var s = newSeq[int]() + + for i in 0..high(thr): + createThread(thr[i], threadFunc, s.addr) + joinThreads(thr) + echo s + +initLock(l) +threadHandler() +deinitLock(l) +``` +]## + + +import std/private/[threadtypes] +export Thread + +import system/ansi_c + +when defined(nimPreviewSlimSystem): + import std/assertions + +when defined(genode): + import genode/env + +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} + +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize + + +when defined(gcDestructors): + proc allocThreadStorage(size: int): pointer = + result = c_malloc(csize_t size) + zeroMem(result, size) +else: + template allocThreadStorage(size: untyped): untyped = allocShared0(size) + +#const globalsSlot = ThreadVarSlot(0) +#sysAssert checkSlot.int == globalsSlot.int + +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include <pthread.h> + """.} + + +# We jump through some hops here to ensure that Nim thread procs can have +# the Nim calling convention. This is needed because thread procs are +# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just +# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. + + + +{.push stack_trace:off.} +when defined(windows): + proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = + nimThreadProcWrapperBody(closure) + # implicitly return 0 +elif defined(genode): + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + nimThreadProcWrapperBody(closure) +else: + proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = + nimThreadProcWrapperBody(closure) +{.pop.} + +proc running*[TArg](t: Thread[TArg]): bool {.inline.} = + ## Returns true if `t` is running. + result = t.dataFn != nil + +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## Returns the thread handle of `t`. + result = t.sys + +when hostOS == "windows": + const MAXIMUM_WAIT_OBJECTS = 64 + + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard waitForSingleObject(t.sys, -1'i32) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] + var k = 0 + while k < len(t): + var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) + for i in 0..(count - 1): a[i] = t[i + k].sys + discard waitForMultipleObjects(int32(count), + cast[ptr SysThread](addr(a)), 1, -1) + inc(k, MAXIMUM_WAIT_OBJECTS) + +elif defined(genode): + proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} + ## Waits for the thread `t` to finish. + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +else: + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard pthread_join(t.sys, nil) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +when false: + # XXX a thread should really release its heap here somehow: + proc destroyThread*[TArg](t: var Thread[TArg]) = + ## Forces the thread `t` to terminate. This is potentially dangerous if + ## you don't have full control over `t` and its acquired resources. + when hostOS == "windows": + discard TerminateThread(t.sys, 1'i32) + else: + discard pthread_cancel(t.sys) + when declared(registerThread): unregisterThread(addr(t)) + t.dataFn = nil + ## if thread `t` already exited, `t.core` will be `null`. + if not isNil(t.core): + deallocThreadStorage(t.core) + t.core = nil + +when hostOS == "windows": + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. + ## `param` is passed to `tp`. `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var dummyThreadId: int32 + t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], + addr(t), 0'i32, dummyThreadId) + if t.sys <= 0: + raise newException(ResourceExhaustedError, "cannot create thread") + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + setThreadAffinityMask(t.sys, uint(1 shl cpu)) + +elif defined(genode): + var affinityOffset: cuint = 1 + ## CPU affinity offset for next thread, safe to roll-over. + + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.stackSize = ThreadStackSize + t.sys.initThread( + runtimeEnv, + ThreadStackSize.culonglong, + threadProcWrapper[TArg], addr(t), affinityOffset) + inc affinityOffset + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + {.hint: "cannot change Genode thread CPU affinity after initialization".} + discard + +else: + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. `param` is passed to `tp`. + ## `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var a {.noinit.}: Pthread_attr + doAssert pthread_attr_init(a) == 0 + when hasAllocStack: + var + rawstk = allocThreadStorage(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + + when not defined(ios): + # This fails on iOS + doAssert(setstacksizeResult == 0) + if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: + raise newException(ResourceExhaustedError, "cannot create thread") + doAssert pthread_attr_destroy(a) == 0 + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + when not defined(macosx): + var s {.noinit.}: CpuSet + cpusetZero(s) + cpusetIncl(cpu.cint, s) + setAffinity(t.sys, csize_t(sizeof(s)), s) + +proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = + createThread[void](t, tp) + +when not defined(gcOrc): + include system/threadids diff --git a/lib/std/varints.nim b/lib/std/varints.nim new file mode 100644 index 000000000..32fe2fffb --- /dev/null +++ b/lib/std/varints.nim @@ -0,0 +1,121 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## A variable length integer +## encoding implementation inspired by SQLite. +## +## Unstable API. + +const + maxVarIntLen* = 9 ## the maximal number of bytes a varint can take + +proc readVu64*(z: openArray[byte]; pResult: var uint64): int = + if z[0] <= 240: + pResult = z[0] + return 1 + if z[0] <= 248: + if z.len < 2: return 0 + pResult = (uint64 z[0] - 241) * 256 + z[1].uint64 + 240 + return 2 + if z.len < int(z[0]-246): return 0 + if z[0] == 249: + pResult = 2288u64 + 256u64*z[1].uint64 + z[2].uint64 + return 3 + if z[0] == 250: + pResult = (z[1].uint64 shl 16u64) + (z[2].uint64 shl 8u64) + z[3].uint64 + return 4 + let x = (z[1].uint64 shl 24) + (z[2].uint64 shl 16) + (z[3].uint64 shl 8) + z[4].uint64 + if z[0] == 251: + pResult = x + return 5 + if z[0] == 252: + pResult = (((uint64)x) shl 8) + z[5].uint64 + return 6 + if z[0] == 253: + pResult = (((uint64)x) shl 16) + (z[5].uint64 shl 8) + z[6].uint64 + return 7 + if z[0] == 254: + pResult = (((uint64)x) shl 24) + (z[5].uint64 shl 16) + (z[6].uint64 shl 8) + z[7].uint64 + return 8 + pResult = (((uint64)x) shl 32) + + (0xffffffff'u64 and ((z[5].uint64 shl 24) + + (z[6].uint64 shl 16) + (z[7].uint64 shl 8) + z[8].uint64)) + return 9 + +proc varintWrite32(z: var openArray[byte]; y: uint32) = + z[0] = cast[uint8](y shr 24) + z[1] = cast[uint8](y shr 16) + z[2] = cast[uint8](y shr 8) + z[3] = cast[uint8](y) + +proc writeVu64*(z: var openArray[byte], x: uint64): int = + ## Write a varint into z. The buffer z must be at least 9 characters + ## long to accommodate the largest possible varint. Returns the number of + ## bytes used. + if x <= 240: + z[0] = cast[uint8](x) + return 1 + if x <= 2287: + let y = cast[uint32](x - 240) + z[0] = cast[uint8](y shr 8 + 241) + z[1] = cast[uint8](y and 255) + return 2 + if x <= 67823: + let y = cast[uint32](x - 2288) + z[0] = 249 + z[1] = cast[uint8](y shr 8) + z[2] = cast[uint8](y and 255) + return 3 + let y = cast[uint32](x) + let w = cast[uint32](x shr 32) + if w == 0: + if y <= 16777215: + z[0] = 250 + z[1] = cast[uint8](y shr 16) + z[2] = cast[uint8](y shr 8) + z[3] = cast[uint8](y) + return 4 + z[0] = 251 + varintWrite32(toOpenArray(z, 1, 4), y) + return 5 + if w <= 255: + z[0] = 252 + z[1] = cast[uint8](w) + varintWrite32(toOpenArray(z, 2, 5), y) + return 6 + if w <= 65535: + z[0] = 253 + z[1] = cast[uint8](w shr 8) + z[2] = cast[uint8](w) + varintWrite32(toOpenArray(z, 3, 6), y) + return 7 + if w <= 16777215: + z[0] = 254 + z[1] = cast[uint8](w shr 16) + z[2] = cast[uint8](w shr 8) + z[3] = cast[uint8](w) + varintWrite32(toOpenArray(z, 4, 7), y) + return 8 + z[0] = 255 + varintWrite32(toOpenArray(z, 1, 4), w) + varintWrite32(toOpenArray(z, 5, 8), y) + return 9 + +proc sar(a, b: int64): int64 = + {.emit: [result, " = ", a, " >> ", b, ";"].} + +proc sal(a, b: int64): int64 = + {.emit: [result, " = ", a, " << ", b, ";"].} + +proc encodeZigzag*(x: int64): uint64 {.inline.} = + uint64(sal(x, 1)) xor uint64(sar(x, 63)) + +proc decodeZigzag*(x: uint64): int64 {.inline.} = + let casted = cast[int64](x) + result = (`shr`(casted, 1)) xor (-(casted and 1)) diff --git a/lib/std/vmutils.nim b/lib/std/vmutils.nim new file mode 100644 index 000000000..e16912a3c --- /dev/null +++ b/lib/std/vmutils.nim @@ -0,0 +1,11 @@ +##[ +Experimental API, subject to change. +]## + +proc vmTrace*(on: bool) {.compileTime.} = + runnableExamples: + static: vmTrace(true) + proc fn = + var a = 1 + vmTrace(false) + static: fn() diff --git a/lib/std/widestrs.nim b/lib/std/widestrs.nim new file mode 100644 index 000000000..2ddf80d14 --- /dev/null +++ b/lib/std/widestrs.nim @@ -0,0 +1,239 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim support for C/C++'s `wide strings`:idx:. + +#when not declared(ThisIsSystem): +# {.error: "You must not import this module explicitly".} + +type + Utf16Char* = distinct int16 + +when not (defined(cpu16) or defined(cpu8)): + when defined(nimv2): + + type + WideCString* = ptr UncheckedArray[Utf16Char] + + WideCStringObj* = object + bytes: int + data: WideCString + + const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`(a: WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + else: + proc `=destroy`(a: var WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + + proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.} + + proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = + a.bytes = b.bytes + a.data = b.data + + proc createWide(a: var WideCStringObj; bytes: int) = + a.bytes = bytes + when compileOption("threads"): + a.data = cast[typeof(a.data)](allocShared0(bytes)) + else: + a.data = cast[typeof(a.data)](alloc0(bytes)) + + template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] + template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val + + template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) + + converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = + result = x.data + + else: + template nullWide(): untyped = nil + + type + WideCString* = ref UncheckedArray[Utf16Char] + WideCStringObj* = WideCString + + template createWide(a; L) = + unsafeNew(a, L) + + proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) + + proc len*(w: WideCString): int = + ## returns the length of a widestring. This traverses the whole string to + ## find the binary zero end marker! + result = 0 + while int16(w[result]) != 0'i16: inc result + + const + UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) + UNI_MAX_BMP = 0x0000FFFF + UNI_MAX_UTF16 = 0x0010FFFF + # UNI_MAX_UTF32 = 0x7FFFFFFF + # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF + + halfShift = 10 + halfBase = 0x0010000 + halfMask = 0x3FF + + UNI_SUR_HIGH_START = 0xD800 + UNI_SUR_HIGH_END = 0xDBFF + UNI_SUR_LOW_START = 0xDC00 + UNI_SUR_LOW_END = 0xDFFF + UNI_REPL = 0xFFFD + + template ones(n: untyped): untyped = ((1 shl n)-1) + + template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = + ## Returns the unicode character `s[i]` in `result`. If `doInc == true` + ## `i` is incremented by the number of bytes that have been processed. + bind ones + + if ord(s[i]) <= 127: + result = ord(s[i]) + when doInc: inc(i) + elif ord(s[i]) shr 5 == 0b110: + #assert(ord(s[i+1]) shr 6 == 0b10) + if i <= L - 2: + result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) + when doInc: inc(i, 2) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 4 == 0b1110: + if i <= L - 3: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + result = (ord(s[i]) and ones(4)) shl 12 or + (ord(s[i+1]) and ones(6)) shl 6 or + (ord(s[i+2]) and ones(6)) + when doInc: inc(i, 3) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 3 == 0b11110: + if i <= L - 4: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + #assert(ord(s[i+3]) shr 6 == 0b10) + result = (ord(s[i]) and ones(3)) shl 18 or + (ord(s[i+1]) and ones(6)) shl 12 or + (ord(s[i+2]) and ones(6)) shl 6 or + (ord(s[i+3]) and ones(6)) + when doInc: inc(i, 4) + else: + result = UNI_REPL + when doInc: inc(i) + else: + result = 0xFFFD + when doInc: inc(i) + + iterator runes(s: cstring, L: int): int = + var + i = 0 + result: int + while i < L: + fastRuneAt(s, i, L, result, true) + yield result + + proc newWideCString*(size: int): WideCStringObj = + createWide(result, size * 2 + 2) + + proc newWideCString*(source: cstring, L: int): WideCStringObj = + ## Warning:: `source` needs to be preallocated with the length `L` + createWide(result, L * 2 + 2) + var d = 0 + for ch in runes(source, L): + + if ch <= UNI_MAX_BMP: + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: + result[d] = UNI_REPLACEMENT_CHAR + else: + result[d] = cast[Utf16Char](uint16(ch)) + elif ch > UNI_MAX_UTF16: + result[d] = UNI_REPLACEMENT_CHAR + else: + let ch = ch - halfBase + result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) + inc d + result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) + inc d + result[d] = Utf16Char(0) + + proc newWideCString*(s: cstring): WideCStringObj = + if s.isNil: return nullWide + + result = newWideCString(s, s.len) + + proc newWideCString*(s: string): WideCStringObj = + result = newWideCString(cstring s, s.len) + + proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = + result = newStringOfCap(estimate + estimate shr 2) + + var i = 0 + while w[i].int16 != 0'i16: + var ch = ord(w[i]) + inc i + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: + # If the 16 bits following the high surrogate are in the source buffer... + let ch2 = ord(w[i]) + + # If it's a low surrogate, convert to UTF32: + if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: + ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase + inc i + else: + #invalid UTF-16 + ch = replacement + elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: + #invalid UTF-16 + ch = replacement + + if ch < 0x80: + result.add chr(ch) + elif ch < 0x800: + result.add chr((ch shr 6) or 0xc0) + result.add chr((ch and 0x3f) or 0x80) + elif ch < 0x10000: + result.add chr((ch shr 12) or 0xe0) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + elif ch <= 0x10FFFF: + result.add chr((ch shr 18) or 0xf0) + result.add chr(((ch shr 12) and 0x3f) or 0x80) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + else: + # replacement char(in case user give very large number): + result.add chr(0xFFFD shr 12 or 0b1110_0000) + result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) + result.add chr(0xFFFD and ones(6) or 0b10_0000_00) + + proc `$`*(s: WideCString): string = + result = s $ 80 + + when defined(nimv2): + proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string = + `$`(s.data, estimate, replacement) + + proc `$`*(s: WideCStringObj): string = + $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/lib/std/with.nim b/lib/std/with.nim new file mode 100644 index 000000000..c2eaa4bef --- /dev/null +++ b/lib/std/with.nim @@ -0,0 +1,48 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the `with` macro for easy +## function chaining. See https://github.com/nim-lang/RFCs/issues/193 +## and https://github.com/nim-lang/RFCs/issues/192 for details leading to this +## particular design. +## +## **Since:** version 1.2. + +import std/[macros, private / underscored_calls] + +macro with*(arg: typed; calls: varargs[untyped]): untyped = + ## This macro provides `chaining`:idx: of function calls. + ## It does so by patching every call in `calls` to + ## use `arg` as the first argument. + ## + ## .. caution:: This evaluates `arg` multiple times! + runnableExamples: + var x = "yay" + with x: + add "abc" + add "efg" + doAssert x == "yayabcefg" + + var a = 44 + with a: + += 4 + -= 5 + doAssert a == 43 + + # Nesting works for object types too! + var foo = (bar: 1, qux: (baz: 2)) + with foo: + bar = 2 + with qux: + baz = 3 + doAssert foo.bar == 2 + doAssert foo.qux.baz == 3 + + result = newNimNode(nnkStmtList, arg) + underscoredCalls(result, calls, arg) diff --git a/lib/std/wordwrap.nim b/lib/std/wordwrap.nim new file mode 100644 index 000000000..9333f880b --- /dev/null +++ b/lib/std/wordwrap.nim @@ -0,0 +1,74 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains an algorithm to wordwrap a Unicode string. + +import std/[strutils, unicode] + +proc olen(s: string; start, lastExclusive: int): int = + var i = start + result = 0 + while i < lastExclusive: + inc result + let L = graphemeLen(s, i) + inc i, L + +proc wrapWords*(s: string, maxLineWidth = 80, + splitLongWords = true, + seps: set[char] = Whitespace, + newLine = "\n"): string {.noSideEffect.} = + ## Word wraps `s`. + runnableExamples: + doAssert "12345678901234567890".wrapWords() == "12345678901234567890" + doAssert "123456789012345678901234567890".wrapWords(20) == "12345678901234567890\n1234567890" + doAssert "Hello Bob. Hello John.".wrapWords(13, false) == "Hello Bob.\nHello John." + doAssert "Hello Bob. Hello John.".wrapWords(13, true, {';'}) == "Hello Bob. He\nllo John." + result = newStringOfCap(s.len + s.len shr 6) + var spaceLeft = maxLineWidth + var lastSep = "" + + var i = 0 + while true: + var j = i + let isSep = j < s.len and s[j] in seps + while j < s.len and (s[j] in seps) == isSep: inc(j) + if j <= i: break + #yield (substr(s, i, j-1), isSep) + if isSep: + lastSep.setLen 0 + for k in i..<j: + if s[k] notin {'\L', '\C'}: lastSep.add s[k] + if lastSep.len == 0: + lastSep.add ' ' + dec spaceLeft + else: + spaceLeft = spaceLeft - olen(lastSep, 0, lastSep.len) + else: + let wlen = olen(s, i, j) + if wlen > spaceLeft: + if splitLongWords and wlen > maxLineWidth: + var k = 0 + while k < j - i: + if spaceLeft <= 0: + spaceLeft = maxLineWidth + result.add newLine + dec spaceLeft + let L = graphemeLen(s, k+i) + for m in 0 ..< L: result.add s[i+k+m] + inc k, L + else: + spaceLeft = maxLineWidth - wlen + result.add(newLine) + for k in i..<j: result.add(s[k]) + else: + spaceLeft = spaceLeft - wlen + result.add(lastSep) + for k in i..<j: result.add(s[k]) + #lastSep.setLen(0) + i = j diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim new file mode 100644 index 000000000..0b75c270e --- /dev/null +++ b/lib/std/wrapnils.nim @@ -0,0 +1,193 @@ +## This module allows evaluating expressions safely against the following conditions: +## * nil dereferences +## * field accesses with incorrect discriminant in case objects +## +## `default(T)` is returned in those cases when evaluating an expression of type `T`. +## This simplifies code by reducing need for if-else branches. +## +## Note: experimental module, unstable API. + +#[ +TODO: +consider handling indexing operations, eg: +doAssert ?.default(seq[int])[3] == default(int) +]# + +import std/macros + +runnableExamples: + type Foo = ref object + x1: string + x2: Foo + x3: ref int + + var f: Foo + assert ?.f.x2.x1 == "" # returns default value since `f` is nil + + var f2 = Foo(x1: "a") + f2.x2 = f2 + assert ?.f2.x1 == "a" # same as f2.x1 (no nil LHS in this chain) + assert ?.Foo(x1: "a").x1 == "a" # can use constructor inside + + # when you know a sub-expression doesn't involve a `nil` (e.g. `f2.x2.x2`), + # you can scope it as follows: + assert ?.(f2.x2.x2).x3[] == 0 + + assert (?.f2.x2.x2).x3 == nil # this terminates ?. early + +runnableExamples: + # ?. also allows case object + type B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + var b = B(cond: false, b0: 3) + doAssertRaises(FieldDefect): discard b.b1 # wrong discriminant + doAssert ?.b.b1 == 0.0 # safe + b = B(cond: true, b1: 4.5) + doAssert ?.b.b1 == 4.5 + + # lvalue semantics are preserved: + if (let p = ?.b.b1.addr; p != nil): p[] = 4.7 + doAssert b.b1 == 4.7 + +proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = + if level == 0: + result = quote: `lhs` = `n` + else: + result = quote: (let `lhs` = `n`) + +proc process(n: NimNode, lhs: NimNode, label: NimNode, level: int): NimNode = + var n = n.copyNimTree + var it = n + let addr2 = bindSym"addr" + var old: tuple[n: NimNode, index: int] + while true: + if it.len == 0: + result = finalize(n, lhs, level) + break + elif it.kind == nnkCheckedFieldExpr: + let dot = it[0] + let obj = dot[0] + let objRef = quote do: `addr2`(`obj`) + # avoids a copy and preserves lvalue semantics, see tests + let check = it[1] + let okSet = check[1] + let kind1 = check[2] + let tmp = genSym(nskLet, "tmpCase") + let body = process(objRef, tmp, label, level + 1) + let tmp3 = nnkDerefExpr.newTree(tmp) + it[0][0] = tmp3 + let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) + if old.n != nil: old.n[old.index] = dot2 + else: n = dot2 + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp3`.`kind1` notin `okSet`: break `label` + `assgn` + break + elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: + let tmp = genSym(nskLet, "tmp") + let body = process(it[0], tmp, label, level + 1) + it[0] = tmp + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp` == nil: break `label` + `assgn` + break + elif it.kind == nnkCall: # consider extending to `nnkCallKinds` + # `copyNimTree` needed to avoid `typ = nil` issues + old = (it, 1) + it = it[1].copyNimTree + else: + old = (it, 0) + it = it[0] + +macro `?.`*(a: typed): auto = + ## Transforms `a` into an expression that can be safely evaluated even in + ## presence of intermediate nil pointers/references, in which case a default + ## value is produced. + let lhs = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs, label, 0) + result = quote do: + var `lhs`: type(`a`) + block `label`: + `body` + `lhs` + +# the code below is not needed for `?.` +from std/options import Option, isSome, get, option, unsafeGet, UnpackDefect + +macro `??.`*(a: typed): Option = + ## Same as `?.` but returns an `Option`. + runnableExamples: + import std/options + type Foo = ref object + x1: ref int + x2: int + # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: + var f1 = Foo(x1: int.new, x2: 2) + doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. + doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain + doAssert (??.f1.x2).get == 2 + + var f2: Foo + doAssert not (??.f2.x1[]).isSome # f2 was nil + + doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get + doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) + + let lhs = genSym(nskVar, "lhs") + let lhs2 = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs2, label, 0) + result = quote do: + var `lhs`: Option[type(`a`)] + block `label`: + var `lhs2`: type(`a`) + `body` + `lhs` = option(`lhs2`) + `lhs` + +template fakeDot*(a: Option, b): untyped = + ## See top-level example. + let a1 = a # to avoid double evaluations + type T = Option[typeof(unsafeGet(a1).b)] + if isSome(a1): + let a2 = unsafeGet(a1) + when typeof(a2) is ref|ptr: + if a2 == nil: + default(T) + else: + option(a2.b) + else: + option(a2.b) + else: + # nil is "sticky"; this is needed, see tests + default(T) + +# xxx this should but doesn't work: func `[]`*[T, I](a: Option[T], i: I): Option {.inline.} = + +func `[]`*[T, I](a: Option[T], i: I): auto {.inline.} = + ## See top-level example. + if isSome(a): + # correctly will raise IndexDefect if a is valid but wraps an empty container + result = option(a.unsafeGet[i]) + +func `[]`*[U](a: Option[U]): auto {.inline.} = + ## See top-level example. + if isSome(a): + let a2 = a.unsafeGet + if a2 != nil: + result = option(a2[]) + +when false: + # xxx: expose a way to do this directly in std/options, e.g.: `getAsIs` + proc safeGet[T](a: Option[T]): T {.inline.} = + get(a, default(T)) diff --git a/lib/stdlib.nimble b/lib/stdlib.nimble index 0805ead54..5965d6436 100644 --- a/lib/stdlib.nimble +++ b/lib/stdlib.nimble @@ -1,6 +1,5 @@ -[Package] -name = "stdlib" -version = "0.9.0" + +version = system.NimVersion author = "Dominik Picheta" description = "Nim's standard library." license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index bb8254364..2f9cdc5f9 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -7,1111 +7,759 @@ # distribution, for details about the copyright. # + ## The compiler depends on the System module to work properly and the System ## module depends on the compiler. Most of the routines listed here use ## special compiler magic. +## ## Each module implicitly imports the System module; it must not be listed ## explicitly. Because of this there cannot be a user-defined module named -## ``system``. +## `system`. ## -## Module system +## System module ## ============= ## +## .. include:: ./system_overview.rst -# That lonesome header above is to prevent :idx: entries from being mentioned -# in the global index as part of the previous header (Exception hierarchy). -type - int* {.magic: Int.} ## default integer type; bitwidth depends on - ## architecture, but is always the same as a pointer - int8* {.magic: Int8.} ## signed 8 bit integer type - int16* {.magic: Int16.} ## signed 16 bit integer type - int32* {.magic: Int32.} ## signed 32 bit integer type - int64* {.magic: Int64.} ## signed 64 bit integer type - uint* {.magic: UInt.} ## unsigned default integer type - uint8* {.magic: UInt8.} ## unsigned 8 bit integer type - uint16* {.magic: UInt16.} ## unsigned 16 bit integer type - uint32* {.magic: UInt32.} ## unsigned 32 bit integer type - uint64* {.magic: UInt64.} ## unsigned 64 bit integer type - float* {.magic: Float.} ## default floating point type - float32* {.magic: Float32.} ## 32 bit floating point type - float64* {.magic: Float.} ## 64 bit floating point type - -# 'float64' is now an alias to 'float'; this solves many problems - -type # we need to start a new type section here, so that ``0`` can have a type - bool* {.magic: Bool.} = enum ## built-in boolean type - false = 0, true = 1 +include "system/basic_types" -type - char* {.magic: Char.} ## built-in 8 bit character type (unsigned) - string* {.magic: String.} ## built-in string type - cstring* {.magic: Cstring.} ## built-in cstring (*compatible string*) type - pointer* {.magic: Pointer.} ## built-in pointer type, use the ``addr`` - ## operator to get a pointer to a variable -const - on* = true ## alias for ``true`` - off* = false ## alias for ``false`` +func zeroDefault*[T](_: typedesc[T]): T {.magic: "ZeroDefault".} = + ## Returns the binary zeros representation of the type `T`. It ignores + ## default fields of an object. + ## + ## See also: + ## * `default <#default,typedesc[T]>`_ + +include "system/compilation" {.push warning[GcMem]: off, warning[Uninit]: off.} -{.push hints: off.} +# {.push hints: off.} type - Ordinal* {.magic: Ordinal.}[T] ## Generic ordinal type. Includes integer, - ## bool, character, and enumeration types - ## as well as their subtypes. Note `uint` - ## and `uint64` are not ordinal types for - ## implementation reasons - `ptr`* {.magic: Pointer.}[T] ## built-in generic untraced pointer type - `ref`* {.magic: Pointer.}[T] ## built-in generic traced pointer type - - `nil` {.magic: "Nil".} - expr* {.magic: Expr.} ## meta type to denote an expression (for templates) - stmt* {.magic: Stmt.} ## meta type to denote a statement (for templates) - typedesc* {.magic: TypeDesc.} ## meta type to denote a type description - void* {.magic: "VoidType".} ## meta type to denote the absence of any type - auto* {.magic: Expr.} ## meta type for automatic type determination - any* = distinct auto ## meta type for any supported type - untyped* {.magic: Expr.} ## meta type to denote an expression that - ## is not resolved (for templates) - typed* {.magic: Stmt.} ## meta type to denote an expression that - ## is resolved (for templates) - - SomeSignedInt* = int|int8|int16|int32|int64 - ## type class matching all signed integer types - - SomeUnsignedInt* = uint|uint8|uint16|uint32|uint64 - ## type class matching all unsigned integer types - - SomeInteger* = SomeSignedInt|SomeUnsignedInt - ## type class matching all integer types - - SomeOrdinal* = int|int8|int16|int32|int64|bool|enum|uint8|uint16|uint32 - ## type class matching all ordinal types; however this includes enums with - ## holes. - - SomeReal* = float|float32|float64 - ## type class matching all floating point number types - - SomeNumber* = SomeInteger|SomeReal - ## type class matching all number types - -proc defined*(x: expr): bool {.magic: "Defined", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## defined. - ## `x` is an external symbol introduced through the compiler's - ## `-d:x switch <nimc.html#compile-time-symbols>`_ to enable build time - ## conditionals: - ## - ## .. code-block:: Nim - ## when not defined(release): - ## # Do here programmer friendly expensive sanity checks. - ## # Put here the normal code - -when defined(nimalias): - {.deprecated: [ - TSignedInt: SomeSignedInt, - TUnsignedInt: SomeUnsignedInt, - TInteger: SomeInteger, - TReal: SomeReal, - TNumber: SomeNumber, - TOrdinal: SomeOrdinal].} - -proc declared*(x: expr): bool {.magic: "Defined", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared. `x` has to be an identifier or a qualified identifier. - ## This can be used to check whether a library provides a certain - ## feature or not: - ## - ## .. code-block:: Nim - ## when not declared(strutils.toUpper): - ## # provide our own toUpper proc here, because strutils is - ## # missing it. - -when defined(useNimRtl): - {.deadCodeElim: on.} - -proc definedInScope*(x: expr): bool {. - magic: "DefinedInScope", noSideEffect, deprecated, compileTime.} - ## **Deprecated since version 0.9.6**: Use ``declaredInScope`` instead. - -proc declaredInScope*(x: expr): bool {. - magic: "DefinedInScope", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared in the current scope. `x` has to be an identifier. - -proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = - ## Builtin 'addr' operator for taking the address of a memory location. - ## Cannot be overloaded. - ## - ## .. code-block:: nim - ## var - ## buf: seq[char] = @['a','b','c'] - ## p: pointer = buf[1].addr - ## echo cast[ptr char](p)[] # b - discard + `static`*[T] {.magic: "Static".} + ## Meta type representing all values that can be evaluated at compile-time. + ## + ## The type coercion `static(x)` can be used to force the compile-time + ## evaluation of the given expression `x`. -proc unsafeAddr*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = - ## Builtin 'addr' operator for taking the address of a memory location. - ## This works even for ``let`` variables or parameters for better interop - ## with C and so it is considered even more unsafe than the ordinary ``addr``. - ## Cannot be overloaded. - discard + `type`*[T] {.magic: "Type".} + ## Meta type representing the type of all type values. + ## + ## The coercion `type(x)` can be used to obtain the type of the given + ## expression `x`. + +type + TypeOfMode* = enum ## Possible modes of `typeof`. + typeOfProc, ## Prefer the interpretation that means `x` is a proc call. + typeOfIter ## Prefer the interpretation that means `x` is an iterator call. + +proc typeof*(x: untyped; mode = typeOfIter): typedesc {. + magic: "TypeOf", noSideEffect, compileTime.} = + ## Builtin `typeof` operation for accessing the type of an expression. + ## Since version 0.20.0. + runnableExamples: + proc myFoo(): float = 0.0 + iterator myFoo(): string = yield "abc" + iterator myFoo2(): string = yield "abc" + iterator myFoo3(): string {.closure.} = yield "abc" + doAssert type(myFoo()) is string + doAssert typeof(myFoo()) is string + doAssert typeof(myFoo(), typeOfIter) is string + doAssert typeof(myFoo3) is iterator + + doAssert typeof(myFoo(), typeOfProc) is float + doAssert typeof(0.0, typeOfProc) is float + doAssert typeof(myFoo3, typeOfProc) is iterator + doAssert not compiles(typeof(myFoo2(), typeOfProc)) + # this would give: Error: attempting to call routine: 'myFoo2' + # since `typeOfProc` expects a typed expression and `myFoo2()` can + # only be used in a `for` context. + +proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} + ## Constructs an `or` meta class. + +proc `and`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} + ## Constructs an `and` meta class. + +proc `not`*(a: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} + ## Constructs an `not` meta class. + +when defined(nimHasIterable): + type + iterable*[T] {.magic: IterableType.} ## Represents an expression that yields `T` + +type + Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, + ## bool, character, and enumeration types + ## as well as their subtypes. See also + ## `SomeOrdinal`. -proc `type`*(x: expr): typeDesc {.magic: "TypeOf", noSideEffect, compileTime.} = - ## Builtin 'type' operator for accessing the type of an expression. + +proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = + ## Builtin `addr` operator for taking the address of a memory location. + ## + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. + ## ## Cannot be overloaded. + ## + ## ```nim + ## var + ## buf: seq[char] = @['a','b','c'] + ## p = buf[1].addr + ## echo p.repr # ref 0x7faa35c40059 --> 'b' + ## echo p[] # b + ## ``` discard -proc `not` *(x: bool): bool {.magic: "Not", noSideEffect.} - ## Boolean not; returns true iff ``x == false``. - -proc `and`*(x, y: bool): bool {.magic: "And", noSideEffect.} - ## Boolean ``and``; returns true iff ``x == y == true``. - ## Evaluation is lazy: if ``x`` is false, - ## ``y`` will not even be evaluated. -proc `or`*(x, y: bool): bool {.magic: "Or", noSideEffect.} - ## Boolean ``or``; returns true iff ``not (not x and not y)``. - ## Evaluation is lazy: if ``x`` is true, - ## ``y`` will not even be evaluated. -proc `xor`*(x, y: bool): bool {.magic: "Xor", noSideEffect.} - ## Boolean `exclusive or`; returns true iff ``x != y``. - -proc new*[T](a: var ref T) {.magic: "New", noSideEffect.} - ## creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. - -proc new*(T: typedesc): auto = - ## creates a new object of type ``T`` and returns a safe (traced) - ## reference to it as result value. - ## - ## When ``T`` is a ref type then the resulting type will be ``T``, - ## otherwise it will be ``ref T``. - when (T is ref): - var r: T - else: - var r: ref T - new(r) - return r +proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = + ## .. warning:: `unsafeAddr` is a deprecated alias for `addr`, + ## use `addr` instead. + discard -proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.} - ## leaked implementation detail. Do not use. +const ThisIsSystem = true proc new*[T](a: var ref T, finalizer: proc (x: ref T) {.nimcall.}) {. magic: "NewFinalize", noSideEffect.} - ## creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. When the garbage collector frees the object, - ## `finalizer` is called. The `finalizer` may not keep a reference to the + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. + ## + ## When the garbage collector frees the object, `finalizer` is called. + ## The `finalizer` may not keep a reference to the ## object pointed to by `x`. The `finalizer` cannot prevent the GC from - ## freeing the object. Note: The `finalizer` refers to the type `T`, not to - ## the object! This means that for each object of type `T` the finalizer - ## will be called! - -proc reset*[T](obj: var T) {.magic: "Reset", noSideEffect.} - ## resets an object `obj` to its initial (binary zero) value. This needs to - ## be called before any possible `object branch transition`:idx:. - -# for low and high the return type T may not be correct, but -# we handle that with compiler magic in semLowHigh() -proc high*[T](x: T): T {.magic: "High", noSideEffect.} - ## returns the highest possible index of an array, a sequence, a string or - ## the highest possible value of an ordinal value `x`. As a special - ## semantic rule, `x` may also be a type identifier. - ## ``high(int)`` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:. + ## freeing the object. + ## + ## **Note**: The `finalizer` refers to the type `T`, not to the object! + ## This means that for each object of type `T` the finalizer will be called! + +proc `=wasMoved`*[T](obj: var T) {.magic: "WasMoved", noSideEffect.} = + ## Generic `wasMoved`:idx: implementation that can be overridden. + +proc wasMoved*[T](obj: var T) {.inline, noSideEffect.} = + ## Resets an object `obj` to its initial (binary zero) value to signify + ## it was "moved" and to signify its destructor should do nothing and + ## ideally be optimized away. + {.cast(raises: []), cast(tags: []).}: + `=wasMoved`(obj) + +proc move*[T](x: var T): T {.magic: "Move", noSideEffect.} = + result = x + {.cast(raises: []), cast(tags: []).}: + `=wasMoved`(x) + +when defined(nimHasEnsureMove): + proc ensureMove*[T](x: T): T {.magic: "EnsureMove", noSideEffect.} = + ## Ensures that `x` is moved to the new location, otherwise it gives + ## an error at the compile time. + runnableExamples: + proc foo = + var x = "Hello" + let y = ensureMove(x) + doAssert y == "Hello" + foo() + discard "implemented in injectdestructors" + +type + range*[T]{.magic: "Range".} ## Generic type to construct range types. + array*[I, T]{.magic: "Array".} ## Generic type to construct + ## fixed-length arrays. + openArray*[T]{.magic: "OpenArray".} ## Generic type to construct open arrays. + ## Open arrays are implemented as a + ## pointer to the array data and a + ## length field. + varargs*[T]{.magic: "Varargs".} ## Generic type to construct a varargs type. + seq*[T]{.magic: "Seq".} ## Generic type to construct sequences. + set*[T]{.magic: "Set".} ## Generic type to construct bit sets. + +type + UncheckedArray*[T]{.magic: "UncheckedArray".} + ## Array with no bounds checking. + +type sink*[T]{.magic: "BuiltinType".} +type lent*[T]{.magic: "BuiltinType".} + +proc high*[T: Ordinal|enum|range](x: T): T {.magic: "High", noSideEffect, + deprecated: "Deprecated since v1.4; there should not be `high(value)`. Use `high(type)`.".} + ## Returns the highest possible value of an ordinal value `x`. + ## + ## As a special semantic rule, `x` may also be a type identifier. + ## + ## **This proc is deprecated**, use this one instead: + ## * `high(typedesc) <#high,typedesc[T]>`_ + ## + ## ```nim + ## high(2) # => 9223372036854775807 + ## ``` + +proc high*[T: Ordinal|enum|range](x: typedesc[T]): T {.magic: "High", noSideEffect.} + ## Returns the highest possible value of an ordinal or enum type. + ## + ## `high(int)` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:. + ## ```nim + ## high(int) # => 9223372036854775807 + ## ``` ## - ## .. code-block:: nim - ## var arr = [1,2,3,4,5,6,7] - ## high(arr) #=> 6 - ## high(2) #=> 9223372036854775807 + ## See also: + ## * `low(typedesc) <#low,typedesc[T]>`_ -proc low*[T](x: T): T {.magic: "Low", noSideEffect.} - ## returns the lowest possible index of an array, a sequence, a string or - ## the lowest possible value of an ordinal value `x`. As a special +proc high*[T](x: openArray[T]): int {.magic: "High", noSideEffect.} + ## Returns the highest possible index of a sequence `x`. + ## ```nim + ## var s = @[1, 2, 3, 4, 5, 6, 7] + ## high(s) # => 6 + ## for i in low(s)..high(s): + ## echo s[i] + ## ``` + ## + ## See also: + ## * `low(openArray) <#low,openArray[T]>`_ + +proc high*[I, T](x: array[I, T]): I {.magic: "High", noSideEffect.} + ## Returns the highest possible index of an array `x`. + ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## var arr = [1, 2, 3, 4, 5, 6, 7] + ## high(arr) # => 6 + ## for i in low(arr)..high(arr): + ## echo arr[i] + ## ``` + ## + ## See also: + ## * `low(array) <#low,array[I,T]>`_ + +proc high*[I, T](x: typedesc[array[I, T]]): I {.magic: "High", noSideEffect.} + ## Returns the highest possible index of an array type. + ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## high(array[7, int]) # => 6 + ## ``` + ## + ## See also: + ## * `low(typedesc[array]) <#low,typedesc[array[I,T]]>`_ + +proc high*(x: cstring): int {.magic: "High", noSideEffect.} + ## Returns the highest possible index of a compatible string `x`. + ## This is sometimes an O(n) operation. + ## + ## See also: + ## * `low(cstring) <#low,cstring>`_ + +proc high*(x: string): int {.magic: "High", noSideEffect.} + ## Returns the highest possible index of a string `x`. + ## ```nim + ## var str = "Hello world!" + ## high(str) # => 11 + ## ``` + ## + ## See also: + ## * `low(string) <#low,string>`_ + +proc low*[T: Ordinal|enum|range](x: T): T {.magic: "Low", noSideEffect, + deprecated: "Deprecated since v1.4; there should not be `low(value)`. Use `low(type)`.".} + ## Returns the lowest possible value of an ordinal value `x`. As a special ## semantic rule, `x` may also be a type identifier. ## - ## .. code-block:: nim - ## var arr = [1,2,3,4,5,6,7] - ## low(arr) #=> 0 - ## low(2) #=> -9223372036854775808 + ## **This proc is deprecated**, use this one instead: + ## * `low(typedesc) <#low,typedesc[T]>`_ + ## + ## ```nim + ## low(2) # => -9223372036854775808 + ## ``` -type - range*{.magic: "Range".}[T] ## Generic type to construct range types. - array*{.magic: "Array".}[I, T] ## Generic type to construct - ## fixed-length arrays. - openArray*{.magic: "OpenArray".}[T] ## Generic type to construct open arrays. - ## Open arrays are implemented as a - ## pointer to the array data and a - ## length field. - varargs*{.magic: "Varargs".}[T] ## Generic type to construct a varargs type. - seq*{.magic: "Seq".}[T] ## Generic type to construct sequences. - set*{.magic: "Set".}[T] ## Generic type to construct bit sets. - -when defined(nimArrIdx): - # :array|openarray|string|seq|cstring|tuple - proc `[]`*[I: Ordinal;T](a: T; i: I): T {. - noSideEffect, magic: "ArrGet".} - proc `[]=`*[I: Ordinal;T,S](a: T; i: I; - x: S) {.noSideEffect, magic: "ArrPut".} - proc `=`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} +proc low*[T: Ordinal|enum|range](x: typedesc[T]): T {.magic: "Low", noSideEffect.} + ## Returns the lowest possible value of an ordinal or enum type. + ## + ## `low(int)` is Nim's way of writing `INT_MIN`:idx: or `MIN_INT`:idx:. + ## ```nim + ## low(int) # => -9223372036854775808 + ## ``` + ## + ## See also: + ## * `high(typedesc) <#high,typedesc[T]>`_ + +proc low*[T](x: openArray[T]): int {.magic: "Low", noSideEffect.} + ## Returns the lowest possible index of a sequence `x`. + ## ```nim + ## var s = @[1, 2, 3, 4, 5, 6, 7] + ## low(s) # => 0 + ## for i in low(s)..high(s): + ## echo s[i] + ## ``` + ## + ## See also: + ## * `high(openArray) <#high,openArray[T]>`_ + +proc low*[I, T](x: array[I, T]): I {.magic: "Low", noSideEffect.} + ## Returns the lowest possible index of an array `x`. + ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## var arr = [1, 2, 3, 4, 5, 6, 7] + ## low(arr) # => 0 + ## for i in low(arr)..high(arr): + ## echo arr[i] + ## ``` + ## + ## See also: + ## * `high(array) <#high,array[I,T]>`_ + +proc low*[I, T](x: typedesc[array[I, T]]): I {.magic: "Low", noSideEffect.} + ## Returns the lowest possible index of an array type. + ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## low(array[7, int]) # => 0 + ## ``` + ## + ## See also: + ## * `high(typedesc[array]) <#high,typedesc[array[I,T]]>`_ + +proc low*(x: cstring): int {.magic: "Low", noSideEffect.} + ## Returns the lowest possible index of a compatible string `x`. + ## + ## See also: + ## * `high(cstring) <#high,cstring>`_ + +proc low*(x: string): int {.magic: "Low", noSideEffect.} + ## Returns the lowest possible index of a string `x`. + ## ```nim + ## var str = "Hello world!" + ## low(str) # => 0 + ## ``` + ## + ## See also: + ## * `high(string) <#high,string>`_ + +when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} + ## Use this instead of `=` for a `shallow copy`:idx:. + ## + ## The shallow copy only changes the semantics for sequences and strings + ## (and types which contain those). + ## + ## Be careful with the changed semantics though! + ## There is a reason why the default assignment does a deep copy of sequences + ## and strings. + +# :array|openArray|string|seq|cstring|tuple +proc `[]`*[I: Ordinal;T](a: T; i: I): T {. + noSideEffect, magic: "ArrGet".} +proc `[]=`*[I: Ordinal;T,S](a: T; i: I; + x: sink S) {.noSideEffect, magic: "ArrPut".} +proc `=`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} +proc `=copy`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} + +proc arrGet[I: Ordinal;T](a: T; i: I): T {. + noSideEffect, magic: "ArrGet".} +proc arrPut[I: Ordinal;T,S](a: T; i: I; + x: S) {.noSideEffect, magic: "ArrPut".} + +const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + + +when defined(nimAllowNonVarDestructor) and arcLikeMem and defined(nimPreviewNonVarDestructor): + proc `=destroy`*[T](x: T) {.inline, magic: "Destroy".} = + ## Generic `destructor`:idx: implementation that can be overridden. + discard +else: + proc `=destroy`*[T](x: var T) {.inline, magic: "Destroy".} = + ## Generic `destructor`:idx: implementation that can be overridden. + discard + + when defined(nimAllowNonVarDestructor) and arcLikeMem: + proc `=destroy`*(x: string) {.inline, magic: "Destroy", enforceNoRaises.} = + discard + + proc `=destroy`*[T](x: seq[T]) {.inline, magic: "Destroy".} = + discard + + proc `=destroy`*[T](x: ref T) {.inline, magic: "Destroy".} = + discard + +when defined(nimHasDup): + proc `=dup`*[T](x: T): T {.inline, magic: "Dup".} = + ## Generic `dup`:idx: implementation that can be overridden. + discard + +proc `=sink`*[T](x: var T; y: T) {.inline, nodestroy, magic: "Asgn".} = + ## Generic `sink`:idx: implementation that can be overridden. + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + x = y + else: + shallowCopy(x, y) + +when defined(nimHasTrace): + proc `=trace`*[T](x: var T; env: pointer) {.inline, magic: "Trace".} = + ## Generic `trace`:idx: implementation that can be overridden. + discard type - Slice*[T] = object ## builtin slice type - a*, b*: T ## the bounds - -when defined(nimalias): - {.deprecated: [TSlice: Slice].} - -proc `..`*[T](a, b: T): Slice[T] {.noSideEffect, inline, magic: "DotDot".} = - ## `slice`:idx: operator that constructs an interval ``[a, b]``, both `a` - ## and `b` are inclusive. Slices can also be used in the set constructor - ## and in ordinal case statements, but then they are special-cased by the - ## compiler. - result.a = a - result.b = b - -proc `..`*[T](b: T): Slice[T] {.noSideEffect, inline, magic: "DotDot".} = - ## `slice`:idx: operator that constructs an interval ``[default(T), b]`` - result.b = b - -when not defined(niminheritable): - {.pragma: inheritable.} -when not defined(nimunion): - {.pragma: unchecked.} - -when defined(nimNewShared): - type - `shared`* {.magic: "Shared".} - guarded* {.magic: "Guarded".} - -# comparison operators: -proc `==` *[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} - ## Checks whether values within the *same enum* have the same underlying value - ## - ## .. code-block:: nim - ## type - ## Enum1 = enum - ## Field1 = 3, Field2 - ## Enum2 = enum - ## Place1, Place2 = 3 - ## var - ## e1 = Field1 - ## e2 = Enum1(Place2) - ## echo (e1 == e2) # true - ## echo (e1 == Place2) # raises error -proc `==` *(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} - ## .. code-block:: nim - ## var # this is a wildly dangerous example - ## a = cast[pointer](0) - ## b = cast[pointer](nil) - ## echo (a == b) # true due to the special meaning of `nil`/0 as a pointer -proc `==` *(x, y: string): bool {.magic: "EqStr", noSideEffect.} - ## Checks for equality between two `string` variables -proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect.} - ## Checks for equality between two `cstring` variables -proc `==` *(x, y: char): bool {.magic: "EqCh", noSideEffect.} - ## Checks for equality between two `char` variables -proc `==` *(x, y: bool): bool {.magic: "EqB", noSideEffect.} - ## Checks for equality between two `bool` variables -proc `==` *[T](x, y: set[T]): bool {.magic: "EqSet", noSideEffect.} - ## Checks for equality between two variables of type `set` - ## - ## .. code-block:: nim - ## var a = {1, 2, 2, 3} # duplication in sets is ignored - ## var b = {1, 2, 3} - ## echo (a == b) # true -proc `==` *[T](x, y: ref T): bool {.magic: "EqRef", noSideEffect.} - ## Checks that two `ref` variables refer to the same item -proc `==` *[T](x, y: ptr T): bool {.magic: "EqRef", noSideEffect.} - ## Checks that two `ptr` variables refer to the same item -proc `==` *[T: proc](x, y: T): bool {.magic: "EqProc", noSideEffect.} - ## Checks that two `proc` variables refer to the same procedure - -proc `<=` *[Enum: enum](x, y: Enum): bool {.magic: "LeEnum", noSideEffect.} -proc `<=` *(x, y: string): bool {.magic: "LeStr", noSideEffect.} -proc `<=` *(x, y: char): bool {.magic: "LeCh", noSideEffect.} -proc `<=` *[T](x, y: set[T]): bool {.magic: "LeSet", noSideEffect.} -proc `<=` *(x, y: bool): bool {.magic: "LeB", noSideEffect.} -proc `<=` *[T](x, y: ref T): bool {.magic: "LePtr", noSideEffect.} -proc `<=` *(x, y: pointer): bool {.magic: "LePtr", noSideEffect.} - -proc `<` *[Enum: enum](x, y: Enum): bool {.magic: "LtEnum", noSideEffect.} -proc `<` *(x, y: string): bool {.magic: "LtStr", noSideEffect.} -proc `<` *(x, y: char): bool {.magic: "LtCh", noSideEffect.} -proc `<` *[T](x, y: set[T]): bool {.magic: "LtSet", noSideEffect.} -proc `<` *(x, y: bool): bool {.magic: "LtB", noSideEffect.} -proc `<` *[T](x, y: ref T): bool {.magic: "LtPtr", noSideEffect.} -proc `<` *[T](x, y: ptr T): bool {.magic: "LtPtr", noSideEffect.} -proc `<` *(x, y: pointer): bool {.magic: "LtPtr", noSideEffect.} - -template `!=` * (x, y: expr): expr {.immediate.} = - ## unequals operator. This is a shorthand for ``not (x == y)``. - not (x == y) - -template `>=` * (x, y: expr): expr {.immediate.} = - ## "is greater or equals" operator. This is the same as ``y <= x``. - y <= x - -template `>` * (x, y: expr): expr {.immediate.} = - ## "is greater" operator. This is the same as ``y < x``. - y < x + HSlice*[T, U] = object ## "Heterogeneous" slice type. + a*: T ## The lower bound (inclusive). + b*: U ## The upper bound (inclusive). + Slice*[T] = HSlice[T, T] ## An alias for `HSlice[T, T]`. + +proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, magic: "DotDot".} = + ## Binary `slice`:idx: operator that constructs an interval `[a, b]`, both `a` + ## and `b` are inclusive. + ## + ## Slices can also be used in the set constructor and in ordinal case + ## statements, but then they are special-cased by the compiler. + ## ```nim + ## let a = [10, 20, 30, 40, 50] + ## echo a[2 .. 3] # @[30, 40] + ## ``` + result = HSlice[T, U](a: a, b: b) + +proc `..`*[T](b: sink T): HSlice[int, T] + {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} = + ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`. + ## ```nim + ## let a = [10, 20, 30, 40, 50] + ## echo a[.. 2] # @[10, 20, 30] + ## ``` + result = HSlice[int, T](a: 0, b: b) + +when defined(hotCodeReloading): + {.pragma: hcrInline, inline.} +else: + {.pragma: hcrInline.} + +include "system/arithmetics" +include "system/comparisons" const - appType* {.magic: "AppType"}: string = "" - ## a string that describes the application type. Possible values: - ## "console", "gui", "lib". + appType* {.magic: "AppType".}: string = "" + ## A string that describes the application type. Possible values: + ## `"console"`, `"gui"`, `"lib"`. include "system/inclrtl" -const NoFakeVars* = defined(nimscript) ## true if the backend doesn't support \ - ## "fake variables" like 'var EBADF {.importc.}: cint'. +const NoFakeVars = defined(nimscript) ## `true` if the backend doesn't support \ + ## "fake variables" like `var EBADF {.importc.}: cint`. -const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000 +const notJSnotNims = not defined(js) and not defined(nimscript) -when not defined(JS): +when not defined(js) and not defined(nimSeqsV2): type TGenericSeq {.compilerproc, pure, inheritable.} = object len, reserved: int when defined(gogc): elemSize: int + elemAlign: int PGenericSeq {.exportc.} = ptr TGenericSeq - UncheckedCharArray {.unchecked.} = array[0..ArrayDummySize, char] # len and space without counting the terminating zero: NimStringDesc {.compilerproc, final.} = object of TGenericSeq - data: UncheckedCharArray + data: UncheckedArray[char] NimString = ptr NimStringDesc -when not defined(JS) and not defined(nimscript): - template space(s: PGenericSeq): int {.dirty.} = - s.reserved and not seqShallowFlag - +when notJSnotNims: include "system/hti" type - byte* = uint8 ## this is an alias for ``uint8``, that is an unsigned - ## int 8 bits wide. + byte* = uint8 ## This is an alias for `uint8`, that is an unsigned + ## integer, 8 bits wide. Natural* = range[0..high(int)] - ## is an int type ranging from zero to the maximum value - ## of an int. This type is often useful for documentation and debugging. + ## is an `int` type ranging from zero to the maximum value + ## of an `int`. This type is often useful for documentation and debugging. Positive* = range[1..high(int)] - ## is an int type ranging from one to the maximum value - ## of an int. This type is often useful for documentation and debugging. + ## is an `int` type ranging from one to the maximum value + ## of an `int`. This type is often useful for documentation and debugging. + +type + RootObj* {.compilerproc, inheritable.} = + object ## The root of Nim's object hierarchy. + ## + ## Objects should inherit from `RootObj` or one of its descendants. + ## However, objects that have no ancestor are also allowed. + RootRef* = ref RootObj ## Reference to `RootObj`. - RootObj* {.exportc: "TNimObject", inheritable.} = - object ## the root of Nim's object hierarchy. Objects should - ## inherit from RootObj or one of its descendants. However, - ## objects that have no ancestor are allowed. - RootRef* = ref RootObj ## reference to RootObj +const NimStackTraceMsgs = compileOption("stacktraceMsgs") +type RootEffect* {.compilerproc.} = object of RootObj ## \ - ## base effect class; each effect should - ## inherit from `TEffect` unless you know what - ## you doing. - TimeEffect* = object of RootEffect ## Time effect. - IOEffect* = object of RootEffect ## IO effect. - ReadIOEffect* = object of IOEffect ## Effect describing a read IO operation. - WriteIOEffect* = object of IOEffect ## Effect describing a write IO operation. - ExecIOEffect* = object of IOEffect ## Effect describing an executing IO operation. - - Exception* {.compilerproc.} = object of RootObj ## \ + ## Base effect class. + ## + ## Each effect should inherit from `RootEffect` unless you know what + ## you're doing. + +type + StackTraceEntry* = object ## In debug mode exceptions store the stack trace that led + ## to them. A `StackTraceEntry` is a single entry of the + ## stack trace. + procname*: cstring ## Name of the proc that is currently executing. + line*: int ## Line number of the proc that is currently executing. + filename*: cstring ## Filename of the proc that is currently executing. + when NimStackTraceMsgs: + frameMsg*: string ## When a stacktrace is generated in a given frame and + ## rendered at a later time, we should ensure the stacktrace + ## data isn't invalidated; any pointer into PFrame is + ## subject to being invalidated so shouldn't be stored. + when defined(nimStackTraceOverride): + programCounter*: uint ## Program counter - will be used to get the rest of the info, + ## when `$` is called on this type. We can't use + ## "cuintptr_t" in here. + procnameStr*, filenameStr*: string ## GC-ed alternatives to "procname" and "filename" + + Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ ## Base exception class. ## ## Each exception has to inherit from `Exception`. See the full `exception - ## hierarchy`_. - parent*: ref Exception ## parent exception (can be used as a stack) - name*: cstring ## The exception's name is its Nim identifier. - ## This field is filled automatically in the - ## ``raise`` statement. - msg* {.exportc: "message".}: string ## the exception's message. Not + ## hierarchy <manual.html#exception-handling-exception-hierarchy>`_. + parent*: ref Exception ## Parent exception (can be used as a stack). + name*: cstring ## The exception's name is its Nim identifier. + ## This field is filled automatically in the + ## `raise` statement. + msg* {.exportc: "message".}: string ## The exception's message. Not ## providing an exception message ## is bad style. - trace: string + when defined(js): + trace*: string + else: + trace*: seq[StackTraceEntry] + up: ref Exception # used for stacking exceptions. Not exported! - SystemError* = object of Exception ## \ - ## Abstract class for exceptions that the runtime system raises. - ## - ## See the full `exception hierarchy`_. - IOError* = object of SystemError ## \ - ## Raised if an IO error occurred. - ## - ## See the full `exception hierarchy`_. - OSError* = object of SystemError ## \ - ## Raised if an operating system service failed. - ## - ## See the full `exception hierarchy`_. - errorCode*: int32 ## OS-defined error code describing this error. - LibraryError* = object of OSError ## \ - ## Raised if a dynamic library could not be loaded. - ## - ## See the full `exception hierarchy`_. - ResourceExhaustedError* = object of SystemError ## \ - ## Raised if a resource request could not be fulfilled. - ## - ## See the full `exception hierarchy`_. - ArithmeticError* = object of Exception ## \ - ## Raised if any kind of arithmetic error occurred. - ## - ## See the full `exception hierarchy`_. - DivByZeroError* = object of ArithmeticError ## \ - ## Raised for runtime integer divide-by-zero errors. - ## - ## See the full `exception hierarchy`_. + Defect* = object of Exception ## \ + ## Abstract base class for all exceptions that Nim's runtime raises + ## but that are strictly uncatchable as they can also be mapped to + ## a `quit` / `trap` / `exit` operation. - OverflowError* = object of ArithmeticError ## \ - ## Raised for runtime integer overflows. - ## - ## This happens for calculations whose results are too large to fit in the - ## provided bits. See the full `exception hierarchy`_. - AccessViolationError* = object of Exception ## \ - ## Raised for invalid memory access errors - ## - ## See the full `exception hierarchy`_. - AssertionError* = object of Exception ## \ - ## Raised when assertion is proved wrong. - ## - ## Usually the result of using the `assert() template <#assert>`_. See the - ## full `exception hierarchy`_. - ValueError* = object of Exception ## \ - ## Raised for string and object conversion errors. - KeyError* = object of ValueError ## \ - ## Raised if a key cannot be found in a table. - ## - ## Mostly used by the `tables <tables.html>`_ module, it can also be raised - ## by other collection modules like `sets <sets.html>`_ or `strtabs - ## <strtabs.html>`_. See the full `exception hierarchy`_. - OutOfMemError* = object of SystemError ## \ - ## Raised for unsuccessful attempts to allocate memory. - ## - ## See the full `exception hierarchy`_. - IndexError* = object of Exception ## \ - ## Raised if an array index is out of bounds. - ## - ## See the full `exception hierarchy`_. + CatchableError* = object of Exception ## \ + ## Abstract class for all exceptions that are catchable. - FieldError* = object of Exception ## \ - ## Raised if a record field is not accessible because its dicriminant's - ## value does not fit. - ## - ## See the full `exception hierarchy`_. - RangeError* = object of Exception ## \ - ## Raised if a range check error occurred. - ## - ## See the full `exception hierarchy`_. - StackOverflowError* = object of SystemError ## \ - ## Raised if the hardware stack used for subroutine calls overflowed. - ## - ## See the full `exception hierarchy`_. - ReraiseError* = object of Exception ## \ - ## Raised if there is no exception to reraise. - ## - ## See the full `exception hierarchy`_. - ObjectAssignmentError* = object of Exception ## \ - ## Raised if an object gets assigned to its parent's object. - ## - ## See the full `exception hierarchy`_. - ObjectConversionError* = object of Exception ## \ - ## Raised if an object is converted to an incompatible object type. - ## You can use ``of`` operator to check if conversion will succeed. - ## - ## See the full `exception hierarchy`_. - FloatingPointError* = object of Exception ## \ - ## Base class for floating point exceptions. - ## - ## See the full `exception hierarchy`_. - FloatInvalidOpError* = object of FloatingPointError ## \ - ## Raised by invalid operations according to IEEE. - ## - ## Raised by ``0.0/0.0``, for example. See the full `exception - ## hierarchy`_. - FloatDivByZeroError* = object of FloatingPointError ## \ - ## Raised by division by zero. - ## - ## Divisor is zero and dividend is a finite nonzero number. See the full - ## `exception hierarchy`_. - FloatOverflowError* = object of FloatingPointError ## \ - ## Raised for overflows. - ## - ## The operation produced a result that exceeds the range of the exponent. - ## See the full `exception hierarchy`_. - FloatUnderflowError* = object of FloatingPointError ## \ - ## Raised for underflows. - ## - ## The operation produced a result that is too small to be represented as a - ## normal number. See the full `exception hierarchy`_. - FloatInexactError* = object of FloatingPointError ## \ - ## Raised for inexact results. - ## - ## The operation produced a result that cannot be represented with infinite - ## precision -- for example: ``2.0 / 3.0, log(1.1)`` - ## - ## **NOTE**: Nim currently does not detect these! See the full - ## `exception hierarchy`_. - DeadThreadError* = object of Exception ## \ - ## Raised if it is attempted to send a message to a dead thread. - ## - ## See the full `exception hierarchy`_. - - TResult* {.deprecated.} = enum Failure, Success - -{.deprecated: [TObject: RootObj, PObject: RootRef, TEffect: RootEffect, - FTime: TimeEffect, FIO: IOEffect, FReadIO: ReadIOEffect, - FWriteIO: WriteIOEffect, FExecIO: ExecIOEffect, - - E_Base: Exception, ESystem: SystemError, EIO: IOError, - EOS: OSError, EInvalidLibrary: LibraryError, - EResourceExhausted: ResourceExhaustedError, - EArithmetic: ArithmeticError, EDivByZero: DivByZeroError, - EOverflow: OverflowError, EAccessViolation: AccessViolationError, - EAssertionFailed: AssertionError, EInvalidValue: ValueError, - EInvalidKey: KeyError, EOutOfMemory: OutOfMemError, - EInvalidIndex: IndexError, EInvalidField: FieldError, - EOutOfRange: RangeError, EStackOverflow: StackOverflowError, - ENoExceptionToReraise: ReraiseError, - EInvalidObjectAssignment: ObjectAssignmentError, - EInvalidObjectConversion: ObjectConversionError, - EDeadThread: DeadThreadError, - EFloatInexact: FloatInexactError, - EFloatUnderflow: FloatUnderflowError, - EFloatingPoint: FloatingPointError, - EFloatInvalidOp: FloatInvalidOpError, - EFloatDivByZero: FloatDivByZeroError, - EFloatOverflow: FloatOverflowError, - ESynch: Exception -].} +when defined(nimIcIntegrityChecks): + include "system/exceptions" +else: + import system/exceptions + export exceptions + +when defined(js) or defined(nimdoc): + type + JsRoot* = ref object of RootObj + ## Root type of the JavaScript object hierarchy proc unsafeNew*[T](a: var ref T, size: Natural) {.magic: "New", noSideEffect.} - ## creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. This is **unsafe** as it allocates an object - ## of the passed ``size``. This should only be used for optimization - ## purposes when you know what you're doing! + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. + ## + ## This is **unsafe** as it allocates an object of the passed `size`. + ## This should only be used for optimization purposes when you know + ## what you're doing! + ## + ## See also: + ## * `new <#new,ref.T,proc(ref.T)>`_ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.} - ## returns the size of ``x`` in bytes. Since this is a low-level proc, - ## its usage is discouraged - using ``new`` for the most cases suffices - ## that one never needs to know ``x``'s size. As a special semantic rule, - ## ``x`` may also be a type identifier (``sizeof(int)`` is valid). + ## Returns the size of `x` in bytes. ## - ## .. code-block:: nim - ## sizeof('A') #=> 1 - ## sizeof(2) #=> 8 - -when defined(nimtypedescfixed): - proc sizeof*(x: typedesc): int {.magic: "SizeOf", noSideEffect.} - -proc `<`*[T](x: Ordinal[T]): T {.magic: "UnaryLt", noSideEffect.} - ## unary ``<`` that can be used for nice looking excluding ranges: + ## Since this is a low-level proc, + ## its usage is discouraged - using `new <#new,ref.T,proc(ref.T)>`_ for + ## the most cases suffices that one never needs to know `x`'s size. ## - ## .. code-block:: nim - ## for i in 0 .. <10: echo i + ## As a special semantic rule, `x` may also be a type identifier + ## (`sizeof(int)` is valid). ## - ## Semantically this is the same as ``pred``. + ## Limitations: If used for types that are imported from C or C++, + ## sizeof should fallback to the `sizeof` in the C compiler. The + ## result isn't available for the Nim compiler and therefore can't + ## be used inside of macros. + ## ```nim + ## sizeof('A') # => 1 + ## sizeof(2) # => 8 + ## ``` -proc succ*[T](x: Ordinal[T], y = 1): T {.magic: "Succ", noSideEffect.} - ## returns the ``y``-th successor of the value ``x``. ``T`` has to be - ## an ordinal type. If such a value does not exist, ``EOutOfRange`` is raised - ## or a compile time error occurs. +proc alignof*[T](x: T): int {.magic: "AlignOf", noSideEffect.} +proc alignof*(x: typedesc): int {.magic: "AlignOf", noSideEffect.} -proc pred*[T](x: Ordinal[T], y = 1): T {.magic: "Pred", noSideEffect.} - ## returns the ``y``-th predecessor of the value ``x``. ``T`` has to be - ## an ordinal type. If such a value does not exist, ``EOutOfRange`` is raised - ## or a compile time error occurs. +proc offsetOfDotExpr(typeAccess: typed): int {.magic: "OffsetOf", noSideEffect, compileTime.} -proc inc*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Inc", noSideEffect.} - ## increments the ordinal ``x`` by ``y``. If such a value does not - ## exist, ``EOutOfRange`` is raised or a compile time error occurs. This is a - ## short notation for: ``x = succ(x, y)``. - ## - ## .. code-block:: nim - ## var i = 2 - ## inc(i) #=> 3 - ## inc(i, 3) #=> 6 +template offsetOf*[T](t: typedesc[T]; member: untyped): int = + var tmp {.noinit.}: ptr T + offsetOfDotExpr(tmp[].member) + +template offsetOf*[T](value: T; member: untyped): int = + offsetOfDotExpr(value.member) + +#proc offsetOf*(memberaccess: typed): int {.magic: "OffsetOf", noSideEffect.} + +proc sizeof*(x: typedesc): int {.magic: "SizeOf", noSideEffect.} -proc dec*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Dec", noSideEffect.} - ## decrements the ordinal ``x`` by ``y``. If such a value does not - ## exist, ``EOutOfRange`` is raised or a compile time error occurs. This is a - ## short notation for: ``x = pred(x, y)``. - ## - ## .. code-block:: nim - ## var i = 2 - ## dec(i) #=> 1 - ## dec(i, 3) #=> -2 proc newSeq*[T](s: var seq[T], len: Natural) {.magic: "NewSeq", noSideEffect.} - ## creates a new sequence of type ``seq[T]`` with length ``len``. - ## This is equivalent to ``s = @[]; setlen(s, len)``, but more + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## This is equivalent to `s = @[]; setlen(s, len)`, but more ## efficient since no reallocation is needed. ## - ## Note that the sequence will be filled with zeroed entries, which can be a - ## problem for sequences containing strings since their value will be - ## ``nil``. After the creation of the sequence you should assign entries to + ## Note that the sequence will be filled with zeroed entries. + ## After the creation of the sequence you should assign entries to ## the sequence instead of adding them. Example: - ## - ## .. code-block:: nim - ## var inputStrings : seq[string] + ## ```nim + ## var inputStrings: seq[string] ## newSeq(inputStrings, 3) + ## assert len(inputStrings) == 3 ## inputStrings[0] = "The fourth" ## inputStrings[1] = "assignment" ## inputStrings[2] = "would crash" ## #inputStrings[3] = "out of bounds" + ## ``` proc newSeq*[T](len = 0.Natural): seq[T] = - ## creates a new sequence of type ``seq[T]`` with length ``len``. + ## Creates a new sequence of type `seq[T]` with length `len`. ## - ## Note that the sequence will be filled with zeroed entries, which can be a - ## problem for sequences containing strings since their value will be - ## ``nil``. After the creation of the sequence you should assign entries to - ## the sequence instead of adding them. Example: - ## - ## .. code-block:: nim + ## Note that the sequence will be filled with zeroed entries. + ## After the creation of the sequence you should assign entries to + ## the sequence instead of adding them. + ## ```nim ## var inputStrings = newSeq[string](3) + ## assert len(inputStrings) == 3 ## inputStrings[0] = "The fourth" ## inputStrings[1] = "assignment" ## inputStrings[2] = "would crash" ## #inputStrings[3] = "out of bounds" - newSeq(result, len) - -proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {. - magic: "LengthOpenArray", noSideEffect.} -proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} -proc len*(x: cstring): int {.magic: "LengthStr", noSideEffect.} -proc len*[I, T](x: array[I, T]): int {.magic: "LengthArray", noSideEffect.} -proc len*[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.} - ## returns the length of an array, an openarray, a sequence or a string. - ## This is roughly the same as ``high(T)-low(T)+1``, but its resulting type is - ## always an int. - ## - ## .. code-block:: nim - ## var arr = [1,1,1,1,1] - ## len(arr) #=> 5 - ## for i in 0..<arr.len: - ## echo arr[i] #=> 1,1,1,1,1 - -# set routines: -proc incl*[T](x: var set[T], y: T) {.magic: "Incl", noSideEffect.} - ## includes element ``y`` to the set ``x``. This is the same as - ## ``x = x + {y}``, but it might be more efficient. - ## - ## .. code-block:: nim - ## var a = initSet[int](4) - ## a.incl(2) #=> {2} - ## a.incl(3) #=> {2, 3} - -template incl*[T](s: var set[T], flags: set[T]) = - ## includes the set of flags to the set ``x``. - s = s + flags - -proc excl*[T](x: var set[T], y: T) {.magic: "Excl", noSideEffect.} - ## excludes element ``y`` to the set ``x``. This is the same as - ## ``x = x - {y}``, but it might be more efficient. - ## - ## .. code-block:: nim - ## var b = {2,3,5,6,12,545} - ## b.excl(5) #=> {2,3,6,12,545} - -template excl*[T](s: var set[T], flags: set[T]) = - ## excludes the set of flags to ``x``. - s = s - flags - -proc card*[T](x: set[T]): int {.magic: "Card", noSideEffect.} - ## returns the cardinality of the set ``x``, i.e. the number of elements - ## in the set. - ## - ## .. code-block:: nim - ## var i = {1,2,3,4} - ## card(i) #=> 4 - -proc ord*[T](x: T): int {.magic: "Ord", noSideEffect.} - ## returns the internal int value of an ordinal value ``x``. - ## - ## .. code-block:: nim - ## ord('A') #=> 65 - -proc chr*(u: range[0..255]): char {.magic: "Chr", noSideEffect.} - ## converts an int in the range 0..255 to a character. - ## - ## .. code-block:: nim - ## chr(65) #=> A - -# -------------------------------------------------------------------------- -# built-in operators - -when not defined(JS): - proc ze*(x: int8): int {.magic: "Ze8ToI", noSideEffect.} - ## zero extends a smaller integer type to ``int``. This treats `x` as - ## unsigned. - proc ze*(x: int16): int {.magic: "Ze16ToI", noSideEffect.} - ## zero extends a smaller integer type to ``int``. This treats `x` as - ## unsigned. - - proc ze64*(x: int8): int64 {.magic: "Ze8ToI64", noSideEffect.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - proc ze64*(x: int16): int64 {.magic: "Ze16ToI64", noSideEffect.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - - proc ze64*(x: int32): int64 {.magic: "Ze32ToI64", noSideEffect.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - proc ze64*(x: int): int64 {.magic: "ZeIToI64", noSideEffect.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. Does nothing if the size of an ``int`` is the same as ``int64``. - ## (This is the case on 64 bit processors.) - - proc toU8*(x: int): int8 {.magic: "ToU8", noSideEffect.} - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - proc toU16*(x: int): int16 {.magic: "ToU16", noSideEffect.} - ## treats `x` as unsigned and converts it to an ``int16`` by taking the last - ## 16 bits from `x`. - proc toU32*(x: int64): int32 {.magic: "ToU32", noSideEffect.} - ## treats `x` as unsigned and converts it to an ``int32`` by taking the - ## last 32 bits from `x`. - -# integer calculations: -proc `+` *(x: int): int {.magic: "UnaryPlusI", noSideEffect.} -proc `+` *(x: int8): int8 {.magic: "UnaryPlusI", noSideEffect.} -proc `+` *(x: int16): int16 {.magic: "UnaryPlusI", noSideEffect.} -proc `+` *(x: int32): int32 {.magic: "UnaryPlusI", noSideEffect.} -proc `+` *(x: int64): int64 {.magic: "UnaryPlusI", noSideEffect.} - ## Unary `+` operator for an integer. Has no effect. - -proc `-` *(x: int): int {.magic: "UnaryMinusI", noSideEffect.} -proc `-` *(x: int8): int8 {.magic: "UnaryMinusI", noSideEffect.} -proc `-` *(x: int16): int16 {.magic: "UnaryMinusI", noSideEffect.} -proc `-` *(x: int32): int32 {.magic: "UnaryMinusI", noSideEffect.} -proc `-` *(x: int64): int64 {.magic: "UnaryMinusI64", noSideEffect.} - ## Unary `-` operator for an integer. Negates `x`. - -proc `not` *(x: int): int {.magic: "BitnotI", noSideEffect.} -proc `not` *(x: int8): int8 {.magic: "BitnotI", noSideEffect.} -proc `not` *(x: int16): int16 {.magic: "BitnotI", noSideEffect.} -proc `not` *(x: int32): int32 {.magic: "BitnotI", noSideEffect.} - ## computes the `bitwise complement` of the integer `x`. - -when defined(nimnomagic64): - proc `not` *(x: int64): int64 {.magic: "BitnotI", noSideEffect.} -else: - proc `not` *(x: int64): int64 {.magic: "BitnotI64", noSideEffect.} - -proc `+` *(x, y: int): int {.magic: "AddI", noSideEffect.} -proc `+` *(x, y: int8): int8 {.magic: "AddI", noSideEffect.} -proc `+` *(x, y: int16): int16 {.magic: "AddI", noSideEffect.} -proc `+` *(x, y: int32): int32 {.magic: "AddI", noSideEffect.} - ## Binary `+` operator for an integer. - -when defined(nimnomagic64): - proc `+` *(x, y: int64): int64 {.magic: "AddI", noSideEffect.} -else: - proc `+` *(x, y: int64): int64 {.magic: "AddI64", noSideEffect.} - -proc `-` *(x, y: int): int {.magic: "SubI", noSideEffect.} -proc `-` *(x, y: int8): int8 {.magic: "SubI", noSideEffect.} -proc `-` *(x, y: int16): int16 {.magic: "SubI", noSideEffect.} -proc `-` *(x, y: int32): int32 {.magic: "SubI", noSideEffect.} - ## Binary `-` operator for an integer. - -when defined(nimnomagic64): - proc `-` *(x, y: int64): int64 {.magic: "SubI", noSideEffect.} -else: - proc `-` *(x, y: int64): int64 {.magic: "SubI64", noSideEffect.} - -proc `*` *(x, y: int): int {.magic: "MulI", noSideEffect.} -proc `*` *(x, y: int8): int8 {.magic: "MulI", noSideEffect.} -proc `*` *(x, y: int16): int16 {.magic: "MulI", noSideEffect.} -proc `*` *(x, y: int32): int32 {.magic: "MulI", noSideEffect.} - ## Binary `*` operator for an integer. - -when defined(nimnomagic64): - proc `*` *(x, y: int64): int64 {.magic: "MulI", noSideEffect.} -else: - proc `*` *(x, y: int64): int64 {.magic: "MulI64", noSideEffect.} - -proc `div` *(x, y: int): int {.magic: "DivI", noSideEffect.} -proc `div` *(x, y: int8): int8 {.magic: "DivI", noSideEffect.} -proc `div` *(x, y: int16): int16 {.magic: "DivI", noSideEffect.} -proc `div` *(x, y: int32): int32 {.magic: "DivI", noSideEffect.} - ## computes the integer division. This is roughly the same as - ## ``floor(x/y)``. - ## - ## .. code-block:: Nim - ## 1 div 2 == 0 - ## 2 div 2 == 1 - ## 3 div 2 == 1 - ## 7 div 5 == 1 - -when defined(nimnomagic64): - proc `div` *(x, y: int64): int64 {.magic: "DivI", noSideEffect.} -else: - proc `div` *(x, y: int64): int64 {.magic: "DivI64", noSideEffect.} - -proc `mod` *(x, y: int): int {.magic: "ModI", noSideEffect.} -proc `mod` *(x, y: int8): int8 {.magic: "ModI", noSideEffect.} -proc `mod` *(x, y: int16): int16 {.magic: "ModI", noSideEffect.} -proc `mod` *(x, y: int32): int32 {.magic: "ModI", noSideEffect.} - ## computes the integer modulo operation (remainder). - ## This is the same as - ## ``x - (x div y) * y``. + ## ``` ## - ## .. code-block:: Nim - ## (7 mod 5) == 2 + ## See also: + ## * `newSeqOfCap <#newSeqOfCap,Natural>`_ + ## * `newSeqUninit <#newSeqUninit,Natural>`_ + newSeq(result, len) -when defined(nimnomagic64): - proc `mod` *(x, y: int64): int64 {.magic: "ModI", noSideEffect.} -else: - proc `mod` *(x, y: int64): int64 {.magic: "ModI64", noSideEffect.} - -proc `shr` *(x, y: int): int {.magic: "ShrI", noSideEffect.} -proc `shr` *(x, y: int8): int8 {.magic: "ShrI", noSideEffect.} -proc `shr` *(x, y: int16): int16 {.magic: "ShrI", noSideEffect.} -proc `shr` *(x, y: int32): int32 {.magic: "ShrI", noSideEffect.} -proc `shr` *(x, y: int64): int64 {.magic: "ShrI", noSideEffect.} - ## computes the `shift right` operation of `x` and `y`, filling - ## vacant bit positions with zeros. - ## - ## .. code-block:: Nim - ## 0b0001_0000'i8 shr 2 == 0b0000_0100'i8 - ## 0b1000_0000'i8 shr 8 == 0b0000_0000'i8 - ## 0b0000_0001'i8 shr 1 == 0b0000_0000'i8 - -proc `shl` *(x, y: int): int {.magic: "ShlI", noSideEffect.} -proc `shl` *(x, y: int8): int8 {.magic: "ShlI", noSideEffect.} -proc `shl` *(x, y: int16): int16 {.magic: "ShlI", noSideEffect.} -proc `shl` *(x, y: int32): int32 {.magic: "ShlI", noSideEffect.} -proc `shl` *(x, y: int64): int64 {.magic: "ShlI", noSideEffect.} - ## computes the `shift left` operation of `x` and `y`. - ## - ## .. code-block:: Nim - ## 1'i32 shl 4 == 0x0000_0010 - ## 1'i64 shl 4 == 0x0000_0000_0000_0010 - -proc `and` *(x, y: int): int {.magic: "BitandI", noSideEffect.} -proc `and` *(x, y: int8): int8 {.magic: "BitandI", noSideEffect.} -proc `and` *(x, y: int16): int16 {.magic: "BitandI", noSideEffect.} -proc `and` *(x, y: int32): int32 {.magic: "BitandI", noSideEffect.} -proc `and` *(x, y: int64): int64 {.magic: "BitandI", noSideEffect.} - ## computes the `bitwise and` of numbers `x` and `y`. - ## - ## .. code-block:: Nim - ## (0xffff'i16 and 0x0010'i16) == 0x0010 - -proc `or` *(x, y: int): int {.magic: "BitorI", noSideEffect.} -proc `or` *(x, y: int8): int8 {.magic: "BitorI", noSideEffect.} -proc `or` *(x, y: int16): int16 {.magic: "BitorI", noSideEffect.} -proc `or` *(x, y: int32): int32 {.magic: "BitorI", noSideEffect.} -proc `or` *(x, y: int64): int64 {.magic: "BitorI", noSideEffect.} - ## computes the `bitwise or` of numbers `x` and `y`. - ## - ## .. code-block:: Nim - ## (0x0005'i16 or 0x0010'i16) == 0x0015 - -proc `xor` *(x, y: int): int {.magic: "BitxorI", noSideEffect.} -proc `xor` *(x, y: int8): int8 {.magic: "BitxorI", noSideEffect.} -proc `xor` *(x, y: int16): int16 {.magic: "BitxorI", noSideEffect.} -proc `xor` *(x, y: int32): int32 {.magic: "BitxorI", noSideEffect.} -proc `xor` *(x, y: int64): int64 {.magic: "BitxorI", noSideEffect.} - ## computes the `bitwise xor` of numbers `x` and `y`. - ## - ## .. code-block:: Nim - ## (0x1011'i16 xor 0x0101'i16) == 0x1110 - -proc `==` *(x, y: int): bool {.magic: "EqI", noSideEffect.} -proc `==` *(x, y: int8): bool {.magic: "EqI", noSideEffect.} -proc `==` *(x, y: int16): bool {.magic: "EqI", noSideEffect.} -proc `==` *(x, y: int32): bool {.magic: "EqI", noSideEffect.} -proc `==` *(x, y: int64): bool {.magic: "EqI", noSideEffect.} - ## Compares two integers for equality. - -proc `<=` *(x, y: int): bool {.magic: "LeI", noSideEffect.} -proc `<=` *(x, y: int8): bool {.magic: "LeI", noSideEffect.} -proc `<=` *(x, y: int16): bool {.magic: "LeI", noSideEffect.} -proc `<=` *(x, y: int32): bool {.magic: "LeI", noSideEffect.} -proc `<=` *(x, y: int64): bool {.magic: "LeI", noSideEffect.} - ## Returns true iff `x` is less than or equal to `y`. - -proc `<` *(x, y: int): bool {.magic: "LtI", noSideEffect.} -proc `<` *(x, y: int8): bool {.magic: "LtI", noSideEffect.} -proc `<` *(x, y: int16): bool {.magic: "LtI", noSideEffect.} -proc `<` *(x, y: int32): bool {.magic: "LtI", noSideEffect.} -proc `<` *(x, y: int64): bool {.magic: "LtI", noSideEffect.} - ## Returns true iff `x` is less than `y`. +proc newSeqOfCap*[T](cap: Natural): seq[T] {. + magic: "NewSeqOfCap", noSideEffect.} = + ## Creates a new sequence of type `seq[T]` with length zero and capacity + ## `cap`. Example: + ## ```nim + ## var x = newSeqOfCap[int](5) + ## assert len(x) == 0 + ## x.add(10) + ## assert len(x) == 1 + ## ``` + discard -type - IntMax32 = int|int8|int16|int32 - -proc `+%` *(x, y: IntMax32): IntMax32 {.magic: "AddU", noSideEffect.} -proc `+%` *(x, y: int64): int64 {.magic: "AddU", noSideEffect.} - ## treats `x` and `y` as unsigned and adds them. The result is truncated to - ## fit into the result. This implements modulo arithmetic. No overflow - ## errors are possible. - -proc `-%` *(x, y: IntMax32): IntMax32 {.magic: "SubU", noSideEffect.} -proc `-%` *(x, y: int64): int64 {.magic: "SubU", noSideEffect.} - ## treats `x` and `y` as unsigned and subtracts them. The result is - ## truncated to fit into the result. This implements modulo arithmetic. - ## No overflow errors are possible. - -proc `*%` *(x, y: IntMax32): IntMax32 {.magic: "MulU", noSideEffect.} -proc `*%` *(x, y: int64): int64 {.magic: "MulU", noSideEffect.} - ## treats `x` and `y` as unsigned and multiplies them. The result is - ## truncated to fit into the result. This implements modulo arithmetic. - ## No overflow errors are possible. - -proc `/%` *(x, y: IntMax32): IntMax32 {.magic: "DivU", noSideEffect.} -proc `/%` *(x, y: int64): int64 {.magic: "DivU", noSideEffect.} - ## treats `x` and `y` as unsigned and divides them. The result is - ## truncated to fit into the result. This implements modulo arithmetic. - ## No overflow errors are possible. - -proc `%%` *(x, y: IntMax32): IntMax32 {.magic: "ModU", noSideEffect.} -proc `%%` *(x, y: int64): int64 {.magic: "ModU", noSideEffect.} - ## treats `x` and `y` as unsigned and compute the modulo of `x` and `y`. - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. - ## No overflow errors are possible. - -proc `<=%` *(x, y: IntMax32): bool {.magic: "LeU", noSideEffect.} -proc `<=%` *(x, y: int64): bool {.magic: "LeU64", noSideEffect.} - ## treats `x` and `y` as unsigned and compares them. - ## Returns true iff ``unsigned(x) <= unsigned(y)``. - -proc `<%` *(x, y: IntMax32): bool {.magic: "LtU", noSideEffect.} -proc `<%` *(x, y: int64): bool {.magic: "LtU64", noSideEffect.} - ## treats `x` and `y` as unsigned and compares them. - ## Returns true iff ``unsigned(x) < unsigned(y)``. - -# unsigned integer operations: -proc `not`*[T: SomeUnsignedInt](x: T): T {.magic: "BitnotI", noSideEffect.} - ## computes the `bitwise complement` of the integer `x`. - -proc `shr`*[T: SomeUnsignedInt](x, y: T): T {.magic: "ShrI", noSideEffect.} - ## computes the `shift right` operation of `x` and `y`. - -proc `shl`*[T: SomeUnsignedInt](x, y: T): T {.magic: "ShlI", noSideEffect.} - ## computes the `shift left` operation of `x` and `y`. - -proc `and`*[T: SomeUnsignedInt](x, y: T): T {.magic: "BitandI", noSideEffect.} - ## computes the `bitwise and` of numbers `x` and `y`. - -proc `or`*[T: SomeUnsignedInt](x, y: T): T {.magic: "BitorI", noSideEffect.} - ## computes the `bitwise or` of numbers `x` and `y`. - -proc `xor`*[T: SomeUnsignedInt](x, y: T): T {.magic: "BitxorI", noSideEffect.} - ## computes the `bitwise xor` of numbers `x` and `y`. - -proc `==`*[T: SomeUnsignedInt](x, y: T): bool {.magic: "EqI", noSideEffect.} - ## Compares two unsigned integers for equality. - -proc `+`*[T: SomeUnsignedInt](x, y: T): T {.magic: "AddU", noSideEffect.} - ## Binary `+` operator for unsigned integers. - -proc `-`*[T: SomeUnsignedInt](x, y: T): T {.magic: "SubU", noSideEffect.} - ## Binary `-` operator for unsigned integers. - -proc `*`*[T: SomeUnsignedInt](x, y: T): T {.magic: "MulU", noSideEffect.} - ## Binary `*` operator for unsigned integers. - -proc `div`*[T: SomeUnsignedInt](x, y: T): T {.magic: "DivU", noSideEffect.} - ## computes the integer division. This is roughly the same as - ## ``floor(x/y)``. - ## - ## .. code-block:: Nim - ## (7 div 5) == 2 - -proc `mod`*[T: SomeUnsignedInt](x, y: T): T {.magic: "ModU", noSideEffect.} - ## computes the integer modulo operation (remainder). - ## This is the same as - ## ``x - (x div y) * y``. - ## - ## .. code-block:: Nim - ## (7 mod 5) == 2 - -proc `<=`*[T: SomeUnsignedInt](x, y: T): bool {.magic: "LeU", noSideEffect.} - ## Returns true iff ``x <= y``. - -proc `<`*[T: SomeUnsignedInt](x, y: T): bool {.magic: "LtU", noSideEffect.} - ## Returns true iff ``unsigned(x) < unsigned(y)``. - -# floating point operations: -proc `+` *(x: float32): float32 {.magic: "UnaryPlusF64", noSideEffect.} -proc `-` *(x: float32): float32 {.magic: "UnaryMinusF64", noSideEffect.} -proc `+` *(x, y: float32): float32 {.magic: "AddF64", noSideEffect.} -proc `-` *(x, y: float32): float32 {.magic: "SubF64", noSideEffect.} -proc `*` *(x, y: float32): float32 {.magic: "MulF64", noSideEffect.} -proc `/` *(x, y: float32): float32 {.magic: "DivF64", noSideEffect.} - -proc `+` *(x: float): float {.magic: "UnaryPlusF64", noSideEffect.} -proc `-` *(x: float): float {.magic: "UnaryMinusF64", noSideEffect.} -proc `+` *(x, y: float): float {.magic: "AddF64", noSideEffect.} -proc `-` *(x, y: float): float {.magic: "SubF64", noSideEffect.} -proc `*` *(x, y: float): float {.magic: "MulF64", noSideEffect.} -proc `/` *(x, y: float): float {.magic: "DivF64", noSideEffect.} - ## computes the floating point division - -proc `==` *(x, y: float32): bool {.magic: "EqF64", noSideEffect.} -proc `<=` *(x, y: float32): bool {.magic: "LeF64", noSideEffect.} -proc `<` *(x, y: float32): bool {.magic: "LtF64", noSideEffect.} - -proc `==` *(x, y: float): bool {.magic: "EqF64", noSideEffect.} -proc `<=` *(x, y: float): bool {.magic: "LeF64", noSideEffect.} -proc `<` *(x, y: float): bool {.magic: "LtF64", noSideEffect.} - -# set operators -proc `*` *[T](x, y: set[T]): set[T] {.magic: "MulSet", noSideEffect.} - ## This operator computes the intersection of two sets. -proc `+` *[T](x, y: set[T]): set[T] {.magic: "PlusSet", noSideEffect.} - ## This operator computes the union of two sets. -proc `-` *[T](x, y: set[T]): set[T] {.magic: "MinusSet", noSideEffect.} - ## This operator computes the difference of two sets. - -proc contains*[T](x: set[T], y: T): bool {.magic: "InSet", noSideEffect.} - ## One should overload this proc if one wants to overload the ``in`` operator. - ## The parameters are in reverse order! ``a in b`` is a template for - ## ``contains(b, a)``. - ## This is because the unification algorithm that Nim uses for overload - ## resolution works from left to right. - ## But for the ``in`` operator that would be the wrong direction for this - ## piece of code: - ## - ## .. code-block:: Nim - ## var s: set[range['a'..'z']] = {'a'..'c'} - ## writeLine(stdout, 'b' in s) - ## - ## If ``in`` had been declared as ``[T](elem: T, s: set[T])`` then ``T`` would - ## have been bound to ``char``. But ``s`` is not compatible to type - ## ``set[char]``! The solution is to bind ``T`` to ``range['a'..'z']``. This - ## is achieved by reversing the parameters for ``contains``; ``in`` then - ## passes its arguments in reverse order. - -proc contains*[T](s: Slice[T], value: T): bool {.noSideEffect, inline.} = - ## Checks if `value` is within the range of `s`; returns true iff - ## `value >= s.a and value <= s.b` - ## - ## .. code-block:: Nim +func len*[TOpenArray: openArray|varargs](x: TOpenArray): int {.magic: "LengthOpenArray".} = + ## Returns the length of an openArray. + runnableExamples: + proc bar[T](a: openArray[T]): int = len(a) + assert bar([1,2]) == 2 + assert [1,2].len == 2 + +func len*(x: string): int {.magic: "LengthStr".} = + ## Returns the length of a string. + runnableExamples: + assert "abc".len == 3 + assert "".len == 0 + assert string.default.len == 0 + +proc len*(x: cstring): int {.magic: "LengthStr", noSideEffect.} = + ## Returns the length of a compatible string. This is an O(n) operation except + ## in js at runtime. + ## + ## **Note:** On the JS backend this currently counts UTF-16 code points + ## instead of bytes at runtime (not at compile time). For now, if you + ## need the byte length of the UTF-8 encoding, convert to string with + ## `$` first then call `len`. + runnableExamples: + doAssert len(cstring"abc") == 3 + doAssert len(cstring r"ab\0c") == 5 # \0 is escaped + doAssert len(cstring"ab\0c") == 5 # ditto + var a: cstring = "ab\0c" + when defined(js): doAssert a.len == 4 # len ignores \0 for js + else: doAssert a.len == 2 # \0 is a null terminator + static: + var a2: cstring = "ab\0c" + doAssert a2.len == 2 # \0 is a null terminator, even in js vm + +func len*(x: (type array)|array): int {.magic: "LengthArray".} = + ## Returns the length of an array or an array type. + ## This is roughly the same as `high(T)-low(T)+1`. + runnableExamples: + var a = [1, 1, 1] + assert a.len == 3 + assert array[0, float].len == 0 + static: assert array[-2..2, float].len == 5 + +func len*[T](x: seq[T]): int {.magic: "LengthSeq".} = + ## Returns the length of `x`. + runnableExamples: + assert @[0, 1].len == 2 + assert seq[int].default.len == 0 + assert newSeq[int](3).len == 3 + let s = newSeqOfCap[int](3) + assert s.len == 0 + # xxx this gives cgen error: assert newSeqOfCap[int](3).len == 0 + +func ord*[T: Ordinal|enum](x: T): int {.magic: "Ord".} = + ## Returns the internal `int` value of `x`, including for enum with holes + ## and distinct ordinal types. + runnableExamples: + assert ord('A') == 65 + type Foo = enum + f0 = 0, f1 = 3 + assert f1.ord == 3 + type Bar = distinct int + assert 3.Bar.ord == 3 + +func chr*(u: range[0..255]): char {.magic: "Chr".} = + ## Converts `u` to a `char`, same as `char(u)`. + runnableExamples: + doAssert chr(65) == 'A' + doAssert chr(255) == '\255' + doAssert chr(255) == char(255) + doAssert not compiles chr(256) + doAssert not compiles char(256) + var x = 256 + doAssertRaises(RangeDefect): discard chr(x) + doAssertRaises(RangeDefect): discard char(x) + + +include "system/setops" + + +proc contains*[U, V, W](s: HSlice[U, V], value: W): bool {.noSideEffect, inline.} = + ## Checks if `value` is within the range of `s`; returns true if + ## `value >= s.a and value <= s.b`. + ## ```nim ## assert((1..3).contains(1) == true) ## assert((1..3).contains(2) == true) ## assert((1..3).contains(4) == false) + ## ``` result = s.a <= value and value <= s.b -template `in` * (x, y: expr): expr {.immediate, dirty.} = contains(y, x) - ## Sugar for contains - ## - ## .. code-block:: Nim +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template `in`*(x, y: untyped): untyped {.dirty, callsite.} = contains(y, x) + ## Sugar for `contains`. + ## ```nim ## assert(1 in (1..3) == true) ## assert(5 in (1..3) == false) -template `notin` * (x, y: expr): expr {.immediate, dirty.} = not contains(y, x) - ## Sugar for not containing - ## - ## .. code-block:: Nim + ## ``` +template `notin`*(x, y: untyped): untyped {.dirty, callsite.} = not contains(y, x) + ## Sugar for `not contains`. + ## ```nim ## assert(1 notin (1..3) == false) ## assert(5 notin (1..3) == true) + ## ``` -proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} - ## Checks if T is of the same type as S +proc `is`*[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} + ## Checks if `T` is of the same type as `S`. + ## + ## For a negated version, use `isnot <#isnot.t,untyped,untyped>`_. + ## + ## ```nim + ## assert 42 is int + ## assert @[1, 2] is seq ## - ## .. code-block:: Nim ## proc test[T](a: T): int = ## when (T is int): ## return a @@ -1120,207 +768,316 @@ proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## ## assert(test[int](3) == 3) ## assert(test[string]("xyz") == 0) -template `isnot` *(x, y: expr): expr {.immediate.} = not (x is y) - ## Negated version of `is`. Equivalent to ``not(x is y)``. + ## ``` +template `isnot`*(x, y: untyped): untyped {.callsite.} = not (x is y) + ## Negated version of `is <#is,T,S>`_. Equivalent to `not(x is y)`. + ## ```nim + ## assert 42 isnot float + ## assert @[1, 2] isnot enum + ## ``` + +when (defined(nimOwnedEnabled) and not defined(nimscript)) or defined(nimFixedOwned): + type owned*[T]{.magic: "BuiltinType".} ## type constructor to mark a ref/ptr or a closure as `owned`. +else: + template owned*(t: typedesc): typedesc = t -proc `of` *[T, S](x: T, y: S): bool {.magic: "Of", noSideEffect.} - ## Checks if `x` has a type of `y` - ## - ## .. code-block:: Nim - ## assert(FloatingPointError of Exception) - ## assert(DivByZeroError of Exception) +when defined(nimOwnedEnabled) and not defined(nimscript): + proc new*[T](a: var owned(ref T)) {.magic: "New", noSideEffect.} + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. -proc cmp*[T](x, y: T): int {.procvar.} = - ## Generic compare proc. Returns a value < 0 iff x < y, a value > 0 iff x > y - ## and 0 iff x == y. This is useful for writing generic algorithms without - ## performance loss. This generic implementation uses the `==` and `<` - ## operators. - ## - ## .. code-block:: Nim - ## import algorithm - ## echo sorted(@[4,2,6,5,8,7], cmp[int]) + proc new*(t: typedesc): auto = + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it as result value. + ## + ## When `T` is a ref type then the resulting type will be `T`, + ## otherwise it will be `ref T`. + when (t is ref): + var r: owned t + else: + var r: owned(ref t) + new(r) + return r + + proc unown*[T](x: T): T {.magic: "Unown", noSideEffect.} + ## Use the expression `x` ignoring its ownership attribute. + + +else: + template unown*(x: typed): untyped = x + + proc new*[T](a: var ref T) {.magic: "New", noSideEffect.} + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. + + proc new*(t: typedesc): auto = + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it as result value. + ## + ## When `T` is a ref type then the resulting type will be `T`, + ## otherwise it will be `ref T`. + when (t is ref): + var r: t + else: + var r: ref t + new(r) + return r + + +template disarm*(x: typed) = + ## Useful for `disarming` dangling pointers explicitly for `--newruntime`. + ## Regardless of whether `--newruntime` is used or not + ## this sets the pointer or callback `x` to `nil`. This is an + ## experimental API! + x = nil + +proc `of`*[T, S](x: T, y: typedesc[S]): bool {.magic: "Of", noSideEffect.} = + ## Checks if `x` is an instance of `y`. + runnableExamples: + type + Base = ref object of RootObj + Sub1 = ref object of Base + Sub2 = ref object of Base + Unrelated = ref object + + var base: Base = Sub1() # downcast + doAssert base of Base # generates `CondTrue` (statically true) + doAssert base of Sub1 + doAssert base isnot Sub1 + doAssert not (base of Sub2) + + base = Sub2() # re-assign + doAssert base of Sub2 + doAssert Sub2(base) != nil # upcast + doAssertRaises(ObjectConversionDefect): discard Sub1(base) + + var sub1 = Sub1() + doAssert sub1 of Base + doAssert sub1.Base of Sub1 + + doAssert not compiles(base of Unrelated) + +proc cmp*[T](x, y: T): int = + ## Generic compare proc. + ## + ## Returns: + ## * a value less than zero, if `x < y` + ## * a value greater than zero, if `x > y` + ## * zero, if `x == y` + ## + ## This is useful for writing generic algorithms without performance loss. + ## This generic implementation uses the `==` and `<` operators. + ## ```nim + ## import std/algorithm + ## echo sorted(@[4, 2, 6, 5, 8, 7], cmp[int]) + ## ``` if x == y: return 0 if x < y: return -1 return 1 -proc cmp*(x, y: string): int {.noSideEffect, procvar.} +proc cmp*(x, y: string): int {.noSideEffect.} ## Compare proc for strings. More efficient than the generic version. + ## + ## **Note**: The precise result values depend on the used C runtime library and + ## can differ between operating systems! -proc `@` * [IDX, T](a: array[IDX, T]): seq[T] {. - magic: "ArrToSeq", nosideeffect.} - ## turns an array into a sequence. This most often useful for constructing - ## sequences with the array constructor: ``@[1, 2, 3]`` has the type - ## ``seq[int]``, while ``[1, 2, 3]`` has the type ``array[0..2, int]``. +proc `@`* [IDX, T](a: sink array[IDX, T]): seq[T] {.magic: "ArrToSeq", noSideEffect.} + ## Turns an array into a sequence. + ## + ## This most often useful for constructing + ## sequences with the array constructor: `@[1, 2, 3]` has the type + ## `seq[int]`, while `[1, 2, 3]` has the type `array[0..2, int]`. + ## + ## ```nim + ## let + ## a = [1, 3, 5] + ## b = "foo" + ## + ## echo @a # => @[1, 3, 5] + ## echo @b # => @['f', 'o', 'o'] + ## ``` + +proc default*[T](_: typedesc[T]): T {.magic: "Default", noSideEffect.} = + ## Returns the default value of the type `T`. Contrary to `zeroDefault`, it takes default fields + ## of an object into consideration. + ## + ## See also: + ## * `zeroDefault <#zeroDefault,typedesc[T]>`_ + ## + runnableExamples("-d:nimPreviewRangeDefault"): + assert (int, float).default == (0, 0.0) + type Foo = object + a: range[2..6] + var x = Foo.default + assert x.a == 2 + + +proc reset*[T](obj: var T) {.noSideEffect.} = + ## Resets an object `obj` to its default value. + when nimvm: + obj = default(typeof(obj)) + else: + when defined(gcDestructors): + {.cast(noSideEffect), cast(raises: []), cast(tags: []).}: + `=destroy`(obj) + `=wasMoved`(obj) + else: + obj = default(typeof(obj)) proc setLen*[T](s: var seq[T], newlen: Natural) {. - magic: "SetLengthSeq", noSideEffect.} - ## sets the length of `s` to `newlen`. - ## ``T`` may be any sequence type. + magic: "SetLengthSeq", noSideEffect, nodestroy.} + ## Sets the length of seq `s` to `newlen`. `T` may be any sequence type. + ## ## If the current length is greater than the new length, - ## ``s`` will be truncated. `s` cannot be nil! To initialize a sequence with - ## a size, use ``newSeq`` instead. + ## `s` will be truncated. + ## ```nim + ## var x = @[10, 20] + ## x.setLen(5) + ## x[4] = 50 + ## assert x == @[10, 20, 0, 0, 50] + ## x.setLen(1) + ## assert x == @[10] + ## ``` proc setLen*(s: var string, newlen: Natural) {. magic: "SetLengthStr", noSideEffect.} - ## sets the length of `s` to `newlen`. - ## If the current length is greater than the new length, - ## ``s`` will be truncated. `s` cannot be nil! To initialize a string with - ## a size, use ``newString`` instead. + ## Sets the length of string `s` to `newlen`. ## - ## .. code-block:: Nim - ## var myS = "Nim is great!!" - ## myS.setLen(3) - ## echo myS, " is fantastic!!" + ## If the current length is greater than the new length, + ## `s` will be truncated. + ## ```nim + ## var myS = "Nim is great!!" + ## myS.setLen(3) # myS <- "Nim" + ## echo myS, " is fantastic!!" + ## ``` proc newString*(len: Natural): string {. magic: "NewString", importc: "mnewString", noSideEffect.} - ## returns a new string of length ``len`` but with uninitialized - ## content. One needs to fill the string character after character - ## with the index operator ``s[i]``. This procedure exists only for - ## optimization purposes; the same effect can be achieved with the - ## ``&`` operator or with ``add``. + ## Returns a new string of length `len`. + ## One needs to fill the string character after character + ## with the index operator `s[i]`. + ## + ## This procedure exists only for optimization purposes; + ## the same effect can be achieved with the `&` operator or with `add`. proc newStringOfCap*(cap: Natural): string {. magic: "NewStringOfCap", importc: "rawNewString", noSideEffect.} - ## returns a new string of length ``0`` but with capacity `cap`.This - ## procedure exists only for optimization purposes; the same effect can - ## be achieved with the ``&`` operator or with ``add``. - -proc `&` * (x: string, y: char): string {. - magic: "ConStrStr", noSideEffect, merge.} - ## Concatenates `x` with `y` + ## Returns a new string of length `0` but with capacity `cap`. ## - ## .. code-block:: Nim + ## This procedure exists only for optimization purposes; the same effect can + ## be achieved with the `&` operator or with `add`. + +proc `&`*(x: string, y: char): string {. + magic: "ConStrStr", noSideEffect.} + ## Concatenates `x` with `y`. + ## ```nim ## assert("ab" & 'c' == "abc") -proc `&` * (x, y: char): string {. - magic: "ConStrStr", noSideEffect, merge.} - ## Concatenates `x` and `y` into a string - ## - ## .. code-block:: Nim + ## ``` +proc `&`*(x, y: char): string {. + magic: "ConStrStr", noSideEffect.} + ## Concatenates characters `x` and `y` into a string. + ## ```nim ## assert('a' & 'b' == "ab") -proc `&` * (x, y: string): string {. - magic: "ConStrStr", noSideEffect, merge.} - ## Concatenates `x` and `y` - ## - ## .. code-block:: Nim + ## ``` +proc `&`*(x, y: string): string {. + magic: "ConStrStr", noSideEffect.} + ## Concatenates strings `x` and `y`. + ## ```nim ## assert("ab" & "cd" == "abcd") -proc `&` * (x: char, y: string): string {. - magic: "ConStrStr", noSideEffect, merge.} - ## Concatenates `x` with `y` - ## - ## .. code-block:: Nim + ## ``` +proc `&`*(x: char, y: string): string {. + magic: "ConStrStr", noSideEffect.} + ## Concatenates `x` with `y`. + ## ```nim ## assert('a' & "bc" == "abc") + ## ``` # implementation note: These must all have the same magic value "ConStrStr" so # that the merge optimization works properly. proc add*(x: var string, y: char) {.magic: "AppendStrCh", noSideEffect.} - ## Appends `y` to `x` in place - ## - ## .. code-block:: Nim + ## Appends `y` to `x` in place. + ## ```nim ## var tmp = "" ## tmp.add('a') ## tmp.add('b') ## assert(tmp == "ab") -proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} - ## Concatenates `x` and `y` in place + ## ``` + +proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} = + ## Concatenates `x` and `y` in place. ## - ## .. code-block:: Nim - ## var tmp = "" - ## tmp.add("ab") - ## tmp.add("cd") - ## assert(tmp == "abcd") + ## See also `strbasics.add`. + runnableExamples: + var tmp = "" + tmp.add("ab") + tmp.add("cd") + assert tmp == "abcd" type - Endianness* = enum ## is a type describing the endianness of a processor. + Endianness* = enum ## Type describing the endianness of a processor. littleEndian, bigEndian const - isMainModule* {.magic: "IsMainModule".}: bool = false - ## is true only when accessed in the main module. This works thanks to - ## compiler magic. It is useful to embed testing code in a module. - - CompileDate* {.magic: "CompileDate"}: string = "0000-00-00" - ## is the date of compilation as a string of the form - ## ``YYYY-MM-DD``. This works thanks to compiler magic. - - CompileTime* {.magic: "CompileTime"}: string = "00:00:00" - ## is the time of compilation as a string of the form - ## ``HH:MM:SS``. This works thanks to compiler magic. - - cpuEndian* {.magic: "CpuEndian"}: Endianness = littleEndian - ## is the endianness of the target CPU. This is a valuable piece of + cpuEndian* {.magic: "CpuEndian".}: Endianness = littleEndian + ## The endianness of the target CPU. This is a valuable piece of ## information for low-level code only. This works thanks to compiler ## magic. hostOS* {.magic: "HostOS".}: string = "" - ## a string that describes the host operating system. Possible values: - ## "windows", "macosx", "linux", "netbsd", "freebsd", "openbsd", "solaris", - ## "aix", "standalone". + ## A string that describes the host operating system. + ## + ## Possible values: + ## `"windows"`, `"macosx"`, `"linux"`, `"netbsd"`, `"freebsd"`, + ## `"openbsd"`, `"solaris"`, `"aix"`, `"haiku"`, `"standalone"`. hostCPU* {.magic: "HostCPU".}: string = "" - ## a string that describes the host CPU. Possible values: - ## "i386", "alpha", "powerpc", "powerpc64", "powerpc64el", "sparc", - ## "amd64", "mips", "mipsel", "arm", "arm64". + ## A string that describes the host CPU. + ## + ## Possible values: + ## `"i386"`, `"alpha"`, `"powerpc"`, `"powerpc64"`, `"powerpc64el"`, + ## `"sparc"`, `"amd64"`, `"mips"`, `"mipsel"`, `"arm"`, `"arm64"`, + ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`, `"loongarch64"`. seqShallowFlag = low(int) - -when defined(nimKnowsNimvm): - let nimvm* {.magic: "Nimvm".}: bool = false - ## may be used only in "when" expression. - ## It is true in Nim VM context and false otherwise -else: - const nimvm*: bool = false - -proc compileOption*(option: string): bool {. - magic: "CompileOption", noSideEffect.} - ## can be used to determine an on|off compile-time option. Example: - ## - ## .. code-block:: nim - ## when compileOption("floatchecks"): - ## echo "compiled with floating point NaN and Inf checks" - -proc compileOption*(option, arg: string): bool {. - magic: "CompileOptionArg", noSideEffect.} - ## can be used to determine an enum compile-time option. Example: - ## - ## .. code-block:: nim - ## when compileOption("opt", "size") and compileOption("gc", "boehm"): - ## echo "compiled with optimization for size and uses Boehm's GC" + strlitFlag = 1 shl (sizeof(int)*8 - 2) # later versions of the codegen \ + # emit this flag + # for string literals, it allows for some optimizations. const hasThreadSupport = compileOption("threads") and not defined(nimscript) hasSharedHeap = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own - taintMode = compileOption("taintmode") + +when notJSnotNims and not defined(nimSeqsV2): + template space(s: PGenericSeq): int = + s.reserved and not (seqShallowFlag or strlitFlag) + +when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): + # tcc doesn't support TLS + {.error: "`--tlsEmulation:on` must be used when using threads with tcc backend".} when defined(boehmgc): when defined(windows): - const boehmLib = "boehmgc.dll" + when sizeof(int) == 8: + const boehmLib = "boehmgc64.dll" + else: + const boehmLib = "boehmgc.dll" elif defined(macosx): const boehmLib = "libgc.dylib" + elif defined(openbsd): + const boehmLib = "libgc.so.(4|5).0" + elif defined(freebsd): + const boehmLib = "libgc-threaded.so.1" else: const boehmLib = "libgc.so.1" {.pragma: boehmGC, noconv, dynlib: boehmLib.} -when taintMode: - type TaintedString* = distinct string ## a distinct string type that - ## is `tainted`:idx:. It is an alias for - ## ``string`` if the taint mode is not - ## turned on. Use the ``-d:taintMode`` - ## command line switch to turn the taint - ## mode on. +when not defined(nimPreviewSlimSystem): + type TaintedString* {.deprecated: "Deprecated since 1.5".} = string - proc len*(s: TaintedString): int {.borrow.} -else: - type TaintedString* = string ## a distinct string type that - ## is `tainted`:idx:. It is an alias for - ## ``string`` if the taint mode is not - ## turned on. Use the ``-d:taintMode`` - ## command line switch to turn the taint - ## mode on. - -when defined(profiler): - proc nimProfile() {.compilerProc, noinline.} + +when defined(profiler) and not defined(nimscript): + proc nimProfile() {.compilerproc, noinline.} when hasThreadSupport: {.pragma: rtlThreadVar, threadvar.} else: @@ -1328,864 +1085,661 @@ else: const QuitSuccess* = 0 - ## is the value that should be passed to `quit <#quit>`_ to indicate + ## is the value that should be passed to `quit <#quit,int>`_ to indicate ## success. QuitFailure* = 1 - ## is the value that should be passed to `quit <#quit>`_ to indicate + ## is the value that should be passed to `quit <#quit,int>`_ to indicate ## failure. -var programResult* {.exportc: "nim_program_result".}: int - ## modify this variable to specify the exit code of the program - ## under normal circumstances. When the program is terminated - ## prematurely using ``quit``, this value is ignored. +when not defined(js) and hostOS != "standalone": + var programResult* {.compilerproc, exportc: "nim_program_result".}: int + ## deprecated, prefer `quit` or `exitprocs.getProgramResult`, `exitprocs.setProgramResult`. -proc quit*(errorcode: int = QuitSuccess) {. - magic: "Exit", importc: "exit", header: "<stdlib.h>", noreturn.} - ## Stops the program immediately with an exit code. - ## - ## Before stopping the program the "quit procedures" are called in the - ## opposite order they were added with `addQuitProc <#addQuitProc>`_. - ## ``quit`` never returns and ignores any exception that may have been raised - ## by the quit procedures. It does *not* call the garbage collector to free - ## all the memory, unless a quit procedure calls `GC_fullCollect - ## <#GC_fullCollect>`_. - ## - ## The proc ``quit(QuitSuccess)`` is called implicitly when your nim - ## program finishes without incident. A raised unhandled exception is - ## equivalent to calling ``quit(QuitFailure)``. - ## - ## Note that this is a *runtime* call and using ``quit`` inside a macro won't - ## have any compile time effect. If you need to stop the compiler inside a - ## macro, use the `error <manual.html#error-pragma>`_ or `fatal - ## <manual.html#fatal-pragma>`_ pragmas. +import std/private/since +import system/ctypes +export ctypes + +proc align(address, alignment: int): int = + if alignment == 0: # Actually, this is illegal. This branch exists to actively + # hide problems. + result = address + else: + result = (address + (alignment - 1)) and not (alignment - 1) + +include system/rawquits +when defined(genode): + export GenodeEnv template sysAssert(cond: bool, msg: string) = when defined(useSysAssert): if not cond: - echo "[SYSASSERT] ", msg - quit 1 + cstderr.rawWrite "[SYSASSERT] " + cstderr.rawWrite msg + cstderr.rawWrite "\n" + rawQuit 1 const hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(nimscript) -when not defined(JS) and not defined(nimscript) and hostOS != "standalone": - include "system/cgprocs" -when not defined(JS) and not defined(nimscript) and hasAlloc: - proc setStackBottom(theStackBottom: pointer) {.compilerRtl, noinline, benign.} - proc addChar(s: NimString, c: char): NimString {.compilerProc, benign.} - -proc add *[T](x: var seq[T], y: T) {.magic: "AppendSeqElem", noSideEffect.} -proc add *[T](x: var seq[T], y: openArray[T]) {.noSideEffect.} = - ## Generic proc for adding a data item `y` to a container `x`. - ## For containers that have an order, `add` means *append*. New generic - ## containers should also call their adding proc `add` for consistency. - ## Generic code becomes much easier to write if the Nim naming scheme is - ## respected. - ## - ## .. code-block:: nim - ## var s: seq[string] = @["test2","test2"] - ## s.add("test") #=> @[test2, test2, test] - let xl = x.len - setLen(x, xl + y.len) - for i in 0..high(y): x[xl+i] = y[i] - -proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} - ## use this instead of `=` for a `shallow copy`:idx:. The shallow copy - ## only changes the semantics for sequences and strings (and types which - ## contain those). Be careful with the changed semantics though! There - ## is a reason why the default assignment does a deep copy of sequences - ## and strings. +when notJSnotNims and hasAlloc and not defined(nimSeqsV2): + proc addChar(s: NimString, c: char): NimString {.compilerproc, benign.} + +when defined(nimscript) or not defined(nimSeqsV2): + proc add*[T](x: var seq[T], y: sink T) {.magic: "AppendSeqElem", noSideEffect.} + ## Generic proc for adding a data item `y` to a container `x`. + ## + ## For containers that have an order, `add` means *append*. New generic + ## containers should also call their adding proc `add` for consistency. + ## Generic code becomes much easier to write if the Nim naming scheme is + ## respected. + ## ```nim + ## var s: seq[string] = @["test2","test2"] + ## s.add("test") + ## assert s == @["test2", "test2", "test"] + ## ``` + +when false: # defined(gcDestructors): + proc add*[T](x: var seq[T], y: sink openArray[T]) {.noSideEffect.} = + ## Generic proc for adding a container `y` to a container `x`. + ## + ## For containers that have an order, `add` means *append*. New generic + ## containers should also call their adding proc `add` for consistency. + ## Generic code becomes much easier to write if the Nim naming scheme is + ## respected. + ## ```nim + ## var s: seq[string] = @["test2","test2"] + ## s.add("test") # s <- @[test2, test2, test] + ## ``` + ## + ## See also: + ## * `& proc <#&,seq[T],seq[T]>`_ + {.noSideEffect.}: + let xl = x.len + setLen(x, xl + y.len) + for i in 0..high(y): + when nimvm: + # workaround the fact that the VM does not yet + # handle sink parameters properly: + x[xl+i] = y[i] + else: + x[xl+i] = move y[i] +else: + proc add*[T](x: var seq[T], y: openArray[T]) {.noSideEffect.} = + ## Generic proc for adding a container `y` to a container `x`. + ## + ## For containers that have an order, `add` means *append*. New generic + ## containers should also call their adding proc `add` for consistency. + ## Generic code becomes much easier to write if the Nim naming scheme is + ## respected. + ## + ## See also: + ## * `& proc <#&,seq[T],seq[T]>`_ + runnableExamples: + var a = @["a1", "a2"] + a.add(["b1", "b2"]) + assert a == @["a1", "a2", "b1", "b2"] + var c = @["c0", "c1", "c2", "c3"] + a.add(c.toOpenArray(1, 2)) + assert a == @["a1", "a2", "b1", "b2", "c1", "c2"] + + {.noSideEffect.}: + let xl = x.len + setLen(x, xl + y.len) + for i in 0..high(y): x[xl+i] = y[i] + + +when defined(nimSeqsV2): + template movingCopy(a, b: typed) = + a = move(b) +else: + template movingCopy(a, b: typed) = + shallowCopy(a, b) proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## deletes the item at index `i` by putting ``x[high(x)]`` into position `i`. - ## This is an O(1) operation. + ## Deletes the item at index `i` by putting `x[high(x)]` into position `i`. + ## + ## This is an `O(1)` operation. ## - ## .. code-block:: nim - ## var i = @[1,2,3,4,5] - ## i.del(2) #=> @[1, 2, 5, 4] + ## See also: + ## * `delete <#delete,seq[T],Natural>`_ for preserving the order + runnableExamples: + var a = @[10, 11, 12, 13, 14] + a.del(2) + assert a == @[10, 11, 14, 13] let xl = x.len - 1 - shallowCopy(x[i], x[xl]) + movingCopy(x[i], x[xl]) setLen(x, xl) -proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## deletes the item at index `i` by moving ``x[i+1..]`` by one position. - ## This is an O(n) operation. +proc insert*[T](x: var seq[T], item: sink T, i = 0.Natural) {.noSideEffect.} = + ## Inserts `item` into `x` at position `i`. + ## ```nim + ## var i = @[1, 3, 5] + ## i.insert(99, 0) # i <- @[99, 1, 3, 5] + ## ``` + {.noSideEffect.}: + template defaultImpl = + let xl = x.len + setLen(x, xl+1) + var j = xl-1 + while j >= i: + movingCopy(x[j+1], x[j]) + dec(j) + when nimvm: + defaultImpl() + else: + when defined(js): + var it : T + {.emit: "`x` = `x` || []; `x`.splice(`i`, 0, `it`);".} + else: + defaultImpl() + x[i] = item + +when not defined(nimV2): + proc repr*[T](x: T): string {.magic: "Repr", noSideEffect.} + ## Takes any Nim variable and returns its string representation. + ## No trailing newline is inserted (so `echo` won't add an empty newline). + ## Use `-d:nimLegacyReprWithNewline` to revert to old behavior where newlines + ## were added in some cases. + ## + ## It works even for complex data graphs with cycles. This is a great + ## debugging tool. + ## ```nim + ## var s: seq[string] = @["test2", "test2"] + ## var i = @[1, 2, 3, 4, 5] + ## echo repr(s) # => 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] + ## echo repr(i) # => 0x1055ed050[1, 2, 3, 4, 5] + ## ``` + +when not defined(nimPreviewSlimSystem): + type + csize* {.importc: "size_t", nodecl, deprecated: "use `csize_t` instead".} = int + ## This isn't the same as `size_t` in *C*. Don't use it. + +const + Inf* = 0x7FF0000000000000'f64 + ## Contains the IEEE floating point value of positive infinity. + NegInf* = 0xFFF0000000000000'f64 + ## Contains the IEEE floating point value of negative infinity. + NaN* = 0x7FF7FFFFFFFFFFFF'f64 + ## Contains an IEEE floating point value of *Not A Number*. + ## + ## Note that you cannot compare a floating point value to this value + ## and expect a reasonable result - use the `isNaN` or `classify` procedure + ## in the `math module <math.html>`_ for checking for NaN. + +proc high*(T: typedesc[SomeFloat]): T = Inf +proc low*(T: typedesc[SomeFloat]): T = NegInf + +proc toFloat*(i: int): float {.noSideEffect, inline.} = + ## Converts an integer `i` into a `float`. Same as `float(i)`. ## - ## .. code-block:: nim - ## var i = @[1,2,3,4,5] - ## i.delete(2) #=> @[1, 2, 4, 5] - template defaultImpl = - let xl = x.len - for j in i..xl-2: shallowCopy(x[j], x[j+1]) - setLen(x, xl-1) + ## If the conversion fails, `ValueError` is raised. + ## However, on most platforms the conversion cannot fail. + ## + ## ```nim + ## let + ## a = 2 + ## b = 3.7 + ## + ## echo a.toFloat + b # => 5.7 + ## ``` + float(i) - when nimvm: - defaultImpl() - else: - when defined(js): - {.emit: "`x`[`x`_Idx].splice(`i`, 1);".} - else: - defaultImpl() +proc toBiggestFloat*(i: BiggestInt): BiggestFloat {.noSideEffect, inline.} = + ## Same as `toFloat <#toFloat,int>`_ but for `BiggestInt` to `BiggestFloat`. + BiggestFloat(i) -proc insert*[T](x: var seq[T], item: T, i = 0.Natural) {.noSideEffect.} = - ## inserts `item` into `x` at position `i`. +proc toInt*(f: float): int {.noSideEffect.} = + ## Converts a floating point number `f` into an `int`. ## - ## .. code-block:: nim - ## var i = @[1,2,3,4,5] - ## i.insert(2,4) #=> @[1, 2, 3, 4, 2, 5] - template defaultImpl = - let xl = x.len - setLen(x, xl+1) - var j = xl-1 - while j >= i: - shallowCopy(x[j+1], x[j]) - dec(j) + ## Conversion rounds `f` half away from 0, see + ## `Round half away from zero + ## <https://en.wikipedia.org/wiki/Rounding#Round_half_away_from_zero>`_, + ## as opposed to a type conversion which rounds towards zero. + ## + ## Note that some floating point numbers (e.g. infinity or even 1e19) + ## cannot be accurately converted. + ## ```nim + ## doAssert toInt(0.49) == 0 + ## doAssert toInt(0.5) == 1 + ## doAssert toInt(-0.5) == -1 # rounding is symmetrical + ## ``` + if f >= 0: int(f+0.5) else: int(f-0.5) + +proc toBiggestInt*(f: BiggestFloat): BiggestInt {.noSideEffect.} = + ## Same as `toInt <#toInt,float>`_ but for `BiggestFloat` to `BiggestInt`. + if f >= 0: BiggestInt(f+0.5) else: BiggestInt(f-0.5) + +proc `/`*(x, y: int): float {.inline, noSideEffect.} = + ## Division of integers that results in a float. + ## ```nim + ## echo 7 / 5 # => 1.4 + ## ``` + ## + ## See also: + ## * `div <system.html#div,int,int>`_ + ## * `mod <system.html#mod,int,int>`_ + result = toFloat(x) / toFloat(y) + +{.push stackTrace: off.} + +when defined(js): + proc js_abs[T: SomeNumber](x: T): T {.importc: "Math.abs".} +else: + proc c_fabs(x: cdouble): cdouble {.importc: "fabs", header: "<math.h>".} + proc c_fabsf(x: cfloat): cfloat {.importc: "fabsf", header: "<math.h>".} + +proc abs*[T: float64 | float32](x: T): T {.noSideEffect, inline.} = when nimvm: - defaultImpl() + if x < 0.0: result = -x + elif x == 0.0: result = 0.0 # handle 0.0, -0.0 + else: result = x # handle NaN, > 0 else: - when defined(js): - var it : T - {.emit: "`x`[`x`_Idx].splice(`i`, 0, `it`);".} + when defined(js): result = js_abs(x) else: - defaultImpl() - x[i] = item + when T is float64: + result = c_fabs(x) + else: + result = c_fabsf(x) -proc repr*[T](x: T): string {.magic: "Repr", noSideEffect.} - ## takes any Nim variable and returns its string representation. It - ## works even for complex data graphs with cycles. This is a great - ## debugging tool. +func abs*(x: int): int {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int8): int8 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int16): int16 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int32): int32 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int64): int64 {.magic: "AbsI", inline.} = + ## Returns the absolute value of `x`. ## - ## .. code-block:: nim - ## var s: seq[string] = @["test2","test2"] - ## var i = @[1,2,3,4,5] - ## repr(s) #=> 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] - ## repr(i) #=> 0x1055ed050[1, 2, 3, 4, 5] + ## If `x` is `low(x)` (that is -MININT for its type), + ## an overflow exception is thrown (if overflow checking is turned on). + result = if x < 0: -x else: x -type - ByteAddress* = int - ## is the signed integer type that should be used for converting - ## pointers to integer addresses for readability. +{.pop.} # stackTrace: off - BiggestInt* = int64 - ## is an alias for the biggest signed integer type the Nim compiler - ## supports. Currently this is ``int64``, but it is platform-dependant - ## in general. - - BiggestFloat* = float64 - ## is an alias for the biggest floating point type the Nim - ## compiler supports. Currently this is ``float64``, but it is - ## platform-dependant in general. +when not defined(nimPreviewSlimSystem): + proc addQuitProc*(quitProc: proc() {.noconv.}) {. + importc: "atexit", header: "<stdlib.h>", deprecated: "use exitprocs.addExitProc".} + ## Adds/registers a quit procedure. + ## + ## Each call to `addQuitProc` registers another quit procedure. Up to 30 + ## procedures can be registered. They are executed on a last-in, first-out + ## basis (that is, the last function registered is the first to be executed). + ## `addQuitProc` raises an EOutOfIndex exception if `quitProc` cannot be + ## registered. + # Support for addQuitProc() is done by Ansi C's facilities here. + # In case of an unhandled exception the exit handlers should + # not be called explicitly! The user may decide to do this manually though. -{.deprecated: [TAddress: ByteAddress].} +proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} + ## Swaps the values `a` and `b`. + ## + ## This is often more efficient than `tmp = a; a = b; b = tmp`. + ## Particularly useful for sorting algorithms. + ## + ## ```nim + ## var + ## a = 5 + ## b = 9 + ## + ## swap(a, b) + ## + ## assert a == 9 + ## assert b == 5 + ## ``` -when defined(windows): - type - clong* {.importc: "long", nodecl.} = int32 - ## This is the same as the type ``long`` in *C*. - culong* {.importc: "unsigned long", nodecl.} = uint32 - ## This is the same as the type ``unsigned long`` in *C*. -else: - type - clong* {.importc: "long", nodecl.} = int - ## This is the same as the type ``long`` in *C*. - culong* {.importc: "unsigned long", nodecl.} = uint - ## This is the same as the type ``unsigned long`` in *C*. - -type # these work for most platforms: - cchar* {.importc: "char", nodecl.} = char - ## This is the same as the type ``char`` in *C*. - cschar* {.importc: "signed char", nodecl.} = int8 - ## This is the same as the type ``signed char`` in *C*. - cshort* {.importc: "short", nodecl.} = int16 - ## This is the same as the type ``short`` in *C*. - cint* {.importc: "int", nodecl.} = int32 - ## This is the same as the type ``int`` in *C*. - csize* {.importc: "size_t", nodecl.} = int - ## This is the same as the type ``size_t`` in *C*. - clonglong* {.importc: "long long", nodecl.} = int64 - ## This is the same as the type ``long long`` in *C*. - cfloat* {.importc: "float", nodecl.} = float32 - ## This is the same as the type ``float`` in *C*. - cdouble* {.importc: "double", nodecl.} = float64 - ## This is the same as the type ``double`` in *C*. - clongdouble* {.importc: "long double", nodecl.} = BiggestFloat - ## This is the same as the type ``long double`` in *C*. - ## This C type is not supported by Nim's code generator - - cuchar* {.importc: "unsigned char", nodecl.} = char - ## This is the same as the type ``unsigned char`` in *C*. - cushort* {.importc: "unsigned short", nodecl.} = uint16 - ## This is the same as the type ``unsigned short`` in *C*. - cuint* {.importc: "unsigned int", nodecl.} = uint32 - ## This is the same as the type ``unsigned int`` in *C*. - culonglong* {.importc: "unsigned long long", nodecl.} = uint64 - ## This is the same as the type ``unsigned long long`` in *C*. - - cstringArray* {.importc: "char**", nodecl.} = ptr - array [0..ArrayDummySize, cstring] - ## This is binary compatible to the type ``char**`` in *C*. The array's - ## high value is large enough to disable bounds checking in practice. - ## Use `cstringArrayToSeq` to convert it into a ``seq[string]``. - - PFloat32* = ptr float32 ## an alias for ``ptr float32`` - PFloat64* = ptr float64 ## an alias for ``ptr float64`` - PInt64* = ptr int64 ## an alias for ``ptr int64`` - PInt32* = ptr int32 ## an alias for ``ptr int32`` - -proc toFloat*(i: int): float {. - magic: "ToFloat", noSideEffect, importc: "toFloat".} - ## converts an integer `i` into a ``float``. If the conversion - ## fails, `EInvalidValue` is raised. However, on most platforms the - ## conversion cannot fail. - -proc toBiggestFloat*(i: BiggestInt): BiggestFloat {. - magic: "ToBiggestFloat", noSideEffect, importc: "toBiggestFloat".} - ## converts an biggestint `i` into a ``biggestfloat``. If the conversion - ## fails, `EInvalidValue` is raised. However, on most platforms the - ## conversion cannot fail. - -proc toInt*(f: float): int {. - magic: "ToInt", noSideEffect, importc: "toInt".} - ## converts a floating point number `f` into an ``int``. Conversion - ## rounds `f` if it does not contain an integer value. If the conversion - ## fails (because `f` is infinite for example), `EInvalidValue` is raised. - -proc toBiggestInt*(f: BiggestFloat): BiggestInt {. - magic: "ToBiggestInt", noSideEffect, importc: "toBiggestInt".} - ## converts a biggestfloat `f` into a ``biggestint``. Conversion - ## rounds `f` if it does not contain an integer value. If the conversion - ## fails (because `f` is infinite for example), `EInvalidValue` is raised. - -proc addQuitProc*(QuitProc: proc() {.noconv.}) {. - importc: "atexit", header: "<stdlib.h>".} - ## Adds/registers a quit procedure. - ## - ## Each call to ``addQuitProc`` registers another quit procedure. Up to 30 - ## procedures can be registered. They are executed on a last-in, first-out - ## basis (that is, the last function registered is the first to be executed). - ## ``addQuitProc`` raises an EOutOfIndex exception if ``QuitProc`` cannot be - ## registered. - -# Support for addQuitProc() is done by Ansi C's facilities here. -# In case of an unhandled exeption the exit handlers should -# not be called explicitly! The user may decide to do this manually though. - -proc copy*(s: string, first = 0): string {. - magic: "CopyStr", importc: "copyStr", noSideEffect, deprecated.} -proc copy*(s: string, first, last: int): string {. - magic: "CopyStrLast", importc: "copyStrLast", noSideEffect, - deprecated.} - ## copies a slice of `s` into a new string and returns this new - ## string. The bounds `first` and `last` denote the indices of - ## the first and last characters that shall be copied. If ``last`` - ## is omitted, it is treated as ``high(s)``. - ## **Deprecated since version 0.8.12**: Use ``substr`` instead. - -proc substr*(s: string, first = 0): string {. - magic: "CopyStr", importc: "copyStr", noSideEffect.} -proc substr*(s: string, first, last: int): string {. - magic: "CopyStrLast", importc: "copyStrLast", noSideEffect.} - ## copies a slice of `s` into a new string and returns this new - ## string. The bounds `first` and `last` denote the indices of - ## the first and last characters that shall be copied. If ``last`` - ## is omitted, it is treated as ``high(s)``. If ``last >= s.len``, ``s.len`` - ## is used instead: This means ``substr`` can also be used to `cut`:idx: - ## or `limit`:idx: a string's length. +when not defined(js) and not defined(booting) and defined(nimTrMacros): + template swapRefsInArray*{swap(arr[a], arr[b])}(arr: openArray[ref], a, b: int) = + # Optimize swapping of array elements if they are refs. Default swap + # implementation will cause unsureAsgnRef to be emitted which causes + # unnecessary slow down in this case. + swap(cast[ptr pointer](addr arr[a])[], cast[ptr pointer](addr arr[b])[]) when not defined(nimscript): - proc zeroMem*(p: pointer, size: Natural) {.importc, noDecl, benign.} - ## overwrites the contents of the memory at ``p`` with the value 0. - ## Exactly ``size`` bytes will be overwritten. Like any procedure - ## dealing with raw memory this is *unsafe*. - - proc copyMem*(dest, source: pointer, size: Natural) {. - importc: "memcpy", header: "<string.h>", benign.} - ## copies the contents from the memory at ``source`` to the memory - ## at ``dest``. Exactly ``size`` bytes will be copied. The memory - ## regions may not overlap. Like any procedure dealing with raw - ## memory this is *unsafe*. - - proc moveMem*(dest, source: pointer, size: Natural) {. - importc: "memmove", header: "<string.h>", benign.} - ## copies the contents from the memory at ``source`` to the memory - ## at ``dest``. Exactly ``size`` bytes will be copied. The memory - ## regions may overlap, ``moveMem`` handles this case appropriately - ## and is thus somewhat more safe than ``copyMem``. Like any procedure - ## dealing with raw memory this is still *unsafe*, though. - - proc equalMem*(a, b: pointer, size: Natural): bool {. - importc: "equalMem", noDecl, noSideEffect.} - ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will - ## be compared. If the blocks are equal, true is returned, false - ## otherwise. Like any procedure dealing with raw memory this is - ## *unsafe*. + {.push stackTrace: off, profiler: off.} - when hasAlloc: - proc alloc*(size: Natural): pointer {.noconv, rtl, tags: [], benign.} - ## allocates a new memory block with at least ``size`` bytes. The - ## block has to be freed with ``realloc(block, 0)`` or - ## ``dealloc(block)``. The block is not initialized, so reading - ## from it before writing to it is undefined behaviour! - ## The allocated memory belongs to its allocating thread! - ## Use `allocShared` to allocate from a shared heap. - proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline, benign.} = - ## allocates a new memory block with at least ``T.sizeof * size`` - ## bytes. The block has to be freed with ``resize(block, 0)`` or - ## ``free(block)``. The block is not initialized, so reading - ## from it before writing to it is undefined behaviour! - ## The allocated memory belongs to its allocating thread! - ## Use `createSharedU` to allocate from a shared heap. - cast[ptr T](alloc(T.sizeof * size)) - proc alloc0*(size: Natural): pointer {.noconv, rtl, tags: [], benign.} - ## allocates a new memory block with at least ``size`` bytes. The - ## block has to be freed with ``realloc(block, 0)`` or - ## ``dealloc(block)``. The block is initialized with all bytes - ## containing zero, so it is somewhat safer than ``alloc``. - ## The allocated memory belongs to its allocating thread! - ## Use `allocShared0` to allocate from a shared heap. - proc create*(T: typedesc, size = 1.Positive): ptr T {.inline, benign.} = - ## allocates a new memory block with at least ``T.sizeof * size`` - ## bytes. The block has to be freed with ``resize(block, 0)`` or - ## ``free(block)``. The block is initialized with all bytes - ## containing zero, so it is somewhat safer than ``createU``. - ## The allocated memory belongs to its allocating thread! - ## Use `createShared` to allocate from a shared heap. - cast[ptr T](alloc0(sizeof(T) * size)) - proc realloc*(p: pointer, newSize: Natural): pointer {.noconv, rtl, tags: [], - benign.} - ## grows or shrinks a given memory block. If p is **nil** then a new - ## memory block is returned. In either way the block has at least - ## ``newSize`` bytes. If ``newSize == 0`` and p is not **nil** - ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to - ## be freed with ``dealloc``. - ## The allocated memory belongs to its allocating thread! - ## Use `reallocShared` to reallocate from a shared heap. - proc resize*[T](p: ptr T, newSize: Natural): ptr T {.inline, benign.} = - ## grows or shrinks a given memory block. If p is **nil** then a new - ## memory block is returned. In either way the block has at least - ## ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is not - ## **nil** ``resize`` calls ``free(p)``. In other cases the block - ## has to be freed with ``free``. The allocated memory belongs to - ## its allocating thread! - ## Use `resizeShared` to reallocate from a shared heap. - cast[ptr T](realloc(p, T.sizeof * newSize)) - proc dealloc*(p: pointer) {.noconv, rtl, tags: [], benign.} - ## frees the memory allocated with ``alloc``, ``alloc0`` or - ## ``realloc``. This procedure is dangerous! If one forgets to - ## free the memory a leak occurs; if one tries to access freed - ## memory (or just freeing it twice!) a core dump may happen - ## or other memory may be corrupted. - ## The freed memory must belong to its allocating thread! - ## Use `deallocShared` to deallocate from a shared heap. - - proc allocShared*(size: Natural): pointer {.noconv, rtl, benign.} - ## allocates a new memory block on the shared heap with at - ## least ``size`` bytes. The block has to be freed with - ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block - ## is not initialized, so reading from it before writing to it is - ## undefined behaviour! - proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline, - benign.} = - ## allocates a new memory block on the shared heap with at - ## least ``T.sizeof * size`` bytes. The block has to be freed with - ## ``resizeShared(block, 0)`` or ``freeShared(block)``. The block - ## is not initialized, so reading from it before writing to it is - ## undefined behaviour! - cast[ptr T](allocShared(T.sizeof * size)) - proc allocShared0*(size: Natural): pointer {.noconv, rtl, benign.} - ## allocates a new memory block on the shared heap with at - ## least ``size`` bytes. The block has to be freed with - ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. - ## The block is initialized with all bytes - ## containing zero, so it is somewhat safer than ``allocShared``. - proc createShared*(T: typedesc, size = 1.Positive): ptr T {.inline.} = - ## allocates a new memory block on the shared heap with at - ## least ``T.sizeof * size`` bytes. The block has to be freed with - ## ``resizeShared(block, 0)`` or ``freeShared(block)``. - ## The block is initialized with all bytes - ## containing zero, so it is somewhat safer than ``createSharedU``. - cast[ptr T](allocShared0(T.sizeof * size)) - proc reallocShared*(p: pointer, newSize: Natural): pointer {.noconv, rtl, - benign.} - ## grows or shrinks a given memory block on the heap. If p is **nil** - ## then a new memory block is returned. In either way the block has at - ## least ``newSize`` bytes. If ``newSize == 0`` and p is not **nil** - ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the - ## block has to be freed with ``deallocShared``. - proc resizeShared*[T](p: ptr T, newSize: Natural): ptr T {.inline.} = - ## grows or shrinks a given memory block on the heap. If p is **nil** - ## then a new memory block is returned. In either way the block has at - ## least ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is - ## not **nil** ``resizeShared`` calls ``freeShared(p)``. In other - ## cases the block has to be freed with ``freeShared``. - cast[ptr T](reallocShared(p, T.sizeof * newSize)) - proc deallocShared*(p: pointer) {.noconv, rtl, benign.} - ## frees the memory allocated with ``allocShared``, ``allocShared0`` or - ## ``reallocShared``. This procedure is dangerous! If one forgets to - ## free the memory a leak occurs; if one tries to access freed - ## memory (or just freeing it twice!) a core dump may happen - ## or other memory may be corrupted. - proc freeShared*[T](p: ptr T) {.inline, benign.} = - ## frees the memory allocated with ``createShared``, ``createSharedU`` or - ## ``resizeShared``. This procedure is dangerous! If one forgets to - ## free the memory a leak occurs; if one tries to access freed - ## memory (or just freeing it twice!) a core dump may happen - ## or other memory may be corrupted. - deallocShared(p) + when not defined(nimPreviewSlimSystem): + import std/sysatomics + export sysatomics + else: + import std/sysatomics -proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} - ## swaps the values `a` and `b`. This is often more efficient than - ## ``tmp = a; a = b; b = tmp``. Particularly useful for sorting algorithms. + {.pop.} -template `>=%` *(x, y: expr): expr {.immediate.} = y <=% x - ## treats `x` and `y` as unsigned and compares them. - ## Returns true iff ``unsigned(x) >= unsigned(y)``. +include "system/memalloc" -template `>%` *(x, y: expr): expr {.immediate.} = y <% x - ## treats `x` and `y` as unsigned and compares them. - ## Returns true iff ``unsigned(x) > unsigned(y)``. -proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. ``$`` is Nim's general way of - ## spelling `toString`:idx:. +proc `|`*(a, b: typedesc): typedesc = discard -proc `$`*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. +include "system/iterators_1" -when not defined(nimscript): - when not defined(JS) and hasAlloc: - proc `$` *(x: uint64): string {.noSideEffect.} - ## The stringify operator for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - -proc `$` *(x: float): string {.magic: "FloatToStr", noSideEffect.} - ## The stringify operator for a float argument. Returns `x` - ## converted to a decimal string. - -proc `$` *(x: bool): string {.magic: "BoolToStr", noSideEffect.} - ## The stringify operator for a boolean argument. Returns `x` - ## converted to the string "false" or "true". - -proc `$` *(x: char): string {.magic: "CharToStr", noSideEffect.} - ## The stringify operator for a character argument. Returns `x` - ## converted to a string. - -proc `$` *(x: cstring): string {.magic: "CStrToStr", noSideEffect.} - ## The stringify operator for a CString argument. Returns `x` - ## converted to a string. - -proc `$` *(x: string): string {.magic: "StrToStr", noSideEffect.} - ## The stringify operator for a string argument. Returns `x` - ## as it is. This operator is useful for generic code, so - ## that ``$expr`` also works if ``expr`` is already a string. - -proc `$` *[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} - ## The stringify operator for an enumeration argument. This works for - ## any enumeration type thanks to compiler magic. If - ## a ``$`` operator for a concrete enumeration is provided, this is - ## used instead. (In other words: *Overwriting* is possible.) - -# undocumented: -proc getRefcount*[T](x: ref T): int {.importc: "getRefcount", noSideEffect.} -proc getRefcount*(x: string): int {.importc: "getRefcount", noSideEffect.} -proc getRefcount*[T](x: seq[T]): int {.importc: "getRefcount", noSideEffect.} - ## retrieves the reference count of an heap-allocated object. The - ## value is implementation-dependent. +proc len*[U: Ordinal; V: Ordinal](x: HSlice[U, V]): int {.noSideEffect, inline.} = + ## Length of ordinal slice. When x.b < x.a returns zero length. + ## ```nim + ## assert((0..5).len == 6) + ## assert((5..2).len == 0) + ## ``` + result = max(0, ord(x.b) - ord(x.a) + 1) -const - Inf* {.magic: "Inf".} = 1.0 / 0.0 - ## contains the IEEE floating point value of positive infinity. - NegInf* {.magic: "NegInf".} = -Inf - ## contains the IEEE floating point value of negative infinity. - NaN* {.magic: "NaN".} = 0.0 / 0.0 - ## contains an IEEE floating point value of *Not A Number*. Note - ## that you cannot compare a floating point value to this value - ## and expect a reasonable result - use the `classify` procedure - ## in the module ``math`` for checking for NaN. - NimMajor*: int = 0 - ## is the major number of Nim's version. - - NimMinor*: int = 12 - ## is the minor number of Nim's version. - - NimPatch*: int = 1 - ## is the patch number of Nim's version. +proc isNil*[T](x: ref T): bool {.noSideEffect, magic: "IsNil".} + +proc isNil*[T](x: ptr T): bool {.noSideEffect, magic: "IsNil".} +proc isNil*(x: pointer): bool {.noSideEffect, magic: "IsNil".} +proc isNil*(x: cstring): bool {.noSideEffect, magic: "IsNil".} +proc isNil*[T: proc | iterator {.closure.}](x: T): bool {.noSideEffect, magic: "IsNil".} + ## Fast check whether `x` is nil. This is sometimes more efficient than + ## `== nil`. + +when defined(nimHasTopDownInference): + # magic used for seq type inference + proc `@`*[T](a: openArray[T]): seq[T] {.magic: "OpenArrayToSeq".} = + ## Turns an *openArray* into a sequence. + ## + ## This is not as efficient as turning a fixed length array into a sequence + ## as it always copies every element of `a`. + newSeq(result, a.len) + for i in 0..a.len-1: result[i] = a[i] +else: + proc `@`*[T](a: openArray[T]): seq[T] = + ## Turns an *openArray* into a sequence. + ## + ## This is not as efficient as turning a fixed length array into a sequence + ## as it always copies every element of `a`. + newSeq(result, a.len) + for i in 0..a.len-1: result[i] = a[i] - NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch - ## is the version of Nim as a string. -{.deprecated: [TEndian: Endianness, NimrodVersion: NimVersion, - NimrodMajor: NimMajor, NimrodMinor: NimMinor, NimrodPatch: NimPatch].} +when defined(nimSeqsV2): -# GC interface: + proc `&`*[T](x, y: sink seq[T]): seq[T] {.noSideEffect.} = + ## Concatenates two sequences. + ## + ## Requires copying of the sequences. + ## ```nim + ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], openArray[T]) <#add,seq[T],openArray[T]>`_ + newSeq(result, x.len + y.len) + for i in 0..x.len-1: + result[i] = move(x[i]) + for i in 0..y.len-1: + result[i+x.len] = move(y[i]) + + proc `&`*[T](x: sink seq[T], y: sink T): seq[T] {.noSideEffect.} = + ## Appends element y to the end of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], T) <#add,seq[T],sinkT>`_ + newSeq(result, x.len + 1) + for i in 0..x.len-1: + result[i] = move(x[i]) + result[x.len] = move(y) + + proc `&`*[T](x: sink T, y: sink seq[T]): seq[T] {.noSideEffect.} = + ## Prepends the element x to the beginning of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) + ## ``` + newSeq(result, y.len + 1) + result[0] = move(x) + for i in 0..y.len-1: + result[i+1] = move(y[i]) -when not defined(nimscript) and hasAlloc: - proc getOccupiedMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process and hold data. +else: - proc getFreeMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process, but do not - ## hold any meaningful data. + proc `&`*[T](x, y: seq[T]): seq[T] {.noSideEffect.} = + ## Concatenates two sequences. + ## + ## Requires copying of the sequences. + ## ```nim + ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], openArray[T]) <#add,seq[T],openArray[T]>`_ + newSeq(result, x.len + y.len) + for i in 0..x.len-1: + result[i] = x[i] + for i in 0..y.len-1: + result[i+x.len] = y[i] + + proc `&`*[T](x: seq[T], y: T): seq[T] {.noSideEffect.} = + ## Appends element y to the end of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], T) <#add,seq[T],sinkT>`_ + newSeq(result, x.len + 1) + for i in 0..x.len-1: + result[i] = x[i] + result[x.len] = y + + proc `&`*[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = + ## Prepends the element x to the beginning of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) + ## ``` + newSeq(result, y.len + 1) + result[0] = x + for i in 0..y.len-1: + result[i+1] = y[i] - proc getTotalMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process. - when hasThreadSupport: - proc getOccupiedSharedMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process - ## on the shared heap and hold data. This is only available when - ## threads are enabled. - - proc getFreeSharedMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the - ## process on the shared heap, but do not hold any meaningful data. - ## This is only available when threads are enabled. - - proc getTotalSharedMem*(): int {.rtl.} - ## returns the number of bytes on the shared heap that are owned by the - ## process. This is only available when threads are enabled. - -when sizeof(int) <= 2: - type IntLikeForCount = int|int8|int16|char|bool|uint8|enum +proc instantiationInfo*(index = -1, fullPaths = false): tuple[ + filename: string, line: int, column: int] {.magic: "InstantiationInfo", noSideEffect.} + ## Provides access to the compiler's instantiation stack line information + ## of a template. + ## + ## While similar to the `caller info`:idx: of other languages, it is determined + ## at compile time. + ## + ## This proc is mostly useful for meta programming (eg. `assert` template) + ## to retrieve information about the current filename and line number. + ## Example: + ## + ## ```nim + ## import std/strutils + ## + ## template testException(exception, code: untyped): typed = + ## try: + ## let pos = instantiationInfo() + ## discard(code) + ## echo "Test failure at $1:$2 with '$3'" % [pos.filename, + ## $pos.line, astToStr(code)] + ## assert false, "A test expecting failure succeeded?" + ## except exception: + ## discard + ## + ## proc tester(pos: int): int = + ## let + ## a = @[1, 2, 3] + ## result = a[pos] + ## + ## when isMainModule: + ## testException(IndexDefect, tester(30)) + ## testException(IndexDefect, tester(1)) + ## # --> Test failure at example.nim:20 with 'tester(1)' + ## ``` + + +when notJSnotNims: + import system/ansi_c + import system/memory + + +{.push stackTrace: off.} + +when not defined(js) and hasThreadSupport and hostOS != "standalone": + import std/private/syslocks + include "system/threadlocalstorage" + +when not defined(js) and defined(nimV2): + type + DestructorProc = proc (p: pointer) {.nimcall, benign, raises: [].} + TNimTypeV2 {.compilerproc.} = object + destructor: pointer + size: int + align: int16 + depth: int16 + display: ptr UncheckedArray[uint32] # classToken + when defined(nimTypeNames) or defined(nimArcIds): + name: cstring + traceImpl: pointer + typeInfoV1: pointer # for backwards compat, usually nil + flags: int + when defined(gcDestructors): + when defined(cpp): + vTable: ptr UncheckedArray[pointer] # vtable for types + else: + vTable: UncheckedArray[pointer] # vtable for types + PNimTypeV2 = ptr TNimTypeV2 + +proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} + +when notJSnotNims and defined(nimSeqsV2): + include "system/strs_v2" + include "system/seqs_v2" + +when not defined(js): + template newSeqImpl(T, len) = + result = newSeqOfCap[T](len) + {.cast(noSideEffect).}: + when defined(nimSeqsV2): + cast[ptr int](addr result)[] = len + else: + var s = cast[PGenericSeq](result) + s.len = len + + proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] {.deprecated: "Use `newSeqUninit` instead".} = + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## Only available for numbers types. Note that the sequence will be + ## uninitialized. After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + ## Example: + ## ```nim + ## var x = newSeqUninitialized[int](3) + ## assert len(x) == 3 + ## x[0] = 10 + ## ``` + result = newSeqOfCap[T](len) + when defined(nimSeqsV2): + cast[ptr int](addr result)[] = len + else: + var s = cast[PGenericSeq](result) + s.len = len + + func newSeqUninit*[T](len: Natural): seq[T] = + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## Only available for types, which don't contain + ## managed memory or have destructors. + ## Note that the sequence will be uninitialized. + ## After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + runnableExamples: + var x = newSeqUninit[int](3) + assert len(x) == 3 + x[0] = 10 + when supportsCopyMem(T): + when nimvm: + result = newSeq[T](len) + else: + newSeqImpl(T, len) + else: + {.error: "The type T cannot contain managed memory or have destructors".} + + proc newStringUninit*(len: Natural): string = + ## Returns a new string of length `len` but with uninitialized + ## content. One needs to fill the string character after character + ## with the index operator `s[i]`. + ## + ## This procedure exists only for optimization purposes; + ## the same effect can be achieved with the `&` operator or with `add`. + when nimvm: + result = newString(len) + else: + result = newStringOfCap(len) + when defined(nimSeqsV2): + let s = cast[ptr NimStringV2](addr result) + if len > 0: + s.len = len + s.p.data[len] = '\0' + else: + let s = cast[NimString](result) + s.len = len + s.data[len] = '\0' else: - type IntLikeForCount = int|int8|int16|int32|char|bool|uint8|uint16|enum - -iterator countdown*[T](a, b: T, step = 1): T {.inline.} = - ## Counts from ordinal value `a` down to `b` (inclusive) with the given - ## step count. `T` may be any ordinal type, `step` may only - ## be positive. **Note**: This fails to count to ``low(int)`` if T = int for - ## efficiency reasons. - when T is IntLikeForCount: - var res = int(a) - while res >= int(b): - yield T(res) - dec(res, step) - else: - var res = a - while res >= b: - yield res - dec(res, step) - -template countupImpl(incr: stmt) {.immediate, dirty.} = - when T is IntLikeForCount: - var res = int(a) - while res <= int(b): - yield T(res) - incr - else: - var res: T = T(a) - while res <= b: - yield res - incr - -iterator countup*[S, T](a: S, b: T, step = 1): T {.inline.} = - ## Counts from ordinal value `a` up to `b` (inclusive) with the given - ## step count. `S`, `T` may be any ordinal type, `step` may only - ## be positive. **Note**: This fails to count to ``high(int)`` if T = int for - ## efficiency reasons. - countupImpl: - inc(res, step) - -iterator `..`*[S, T](a: S, b: T): T {.inline.} = - ## An alias for `countup`. - countupImpl: - inc(res) - -iterator `||`*[S, T](a: S, b: T, annotation=""): T {. - inline, magic: "OmpParFor", sideEffect.} = - ## parallel loop iterator. Same as `..` but the loop may run in parallel. - ## `annotation` is an additional annotation for the code generator to use. - ## Note that the compiler maps that to - ## the ``#pragma omp parallel for`` construct of `OpenMP`:idx: and as - ## such isn't aware of the parallelism in your code! Be careful! Later - ## versions of ``||`` will get proper support by Nim's code generator - ## and GC. - discard + proc newStringUninit*(len: Natural): string {. + magic: "NewString", importc: "mnewString", noSideEffect.} -{.push stackTrace:off.} -proc min*(x, y: int): int {.magic: "MinI", noSideEffect.} = - if x <= y: x else: y -proc min*(x, y: int8): int8 {.magic: "MinI", noSideEffect.} = - if x <= y: x else: y -proc min*(x, y: int16): int16 {.magic: "MinI", noSideEffect.} = - if x <= y: x else: y -proc min*(x, y: int32): int32 {.magic: "MinI", noSideEffect.} = - if x <= y: x else: y -proc min*(x, y: int64): int64 {.magic: "MinI", noSideEffect.} = - ## The minimum value of two integers. - if x <= y: x else: y - -proc min*[T](x: varargs[T]): T = - ## The minimum value of `x`. ``T`` needs to have a ``<`` operator. - result = x[0] - for i in 1..high(x): - if x[i] < result: result = x[i] - -proc max*(x, y: int): int {.magic: "MaxI", noSideEffect.} = - if y <= x: x else: y -proc max*(x, y: int8): int8 {.magic: "MaxI", noSideEffect.} = - if y <= x: x else: y -proc max*(x, y: int16): int16 {.magic: "MaxI", noSideEffect.} = - if y <= x: x else: y -proc max*(x, y: int32): int32 {.magic: "MaxI", noSideEffect.} = - if y <= x: x else: y -proc max*(x, y: int64): int64 {.magic: "MaxI", noSideEffect.} = - ## The maximum value of two integers. - if y <= x: x else: y - -proc max*[T](x: varargs[T]): T = - ## The maximum value of `x`. ``T`` needs to have a ``<`` operator. - result = x[0] - for i in 1..high(x): - if result < x[i]: result = x[i] - -proc abs*(x: float): float {.magic: "AbsF64", noSideEffect.} = - if x < 0.0: -x else: x -proc min*(x, y: float): float {.magic: "MinF64", noSideEffect.} = - if x <= y: x else: y -proc max*(x, y: float): float {.magic: "MaxF64", noSideEffect.} = - if y <= x: x else: y {.pop.} -proc clamp*[T](x, a, b: T): T = - ## limits the value ``x`` within the interval [a, b] - ## - ## .. code-block:: Nim - ## assert((1.4).clamp(0.0, 1.0) == 1.0) - ## assert((0.5).clamp(0.0, 1.0) == 0.5) - if x < a: return a - if x > b: return b - return x - -iterator items*[T](a: openArray[T]): T {.inline.} = - ## iterates over each item of `a`. - var i = 0 - while i < len(a): - yield a[i] - inc(i) - -iterator mitems*[T](a: var openArray[T]): var T {.inline.} = - ## iterates over each item of `a` so that you can modify the yielded value. - var i = 0 - while i < len(a): - yield a[i] - inc(i) - -iterator items*[IX, T](a: array[IX, T]): T {.inline.} = - ## iterates over each item of `a`. - var i = low(IX) - if i <= high(IX): - while true: - yield a[i] - if i >= high(IX): break - inc(i) +when not defined(nimscript): + proc writeStackTrace*() {.tags: [], gcsafe, raises: [].} + ## Writes the current stack trace to `stderr`. This is only works + ## for debug builds. Since it's usually used for debugging, this + ## is proclaimed to have no IO effect! -iterator mitems*[IX, T](a: var array[IX, T]): var T {.inline.} = - ## iterates over each item of `a` so that you can modify the yielded value. - var i = low(IX) - if i <= high(IX): - while true: - yield a[i] - if i >= high(IX): break - inc(i) +when not declared(sysFatal): + include "system/fatal" -iterator items*[T](a: set[T]): T {.inline.} = - ## iterates over each element of `a`. `items` iterates only over the - ## elements that are really in the set (and not over the ones the set is - ## able to hold). - var i = low(T).int - while i <= high(T).int: - if T(i) in a: yield T(i) - inc(i) - -iterator items*(a: cstring): char {.inline.} = - ## iterates over each item of `a`. - var i = 0 - while a[i] != '\0': - yield a[i] - inc(i) - -iterator mitems*(a: var cstring): var char {.inline.} = - ## iterates over each item of `a` so that you can modify the yielded value. - var i = 0 - while a[i] != '\0': - yield a[i] - inc(i) - -iterator items*(E: typedesc[enum]): E = - ## iterates over the values of the enum ``E``. - for v in low(E)..high(E): - yield v - -iterator items*[T](s: Slice[T]): T = - ## iterates over the slice `s`, yielding each value between `s.a` and `s.b` - ## (inclusively). - for x in s.a..s.b: - yield x - -iterator pairs*[T](a: openArray[T]): tuple[key: int, val: T] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - var i = 0 - while i < len(a): - yield (i, a[i]) - inc(i) - -iterator mpairs*[T](a: var openArray[T]): tuple[key:int, val:var T]{.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. - var i = 0 - while i < len(a): - yield (i, a[i]) - inc(i) - -iterator pairs*[IX, T](a: array[IX, T]): tuple[key: IX, val: T] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - var i = low(IX) - if i <= high(IX): - while true: - yield (i, a[i]) - if i >= high(IX): break - inc(i) +type + PFrame* = ptr TFrame ## Represents a runtime frame of the call stack; + ## part of the debugger API. + # keep in sync with nimbase.h `struct TFrame_` + TFrame* {.importc, nodecl, final.} = object ## The frame itself. + prev*: PFrame ## Previous frame; used for chaining the call stack. + procname*: cstring ## Name of the proc that is currently executing. + line*: int ## Line number of the proc that is currently executing. + filename*: cstring ## Filename of the proc that is currently executing. + len*: int16 ## Length of the inspectable slots. + calldepth*: int16 ## Used for max call depth checking. + when NimStackTraceMsgs: + frameMsgLen*: int ## end position in frameMsgBuf for this frame. + +when defined(nimV2): + var + framePtr {.threadvar.}: PFrame -iterator mpairs*[IX, T](a:var array[IX, T]):tuple[key:IX,val:var T] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. - var i = low(IX) - if i <= high(IX): - while true: - yield (i, a[i]) - if i >= high(IX): break - inc(i) + include system/arc -iterator pairs*[T](a: seq[T]): tuple[key: int, val: T] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - var i = 0 - while i < len(a): - yield (i, a[i]) - inc(i) - -iterator mpairs*[T](a: var seq[T]): tuple[key: int, val: var T] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. - var i = 0 - while i < len(a): - yield (i, a[i]) - inc(i) - -iterator pairs*(a: string): tuple[key: int, val: char] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - var i = 0 - while i < len(a): - yield (i, a[i]) - inc(i) - -iterator mpairs*(a: var string): tuple[key: int, val: var char] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. - var i = 0 - while i < len(a): - yield (i, a[i]) - inc(i) - -iterator pairs*(a: cstring): tuple[key: int, val: char] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - var i = 0 - while a[i] != '\0': - yield (i, a[i]) - inc(i) - -iterator mpairs*(a: var cstring): tuple[key: int, val: var char] {.inline.} = - ## iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. - var i = 0 - while a[i] != '\0': - yield (i, a[i]) - inc(i) - - -proc isNil*[T](x: seq[T]): bool {.noSideEffect, magic: "IsNil".} -proc isNil*[T](x: ref T): bool {.noSideEffect, magic: "IsNil".} -proc isNil*(x: string): bool {.noSideEffect, magic: "IsNil".} -proc isNil*[T](x: ptr T): bool {.noSideEffect, magic: "IsNil".} -proc isNil*(x: pointer): bool {.noSideEffect, magic: "IsNil".} -proc isNil*(x: cstring): bool {.noSideEffect, magic: "IsNil".} -proc isNil*[T: proc](x: T): bool {.noSideEffect, magic: "IsNil".} - ## Fast check whether `x` is nil. This is sometimes more efficient than - ## ``== nil``. +template newException*(exceptn: typedesc, message: string; + parentException: ref Exception = nil): untyped = + ## Creates an exception object of type `exceptn` and sets its `msg` field + ## to `message`. Returns the new exception object. + (ref exceptn)(msg: message, parent: parentException) -proc `==` *[I, T](x, y: array[I, T]): bool = - for f in low(x)..high(x): - if x[f] != y[f]: - return - result = true - -proc `@`*[T](a: openArray[T]): seq[T] = - ## turns an openarray into a sequence. This is not as efficient as turning - ## a fixed length array into a sequence as it always copies every element - ## of `a`. - newSeq(result, a.len) - for i in 0..a.len-1: result[i] = a[i] - -proc `&` *[T](x, y: seq[T]): seq[T] {.noSideEffect.} = - ## Concatenates two sequences. - ## Requires copying of the sequences. - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) - newSeq(result, x.len + y.len) - for i in 0..x.len-1: - result[i] = x[i] - for i in 0..y.len-1: - result[i+x.len] = y[i] - -proc `&` *[T](x: seq[T], y: T): seq[T] {.noSideEffect.} = - ## Appends element y to the end of the sequence. - ## Requires copying of the sequence - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) - newSeq(result, x.len + 1) - for i in 0..x.len-1: - result[i] = x[i] - result[x.len] = y - -proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = - ## Prepends the element x to the beginning of the sequence. - ## Requires copying of the sequence - ## - ## .. code-block:: Nim - ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) - newSeq(result, y.len + 1) - result[0] = x - for i in 0..y.len-1: - result[i+1] = y[i] +when not defined(nimPreviewSlimSystem): + import std/assertions + export assertions + +import system/iterators +export iterators -when not defined(nimscript): - when not defined(JS): - proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = - result = cast[pointer](x) - else: - proc seqToPtr[T](x: seq[T]): pointer {.asmNoStackFrame, nosideeffect.} = - asm """return `x`""" - - proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} = - ## Generic equals operator for sequences: relies on a equals operator for - ## the element type `T`. - if seqToPtr(x) == seqToPtr(y): - result = true - elif seqToPtr(x) == nil or seqToPtr(y) == nil: - result = false - elif x.len == y.len: - for i in 0..x.len-1: - if x[i] != y[i]: return false - result = true proc find*[T, S](a: T, item: S): int {.inline.}= ## Returns the first index of `item` in `a` or -1 if not found. This requires ## appropriate `items` and `==` operations to work. + result = 0 for i in items(a): if i == item: return inc(result) @@ -2193,79 +1747,47 @@ proc find*[T, S](a: T, item: S): int {.inline.}= proc contains*[T](a: openArray[T], item: T): bool {.inline.}= ## Returns true if `item` is in `a` or false if not found. This is a shortcut - ## for ``find(a, item) >= 0``. + ## for `find(a, item) >= 0`. + ## + ## This allows the `in` operator: `a.contains(item)` is the same as + ## `item in a`. + ## ```nim + ## var a = @[1, 3, 5] + ## assert a.contains(5) + ## assert 3 in a + ## assert 99 notin a + ## ``` return find(a, item) >= 0 proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = - ## returns the last item of `s` and decreases ``s.len`` by one. This treats + ## Returns the last item of `s` and decreases `s.len` by one. This treats ## `s` as a stack and implements the common *pop* operation. + ## + ## Raises `IndexDefect` if `s` is empty. + runnableExamples: + var a = @[1, 3, 5, 7] + let b = pop(a) + assert b == 7 + assert a == @[1, 3, 5] + var L = s.len-1 - result = s[L] - setLen(s, L) - -iterator fields*[T: tuple|object](x: T): RootObj {. - magic: "Fields", noSideEffect.} - ## iterates over every field of `x`. Warning: This really transforms - ## the 'for' and unrolls the loop. The current implementation also has a bug - ## that affects symbol binding in the loop body. -iterator fields*[S:tuple|object, T:tuple|object](x: S, y: T): tuple[a,b: expr] {. - magic: "Fields", noSideEffect.} - ## iterates over every field of `x` and `y`. - ## Warning: This is really transforms the 'for' and unrolls the loop. - ## The current implementation also has a bug that affects symbol binding - ## in the loop body. -iterator fieldPairs*[T: tuple|object](x: T): RootObj {. - magic: "FieldPairs", noSideEffect.} - ## Iterates over every field of `x` returning their name and value. - ## - ## When you iterate over objects with different field types you have to use - ## the compile time ``when`` instead of a runtime ``if`` to select the code - ## you want to run for each type. To perform the comparison use the `is - ## operator <manual.html#is-operator>`_. Example: - ## - ## .. code-block:: Nim - ## - ## type - ## Custom = object - ## foo: string - ## bar: bool - ## - ## proc `$`(x: Custom): string = - ## result = "Custom:" - ## for name, value in x.fieldPairs: - ## when value is bool: - ## result.add("\n\t" & name & " is " & $value) - ## else: - ## if value.isNil: - ## result.add("\n\t" & name & " (nil)") - ## else: - ## result.add("\n\t" & name & " '" & value & "'") - ## - ## Another way to do the same without ``when`` is to leave the task of - ## picking the appropriate code to a secondary proc which you overload for - ## each field type and pass the `value` to. - ## - ## Warning: This really transforms the 'for' and unrolls the loop. The - ## current implementation also has a bug that affects symbol binding in the - ## loop body. -iterator fieldPairs*[S: tuple|object, T: tuple|object](x: S, y: T): tuple[ - a, b: expr] {. - magic: "FieldPairs", noSideEffect.} - ## iterates over every field of `x` and `y`. - ## Warning: This really transforms the 'for' and unrolls the loop. - ## The current implementation also has a bug that affects symbol binding - ## in the loop body. + when defined(nimV2): + result = move s[L] + shrink(s, L) + else: + result = s[L] + setLen(s, L) proc `==`*[T: tuple|object](x, y: T): bool = - ## generic ``==`` operator for tuples that is lifted from the components + ## Generic `==` operator for tuples that is lifted from the components. ## of `x` and `y`. for a, b in fields(x, y): if a != b: return false return true proc `<=`*[T: tuple](x, y: T): bool = - ## generic ``<=`` operator for tuples that is lifted from the components - ## of `x` and `y`. This implementation uses `cmp`. + ## Generic lexicographic `<=` operator for tuples that is lifted from the + ## components of `x` and `y`. This implementation uses `cmp`. for a, b in fields(x, y): var c = cmp(a, b) if c < 0: return true @@ -2273,1230 +1795,862 @@ proc `<=`*[T: tuple](x, y: T): bool = return true proc `<`*[T: tuple](x, y: T): bool = - ## generic ``<`` operator for tuples that is lifted from the components - ## of `x` and `y`. This implementation uses `cmp`. + ## Generic lexicographic `<` operator for tuples that is lifted from the + ## components of `x` and `y`. This implementation uses `cmp`. for a, b in fields(x, y): var c = cmp(a, b) if c < 0: return true if c > 0: return false return false -proc `$`*[T: tuple|object](x: T): string = - ## generic ``$`` operator for tuples that is lifted from the components - ## of `x`. Example: - ## - ## .. code-block:: nim - ## $(23, 45) == "(23, 45)" - ## $() == "()" - result = "(" - var firstElement = true - for name, value in fieldPairs(x): - if not firstElement: result.add(", ") - result.add(name) - result.add(": ") - when compiles(value.isNil): - if value.isNil: result.add "nil" - else: result.add($value) - else: - result.add($value) - firstElement = false - result.add(")") - -proc collectionToString[T: set | seq](x: T, b, e: string): string = - when x is seq: - if x.isNil: return "nil" - result = b - var firstElement = true - for value in items(x): - if not firstElement: result.add(", ") - result.add($value) - firstElement = false - result.add(e) - -proc `$`*[T](x: set[T]): string = - ## generic ``$`` operator for sets that is lifted from the components - ## of `x`. Example: - ## - ## .. code-block:: nim - ## ${23, 45} == "{23, 45}" - collectionToString(x, "{", "}") - -proc `$`*[T](x: seq[T]): string = - ## generic ``$`` operator for seqs that is lifted from the components - ## of `x`. Example: - ## - ## .. code-block:: nim - ## $(@[23, 45]) == "@[23, 45]" - collectionToString(x, "@[", "]") - -when false: - # causes bootstrapping to fail as we use array of chars and cstring should - # match better ... - proc `$`*[T, IDX](x: array[IDX, T]): string = - collectionToString(x, "[", "]") -# ----------------- GC interface --------------------------------------------- - -when not defined(nimscript) and hasAlloc: - proc GC_disable*() {.rtl, inl, benign.} - ## disables the GC. If called n-times, n calls to `GC_enable` are needed to - ## reactivate the GC. Note that in most circumstances one should only disable - ## the mark and sweep phase with `GC_disableMarkAndSweep`. - - proc GC_enable*() {.rtl, inl, benign.} - ## enables the GC again. - - proc GC_fullCollect*() {.rtl, benign.} - ## forces a full garbage collection pass. - ## Ordinary code does not need to call this (and should not). - - type - GC_Strategy* = enum ## the strategy the GC should use for the application - gcThroughput, ## optimize for throughput - gcResponsiveness, ## optimize for responsiveness (default) - gcOptimizeTime, ## optimize for speed - gcOptimizeSpace ## optimize for memory footprint - - {.deprecated: [TGC_Strategy: GC_Strategy].} - - proc GC_setStrategy*(strategy: GC_Strategy) {.rtl, deprecated, benign.} - ## tells the GC the desired strategy for the application. - ## **Deprecated** since version 0.8.14. This has always been a nop. - - proc GC_enableMarkAndSweep*() {.rtl, benign.} - proc GC_disableMarkAndSweep*() {.rtl, benign.} - ## the current implementation uses a reference counting garbage collector - ## with a seldomly run mark and sweep phase to free cycles. The mark and - ## sweep phase may take a long time and is not needed if the application - ## does not create cycles. Thus the mark and sweep phase can be deactivated - ## and activated separately from the rest of the GC. - - proc GC_getStatistics*(): string {.rtl, benign.} - ## returns an informative string about the GC's activity. This may be useful - ## for tweaking. - - proc GC_ref*[T](x: ref T) {.magic: "GCref", benign.} - proc GC_ref*[T](x: seq[T]) {.magic: "GCref", benign.} - proc GC_ref*(x: string) {.magic: "GCref", benign.} - ## marks the object `x` as referenced, so that it will not be freed until - ## it is unmarked via `GC_unref`. If called n-times for the same object `x`, - ## n calls to `GC_unref` are needed to unmark `x`. - - proc GC_unref*[T](x: ref T) {.magic: "GCunref", benign.} - proc GC_unref*[T](x: seq[T]) {.magic: "GCunref", benign.} - proc GC_unref*(x: string) {.magic: "GCunref", benign.} - ## see the documentation of `GC_ref`. - -template accumulateResult*(iter: expr) = - ## helps to convert an iterator to a proc. - result = @[] - for x in iter: add(result, x) +include "system/gc_interface" # we have to compute this here before turning it off in except.nim anyway ... const NimStackTrace = compileOption("stacktrace") +import system/coro_detection + {.push checks: off.} # obviously we cannot generate checking operations here :-) # because it would yield into an endless recursion # however, stack-traces are available for most parts # of the code -var - globalRaiseHook*: proc (e: ref Exception): bool {.nimcall, benign.} - ## with this hook you can influence exception handling on a global level. - ## If not nil, every 'raise' statement ends up calling this hook. Ordinary - ## application code should never set this hook! You better know what you - ## do when setting this. If ``globalRaiseHook`` returns false, the - ## exception is caught and does not propagate further through the call - ## stack. - - localRaiseHook* {.threadvar.}: proc (e: ref Exception): bool {.nimcall, benign.} - ## with this hook you can influence exception handling on a - ## thread local level. - ## If not nil, every 'raise' statement ends up calling this hook. Ordinary - ## application code should never set this hook! You better know what you - ## do when setting this. If ``localRaiseHook`` returns false, the exception - ## is caught and does not propagate further through the call stack. - - outOfMemHook*: proc () {.nimcall, tags: [], benign.} - ## set this variable to provide a procedure that should be called - ## in case of an `out of memory`:idx: event. The standard handler - ## writes an error message and terminates the program. `outOfMemHook` can - ## be used to raise an exception in case of OOM like so: - ## - ## .. code-block:: nim - ## - ## var gOutOfMem: ref EOutOfMemory - ## new(gOutOfMem) # need to be allocated *before* OOM really happened! - ## gOutOfMem.msg = "out of memory" - ## - ## proc handleOOM() = - ## raise gOutOfMem - ## - ## system.outOfMemHook = handleOOM - ## - ## If the handler does not raise an exception, ordinary control flow - ## continues and the program is terminated. +when notJSnotNims: + var + globalRaiseHook*: proc (e: ref Exception): bool {.nimcall, benign.} + ## With this hook you can influence exception handling on a global level. + ## If not nil, every 'raise' statement ends up calling this hook. + ## + ## .. warning:: Ordinary application code should never set this hook! You better know what you do when setting this. + ## + ## If `globalRaiseHook` returns false, the exception is caught and does + ## not propagate further through the call stack. -type - PFrame* = ptr TFrame ## represents a runtime frame of the call stack; - ## part of the debugger API. - TFrame* {.importc, nodecl, final.} = object ## the frame itself - prev*: PFrame ## previous frame; used for chaining the call stack - procname*: cstring ## name of the proc that is currently executing - line*: int ## line number of the proc that is currently executing - filename*: cstring ## filename of the proc that is currently executing - len*: int16 ## length of the inspectable slots - calldepth*: int16 ## used for max call depth checking -#{.deprecated: [TFrame: Frame].} - -when defined(JS): + localRaiseHook* {.threadvar.}: proc (e: ref Exception): bool {.nimcall, benign.} + ## With this hook you can influence exception handling on a + ## thread local level. + ## If not nil, every 'raise' statement ends up calling this hook. + ## + ## .. warning:: Ordinary application code should never set this hook! You better know what you do when setting this. + ## + ## If `localRaiseHook` returns false, the exception + ## is caught and does not propagate further through the call stack. + + outOfMemHook*: proc () {.nimcall, tags: [], benign, raises: [].} + ## Set this variable to provide a procedure that should be called + ## in case of an `out of memory`:idx: event. The standard handler + ## writes an error message and terminates the program. + ## + ## `outOfMemHook` can be used to raise an exception in case of OOM like so: + ## + ## ```nim + ## var gOutOfMem: ref EOutOfMemory + ## new(gOutOfMem) # need to be allocated *before* OOM really happened! + ## gOutOfMem.msg = "out of memory" + ## + ## proc handleOOM() = + ## raise gOutOfMem + ## + ## system.outOfMemHook = handleOOM + ## ``` + ## + ## If the handler does not raise an exception, ordinary control flow + ## continues and the program is terminated. + unhandledExceptionHook*: proc (e: ref Exception) {.nimcall, tags: [], benign, raises: [].} + ## Set this variable to provide a procedure that should be called + ## in case of an `unhandle exception` event. The standard handler + ## writes an error message and terminates the program, except when + ## using `--os:any` + +when defined(js) or defined(nimdoc): proc add*(x: var string, y: cstring) {.asmNoStackFrame.} = - asm """ - var len = `x`[0].length-1; + ## Appends `y` to `x` in place. + runnableExamples: + var tmp = "" + tmp.add(cstring("ab")) + tmp.add(cstring("cd")) + doAssert tmp == "abcd" + {.emit: """ + if (`x` === null) { `x` = []; } + var off = `x`.length; + `x`.length += `y`.length; for (var i = 0; i < `y`.length; ++i) { - `x`[0][len] = `y`.charCodeAt(i); - ++len; + `x`[off+i] = `y`.charCodeAt(i); } - `x`[0][len] = 0 - """ - proc add*(x: var cstring, y: cstring) {.magic: "AppendStrStr".} + """.} + proc add*(x: var cstring, y: cstring) {.magic: "AppendStrStr".} = + ## Appends `y` to `x` in place. + ## Only implemented for JS backend. + runnableExamples: + when defined(js): + var tmp: cstring = "" + tmp.add(cstring("ab")) + tmp.add(cstring("cd")) + doAssert tmp == cstring("abcd") elif hasAlloc: - {.push stack_trace:off, profiler:off.} + {.push stackTrace: off, profiler: off.} proc add*(x: var string, y: cstring) = var i = 0 - while y[i] != '\0': - add(x, y[i]) - inc(i) + if y != nil: + while y[i] != '\0': + add(x, y[i]) + inc(i) {.pop.} -when defined(nimvarargstyped): - proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} - ## Writes and flushes the parameters to the standard output. - ## - ## Special built-in that takes a variable number of arguments. Each argument - ## is converted to a string via ``$``, so it works for user-defined - ## types that have an overloaded ``$`` operator. - ## It is roughly equivalent to ``writeLine(stdout, x); flushFile(stdout)``, but - ## available for the JavaScript target too. - ## - ## Unlike other IO operations this is guaranteed to be thread-safe as - ## ``echo`` is very often used for debugging convenience. If you want to use - ## ``echo`` inside a `proc without side effects - ## <manual.html#pragmas-nosideeffect-pragma>`_ you can use `debugEcho <#debugEcho>`_ - ## instead. - - proc debugEcho*(x: varargs[typed, `$`]) {.magic: "Echo", noSideEffect, - tags: [], raises: [].} - ## Same as `echo <#echo>`_, but as a special semantic rule, ``debugEcho`` - ## pretends to be free of side effects, so that it can be used for debugging - ## routines marked as `noSideEffect <manual.html#pragmas-nosideeffect-pragma>`_. -else: - proc echo*(x: varargs[expr, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} - proc debugEcho*(x: varargs[expr, `$`]) {.magic: "Echo", noSideEffect, - tags: [], raises: [].} +proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", benign, sideEffect.} + ## Writes and flushes the parameters to the standard output. + ## + ## Special built-in that takes a variable number of arguments. Each argument + ## is converted to a string via `$`, so it works for user-defined + ## types that have an overloaded `$` operator. + ## It is roughly equivalent to `writeLine(stdout, x); flushFile(stdout)`, but + ## available for the JavaScript target too. + ## + ## Unlike other IO operations this is guaranteed to be thread-safe as + ## `echo` is very often used for debugging convenience. If you want to use + ## `echo` inside a `proc without side effects + ## <manual.html#pragmas-nosideeffect-pragma>`_ you can use `debugEcho + ## <#debugEcho,varargs[typed,]>`_ instead. + +proc debugEcho*(x: varargs[typed, `$`]) {.magic: "Echo", noSideEffect, + tags: [], raises: [].} + ## Same as `echo <#echo,varargs[typed,]>`_, but as a special semantic rule, + ## `debugEcho` pretends to be free of side effects, so that it can be used + ## for debugging routines marked as `noSideEffect + ## <manual.html#pragmas-nosideeffect-pragma>`_. + +when hostOS == "standalone" and defined(nogc): + proc nimToCStringConv(s: NimString): cstring {.compilerproc, inline.} = + if s == nil or s.len == 0: result = cstring"" + else: result = cast[cstring](addr s.data) -template newException*(exceptn: typedesc, message: string): expr = - ## creates an exception object of type ``exceptn`` and sets its ``msg`` field - ## to `message`. Returns the new exception object. - var - e: ref exceptn - new(e) - e.msg = message - e +proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} + ## Get type information for `x`. + ## + ## Ordinary code should not use this, but the `typeinfo module + ## <typeinfo.html>`_ instead. -when hostOS == "standalone": - include panicoverride -when not declared(sysFatal): - when hostOS == "standalone": - proc sysFatal(exceptn: typedesc, message: string) {.inline.} = - panic(message) +when not defined(js): - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline.} = - rawoutput(message) - panic(arg) + proc likelyProc(val: bool): bool {.importc: "NIM_LIKELY", nodecl, noSideEffect.} + proc unlikelyProc(val: bool): bool {.importc: "NIM_UNLIKELY", nodecl, noSideEffect.} + +template likely*(val: bool): bool = + ## Hints the optimizer that `val` is likely going to be true. + ## + ## You can use this template to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## ```nim + ## for value in inputValues: + ## if likely(value <= 100): + ## process(value) + ## else: + ## echo "Value too big!" + ## ``` + ## + ## On backends without branch prediction (JS and the nimscript VM), this + ## template will not affect code execution. + when nimvm: + val + else: + when defined(js): + val + else: + likelyProc(val) + +template unlikely*(val: bool): bool = + ## Hints the optimizer that `val` is likely going to be false. + ## + ## You can use this proc to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## ```nim + ## for value in inputValues: + ## if unlikely(value > 100): + ## echo "Value too big!" + ## else: + ## process(value) + ## ``` + ## + ## On backends without branch prediction (JS and the nimscript VM), this + ## template will not affect code execution. + when nimvm: + val else: - proc sysFatal(exceptn: typedesc, message: string) {.inline, noReturn.} = - var e: ref exceptn - new(e) - e.msg = message - raise e - - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noReturn.} = - var e: ref exceptn - new(e) - e.msg = message & arg - raise e + when defined(js): + val + else: + unlikelyProc(val) -proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} - ## get type information for `x`. Ordinary code should not use this, but - ## the `typeinfo` module instead. +import system/dollars +export dollars -{.push stackTrace: off.} -proc abs*(x: int): int {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int8): int8 {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int16): int16 {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int32): int32 {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -when defined(nimnomagic64): - proc abs*(x: int64): int64 {.magic: "AbsI", noSideEffect.} = - ## returns the absolute value of `x`. If `x` is ``low(x)`` (that - ## is -MININT for its type), an overflow exception is thrown (if overflow - ## checking is turned on). - if x < 0: -x else: x +when defined(nimAuditDelete): + {.pragma: auditDelete, deprecated: "review this call for out of bounds behavior".} else: - proc abs*(x: int64): int64 {.magic: "AbsI64", noSideEffect.} = - ## returns the absolute value of `x`. If `x` is ``low(x)`` (that - ## is -MININT for its type), an overflow exception is thrown (if overflow - ## checking is turned on). - if x < 0: -x else: x -{.pop.} + {.pragma: auditDelete.} + +proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, systemRaisesDefect, auditDelete.} = + ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position. + ## + ## This is an `O(n)` operation. + ## + ## See also: + ## * `del <#del,seq[T],Natural>`_ for O(1) operation + ## + runnableExamples: + var s = @[1, 2, 3, 4, 5] + s.delete(2) + doAssert s == @[1, 2, 4, 5] + + when not defined(nimAuditDelete): + if i > high(x): + # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring + raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed") + + template defaultImpl = + let xl = x.len + for j in i.int..xl-2: movingCopy(x[j], x[j+1]) + setLen(x, xl-1) + + when nimvm: + defaultImpl() + else: + when defined(js): + {.emit: "`x`.splice(`i`, 1);".} + else: + defaultImpl() + + +const + NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch + ## is the version of Nim as a string. -when not defined(JS): #and not defined(nimscript): - {.push stack_trace: off, profiler:off.} +when not defined(js): + {.push stackTrace: off, profiler: off.} - when not defined(nimscript) and not defined(nogc): - proc initGC() - when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc): - proc initAllocator() {.inline.} + when hasAlloc: + when not defined(gcRegions) and not usesDestructors: + proc initGC() {.gcsafe, raises: [].} proc initStackBottom() {.inline, compilerproc.} = # WARNING: This is very fragile! An array size of 8 does not work on my # Linux 64bit system. -- That's because the stack direction is the other - # way round. - when declared(setStackBottom): - var locals {.volatile.}: pointer + # way around. + when declared(nimGC_setStackBottom): + var locals {.volatile, noinit.}: pointer locals = addr(locals) - setStackBottom(locals) + nimGC_setStackBottom(locals) proc initStackBottomWith(locals: pointer) {.inline, compilerproc.} = # We need to keep initStackBottom around for now to avoid # bootstrapping problems. - when declared(setStackBottom): - setStackBottom(locals) + when declared(nimGC_setStackBottom): + nimGC_setStackBottom(locals) - when hasAlloc: - var - strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic}) + when not usesDestructors: + {.push profiler: off.} + var + strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic}) + {.pop.} + + {.pop.} - when not defined(nimscript): - include "system/ansi_c" - proc cmp(x, y: string): int = - result = int(c_strcmp(x, y)) - else: - proc cmp(x, y: string): int = +when not defined(js): + # ugly hack, see the accompanying .pop for + # the mysterious error message + {.push stackTrace: off, profiler: off.} + +when notJSnotNims: + proc zeroMem(p: pointer, size: Natural) = + nimZeroMem(p, size) + when declared(memTrackerOp): + memTrackerOp("zeroMem", p, size) + proc copyMem(dest, source: pointer, size: Natural) = + nimCopyMem(dest, source, size) + when declared(memTrackerOp): + memTrackerOp("copyMem", dest, size) + proc moveMem(dest, source: pointer, size: Natural) = + c_memmove(dest, source, csize_t(size)) + when declared(memTrackerOp): + memTrackerOp("moveMem", dest, size) + proc equalMem(a, b: pointer, size: Natural): bool = + nimCmpMem(a, b, size) == 0 + proc cmpMem(a, b: pointer, size: Natural): int = + nimCmpMem(a, b, size).int + +when not defined(js) or defined(nimscript): + # nimscript can be defined if config file for js compilation + proc cmp(x, y: string): int = + when nimvm: if x < y: result = -1 elif x > y: result = 1 else: result = 0 - - const pccHack = if defined(pcc): "_" else: "" # Hack for PCC - when not defined(nimscript): - when defined(windows): - # work-around C's sucking abstraction: - # BUGFIX: stdin and stdout should be binary files! - proc setmode(handle, mode: int) {.importc: pccHack & "setmode", - header: "<io.h>".} - proc fileno(f: C_TextFileStar): int {.importc: pccHack & "fileno", - header: "<fcntl.h>".} - var - O_BINARY {.importc: pccHack & "O_BINARY", nodecl.}: int - - # we use binary mode in Windows: - setmode(fileno(c_stdin), O_BINARY) - setmode(fileno(c_stdout), O_BINARY) - - when defined(endb): - proc endbStep() - - # ----------------- IO Part ------------------------------------------------ - when hostOS != "standalone": - type - CFile {.importc: "FILE", header: "<stdio.h>", - final, incompletestruct.} = object - File* = ptr CFile ## The type representing a file handle. - - FileMode* = enum ## The file mode when opening a file. - fmRead, ## Open the file for read access only. - fmWrite, ## Open the file for write access only. - fmReadWrite, ## Open the file for read and write access. - ## If the file does not exist, it will be - ## created. - fmReadWriteExisting, ## Open the file for read and write access. - ## If the file does not exist, it will not be - ## created. - fmAppend ## Open the file for writing only; append data - ## at the end. - - FileHandle* = cint ## type that represents an OS file handle; this is - ## useful for low-level file access - - {.deprecated: [TFile: File, TFileHandle: FileHandle, TFileMode: FileMode].} - - when not defined(nimscript): - # text file handling: - var - stdin* {.importc: "stdin", header: "<stdio.h>".}: File - ## The standard input stream. - stdout* {.importc: "stdout", header: "<stdio.h>".}: File - ## The standard output stream. - stderr* {.importc: "stderr", header: "<stdio.h>".}: File - ## The standard error stream. - - when defined(useStdoutAsStdmsg): - template stdmsg*: File = stdout else: - template stdmsg*: File = stderr - ## Template which expands to either stdout or stderr depending on - ## `useStdoutAsStdmsg` compile-time switch. - - proc open*(f: var File, filename: string, - mode: FileMode = fmRead, bufSize: int = -1): bool {.tags: [], - benign.} - ## Opens a file named `filename` with given `mode`. - ## - ## Default mode is readonly. Returns true iff the file could be opened. - ## This throws no exception if the file could not be opened. - - proc open*(f: var File, filehandle: FileHandle, - mode: FileMode = fmRead): bool {.tags: [], benign.} - ## Creates a ``File`` from a `filehandle` with given `mode`. - ## - ## Default mode is readonly. Returns true iff the file could be opened. - - proc open*(filename: string, - mode: FileMode = fmRead, bufSize: int = -1): File = - ## Opens a file named `filename` with given `mode`. - ## - ## Default mode is readonly. Raises an ``IO`` exception if the file - ## could not be opened. - if not open(result, filename, mode, bufSize): - sysFatal(IOError, "cannot open: ", filename) - - proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. - tags: [], benign.} - ## reopens the file `f` with given `filename` and `mode`. This - ## is often used to redirect the `stdin`, `stdout` or `stderr` - ## file variables. - ## - ## Default mode is readonly. Returns true iff the file could be reopened. - - proc close*(f: File) {.importc: "fclose", header: "<stdio.h>", tags: [].} - ## Closes the file. - - proc endOfFile*(f: File): bool {.tags: [], benign.} - ## Returns true iff `f` is at the end. - - proc readChar*(f: File): char {. - importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} - ## Reads a single character from the stream `f`. - proc flushFile*(f: File) {. - importc: "fflush", header: "<stdio.h>", tags: [WriteIOEffect].} - ## Flushes `f`'s buffer. - - proc readAll*(file: File): TaintedString {.tags: [ReadIOEffect], benign.} - ## Reads all data from the stream `file`. - ## - ## Raises an IO exception in case of an error. It is an error if the - ## current file position is not at the beginning of the file. - - proc readFile*(filename: string): TaintedString {.tags: [ReadIOEffect], benign.} - ## Opens a file named `filename` for reading. - ## - ## Then calls `readAll <#readAll>`_ and closes the file afterwards. - ## Returns the string. Raises an IO exception in case of an error. If - ## you need to call this inside a compile time macro you can use - ## `staticRead <#staticRead>`_. - - proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} - ## Opens a file named `filename` for writing. Then writes the - ## `content` completely to the file and closes the file afterwards. - ## Raises an IO exception in case of an error. - - proc write*(f: File, r: float32) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, i: int) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, i: BiggestInt) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, r: BiggestFloat) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, b: bool) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, c: char) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} - proc write*(f: File, a: varargs[string, `$`]) {.tags: [WriteIOEffect], benign.} - ## Writes a value to the file `f`. May throw an IO exception. - - proc readLine*(f: File): TaintedString {.tags: [ReadIOEffect], benign.} - ## reads a line of text from the file `f`. May throw an IO exception. - ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline - ## character(s) are not part of the returned string. - - proc readLine*(f: File, line: var TaintedString): bool {.tags: [ReadIOEffect], - benign.} - ## reads a line of text from the file `f` into `line`. `line` must not be - ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline - ## character(s) are not part of the returned string. Returns ``false`` - ## if the end of the file has been reached, ``true`` otherwise. If - ## ``false`` is returned `line` contains no new data. - - proc writeLn*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, - tags: [WriteIOEffect], benign, deprecated.} - ## **Deprecated since version 0.11.4:** Use **writeLine** instead. - - proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, - tags: [WriteIOEffect], benign.} - ## writes the values `x` to `f` and then writes "\n". - ## May throw an IO exception. - - proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} - ## retrieves the file size (in bytes) of `f`. - - proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. - tags: [ReadIOEffect], benign.} - ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns - ## the actual number of bytes that have been read which may be less than - ## `len` (if not as many bytes are remaining), but not greater. - - proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. - tags: [ReadIOEffect], benign.} - ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns - ## the actual number of bytes that have been read which may be less than - ## `len` (if not as many bytes are remaining), but not greater. - - proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. - tags: [ReadIOEffect], benign.} - ## reads `len` bytes into the buffer pointed to by `buffer`. Returns - ## the actual number of bytes that have been read which may be less than - ## `len` (if not as many bytes are remaining), but not greater. - - proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. - tags: [WriteIOEffect], benign.} - ## writes the bytes of ``a[start..start+len-1]`` to the file `f`. Returns - ## the number of actual written bytes, which may be less than `len` in case - ## of an error. - - proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. - tags: [WriteIOEffect], benign.} - ## writes the bytes of ``a[start..start+len-1]`` to the file `f`. Returns - ## the number of actual written bytes, which may be less than `len` in case - ## of an error. - - proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. - tags: [WriteIOEffect], benign.} - ## writes the bytes of buffer pointed to by the parameter `buffer` to the - ## file `f`. Returns the number of actual written bytes, which may be less - ## than `len` in case of an error. - - proc setFilePos*(f: File, pos: int64) {.benign.} - ## sets the position of the file pointer that is used for read/write - ## operations. The file's first byte has the index zero. - - proc getFilePos*(f: File): int64 {.benign.} - ## retrieves the current position of the file pointer that is used to - ## read from the file `f`. The file's first byte has the index zero. - - proc getFileHandle*(f: File): FileHandle {.importc: "fileno", - header: "<stdio.h>"} - ## returns the OS file handle of the file ``f``. This is only useful for - ## platform specific programming. - - when not defined(nimfix): - {.deprecated: [fileHandle: getFileHandle].} + when not defined(nimscript): # avoid semantic checking + let minlen = min(x.len, y.len) + result = int(nimCmpMem(x.cstring, y.cstring, cast[csize_t](minlen))) + if result == 0: + result = x.len - y.len when declared(newSeq): proc cstringArrayToSeq*(a: cstringArray, len: Natural): seq[string] = - ## converts a ``cstringArray`` to a ``seq[string]``. `a` is supposed to be - ## of length ``len``. + ## Converts a `cstringArray` to a `seq[string]`. `a` is supposed to be + ## of length `len`. + if a == nil: return @[] newSeq(result, len) for i in 0..len-1: result[i] = $a[i] proc cstringArrayToSeq*(a: cstringArray): seq[string] = - ## converts a ``cstringArray`` to a ``seq[string]``. `a` is supposed to be - ## terminated by ``nil``. + ## Converts a `cstringArray` to a `seq[string]`. `a` is supposed to be + ## terminated by `nil`. + if a == nil: return @[] var L = 0 while a[L] != nil: inc(L) result = cstringArrayToSeq(a, L) - # ------------------------------------------------------------------------- - - when declared(alloc0) and declared(dealloc): - proc allocCStringArray*(a: openArray[string]): cstringArray = - ## creates a NULL terminated cstringArray from `a`. The result has to - ## be freed with `deallocCStringArray` after it's not needed anymore. - result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring))) - let x = cast[ptr array[0..ArrayDummySize, string]](a) - for i in 0 .. a.high: - result[i] = cast[cstring](alloc0(x[i].len+1)) - copyMem(result[i], addr(x[i][0]), x[i].len) - - proc deallocCStringArray*(a: cstringArray) = - ## frees a NULL terminated cstringArray. - var i = 0 - while a[i] != nil: - dealloc(a[i]) - inc(i) - dealloc(a) - when not defined(nimscript): - proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, - discardable, benign.} - ## atomic increment of `memLoc`. Returns the value after the operation. +when not defined(js) and declared(alloc0) and declared(dealloc): + proc allocCStringArray*(a: openArray[string]): cstringArray = + ## Creates a NULL terminated cstringArray from `a`. The result has to + ## be freed with `deallocCStringArray` after it's not needed anymore. + result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring))) - proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, - discardable, benign.} - ## atomic decrement of `memLoc`. Returns the value after the operation. + let x = cast[ptr UncheckedArray[string]](a) + for i in 0 .. a.high: + result[i] = cast[cstring](alloc0(x[i].len+1)) + copyMem(result[i], addr(x[i][0]), x[i].len) - include "system/atomics" - - type - PSafePoint = ptr TSafePoint - TSafePoint {.compilerproc, final.} = object - prev: PSafePoint # points to next safe point ON THE STACK - status: int - context: C_JmpBuf - hasRaiseAction: bool - raiseAction: proc (e: ref Exception): bool {.closure.} - SafePoint = TSafePoint - # {.deprecated: [TSafePoint: SafePoint].} + proc deallocCStringArray*(a: cstringArray) = + ## Frees a NULL terminated cstringArray. + var i = 0 + while a[i] != nil: + dealloc(a[i]) + inc(i) + dealloc(a) +when notJSnotNims and not gotoBasedExceptions: + type + PSafePoint = ptr TSafePoint + TSafePoint {.compilerproc, final.} = object + prev: PSafePoint # points to next safe point ON THE STACK + status: int + context: C_JmpBuf + SafePoint = TSafePoint + +when not defined(js): when declared(initAllocator): initAllocator() when hasThreadSupport: - include "system/syslocks" - when hostOS != "standalone": include "system/threads" + when hostOS != "standalone": + include system/threadimpl + when not defined(nimPreviewSlimSystem): + import std/typedthreads + export typedthreads + elif not defined(nogc) and not defined(nimscript): when not defined(useNimRtl) and not defined(createNimRtl): initStackBottom() when declared(initGC): initGC() - when not defined(nimscript): - proc setControlCHook*(hook: proc () {.noconv.} not nil) - ## allows you to override the behaviour of your application when CTRL+C - ## is pressed. Only one such hook is supported. +when notJSnotNims: + proc setControlCHook*(hook: proc () {.noconv.}) + ## Allows you to override the behaviour of your application when CTRL+C + ## is pressed. Only one such hook is supported. + ## Example: + ## + ## ```nim + ## proc ctrlc() {.noconv.} = + ## echo "Ctrl+C fired!" + ## # do clean up stuff + ## quit() + ## + ## setControlCHook(ctrlc) + ## ``` - proc writeStackTrace*() {.tags: [WriteIOEffect].} - ## writes the current stack trace to ``stderr``. This is only works - ## for debug builds. - when hostOS != "standalone": - proc getStackTrace*(): string - ## gets the current stack trace. This only works for debug builds. + when not defined(noSignalHandler) and not defined(useNimRtl): + proc unsetControlCHook*() + ## Reverts a call to setControlCHook. - proc getStackTrace*(e: ref Exception): string - ## gets the stack trace associated with `e`, which is the stack that - ## lead to the ``raise`` statement. This only works for debug builds. + when hostOS != "standalone": + proc getStackTrace*(): string {.gcsafe.} + ## Gets the current stack trace. This only works for debug builds. - {.push stack_trace: off, profiler:off.} - when hostOS == "standalone": - include "system/embedded" - else: - include "system/excpt" - include "system/chcks" + proc getStackTrace*(e: ref Exception): string {.gcsafe.} + ## Gets the stack trace associated with `e`, which is the stack that + ## lead to the `raise` statement. This only works for debug builds. - # we cannot compile this with stack tracing on - # as it would recurse endlessly! - include "system/arithm" - {.pop.} # stack trace - {.pop.} # stack trace + {.push stackTrace: off, profiler: off.} + when defined(memtracker): + include "system/memtracker" - when hostOS != "standalone" and not defined(nimscript): - include "system/dyncalls" - when not defined(nimscript): - include "system/sets" + when hostOS == "standalone": + include "system/embedded" + else: + include "system/excpt" + include "system/chcks" - when defined(gogc): - const GenericSeqSize = (3 * sizeof(int)) - else: - const GenericSeqSize = (2 * sizeof(int)) - - proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = - sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase") - var d: int - var a = cast[ByteAddress](aa) - case n.typ.size - of 1: d = ze(cast[ptr int8](a +% n.offset)[]) - of 2: d = ze(cast[ptr int16](a +% n.offset)[]) - of 4: d = int(cast[ptr int32](a +% n.offset)[]) - else: sysAssert(false, "getDiscriminant: invalid n.typ.size") - return d - - proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = - var discr = getDiscriminant(aa, n) - if discr <% n.len: - result = n.sons[discr] - if result == nil: result = n.sons[n.len] - # n.sons[n.len] contains the ``else`` part (but may be nil) - else: - result = n.sons[n.len] + # we cannot compile this with stack tracing on + # as it would recurse endlessly! + include "system/integerops" + {.pop.} - when hasAlloc: include "system/mmdisp" - {.push stack_trace: off, profiler:off.} - when hasAlloc: include "system/sysstr" - {.pop.} - when hostOS != "standalone": include "system/sysio" - when hasThreadSupport: - when hostOS != "standalone": include "system/channels" - else: - include "system/sysio" +when not defined(js): + # this is a hack: without this when statement, you would get: + # Error: system module needs: nimGCvisit + {.pop.} # stackTrace: off, profiler: off - when declared(open) and declared(close) and declared(readline): - iterator lines*(filename: string): TaintedString {.tags: [ReadIOEffect].} = - ## Iterates over any line in the file named `filename`. - ## - ## If the file does not exist `EIO` is raised. The trailing newline - ## character(s) are removed from the iterated lines. Example: - ## - ## .. code-block:: nim - ## import strutils - ## - ## proc transformLetters(filename: string) = - ## var buffer = "" - ## for line in filename.lines: - ## buffer.add(line.replace("a", "0") & '\x0A') - ## writeFile(filename, buffer) - var f = open(filename, bufSize=8000) - var res = TaintedString(newStringOfCap(80)) - while f.readLine(res): yield res - close(f) - - iterator lines*(f: File): TaintedString {.tags: [ReadIOEffect].} = - ## Iterate over any line in the file `f`. - ## - ## The trailing newline character(s) are removed from the iterated lines. - ## Example: - ## - ## .. code-block:: nim - ## proc countZeros(filename: File): tuple[lines, zeros: int] = - ## for line in filename.lines: - ## for letter in line: - ## if letter == '0': - ## result.zeros += 1 - ## result.lines += 1 - var res = TaintedString(newStringOfCap(80)) - while f.readLine(res): yield res - - when not defined(nimscript) and hasAlloc: - include "system/assign" - include "system/repr" - when hostOS != "standalone" and not defined(nimscript): - proc getCurrentException*(): ref Exception {.compilerRtl, inl, benign.} = - ## retrieves the current exception; if there is none, nil is returned. - result = currException - - proc getCurrentExceptionMsg*(): string {.inline, benign.} = - ## retrieves the error message that was attached to the current - ## exception; if there is none, "" is returned. - var e = getCurrentException() - return if e == nil: "" else: e.msg - - proc onRaise*(action: proc(e: ref Exception): bool{.closure.}) = - ## can be used in a ``try`` statement to setup a Lisp-like - ## `condition system`:idx:\: This prevents the 'raise' statement to - ## raise an exception but instead calls ``action``. - ## If ``action`` returns false, the exception has been handled and - ## does not propagate further through the call stack. - if not isNil(excHandler): - excHandler.hasRaiseAction = true - excHandler.raiseAction = action - - proc setCurrentException*(exc: ref Exception) {.inline, benign.} = - ## sets the current exception. - ## - ## **Warning**: Only use this if you know what you are doing. - currException = exc - {.push stack_trace: off, profiler:off.} - when defined(endb) and not defined(nimscript): - include "system/debugger" +when notJSnotNims: + when hostOS != "standalone" and hostOS != "any": + type + LibHandle = pointer # private type + ProcAddr = pointer # library loading and loading of procs: - when defined(profiler) or defined(memProfiler): - include "system/profiler" - {.pop.} # stacktrace + proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} + proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} - when not defined(nimscript): - proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} - ## Hints the optimizer that `val` is likely going to be true. - ## - ## You can use this proc to decorate a branch condition. On certain - ## platforms this can help the processor predict better which branch is - ## going to be run. Example: - ## - ## .. code-block:: nim - ## for value in inputValues: - ## if likely(value <= 100): - ## process(value) - ## else: - ## echo "Value too big!" - - proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} - ## Hints the optimizer that `val` is likely going to be false. - ## - ## You can use this proc to decorate a branch condition. On certain - ## platforms this can help the processor predict better which branch is - ## going to be run. Example: - ## - ## .. code-block:: nim - ## for value in inputValues: - ## if unlikely(value > 100): - ## echo "Value too big!" - ## else: - ## process(value) - - proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## retrieves the raw proc pointer of the closure `x`. This is - ## useful for interfacing closures with C. - {.emit: """ - `result` = `x`.ClPrc; - """.} - - proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## retrieves the raw environment pointer of the closure `x`. This is - ## useful for interfacing closures with C. - {.emit: """ - `result` = `x`.ClEnv; - """.} - - proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = - ## can be used to determine if a first class iterator has finished. - {.emit: """ - `result` = *((NI*) `x`.ClEnv) < 0; - """.} - -elif defined(JS): - # Stubs: - proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = discard - - proc GC_disable() = discard - proc GC_enable() = discard - proc GC_fullCollect() = discard - proc GC_setStrategy(strategy: GC_Strategy) = discard - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - proc GC_getStatistics(): string = return "" - - proc getOccupiedMem(): int = return -1 - proc getFreeMem(): int = return -1 - proc getTotalMem(): int = return -1 - - proc dealloc(p: pointer) = discard - proc alloc(size: Natural): pointer = discard - proc alloc0(size: Natural): pointer = discard - proc realloc(p: pointer, newsize: Natural): pointer = discard - - proc allocShared(size: Natural): pointer = discard - proc allocShared0(size: Natural): pointer = discard - proc deallocShared(p: pointer) = discard - proc reallocShared(p: pointer, newsize: Natural): pointer = discard - - when defined(JS): - include "system/jssys" - include "system/reprjs" - elif defined(nimscript): - proc cmp(x, y: string): int = - if x == y: return 0 - if x < y: return -1 - return 1 - - when defined(nimffi): - include "system/sysio" - - -proc quit*(errormsg: string, errorcode = QuitFailure) {.noReturn.} = - ## a shorthand for ``echo(errormsg); quit(errorcode)``. - echo(errormsg) - quit(errorcode) + proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} -{.pop.} # checks -{.pop.} # hints + include "system/dyncalls" -proc `/`*(x, y: int): float {.inline, noSideEffect.} = - ## integer division that results in a float. - result = toFloat(x) / toFloat(y) + import system/countbits_impl + include "system/sets" -template spliceImpl(s, a, L, b: expr): stmt {.immediate.} = - # make room for additional elements or cut: - var slen = s.len - var shift = b.len - L - var newLen = slen + shift - if shift > 0: - # enlarge: - setLen(s, newLen) - for i in countdown(newLen-1, a+shift+1): shallowCopy(s[i], s[i-shift]) + when defined(gogc): + const GenericSeqSize = (3 * sizeof(int)) else: - for i in countup(a+b.len, s.len-1+shift): shallowCopy(s[i], s[i-shift]) - # cut down: - setLen(s, newLen) - # fill the hole: - for i in 0 .. <b.len: s[i+a] = b[i] - -when hasAlloc or defined(nimscript): - proc `[]`*(s: string, x: Slice[int]): string {.inline.} = - ## slice operation for strings. - result = s.substr(x.a, x.b) - - proc `[]=`*(s: var string, x: Slice[int], b: string) = - ## slice assignment for strings. If - ## ``b.len`` is not exactly the number of elements that are referred to - ## by `x`, a `splice`:idx: is performed: - ## - ## .. code-block:: nim - ## var s = "abcdef" - ## s[1 .. ^2] = "xyz" - ## assert s == "axyzf" - var a = x.a - var L = x.b - a + 1 - if L == b.len: - for i in 0 .. <L: s[i+a] = b[i] + const GenericSeqSize = (2 * sizeof(int)) + + proc getDiscriminant(aa: pointer, n: ptr TNimNode): uint = + sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase") + var d: uint + var a = cast[uint](aa) + case n.typ.size + of 1: d = uint(cast[ptr uint8](a + uint(n.offset))[]) + of 2: d = uint(cast[ptr uint16](a + uint(n.offset))[]) + of 4: d = uint(cast[ptr uint32](a + uint(n.offset))[]) + of 8: d = uint(cast[ptr uint64](a + uint(n.offset))[]) else: - spliceImpl(s, a, L, b) - -proc `[]`*[Idx, T](a: array[Idx, T], x: Slice[int]): seq[T] = - ## slice operation for arrays. - when low(a) < 0: - {.error: "Slicing for arrays with negative indices is unsupported.".} - var L = x.b - x.a + 1 - newSeq(result, L) - for i in 0.. <L: result[i] = a[i + x.a] - -proc `[]=`*[Idx, T](a: var array[Idx, T], x: Slice[int], b: openArray[T]) = - ## slice assignment for arrays. - when low(a) < 0: - {.error: "Slicing for arrays with negative indices is unsupported.".} - var L = x.b - x.a + 1 - if L == b.len: - for i in 0 .. <L: a[i+x.a] = b[i] - else: - sysFatal(RangeError, "different lengths for slice assignment") - -proc `[]`*[Idx, T](a: array[Idx, T], x: Slice[Idx]): seq[T] = - ## slice operation for arrays. - var L = ord(x.b) - ord(x.a) + 1 - newSeq(result, L) - for i in 0.. <L: - result[i] = a[Idx(ord(x.a) + i)] - -proc `[]=`*[Idx, T](a: var array[Idx, T], x: Slice[Idx], b: openArray[T]) = - ## slice assignment for arrays. - var L = ord(x.b) - ord(x.a) + 1 - if L == b.len: - for i in 0 .. <L: - a[Idx(ord(x.a) + i)] = b[i] - else: - sysFatal(RangeError, "different lengths for slice assignment") - -proc `[]`*[T](s: seq[T], x: Slice[int]): seq[T] = - ## slice operation for sequences. - var a = x.a - var L = x.b - a + 1 - newSeq(result, L) - for i in 0.. <L: result[i] = s[i + a] - -proc `[]=`*[T](s: var seq[T], x: Slice[int], b: openArray[T]) = - ## slice assignment for sequences. If - ## ``b.len`` is not exactly the number of elements that are referred to - ## by `x`, a `splice`:idx: is performed. - var a = x.a - var L = x.b - a + 1 - if L == b.len: - for i in 0 .. <L: s[i+a] = b[i] - else: - spliceImpl(s, a, L, b) + d = 0'u + sysAssert(false, "getDiscriminant: invalid n.typ.size") + return d + + proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = + var discr = getDiscriminant(aa, n) + if discr < cast[uint](n.len): + result = n.sons[discr] + if result == nil: result = n.sons[n.len] + # n.sons[n.len] contains the `else` part (but may be nil) + else: + result = n.sons[n.len] -proc slurp*(filename: string): string {.magic: "Slurp".} - ## This is an alias for `staticRead <#staticRead>`_. +when notJSnotNims and hasAlloc: + {.push profiler: off.} + include "system/mmdisp" + {.pop.} + {.push stackTrace: off, profiler: off.} + when not defined(nimSeqsV2): + include "system/sysstr" + {.pop.} -proc staticRead*(filename: string): string {.magic: "Slurp".} - ## Compile-time `readFile <#readFile>`_ proc for easy `resource`:idx: - ## embedding: - ## - ## .. code-block:: nim - ## const myResource = staticRead"mydatafile.bin" - ## - ## `slurp <#slurp>`_ is an alias for ``staticRead``. + include "system/strmantle" + include "system/assign" -proc gorge*(command: string, input = "", cache = ""): string {. - magic: "StaticExec".} = discard - ## This is an alias for `staticExec <#staticExec>`_. + when not defined(nimV2): + include "system/repr" -proc staticExec*(command: string, input = "", cache = ""): string {. - magic: "StaticExec".} = discard - ## Executes an external process at compile-time. - ## if `input` is not an empty string, it will be passed as a standard input - ## to the executed program. - ## - ## .. code-block:: nim - ## const buildInfo = "Revision " & staticExec("git rev-parse HEAD") & - ## "\nCompiled on " & staticExec("uname -v") - ## - ## `gorge <#gorge>`_ is an alias for ``staticExec``. Note that you can use - ## this proc inside a pragma like `passC <nimc.html#passc-pragma>`_ or `passL - ## <nimc.html#passl-pragma>`_. - ## - ## If ``cache`` is not empty, the results of ``staticExec`` are cached within - ## the ``nimcache`` directory. Use ``--forceBuild`` to get rid of this caching - ## behaviour then. ``command & input & cache`` (the concatenated string) is - ## used to determine wether the entry in the cache is still valid. You can - ## use versioning information for ``cache``: - ## - ## .. code-block:: nim - ## const stateMachine = staticExec("dfaoptimizer", "input", "0.8.0") +when notJSnotNims and hasThreadSupport and hostOS != "standalone": + when not defined(nimPreviewSlimSystem): + include "system/channels_builtin" -proc `+=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {. - magic: "Inc", noSideEffect.} - ## Increments an ordinal -proc `-=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {. - magic: "Dec", noSideEffect.} - ## Decrements an ordinal +when notJSnotNims and hostOS != "standalone": + proc getCurrentException*(): ref Exception {.compilerRtl, inl, benign.} = + ## Retrieves the current exception; if there is none, `nil` is returned. + result = currException -proc `*=`*[T: SomeOrdinal|uint|uint64](x: var T, y: T) {. - inline, noSideEffect.} = - ## Binary `*=` operator for ordinals - x = x * y + proc nimBorrowCurrentException(): ref Exception {.compilerRtl, inl, benign, nodestroy.} = + # .nodestroy here so that we do not produce a write barrier as the + # C codegen only uses it in a borrowed way: + result = currException -proc `+=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Increments in placee a floating point number - x = x + y + proc getCurrentExceptionMsg*(): string {.inline, benign.} = + ## Retrieves the error message that was attached to the current + ## exception; if there is none, `""` is returned. + return if currException == nil: "" else: currException.msg -proc `-=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Decrements in place a floating point number - x = x - y + proc setCurrentException*(exc: ref Exception) {.inline, benign.} = + ## Sets the current exception. + ## + ## .. warning:: Only use this if you know what you are doing. + currException = exc +elif defined(nimscript): + proc getCurrentException*(): ref Exception {.compilerRtl.} = discard + +when notJSnotNims: + {.push stackTrace: off, profiler: off.} + when (defined(profiler) or defined(memProfiler)): + include "system/profiler" + {.pop.} -proc `*=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Multiplies in place a floating point number - x = x * y + proc rawProc*[T: proc {.closure.} | iterator {.closure.}](x: T): pointer {.noSideEffect, inline.} = + ## Retrieves the raw proc pointer of the closure `x`. This is + ## useful for interfacing closures with C/C++, hash computations, etc. + ## If `rawEnv(x)` returns `nil`, the proc which the result points to + ## takes as many parameters as `x`, but with `{.nimcall.}` as its calling + ## convention instead of `{.closure.}`, otherwise it takes one more parameter + ## which is a `pointer`, and it still has `{.nimcall.}` as its calling convention. + ## To invoke the resulted proc, what this returns has to be casted into a `proc`, + ## not a `ptr proc`, and, in a case where `rawEnv(x)` returns non-`nil`, + ## the last and additional argument has to be the result of `rawEnv(x)`. + ## This is not available for the JS target. + #[ + The conversion from function pointer to `void*` is a tricky topic, but this + should work at least for c++ >= c++11, e.g. for `dlsym` support. + refs: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57869, + https://stackoverflow.com/questions/14125474/casts-between-pointer-to-function-and-pointer-to-object-in-c-and-c + ]# + runnableExamples: + proc makeClosure(x: int): (proc(y: int): int) = + var n = x + result = ( + proc(y: int): int = + n += y + return n + ) -proc `/=`*(x: var float64, y: float64) {.inline, noSideEffect.} = - ## Divides in place a floating point number - x = x / y + var + c1 = makeClosure(10) + e = c1.rawEnv() + p = c1.rawProc() -proc `/=`*[T: float|float32](x: var T, y: T) {.inline, noSideEffect.} = - ## Divides in place a floating point number - x = x / y + if e.isNil(): + let c2 = cast[proc(y: int): int {.nimcall.}](p) + echo c2(2) + else: + let c3 = cast[proc(y: int; env: pointer): int {.nimcall.}](p) + echo c3(3, e) + + {.emit: """ + `result` = (void*)`x`.ClP_0; + """.} + + proc rawEnv*[T: proc {.closure.} | iterator {.closure.}](x: T): pointer {.noSideEffect, inline.} = + ## Retrieves the raw environment pointer of the closure `x`. See also `rawProc`. + ## This is not available for the JS target. + {.emit: """ + `result` = `x`.ClE_0; + """.} + +proc finished*[T: iterator {.closure.}](x: T): bool {.noSideEffect, inline, magic: "Finished".} = + ## It can be used to determine if a first class iterator has finished. + when defined(js): + # TODO: mangle `:state` + {.emit: """ + `result` = (`x`.ClE_0).HEX3Astate < 0; + """.} + else: + {.emit: """ + `result` = ((NI*) `x`.ClE_0)[1] < 0; + """.} -proc `&=`* (x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} +from std/private/digitsutils import addInt +export addInt -proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} - ## converts the AST of `x` into a string representation. This is very useful - ## for debugging. +when defined(js) and not defined(nimscript): + # nimscript can be defined if config file for js compilation + include "system/jssys" + include "system/reprjs" -proc instantiationInfo*(index = -1, fullPaths = false): tuple[ - filename: string, line: int] {. magic: "InstantiationInfo", noSideEffect.} - ## provides access to the compiler's instantiation stack line information. - ## - ## This proc is mostly useful for meta programming (eg. ``assert`` template) - ## to retrieve information about the current filename and line number. - ## Example: - ## - ## .. code-block:: nim - ## import strutils - ## - ## template testException(exception, code: expr): stmt = - ## try: - ## let pos = instantiationInfo() - ## discard(code) - ## echo "Test failure at $1:$2 with '$3'" % [pos.filename, - ## $pos.line, astToStr(code)] - ## assert false, "A test expecting failure succeeded?" - ## except exception: - ## discard - ## - ## proc tester(pos: int): int = - ## let - ## a = @[1, 2, 3] - ## result = a[pos] + +when defined(nimNoQuit): + proc quit*(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(nimdoc): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} + ## Stops the program immediately with an exit code. + ## + ## Before stopping the program the "exit procedures" are called in the + ## opposite order they were added with `addExitProc <exitprocs.html#addExitProc,proc)>`_. + ## + ## The proc `quit(QuitSuccess)` is called implicitly when your nim + ## program finishes without incident for platforms where this is the + ## expected behavior. A raised unhandled exception is + ## equivalent to calling `quit(QuitFailure)`. + ## + ## Note that this is a *runtime* call and using `quit` inside a macro won't + ## have any compile time effect. If you need to stop the compiler inside a + ## macro, use the `error <manual.html#pragmas-error-pragma>`_ or `fatal + ## <manual.html#pragmas-fatal-pragma>`_ pragmas. + ## + ## .. warning:: `errorcode` gets saturated when it exceeds the valid range + ## on the specific platform. On Posix, the valid range is `low(int8)..high(int8)`. + ## On Windows, the valid range is `low(int32)..high(int32)`. For instance, + ## `quit(int(0x100000000))` is equal to `quit(127)` on Linux. + ## + ## .. danger:: In almost all cases, in particular in library code, prefer + ## alternatives, e.g. `raiseAssert` or raise a `Defect`. + ## `quit` bypasses regular control flow in particular `defer`, + ## `try`, `catch`, `finally` and `destructors`, and exceptions that may have been + ## raised by an `addExitProc` proc, as well as cleanup code in other threads. + ## It does *not* call the garbage collector to free all the memory, + ## unless an `addExitProc` proc calls `GC_fullCollect <#GC_fullCollect>`_. + +elif defined(genode): + proc quit*(errorcode: int = QuitSuccess) {.inline, noreturn.} = + rawQuit(errorcode) + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc quit*(errorcode: int = QuitSuccess) {.inline, noreturn.} = + when defined(posix): # posix uses low 8 bits + type ExitCodeRange = int8 + else: # win32 uses low 32 bits + type ExitCodeRange = cint + when sizeof(errorcode) > sizeof(ExitCodeRange): + if errorcode < low(ExitCodeRange): + rawQuit(low(ExitCodeRange).cint) + elif errorcode > high(ExitCodeRange): + rawQuit(high(ExitCodeRange).cint) + else: + rawQuit(errorcode.cint) + else: + rawQuit(errorcode.cint) + +proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = + ## A shorthand for `echo(errormsg); quit(errorcode)`. + when defined(nimscript) or defined(js) or (hostOS == "standalone"): + echo errormsg + else: + when nimvm: + echo errormsg + else: + cstderr.rawWrite(errormsg) + cstderr.rawWrite("\n") + quit(errorcode) + +{.pop.} # checks: off +# {.pop.} # hints: off + +include "system/indices" + +proc `&=`*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} + ## Appends in place to a string. + ## ```nim + ## var a = "abc" + ## a &= "de" # a <- "abcde" + ## ``` + +template `&=`*(x, y: typed) = + ## Generic 'sink' operator for Nim. ## - ## when isMainModule: - ## testException(IndexError, tester(30)) - ## testException(IndexError, tester(1)) - ## # --> Test failure at example.nim:20 with 'tester(1)' + ## If not specialized further, an alias for `add`. + add(x, y) -template currentSourcePath*: string = instantiationInfo(-1, true).filename - ## returns the full file-system path of the current source - -proc raiseAssert*(msg: string) {.noinline.} = - sysFatal(AssertionError, msg) - -proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = - # trick the compiler to not list ``AssertionError`` when called - # by ``assert``. - type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, - tags: [].} - {.deprecated: [THide: Hide].} - Hide(raiseAssert)(msg) - -template assert*(cond: bool, msg = "") = - ## Raises ``AssertionError`` with `msg` if `cond` is false. Note - ## that ``AssertionError`` is hidden from the effect system, so it doesn't - ## produce ``{.raises: [AssertionError].}``. This exception is only supposed - ## to be caught by unit testing frameworks. - ## The compiler may not generate any code at all for ``assert`` if it is - ## advised to do so through the ``-d:release`` or ``--assertions:off`` - ## `command line switches <nimc.html#command-line-switches>`_. - bind instantiationInfo - mixin failedAssertImpl - when compileOption("assertions"): - {.line.}: - if not cond: failedAssertImpl(astToStr(cond) & ' ' & msg) - -template doAssert*(cond: bool, msg = "") = - ## same as `assert` but is always turned on and not affected by the - ## ``--assertions`` command line switch. - bind instantiationInfo - {.line: instantiationInfo().}: - if not cond: - raiseAssert(astToStr(cond) & ' ' & msg) - -iterator items*[T](a: seq[T]): T {.inline.} = - ## iterates over each item of `a`. - var i = 0 - let L = len(a) - while i < L: - yield a[i] - inc(i) - assert(len(a) == L, "seq modified while iterating over it") - -iterator mitems*[T](a: var seq[T]): var T {.inline.} = - ## iterates over each item of `a` so that you can modify the yielded value. - var i = 0 - let L = len(a) - while i < L: - yield a[i] - inc(i) - assert(len(a) == L, "seq modified while iterating over it") - -iterator items*(a: string): char {.inline.} = - ## iterates over each item of `a`. - var i = 0 - let L = len(a) - while i < L: - yield a[i] - inc(i) - assert(len(a) == L, "string modified while iterating over it") - -iterator mitems*(a: var string): var char {.inline.} = - ## iterates over each item of `a` so that you can modify the yielded value. - var i = 0 - let L = len(a) - while i < L: - yield a[i] - inc(i) - assert(len(a) == L, "string modified while iterating over it") - -when not defined(nimhygiene): - {.pragma: inject.} - -template onFailedAssert*(msg: expr, code: stmt): stmt {.dirty, immediate.} = - ## Sets an assertion failure handler that will intercept any assert - ## statements following `onFailedAssert` in the current module scope. - ## - ## .. code-block:: nim - ## # module-wide policy to change the failed assert - ## # exception type in order to include a lineinfo - ## onFailedAssert(msg): - ## var e = new(TMyError) - ## e.msg = msg - ## e.lineinfo = instantiationInfo(-2) - ## raise e - ## - template failedAssertImpl(msgIMPL: string): stmt {.dirty.} = - let msg = msgIMPL - code - -proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = - ## marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not - ## perform deep copies of `s`. This is only useful for optimization - ## purposes. - when not defined(JS) and not defined(nimscript): - var s = cast[PGenericSeq](s) - s.reserved = s.reserved or seqShallowFlag - -proc shallow*(s: var string) {.noSideEffect, inline.} = - ## marks a string `s` as `shallow`:idx:. Subsequent assignments will not - ## perform deep copies of `s`. This is only useful for optimization - ## purposes. - when not defined(JS) and not defined(nimscript): - var s = cast[PGenericSeq](s) - s.reserved = s.reserved or seqShallowFlag +when compileOption("rangechecks"): + template rangeCheck*(cond) = + ## Helper for performing user-defined range checks. + ## Such checks will be performed only when the `rangechecks` + ## compile-time option is enabled. + if not cond: sysFatal(RangeDefect, "range check failed") +else: + template rangeCheck*(cond) = discard + +when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = + ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not + ## perform deep copies of `s`. + ## + ## This is only useful for optimization purposes. + if s.len == 0: return + when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): + var s = cast[PGenericSeq](s) + {.noSideEffect.}: + s.reserved = s.reserved or seqShallowFlag + + proc shallow*(s: var string) {.noSideEffect, inline.} = + ## Marks a string `s` as `shallow`:idx:. Subsequent assignments will not + ## perform deep copies of `s`. + ## + ## This is only useful for optimization purposes. + when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): + var s = cast[PGenericSeq](s) + if s == nil: + s = cast[PGenericSeq](newString(0)) + # string literals cannot become 'shallow': + if (s.reserved and strlitFlag) == 0: + {.noSideEffect.}: + s.reserved = s.reserved or seqShallowFlag type NimNodeObj = object NimNode* {.magic: "PNimrodNode".} = ref NimNodeObj - ## represents a Nim AST node. Macros operate on this type. -{.deprecated: [PNimrodNode: NimNode].} - -when false: - template eval*(blk: stmt): stmt = - ## executes a block of code at compile time just as if it was a macro - ## optionally, the block can return an AST tree that will replace the - ## eval expression - macro payload: stmt {.gensym.} = blk - payload() - -when hasAlloc: + ## Represents a Nim AST node. Macros operate on this type. + +type + ForLoopStmt* {.compilerproc.} = object ## \ + ## A special type that marks a macro as a `for-loop macro`:idx:. + ## See `"For Loop Macro" <manual.html#macros-for-loop-macro>`_. + +macro varargsLen*(x: varargs[untyped]): int {.since: (1, 1).} = + ## returns number of variadic arguments in `x` + proc varargsLenImpl(x: NimNode): NimNode {.magic: "LengthOpenArray", noSideEffect.} + varargsLenImpl(x) + +when defined(nimV2): + import system/repr_v2 + export repr_v2 + +proc repr*[T, U](x: HSlice[T, U]): string = + ## Generic `repr` operator for slices that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(1 .. 5) == "1 .. 5" + ## ``` + result = repr(x.a) + result.add(" .. ") + result.add(repr(x.b)) + +when hasAlloc or defined(nimscript): proc insert*(x: var string, item: string, i = 0.Natural) {.noSideEffect.} = - ## inserts `item` into `x` at position `i`. + ## Inserts `item` into `x` at position `i`. + ## ```nim + ## var a = "abc" + ## a.insert("zz", 0) # a <- "zzabc" + ## ``` + if item.len == 0: # prevents self-assignment + return var xl = x.len setLen(x, xl+item.len) var j = xl-1 while j >= i: - shallowCopy(x[j+item.len], x[j]) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + x[j+item.len] = move x[j] + else: + shallowCopy(x[j+item.len], x[j]) dec(j) j = 0 while j < item.len: x[j+i] = item[j] inc(j) -proc compiles*(x: expr): bool {.magic: "Compiles", noSideEffect, compileTime.} = - ## Special compile-time procedure that checks whether `x` can be compiled - ## without any semantic error. - ## This can be used to check whether a type supports some operation: - ## - ## .. code-block:: Nim - ## when not compiles(3 + 4): - ## echo "'+' for integers is available" - discard - when declared(initDebugger): initDebugger() -when hasAlloc: - # XXX: make these the default (or implement the NilObject optimization) - proc safeAdd*[T](x: var seq[T], y: T) {.noSideEffect.} = - ## Adds ``y`` to ``x`` unless ``x`` is not yet initialized; in that case, - ## ``x`` becomes ``@[y]`` - if x == nil: x = @[y] - else: x.add(y) - - proc safeAdd*(x: var string, y: char) = - ## Adds ``y`` to ``x``. If ``x`` is ``nil`` it is initialized to ``""`` - if x == nil: x = "" - x.add(y) - - proc safeAdd*(x: var string, y: string) = - ## Adds ``y`` to ``x`` unless ``x`` is not yet initalized; in that - ## case, ``x`` becomes ``y`` - if x == nil: x = y - else: x.add(y) +proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = + ## Adds a char to string `s` and applies the following escaping: + ## + ## * replaces any ``\`` by `\\` + ## * replaces any `'` by `\'` + ## * replaces any `"` by `\"` + ## * replaces any `\a` by `\\a` + ## * replaces any `\b` by `\\b` + ## * replaces any `\t` by `\\t` + ## * replaces any `\n` by `\\n` + ## * replaces any `\v` by `\\v` + ## * replaces any `\f` by `\\f` + ## * replaces any `\r` by `\\r` + ## * replaces any `\e` by `\\e` + ## * replaces any other character not in the set `{\21..\126}` + ## by `\xHH` where `HH` is its hexadecimal value + ## + ## The procedure has been designed so that its output is usable for many + ## different common syntaxes. + ## + ## .. warning:: This is **not correct** for producing ANSI C code! + ## + case c + of '\a': s.add "\\a" # \x07 + of '\b': s.add "\\b" # \x08 + of '\t': s.add "\\t" # \x09 + of '\n': s.add "\\n" # \x0A + of '\v': s.add "\\v" # \x0B + of '\f': s.add "\\f" # \x0C + of '\r': (when defined(nimLegacyAddEscapedCharx0D): s.add "\\c" else: s.add "\\r") # \x0D + of '\e': s.add "\\e" # \x1B + of '\\': s.add("\\\\") + of '\'': s.add("\\'") + of '\"': s.add("\\\"") + of {'\32'..'\126'} - {'\\', '\'', '\"'}: s.add(c) + else: + s.add("\\x") + const HexChars = "0123456789ABCDEF" + let n = ord(c) + s.add(HexChars[int((n and 0xF0) shr 4)]) + s.add(HexChars[int(n and 0xF)]) + +proc addQuoted*[T](s: var string, x: T) = + ## Appends `x` to string `s` in place, applying quoting and escaping + ## if `x` is a string or char. + ## + ## See `addEscapedChar <#addEscapedChar,string,char>`_ + ## for the escaping scheme. When `x` is a string, characters in the + ## range `{\128..\255}` are never escaped so that multibyte UTF-8 + ## characters are untouched (note that this behavior is different from + ## `addEscapedChar`). + ## + ## The Nim standard library uses this function on the elements of + ## collections when producing a string representation of a collection. + ## It is recommended to use this function as well for user-side collections. + ## Users may overload `addQuoted` for custom (string-like) types if + ## they want to implement a customized element representation. + ## + ## ```nim + ## var tmp = "" + ## tmp.addQuoted(1) + ## tmp.add(", ") + ## tmp.addQuoted("string") + ## tmp.add(", ") + ## tmp.addQuoted('c') + ## assert(tmp == """1, "string", 'c'""") + ## ``` + when T is string or T is cstring: + s.add("\"") + for c in x: + # Only ASCII chars are escaped to avoid butchering + # multibyte UTF-8 characters. + if c <= 127.char: + s.addEscapedChar(c) + else: + s.add c + s.add("\"") + elif T is char: + s.add("'") + s.addEscapedChar(x) + s.add("'") + # prevent temporary string allocation + elif T is SomeInteger: + s.addInt(x) + elif T is SomeFloat: + s.addFloat(x) + elif compiles(s.add(x)): + s.add(x) + else: + s.add($x) proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} = - ## generates a tuple constructor expression listing all the local variables - ## in the current scope. This is quite fast as it does not rely - ## on any debug or runtime information. Note that in constrast to what - ## the official signature says, the return type is not ``RootObj`` but a + ## Generates a tuple constructor expression listing all the local variables + ## in the current scope. + ## + ## This is quite fast as it does not rely + ## on any debug or runtime information. Note that in contrast to what + ## the official signature says, the return type is *not* `RootObj` but a ## tuple of a structure that depends on the current scope. Example: ## - ## .. code-block:: nim + ## ```nim ## proc testLocals() = ## var ## a = "something" @@ -3511,51 +2665,291 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} = ## # -> name a with value something ## # -> name b with value 4 ## # -> B is 1 + ## ``` discard -when hasAlloc and not defined(nimscript) and not defined(JS): +when hasAlloc and notJSnotNims: + # XXX how to implement 'deepCopy' is an open problem. proc deepCopy*[T](x: var T, y: T) {.noSideEffect, magic: "DeepCopy".} = - ## performs a deep copy of `x`. This is also used by the code generator - ## for the implementation of ``spawn``. + ## Performs a deep copy of `y` and copies it into `x`. + ## + ## This is also used by the code generator + ## for the implementation of `spawn`. + ## + ## For `--mm:arc` or `--mm:orc` deepcopy support has to be enabled + ## via `--deepcopy:on`. discard + proc deepCopy*[T](y: T): T = + ## Convenience wrapper around `deepCopy` overload. + deepCopy(result, y) + include "system/deepcopy" -proc procCall*(x: expr) {.magic: "ProcCall", compileTime.} = - ## special magic to prohibit dynamic binding for `method`:idx: calls. +proc procCall*(x: untyped) {.magic: "ProcCall", compileTime.} = + ## Special magic to prohibit dynamic binding for `method`:idx: calls. ## This is similar to `super`:idx: in ordinary OO languages. - ## - ## .. code-block:: nim + ## ```nim ## # 'someMethod' will be resolved fully statically: ## procCall someMethod(a, b) + ## ``` discard -proc `^`*[T](x: int; y: openArray[T]): int {.noSideEffect, magic: "Roof".} -proc `^`*(x: int): int {.noSideEffect, magic: "Roof".} = - ## builtin `roof`:idx: operator that can be used for convenient array access. - ## ``a[^x]`` is rewritten to ``a[a.len-x]``. However currently the ``a`` - ## expression must not have side effects for this to compile. Note that since - ## this is a builtin, it automatically works for all kinds of - ## overloaded ``[]`` or ``[]=`` accessors. - discard -template `..^`*(a, b: expr): expr = - ## a shortcut for '.. ^' to avoid the common gotcha that a space between - ## '..' and '^' is required. - a .. ^b +proc `==`*(x, y: cstring): bool {.magic: "EqCString", noSideEffect, + inline.} = + ## Checks for equality between two `cstring` variables. + proc strcmp(a, b: cstring): cint {.noSideEffect, + importc, header: "<string.h>".} + if pointer(x) == pointer(y): result = true + elif pointer(x) == nil or pointer(y) == nil: result = false + else: result = strcmp(x, y) == 0 -template `..<`*(a, b: expr): expr = - ## a shortcut for '.. <' to avoid the common gotcha that a space between - ## '..' and '<' is required. - a .. <b +template closureScope*(body: untyped): untyped = + ## Useful when creating a closure in a loop to capture local loop variables by + ## their current iteration values. + ## + ## Note: This template may not work in some cases, use + ## `capture <sugar.html#capture.m,varargs[typed],untyped>`_ instead. + ## + ## Example: + ## + ## ```nim + ## var myClosure : proc() + ## # without closureScope: + ## for i in 0 .. 5: + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 5. `j` is changed after closure creation + ## # with closureScope: + ## for i in 0 .. 5: + ## closureScope: # Everything in this scope is locked after closure creation + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 3 + ## ``` + (proc() = body)() + +template once*(body: untyped): untyped = + ## Executes a block of code only once (the first time the block is reached). + ## ```nim + ## proc draw(t: Triangle) = + ## once: + ## graphicsInit() + ## line(t.p1, t.p2) + ## line(t.p2, t.p3) + ## line(t.p3, t.p1) + ## ``` + var alreadyExecuted {.global.} = false + if not alreadyExecuted: + alreadyExecuted = true + body + +{.pop.} # warning[GcMem]: off, warning[Uninit]: off + +proc substr*(s: openArray[char]): string = + ## Copies a slice of `s` into a new string and returns this new + ## string. + runnableExamples: + let a = "abcdefgh" + assert a.substr(2, 5) == "cdef" + assert a.substr(2) == "cdefgh" + assert a.substr(5, 99) == "fgh" + result = newString(s.len) + for i, ch in s: + result[i] = ch + +proc substr*(s: string, first, last: int): string = # A bug with `magic: Slice` requires this to exist this way + ## Copies a slice of `s` into a new string and returns this new + ## string. + ## + ## The bounds `first` and `last` denote the indices of + ## the first and last characters that shall be copied. If `last` + ## is omitted, it is treated as `high(s)`. If `last >= s.len`, `s.len` + ## is used instead: This means `substr` can also be used to `cut`:idx: + ## or `limit`:idx: a string's length. + runnableExamples: + let a = "abcdefgh" + assert a.substr(2, 5) == "cdef" + assert a.substr(2) == "cdefgh" + assert a.substr(5, 99) == "fgh" -proc xlen*(x: string): int {.magic: "XLenStr", noSideEffect.} = discard -proc xlen*[T](x: seq[T]): int {.magic: "XLenSeq", noSideEffect.} = - ## returns the length of a sequence or a string without testing for 'nil'. - ## This is an optimization that rarely makes sense. - discard + let first = max(first, 0) + let L = max(min(last, high(s)) - first + 1, 0) + result = newString(L) + for i in 0 .. L-1: + result[i] = s[i+first] -{.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.} +proc substr*(s: string, first = 0): string = + result = substr(s, first, high(s)) when defined(nimconfig): include "system/nimscript" + +when not defined(js): + proc toOpenArray*[T](x: ptr UncheckedArray[T]; first, last: int): openArray[T] {. + magic: "Slice".} + proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. + magic: "Slice".} + proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. + magic: "Slice".} + +proc toOpenArray*[T](x: seq[T]; first, last: int): openArray[T] {. + magic: "Slice".} + ## Allows passing the slice of `x` from the element at `first` to the element + ## at `last` to `openArray[T]` parameters without copying it. + ## + ## Example: + ## ```nim + ## proc test(x: openArray[int]) = + ## doAssert x == [1, 2, 3] + ## + ## let s = @[0, 1, 2, 3, 4] + ## s.toOpenArray(1, 3).test + ## ``` + +proc toOpenArray*[T](x: openArray[T]; first, last: int): openArray[T] {. + magic: "Slice".} +proc toOpenArray*[I, T](x: array[I, T]; first, last: I): openArray[T] {. + magic: "Slice".} +proc toOpenArray*(x: string; first, last: int): openArray[char] {. + magic: "Slice".} + +proc toOpenArrayByte*(x: string; first, last: int): openArray[byte] {. + magic: "Slice".} +proc toOpenArrayByte*(x: openArray[char]; first, last: int): openArray[byte] {. + magic: "Slice".} +proc toOpenArrayByte*(x: seq[char]; first, last: int): openArray[byte] {. + magic: "Slice".} + +proc toOpenArrayChar*(x: openArray[byte]; first, last: int): openArray[char] {. + magic: "Slice".} + +when defined(genode): + var componentConstructHook*: proc (env: GenodeEnv) {.nimcall.} + ## Hook into the Genode component bootstrap process. + ## + ## This hook is called after all globals are initialized. + ## When this hook is set the component will not automatically exit, + ## call `quit` explicitly to do so. This is the only available method + ## of accessing the initial Genode environment. + + proc nim_component_construct(env: GenodeEnv) {.exportc.} = + ## Procedure called during `Component::construct` by the loader. + if componentConstructHook.isNil: + env.rawQuit(programResult) + # No native Genode application initialization, + # exit as would POSIX. + else: + componentConstructHook(env) + # Perform application initialization + # and return to thread entrypoint. + + +when not defined(nimPreviewSlimSystem): + import std/widestrs + export widestrs + +when notJSnotNims: + when defined(windows) and compileOption("threads"): + when not declared(addSysExitProc): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} + var echoLock: SysLock + initSysLock echoLock + addSysExitProc(proc() {.noconv.} = deinitSys(echoLock)) + + const stdOutLock = compileOption("threads") and + not defined(windows) and + not defined(android) and + not defined(nintendoswitch) and + not defined(freertos) and + not defined(zephyr) and + not defined(nuttx) and + hostOS != "any" + + proc raiseEIO(msg: string) {.noinline, noreturn.} = + raise newException(IOError, msg) + + proc echoBinSafe(args: openArray[string]) {.compilerproc.} = + when defined(androidNDK): + # When running nim in android app, stdout goes nowhere, so echo gets ignored + # To redirect echo to the android logcat, use -d:androidNDK + const ANDROID_LOG_VERBOSE = 2.cint + proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint + {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.} + var s = "" + for arg in args: + s.add arg + android_log_print(ANDROID_LOG_VERBOSE, "nim", s) + else: + # flockfile deadlocks some versions of Android 5.x.x + when stdOutLock: + proc flockfile(f: CFilePtr) {.importc, nodecl.} + proc funlockfile(f: CFilePtr) {.importc, nodecl.} + flockfile(cstdout) + when defined(windows) and compileOption("threads"): + acquireSys echoLock + for s in args: + when defined(windows): + # equivalent to syncio.writeWindows + proc writeWindows(f: CFilePtr; s: string; doRaise = false) = + # Don't ask why but the 'printf' family of function is the only thing + # that writes utf-8 strings reliably on Windows. At least on my Win 10 + # machine. We also enable `setConsoleOutputCP(65001)` now by default. + # But we cannot call printf directly as the string might contain \0. + # So we have to loop over all the sections separated by potential \0s. + var i = int c_fprintf(f, "%s", s) + while i < s.len: + if s[i] == '\0': + let w = c_fputc('\0', f) + if w != 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i + else: + let w = c_fprintf(f, "%s", unsafeAddr s[i]) + if w <= 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i, w + writeWindows(cstdout, s) + else: + discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, cstdout) + const linefeed = "\n" + discard c_fwrite(linefeed.cstring, linefeed.len, 1, cstdout) + discard c_fflush(cstdout) + when stdOutLock: + funlockfile(cstdout) + when defined(windows) and compileOption("threads"): + releaseSys echoLock + +when not defined(nimPreviewSlimSystem): + import std/syncio + export syncio + +when not defined(createNimHcr) and not defined(nimscript): + include nimhcr + +when notJSnotNims and not defined(nimSeqsV2): + proc prepareMutation*(s: var string) {.inline.} = + ## String literals (e.g. "abc", etc) in the ARC/ORC mode are "copy on write", + ## therefore you should call `prepareMutation` before modifying the strings + ## via `addr`. + runnableExamples: + var x = "abc" + var y = "defgh" + prepareMutation(y) # without this, you may get a `SIGBUS` or `SIGSEGV` + moveMem(addr y[0], addr x[0], x.len) + assert y == "abcgh" + discard + +proc arrayWith*[T](y: T, size: static int): array[size, T] {.raises: [].} = + ## Creates a new array filled with `y`. + for i in 0..size-1: + when nimvm: + result[i] = y + else: + # TODO: fixme it should be `=dup` + result[i] = y diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index b4462ed83..3de6d8713 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -8,213 +8,74 @@ # # Low level allocator for Nim. Has been designed to support the GC. -# TODO: -# - eliminate "used" field -# - make searching for block O(1) {.push profiler:off.} -proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - sysAssert(result >= x, "roundup: result < x") - #return ((-x) and (v-1)) +% x - -sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") -sysAssert(roundup(15, 8) == 16, "roundup broken") -sysAssert(roundup(65, 8) == 72, "roundup broken 2") - -# ------------ platform specific chunk allocation code ----------------------- - -# some platforms have really weird unmap behaviour: unmap(blockStart, PageSize) -# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 -# and x86 are safe though; Windows is special because MEM_RELEASE can only be -# used with a size of 0. We also allow unmapping to be turned off with -# -d:nimAllocNoUnmap: -const doNotUnmap = not (defined(amd64) or defined(i386)) or - defined(windows) or defined(nimAllocNoUnmap) - - -when defined(emscripten): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - type - PEmscriptenMMapBlock = ptr EmscriptenMMapBlock - EmscriptenMMapBlock {.pure, inheritable.} = object - realSize: int # size of previous chunk; for coalescing - realPointer: pointer # if < PageSize it is a small chunk - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "<sys/mman.h>".} - - proc munmap(adr: pointer, len: int) {.header: "<sys/mman.h>".} - - proc osAllocPages(block_size: int): pointer {.inline.} = - let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 - result = mmap(nil, realSize, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - let realPointer = result - let pos = cast[int](result) - - # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) - if (new_pos-pos)< sizeof(EmscriptenMMapBlock): - new_pos = new_pos +% PageSize - result = cast[pointer](new_pos) - - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) - - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - mmapDescr.realSize = realSize - mmapDescr.realPointer = realPointer - - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) - - proc osDeallocPages(p: pointer, size: int) {.inline} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - munmap(mmapDescr.realPointer, mmapDescr.realSize) - -elif defined(posix): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - when defined(macosx) or defined(bsd): - const MAP_ANONYMOUS = 0x1000 - elif defined(solaris): - const MAP_ANONYMOUS = 0x100 - else: - var - MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "<sys/mman.h>".} - - proc munmap(adr: pointer, len: int) {.header: "<sys/mman.h>".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = mmap(nil, size, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline} = - when reallyOsDealloc: munmap(p, size) - -elif defined(windows): - const - MEM_RESERVE = 0x2000 - MEM_COMMIT = 0x1000 - MEM_TOP_DOWN = 0x100000 - PAGE_READWRITE = 0x04 - - MEM_DECOMMIT = 0x4000 - MEM_RELEASE = 0x8000 - - proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, - flProtect: int32): pointer {. - header: "<windows.h>", stdcall, importc: "VirtualAlloc".} - - proc virtualFree(lpAddress: pointer, dwSize: int, - dwFreeType: int32) {.header: "<windows.h>", stdcall, - importc: "VirtualFree".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, - PAGE_READWRITE) - if result == nil: raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - # according to Microsoft, 0 is the only correct value for MEM_RELEASE: - # This means that the OS has some different view over how big the block is - # that we want to free! So, we cannot reliably release the memory back to - # Windows :-(. We have to live with MEM_DECOMMIT instead. - # Well that used to be the case but MEM_DECOMMIT fragments the address - # space heavily, so we now treat Windows as a strange unmap target. - when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) - #VirtualFree(p, size, MEM_DECOMMIT) - -elif hostOS == "standalone": - var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment - bumpPointer = cast[int](addr theHeap) +include osalloc +import std/private/syslocks +import std/sysatomics - proc osAllocPages(size: int): pointer {.inline.} = - if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): - result = cast[pointer](bumpPointer) - inc bumpPointer, size - else: - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - if bumpPointer-size == cast[int](p): - dec bumpPointer, size -else: - {.error: "Port memory manager to your platform".} - -# --------------------- end of non-portable code ----------------------------- +template track(op, address, size) = + when defined(memTracker): + memTrackerOp(op, address, size) # We manage *chunks* of memory. Each chunk is a multiple of the page size. -# Each chunk starts at an address that is divisible by the page size. Chunks -# that are bigger than ``ChunkOsReturn`` are returned back to the operating -# system immediately. +# Each chunk starts at an address that is divisible by the page size. +# Small chunks may be divided into smaller cells of reusable pointers to reduce the number of page allocations. + +# An allocation of a small pointer looks approximately like this +#[ + + alloc -> rawAlloc -> No free chunk available > Request a new page from tslf -> result = chunk.data -------------+ + | | + v | + Free chunk available | + | | + v v + Fetch shared cells -> No free cells available -> Advance acc -> result = chunk.data + chunk.acc -------> return + (may not add new cells) ^ + | | + v | + Free cells available -> result = chunk.freeList -> Advance chunk.freeList -----------------------------------+ +]# +# so it is split into 3 paths, where the last path is preferred to prevent unnecessary allocations. +# +# +# A deallocation of a small pointer then looks like this +#[ + dealloc -> rawDealloc -> chunk.owner == addr(a) --------------> This thread owns the chunk ------> The current chunk is active -> Chunk is completely unused -----> Chunk references no foreign cells + | | (Add cell into the current chunk) | Return the current chunk back to tlsf + | | | | + v v v v + A different thread owns this chunk. The current chunk is not active. chunk.free was < size Chunk references foreign cells, noop + Add the cell to a.sharedFreeLists Add the cell into the active chunk Activate the chunk (end) + (end) (end) (end) +]# +# So "true" deallocation is delayed for as long as possible in favor of reusing cells. const - ChunkOsReturn = 256 * PageSize # 1 MB - InitialMemoryRequest = ChunkOsReturn div 2 # < ChunkOsReturn! + nimMinHeapPages {.intdefine.} = 128 # 0.5 MB SmallChunkSize = PageSize + MaxFli = when sizeof(int) > 2: 30 else: 14 + MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32' + # everywhere! + MaxSli = 1 shl MaxLog2Sli + FliOffset = 6 + RealFli = MaxFli - FliOffset + + # size of chunks in last matrix bin + MaxBigChunkSize = int(1'i32 shl MaxFli - 1'i32 shl (MaxFli-MaxLog2Sli-1)) + HugeChunkSize = MaxBigChunkSize + 1 type PTrunk = ptr Trunk - Trunk {.final.} = object + Trunk = object next: PTrunk # all nodes are connected with this pointer key: int # start address at bit 0 - bits: array[0..IntsPerTrunk-1, int] # a bit vector + bits: array[0..IntsPerTrunk-1, uint] # a bit vector TrunkBuckets = array[0..255, PTrunk] - IntSet {.final.} = object + IntSet = object data: TrunkBuckets -{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkBuckets: TrunkBuckets].} - -type - AlignType = BiggestFloat - FreeCell {.final, pure.} = object - next: ptr FreeCell # next free cell in chunk (overlaid with refcount) - zeroField: int # 0 means cell is not used (overlaid with typ field) - # 1 means cell is manually managed pointer - # otherwise a PNimType is stored in there - - PChunk = ptr BaseChunk - PBigChunk = ptr BigChunk - PSmallChunk = ptr SmallChunk - BaseChunk {.pure, inheritable.} = object - prevSize: int # size of previous chunk; for coalescing - size: int # if < PageSize it is a small chunk - used: bool # later will be optimized into prevSize... - - SmallChunk = object of BaseChunk - next, prev: PSmallChunk # chunks of the same size - freeList: ptr FreeCell - free: int # how many bytes remain - acc: int # accumulator for small object allocation - data: AlignType # start of usable memory - - BigChunk = object of BaseChunk # not necessarily > PageSize! - next, prev: PBigChunk # chunks of the same (or bigger) size - align: int - data: AlignType # start of usable memory -{.deprecated: [TAlignType: AlignType, TFreeCell: FreeCell, TBaseChunk: BaseChunk, - TBigChunk: BigChunk, TSmallChunk: SmallChunk].} - -template smallChunkOverhead(): expr = sizeof(SmallChunk)-sizeof(AlignType) -template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) # ------------- chunk table --------------------------------------------------- # We use a PtrSet of chunk starts and a table[Page, chunksize] for chunk @@ -225,47 +86,226 @@ template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) type PLLChunk = ptr LLChunk - LLChunk {.pure.} = object ## *low-level* chunk + LLChunk = object ## *low-level* chunk size: int # remaining size acc: int # accumulator next: PLLChunk # next low-level chunk; only needed for dealloc PAvlNode = ptr AvlNode - AvlNode {.pure, final.} = object + AvlNode = object link: array[0..1, PAvlNode] # Left (0) and right (1) links key, upperBound: int level: int - MemRegion {.final, pure.} = object - minLargeObj, maxLargeObj: int - freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] +const + RegionHasLock = false # hasThreadSupport and defined(gcDestructors) + +type + FreeCell {.final, pure.} = object + # A free cell is a pointer that has been freed, meaning it became available for reuse. + # It may become foreign if it is lent to a chunk that did not create it, doing so reduces the amount of needed pages. + next: ptr FreeCell # next free cell in chunk (overlaid with refcount) + when not defined(gcDestructors): + zeroField: int # 0 means cell is not used (overlaid with typ field) + # 1 means cell is manually managed pointer + # otherwise a PNimType is stored in there + else: + alignment: int + + PChunk = ptr BaseChunk + PBigChunk = ptr BigChunk + PSmallChunk = ptr SmallChunk + BaseChunk {.pure, inheritable.} = object + prevSize: int # size of previous chunk; for coalescing + # 0th bit == 1 if 'used + size: int # if < PageSize it is a small chunk + owner: ptr MemRegion + + SmallChunk = object of BaseChunk + next, prev: PSmallChunk # chunks of the same size + freeList: ptr FreeCell # Singly linked list of cells. They may be from foreign chunks or from the current chunk. + # Should be `nil` when the chunk isn't active in `a.freeSmallChunks`. + free: int32 # Bytes this chunk is able to provide using both the accumulator and free cells. + # When a cell is considered foreign, its source chunk's free field is NOT adjusted until it + # reaches dealloc while the source chunk is active. + # Instead, the receiving chunk gains the capacity and thus reserves space in the foreign chunk. + acc: uint32 # Offset from data, used when there are no free cells available but the chunk is considered free. + foreignCells: int # When a free cell is given to a chunk that is not its origin, + # both the cell and the source chunk are considered foreign. + # Receiving a foreign cell can happen both when deallocating from another thread or when + # the active chunk in `a.freeSmallChunks` is not the current chunk. + # Freeing a chunk while `foreignCells > 0` leaks memory as all references to it become lost. + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory + + BigChunk = object of BaseChunk # not necessarily > PageSize! + next, prev: PBigChunk # chunks of the same (or bigger) size + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory + + HeapLinks = object + len: int + chunks: array[30, (PBigChunk, int)] + next: ptr HeapLinks + + MemRegion = object + when not defined(gcDestructors): + minLargeObj, maxLargeObj: int + freeSmallChunks: array[0..max(1, SmallChunkSize div MemAlign-1), PSmallChunk] + # List of available chunks per size class. Only one is expected to be active per class. + when defined(gcDestructors): + sharedFreeLists: array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell] + # When a thread frees a pointer it did not create, it must not adjust the counters. + # Instead, the cell is placed here and deferred until the next allocation. + flBitmap: uint32 + slBitmap: array[RealFli, uint32] + matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk - currMem, maxMem, freeMem: int # memory sizes (allocated from OS) + currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly - freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access + when RegionHasLock: + lock: SysLock + when defined(gcDestructors): + sharedFreeListBigChunks: PBigChunk # make no attempt at avoiding false sharing for now for this object field + chunkStarts: IntSet - root, deleted, last, freeAvlNodes: PAvlNode -{.deprecated: [TLLChunk: LLChunk, TAvlNode: AvlNode, TMemRegion: MemRegion].} - -# shared: -var - bottomData: AvlNode - bottom: PAvlNode - -{.push stack_trace: off.} -proc initAllocator() = - when not defined(useNimRtl): - bottom = addr(bottomData) - bottom.link[0] = bottom - bottom.link[1] = bottom -{.pop.} + when not defined(gcDestructors): + root, deleted, last, freeAvlNodes: PAvlNode + lockActive, locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. + nextChunkSize: int + when not defined(gcDestructors): + bottomData: AvlNode + heapLinks: HeapLinks + when defined(nimTypeNames): + allocCounter, deallocCounter: int + +template smallChunkOverhead(): untyped = sizeof(SmallChunk) +template bigChunkOverhead(): untyped = sizeof(BigChunk) + +when hasThreadSupport: + template loada(x: untyped): untyped = atomicLoadN(unsafeAddr x, ATOMIC_RELAXED) + template storea(x, y: untyped) = atomicStoreN(unsafeAddr x, y, ATOMIC_RELAXED) + + when false: + # not yet required + template atomicStatDec(x, diff: untyped) = discard atomicSubFetch(unsafeAddr x, diff, ATOMIC_RELAXED) + template atomicStatInc(x, diff: untyped) = discard atomicAddFetch(unsafeAddr x, diff, ATOMIC_RELAXED) +else: + template loada(x: untyped): untyped = x + template storea(x, y: untyped) = x = y + +template atomicStatDec(x, diff: untyped) = dec x, diff +template atomicStatInc(x, diff: untyped) = inc x, diff + +const + fsLookupTable: array[byte, int8] = [ + -1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7 + ] + +proc msbit(x: uint32): int {.inline.} = + let a = if x <= 0xff_ff'u32: + (if x <= 0xff: 0 else: 8) + else: + (if x <= 0xff_ff_ff'u32: 16 else: 24) + result = int(fsLookupTable[byte(x shr a)]) + a + +proc lsbit(x: uint32): int {.inline.} = + msbit(x and ((not x) + 1)) + +proc setBit(nr: int; dest: var uint32) {.inline.} = + dest = dest or (1u32 shl (nr and 0x1f)) + +proc clearBit(nr: int; dest: var uint32) {.inline.} = + dest = dest and not (1u32 shl (nr and 0x1f)) + +proc mappingSearch(r, fl, sl: var int) {.inline.} = + #let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1 + # This diverges from the standard TLSF algorithm because we need to ensure + # PageSize alignment: + let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1 + r = r + t + r = r and not t + r = min(r, MaxBigChunkSize).int + fl = msbit(uint32 r) + sl = (r shr (fl - MaxLog2Sli)) - MaxSli + dec fl, FliOffset + sysAssert((r and PageMask) == 0, "mappingSearch: still not aligned") + +# See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of +# this algorithm. + +proc mappingInsert(r: int): tuple[fl, sl: int] {.inline.} = + sysAssert((r and PageMask) == 0, "mappingInsert: still not aligned") + result.fl = msbit(uint32 r) + result.sl = (r shr (result.fl - MaxLog2Sli)) - MaxSli + dec result.fl, FliOffset + +template mat(): untyped = a.matrix[fl][sl] + +proc findSuitableBlock(a: MemRegion; fl, sl: var int): PBigChunk {.inline.} = + let tmp = a.slBitmap[fl] and (not 0u32 shl sl) + result = nil + if tmp != 0: + sl = lsbit(tmp) + result = mat() + else: + fl = lsbit(a.flBitmap and (not 0u32 shl (fl + 1))) + if fl > 0: + sl = lsbit(a.slBitmap[fl]) + result = mat() + +template clearBits(sl, fl) = + clearBit(sl, a.slBitmap[fl]) + if a.slBitmap[fl] == 0u32: + # do not forget to cascade: + clearBit(fl, a.flBitmap) + +proc removeChunkFromMatrix(a: var MemRegion; b: PBigChunk) = + let (fl, sl) = mappingInsert(b.size) + if b.next != nil: b.next.prev = b.prev + if b.prev != nil: b.prev.next = b.next + if mat() == b: + mat() = b.next + if mat() == nil: + clearBits(sl, fl) + b.prev = nil + b.next = nil + +proc removeChunkFromMatrix2(a: var MemRegion; b: PBigChunk; fl, sl: int) = + mat() = b.next + if mat() != nil: + mat().prev = nil + else: + clearBits(sl, fl) + b.prev = nil + b.next = nil + +proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) = + let (fl, sl) = mappingInsert(b.size) + b.prev = nil + b.next = mat() + if mat() != nil: + mat().prev = b + mat() = b + setBit(sl, a.slBitmap[fl]) + setBit(fl, a.flBitmap) proc incCurrMem(a: var MemRegion, bytes: int) {.inline.} = - inc(a.currMem, bytes) + atomicStatInc(a.currMem, bytes) proc decCurrMem(a: var MemRegion, bytes: int) {.inline.} = a.maxMem = max(a.maxMem, a.currMem) - dec(a.currMem, bytes) + atomicStatDec(a.currMem, bytes) proc getMaxMem(a: var MemRegion): int = # Since we update maxPagesCount only when freeing pages, @@ -273,6 +313,20 @@ proc getMaxMem(a: var MemRegion): int = # maximum of these both values here: result = max(a.currMem, a.maxMem) +const nimMaxHeap {.intdefine.} = 0 + +proc allocPages(a: var MemRegion, size: int): pointer = + when nimMaxHeap != 0: + if a.occ + size > nimMaxHeap * 1024 * 1024: + raiseOutOfMem() + osAllocPages(size) + +proc tryAllocPages(a: var MemRegion, size: int): pointer = + when nimMaxHeap != 0: + if a.occ + size > nimMaxHeap * 1024 * 1024: + raiseOutOfMem() + osTryAllocPages(size) + proc llAlloc(a: var MemRegion, size: int): pointer = # *low-level* alloc for the memory managers data structures. Deallocation # is done at the end of the allocator's life time. @@ -282,36 +336,67 @@ proc llAlloc(a: var MemRegion, size: int): pointer = # is one page: sysAssert roundup(size+sizeof(LLChunk), PageSize) == PageSize, "roundup 6" var old = a.llmem # can be nil and is correct with nil - a.llmem = cast[PLLChunk](osAllocPages(PageSize)) + a.llmem = cast[PLLChunk](allocPages(a, PageSize)) + when defined(nimAvlcorruption): + trackLocation(a.llmem, PageSize) incCurrMem(a, PageSize) a.llmem.size = PageSize - sizeof(LLChunk) a.llmem.acc = sizeof(LLChunk) a.llmem.next = old - result = cast[pointer](cast[ByteAddress](a.llmem) + a.llmem.acc) + result = cast[pointer](cast[int](a.llmem) + a.llmem.acc) dec(a.llmem.size, size) inc(a.llmem.acc, size) zeroMem(result, size) -proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode = - if a.freeAvlNodes != nil: - result = a.freeAvlNodes - a.freeAvlNodes = a.freeAvlNodes.link[0] +when not defined(gcDestructors): + proc getBottom(a: var MemRegion): PAvlNode = + result = addr(a.bottomData) + if result.link[0] == nil: + result.link[0] = result + result.link[1] = result + + proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode = + if a.freeAvlNodes != nil: + result = a.freeAvlNodes + a.freeAvlNodes = a.freeAvlNodes.link[0] + else: + result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode))) + when defined(nimAvlcorruption): + cprintf("tracking location: %p\n", result) + result.key = key + result.upperBound = upperBound + let bottom = getBottom(a) + result.link[0] = bottom + result.link[1] = bottom + result.level = 1 + #when defined(nimAvlcorruption): + # track("allocAvlNode", result, sizeof(AvlNode)) + sysAssert(bottom == addr(a.bottomData), "bottom data") + sysAssert(bottom.link[0] == bottom, "bottom link[0]") + sysAssert(bottom.link[1] == bottom, "bottom link[1]") + + proc deallocAvlNode(a: var MemRegion, n: PAvlNode) {.inline.} = + n.link[0] = a.freeAvlNodes + a.freeAvlNodes = n + +proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int): ptr HeapLinks = + var it = addr(a.heapLinks) + while it != nil and it.len >= it.chunks.len: it = it.next + if it == nil: + var n = cast[ptr HeapLinks](llAlloc(a, sizeof(HeapLinks))) + n.next = a.heapLinks.next + a.heapLinks.next = n + n.chunks[0] = (p, size) + n.len = 1 + result = n else: - result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode))) - result.key = key - result.upperBound = upperBound - result.link[0] = bottom - result.link[1] = bottom - result.level = 1 - sysAssert(bottom == addr(bottomData), "bottom data") - sysAssert(bottom.link[0] == bottom, "bottom link[0]") - sysAssert(bottom.link[1] == bottom, "bottom link[1]") - -proc deallocAvlNode(a: var MemRegion, n: PAvlNode) {.inline.} = - n.link[0] = a.freeAvlNodes - a.freeAvlNodes = n - -include "system/avltree" + let L = it.len + it.chunks[L] = (p, size) + inc it.len + result = it + +when not defined(gcDestructors): + include "system/avltree" proc llDeallocAll(a: var MemRegion) = var it = a.llmem @@ -320,6 +405,7 @@ proc llDeallocAll(a: var MemRegion) = var next = it.next osDeallocPages(it, PageSize) it = next + a.llmem = nil proc intSetGet(t: IntSet, key: int): PTrunk = var it = t.data[key and high(t.data)] @@ -340,21 +426,21 @@ proc contains(s: IntSet, key: int): bool = var t = intSetGet(s, key shr TrunkShift) if t != nil: var u = key and TrunkMask - result = (t.bits[u shr IntShift] and (1 shl (u and IntMask))) != 0 + result = (t.bits[u shr IntShift] and (uint(1) shl (u and IntMask))) != 0 else: result = false proc incl(a: var MemRegion, s: var IntSet, key: int) = var t = intSetPut(a, s, key shr TrunkShift) var u = key and TrunkMask - t.bits[u shr IntShift] = t.bits[u shr IntShift] or (1 shl (u and IntMask)) + t.bits[u shr IntShift] = t.bits[u shr IntShift] or (uint(1) shl (u and IntMask)) proc excl(s: var IntSet, key: int) = var t = intSetGet(s, key shr TrunkShift) if t != nil: var u = key and TrunkMask t.bits[u shr IntShift] = t.bits[u shr IntShift] and not - (1 shl (u and IntMask)) + (uint(1) shl (u and IntMask)) iterator elements(t: IntSet): int {.inline.} = # while traversing it is forbidden to change the set! @@ -375,12 +461,13 @@ iterator elements(t: IntSet): int {.inline.} = r = r.next proc isSmallChunk(c: PChunk): bool {.inline.} = - return c.size <= SmallChunkSize-smallChunkOverhead() + result = c.size <= SmallChunkSize-smallChunkOverhead() proc chunkUnused(c: PChunk): bool {.inline.} = - result = not c.used + result = (c.prevSize and 1) == 0 -iterator allObjects(m: MemRegion): pointer {.inline.} = +iterator allObjects(m: var MemRegion): pointer {.inline.} = + m.locked = true for s in elements(m.chunkStarts): # we need to check here again as it could have been modified: if s in m.chunkStarts: @@ -390,74 +477,99 @@ iterator allObjects(m: MemRegion): pointer {.inline.} = var c = cast[PSmallChunk](c) let size = c.size - var a = cast[ByteAddress](addr(c.data)) - let limit = a + c.acc + var a = cast[int](addr(c.data)) + let limit = a + c.acc.int while a <% limit: yield cast[pointer](a) a = a +% size else: let c = cast[PBigChunk](c) yield addr(c.data) + m.locked = false proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {. magic: "Plugin", compileTime.} -proc isCell(p: pointer): bool {.inline.} = - result = cast[ptr FreeCell](p).zeroField >% 1 +when not defined(gcDestructors): + proc isCell(p: pointer): bool {.inline.} = + result = cast[ptr FreeCell](p).zeroField >% 1 # ------------- chunk management ---------------------------------------------- proc pageIndex(c: PChunk): int {.inline.} = - result = cast[ByteAddress](c) shr PageShift + result = cast[int](c) shr PageShift proc pageIndex(p: pointer): int {.inline.} = - result = cast[ByteAddress](p) shr PageShift + result = cast[int](p) shr PageShift proc pageAddr(p: pointer): PChunk {.inline.} = - result = cast[PChunk](cast[ByteAddress](p) and not PageMask) + result = cast[PChunk](cast[int](p) and not PageMask) #sysAssert(Contains(allocator.chunkStarts, pageIndex(result))) +when false: + proc writeFreeList(a: MemRegion) = + var it = a.freeChunksList + c_fprintf(stdout, "freeChunksList: %p\n", it) + while it != nil: + c_fprintf(stdout, "it: %p, next: %p, prev: %p, size: %ld\n", + it, it.next, it.prev, it.size) + it = it.next + proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = + when not defined(emscripten): + if not a.blockChunkSizeIncrease: + let usedMem = a.occ #a.currMem # - a.freeMem + if usedMem < 64 * 1024: + a.nextChunkSize = PageSize*4 + else: + a.nextChunkSize = min(roundup(usedMem shr 2, PageSize), a.nextChunkSize * 2) + a.nextChunkSize = min(a.nextChunkSize, MaxBigChunkSize).int + + var size = size + if size > a.nextChunkSize: + result = cast[PBigChunk](allocPages(a, size)) + else: + result = cast[PBigChunk](tryAllocPages(a, a.nextChunkSize)) + if result == nil: + result = cast[PBigChunk](allocPages(a, size)) + a.blockChunkSizeIncrease = true + else: + size = a.nextChunkSize + incCurrMem(a, size) inc(a.freeMem, size) - result = cast[PBigChunk](osAllocPages(size)) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "requestOsChunks 1") + let heapLink = a.addHeapLink(result, size) + when defined(debugHeapLinks): + cprintf("owner: %p; result: %p; next pointer %p; size: %ld\n", addr(a), + result, heapLink, size) + + when defined(memtracker): + trackLocation(addr result.size, sizeof(int)) + + sysAssert((cast[int](result) and PageMask) == 0, "requestOsChunks 1") #zeroMem(result, size) result.next = nil result.prev = nil - result.used = false result.size = size # update next.prevSize: - var nxt = cast[ByteAddress](result) +% size + var nxt = cast[int](result) +% size sysAssert((nxt and PageMask) == 0, "requestOsChunks 2") var next = cast[PChunk](nxt) if pageIndex(next) in a.chunkStarts: #echo("Next already allocated!") - next.prevSize = size + next.prevSize = size or (next.prevSize and 1) # set result.prevSize: var lastSize = if a.lastSize != 0: a.lastSize else: PageSize - var prv = cast[ByteAddress](result) -% lastSize + var prv = cast[int](result) -% lastSize sysAssert((nxt and PageMask) == 0, "requestOsChunks 3") var prev = cast[PChunk](prv) if pageIndex(prev) in a.chunkStarts and prev.size == lastSize: #echo("Prev already allocated!") - result.prevSize = lastSize + result.prevSize = lastSize or (result.prevSize and 1) else: - result.prevSize = 0 # unknown + result.prevSize = 0 or (result.prevSize and 1) # unknown + # but do not overwrite 'used' field a.lastSize = size # for next request - -proc freeOsChunks(a: var MemRegion, p: pointer, size: int) = - # update next.prevSize: - var c = cast[PChunk](p) - var nxt = cast[ByteAddress](p) +% c.size - sysAssert((nxt and PageMask) == 0, "freeOsChunks") - var next = cast[PChunk](nxt) - if pageIndex(next) in a.chunkStarts: - next.prevSize = 0 # XXX used - excl(a.chunkStarts, pageIndex(p)) - osDeallocPages(p, size) - decCurrMem(a, size) - dec(a.freeMem, size) - #c_fprintf(c_stdout, "[Alloc] back to OS: %ld\n", size) + sysAssert((cast[int](result) and PageMask) == 0, "requestOschunks: unaligned chunk") proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} = result = contains(a.chunkStarts, pageIndex(p)) @@ -468,14 +580,6 @@ proc contains[T](list, x: T): bool = if it == x: return true it = it.next -proc writeFreeList(a: MemRegion) = - var it = a.freeChunksList - c_fprintf(c_stdout, "freeChunksList: %p\n", it) - while it != nil: - c_fprintf(c_stdout, "it: %p, next: %p, prev: %p\n", - it, it.next, it.prev) - it = it.next - proc listAdd[T](head: var T, c: T) {.inline.} = sysAssert(c notin head, "listAdd 1") sysAssert c.prev == nil, "listAdd 2" @@ -501,83 +605,137 @@ proc listRemove[T](head: var T, c: T) {.inline.} = proc updatePrevSize(a: var MemRegion, c: PBigChunk, prevSize: int) {.inline.} = - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "updatePrevSize") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "updatePrevSize") if isAccessible(a, ri): - ri.prevSize = prevSize + ri.prevSize = prevSize or (ri.prevSize and 1) + +proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = + result = cast[PBigChunk](cast[int](c) +% size) + result.size = c.size - size + track("result.size", addr result.size, sizeof(int)) + when not defined(nimOptimizedSplitChunk): + # still active because of weird codegen issue on some of our CIs: + result.next = nil + result.prev = nil + # size and not used: + result.prevSize = size + result.owner = addr a + sysAssert((size and 1) == 0, "splitChunk 2") + sysAssert((size and PageMask) == 0, + "splitChunk: size is not a multiple of the PageSize") + updatePrevSize(a, c, result.size) + c.size = size + incl(a, a.chunkStarts, pageIndex(result)) + +proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = + let rest = splitChunk2(a, c, size) + addChunkToMatrix(a, rest) proc freeBigChunk(a: var MemRegion, c: PBigChunk) = var c = c sysAssert(c.size >= PageSize, "freeBigChunk") inc(a.freeMem, c.size) - when coalescRight: - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") - if isAccessible(a, ri) and chunkUnused(ri): - sysAssert(not isSmallChunk(ri), "freeBigChunk 3") - if not isSmallChunk(ri): - listRemove(a.freeChunksList, cast[PBigChunk](ri)) - inc(c.size, ri.size) - excl(a.chunkStarts, pageIndex(ri)) + c.prevSize = c.prevSize and not 1 # set 'used' to false when coalescLeft: - if c.prevSize != 0: - var le = cast[PChunk](cast[ByteAddress](c) -% c.prevSize) - sysAssert((cast[ByteAddress](le) and PageMask) == 0, "freeBigChunk 4") + let prevSize = c.prevSize + if prevSize != 0: + var le = cast[PChunk](cast[int](c) -% prevSize) + sysAssert((cast[int](le) and PageMask) == 0, "freeBigChunk 4") if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") - if not isSmallChunk(le): - listRemove(a.freeChunksList, cast[PBigChunk](le)) + if not isSmallChunk(le) and le.size < MaxBigChunkSize: + removeChunkFromMatrix(a, cast[PBigChunk](le)) inc(le.size, c.size) excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) - - if c.size < ChunkOsReturn or doNotUnmap: - incl(a, a.chunkStarts, pageIndex(c)) - updatePrevSize(a, c, c.size) - listAdd(a.freeChunksList, c) - c.used = false - else: - freeOsChunks(a, c, c.size) - -proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = - var rest = cast[PBigChunk](cast[ByteAddress](c) +% size) - sysAssert(rest notin a.freeChunksList, "splitChunk") - rest.size = c.size - size - rest.used = false - rest.next = nil - rest.prev = nil - rest.prevSize = size - updatePrevSize(a, c, rest.size) - c.size = size - incl(a, a.chunkStarts, pageIndex(rest)) - listAdd(a.freeChunksList, rest) + if c.size > MaxBigChunkSize: + let rest = splitChunk2(a, c, MaxBigChunkSize) + when defined(nimOptimizedSplitChunk): + rest.next = nil + rest.prev = nil + addChunkToMatrix(a, c) + c = rest + when coalescRight: + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "freeBigChunk 2") + if isAccessible(a, ri) and chunkUnused(ri): + sysAssert(not isSmallChunk(ri), "freeBigChunk 3") + if not isSmallChunk(ri) and c.size < MaxBigChunkSize: + removeChunkFromMatrix(a, cast[PBigChunk](ri)) + inc(c.size, ri.size) + excl(a.chunkStarts, pageIndex(ri)) + if c.size > MaxBigChunkSize: + let rest = splitChunk2(a, c, MaxBigChunkSize) + addChunkToMatrix(a, rest) + addChunkToMatrix(a, c) proc getBigChunk(a: var MemRegion, size: int): PBigChunk = - # use first fit for now: - sysAssert((size and PageMask) == 0, "getBigChunk 1") sysAssert(size > 0, "getBigChunk 2") - result = a.freeChunksList - block search: - while result != nil: - sysAssert chunkUnused(result), "getBigChunk 3" - if result.size == size: - listRemove(a.freeChunksList, result) - break search - elif result.size > size: - listRemove(a.freeChunksList, result) - splitChunk(a, result, size) - break search - result = result.next - sysAssert result != a.freeChunksList, "getBigChunk 4" - if size < InitialMemoryRequest: - result = requestOsChunks(a, InitialMemoryRequest) + var size = size # roundup(size, PageSize) + var fl = 0 + var sl = 0 + mappingSearch(size, fl, sl) + sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") + result = findSuitableBlock(a, fl, sl) + + when RegionHasLock: + if not a.lockActive: + a.lockActive = true + initSysLock(a.lock) + acquireSys a.lock + + if result == nil: + if size < nimMinHeapPages * PageSize: + result = requestOsChunks(a, nimMinHeapPages * PageSize) splitChunk(a, result, size) else: result = requestOsChunks(a, size) - result.prevSize = 0 # XXX why is this needed? - result.used = true + # if we over allocated split the chunk: + if result.size > size: + splitChunk(a, result, size) + result.owner = addr a + else: + removeChunkFromMatrix2(a, result, fl, sl) + if result.size >= size + PageSize: + splitChunk(a, result, size) + # set 'used' to to true: + result.prevSize = 1 + track("setUsedToFalse", addr result.size, sizeof(int)) + sysAssert result.owner == addr a, "getBigChunk: No owner set!" + incl(a, a.chunkStarts, pageIndex(result)) dec(a.freeMem, size) + when RegionHasLock: + releaseSys a.lock + +proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = + result = cast[PBigChunk](allocPages(a, size)) + when RegionHasLock: + if not a.lockActive: + a.lockActive = true + initSysLock(a.lock) + acquireSys a.lock + incCurrMem(a, size) + # XXX add this to the heap links. But also remove it from it later. + when false: a.addHeapLink(result, size) + sysAssert((cast[int](result) and PageMask) == 0, "getHugeChunk") + result.next = nil + result.prev = nil + result.size = size + # set 'used' to to true: + result.prevSize = 1 + result.owner = addr a + incl(a, a.chunkStarts, pageIndex(result)) + when RegionHasLock: + releaseSys a.lock + +proc freeHugeChunk(a: var MemRegion; c: PBigChunk) = + let size = c.size + sysAssert(size >= HugeChunkSize, "freeHugeChunk: invalid size") + excl(a.chunkStarts, pageIndex(c)) + decCurrMem(a, size) + osDeallocPages(c, size) proc getSmallChunk(a: var MemRegion): PSmallChunk = var res = getBigChunk(a, PageSize) @@ -586,278 +744,498 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk = result = cast[PSmallChunk](res) # ----------------------------------------------------------------------------- -proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.} +when not defined(gcDestructors): + proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.} -proc allocInv(a: MemRegion): bool = - ## checks some (not all yet) invariants of the allocator's data structures. - for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): - var c = a.freeSmallChunks[s] - while c != nil: - if c.next == c: - echo "[SYSASSERT] c.next == c" - return false - if c.size != s * MemAlign: - echo "[SYSASSERT] c.size != s * MemAlign" - return false - var it = c.freeList - while it != nil: - if it.zeroField != 0: - echo "[SYSASSERT] it.zeroField != 0" - c_printf("%ld %p\n", it.zeroField, it) +when true: + template allocInv(a: MemRegion): bool = true +else: + proc allocInv(a: MemRegion): bool = + ## checks some (not all yet) invariants of the allocator's data structures. + for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): + var c = a.freeSmallChunks[s] + while not (c == nil): + if c.next == c: + echo "[SYSASSERT] c.next == c" return false - it = it.next - c = c.next - result = true + if not (c.size == s * MemAlign): + echo "[SYSASSERT] c.size != s * MemAlign" + return false + var it = c.freeList + while not (it == nil): + if not (it.zeroField == 0): + echo "[SYSASSERT] it.zeroField != 0" + c_printf("%ld %p\n", it.zeroField, it) + return false + it = it.next + c = c.next + result = true + +when false: + var + rsizes: array[50_000, int] + rsizesLen: int + + proc trackSize(size: int) = + rsizes[rsizesLen] = size + inc rsizesLen + + proc untrackSize(size: int) = + for i in 0 .. rsizesLen-1: + if rsizes[i] == size: + rsizes[i] = rsizes[rsizesLen-1] + dec rsizesLen + return + c_fprintf(stdout, "%ld\n", size) + sysAssert(false, "untracked size!") +else: + template trackSize(x) = discard + template untrackSize(x) = discard + +proc deallocBigChunk(a: var MemRegion, c: PBigChunk) = + when RegionHasLock: + acquireSys a.lock + dec a.occ, c.size + untrackSize(c.size) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" + when not defined(gcDestructors): + a.deleted = getBottom(a) + del(a, a.root, cast[int](addr(c.data))) + if c.size >= HugeChunkSize: freeHugeChunk(a, c) + else: freeBigChunk(a, c) + when RegionHasLock: + releaseSys a.lock + +when defined(gcDestructors): + template atomicPrepend(head, elem: untyped) = + # see also https://en.cppreference.com/w/cpp/atomic/atomic_compare_exchange + when hasThreadSupport: + while true: + elem.next.storea head.loada + if atomicCompareExchangeN(addr head, addr elem.next, elem, weak = true, ATOMIC_RELEASE, ATOMIC_RELAXED): + break + else: + elem.next.storea head.loada + head.storea elem + + proc addToSharedFreeListBigChunks(a: var MemRegion; c: PBigChunk) {.inline.} = + sysAssert c.next == nil, "c.next pointer must be nil" + atomicPrepend a.sharedFreeListBigChunks, c + + proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} = + atomicPrepend c.owner.sharedFreeLists[size], f + + const MaxSteps = 20 + + proc compensateCounters(a: var MemRegion; c: PSmallChunk; size: int) = + # rawDealloc did NOT do the usual: + # `inc(c.free, size); dec(a.occ, size)` because it wasn't the owner of these + # memory locations. We have to compensate here for these for the entire list. + var it = c.freeList + var total = 0 + while it != nil: + inc total, size + let chunk = cast[PSmallChunk](pageAddr(it)) + if c != chunk: + # The cell is foreign, potentially even from a foreign thread. + # It must block the current chunk from being freed, as doing so would leak memory. + inc c.foreignCells + it = it.next + # By not adjusting the foreign chunk we reserve space in it to prevent deallocation + inc(c.free, total) + dec(a.occ, total) + + proc freeDeferredObjects(a: var MemRegion; root: PBigChunk) = + var it = root + var maxIters = MaxSteps # make it time-bounded + while true: + let rest = it.next.loada + it.next.storea nil + deallocBigChunk(a, cast[PBigChunk](it)) + if maxIters == 0: + if rest != nil: + addToSharedFreeListBigChunks(a, rest) + sysAssert a.sharedFreeListBigChunks != nil, "re-enqueing failed" + break + it = rest + dec maxIters + if it == nil: break proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = + when defined(nimTypeNames): + inc(a.allocCounter) sysAssert(allocInv(a), "rawAlloc: begin") sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") - sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") var size = roundup(requestedSize, MemAlign) + sysAssert(size >= sizeof(FreeCell), "rawAlloc: requested size too small") sysAssert(size >= requestedSize, "insufficient allocated size!") - #c_fprintf(c_stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): + template fetchSharedCells(tc: PSmallChunk) = + # Consumes cells from (potentially) foreign threads from `a.sharedFreeLists[s]` + when defined(gcDestructors): + if tc.freeList == nil: + when hasThreadSupport: + # Steal the entire list from `sharedFreeList`: + tc.freeList = atomicExchangeN(addr a.sharedFreeLists[s], nil, ATOMIC_RELAXED) + else: + tc.freeList = a.sharedFreeLists[s] + a.sharedFreeLists[s] = nil + # if `tc.freeList` isn't nil, `tc` will gain capacity. + # We must calculate how much it gained and how many foreign cells are included. + compensateCounters(a, tc, size) + # allocate a small block: for small chunks, we use only its next pointer - var s = size div MemAlign + let s = size div MemAlign var c = a.freeSmallChunks[s] if c == nil: + # There is no free chunk of the requested size available, we need a new one. c = getSmallChunk(a) + # init all fields in case memory didn't get zeroed c.freeList = nil + c.foreignCells = 0 sysAssert c.size == PageSize, "rawAlloc 3" c.size = size - c.acc = size - c.free = SmallChunkSize - smallChunkOverhead() - size + c.acc = size.uint32 + c.free = SmallChunkSize - smallChunkOverhead() - size.int32 + sysAssert c.owner == addr(a), "rawAlloc: No owner set!" c.next = nil c.prev = nil - listAdd(a.freeSmallChunks[s], c) + # Shared cells are fetched here in case `c.size * 2 >= SmallChunkSize - smallChunkOverhead()`. + # For those single cell chunks, we would otherwise have to allocate a new one almost every time. + fetchSharedCells(c) + if c.free >= size: + # Because removals from `a.freeSmallChunks[s]` only happen in the other alloc branch and during dealloc, + # we must not add it to the list if it cannot be used the next time a pointer of `size` bytes is needed. + listAdd(a.freeSmallChunks[s], c) result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 4") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 4") else: + # There is a free chunk of the requested size available, use it. sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" #if c.size != size: - # c_fprintf(c_stdout, "csize: %lld; size %lld\n", c.size, size) + # c_fprintf(stdout, "csize: %lld; size %lld\n", c.size, size) sysAssert c.size == size, "rawAlloc 6" if c.freeList == nil: - sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, + sysAssert(c.acc.int + smallChunkOverhead() + size <= SmallChunkSize, "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + result = cast[pointer](cast[int](addr(c.data)) +% c.acc.int) inc(c.acc, size) else: + # There are free cells available, prefer them over the accumulator result = c.freeList - sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") + when not defined(gcDestructors): + sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") c.freeList = c.freeList.next + if cast[PSmallChunk](pageAddr(result)) != c: + # This cell isn't a blocker for the current chunk's deallocation anymore + dec(c.foreignCells) + else: + sysAssert(c == cast[PSmallChunk](pageAddr(result)), "rawAlloc: Bad cell") + # Even if the cell we return is foreign, the local chunk's capacity decreases. + # The capacity was previously reserved in the source chunk (when it first got allocated), + # then added into the current chunk during dealloc, + # so the source chunk will not be freed or leak memory because of this. dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert(allocInv(a), "rawAlloc: end c != nil") - sysAssert(allocInv(a), "rawAlloc: before c.free < size") - if c.free < size: - sysAssert(allocInv(a), "rawAlloc: before listRemove test") - listRemove(a.freeSmallChunks[s], c) - sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + # We fetch deferred cells *after* advancing `c.freeList`/`acc` to adjust `c.free`. + # If after the adjustment it turns out there's free cells available, + # the chunk stays in `a.freeSmallChunks[s]` and the need for a new chunk is delayed. + fetchSharedCells(c) + sysAssert(allocInv(a), "rawAlloc: before c.free < size") + if c.free < size: + # Even after fetching shared cells the chunk has no usable memory left. It is no longer the active chunk + sysAssert(allocInv(a), "rawAlloc: before listRemove test") + listRemove(a.freeSmallChunks[s], c) + sysAssert(allocInv(a), "rawAlloc: end listRemove test") + sysAssert(((cast[int](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") + inc a.occ, size + trackSize(c.size) else: - size = roundup(requestedSize+bigChunkOverhead(), PageSize) + when defined(gcDestructors): + when hasThreadSupport: + let deferredFrees = atomicExchangeN(addr a.sharedFreeListBigChunks, nil, ATOMIC_RELAXED) + else: + let deferredFrees = a.sharedFreeListBigChunks + a.sharedFreeListBigChunks = nil + if deferredFrees != nil: + freeDeferredObjects(a, deferredFrees) + + size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block - var c = getBigChunk(a, size) + var c = if size >= HugeChunkSize: getHugeChunk(a, size) + else: getBigChunk(a, size) sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" - sysAssert c.size == size, "rawAlloc 12" result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13") - if a.root == nil: a.root = bottom - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + sysAssert((cast[int](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[int](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + when not defined(gcDestructors): + if a.root == nil: a.root = getBottom(a) + add(a, a.root, cast[int](result), cast[int](result)+%size) + inc a.occ, c.size + trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") sysAssert(allocInv(a), "rawAlloc: end") - when logAlloc: cprintf("rawAlloc: %ld %p\n", requestedSize, result) + when logAlloc: cprintf("var pointer_%p = alloc(%ld) # %p\n", result, requestedSize, addr a) proc rawAlloc0(a: var MemRegion, requestedSize: int): pointer = result = rawAlloc(a, requestedSize) zeroMem(result, requestedSize) proc rawDealloc(a: var MemRegion, p: pointer) = + when defined(nimTypeNames): + inc(a.deallocCounter) #sysAssert(isAllocatedPtr(a, p), "rawDealloc: no allocated pointer") sysAssert(allocInv(a), "rawDealloc: begin") var c = pageAddr(p) + sysAssert(c != nil, "rawDealloc: begin") if isSmallChunk(c): # `p` is within a small chunk: var c = cast[PSmallChunk](c) - var s = c.size - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% - s == 0, "rawDealloc 3") + let s = c.size + # ^ We might access thread foreign storage here. + # The other thread cannot possibly free this block as it's still alive. var f = cast[ptr FreeCell](p) - #echo("setting to nil: ", $cast[TAddress](addr(f.zeroField))) - sysAssert(f.zeroField != 0, "rawDealloc 1") - f.zeroField = 0 - f.next = c.freeList - c.freeList = f - when overwriteFree: - # set to 0xff to check for usage after free bugs: - c_memset(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32, - s -% sizeof(FreeCell)) - # check if it is not in the freeSmallChunks[s] list: - if c.free < s: - # add it to the freeSmallChunks[s] array: - listAdd(a.freeSmallChunks[s div MemAlign], c) - inc(c.free, s) + if c.owner == addr(a): + # We own the block, there is no foreign thread involved. + dec a.occ, s + untrackSize(s) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% + s == 0, "rawDealloc 3") + when not defined(gcDestructors): + #echo("setting to nil: ", $cast[int](addr(f.zeroField))) + sysAssert(f.zeroField != 0, "rawDealloc 1") + f.zeroField = 0 + when overwriteFree: + # set to 0xff to check for usage after free bugs: + nimSetMem(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32, + s -% sizeof(FreeCell)) + let activeChunk = a.freeSmallChunks[s div MemAlign] + if activeChunk != nil and c != activeChunk: + # This pointer is not part of the active chunk, lend it out + # and do not adjust the current chunk (same logic as compensateCounters.) + # Put the cell into the active chunk, + # may prevent a queue of available chunks from forming in a.freeSmallChunks[s div MemAlign]. + # This queue would otherwise waste memory in the form of free cells until we return to those chunks. + f.next = activeChunk.freeList + activeChunk.freeList = f # lend the cell + inc(activeChunk.free, s) # By not adjusting the current chunk's capacity it is prevented from being freed + inc(activeChunk.foreignCells) # The cell is now considered foreign from the perspective of the active chunk + else: + f.next = c.freeList + c.freeList = f + if c.free < s: + # The chunk could not have been active as it didn't have enough space to give + listAdd(a.freeSmallChunks[s div MemAlign], c) + inc(c.free, s) + else: + inc(c.free, s) + # Free only if the entire chunk is unused and there are no borrowed cells. + # If the chunk were to be freed while it references foreign cells, + # the foreign chunks will leak memory and can never be freed. + if c.free == SmallChunkSize-smallChunkOverhead() and c.foreignCells == 0: + listRemove(a.freeSmallChunks[s div MemAlign], c) + c.size = SmallChunkSize + freeBigChunk(a, cast[PBigChunk](c)) else: - inc(c.free, s) - if c.free == SmallChunkSize-smallChunkOverhead(): - listRemove(a.freeSmallChunks[s div MemAlign], c) - c.size = SmallChunkSize - freeBigChunk(a, cast[PBigChunk](c)) - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + when logAlloc: cprintf("dealloc(pointer_%p) # SMALL FROM %p CALLER %p\n", p, c.owner, addr(a)) + + when defined(gcDestructors): + addToSharedFreeList(c, f, s div MemAlign) + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 2") else: # set to 0xff to check for usage after free bugs: - when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead()) - # free big chunk - var c = cast[PBigChunk](c) - a.deleted = bottom - del(a, a.root, cast[int](addr(c.data))) - freeBigChunk(a, c) + when overwriteFree: nimSetMem(p, -1'i32, c.size -% bigChunkOverhead()) + when logAlloc: cprintf("dealloc(pointer_%p) # BIG %p\n", p, c.owner) + when defined(gcDestructors): + if c.owner == addr(a): + deallocBigChunk(a, cast[PBigChunk](c)) + else: + addToSharedFreeListBigChunks(c.owner[], cast[PBigChunk](c)) + else: + deallocBigChunk(a, cast[PBigChunk](c)) + sysAssert(allocInv(a), "rawDealloc: end") - when logAlloc: cprintf("rawDealloc: %p\n", p) + #when logAlloc: cprintf("dealloc(pointer_%p)\n", p) -proc isAllocatedPtr(a: MemRegion, p: pointer): bool = - if isAccessible(a, p): - var c = pageAddr(p) - if not chunkUnused(c): - if isSmallChunk(c): - var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% - smallChunkOverhead() - result = (c.acc >% offset) and (offset %% c.size == 0) and - (cast[ptr FreeCell](p).zeroField >% 1) - else: - var c = cast[PBigChunk](c) - result = p == addr(c.data) and cast[ptr FreeCell](p).zeroField >% 1 +when not defined(gcDestructors): + proc isAllocatedPtr(a: MemRegion, p: pointer): bool = + if isAccessible(a, p): + var c = pageAddr(p) + if not chunkUnused(c): + if isSmallChunk(c): + var c = cast[PSmallChunk](c) + var offset = (cast[int](p) and (PageSize-1)) -% + smallChunkOverhead() + result = (c.acc.int >% offset) and (offset %% c.size == 0) and + (cast[ptr FreeCell](p).zeroField >% 1) + else: + var c = cast[PBigChunk](c) + result = p == addr(c.data) and cast[ptr FreeCell](p).zeroField >% 1 -proc prepareForInteriorPointerChecking(a: var MemRegion) {.inline.} = - a.minLargeObj = lowGauge(a.root) - a.maxLargeObj = highGauge(a.root) + proc prepareForInteriorPointerChecking(a: var MemRegion) {.inline.} = + a.minLargeObj = lowGauge(a.root) + a.maxLargeObj = highGauge(a.root) -proc interiorAllocatedPtr(a: MemRegion, p: pointer): pointer = - if isAccessible(a, p): - var c = pageAddr(p) - if not chunkUnused(c): - if isSmallChunk(c): - var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% - smallChunkOverhead() - if c.acc >% offset: - sysAssert(cast[ByteAddress](addr(c.data)) +% offset == - cast[ByteAddress](p), "offset is not what you think it is") - var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +% - offset -% (offset %% c.size)) - if d.zeroField >% 1: + proc interiorAllocatedPtr(a: MemRegion, p: pointer): pointer = + if isAccessible(a, p): + var c = pageAddr(p) + if not chunkUnused(c): + if isSmallChunk(c): + var c = cast[PSmallChunk](c) + var offset = (cast[int](p) and (PageSize-1)) -% + smallChunkOverhead() + if c.acc.int >% offset: + sysAssert(cast[int](addr(c.data)) +% offset == + cast[int](p), "offset is not what you think it is") + var d = cast[ptr FreeCell](cast[int](addr(c.data)) +% + offset -% (offset %% c.size)) + if d.zeroField >% 1: + result = d + sysAssert isAllocatedPtr(a, result), " result wrong pointer!" + else: + var c = cast[PBigChunk](c) + var d = addr(c.data) + if p >= d and cast[ptr FreeCell](d).zeroField >% 1: result = d sysAssert isAllocatedPtr(a, result), " result wrong pointer!" - else: - var c = cast[PBigChunk](c) - var d = addr(c.data) - if p >= d and cast[ptr FreeCell](d).zeroField >% 1: - result = d - sysAssert isAllocatedPtr(a, result), " result wrong pointer!" - else: - var q = cast[int](p) - if q >=% a.minLargeObj and q <=% a.maxLargeObj: - # this check is highly effective! Test fails for 99,96% of all checks on - # an x86-64. - var avlNode = inRange(a.root, q) - if avlNode != nil: - var k = cast[pointer](avlNode.key) - var c = cast[PBigChunk](pageAddr(k)) - sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!") - if cast[ptr FreeCell](k).zeroField >% 1: - result = k - sysAssert isAllocatedPtr(a, result), " result wrong pointer!" + else: + var q = cast[int](p) + if q >=% a.minLargeObj and q <=% a.maxLargeObj: + # this check is highly effective! Test fails for 99,96% of all checks on + # an x86-64. + var avlNode = inRange(a.root, q) + if avlNode != nil: + var k = cast[pointer](avlNode.key) + var c = cast[PBigChunk](pageAddr(k)) + sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!") + if cast[ptr FreeCell](k).zeroField >% 1: + result = k + sysAssert isAllocatedPtr(a, result), " result wrong pointer!" proc ptrSize(p: pointer): int = - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) - var c = pageAddr(p) - sysAssert(not chunkUnused(c), "ptrSize") - result = c.size -% sizeof(FreeCell) - if not isSmallChunk(c): - dec result, bigChunkOverhead() - -proc alloc(allocator: var MemRegion, size: Natural): pointer = - result = rawAlloc(allocator, size+sizeof(FreeCell)) - cast[ptr FreeCell](result).zeroField = 1 # mark it as used - sysAssert(not isAllocatedPtr(allocator, result), "alloc") - result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) + when not defined(gcDestructors): + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) + var c = pageAddr(p) + sysAssert(not chunkUnused(c), "ptrSize") + result = c.size -% sizeof(FreeCell) + if not isSmallChunk(c): + dec result, bigChunkOverhead() + else: + var c = pageAddr(p) + sysAssert(not chunkUnused(c), "ptrSize") + result = c.size + if not isSmallChunk(c): + dec result, bigChunkOverhead() + +proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} = + when not defined(gcDestructors): + result = rawAlloc(allocator, size+sizeof(FreeCell)) + cast[ptr FreeCell](result).zeroField = 1 # mark it as used + sysAssert(not isAllocatedPtr(allocator, result), "alloc") + result = cast[pointer](cast[int](result) +% sizeof(FreeCell)) + track("alloc", result, size) + else: + result = rawAlloc(allocator, size) proc alloc0(allocator: var MemRegion, size: Natural): pointer = result = alloc(allocator, size) zeroMem(result, size) proc dealloc(allocator: var MemRegion, p: pointer) = - sysAssert(p != nil, "dealloc 0") - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) - sysAssert(x != nil, "dealloc 1") - sysAssert(isAccessible(allocator, x), "is not accessible") - sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2") - rawDealloc(allocator, x) - sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3") + when not defined(gcDestructors): + sysAssert(p != nil, "dealloc: p is nil") + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) + sysAssert(x != nil, "dealloc: x is nil") + sysAssert(isAccessible(allocator, x), "is not accessible") + sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") + rawDealloc(allocator, x) + sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible") + track("dealloc", p, 0) + else: + rawDealloc(allocator, p) proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer = if newsize > 0: - result = alloc0(allocator, newsize) + result = alloc(allocator, newsize) if p != nil: - copyMem(result, p, ptrSize(p)) + copyMem(result, p, min(ptrSize(p), newsize)) dealloc(allocator, p) elif p != nil: dealloc(allocator, p) +proc realloc0(allocator: var MemRegion, p: pointer, oldsize, newsize: Natural): pointer = + result = realloc(allocator, p, newsize) + if newsize > oldsize: + zeroMem(cast[pointer](cast[uint](result) + uint(oldsize)), newsize - oldsize) + proc deallocOsPages(a: var MemRegion) = # we free every 'ordinarily' allocated page by iterating over the page bits: - for p in elements(a.chunkStarts): - var page = cast[PChunk](p shl PageShift) - when not doNotUnmap: - var size = if page.size < PageSize: PageSize else: page.size - osDeallocPages(page, size) - else: - # Linux on PowerPC for example frees MORE than asked if 'munmap' - # receives the start of an originally mmap'ed memory block. This is not - # too bad, but we must not access 'page.size' then as that could trigger - # a segfault. But we don't need to access 'page.size' here anyway, - # because calling munmap with PageSize suffices: - osDeallocPages(page, PageSize) + var it = addr(a.heapLinks) + while true: + let next = it.next + for i in 0..it.len-1: + let (p, size) = it.chunks[i] + when defined(debugHeapLinks): + cprintf("owner %p; dealloc A: %p size: %ld; next: %p\n", addr(a), + it, size, next) + sysAssert size >= PageSize, "origSize too small" + osDeallocPages(p, size) + it = next + if it == nil: break # And then we free the pages that are in use for the page bits: llDeallocAll(a) proc getFreeMem(a: MemRegion): int {.inline.} = result = a.freeMem proc getTotalMem(a: MemRegion): int {.inline.} = result = a.currMem proc getOccupiedMem(a: MemRegion): int {.inline.} = - result = a.currMem - a.freeMem + result = a.occ + # a.currMem - a.freeMem + +when defined(nimTypeNames): + proc getMemCounters(a: MemRegion): (int, int) {.inline.} = + (a.allocCounter, a.deallocCounter) # ---------------------- thread memory region ------------------------------- -template instantiateForRegion(allocator: expr) = - when defined(fulldebug): +template instantiateForRegion(allocator: untyped) {.dirty.} = + {.push stackTrace: off.} + + when defined(nimFulldebug): proc interiorAllocatedPtr*(p: pointer): pointer = result = interiorAllocatedPtr(allocator, p) proc isAllocatedPtr*(p: pointer): bool = - let p = cast[pointer](cast[ByteAddress](p)-%ByteAddress(sizeof(Cell))) + let p = cast[pointer](cast[int](p)-%ByteAddress(sizeof(Cell))) result = isAllocatedPtr(allocator, p) proc deallocOsPages = deallocOsPages(allocator) - proc alloc(size: Natural): pointer = + proc allocImpl(size: Natural): pointer = result = alloc(allocator, size) - proc alloc0(size: Natural): pointer = + proc alloc0Impl(size: Natural): pointer = result = alloc0(allocator, size) - proc dealloc(p: pointer) = + proc deallocImpl(p: pointer) = dealloc(allocator, p) - proc realloc(p: pointer, newsize: Natural): pointer = + proc reallocImpl(p: pointer, newSize: Natural): pointer = result = realloc(allocator, p, newSize) + proc realloc0Impl(p: pointer, oldSize, newSize: Natural): pointer = + result = realloc(allocator, p, newSize) + if newSize > oldSize: + zeroMem(cast[pointer](cast[uint](result) + uint(oldSize)), newSize - oldSize) + when false: proc countFreeMem(): int = # only used for assertions @@ -866,61 +1244,94 @@ template instantiateForRegion(allocator: expr) = inc(result, it.size) it = it.next + when hasThreadSupport and not defined(gcDestructors): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} + + var sharedHeap: MemRegion + var heapLock: SysLock + initSysLock(heapLock) + addSysExitProc(proc() {.noconv.} = deinitSys(heapLock)) + proc getFreeMem(): int = - result = allocator.freeMem #sysAssert(result == countFreeMem()) + result = allocator.freeMem + + proc getTotalMem(): int = + result = allocator.currMem - proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return getTotalMem() - getFreeMem() + proc getOccupiedMem(): int = + result = allocator.occ #getTotalMem() - getFreeMem() + + proc getMaxMem*(): int = + result = getMaxMem(allocator) + + when defined(nimTypeNames): + proc getMemCounters*(): (int, int) = getMemCounters(allocator) # -------------------- shared heap region ---------------------------------- - when hasThreadSupport: - var sharedHeap: MemRegion - var heapLock: SysLock - initSysLock(heapLock) - proc allocShared(size: Natural): pointer = - when hasThreadSupport: + proc allocSharedImpl(size: Natural): pointer = + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = alloc(sharedHeap, size) releaseSys(heapLock) else: - result = alloc(size) + result = allocImpl(size) - proc allocShared0(size: Natural): pointer = - result = allocShared(size) + proc allocShared0Impl(size: Natural): pointer = + result = allocSharedImpl(size) zeroMem(result, size) - proc deallocShared(p: pointer) = - when hasThreadSupport: + proc deallocSharedImpl(p: pointer) = + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) dealloc(sharedHeap, p) releaseSys(heapLock) else: - dealloc(p) + deallocImpl(p) - proc reallocShared(p: pointer, newsize: Natural): pointer = - when hasThreadSupport: + proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) - result = realloc(sharedHeap, p, newsize) + result = realloc(sharedHeap, p, newSize) releaseSys(heapLock) else: - result = realloc(p, newSize) + result = reallocImpl(p, newSize) - when hasThreadSupport: - - template sharedMemStatsShared(v: int) {.immediate.} = + proc reallocShared0Impl(p: pointer, oldSize, newSize: Natural): pointer = + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) - result = v + result = realloc0(sharedHeap, p, oldSize, newSize) releaseSys(heapLock) + else: + result = realloc0Impl(p, oldSize, newSize) + + when hasThreadSupport: + when defined(gcDestructors): + proc getFreeSharedMem(): int = + allocator.freeMem + + proc getTotalSharedMem(): int = + allocator.currMem + + proc getOccupiedSharedMem(): int = + allocator.occ + + else: + template sharedMemStatsShared(v: int) = + acquireSys(heapLock) + result = v + releaseSys(heapLock) - proc getFreeSharedMem(): int = - sharedMemStatsShared(sharedHeap.freeMem) + proc getFreeSharedMem(): int = + sharedMemStatsShared(sharedHeap.freeMem) - proc getTotalSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem) + proc getTotalSharedMem(): int = + sharedMemStatsShared(sharedHeap.currMem) - proc getOccupiedSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + proc getOccupiedSharedMem(): int = + sharedMemStatsShared(sharedHeap.occ) + #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + {.pop.} {.pop.} diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 702559034..3098e17d6 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -7,162 +7,227 @@ # distribution, for details about the copyright. # -# This include file contains headers of Ansi C procs +# This module contains headers of Ansi C procs # and definitions of Ansi C types in Nim syntax # All symbols are prefixed with 'c_' to avoid ambiguities -{.push hints:off} - -proc c_strcmp(a, b: cstring): cint {.header: "<string.h>", - noSideEffect, importc: "strcmp".} -proc c_memcmp(a, b: cstring, size: int): cint {.header: "<string.h>", - noSideEffect, importc: "memcmp".} -proc c_memcpy(a, b: cstring, size: int) {.header: "<string.h>", importc: "memcpy".} -proc c_strlen(a: cstring): int {.header: "<string.h>", - noSideEffect, importc: "strlen".} -proc c_memset(p: pointer, value: cint, size: int) {. - header: "<string.h>", importc: "memset".} - -type - C_TextFile {.importc: "FILE", header: "<stdio.h>", - final, incompleteStruct.} = object - C_BinaryFile {.importc: "FILE", header: "<stdio.h>", - final, incompleteStruct.} = object - C_TextFileStar = ptr C_TextFile - C_BinaryFileStar = ptr C_BinaryFile - - C_JmpBuf {.importc: "jmp_buf", header: "<setjmp.h>".} = object - -when not defined(vm): - var - c_stdin {.importc: "stdin", nodecl.}: C_TextFileStar - c_stdout {.importc: "stdout", nodecl.}: C_TextFileStar - c_stderr {.importc: "stderr", nodecl.}: C_TextFileStar - -# constants faked as variables: -when not declared(SIGINT): - when NoFakeVars: - when defined(windows): - const - SIGABRT = cint(22) - SIGFPE = cint(8) - SIGILL = cint(4) - SIGINT = cint(2) - SIGSEGV = cint(11) - SIGTERM = cint(15) - elif defined(macosx) or defined(linux): - const - SIGABRT = cint(6) - SIGFPE = cint(8) - SIGILL = cint(4) - SIGINT = cint(2) - SIGSEGV = cint(11) - SIGTERM = cint(15) - SIGPIPE = cint(13) - else: - {.error: "SIGABRT not ported to your platform".} +{.push hints:off, stack_trace: off, profiler: off.} + +proc c_memchr*(s: pointer, c: cint, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} +proc c_memcmp*(a, b: pointer, size: csize_t): cint {. + importc: "memcmp", header: "<string.h>", noSideEffect.} +proc c_memcpy*(a, b: pointer, size: csize_t): pointer {. + importc: "memcpy", header: "<string.h>", discardable.} +proc c_memmove*(a, b: pointer, size: csize_t): pointer {. + importc: "memmove", header: "<string.h>",discardable.} +proc c_memset*(p: pointer, value: cint, size: csize_t): pointer {. + importc: "memset", header: "<string.h>", discardable.} +proc c_strcmp*(a, b: cstring): cint {. + importc: "strcmp", header: "<string.h>", noSideEffect.} +proc c_strlen*(a: cstring): csize_t {. + importc: "strlen", header: "<string.h>", noSideEffect.} +proc c_abort*() {. + importc: "abort", header: "<stdlib.h>", noSideEffect, noreturn.} + + +when defined(nimBuiltinSetjmp): + type + C_JmpBuf* = array[5, pointer] +elif defined(linux) and defined(amd64): + type + C_JmpBuf* {.importc: "jmp_buf", header: "<setjmp.h>", bycopy.} = object + abi: array[200 div sizeof(clong), clong] +else: + type + C_JmpBuf* {.importc: "jmp_buf", header: "<setjmp.h>".} = object + + +type CSighandlerT = proc (a: cint) {.noconv.} +when defined(windows): + const + SIGABRT* = cint(22) + SIGFPE* = cint(8) + SIGILL* = cint(4) + SIGINT* = cint(2) + SIGSEGV* = cint(11) + SIGTERM = cint(15) + SIG_DFL* = cast[CSighandlerT](0) +elif defined(macosx) or defined(linux) or defined(freebsd) or + defined(openbsd) or defined(netbsd) or defined(solaris) or + defined(dragonfly) or defined(nintendoswitch) or defined(genode) or + defined(aix) or hostOS == "standalone": + const + SIGABRT* = cint(6) + SIGFPE* = cint(8) + SIGILL* = cint(4) + SIGINT* = cint(2) + SIGSEGV* = cint(11) + SIGTERM* = cint(15) + SIGPIPE* = cint(13) + SIG_DFL* = CSighandlerT(nil) +elif defined(haiku): + const + SIGABRT* = cint(6) + SIGFPE* = cint(8) + SIGILL* = cint(4) + SIGINT* = cint(2) + SIGSEGV* = cint(11) + SIGTERM* = cint(15) + SIGPIPE* = cint(7) + SIG_DFL* = CSighandlerT(nil) +else: + when defined(nimscript): + {.error: "SIGABRT not ported to your platform".} else: var - SIGINT {.importc: "SIGINT", nodecl.}: cint - SIGSEGV {.importc: "SIGSEGV", nodecl.}: cint - SIGABRT {.importc: "SIGABRT", nodecl.}: cint - SIGFPE {.importc: "SIGFPE", nodecl.}: cint - SIGILL {.importc: "SIGILL", nodecl.}: cint + SIGINT* {.importc: "SIGINT", nodecl.}: cint + SIGSEGV* {.importc: "SIGSEGV", nodecl.}: cint + SIGABRT* {.importc: "SIGABRT", nodecl.}: cint + SIGFPE* {.importc: "SIGFPE", nodecl.}: cint + SIGILL* {.importc: "SIGILL", nodecl.}: cint + SIG_DFL* {.importc: "SIG_DFL", nodecl.}: CSighandlerT when defined(macosx) or defined(linux): - var SIGPIPE {.importc: "SIGPIPE", nodecl.}: cint + var SIGPIPE* {.importc: "SIGPIPE", nodecl.}: cint when defined(macosx): - when NoFakeVars: - const SIGBUS = cint(10) - else: - var SIGBUS {.importc: "SIGBUS", nodecl.}: cint -else: - template SIGBUS: expr = SIGSEGV - -when defined(nimSigSetjmp) and not defined(nimStdSetjmp): - proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. + const SIGBUS* = cint(10) +elif defined(haiku): + const SIGBUS* = cint(30) + +# "nimRawSetjmp" is defined by default for certain platforms, so we need the +# "nimStdSetjmp" escape hatch with it. +when defined(nimSigSetjmp): + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "<setjmp.h>", importc: "siglongjmp".} - template c_setjmp(jmpb: C_JmpBuf): cint = + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_sigsetjmp(jmpb: C_JmpBuf, savemask: cint): cint {. header: "<setjmp.h>", importc: "sigsetjmp".} c_sigsetjmp(jmpb, 0) +elif defined(nimBuiltinSetjmp): + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) = + # Apple's Clang++ has trouble converting array names to pointers, so we need + # to be very explicit here. + proc c_builtin_longjmp(jmpb: ptr pointer, retval: cint) {. + importc: "__builtin_longjmp", nodecl.} + # The second parameter needs to be 1 and sometimes the C/C++ compiler checks it. + c_builtin_longjmp(unsafeAddr jmpb[0], 1) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_builtin_setjmp(jmpb: ptr pointer): cint {. + importc: "__builtin_setjmp", nodecl.} + c_builtin_setjmp(unsafeAddr jmpb[0]) + elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. - header: "<setjmp.h>", importc: "_longjmp".} - proc c_setjmp(jmpb: C_JmpBuf): cint {. - header: "<setjmp.h>", importc: "_setjmp".} + when defined(windows): + # No `_longjmp()` on Windows. + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "<setjmp.h>", importc: "longjmp".} + when defined(vcc) or defined(clangcl): + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "<setjmp.h>", importc: "setjmp".} + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} + + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "<setjmp.h>", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) + + when defined(nimHasStyleChecks): + {.pop.} + else: + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "<setjmp.h>", importc: "_longjmp".} + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "<setjmp.h>", importc: "_setjmp".} else: - proc c_longjmp(jmpb: C_JmpBuf, retval: cint) {. + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "<setjmp.h>", importc: "longjmp".} - proc c_setjmp(jmpb: C_JmpBuf): cint {. + proc c_setjmp*(jmpb: C_JmpBuf): cint {. header: "<setjmp.h>", importc: "setjmp".} -proc c_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {. - importc: "signal", header: "<signal.h>".} -proc c_raise(sig: cint) {.importc: "raise", header: "<signal.h>".} - -proc c_fputs(c: cstring, f: C_TextFileStar) {.importc: "fputs", - header: "<stdio.h>".} -proc c_fgets(c: cstring, n: int, f: C_TextFileStar): cstring {. - importc: "fgets", header: "<stdio.h>".} -proc c_fgetc(stream: C_TextFileStar): int {.importc: "fgetc", - header: "<stdio.h>".} -proc c_ungetc(c: int, f: C_TextFileStar) {.importc: "ungetc", - header: "<stdio.h>".} -proc c_putc(c: char, stream: C_TextFileStar) {.importc: "putc", - header: "<stdio.h>".} -proc c_fprintf(f: C_TextFileStar, frmt: cstring) {. - importc: "fprintf", header: "<stdio.h>", varargs.} -proc c_printf(frmt: cstring) {. - importc: "printf", header: "<stdio.h>", varargs.} - -proc c_fopen(filename, mode: cstring): C_TextFileStar {. - importc: "fopen", header: "<stdio.h>".} -proc c_fclose(f: C_TextFileStar) {.importc: "fclose", header: "<stdio.h>".} - -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} - # we use it only in a way that cannot lead to security issues +proc c_signal*(sign: cint, handler: CSighandlerT): CSighandlerT {. + importc: "signal", header: "<signal.h>", discardable.} +proc c_raise*(sign: cint): cint {.importc: "raise", header: "<signal.h>".} -proc c_fread(buf: pointer, size, n: int, f: C_BinaryFileStar): int {. - importc: "fread", header: "<stdio.h>".} -proc c_fseek(f: C_BinaryFileStar, offset: clong, whence: int): int {. - importc: "fseek", header: "<stdio.h>".} +type + CFile {.importc: "FILE", header: "<stdio.h>", + incompleteStruct.} = object + CFilePtr* = ptr CFile ## The type representing a file handle. + +# duplicated between io and ansi_c +const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined(dragonfly)) and not defined(emscripten) +const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" +const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" +const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" + +var + cstderr* {.importc: stderrName, header: "<stdio.h>".}: CFilePtr + cstdout* {.importc: stdoutName, header: "<stdio.h>".}: CFilePtr + cstdin* {.importc: stdinName, header: "<stdio.h>".}: CFilePtr + +proc c_fprintf*(f: CFilePtr, frmt: cstring): cint {. + importc: "fprintf", header: "<stdio.h>", varargs, discardable.} +proc c_printf*(frmt: cstring): cint {. + importc: "printf", header: "<stdio.h>", varargs, discardable.} + +proc c_fputs*(c: cstring, f: CFilePtr): cint {. + importc: "fputs", header: "<stdio.h>", discardable.} +proc c_fputc*(c: char, f: CFilePtr): cint {. + importc: "fputc", header: "<stdio.h>", discardable.} + +proc c_sprintf*(buf, frmt: cstring): cint {. + importc: "sprintf", header: "<stdio.h>", varargs, noSideEffect.} + # we use it only in a way that cannot lead to security issues -proc c_fwrite(buf: pointer, size, n: int, f: C_BinaryFileStar): int {. +proc c_snprintf*(buf: cstring, n: csize_t, frmt: cstring): cint {. + importc: "snprintf", header: "<stdio.h>", varargs, noSideEffect.} + +when defined(zephyr) and not defined(zephyrUseLibcMalloc): + proc c_malloc*(size: csize_t): pointer {. + importc: "k_malloc", header: "<kernel.h>".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "k_calloc", header: "<kernel.h>".} + proc c_free*(p: pointer) {. + importc: "k_free", header: "<kernel.h>".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer = + # Zephyr's kernel malloc doesn't support realloc + result = c_malloc(newSize) + # match the ansi c behavior + if not result.isNil(): + copyMem(result, p, newSize) + c_free(p) +else: + proc c_malloc*(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "calloc", header: "<stdlib.h>".} + proc c_free*(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer {. + importc: "realloc", header: "<stdlib.h>".} + +proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): csize_t {. importc: "fwrite", header: "<stdio.h>".} -proc c_exit(errorcode: cint) {.importc: "exit", header: "<stdlib.h>".} -proc c_ferror(stream: C_TextFileStar): bool {. - importc: "ferror", header: "<stdio.h>".} -proc c_fflush(stream: C_TextFileStar) {.importc: "fflush", header: "<stdio.h>".} -proc c_abort() {.importc: "abort", header: "<stdlib.h>".} -proc c_feof(stream: C_TextFileStar): bool {. - importc: "feof", header: "<stdio.h>".} - -proc c_malloc(size: int): pointer {.importc: "malloc", header: "<stdlib.h>".} -proc c_free(p: pointer) {.importc: "free", header: "<stdlib.h>".} -proc c_realloc(p: pointer, newsize: int): pointer {. - importc: "realloc", header: "<stdlib.h>".} - -when hostOS != "standalone": - when not declared(errno): - when defined(NimrodVM): - var vmErrnoWrapper {.importc.}: ptr cint - template errno: expr = - bind vmErrnoWrapper - vmErrnoWrapper[] - else: - var errno {.importc, header: "<errno.h>".}: cint ## error variable -proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} +proc c_fflush*(f: CFilePtr): cint {. + importc: "fflush", header: "<stdio.h>".} -proc c_remove(filename: cstring): cint {. - importc: "remove", header: "<stdio.h>".} -proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "<stdio.h>".} +proc rawWriteString*(f: CFilePtr, s: cstring, length: int) {.compilerproc, nonReloadable, inline.} = + # we cannot throw an exception here! + discard c_fwrite(s, 1, cast[csize_t](length), f) + discard c_fflush(f) -proc c_system(cmd: cstring): cint {.importc: "system", header: "<stdlib.h>".} -proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} -proc c_putenv(env: cstring): cint {.importc: "putenv", header: "<stdlib.h>".} +proc rawWrite*(f: CFilePtr, s: cstring) {.compilerproc, nonReloadable, inline.} = + # we cannot throw an exception here! + discard c_fwrite(s, 1, c_strlen(s), f) + discard c_fflush(f) -{.pop} +{.pop.} diff --git a/lib/system/arc.nim b/lib/system/arc.nim new file mode 100644 index 000000000..d001fcaa5 --- /dev/null +++ b/lib/system/arc.nim @@ -0,0 +1,267 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +#[ +In this new runtime we simplify the object layouts a bit: The runtime type +information is only accessed for the objects that have it and it's always +at offset 0 then. The ``ref`` object header is independent from the +runtime type and only contains a reference count. +]# + +when defined(gcOrc): + const + rcIncrement = 0b10000 # so that lowest 4 bits are not touched + rcMask = 0b1111 + rcShift = 4 # shift by rcShift to get the reference counter + +else: + const + rcIncrement = 0b1000 # so that lowest 3 bits are not touched + rcMask = 0b111 + rcShift = 3 # shift by rcShift to get the reference counter + +const + orcLeakDetector = defined(nimOrcLeakDetector) + +type + RefHeader = object + rc: int # the object header is now a single RC field. + # we could remove it in non-debug builds for the 'owned ref' + # design but this seems unwise. + when defined(gcOrc): + rootIdx: int # thanks to this we can delete potential cycle roots + # in O(1) without doubly linked lists + when defined(nimArcDebug) or defined(nimArcIds): + refId: int + when defined(gcOrc) and orcLeakDetector: + filename: cstring + line: int + + Cell = ptr RefHeader + +template setFrameInfo(c: Cell) = + when orcLeakDetector: + if framePtr != nil and framePtr.prev != nil: + c.filename = framePtr.prev.filename + c.line = framePtr.prev.line + else: + c.filename = nil + c.line = 0 + +template head(p: pointer): Cell = + cast[Cell](cast[int](p) -% sizeof(RefHeader)) + +const + traceCollector = defined(traceArc) + +when defined(nimArcDebug): + include cellsets + + const traceId = 20 # 1037 + + var gRefId: int + var freedCells: CellSet +elif defined(nimArcIds): + var gRefId: int + + const traceId = -1 + +when defined(gcAtomicArc) and hasThreadSupport: + template decrement(cell: Cell): untyped = + discard atomicDec(cell.rc, rcIncrement) + template increment(cell: Cell): untyped = + discard atomicInc(cell.rc, rcIncrement) + template count(x: Cell): untyped = + atomicLoadN(x.rc.addr, ATOMIC_ACQUIRE) shr rcShift +else: + template decrement(cell: Cell): untyped = + dec(cell.rc, rcIncrement) + template increment(cell: Cell): untyped = + inc(cell.rc, rcIncrement) + template count(x: Cell): untyped = + x.rc shr rcShift + +proc nimNewObj(size, alignment: int): pointer {.compilerRtl.} = + let hdrSize = align(sizeof(RefHeader), alignment) + let s = size + hdrSize + when defined(nimscript): + discard + else: + result = alignedAlloc0(s, alignment) +! hdrSize + when defined(nimArcDebug) or defined(nimArcIds): + head(result).refId = gRefId + atomicInc gRefId + if head(result).refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[nimNewObj] %p %ld\n", result, head(result).count) + when traceCollector: + cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result) + setFrameInfo head(result) + +proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} = + # Same as 'newNewObj' but do not initialize the memory to zero. + # The codegen proved for us that this is not necessary. + let hdrSize = align(sizeof(RefHeader), alignment) + let s = size + hdrSize + when defined(nimscript): + discard + else: + result = cast[ptr RefHeader](alignedAlloc(s, alignment) +! hdrSize) + head(result).rc = 0 + when defined(gcOrc): + head(result).rootIdx = 0 + when defined(nimArcDebug): + head(result).refId = gRefId + atomicInc gRefId + if head(result).refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[nimNewObjUninit] %p %ld\n", result, head(result).count) + + when traceCollector: + cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result) + setFrameInfo head(result) + +proc nimDecWeakRef(p: pointer) {.compilerRtl, inl.} = + decrement head(p) + +proc isUniqueRef*[T](x: ref T): bool {.inline.} = + ## Returns true if the object `x` points to is uniquely referenced. Such + ## an object can potentially be passed over to a different thread safely, + ## if great care is taken. This queries the internal reference count of + ## the object which is subject to lots of optimizations! In other words + ## the value of `isUniqueRef` can depend on the used compiler version and + ## optimizer setting. + ## Nevertheless it can be used as a very valuable debugging tool and can + ## be used to specify the constraints of a threading related API + ## via `assert isUniqueRef(x)`. + head(cast[pointer](x)).rc == 0 + +proc nimIncRef(p: pointer) {.compilerRtl, inl.} = + when defined(nimArcDebug): + if head(p).refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[IncRef] %p %ld\n", p, head(p).count) + + increment head(p) + when traceCollector: + cprintf("[INCREF] %p\n", head(p)) + +when not defined(gcOrc) or defined(nimThinout): + proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} = + # This is only used by the old RTTI mechanism and we know + # that 'dest[]' is nil and needs no destruction. Which is really handy + # as we cannot destroy the object reliably if it's an object of unknown + # compile-time type. + dest[] = src + if src != nil: nimIncRef src + +when not defined(nimscript) and defined(nimArcDebug): + proc deallocatedRefId*(p: pointer): int = + ## Returns the ref's ID if the ref was already deallocated. This + ## is a memory corruption check. Returns 0 if there is no error. + let c = head(p) + if freedCells.data != nil and freedCells.contains(c): + result = c.refId + else: + result = 0 + +proc nimRawDispose(p: pointer, alignment: int) {.compilerRtl.} = + when not defined(nimscript): + when traceCollector: + cprintf("[Freed] %p\n", p -! sizeof(RefHeader)) + when defined(nimOwnedEnabled): + if head(p).rc >= rcIncrement: + cstderr.rawWrite "[FATAL] dangling references exist\n" + rawQuit 1 + when defined(nimArcDebug): + # we do NOT really free the memory here in order to reliably detect use-after-frees + if freedCells.data == nil: init(freedCells) + freedCells.incl head(p) + else: + let hdrSize = align(sizeof(RefHeader), alignment) + alignedDealloc(p -! hdrSize, alignment) + +template `=dispose`*[T](x: owned(ref T)) = nimRawDispose(cast[pointer](x), T.alignOf) +#proc dispose*(x: pointer) = nimRawDispose(x) + +proc nimDestroyAndDispose(p: pointer) {.compilerRtl, raises: [].} = + let rti = cast[ptr PNimTypeV2](p) + if rti.destructor != nil: + cast[DestructorProc](rti.destructor)(p) + when false: + cstderr.rawWrite cast[ptr PNimTypeV2](p)[].name + cstderr.rawWrite "\n" + if d == nil: + cstderr.rawWrite "bah, nil\n" + else: + cstderr.rawWrite "has destructor!\n" + nimRawDispose(p, rti.align) + +when defined(gcOrc): + when defined(nimThinout): + include cyclebreaker + else: + include orc + #include cyclecollector + +proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + + when defined(nimArcDebug): + if cell.refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[DecRef] %p %ld\n", p, cell.count) + + when defined(gcAtomicArc) and hasThreadSupport: + # `atomicDec` returns the new value + if atomicDec(cell.rc, rcIncrement) == -rcIncrement: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + else: + if cell.count == 0: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + else: + decrement cell + # According to Lins it's correct to do nothing else here. + when traceCollector: + cprintf("[DECREF] %p\n", cell) + +proc GC_unref*[T](x: ref T) = + ## New runtime only supports this operation for 'ref T'. + var y {.cursor.} = x + `=destroy`(y) + +proc GC_ref*[T](x: ref T) = + ## New runtime only supports this operation for 'ref T'. + if x != nil: nimIncRef(cast[pointer](x)) + +when not defined(gcOrc): + template GC_fullCollect* = + ## Forces a full garbage collection pass. With `--mm:arc` a nop. + discard + +template setupForeignThreadGc* = + ## With `--mm:arc` a nop. + discard + +template tearDownForeignThreadGc* = + ## With `--mm:arc` a nop. + discard + +proc isObjDisplayCheck(source: PNimTypeV2, targetDepth: int16, token: uint32): bool {.compilerRtl, inl.} = + result = targetDepth <= source.depth and source.display[targetDepth] == token + +when defined(gcDestructors): + proc nimGetVTable(p: pointer, index: int): pointer + {.compilerRtl, inline, raises: [].} = + result = cast[ptr PNimTypeV2](p).vTable[index] diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim deleted file mode 100644 index 69c558799..000000000 --- a/lib/system/arithm.nim +++ /dev/null @@ -1,411 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# simple integer arithmetic with overflow checking - -proc raiseOverflow {.compilerproc, noinline.} = - # a single proc to reduce code size to a minimum - sysFatal(OverflowError, "over- or underflow") - -proc raiseDivByZero {.compilerproc, noinline.} = - sysFatal(DivByZeroError, "division by zero") - -when defined(builtinOverflow): - # Builtin compiler functions for improved performance - when sizeof(clong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddl_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubl_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smull_overflow", nodecl, nosideeffect.} - - elif sizeof(clonglong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddll_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubll_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smulll_overflow", nodecl, nosideeffect.} - - when sizeof(int) == 8: - proc addIntOverflow(a, b: int, c: var int): bool {.inline.} = - addInt64Overflow(a, b, c) - - proc subIntOverflow(a, b: int, c: var int): bool {.inline.} = - subInt64Overflow(a, b, c) - - proc mulIntOverflow(a, b: int, c: var int): bool {.inline.} = - mulInt64Overflow(a, b, c) - - elif sizeof(int) == 4 and sizeof(cint) == 4: - proc addIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_sadd_overflow", nodecl, nosideeffect.} - - proc subIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_ssub_overflow", nodecl, nosideeffect.} - - proc mulIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_smul_overflow", nodecl, nosideeffect.} - - proc addInt64(a, b: int64): int64 {.compilerProc, inline.} = - if addInt64Overflow(a, b, result): - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerProc, inline.} = - if subInt64Overflow(a, b, result): - raiseOverflow() - - proc mulInt64(a, b: int64): int64 {.compilerproc, inline.} = - if mulInt64Overflow(a, b, result): - raiseOverflow() -else: - proc addInt64(a, b: int64): int64 {.compilerProc, inline.} = - result = a +% b - if (result xor a) >= int64(0) or (result xor b) >= int64(0): - return result - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerProc, inline.} = - result = a -% b - if (result xor a) >= int64(0) or (result xor not b) >= int64(0): - return result - raiseOverflow() - - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if they're - # approximately the same. Else, because the native int product is the only - # one that can lose catastrophic amounts of information, it's the native int - # product that must have overflowed. - # - proc mulInt64(a, b: int64): int64 {.compilerproc.} = - var - resAsFloat, floatProd: float64 - result = a *% b - floatProd = toBiggestFloat(a) # conversion - floatProd = floatProd * toBiggestFloat(b) - resAsFloat = toBiggestFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -proc negInt64(a: int64): int64 {.compilerProc, inline.} = - if a != low(int64): return -a - raiseOverflow() - -proc absInt64(a: int64): int64 {.compilerProc, inline.} = - if a != low(int64): - if a >= 0: return a - else: return -a - raiseOverflow() - -proc divInt64(a, b: int64): int64 {.compilerProc, inline.} = - if b == int64(0): - raiseDivByZero() - if a == low(int64) and b == int64(-1): - raiseOverflow() - return a div b - -proc modInt64(a, b: int64): int64 {.compilerProc, inline.} = - if b == int64(0): - raiseDivByZero() - return a mod b - -proc absInt(a: int): int {.compilerProc, inline.} = - if a != low(int): - if a >= 0: return a - else: return -a - raiseOverflow() - -const - asmVersion = defined(I386) and (defined(vcc) or defined(wcc) or - defined(dmc) or defined(gcc) or defined(llvm_gcc)) - # my Version of Borland C++Builder does not have - # tasm32, which is needed for assembler blocks - # this is why Borland is not included in the 'when' - -when asmVersion and not defined(gcc) and not defined(llvm_gcc): - # assembler optimized versions for compilers that - # have an intel syntax assembler: - proc addInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = - # a in eax, and b in edx - asm """ - mov eax, ecx - add eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc subInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = - asm """ - mov eax, ecx - sub eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc negInt(a: int): int {.compilerProc, asmNoStackFrame.} = - asm """ - mov eax, ecx - neg eax - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc divInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - idiv ecx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc modInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - idiv ecx - jno theEnd - call `raiseOverflow` - theEnd: - mov eax, edx - ret - """ - - proc mulInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - imul ecx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - -elif false: # asmVersion and (defined(gcc) or defined(llvm_gcc)): - proc addInt(a, b: int): int {.compilerProc, inline.} = - # don't use a pure proc here! - asm """ - "addl %%ecx, %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - #".intel_syntax noprefix" - #/* Intel syntax here */ - #".att_syntax" - - proc subInt(a, b: int): int {.compilerProc, inline.} = - asm """ "subl %%ecx,%%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - - proc mulInt(a, b: int): int {.compilerProc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "imull %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc negInt(a: int): int {.compilerProc, inline.} = - asm """ "negl %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`) - """ - - proc divInt(a, b: int): int {.compilerProc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc modInt(a, b: int): int {.compilerProc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - "movl %%edx, %%eax" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - -when not declared(addInt) and defined(builtinOverflow): - proc addInt(a, b: int): int {.compilerProc, inline.} = - if addIntOverflow(a, b, result): - raiseOverflow() - -when not declared(subInt) and defined(builtinOverflow): - proc subInt(a, b: int): int {.compilerProc, inline.} = - if subIntOverflow(a, b, result): - raiseOverflow() - -when not declared(mulInt) and defined(builtinOverflow): - proc mulInt(a, b: int): int {.compilerProc, inline.} = - if mulIntOverflow(a, b, result): - raiseOverflow() - -# Platform independent versions of the above (slower!) -when not declared(addInt): - proc addInt(a, b: int): int {.compilerProc, inline.} = - result = a +% b - if (result xor a) >= 0 or (result xor b) >= 0: - return result - raiseOverflow() - -when not declared(subInt): - proc subInt(a, b: int): int {.compilerProc, inline.} = - result = a -% b - if (result xor a) >= 0 or (result xor not b) >= 0: - return result - raiseOverflow() - -when not declared(negInt): - proc negInt(a: int): int {.compilerProc, inline.} = - if a != low(int): return -a - raiseOverflow() - -when not declared(divInt): - proc divInt(a, b: int): int {.compilerProc, inline.} = - if b == 0: - raiseDivByZero() - if a == low(int) and b == -1: - raiseOverflow() - return a div b - -when not declared(modInt): - proc modInt(a, b: int): int {.compilerProc, inline.} = - if b == 0: - raiseDivByZero() - return a mod b - -when not declared(mulInt): - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if - # they're approximately the same. Else, because the native int product is - # the only one that can lose catastrophic amounts of information, it's the - # native int product that must have overflowed. - # - proc mulInt(a, b: int): int {.compilerProc.} = - var - resAsFloat, floatProd: float - - result = a *% b - floatProd = toFloat(a) * toFloat(b) - resAsFloat = toFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -# We avoid setting the FPU control word here for compatibility with libraries -# written in other languages. - -proc raiseFloatInvalidOp {.noinline.} = - sysFatal(FloatInvalidOpError, "FPU operation caused a NaN result") - -proc nanCheck(x: float64) {.compilerProc, inline.} = - if x != x: raiseFloatInvalidOp() - -proc raiseFloatOverflow(x: float64) {.noinline.} = - if x > 0.0: - sysFatal(FloatOverflowError, "FPU operation caused an overflow") - else: - sysFatal(FloatUnderflowError, "FPU operations caused an underflow") - -proc infCheck(x: float64) {.compilerProc, inline.} = - if x != 0.0 and x*0.5 == x: raiseFloatOverflow(x) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim new file mode 100644 index 000000000..e229a0f4b --- /dev/null +++ b/lib/system/arithmetics.nim @@ -0,0 +1,405 @@ +proc succ*[T, V: Ordinal](x: T, y: V = 1): T {.magic: "Succ", noSideEffect.} = + ## Returns the `y`-th successor (default: 1) of the value `x`. + ## + ## If such a value does not exist, `OverflowDefect` is raised + ## or a compile time error occurs. + runnableExamples: + assert succ(5) == 6 + assert succ(5, 3) == 8 + +proc pred*[T, V: Ordinal](x: T, y: V = 1): T {.magic: "Pred", noSideEffect.} = + ## Returns the `y`-th predecessor (default: 1) of the value `x`. + ## + ## If such a value does not exist, `OverflowDefect` is raised + ## or a compile time error occurs. + runnableExamples: + assert pred(5) == 4 + assert pred(5, 3) == 2 + +proc inc*[T, V: Ordinal](x: var T, y: V = 1) {.magic: "Inc", noSideEffect.} = + ## Increments the ordinal `x` by `y`. + ## + ## If such a value does not exist, `OverflowDefect` is raised or a compile + ## time error occurs. This is a short notation for: `x = succ(x, y)`. + runnableExamples: + var i = 2 + inc(i) + assert i == 3 + inc(i, 3) + assert i == 6 + +proc dec*[T, V: Ordinal](x: var T, y: V = 1) {.magic: "Dec", noSideEffect.} = + ## Decrements the ordinal `x` by `y`. + ## + ## If such a value does not exist, `OverflowDefect` is raised or a compile + ## time error occurs. This is a short notation for: `x = pred(x, y)`. + runnableExamples: + var i = 2 + dec(i) + assert i == 1 + dec(i, 3) + assert i == -2 + + + +# -------------------------------------------------------------------------- +# built-in operators + +# integer calculations: +proc `+`*(x: int): int {.magic: "UnaryPlusI", noSideEffect.} + ## Unary `+` operator for an integer. Has no effect. +proc `+`*(x: int8): int8 {.magic: "UnaryPlusI", noSideEffect.} +proc `+`*(x: int16): int16 {.magic: "UnaryPlusI", noSideEffect.} +proc `+`*(x: int32): int32 {.magic: "UnaryPlusI", noSideEffect.} +proc `+`*(x: int64): int64 {.magic: "UnaryPlusI", noSideEffect.} + +proc `-`*(x: int): int {.magic: "UnaryMinusI", noSideEffect.} + ## Unary `-` operator for an integer. Negates `x`. +proc `-`*(x: int8): int8 {.magic: "UnaryMinusI", noSideEffect.} +proc `-`*(x: int16): int16 {.magic: "UnaryMinusI", noSideEffect.} +proc `-`*(x: int32): int32 {.magic: "UnaryMinusI", noSideEffect.} +proc `-`*(x: int64): int64 {.magic: "UnaryMinusI64", noSideEffect.} + +proc `not`*(x: int): int {.magic: "BitnotI", noSideEffect.} = + ## Computes the `bitwise complement` of the integer `x`. + runnableExamples: + assert not 0'u8 == 255 + assert not 0'i8 == -1 + assert not 1000'u16 == 64535 + assert not 1000'i16 == -1001 +proc `not`*(x: int8): int8 {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: int16): int16 {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: int32): int32 {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: int64): int64 {.magic: "BitnotI", noSideEffect.} + +proc `+`*(x, y: int): int {.magic: "AddI", noSideEffect.} + ## Binary `+` operator for an integer. +proc `+`*(x, y: int8): int8 {.magic: "AddI", noSideEffect.} +proc `+`*(x, y: int16): int16 {.magic: "AddI", noSideEffect.} +proc `+`*(x, y: int32): int32 {.magic: "AddI", noSideEffect.} +proc `+`*(x, y: int64): int64 {.magic: "AddI", noSideEffect.} + +proc `-`*(x, y: int): int {.magic: "SubI", noSideEffect.} + ## Binary `-` operator for an integer. +proc `-`*(x, y: int8): int8 {.magic: "SubI", noSideEffect.} +proc `-`*(x, y: int16): int16 {.magic: "SubI", noSideEffect.} +proc `-`*(x, y: int32): int32 {.magic: "SubI", noSideEffect.} +proc `-`*(x, y: int64): int64 {.magic: "SubI", noSideEffect.} + +proc `*`*(x, y: int): int {.magic: "MulI", noSideEffect.} + ## Binary `*` operator for an integer. +proc `*`*(x, y: int8): int8 {.magic: "MulI", noSideEffect.} +proc `*`*(x, y: int16): int16 {.magic: "MulI", noSideEffect.} +proc `*`*(x, y: int32): int32 {.magic: "MulI", noSideEffect.} +proc `*`*(x, y: int64): int64 {.magic: "MulI", noSideEffect.} + +proc `div`*(x, y: int): int {.magic: "DivI", noSideEffect.} = + ## Computes the integer division. + ## + ## This is roughly the same as `math.trunc(x/y).int`. + runnableExamples: + assert (1 div 2) == 0 + assert (2 div 2) == 1 + assert (3 div 2) == 1 + assert (7 div 3) == 2 + assert (-7 div 3) == -2 + assert (7 div -3) == -2 + assert (-7 div -3) == 2 +proc `div`*(x, y: int8): int8 {.magic: "DivI", noSideEffect.} +proc `div`*(x, y: int16): int16 {.magic: "DivI", noSideEffect.} +proc `div`*(x, y: int32): int32 {.magic: "DivI", noSideEffect.} +proc `div`*(x, y: int64): int64 {.magic: "DivI", noSideEffect.} + +proc `mod`*(x, y: int): int {.magic: "ModI", noSideEffect.} = + ## Computes the integer modulo operation (remainder). + ## + ## This is the same as `x - (x div y) * y`. + runnableExamples: + assert (7 mod 5) == 2 + assert (-7 mod 5) == -2 + assert (7 mod -5) == 2 + assert (-7 mod -5) == -2 +proc `mod`*(x, y: int8): int8 {.magic: "ModI", noSideEffect.} +proc `mod`*(x, y: int16): int16 {.magic: "ModI", noSideEffect.} +proc `mod`*(x, y: int32): int32 {.magic: "ModI", noSideEffect.} +proc `mod`*(x, y: int64): int64 {.magic: "ModI", noSideEffect.} + +when defined(nimOldShiftRight): + const shrDepMessage = "`shr` will become sign preserving." + proc `shr`*(x: int, y: SomeInteger): int {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} + proc `shr`*(x: int8, y: SomeInteger): int8 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} + proc `shr`*(x: int16, y: SomeInteger): int16 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} + proc `shr`*(x: int32, y: SomeInteger): int32 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} + proc `shr`*(x: int64, y: SomeInteger): int64 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} +else: + proc `shr`*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = + ## Computes the `shift right` operation of `x` and `y`, filling + ## vacant bit positions with the sign bit. + ## + ## **Note**: `Operator precedence <manual.html#syntax-precedence>`_ + ## is different than in *C*. + ## + ## See also: + ## * `ashr func<#ashr,int,SomeInteger>`_ for arithmetic shift right + runnableExamples: + assert 0b0001_0000'i8 shr 2 == 0b0000_0100'i8 + assert 0b0000_0001'i8 shr 1 == 0b0000_0000'i8 + assert 0b1000_0000'i8 shr 4 == 0b1111_1000'i8 + assert -1 shr 5 == -1 + assert 1 shr 5 == 0 + assert 16 shr 2 == 4 + assert -16 shr 2 == -4 + proc `shr`*(x: int8, y: SomeInteger): int8 {.magic: "AshrI", noSideEffect.} + proc `shr`*(x: int16, y: SomeInteger): int16 {.magic: "AshrI", noSideEffect.} + proc `shr`*(x: int32, y: SomeInteger): int32 {.magic: "AshrI", noSideEffect.} + proc `shr`*(x: int64, y: SomeInteger): int64 {.magic: "AshrI", noSideEffect.} + + +proc `shl`*(x: int, y: SomeInteger): int {.magic: "ShlI", noSideEffect.} = + ## Computes the `shift left` operation of `x` and `y`. + ## + ## **Note**: `Operator precedence <manual.html#syntax-precedence>`_ + ## is different than in *C*. + runnableExamples: + assert 1'i32 shl 4 == 0x0000_0010 + assert 1'i64 shl 4 == 0x0000_0000_0000_0010 +proc `shl`*(x: int8, y: SomeInteger): int8 {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: int16, y: SomeInteger): int16 {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: int32, y: SomeInteger): int32 {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: int64, y: SomeInteger): int64 {.magic: "ShlI", noSideEffect.} + +proc ashr*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = + ## Shifts right by pushing copies of the leftmost bit in from the left, + ## and let the rightmost bits fall off. + ## + ## Note that `ashr` is not an operator so use the normal function + ## call syntax for it. + ## + ## See also: + ## * `shr func<#shr,int,SomeInteger>`_ + runnableExamples: + assert ashr(0b0001_0000'i8, 2) == 0b0000_0100'i8 + assert ashr(0b1000_0000'i8, 8) == 0b1111_1111'i8 + assert ashr(0b1000_0000'i8, 1) == 0b1100_0000'i8 +proc ashr*(x: int8, y: SomeInteger): int8 {.magic: "AshrI", noSideEffect.} +proc ashr*(x: int16, y: SomeInteger): int16 {.magic: "AshrI", noSideEffect.} +proc ashr*(x: int32, y: SomeInteger): int32 {.magic: "AshrI", noSideEffect.} +proc ashr*(x: int64, y: SomeInteger): int64 {.magic: "AshrI", noSideEffect.} + +proc `and`*(x, y: int): int {.magic: "BitandI", noSideEffect.} = + ## Computes the `bitwise and` of numbers `x` and `y`. + runnableExamples: + assert (0b0011 and 0b0101) == 0b0001 + assert (0b0111 and 0b1100) == 0b0100 +proc `and`*(x, y: int8): int8 {.magic: "BitandI", noSideEffect.} +proc `and`*(x, y: int16): int16 {.magic: "BitandI", noSideEffect.} +proc `and`*(x, y: int32): int32 {.magic: "BitandI", noSideEffect.} +proc `and`*(x, y: int64): int64 {.magic: "BitandI", noSideEffect.} + +proc `or`*(x, y: int): int {.magic: "BitorI", noSideEffect.} = + ## Computes the `bitwise or` of numbers `x` and `y`. + runnableExamples: + assert (0b0011 or 0b0101) == 0b0111 + assert (0b0111 or 0b1100) == 0b1111 +proc `or`*(x, y: int8): int8 {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: int16): int16 {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: int32): int32 {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: int64): int64 {.magic: "BitorI", noSideEffect.} + +proc `xor`*(x, y: int): int {.magic: "BitxorI", noSideEffect.} = + ## Computes the `bitwise xor` of numbers `x` and `y`. + runnableExamples: + assert (0b0011 xor 0b0101) == 0b0110 + assert (0b0111 xor 0b1100) == 0b1011 +proc `xor`*(x, y: int8): int8 {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: int16): int16 {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: int32): int32 {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: int64): int64 {.magic: "BitxorI", noSideEffect.} + +# unsigned integer operations: +proc `not`*(x: uint): uint {.magic: "BitnotI", noSideEffect.} + ## Computes the `bitwise complement` of the integer `x`. +proc `not`*(x: uint8): uint8 {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: uint16): uint16 {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: uint32): uint32 {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: uint64): uint64 {.magic: "BitnotI", noSideEffect.} + +proc `shr`*(x: uint, y: SomeInteger): uint {.magic: "ShrI", noSideEffect.} + ## Computes the `shift right` operation of `x` and `y`. +proc `shr`*(x: uint8, y: SomeInteger): uint8 {.magic: "ShrI", noSideEffect.} +proc `shr`*(x: uint16, y: SomeInteger): uint16 {.magic: "ShrI", noSideEffect.} +proc `shr`*(x: uint32, y: SomeInteger): uint32 {.magic: "ShrI", noSideEffect.} +proc `shr`*(x: uint64, y: SomeInteger): uint64 {.magic: "ShrI", noSideEffect.} + +proc `shl`*(x: uint, y: SomeInteger): uint {.magic: "ShlI", noSideEffect.} + ## Computes the `shift left` operation of `x` and `y`. +proc `shl`*(x: uint8, y: SomeInteger): uint8 {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: uint16, y: SomeInteger): uint16 {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: uint32, y: SomeInteger): uint32 {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: uint64, y: SomeInteger): uint64 {.magic: "ShlI", noSideEffect.} + +proc `and`*(x, y: uint): uint {.magic: "BitandI", noSideEffect.} + ## Computes the `bitwise and` of numbers `x` and `y`. +proc `and`*(x, y: uint8): uint8 {.magic: "BitandI", noSideEffect.} +proc `and`*(x, y: uint16): uint16 {.magic: "BitandI", noSideEffect.} +proc `and`*(x, y: uint32): uint32 {.magic: "BitandI", noSideEffect.} +proc `and`*(x, y: uint64): uint64 {.magic: "BitandI", noSideEffect.} + +proc `or`*(x, y: uint): uint {.magic: "BitorI", noSideEffect.} + ## Computes the `bitwise or` of numbers `x` and `y`. +proc `or`*(x, y: uint8): uint8 {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: uint16): uint16 {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: uint32): uint32 {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: uint64): uint64 {.magic: "BitorI", noSideEffect.} + +proc `xor`*(x, y: uint): uint {.magic: "BitxorI", noSideEffect.} + ## Computes the `bitwise xor` of numbers `x` and `y`. +proc `xor`*(x, y: uint8): uint8 {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: uint16): uint16 {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: uint32): uint32 {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: uint64): uint64 {.magic: "BitxorI", noSideEffect.} + +proc `+`*(x, y: uint): uint {.magic: "AddU", noSideEffect.} + ## Binary `+` operator for unsigned integers. +proc `+`*(x, y: uint8): uint8 {.magic: "AddU", noSideEffect.} +proc `+`*(x, y: uint16): uint16 {.magic: "AddU", noSideEffect.} +proc `+`*(x, y: uint32): uint32 {.magic: "AddU", noSideEffect.} +proc `+`*(x, y: uint64): uint64 {.magic: "AddU", noSideEffect.} + +proc `-`*(x, y: uint): uint {.magic: "SubU", noSideEffect.} + ## Binary `-` operator for unsigned integers. +proc `-`*(x, y: uint8): uint8 {.magic: "SubU", noSideEffect.} +proc `-`*(x, y: uint16): uint16 {.magic: "SubU", noSideEffect.} +proc `-`*(x, y: uint32): uint32 {.magic: "SubU", noSideEffect.} +proc `-`*(x, y: uint64): uint64 {.magic: "SubU", noSideEffect.} + +proc `*`*(x, y: uint): uint {.magic: "MulU", noSideEffect.} + ## Binary `*` operator for unsigned integers. +proc `*`*(x, y: uint8): uint8 {.magic: "MulU", noSideEffect.} +proc `*`*(x, y: uint16): uint16 {.magic: "MulU", noSideEffect.} +proc `*`*(x, y: uint32): uint32 {.magic: "MulU", noSideEffect.} +proc `*`*(x, y: uint64): uint64 {.magic: "MulU", noSideEffect.} + +proc `div`*(x, y: uint): uint {.magic: "DivU", noSideEffect.} + ## Computes the integer division for unsigned integers. + ## This is roughly the same as `trunc(x/y)`. +proc `div`*(x, y: uint8): uint8 {.magic: "DivU", noSideEffect.} +proc `div`*(x, y: uint16): uint16 {.magic: "DivU", noSideEffect.} +proc `div`*(x, y: uint32): uint32 {.magic: "DivU", noSideEffect.} +proc `div`*(x, y: uint64): uint64 {.magic: "DivU", noSideEffect.} + +proc `mod`*(x, y: uint): uint {.magic: "ModU", noSideEffect.} + ## Computes the integer modulo operation (remainder) for unsigned integers. + ## This is the same as `x - (x div y) * y`. +proc `mod`*(x, y: uint8): uint8 {.magic: "ModU", noSideEffect.} +proc `mod`*(x, y: uint16): uint16 {.magic: "ModU", noSideEffect.} +proc `mod`*(x, y: uint32): uint32 {.magic: "ModU", noSideEffect.} +proc `mod`*(x, y: uint64): uint64 {.magic: "ModU", noSideEffect.} + +proc `+=`*[T: SomeInteger](x: var T, y: T) {. + magic: "Inc", noSideEffect.} + ## Increments an integer. + +proc `-=`*[T: SomeInteger](x: var T, y: T) {. + magic: "Dec", noSideEffect.} + ## Decrements an integer. + +proc `*=`*[T: SomeInteger](x: var T, y: T) {. + inline, noSideEffect.} = + ## Binary `*=` operator for integers. + x = x * y + +# floating point operations: +proc `+`*(x: float32): float32 {.magic: "UnaryPlusF64", noSideEffect.} +proc `-`*(x: float32): float32 {.magic: "UnaryMinusF64", noSideEffect.} +proc `+`*(x, y: float32): float32 {.magic: "AddF64", noSideEffect.} +proc `-`*(x, y: float32): float32 {.magic: "SubF64", noSideEffect.} +proc `*`*(x, y: float32): float32 {.magic: "MulF64", noSideEffect.} +proc `/`*(x, y: float32): float32 {.magic: "DivF64", noSideEffect.} + +proc `+`*(x: float): float {.magic: "UnaryPlusF64", noSideEffect.} +proc `-`*(x: float): float {.magic: "UnaryMinusF64", noSideEffect.} +proc `+`*(x, y: float): float {.magic: "AddF64", noSideEffect.} +proc `-`*(x, y: float): float {.magic: "SubF64", noSideEffect.} +proc `*`*(x, y: float): float {.magic: "MulF64", noSideEffect.} +proc `/`*(x, y: float): float {.magic: "DivF64", noSideEffect.} + +proc `+=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Increments in place a floating point number. + x = x + y + +proc `-=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Decrements in place a floating point number. + x = x - y + +proc `*=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Multiplies in place a floating point number. + x = x * y + +proc `/=`*(x: var float64, y: float64) {.inline, noSideEffect.} = + ## Divides in place a floating point number. + x = x / y + +proc `/=`*[T: float|float32](x: var T, y: T) {.inline, noSideEffect.} = + ## Divides in place a floating point number. + x = x / y + +# the following have to be included in system, not imported for some reason: + +proc `+%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and adds them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) + cast[uint](y)) +proc `+%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) + cast[uint8](y)) +proc `+%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) + cast[uint16](y)) +proc `+%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) + cast[uint32](y)) +proc `+%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) + cast[uint64](y)) + +proc `-%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and subtracts them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) - cast[uint](y)) +proc `-%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) - cast[uint8](y)) +proc `-%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) - cast[uint16](y)) +proc `-%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) - cast[uint32](y)) +proc `-%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) - cast[uint64](y)) + +proc `*%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and multiplies them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) * cast[uint](y)) +proc `*%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) * cast[uint8](y)) +proc `*%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) * cast[uint16](y)) +proc `*%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) * cast[uint32](y)) +proc `*%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) * cast[uint64](y)) + +proc `/%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and divides them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) div cast[uint](y)) +proc `/%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) div cast[uint8](y)) +proc `/%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) div cast[uint16](y)) +proc `/%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) div cast[uint32](y)) +proc `/%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) div cast[uint64](y)) + +proc `%%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and compute the modulo of `x` and `y`. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) mod cast[uint](y)) +proc `%%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) mod cast[uint8](y)) +proc `%%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) mod cast[uint16](y)) +proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast[uint32](y)) +proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 55d7572e2..9f4cbc0fe 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -7,14 +7,16 @@ # distribution, for details about the copyright. # +include seqs_v2_reimpl + proc genericResetAux(dest: pointer, n: ptr TNimNode) {.benign.} proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) {.benign.} proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, shallow: bool) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericAssignAux(cast[pointer](d +% n.offset), @@ -38,64 +40,109 @@ proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, # echo "ugh memory corruption! ", n.kind # quit 1 +template deepSeqAssignImpl(operation, additionalArg) {.dirty.} = + var d = cast[ptr NimSeqV2Reimpl](dest) + var s = cast[ptr NimSeqV2Reimpl](src) + d.len = s.len + let elem = mt.base + d.p = cast[ptr NimSeqPayloadReimpl](newSeqPayload(s.len, elem.size, elem.align)) + + let bs = elem.size + let ba = elem.align + let headerSize = align(sizeof(NimSeqPayloadBase), ba) + + for i in 0..d.len-1: + operation(d.p +! (headerSize+i*bs), s.p +! (headerSize+i*bs), mt.base, additionalArg) + proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericAssignAux 2") case mt.kind of tyString: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil or shallow or ( - cast[PGenericSeq](s2).reserved and seqShallowFlag) != 0: - unsureAsgnRef(x, s2) + when defined(nimSeqsV2): + var x = cast[ptr NimStringV2](dest) + var s2 = cast[ptr NimStringV2](s)[] + nimAsgnStrV2(x[], s2) else: - unsureAsgnRef(x, copyString(cast[NimString](s2))) + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil or shallow or ( + cast[PGenericSeq](s2).reserved and seqShallowFlag) != 0: + unsureAsgnRef(x, s2) + else: + unsureAsgnRef(x, copyString(cast[NimString](s2))) of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil or shallow or (seq.reserved and seqShallowFlag) != 0: - # this can happen! nil sequences are allowed - unsureAsgnRef(x, s2) - return - sysAssert(dest != nil, "genericAssignAux 3") - unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - genericAssignAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base, shallow) + when defined(nimSeqsV2): + deepSeqAssignImpl(genericAssignAux, shallow) + else: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil or shallow or (seq.reserved and seqShallowFlag) != 0: + # this can happen! nil sequences are allowed + unsureAsgnRef(x, s2) + return + sysAssert(dest != nil, "genericAssignAux 3") + if ntfNoRefs in mt.base.flags: + var ss = nimNewSeqOfCap(mt, seq.len) + cast[PGenericSeq](ss).len = seq.len + unsureAsgnRef(x, ss) + var dst = cast[int](cast[PPointer](dest)[]) + copyMem(cast[pointer](dst +% align(GenericSeqSize, mt.base.align)), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align)), + seq.len *% mt.base.size) + else: + unsureAsgnRef(x, newSeq(mt, seq.len)) + var dst = cast[int](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericAssignAux( + cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + mt.base, shallow) of tyObject: - if mt.base != nil: - genericAssignAux(dest, src, mt.base, shallow) + var it = mt.base + # don't use recursion here on the PNimType because the subtype + # check should only be done at the very end: + while it != nil: + genericAssignAux(dest, src, it.node, shallow) + it = it.base genericAssignAux(dest, src, mt.node, shallow) # we need to copy m_type field for tyObject, as it could be empty for # sequence reallocations: - var pint = cast[ptr PNimType](dest) - # We need to copy the *static* type not the dynamic type: - # if p of TB: - # var tbObj = TB(p) - # tbObj of TC # needs to be false! - pint[] = mt # cast[ptr PNimType](src)[] + when defined(nimSeqsV2): + var pint = cast[ptr PNimTypeV2](dest) + #chckObjAsgn(cast[ptr PNimTypeV2](src)[].typeInfoV2, mt) + pint[] = cast[PNimTypeV2](mt.typeInfoV2) + else: + var pint = cast[ptr PNimType](dest) + # We need to copy the *static* type not the dynamic type: + # if p of TB: + # var tbObj = TB(p) + # tbObj of TC # needs to be false! + #c_fprintf(stdout, "%s %s\n", pint[].name, mt.name) + let srcType = cast[ptr PNimType](src)[] + if srcType != nil: + # `!= nil` needed because of cases where object is not initialized properly (see bug #16706) + # note that you can have `srcType == nil` yet `src != nil` + chckObjAsgn(srcType, mt) + pint[] = mt # cast[ptr PNimType](src)[] of tyTuple: genericAssignAux(dest, src, mt.node, shallow) of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - genericAssignAux(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base, shallow) + genericAssignAux(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base, shallow) of tyRef: unsureAsgnRef(cast[PPointer](dest), cast[PPointer](s)[]) else: copyMem(dest, src, mt.size) # copy raw bits -proc genericAssign(dest, src: pointer, mt: PNimType) {.compilerProc.} = +proc genericAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = genericAssignAux(dest, src, mt, false) -proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.compilerProc.} = +proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = genericAssignAux(dest, src, mt, true) when false: @@ -120,29 +167,29 @@ when false: of tyPointer: k = "range" of tyOpenArray: k = "openarray" of tyString: k = "string" - of tyCString: k = "cstring" + of tyCstring: k = "cstring" of tyInt: k = "int" of tyInt32: k = "int32" else: k = "other" cprintf("%s %ld\n", k, t.size) debugNimType(t.base) -proc genericSeqAssign(dest, src: pointer, mt: PNimType) {.compilerProc.} = +proc genericSeqAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = var src = src # ugly, but I like to stress the parser sometimes :-) genericAssign(dest, addr(src), mt) proc genericAssignOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: - genericAssign(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) + genericAssign(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base) -proc objectInit(dest: pointer, typ: PNimType) {.compilerProc, benign.} +proc objectInit(dest: pointer, typ: PNimType) {.compilerproc, benign.} proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "objectInitAux") of nkSlot: objectInit(cast[pointer](d +% n.offset), n.typ) @@ -156,13 +203,17 @@ proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = proc objectInit(dest: pointer, typ: PNimType) = # the generic init proc that takes care of initialization of complex # objects on the stack or heap - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case typ.kind of tyObject: # iterate over any structural type # here we have to init the type field: - var pint = cast[ptr PNimType](dest) - pint[] = typ + when defined(nimSeqsV2): + var pint = cast[ptr PNimTypeV2](dest) + pint[] = cast[PNimTypeV2](typ.typeInfoV2) + else: + var pint = cast[ptr PNimType](dest) + pint[] = typ objectInitAux(dest, typ.node) of tyTuple: objectInitAux(dest, typ.node) @@ -173,14 +224,9 @@ proc objectInit(dest: pointer, typ: PNimType) = # ---------------------- assign zero ----------------------------------------- -proc nimDestroyRange[T](r: T) {.compilerProc.} = - # internal proc used for destroying sequences and arrays - mixin `=destroy` - for i in countup(0, r.len - 1): `=destroy`(r[i]) - -proc genericReset(dest: pointer, mt: PNimType) {.compilerProc, benign.} +proc genericReset(dest: pointer, mt: PNimType) {.compilerproc, benign.} proc genericResetAux(dest: pointer, n: ptr TNimNode) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "genericResetAux") of nkSlot: genericReset(cast[pointer](d +% n.offset), n.typ) @@ -192,35 +238,61 @@ proc genericResetAux(dest: pointer, n: ptr TNimNode) = zeroMem(cast[pointer](d +% n.offset), n.typ.size) proc genericReset(dest: pointer, mt: PNimType) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) sysAssert(mt != nil, "genericReset 2") case mt.kind - of tyString, tyRef, tySequence: + of tyRef: unsureAsgnRef(cast[PPointer](dest), nil) + of tyString: + when defined(nimSeqsV2): + var s = cast[ptr NimStringV2](dest) + frees(s[]) + zeroMem(dest, mt.size) + else: + unsureAsgnRef(cast[PPointer](dest), nil) + of tySequence: + when defined(nimSeqsV2): + frees(cast[ptr NimSeqV2Reimpl](dest)[]) + zeroMem(dest, mt.size) + else: + unsureAsgnRef(cast[PPointer](dest), nil) of tyTuple: genericResetAux(dest, mt.node) of tyObject: genericResetAux(dest, mt.node) # also reset the type field for tyObject, for correct branch switching! - var pint = cast[ptr PNimType](dest) - pint[] = nil + when defined(nimSeqsV2): + var pint = cast[ptr PNimTypeV2](dest) + pint[] = nil + else: + var pint = cast[ptr PNimType](dest) + pint[] = nil of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - genericReset(cast[pointer](d +% i*% mt.base.size), mt.base) + genericReset(cast[pointer](d +% i *% mt.base.size), mt.base) else: zeroMem(dest, mt.size) # set raw bits to zero proc selectBranch(discVal, L: int, - a: ptr array [0..0x7fff, ptr TNimNode]): ptr TNimNode = - result = a[L] # a[L] contains the ``else`` part (but may be nil) + a: ptr array[0x7fff, ptr TNimNode]): ptr TNimNode = if discVal <% L: - var x = a[discVal] - if x != nil: result = x + result = a[discVal] + if result == nil: + result = a[L] + else: + result = a[L] # a[L] contains the ``else`` part (but may be nil) proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, - a: ptr array [0..0x7fff, ptr TNimNode], - L: int) {.compilerProc.} = - var oldBranch = selectBranch(oldDiscVal, L, a) - var newBranch = selectBranch(newDiscVal, L, a) - if newBranch != oldBranch and oldDiscVal != 0: - sysFatal(FieldError, "assignment to discriminant changes object branch") + a: ptr array[0x7fff, ptr TNimNode], + L: int) {.compilerproc.} = + let oldBranch = selectBranch(oldDiscVal, L, a) + let newBranch = selectBranch(newDiscVal, L, a) + when defined(nimOldCaseObjects): + if newBranch != oldBranch and oldDiscVal != 0: + sysFatal(FieldDefect, "assignment to discriminant changes object branch") + else: + if newBranch != oldBranch: + if oldDiscVal != 0: + sysFatal(FieldDefect, "assignment to discriminant changes object branch") + else: + sysFatal(FieldDefect, "assignment to discriminant changes object branch; compile with -d:nimOldCaseObjects for a transition period") diff --git a/lib/system/avltree.nim b/lib/system/avltree.nim index d5c901542..8d4b7e897 100644 --- a/lib/system/avltree.nim +++ b/lib/system/avltree.nim @@ -9,7 +9,7 @@ # not really an AVL tree anymore, but still balanced ... -template isBottom(n: PAvlNode): bool = n == bottom +template isBottom(n: PAvlNode): bool = n.link[0] == n proc lowGauge(n: PAvlNode): int = var it = n @@ -52,12 +52,18 @@ proc split(t: var PAvlNode) = inc t.level proc add(a: var MemRegion, t: var PAvlNode, key, upperBound: int) {.benign.} = - if t == bottom: + if t.isBottom: t = allocAvlNode(a, key, upperBound) else: if key <% t.key: + when defined(avlcorruption): + if t.link[0] == nil: + cprintf("bug here %p\n", t) add(a, t.link[0], key, upperBound) elif key >% t.key: + when defined(avlcorruption): + if t.link[1] == nil: + cprintf("bug here B %p\n", t) add(a, t.link[1], key, upperBound) else: sysAssert false, "key already exists" @@ -65,17 +71,17 @@ proc add(a: var MemRegion, t: var PAvlNode, key, upperBound: int) {.benign.} = split(t) proc del(a: var MemRegion, t: var PAvlNode, x: int) {.benign.} = - if t == bottom: return + if isBottom(t): return a.last = t if x <% t.key: del(a, t.link[0], x) else: a.deleted = t del(a, t.link[1], x) - if t == a.last and a.deleted != bottom and x == a.deleted.key: + if t == a.last and not isBottom(a.deleted) and x == a.deleted.key: a.deleted.key = t.key a.deleted.upperBound = t.upperBound - a.deleted = bottom + a.deleted = getBottom(a) t = t.link[1] deallocAvlNode(a, a.last) elif t.link[0].level < t.level-1 or diff --git a/lib/system/basic_types.nim b/lib/system/basic_types.nim new file mode 100644 index 000000000..bf81b9b6a --- /dev/null +++ b/lib/system/basic_types.nim @@ -0,0 +1,94 @@ +type + int* {.magic: Int.} ## Default integer type; bitwidth depends on + ## architecture, but is always the same as a pointer. + int8* {.magic: Int8.} ## Signed 8 bit integer type. + int16* {.magic: Int16.} ## Signed 16 bit integer type. + int32* {.magic: Int32.} ## Signed 32 bit integer type. + int64* {.magic: Int64.} ## Signed 64 bit integer type. + uint* {.magic: UInt.} ## Unsigned default integer type. + uint8* {.magic: UInt8.} ## Unsigned 8 bit integer type. + uint16* {.magic: UInt16.} ## Unsigned 16 bit integer type. + uint32* {.magic: UInt32.} ## Unsigned 32 bit integer type. + uint64* {.magic: UInt64.} ## Unsigned 64 bit integer type. + +type + float* {.magic: Float.} ## Default floating point type. + float32* {.magic: Float32.} ## 32 bit floating point type. + float64* {.magic: Float.} ## 64 bit floating point type. + +# 'float64' is now an alias to 'float'; this solves many problems + +type + char* {.magic: Char.} ## Built-in 8 bit character type (unsigned). + string* {.magic: String.} ## Built-in string type. + cstring* {.magic: Cstring.} ## Built-in cstring (*compatible string*) type. + pointer* {.magic: Pointer.} ## Built-in pointer type, use the `addr` + ## operator to get a pointer to a variable. + + typedesc* {.magic: TypeDesc.} ## Meta type to denote a type description. + +type + `ptr`*[T] {.magic: Pointer.} ## Built-in generic untraced pointer type. + `ref`*[T] {.magic: Pointer.} ## Built-in generic traced pointer type. + + `nil` {.magic: "Nil".} + + void* {.magic: "VoidType".} ## Meta type to denote the absence of any type. + auto* {.magic: Expr.} ## Meta type for automatic type determination. + any* {.deprecated: "Deprecated since v1.5; Use auto instead.".} = distinct auto ## Deprecated; Use `auto` instead. See https://github.com/nim-lang/RFCs/issues/281 + untyped* {.magic: Expr.} ## Meta type to denote an expression that + ## is not resolved (for templates). + typed* {.magic: Stmt.} ## Meta type to denote an expression that + ## is resolved (for templates). + +type # we need to start a new type section here, so that ``0`` can have a type + bool* {.magic: "Bool".} = enum ## Built-in boolean type. + false = 0, true = 1 + +const + on* = true ## Alias for `true`. + off* = false ## Alias for `false`. + +type + SomeSignedInt* = int|int8|int16|int32|int64 + ## Type class matching all signed integer types. + + SomeUnsignedInt* = uint|uint8|uint16|uint32|uint64 + ## Type class matching all unsigned integer types. + + SomeInteger* = SomeSignedInt|SomeUnsignedInt + ## Type class matching all integer types. + + SomeFloat* = float|float32|float64 + ## Type class matching all floating point number types. + + SomeNumber* = SomeInteger|SomeFloat + ## Type class matching all number types. + + SomeOrdinal* = int|int8|int16|int32|int64|bool|enum|uint|uint8|uint16|uint32|uint64 + ## Type class matching all ordinal types; however this includes enums with + ## holes. See also `Ordinal` + + +{.push warning[GcMem]: off, warning[Uninit]: off.} +{.push hints: off.} + +proc `not`*(x: bool): bool {.magic: "Not", noSideEffect.} + ## Boolean not; returns true if `x == false`. + +proc `and`*(x, y: bool): bool {.magic: "And", noSideEffect.} + ## Boolean `and`; returns true if `x == y == true` (if both arguments + ## are true). + ## + ## Evaluation is lazy: if `x` is false, `y` will not even be evaluated. +proc `or`*(x, y: bool): bool {.magic: "Or", noSideEffect.} + ## Boolean `or`; returns true if `not (not x and not y)` (if any of + ## the arguments is true). + ## + ## Evaluation is lazy: if `x` is true, `y` will not even be evaluated. +proc `xor`*(x, y: bool): bool {.magic: "Xor", noSideEffect.} + ## Boolean `exclusive or`; returns true if `x != y` (if either argument + ## is true while the other is false). + +{.pop.} +{.pop.} diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim new file mode 100644 index 000000000..0663247c2 --- /dev/null +++ b/lib/system/bitmasks.nim @@ -0,0 +1,39 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Page size of the system; in most cases 4096 bytes. For exotic OS or +# CPU this needs to be changed: +const + PageShift = when defined(nimPage256) or defined(cpu16): 3 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + + PageSize = 1 shl PageShift + PageMask = PageSize-1 + + + MemAlign = # also minimal allocatable memory block + when defined(nimMemAlignTiny): 4 + elif defined(useMalloc): + when defined(amd64): 16 + else: 8 + else: 16 + + BitsPerPage = PageSize div MemAlign + UnitsPerPage = BitsPerPage div (sizeof(int)*8) + # how many ints do we need to describe a page: + # on 32 bit systems this is only 16 (!) + + TrunkShift = 9 + BitsPerTrunk = 1 shl TrunkShift # needs to be power of 2 and divisible by 64 + TrunkMask = BitsPerTrunk - 1 + IntsPerTrunk = BitsPerTrunk div (sizeof(int)*8) + IntShift = 5 + ord(sizeof(int) == 8) # 5 or 6, depending on int width + IntMask = 1 shl IntShift - 1 diff --git a/lib/system/cellseqs_v1.nim b/lib/system/cellseqs_v1.nim new file mode 100644 index 000000000..1a305aa42 --- /dev/null +++ b/lib/system/cellseqs_v1.nim @@ -0,0 +1,46 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# ------------------- cell seq handling --------------------------------------- + +type + PCellArray = ptr UncheckedArray[PCell] + CellSeq {.final, pure.} = object + len, cap: int + d: PCellArray + +proc contains(s: CellSeq, c: PCell): bool {.inline.} = + for i in 0 ..< s.len: + if s.d[i] == c: + return true + return false + +proc resize(s: var CellSeq) = + s.cap = s.cap * 3 div 2 + let d = cast[PCellArray](alloc(s.cap * sizeof(PCell))) + copyMem(d, s.d, s.len * sizeof(PCell)) + dealloc(s.d) + s.d = d + +proc add(s: var CellSeq, c: PCell) {.inline.} = + if s.len >= s.cap: + resize(s) + s.d[s.len] = c + inc(s.len) + +proc init(s: var CellSeq, cap: int = 1024) = + s.len = 0 + s.cap = cap + s.d = cast[PCellArray](alloc0(cap * sizeof(PCell))) + +proc deinit(s: var CellSeq) = + dealloc(s.d) + s.d = nil + s.len = 0 + s.cap = 0 diff --git a/lib/system/cellseqs_v2.nim b/lib/system/cellseqs_v2.nim new file mode 100644 index 000000000..c6c7b1a8e --- /dev/null +++ b/lib/system/cellseqs_v2.nim @@ -0,0 +1,53 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Cell seqs for cyclebreaker and cyclicrefs_v2. + +type + CellTuple[T] = (T, PNimTypeV2) + CellArray[T] = ptr UncheckedArray[CellTuple[T]] + CellSeq[T] = object + len, cap: int + d: CellArray[T] + +proc resize[T](s: var CellSeq[T]) = + s.cap = s.cap * 3 div 2 + var newSize = s.cap * sizeof(CellTuple[T]) + when compileOption("threads"): + s.d = cast[CellArray[T]](reallocShared(s.d, newSize)) + else: + s.d = cast[CellArray[T]](realloc(s.d, newSize)) + +proc add[T](s: var CellSeq[T], c: T, t: PNimTypeV2) {.inline.} = + if s.len >= s.cap: + s.resize() + s.d[s.len] = (c, t) + inc(s.len) + +proc init[T](s: var CellSeq[T], cap: int = 1024) = + s.len = 0 + s.cap = cap + when compileOption("threads"): + s.d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) + else: + s.d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) + +proc deinit[T](s: var CellSeq[T]) = + if s.d != nil: + when compileOption("threads"): + deallocShared(s.d) + else: + dealloc(s.d) + s.d = nil + s.len = 0 + s.cap = 0 + +proc pop[T](s: var CellSeq[T]): (T, PNimTypeV2) = + result = s.d[s.len-1] + dec s.len diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index 776a2b7ec..92036c226 100644 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -7,69 +7,81 @@ # distribution, for details about the copyright. # -# Efficient set of pointers for the GC (and repr) -type - RefCount = int +#[ + +Efficient set of pointers for the GC (and repr) +----------------------------------------------- + +The GC depends on an extremely efficient datastructure for storing a +set of pointers - this is called a `CellSet` in the source code. +Inserting, deleting and searching are done in constant time. However, +modifying a `CellSet` during traversal leads to undefined behaviour. + +All operations on a CellSet have to perform efficiently. Because a Cellset can +become huge a hash table alone is not suitable for this. + +We use a mixture of bitset and hash table for this. The hash table maps *pages* +to a page descriptor. The page descriptor contains a bit for any possible cell +address within this page. So including a cell is done as follows: + +- Find the page descriptor for the page the cell belongs to. +- Set the appropriate bit in the page descriptor indicating that the + cell points to the start of a memory block. + +Removing a cell is analogous - the bit has to be set to zero. +Single page descriptors are never deleted from the hash table. This is not +needed as the data structures needs to be rebuilt periodically anyway. + +Complete traversal is done in this way:: - Cell {.pure.} = object - refcount: RefCount # the refcount and some flags - typ: PNimType - when trackAllocationSource: - filename: cstring - line: int - when useCellIds: - id: int + for each page descriptor d: + for each bit in d: + if bit == 1: + traverse the pointer belonging to this bit - PCell = ptr Cell +]# +when defined(gcOrc) or defined(gcArc) or defined(gcAtomicArc): + type + PCell = Cell + + when not declaredInScope(PageShift): + include bitmasks + +else: + type + RefCount = int + + Cell {.pure.} = object + refcount: RefCount # the refcount and some flags + typ: PNimType + when trackAllocationSource: + filename: cstring + line: int + when useCellIds: + id: int + + PCell = ptr Cell + +type PPageDesc = ptr PageDesc BitIndex = range[0..UnitsPerPage-1] PageDesc {.final, pure.} = object next: PPageDesc # all nodes are connected with this pointer - key: ByteAddress # start address at bit 0 + key: uint # start address at bit 0 bits: array[BitIndex, int] # a bit vector - PPageDescArray = ptr array[0..1000_000, PPageDesc] + PPageDescArray = ptr UncheckedArray[PPageDesc] CellSet {.final, pure.} = object counter, max: int head: PPageDesc data: PPageDescArray - PCellArray = ptr array[0..100_000_000, PCell] - CellSeq {.final, pure.} = object - len, cap: int - d: PCellArray -{.deprecated: [TCell: Cell, TBitIndex: BitIndex, TPageDesc: PageDesc, - TRefCount: RefCount, TCellSet: CellSet, TCellSeq: CellSeq].} -# ------------------- cell seq handling --------------------------------------- - -proc contains(s: CellSeq, c: PCell): bool {.inline.} = - for i in 0 .. s.len-1: - if s.d[i] == c: return true - return false - -proc add(s: var CellSeq, c: PCell) {.inline.} = - if s.len >= s.cap: - s.cap = s.cap * 3 div 2 - var d = cast[PCellArray](alloc(s.cap * sizeof(PCell))) - copyMem(d, s.d, s.len * sizeof(PCell)) - dealloc(s.d) - s.d = d - # XXX: realloc? - s.d[s.len] = c - inc(s.len) - -proc init(s: var CellSeq, cap: int = 1024) = - s.len = 0 - s.cap = cap - s.d = cast[PCellArray](alloc0(cap * sizeof(PCell))) - -proc deinit(s: var CellSeq) = - dealloc(s.d) - s.d = nil - s.len = 0 - s.cap = 0 +when defined(gcOrc) or defined(gcArc) or defined(gcAtomicArc): + discard +else: + include cellseqs_v1 # ------------------- cell set handling --------------------------------------- @@ -99,7 +111,7 @@ proc nextTry(h, maxHash: int): int {.inline.} = # generates each int in range(maxHash) exactly once (see any text on # random-number generation for proof). -proc cellSetGet(t: CellSet, key: ByteAddress): PPageDesc = +proc cellSetGet(t: CellSet, key: uint): PPageDesc = var h = cast[int](key) and t.max while t.data[h] != nil: if t.data[h].key == key: return t.data[h] @@ -124,7 +136,7 @@ proc cellSetEnlarge(t: var CellSet) = dealloc(t.data) t.data = n -proc cellSetPut(t: var CellSet, key: ByteAddress): PPageDesc = +proc cellSetPut(t: var CellSet, key: uint): PPageDesc = var h = cast[int](key) and t.max while true: var x = t.data[h] @@ -148,33 +160,33 @@ proc cellSetPut(t: var CellSet, key: ByteAddress): PPageDesc = # ---------- slightly higher level procs -------------------------------------- proc contains(s: CellSet, cell: PCell): bool = - var u = cast[ByteAddress](cell) + var u = cast[uint](cell) var t = cellSetGet(s, u shr PageShift) if t != nil: - u = (u %% PageSize) /% MemAlign + u = (u mod PageSize) div MemAlign result = (t.bits[u shr IntShift] and (1 shl (u and IntMask))) != 0 else: result = false -proc incl(s: var CellSet, cell: PCell) {.noinline.} = - var u = cast[ByteAddress](cell) +proc incl(s: var CellSet, cell: PCell) = + var u = cast[uint](cell) var t = cellSetPut(s, u shr PageShift) - u = (u %% PageSize) /% MemAlign + u = (u mod PageSize) div MemAlign t.bits[u shr IntShift] = t.bits[u shr IntShift] or (1 shl (u and IntMask)) proc excl(s: var CellSet, cell: PCell) = - var u = cast[ByteAddress](cell) + var u = cast[uint](cell) var t = cellSetGet(s, u shr PageShift) if t != nil: - u = (u %% PageSize) /% MemAlign + u = (u mod PageSize) div MemAlign t.bits[u shr IntShift] = (t.bits[u shr IntShift] and not (1 shl (u and IntMask))) proc containsOrIncl(s: var CellSet, cell: PCell): bool = - var u = cast[ByteAddress](cell) + var u = cast[uint](cell) var t = cellSetGet(s, u shr PageShift) if t != nil: - u = (u %% PageSize) /% MemAlign + u = (u mod PageSize) div MemAlign result = (t.bits[u shr IntShift] and (1 shl (u and IntMask))) != 0 if not result: t.bits[u shr IntShift] = t.bits[u shr IntShift] or @@ -187,15 +199,15 @@ iterator elements(t: CellSet): PCell {.inline.} = # while traversing it is forbidden to add pointers to the tree! var r = t.head while r != nil: - var i = 0 - while i <= high(r.bits): + var i: uint = 0 + while int(i) <= high(r.bits): var w = r.bits[i] # taking a copy of r.bits[i] here is correct, because # modifying operations are not allowed during traversation - var j = 0 + var j: uint = 0 while w != 0: # test all remaining bits for zero if (w and 1) != 0: # the bit is set! yield cast[PCell]((r.key shl PageShift) or - (i shl IntShift +% j) *% MemAlign) + (i shl IntShift + j) * MemAlign) inc(j) w = w shr 1 inc(i) @@ -240,16 +252,16 @@ iterator elementsExcept(t, s: CellSet): PCell {.inline.} = var r = t.head while r != nil: let ss = cellSetGet(s, r.key) - var i = 0 - while i <= high(r.bits): + var i:uint = 0 + while int(i) <= high(r.bits): var w = r.bits[i] if ss != nil: w = w and not ss.bits[i] - var j = 0 + var j:uint = 0 while w != 0: if (w and 1) != 0: yield cast[PCell]((r.key shl PageShift) or - (i shl IntShift +% j) *% MemAlign) + (i shl IntShift + j) * MemAlign) inc(j) w = w shr 1 inc(i) diff --git a/lib/system/cgprocs.nim b/lib/system/cgprocs.nim index 660c68116..9a7645f9b 100644 --- a/lib/system/cgprocs.nim +++ b/lib/system/cgprocs.nim @@ -8,14 +8,3 @@ # # Headers for procs that the code generator depends on ("compilerprocs") - -type - LibHandle = pointer # private type - ProcAddr = pointer # library loading and loading of procs: -{.deprecated: [TLibHandle: LibHandle, TProcAddr: ProcAddr].} - -proc nimLoadLibrary(path: string): LibHandle {.compilerproc.} -proc nimUnloadLibrary(lib: LibHandle) {.compilerproc.} -proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc.} - -proc nimLoadLibraryError(path: string) {.compilerproc, noinline.} diff --git a/lib/system/channels.nim b/lib/system/channels.nim deleted file mode 100644 index 68c0e32d2..000000000 --- a/lib/system/channels.nim +++ /dev/null @@ -1,267 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Channel support for threads. **Note**: This is part of the system module. -## Do not import it directly. To activate thread support you need to compile -## with the ``--threads:on`` command line switch. -## -## **Note:** The current implementation of message passing is slow and does -## not work with cyclic data structures. - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -type - pbytes = ptr array[0.. 0xffff, byte] - RawChannel {.pure, final.} = object ## msg queue for a thread - rd, wr, count, mask: int - data: pbytes - lock: SysLock - cond: SysCond - elemType: PNimType - ready: bool - region: MemRegion - PRawChannel = ptr RawChannel - LoadStoreMode = enum mStore, mLoad - Channel* {.gcsafe.}[TMsg] = RawChannel ## a channel for thread communication -{.deprecated: [TRawChannel: RawChannel, TLoadStoreMode: LoadStoreMode, - TChannel: Channel].} - -const ChannelDeadMask = -2 - -proc initRawChannel(p: pointer) = - var c = cast[PRawChannel](p) - initSysLock(c.lock) - initSysCond(c.cond) - c.mask = -1 - -proc deinitRawChannel(p: pointer) = - var c = cast[PRawChannel](p) - # we need to grab the lock to be safe against sending threads! - acquireSys(c.lock) - c.mask = ChannelDeadMask - deallocOsPages(c.region) - deinitSys(c.lock) - deinitSysCond(c.cond) - -proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, - mode: LoadStoreMode) {.benign.} -proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, - mode: LoadStoreMode) {.benign.} = - var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) - case n.kind - of nkSlot: storeAux(cast[pointer](d +% n.offset), - cast[pointer](s +% n.offset), n.typ, t, mode) - of nkList: - for i in 0..n.len-1: storeAux(dest, src, n.sons[i], t, mode) - of nkCase: - copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), - n.typ.size) - var m = selectBranch(src, n) - if m != nil: storeAux(dest, src, m, t, mode) - of nkNone: sysAssert(false, "storeAux") - -proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, - mode: LoadStoreMode) = - var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) - sysAssert(mt != nil, "mt == nil") - case mt.kind - of tyString: - if mode == mStore: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil: - x[] = nil - else: - var ss = cast[NimString](s2) - var ns = cast[NimString](alloc(t.region, ss.len+1 + GenericSeqSize)) - copyMem(ns, ss, ss.len+1 + GenericSeqSize) - x[] = ns - else: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil: - unsureAsgnRef(x, s2) - else: - unsureAsgnRef(x, copyString(cast[NimString](s2))) - dealloc(t.region, s2) - of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil: - if mode == mStore: - x[] = nil - else: - unsureAsgnRef(x, nil) - else: - sysAssert(dest != nil, "dest == nil") - if mode == mStore: - x[] = alloc(t.region, seq.len *% mt.base.size +% GenericSeqSize) - else: - unsureAsgnRef(x, newObj(mt, seq.len * mt.base.size + GenericSeqSize)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - storeAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base, t, mode) - var dstseq = cast[PGenericSeq](dst) - dstseq.len = seq.len - dstseq.reserved = seq.len - if mode != mStore: dealloc(t.region, s2) - of tyObject: - # copy type field: - var pint = cast[ptr PNimType](dest) - # XXX use dynamic type here! - pint[] = mt - if mt.base != nil: - storeAux(dest, src, mt.base, t, mode) - storeAux(dest, src, mt.node, t, mode) - of tyTuple: - storeAux(dest, src, mt.node, t, mode) - of tyArray, tyArrayConstr: - for i in 0..(mt.size div mt.base.size)-1: - storeAux(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base, t, mode) - of tyRef: - var s = cast[PPointer](src)[] - var x = cast[PPointer](dest) - if s == nil: - if mode == mStore: - x[] = nil - else: - unsureAsgnRef(x, nil) - else: - if mode == mStore: - x[] = alloc(t.region, mt.base.size) - else: - # XXX we should use the dynamic type here too, but that is not stored - # in the inbox at all --> use source[]'s object type? but how? we need - # a tyRef to the object! - var obj = newObj(mt, mt.base.size) - unsureAsgnRef(x, obj) - storeAux(x[], s, mt.base, t, mode) - if mode != mStore: dealloc(t.region, s) - else: - copyMem(dest, src, mt.size) # copy raw bits - -proc rawSend(q: PRawChannel, data: pointer, typ: PNimType) = - ## adds an `item` to the end of the queue `q`. - var cap = q.mask+1 - if q.count >= cap: - # start with capacity for 2 entries in the queue: - if cap == 0: cap = 1 - var n = cast[pbytes](alloc0(q.region, cap*2*typ.size)) - var z = 0 - var i = q.rd - var c = q.count - while c > 0: - dec c - copyMem(addr(n[z*typ.size]), addr(q.data[i*typ.size]), typ.size) - i = (i + 1) and q.mask - inc z - if q.data != nil: dealloc(q.region, q.data) - q.data = n - q.mask = cap*2 - 1 - q.wr = q.count - q.rd = 0 - storeAux(addr(q.data[q.wr * typ.size]), data, typ, q, mStore) - inc q.count - q.wr = (q.wr + 1) and q.mask - -proc rawRecv(q: PRawChannel, data: pointer, typ: PNimType) = - sysAssert q.count > 0, "rawRecv" - dec q.count - storeAux(data, addr(q.data[q.rd * typ.size]), typ, q, mLoad) - q.rd = (q.rd + 1) and q.mask - -template lockChannel(q: expr, action: stmt) {.immediate.} = - acquireSys(q.lock) - action - releaseSys(q.lock) - -template sendImpl(q: expr) {.immediate.} = - if q.mask == ChannelDeadMask: - sysFatal(DeadThreadError, "cannot send message; thread died") - acquireSys(q.lock) - var m: TMsg - shallowCopy(m, msg) - var typ = cast[PNimType](getTypeInfo(msg)) - rawSend(q, addr(m), typ) - q.elemType = typ - releaseSys(q.lock) - signalSysCond(q.cond) - -proc send*[TMsg](c: var Channel[TMsg], msg: TMsg) = - ## sends a message to a thread. `msg` is deeply copied. - var q = cast[PRawChannel](addr(c)) - sendImpl(q) - -proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) = - # to save space, the generic is as small as possible - q.ready = true - while q.count <= 0: - waitSysCond(q.cond, q.lock) - q.ready = false - if typ != q.elemType: - releaseSys(q.lock) - sysFatal(ValueError, "cannot receive message of wrong type") - rawRecv(q, res, typ) - -proc recv*[TMsg](c: var Channel[TMsg]): TMsg = - ## receives a message from the channel `c`. This blocks until - ## a message has arrived! You may use ``peek`` to avoid the blocking. - var q = cast[PRawChannel](addr(c)) - acquireSys(q.lock) - llRecv(q, addr(result), cast[PNimType](getTypeInfo(result))) - releaseSys(q.lock) - -proc tryRecv*[TMsg](c: var Channel[TMsg]): tuple[dataAvailable: bool, - msg: TMsg] = - ## try to receives a message from the channel `c` if available. Otherwise - ## it returns ``(false, default(msg))``. - var q = cast[PRawChannel](addr(c)) - if q.mask != ChannelDeadMask: - if tryAcquireSys(q.lock): - if q.count > 0: - llRecv(q, addr(result.msg), cast[PNimType](getTypeInfo(result.msg))) - result.dataAvailable = true - releaseSys(q.lock) - -proc peek*[TMsg](c: var Channel[TMsg]): int = - ## returns the current number of messages in the channel `c`. Returns -1 - ## if the channel has been closed. **Note**: This is dangerous to use - ## as it encourages races. It's much better to use ``tryRecv`` instead. - var q = cast[PRawChannel](addr(c)) - if q.mask != ChannelDeadMask: - lockChannel(q): - result = q.count - else: - result = -1 - -proc open*[TMsg](c: var Channel[TMsg]) = - ## opens a channel `c` for inter thread communication. - initRawChannel(addr(c)) - -proc close*[TMsg](c: var Channel[TMsg]) = - ## closes a channel `c` and frees its associated resources. - deinitRawChannel(addr(c)) - -proc ready*[TMsg](c: var Channel[TMsg]): bool = - ## returns true iff some thread is waiting on the channel `c` for - ## new messages. - var q = cast[PRawChannel](addr(c)) - result = q.ready - diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim new file mode 100644 index 000000000..02b4d8cbf --- /dev/null +++ b/lib/system/channels_builtin.nim @@ -0,0 +1,459 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Channel support for threads. +## +## **Note**: This is part of the system module. Do not import it directly. +## To activate thread support compile with the `--threads:on` command line switch. +## +## **Note:** Channels are designed for the `Thread` type. They are unstable when +## used with `spawn` +## +## **Note:** The current implementation of message passing does +## not work with cyclic data structures. +## +## **Note:** Channels cannot be passed between threads. Use globals or pass +## them by `ptr`. +## +## Example +## ======= +## The following is a simple example of two different ways to use channels: +## blocking and non-blocking. +## +## ```Nim +## # Be sure to compile with --threads:on. +## # The channels and threads modules are part of system and should not be +## # imported. +## import std/os +## +## # Channels can either be: +## # - declared at the module level, or +## # - passed to procedures by ptr (raw pointer) -- see note on safety. +## # +## # For simplicity, in this example a channel is declared at module scope. +## # Channels are generic, and they include support for passing objects between +## # threads. +## # Note that objects passed through channels will be deeply copied. +## var chan: Channel[string] +## +## # This proc will be run in another thread using the threads module. +## proc firstWorker() = +## chan.send("Hello World!") +## +## # This is another proc to run in a background thread. This proc takes a while +## # to send the message since it sleeps for 2 seconds (or 2000 milliseconds). +## proc secondWorker() = +## sleep(2000) +## chan.send("Another message") +## +## # Initialize the channel. +## chan.open() +## +## # Launch the worker. +## var worker1: Thread[void] +## createThread(worker1, firstWorker) +## +## # Block until the message arrives, then print it out. +## echo chan.recv() # "Hello World!" +## +## # Wait for the thread to exit before moving on to the next example. +## worker1.joinThread() +## +## # Launch the other worker. +## var worker2: Thread[void] +## createThread(worker2, secondWorker) +## # This time, use a non-blocking approach with tryRecv. +## # Since the main thread is not blocked, it could be used to perform other +## # useful work while it waits for data to arrive on the channel. +## while true: +## let tried = chan.tryRecv() +## if tried.dataAvailable: +## echo tried.msg # "Another message" +## break +## +## echo "Pretend I'm doing useful work..." +## # For this example, sleep in order not to flood stdout with the above +## # message. +## sleep(400) +## +## # Wait for the second thread to exit before cleaning up the channel. +## worker2.joinThread() +## +## # Clean up the channel. +## chan.close() +## ``` +## +## Sample output +## ------------- +## The program should output something similar to this, but keep in mind that +## exact results may vary in the real world: +## +## Hello World! +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Another message +## +## Passing Channels Safely +## ----------------------- +## Note that when passing objects to procedures on another thread by pointer +## (for example through a thread's argument), objects created using the default +## allocator will use thread-local, GC-managed memory. Thus it is generally +## safer to store channel objects in global variables (as in the above example), +## in which case they will use a process-wide (thread-safe) shared heap. +## +## However, it is possible to manually allocate shared memory for channels +## using e.g. `system.allocShared0` and pass these pointers through thread +## arguments: +## +## ```Nim +## proc worker(channel: ptr Channel[string]) = +## let greeting = channel[].recv() +## echo greeting +## +## proc localChannelExample() = +## # Use allocShared0 to allocate some shared-heap memory and zero it. +## # The usual warnings about dealing with raw pointers apply. Exercise caution. +## var channel = cast[ptr Channel[string]]( +## allocShared0(sizeof(Channel[string])) +## ) +## channel[].open() +## # Create a thread which will receive the channel as an argument. +## var thread: Thread[ptr Channel[string]] +## createThread(thread, worker, channel) +## channel[].send("Hello from the main thread!") +## # Clean up resources. +## thread.joinThread() +## channel[].close() +## deallocShared(channel) +## +## localChannelExample() # "Hello from the main thread!" +## ``` + +when not declared(ThisIsSystem): + {.error: "You must not import this module explicitly".} + +import std/private/syslocks + +type + pbytes = ptr UncheckedArray[byte] + RawChannel {.pure, final.} = object ## msg queue for a thread + rd, wr, count, mask, maxItems: int + data: pbytes + lock: SysLock + cond: SysCond + elemType: PNimType + ready: bool + when not usesDestructors: + region: MemRegion + PRawChannel = ptr RawChannel + LoadStoreMode = enum mStore, mLoad + Channel*[TMsg] {.gcsafe.} = RawChannel ## a channel for thread communication + +const ChannelDeadMask = -2 + +proc initRawChannel(p: pointer, maxItems: int) = + var c = cast[PRawChannel](p) + initSysLock(c.lock) + initSysCond(c.cond) + c.mask = -1 + c.maxItems = maxItems + +proc deinitRawChannel(p: pointer) = + var c = cast[PRawChannel](p) + # we need to grab the lock to be safe against sending threads! + acquireSys(c.lock) + c.mask = ChannelDeadMask + when not usesDestructors: + deallocOsPages(c.region) + else: + if c.data != nil: deallocShared(c.data) + deinitSys(c.lock) + deinitSysCond(c.cond) + +when not usesDestructors: + + proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, + mode: LoadStoreMode) {.benign.} + + proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, + mode: LoadStoreMode) {.benign.} = + var + d = cast[int](dest) + s = cast[int](src) + case n.kind + of nkSlot: storeAux(cast[pointer](d +% n.offset), + cast[pointer](s +% n.offset), n.typ, t, mode) + of nkList: + for i in 0..n.len-1: storeAux(dest, src, n.sons[i], t, mode) + of nkCase: + copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), + n.typ.size) + var m = selectBranch(src, n) + if m != nil: storeAux(dest, src, m, t, mode) + of nkNone: sysAssert(false, "storeAux") + + proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, + mode: LoadStoreMode) = + template `+!`(p: pointer; x: int): pointer = + cast[pointer](cast[int](p) +% x) + + var + d = cast[int](dest) + s = cast[int](src) + sysAssert(mt != nil, "mt == nil") + case mt.kind + of tyString: + if mode == mStore: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + x[] = nil + else: + var ss = cast[NimString](s2) + var ns = cast[NimString](alloc(t.region, GenericSeqSize + ss.len+1)) + copyMem(ns, ss, ss.len+1 + GenericSeqSize) + x[] = ns + else: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + unsureAsgnRef(x, s2) + else: + let y = copyDeepString(cast[NimString](s2)) + #echo "loaded ", cast[int](y), " ", cast[string](y) + unsureAsgnRef(x, y) + dealloc(t.region, s2) + of tySequence: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + if mode == mStore: + x[] = nil + else: + unsureAsgnRef(x, nil) + else: + sysAssert(dest != nil, "dest == nil") + if mode == mStore: + x[] = alloc0(t.region, align(GenericSeqSize, mt.base.align) +% seq.len *% mt.base.size) + else: + unsureAsgnRef(x, newSeq(mt, seq.len)) + var dst = cast[int](cast[PPointer](dest)[]) + var dstseq = cast[PGenericSeq](dst) + dstseq.len = seq.len + dstseq.reserved = seq.len + for i in 0..seq.len-1: + storeAux( + cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% + i *% mt.base.size), + mt.base, t, mode) + if mode != mStore: dealloc(t.region, s2) + of tyObject: + if mt.base != nil: + storeAux(dest, src, mt.base, t, mode) + else: + # copy type field: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + storeAux(dest, src, mt.node, t, mode) + of tyTuple: + storeAux(dest, src, mt.node, t, mode) + of tyArray, tyArrayConstr: + for i in 0..(mt.size div mt.base.size)-1: + storeAux(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base, t, mode) + of tyRef: + var s = cast[PPointer](src)[] + var x = cast[PPointer](dest) + if s == nil: + if mode == mStore: + x[] = nil + else: + unsureAsgnRef(x, nil) + else: + #let size = if mt.base.kind == tyObject: cast[ptr PNimType](s)[].size + # else: mt.base.size + if mode == mStore: + let dyntype = when declared(usrToCell): usrToCell(s).typ + else: mt + let size = dyntype.base.size + # we store the real dynamic 'ref type' at offset 0, so that + # no information is lost + let a = alloc0(t.region, size+sizeof(pointer)) + x[] = a + cast[PPointer](a)[] = dyntype + storeAux(a +! sizeof(pointer), s, dyntype.base, t, mode) + else: + let dyntype = cast[ptr PNimType](s)[] + var obj = newObj(dyntype, dyntype.base.size) + unsureAsgnRef(x, obj) + storeAux(x[], s +! sizeof(pointer), dyntype.base, t, mode) + dealloc(t.region, s) + else: + copyMem(dest, src, mt.size) # copy raw bits + +proc rawSend(q: PRawChannel, data: pointer, typ: PNimType) = + ## Adds an `item` to the end of the queue `q`. + var cap = q.mask+1 + if q.count >= cap: + # start with capacity for 2 entries in the queue: + if cap == 0: cap = 1 + when not usesDestructors: + var n = cast[pbytes](alloc0(q.region, cap*2*typ.size)) + else: + var n = cast[pbytes](allocShared0(cap*2*typ.size)) + var z = 0 + var i = q.rd + var c = q.count + while c > 0: + dec c + copyMem(addr(n[z*typ.size]), addr(q.data[i*typ.size]), typ.size) + i = (i + 1) and q.mask + inc z + if q.data != nil: + when not usesDestructors: + dealloc(q.region, q.data) + else: + deallocShared(q.data) + q.data = n + q.mask = cap*2 - 1 + q.wr = q.count + q.rd = 0 + when not usesDestructors: + storeAux(addr(q.data[q.wr * typ.size]), data, typ, q, mStore) + else: + copyMem(addr(q.data[q.wr * typ.size]), data, typ.size) + inc q.count + q.wr = (q.wr + 1) and q.mask + +proc rawRecv(q: PRawChannel, data: pointer, typ: PNimType) = + sysAssert q.count > 0, "rawRecv" + dec q.count + when not usesDestructors: + storeAux(data, addr(q.data[q.rd * typ.size]), typ, q, mLoad) + else: + copyMem(data, addr(q.data[q.rd * typ.size]), typ.size) + q.rd = (q.rd + 1) and q.mask + +template lockChannel(q, action): untyped = + acquireSys(q.lock) + action + releaseSys(q.lock) + +proc sendImpl(q: PRawChannel, typ: PNimType, msg: pointer, noBlock: bool): bool = + if q.mask == ChannelDeadMask: + sysFatal(DeadThreadDefect, "cannot send message; thread died") + acquireSys(q.lock) + if q.maxItems > 0: + # Wait until count is less than maxItems + if noBlock and q.count >= q.maxItems: + releaseSys(q.lock) + return + + while q.count >= q.maxItems: + waitSysCond(q.cond, q.lock) + + rawSend(q, msg, typ) + q.elemType = typ + signalSysCond(q.cond) + releaseSys(q.lock) + result = true + +proc send*[TMsg](c: var Channel[TMsg], msg: sink TMsg) {.inline.} = + ## Sends a message to a thread. `msg` is deeply copied. + discard sendImpl(cast[PRawChannel](addr c), cast[PNimType](getTypeInfo(msg)), unsafeAddr(msg), false) + when defined(gcDestructors): + wasMoved(msg) + +proc trySend*[TMsg](c: var Channel[TMsg], msg: sink TMsg): bool {.inline.} = + ## Tries to send a message to a thread. + ## + ## `msg` is deeply copied. Doesn't block. + ## + ## Returns `false` if the message was not sent because number of pending items + ## in the channel exceeded `maxItems`. + result = sendImpl(cast[PRawChannel](addr c), cast[PNimType](getTypeInfo(msg)), unsafeAddr(msg), true) + when defined(gcDestructors): + if result: + wasMoved(msg) + +proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) = + q.ready = true + while q.count <= 0: + waitSysCond(q.cond, q.lock) + q.ready = false + if typ != q.elemType: + releaseSys(q.lock) + raise newException(ValueError, "cannot receive message of wrong type") + rawRecv(q, res, typ) + if q.maxItems > 0 and q.count == q.maxItems - 1: + # Parent thread is awaiting in send. Wake it up. + signalSysCond(q.cond) + +proc recv*[TMsg](c: var Channel[TMsg]): TMsg = + ## Receives a message from the channel `c`. + ## + ## This blocks until a message has arrived! + ## You may use `peek proc <#peek,Channel[TMsg]>`_ to avoid the blocking. + var q = cast[PRawChannel](addr(c)) + acquireSys(q.lock) + llRecv(q, addr(result), cast[PNimType](getTypeInfo(result))) + releaseSys(q.lock) + +proc tryRecv*[TMsg](c: var Channel[TMsg]): tuple[dataAvailable: bool, + msg: TMsg] = + ## Tries to receive a message from the channel `c`, but this can fail + ## for all sort of reasons, including contention. + ## + ## If it fails, it returns `(false, default(msg))` otherwise it + ## returns `(true, msg)`. + var q = cast[PRawChannel](addr(c)) + if q.mask != ChannelDeadMask: + if tryAcquireSys(q.lock): + if q.count > 0: + llRecv(q, addr(result.msg), cast[PNimType](getTypeInfo(result.msg))) + result.dataAvailable = true + releaseSys(q.lock) + +proc peek*[TMsg](c: var Channel[TMsg]): int = + ## Returns the current number of messages in the channel `c`. + ## + ## Returns -1 if the channel has been closed. + ## + ## **Note**: This is dangerous to use as it encourages races. + ## It's much better to use `tryRecv proc <#tryRecv,Channel[TMsg]>`_ instead. + var q = cast[PRawChannel](addr(c)) + if q.mask != ChannelDeadMask: + lockChannel(q): + result = q.count + else: + result = -1 + +proc open*[TMsg](c: var Channel[TMsg], maxItems: int = 0) = + ## Opens a channel `c` for inter thread communication. + ## + ## The `send` operation will block until number of unprocessed items is + ## less than `maxItems`. + ## + ## For unlimited queue set `maxItems` to 0. + initRawChannel(addr(c), maxItems) + +proc close*[TMsg](c: var Channel[TMsg]) = + ## Closes a channel `c` and frees its associated resources. + deinitRawChannel(addr(c)) + +proc ready*[TMsg](c: var Channel[TMsg]): bool = + ## Returns true if some thread is waiting on the channel `c` for + ## new messages. + var q = cast[PRawChannel](addr(c)) + result = q.ready diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 6caf99d27..b48855964 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -8,24 +8,72 @@ # # Implementation of some runtime checks. +include system/indexerrors +when defined(nimPreviewSlimSystem): + import std/formatfloat proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = when hostOS == "standalone": - sysFatal(RangeError, "value out of range") + sysFatal(RangeDefect, "value out of range") else: - sysFatal(RangeError, "value out of range: ", $val) + sysFatal(RangeDefect, "value out of range: ", $val) + +proc raiseIndexError4(l1, h1, h2: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, "index out of bounds: " & $l1 & ".." & $h1 & " notin 0.." & $(h2 - 1)) + +proc raiseIndexError3(i, a, b: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, formatErrorIndexBound(i, a, b)) + +proc raiseIndexError2(i, n: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, formatErrorIndexBound(i, n)) proc raiseIndexError() {.compilerproc, noinline.} = - sysFatal(IndexError, "index out of bounds") + sysFatal(IndexDefect, "index out of bounds") proc raiseFieldError(f: string) {.compilerproc, noinline.} = - sysFatal(FieldError, f, " is not accessible") + ## remove after bootstrap > 1.5.1 + sysFatal(FieldDefect, f) + +when defined(nimV2): + proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, f & $discVal & "'") + + proc raiseFieldErrorStr(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, formatFieldDefect(f, discVal)) +else: + proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, formatFieldDefect(f, discVal)) + +proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} = + when defined(standalone): + sysFatal(RangeDefect, "value out of range") + else: + sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) + +proc raiseRangeErrorF(i, a, b: float) {.compilerproc, noinline.} = + when defined(standalone): + sysFatal(RangeDefect, "value out of range") + else: + sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) + +proc raiseRangeErrorU(i, a, b: uint64) {.compilerproc, noinline.} = + # todo: better error reporting + sysFatal(RangeDefect, "value out of range") + +proc raiseRangeErrorNoArgs() {.compilerproc, noinline.} = + sysFatal(RangeDefect, "value out of range") + +proc raiseObjectConversionError() {.compilerproc, noinline.} = + sysFatal(ObjectConversionDefect, "invalid object conversion") proc chckIndx(i, a, b: int): int = if i >= a and i <= b: return i else: - raiseIndexError() + raiseIndexError3(i, a, b) proc chckRange(i, a, b: int): int = if i >= a and i <= b: @@ -39,61 +87,75 @@ proc chckRange64(i, a, b: int64): int64 {.compilerproc.} = else: raiseRangeError(i) +proc chckRangeU(i, a, b: uint64): uint64 {.compilerproc.} = + if i >= a and i <= b: + return i + else: + sysFatal(RangeDefect, "value out of range") + proc chckRangeF(x, a, b: float): float = if x >= a and x <= b: return x else: when hostOS == "standalone": - sysFatal(RangeError, "value out of range") + sysFatal(RangeDefect, "value out of range") else: - sysFatal(RangeError, "value out of range: ", $x) + sysFatal(RangeDefect, "value out of range: ", $x) proc chckNil(p: pointer) = if p == nil: - sysFatal(ValueError, "attempt to write to a nil address") - #c_raise(SIGSEGV) - -proc chckObj(obj, subclass: PNimType) {.compilerproc.} = - # checks if obj is of type subclass: - var x = obj - if x == subclass: return # optimized fast path - while x != subclass: - if x == nil: - sysFatal(ObjectConversionError, "invalid object conversion") - break - x = x.base - -proc chckObjAsgn(a, b: PNimType) {.compilerproc, inline.} = - if a != b: - sysFatal(ObjectAssignmentError, "invalid object assignment") - -type ObjCheckCache = array[0..1, PNimType] - -proc isObjSlowPath(obj, subclass: PNimType; - cache: var ObjCheckCache): bool {.noinline.} = - # checks if obj is of type subclass: - var x = obj.base - while x != subclass: - if x == nil: - cache[0] = obj - return false - x = x.base - cache[1] = obj - return true - -proc isObjWithCache(obj, subclass: PNimType; - cache: var ObjCheckCache): bool {.compilerProc, inline.} = - if obj == subclass: return true - if obj.base == subclass: return true - if cache[0] == obj: return false - if cache[1] == obj: return true - return isObjSlowPath(obj, subclass, cache) - -proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = - # checks if obj is of type subclass: - var x = obj - if x == subclass: return true # optimized fast path - while x != subclass: - if x == nil: return false - x = x.base - return true + sysFatal(NilAccessDefect, "attempt to write to a nil address") + +proc chckNilDisp(p: pointer) {.compilerproc.} = + if p == nil: + sysFatal(NilAccessDefect, "cannot dispatch; dispatcher is nil") + +when not defined(nimV2): + + proc chckObj(obj, subclass: PNimType) {.compilerproc.} = + # checks if obj is of type subclass: + var x = obj + if x == subclass: return # optimized fast path + while x != subclass: + if x == nil: + sysFatal(ObjectConversionDefect, "invalid object conversion") + x = x.base + + proc chckObjAsgn(a, b: PNimType) {.compilerproc, inline.} = + if a != b: + sysFatal(ObjectAssignmentDefect, "invalid object assignment") + + type ObjCheckCache = array[0..1, PNimType] + + proc isObjSlowPath(obj, subclass: PNimType; + cache: var ObjCheckCache): bool {.noinline.} = + # checks if obj is of type subclass: + var x = obj.base + while x != subclass: + if x == nil: + cache[0] = obj + return false + x = x.base + cache[1] = obj + return true + + proc isObjWithCache(obj, subclass: PNimType; + cache: var ObjCheckCache): bool {.compilerproc, inline.} = + if obj == subclass: return true + if obj.base == subclass: return true + if cache[0] == obj: return false + if cache[1] == obj: return true + return isObjSlowPath(obj, subclass, cache) + + proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = + # checks if obj is of type subclass: + var x = obj + if x == subclass: return true # optimized fast path + while x != subclass: + if x == nil: return false + x = x.base + return true + +when defined(nimV2): + proc raiseObjectCaseTransition() {.compilerproc.} = + sysFatal(FieldDefect, "assignment to discriminant changes object branch") diff --git a/lib/system/comparisons.nim b/lib/system/comparisons.nim new file mode 100644 index 000000000..a8d78bb93 --- /dev/null +++ b/lib/system/comparisons.nim @@ -0,0 +1,337 @@ +# comparison operators: +proc `==`*[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} = + ## Checks whether values within the *same enum* have the same underlying value. + runnableExamples: + type + Enum1 = enum + field1 = 3, field2 + Enum2 = enum + place1, place2 = 3 + var + e1 = field1 + e2 = place2.ord.Enum1 + assert e1 == e2 + assert not compiles(e1 == place2) # raises error +proc `==`*(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} = + ## Checks for equality between two `pointer` variables. + runnableExamples: + var # this is a wildly dangerous example + a = cast[pointer](0) + b = cast[pointer](nil) + assert a == b # true due to the special meaning of `nil`/0 as a pointer +proc `==`*(x, y: string): bool {.magic: "EqStr", noSideEffect.} + ## Checks for equality between two `string` variables. + +proc `==`*(x, y: char): bool {.magic: "EqCh", noSideEffect.} + ## Checks for equality between two `char` variables. +proc `==`*(x, y: bool): bool {.magic: "EqB", noSideEffect.} + ## Checks for equality between two `bool` variables. +proc `==`*[T](x, y: set[T]): bool {.magic: "EqSet", noSideEffect.} = + ## Checks for equality between two variables of type `set`. + runnableExamples: + assert {1, 2, 2, 3} == {1, 2, 3} # duplication in sets is ignored + +proc `==`*[T](x, y: ref T): bool {.magic: "EqRef", noSideEffect.} + ## Checks that two `ref` variables refer to the same item. +proc `==`*[T](x, y: ptr T): bool {.magic: "EqRef", noSideEffect.} + ## Checks that two `ptr` variables refer to the same item. +proc `==`*[T: proc | iterator](x, y: T): bool {.magic: "EqProc", noSideEffect.} + ## Checks that two `proc` variables refer to the same procedure. + +proc `<=`*[Enum: enum](x, y: Enum): bool {.magic: "LeEnum", noSideEffect.} +proc `<=`*(x, y: string): bool {.magic: "LeStr", noSideEffect.} = + ## Compares two strings and returns true if `x` is lexicographically + ## before `y` (uppercase letters come before lowercase letters). + runnableExamples: + let + a = "abc" + b = "abd" + c = "ZZZ" + assert a <= b + assert a <= a + assert not (a <= c) + +proc `<=`*(x, y: char): bool {.magic: "LeCh", noSideEffect.} = + ## Compares two chars and returns true if `x` is lexicographically + ## before `y` (uppercase letters come before lowercase letters). + runnableExamples: + let + a = 'a' + b = 'b' + c = 'Z' + assert a <= b + assert a <= a + assert not (a <= c) + +proc `<=`*[T](x, y: set[T]): bool {.magic: "LeSet", noSideEffect.} = + ## Returns true if `x` is a subset of `y`. + ## + ## A subset `x` has all of its members in `y` and `y` doesn't necessarily + ## have more members than `x`. That is, `x` can be equal to `y`. + runnableExamples: + let + a = {3, 5} + b = {1, 3, 5, 7} + c = {2} + assert a <= b + assert a <= a + assert not (a <= c) + +proc `<=`*(x, y: bool): bool {.magic: "LeB", noSideEffect.} +proc `<=`*[T](x, y: ref T): bool {.magic: "LePtr", noSideEffect.} +proc `<=`*(x, y: pointer): bool {.magic: "LePtr", noSideEffect.} + +proc `<`*[Enum: enum](x, y: Enum): bool {.magic: "LtEnum", noSideEffect.} +proc `<`*(x, y: string): bool {.magic: "LtStr", noSideEffect.} = + ## Compares two strings and returns true if `x` is lexicographically + ## before `y` (uppercase letters come before lowercase letters). + runnableExamples: + let + a = "abc" + b = "abd" + c = "ZZZ" + assert a < b + assert not (a < a) + assert not (a < c) + +proc `<`*(x, y: char): bool {.magic: "LtCh", noSideEffect.} = + ## Compares two chars and returns true if `x` is lexicographically + ## before `y` (uppercase letters come before lowercase letters). + runnableExamples: + let + a = 'a' + b = 'b' + c = 'Z' + assert a < b + assert not (a < a) + assert not (a < c) + +proc `<`*[T](x, y: set[T]): bool {.magic: "LtSet", noSideEffect.} = + ## Returns true if `x` is a strict or proper subset of `y`. + ## + ## A strict or proper subset `x` has all of its members in `y` but `y` has + ## more elements than `y`. + runnableExamples: + let + a = {3, 5} + b = {1, 3, 5, 7} + c = {2} + assert a < b + assert not (a < a) + assert not (a < c) + +proc `<`*(x, y: bool): bool {.magic: "LtB", noSideEffect.} +proc `<`*[T](x, y: ref T): bool {.magic: "LtPtr", noSideEffect.} +proc `<`*[T](x, y: ptr T): bool {.magic: "LtPtr", noSideEffect.} +proc `<`*(x, y: pointer): bool {.magic: "LtPtr", noSideEffect.} + +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template `!=`*(x, y: untyped): untyped {.callsite.} = + ## Unequals operator. This is a shorthand for `not (x == y)`. + not (x == y) + +template `>=`*(x, y: untyped): untyped {.callsite.} = + ## "is greater or equals" operator. This is the same as `y <= x`. + y <= x + +template `>`*(x, y: untyped): untyped {.callsite.} = + ## "is greater" operator. This is the same as `y < x`. + y < x + + +proc `==`*(x, y: int): bool {.magic: "EqI", noSideEffect.} + ## Compares two integers for equality. +proc `==`*(x, y: int8): bool {.magic: "EqI", noSideEffect.} +proc `==`*(x, y: int16): bool {.magic: "EqI", noSideEffect.} +proc `==`*(x, y: int32): bool {.magic: "EqI", noSideEffect.} +proc `==`*(x, y: int64): bool {.magic: "EqI", noSideEffect.} + +proc `<=`*(x, y: int): bool {.magic: "LeI", noSideEffect.} + ## Returns true if `x` is less than or equal to `y`. +proc `<=`*(x, y: int8): bool {.magic: "LeI", noSideEffect.} +proc `<=`*(x, y: int16): bool {.magic: "LeI", noSideEffect.} +proc `<=`*(x, y: int32): bool {.magic: "LeI", noSideEffect.} +proc `<=`*(x, y: int64): bool {.magic: "LeI", noSideEffect.} + +proc `<`*(x, y: int): bool {.magic: "LtI", noSideEffect.} + ## Returns true if `x` is less than `y`. +proc `<`*(x, y: int8): bool {.magic: "LtI", noSideEffect.} +proc `<`*(x, y: int16): bool {.magic: "LtI", noSideEffect.} +proc `<`*(x, y: int32): bool {.magic: "LtI", noSideEffect.} +proc `<`*(x, y: int64): bool {.magic: "LtI", noSideEffect.} + +proc `<=`*(x, y: uint): bool {.magic: "LeU", noSideEffect.} + ## Returns true if `x <= y`. +proc `<=`*(x, y: uint8): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint16): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint32): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint64): bool {.magic: "LeU", noSideEffect.} + +proc `<`*(x, y: uint): bool {.magic: "LtU", noSideEffect.} + ## Returns true if `x < y`. +proc `<`*(x, y: uint8): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint16): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint32): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint64): bool {.magic: "LtU", noSideEffect.} + +proc `<=%`*(x, y: int): bool {.inline.} = + ## Treats `x` and `y` as unsigned and compares them. + ## Returns true if `unsigned(x) <= unsigned(y)`. + cast[uint](x) <= cast[uint](y) +proc `<=%`*(x, y: int8): bool {.inline.} = cast[uint8](x) <= cast[uint8](y) +proc `<=%`*(x, y: int16): bool {.inline.} = cast[uint16](x) <= cast[uint16](y) +proc `<=%`*(x, y: int32): bool {.inline.} = cast[uint32](x) <= cast[uint32](y) +proc `<=%`*(x, y: int64): bool {.inline.} = cast[uint64](x) <= cast[uint64](y) + +proc `<%`*(x, y: int): bool {.inline.} = + ## Treats `x` and `y` as unsigned and compares them. + ## Returns true if `unsigned(x) < unsigned(y)`. + cast[uint](x) < cast[uint](y) +proc `<%`*(x, y: int8): bool {.inline.} = cast[uint8](x) < cast[uint8](y) +proc `<%`*(x, y: int16): bool {.inline.} = cast[uint16](x) < cast[uint16](y) +proc `<%`*(x, y: int32): bool {.inline.} = cast[uint32](x) < cast[uint32](y) +proc `<%`*(x, y: int64): bool {.inline.} = cast[uint64](x) < cast[uint64](y) + +template `>=%`*(x, y: untyped): untyped = y <=% x + ## Treats `x` and `y` as unsigned and compares them. + ## Returns true if `unsigned(x) >= unsigned(y)`. + +template `>%`*(x, y: untyped): untyped = y <% x + ## Treats `x` and `y` as unsigned and compares them. + ## Returns true if `unsigned(x) > unsigned(y)`. + +proc `==`*(x, y: uint): bool {.magic: "EqI", noSideEffect.} + ## Compares two unsigned integers for equality. +proc `==`*(x, y: uint8): bool {.magic: "EqI", noSideEffect.} +proc `==`*(x, y: uint16): bool {.magic: "EqI", noSideEffect.} +proc `==`*(x, y: uint32): bool {.magic: "EqI", noSideEffect.} +proc `==`*(x, y: uint64): bool {.magic: "EqI", noSideEffect.} + +proc `<=`*(x, y: float32): bool {.magic: "LeF64", noSideEffect.} +proc `<=`*(x, y: float): bool {.magic: "LeF64", noSideEffect.} + +proc `<`*(x, y: float32): bool {.magic: "LtF64", noSideEffect.} +proc `<`*(x, y: float): bool {.magic: "LtF64", noSideEffect.} + +proc `==`*(x, y: float32): bool {.magic: "EqF64", noSideEffect.} +proc `==`*(x, y: float): bool {.magic: "EqF64", noSideEffect.} + +{.push stackTrace: off.} + +proc min*(x, y: int): int {.magic: "MinI", noSideEffect.} = + if x <= y: x else: y +proc min*(x, y: int8): int8 {.magic: "MinI", noSideEffect.} = + if x <= y: x else: y +proc min*(x, y: int16): int16 {.magic: "MinI", noSideEffect.} = + if x <= y: x else: y +proc min*(x, y: int32): int32 {.magic: "MinI", noSideEffect.} = + if x <= y: x else: y +proc min*(x, y: int64): int64 {.magic: "MinI", noSideEffect.} = + ## The minimum value of two integers. + if x <= y: x else: y +proc min*(x, y: float32): float32 {.noSideEffect, inline.} = + if x <= y or y != y: x else: y +proc min*(x, y: float64): float64 {.noSideEffect, inline.} = + if x <= y or y != y: x else: y +proc min*[T: not SomeFloat](x, y: T): T {.inline.} = + ## Generic minimum operator of 2 values based on `<=`. + if x <= y: x else: y + +proc max*(x, y: int): int {.magic: "MaxI", noSideEffect.} = + if y <= x: x else: y +proc max*(x, y: int8): int8 {.magic: "MaxI", noSideEffect.} = + if y <= x: x else: y +proc max*(x, y: int16): int16 {.magic: "MaxI", noSideEffect.} = + if y <= x: x else: y +proc max*(x, y: int32): int32 {.magic: "MaxI", noSideEffect.} = + if y <= x: x else: y +proc max*(x, y: int64): int64 {.magic: "MaxI", noSideEffect.} = + ## The maximum value of two integers. + if y <= x: x else: y +proc max*(x, y: float32): float32 {.noSideEffect, inline.} = + if y <= x or y != y: x else: y +proc max*(x, y: float64): float64 {.noSideEffect, inline.} = + if y <= x or y != y: x else: y +proc max*[T: not SomeFloat](x, y: T): T {.inline.} = + ## Generic maximum operator of 2 values based on `<=`. + if y <= x: x else: y + + +proc min*[T](x: openArray[T]): T = + ## The minimum value of `x`. `T` needs to have a `<` operator. + result = x[0] + for i in 1..high(x): + if x[i] < result: result = x[i] + +proc max*[T](x: openArray[T]): T = + ## The maximum value of `x`. `T` needs to have a `<` operator. + result = x[0] + for i in 1..high(x): + if result < x[i]: result = x[i] + +{.pop.} # stackTrace: off + + +proc clamp*[T](x, a, b: T): T = + ## Limits the value `x` within the interval \[a, b]. + ## This proc is equivalent to but faster than `max(a, min(b, x))`. + ## + ## .. warning:: `a <= b` is assumed and will not be checked (currently). + ## + ## **See also:** + ## `math.clamp` for a version that takes a `Slice[T]` instead. + runnableExamples: + assert (1.4).clamp(0.0, 1.0) == 1.0 + assert (0.5).clamp(0.0, 1.0) == 0.5 + assert 4.clamp(1, 3) == max(1, min(3, 4)) + if x < a: return a + if x > b: return b + return x + + +proc `==`*[I, T](x, y: array[I, T]): bool = + for f in low(x)..high(x): + if x[f] != y[f]: + return + result = true + +proc `==`*[T](x, y: openArray[T]): bool = + if x.len != y.len: + return false + for f in low(x)..high(x): + if x[f] != y[f]: + return false + result = true + + +proc `==`*[T](x, y: seq[T]): bool {.noSideEffect.} = + ## Generic equals operator for sequences: relies on a equals operator for + ## the element type `T`. + when nimvm: + if x.len == 0 and y.len == 0: + return true + else: + when not defined(js): + proc seqToPtr[T](x: seq[T]): pointer {.inline, noSideEffect.} = + when defined(nimSeqsV2): + result = cast[NimSeqV2[T]](x).p + else: + result = cast[pointer](x) + + if seqToPtr(x) == seqToPtr(y): + return true + else: + var sameObject = false + {.emit: """`sameObject` = `x` === `y`;""".} + if sameObject: return true + + if x.len != y.len: + return false + + for i in 0..x.len-1: + if x[i] != y[i]: + return false + + return true diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim new file mode 100644 index 000000000..cdb976ed5 --- /dev/null +++ b/lib/system/compilation.nim @@ -0,0 +1,209 @@ +const + NimMajor* {.intdefine.}: int = 2 + ## is the major number of Nim's version. Example: + ## ```nim + ## when (NimMajor, NimMinor, NimPatch) >= (1, 3, 1): discard + ## ``` + # see also std/private/since + + NimMinor* {.intdefine.}: int = 2 + ## is the minor number of Nim's version. + ## Odd for devel, even for releases. + + NimPatch* {.intdefine.}: int = 1 + ## is the patch number of Nim's version. + ## Odd for devel, even for releases. + +{.push profiler: off.} +let nimvm* {.magic: "Nimvm", compileTime.}: bool = false + ## May be used only in `when` expression. + ## It is true in Nim VM context and false otherwise. +{.pop.} + +const + isMainModule* {.magic: "IsMainModule".}: bool = false + ## True only when accessed in the main module. This works thanks to + ## compiler magic. It is useful to embed testing code in a module. + + CompileDate* {.magic: "CompileDate".}: string = "0000-00-00" + ## The date (in UTC) of compilation as a string of the form + ## `YYYY-MM-DD`. This works thanks to compiler magic. + + CompileTime* {.magic: "CompileTime".}: string = "00:00:00" + ## The time (in UTC) of compilation as a string of the form + ## `HH:MM:SS`. This works thanks to compiler magic. + +proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## defined. + ## + ## `x` is an external symbol introduced through the compiler's + ## `-d:x switch <nimc.html#compiler-usage-compileminustime-symbols>`_ to enable + ## build time conditionals: + ## ```nim + ## when not defined(release): + ## # Do here programmer friendly expensive sanity checks. + ## # Put here the normal code + ## ``` + ## + ## See also: + ## * `compileOption <#compileOption,string>`_ for `on|off` options + ## * `compileOption <#compileOption,string,string>`_ for enum options + ## * `define pragmas <manual.html#implementation-specific-pragmas-compileminustime-define-pragmas>`_ + +proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared. `x` has to be an identifier or a qualified identifier. + ## + ## This can be used to check whether a library provides a certain + ## feature or not: + ## ```nim + ## when not declared(strutils.toUpper): + ## # provide our own toUpper proc here, because strutils is + ## # missing it. + ## ``` + ## + ## See also: + ## * `declaredInScope <#declaredInScope,untyped>`_ + +proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared in the current scope. `x` has to be an identifier. + +proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = + ## Special compile-time procedure that checks whether `x` can be compiled + ## without any semantic error. + ## This can be used to check whether a type supports some operation: + ## ```nim + ## when compiles(3 + 4): + ## echo "'+' for integers is available" + ## ``` + discard + +proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} + ## Converts the AST of `x` into a string representation. This is very useful + ## for debugging. + +proc runnableExamples*(rdoccmd = "", body: untyped) {.magic: "RunnableExamples".} = + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a `runnableExamples` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the `##` doc comment. As the last step of documentation + ## generation each runnableExample is put in its own file `$file_examples$i.nim`, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. + runnableExamples: + proc timesTwo*(x: int): int = + ## This proc doubles a number. + runnableExamples: + # at module scope + const exported* = 123 + assert timesTwo(5) == 10 + block: # at block scope + defer: echo "done" + runnableExamples "-d:foo -b:cpp": + import std/compilesettings + assert querySetting(backend) == "cpp" + assert defined(foo) + runnableExamples "-r:off": ## this one is only compiled + import std/browsers + openDefaultBrowser "https://forum.nim-lang.org/" + 2 * x + +proc compileOption*(option: string): bool {. + magic: "CompileOption", noSideEffect.} = + ## Can be used to determine an `on|off` compile-time option. + ## + ## See also: + ## * `compileOption <#compileOption,string,string>`_ for enum options + ## * `defined <#defined,untyped>`_ + ## * `std/compilesettings module <compilesettings.html>`_ + runnableExamples("--floatChecks:off"): + static: doAssert not compileOption("floatchecks") + {.push floatChecks: on.} + static: doAssert compileOption("floatchecks") + # floating point NaN and Inf checks enabled in this scope + {.pop.} + +proc compileOption*(option, arg: string): bool {. + magic: "CompileOptionArg", noSideEffect.} = + ## Can be used to determine an enum compile-time option. + ## + ## See also: + ## * `compileOption <#compileOption,string>`_ for `on|off` options + ## * `defined <#defined,untyped>`_ + ## * `std/compilesettings module <compilesettings.html>`_ + runnableExamples: + when compileOption("opt", "size") and compileOption("gc", "boehm"): + discard "compiled with optimization for size and uses Boehm's GC" + +template currentSourcePath*: string = instantiationInfo(-1, true).filename + ## Returns the full file-system path of the current source. + ## + ## To get the directory containing the current source, use it with + ## `ospaths2.parentDir() <ospaths2.html#parentDir%2Cstring>`_ as + ## `currentSourcePath.parentDir()`. + ## + ## The path returned by this template is set at compile time. + ## + ## See the docstring of `macros.getProjectPath() <macros.html#getProjectPath>`_ + ## for an example to see the distinction between the `currentSourcePath()` + ## and `getProjectPath()`. + ## + ## See also: + ## * `ospaths2.getCurrentDir() proc <ospaths2.html#getCurrentDir>`_ + +proc slurp*(filename: string): string {.magic: "Slurp".} + ## This is an alias for `staticRead <#staticRead,string>`_. + +proc staticRead*(filename: string): string {.magic: "Slurp".} + ## Compile-time `readFile <syncio.html#readFile,string>`_ proc for easy + ## `resource`:idx: embedding: + ## + ## The maximum file size limit that `staticRead` and `slurp` can read is + ## near or equal to the *free* memory of the device you are using to compile. + ## ```nim + ## const myResource = staticRead"mydatafile.bin" + ## ``` + ## + ## `slurp <#slurp,string>`_ is an alias for `staticRead`. + +proc gorge*(command: string, input = "", cache = ""): string {. + magic: "StaticExec".} = discard + ## This is an alias for `staticExec <#staticExec,string,string,string>`_. + +proc staticExec*(command: string, input = "", cache = ""): string {. + magic: "StaticExec".} = discard + ## Executes an external process at compile-time and returns its text output + ## (stdout + stderr). + ## + ## If `input` is not an empty string, it will be passed as a standard input + ## to the executed program. + ## ```nim + ## const buildInfo = "Revision " & staticExec("git rev-parse HEAD") & + ## "\nCompiled on " & staticExec("uname -v") + ## ``` + ## + ## `gorge <#gorge,string,string,string>`_ is an alias for `staticExec`. + ## + ## Note that you can use this proc inside a pragma like + ## `passc <manual.html#implementation-specific-pragmas-passc-pragma>`_ or + ## `passl <manual.html#implementation-specific-pragmas-passl-pragma>`_. + ## + ## If `cache` is not empty, the results of `staticExec` are cached within + ## the `nimcache` directory. Use `--forceBuild` to get rid of this caching + ## behaviour then. `command & input & cache` (the concatenated string) is + ## used to determine whether the entry in the cache is still valid. You can + ## use versioning information for `cache`: + ## ```nim + ## const stateMachine = staticExec("dfaoptimizer", "input", "0.8.0") + ## ``` + +proc gorgeEx*(command: string, input = "", cache = ""): tuple[output: string, + exitCode: int] = + ## Similar to `gorge <#gorge,string,string,string>`_ but also returns the + ## precious exit code. + discard diff --git a/lib/system/coro_detection.nim b/lib/system/coro_detection.nim new file mode 100644 index 000000000..f6c1b5c15 --- /dev/null +++ b/lib/system/coro_detection.nim @@ -0,0 +1,20 @@ +## Coroutine detection logic + +template coroutinesSupportedPlatform(): bool = + when defined(sparc) or defined(ELATE) or defined(boehmgc) or defined(gogc) or + defined(nogc) or defined(gcRegions) or defined(gcMarkAndSweep): + false + else: + true + +when defined(nimCoroutines): + # Explicit opt-in. + when not coroutinesSupportedPlatform(): + {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} + const nimCoroutines* = true +elif defined(noNimCoroutines): + # Explicit opt-out. + const nimCoroutines* = false +else: + # Autodetect coroutine support. + const nimCoroutines* = false diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim new file mode 100644 index 000000000..34969cb32 --- /dev/null +++ b/lib/system/countbits_impl.nim @@ -0,0 +1,93 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Contains the used algorithms for counting bits. + +from std/private/bitops_utils import forwardImpl, castToUnsigned + +const useBuiltins* = not defined(noIntrinsicsBitOpts) +const noUndefined* = defined(noUndefinedBitOpts) +const useGCC_builtins* = (defined(gcc) or defined(llvm_gcc) or + defined(clang)) and useBuiltins +const useICC_builtins* = defined(icc) and useBuiltins +const useVCC_builtins* = defined(vcc) and useBuiltins +const arch64* = sizeof(int) == 8 + +template countBitsImpl(n: uint32): int = + # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + var v = uint32(n) + v = v - ((v shr 1'u32) and 0x55555555'u32) + v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) + (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int + +template countBitsImpl(n: uint64): int = + # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + var v = uint64(n) + v = v - ((v shr 1'u64) and 0x5555555555555555'u64) + v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) + v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) + ((v * 0x0101010101010101'u64) shr 56'u64).int + + +when useGCC_builtins: + # Returns the number of set 1-bits in value. + proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} + proc builtin_popcountll(x: culonglong): cint {. + importc: "__builtin_popcountll", cdecl.} + +elif useVCC_builtins: + # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. + func builtin_popcnt16(a2: uint16): uint16 {. + importc: "__popcnt16", header: "<intrin.h>".} + func builtin_popcnt32(a2: uint32): uint32 {. + importc: "__popcnt", header: "<intrin.h>".} + func builtin_popcnt64(a2: uint64): uint64 {. + importc: "__popcnt64", header: "<intrin.h>".} + +elif useICC_builtins: + # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm + # see also: https://software.intel.com/en-us/node/523362 + # Count the number of bits set to 1 in an integer a, and return that count in dst. + func builtin_popcnt32(a: cint): cint {. + importc: "_popcnt", header: "<immintrin.h>".} + func builtin_popcnt64(a: uint64): cint {. + importc: "_popcnt64", header: "<immintrin.h>".} + + +func countSetBitsImpl*(x: SomeInteger): int {.inline.} = + ## Counts the set bits in an integer (also called `Hamming weight`:idx:). + # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. + # like GCC and MSVC + let x = x.castToUnsigned + when nimvm: + result = forwardImpl(countBitsImpl, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int + else: result = builtin_popcountll(x.culonglong).int + elif useVCC_builtins: + when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int + elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + + builtin_popcnt32((x.uint64 shr 32'u64).uint32).int + elif useICC_builtins: + when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + + builtin_popcnt32((x.uint64 shr 32'u64).cint).int + else: + when sizeof(x) <= 4: result = countBitsImpl(x.uint32) + else: result = countBitsImpl(x.uint64) + +proc countBits32*(n: uint32): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) + +proc countBits64*(n: uint64): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) diff --git a/lib/system/ctypes.nim b/lib/system/ctypes.nim new file mode 100644 index 000000000..b788274bd --- /dev/null +++ b/lib/system/ctypes.nim @@ -0,0 +1,84 @@ +## Some type definitions for compatibility between different +## backends and platforms. + +type + BiggestInt* = int64 + ## is an alias for the biggest signed integer type the Nim compiler + ## supports. Currently this is `int64`, but it is platform-dependent + ## in general. + + BiggestFloat* = float64 + ## is an alias for the biggest floating point type the Nim + ## compiler supports. Currently this is `float64`, but it is + ## platform-dependent in general. + + BiggestUInt* = uint64 + ## is an alias for the biggest unsigned integer type the Nim compiler + ## supports. Currently this is `uint64`, but it is platform-dependent + ## in general. + +when defined(windows): + type + clong* {.importc: "long", nodecl.} = int32 + ## This is the same as the type `long` in *C*. + culong* {.importc: "unsigned long", nodecl.} = uint32 + ## This is the same as the type `unsigned long` in *C*. +else: + type + clong* {.importc: "long", nodecl.} = int + ## This is the same as the type `long` in *C*. + culong* {.importc: "unsigned long", nodecl.} = uint + ## This is the same as the type `unsigned long` in *C*. + +type # these work for most platforms: + cchar* {.importc: "char", nodecl.} = char + ## This is the same as the type `char` in *C*. + cschar* {.importc: "signed char", nodecl.} = int8 + ## This is the same as the type `signed char` in *C*. + cshort* {.importc: "short", nodecl.} = int16 + ## This is the same as the type `short` in *C*. + cint* {.importc: "int", nodecl.} = int32 + ## This is the same as the type `int` in *C*. + csize_t* {.importc: "size_t", nodecl.} = uint + ## This is the same as the type `size_t` in *C*. + clonglong* {.importc: "long long", nodecl.} = int64 + ## This is the same as the type `long long` in *C*. + cfloat* {.importc: "float", nodecl.} = float32 + ## This is the same as the type `float` in *C*. + cdouble* {.importc: "double", nodecl.} = float64 + ## This is the same as the type `double` in *C*. + clongdouble* {.importc: "long double", nodecl.} = BiggestFloat + ## This is the same as the type `long double` in *C*. + ## This C type is not supported by Nim's code generator. + + cuchar* {.importc: "unsigned char", nodecl, deprecated: "use `char` or `uint8` instead".} = char + ## Deprecated: Use `uint8` instead. + cushort* {.importc: "unsigned short", nodecl.} = uint16 + ## This is the same as the type `unsigned short` in *C*. + cuint* {.importc: "unsigned int", nodecl.} = uint32 + ## This is the same as the type `unsigned int` in *C*. + culonglong* {.importc: "unsigned long long", nodecl.} = uint64 + ## This is the same as the type `unsigned long long` in *C*. + +type + ByteAddress* {.deprecated: "use `uint`".} = int + ## is the signed integer type that should be used for converting + ## pointers to integer addresses for readability. + + cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring] + ## This is binary compatible to the type `char**` in *C*. The array's + ## high value is large enough to disable bounds checking in practice. + ## Use `cstringArrayToSeq proc <#cstringArrayToSeq,cstringArray,Natural>`_ + ## to convert it into a `seq[string]`. + +when not defined(nimPreviewSlimSystem): + # pollutes namespace + type + PFloat32* {.deprecated: "use `ptr float32`".} = ptr float32 + ## An alias for `ptr float32`. + PFloat64* {.deprecated: "use `ptr float64`".} = ptr float64 + ## An alias for `ptr float64`. + PInt64* {.deprecated: "use `ptr int64`".} = ptr int64 + ## An alias for `ptr int64`. + PInt32* {.deprecated: "use `ptr int32`".} = ptr int32 + ## An alias for `ptr int32`. diff --git a/lib/system/cyclebreaker.nim b/lib/system/cyclebreaker.nim new file mode 100644 index 000000000..45b0a5a65 --- /dev/null +++ b/lib/system/cyclebreaker.nim @@ -0,0 +1,184 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +#[ +A Cycle breaker for Nim +----------------------- + +Instead of "collecting" cycles with all of its pitfalls we will break cycles. +We exploit that every 'ref' can be 'nil' for this and so get away without +a distinction between weak and strong pointers. The required runtime +mechanisms are the same though: We need to be able to traverse the graph. +This design has the tremendous benefit that it doesn't require a dedicated +'rawDispose' operation and that it plays well with Nim's cost model. +The cost of freeing a subgraph with cycles is 2 * N rather than N, that's all. + +Cycles do not have to be prepared via .acyclic, there are not multiple +pointless traversals, only a single proc, `breakCycles` is exposed as a +separate module. + +Algorithm +--------- + +We traverse the graph and notice the nodes we've already traversed. If we +marked the node already, we set the pointer that leads to this node to 'nil' +and decrement the reference count of the cell we pointed at. + +We notice that multiple paths to the same object do not mean +we found a cycle, it only means the node is shared. + + + a -------> b <----- c + | ^ ^ + +----------+ | + | | + +-------------------+ + +If we simply remove all links to already processed nodes we end up with: + + a -------> b c + | ^ + + | + | | + +-------------------+ + +That seems acceptable, no leak is produced. This implies that the standard +depth-first traversal suffices. + +]# + +include cellseqs_v2 + +const + colGreen = 0b000 + colYellow = 0b001 + colRed = 0b010 + colorMask = 0b011 + +type + TraceProc = proc (p, env: pointer) {.nimcall, benign.} + DisposeProc = proc (p: pointer) {.nimcall, benign.} + +template color(c): untyped = c.rc and colorMask +template setColor(c, col) = + c.rc = c.rc and not colorMask or col + +proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = + let h = head(p) + inc h.rc, rcIncrement + +proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = discard + +type + GcEnv = object + traceStack: CellSeq[ptr pointer] + +proc trace(p: pointer; desc: PNimTypeV2; j: var GcEnv) {.inline.} = + when false: + cprintf("[Trace] desc: %p %p\n", desc, p) + cprintf("[Trace] trace: %p\n", desc.traceImpl) + if desc.traceImpl != nil: + cast[TraceProc](desc.traceImpl)(p, addr(j)) + +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl.} = + let p = cast[ptr pointer](q) + when traceCollector: + cprintf("[Trace] raw: %p\n", p) + cprintf("[Trace] deref: %p\n", p[]) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, desc) + +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl.} = + let p = cast[ptr pointer](q) + when traceCollector: + cprintf("[TraceDyn] raw: %p\n", p) + cprintf("[TraceDyn] deref: %p\n", p[]) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) + +var markerGeneration: int + +proc breakCycles(s: Cell; desc: PNimTypeV2) = + let markerColor = if (markerGeneration and 1) == 0: colRed + else: colYellow + atomicInc markerGeneration + when traceCollector: + cprintf("[BreakCycles] starting: %p %s RC %ld trace proc %p\n", + s, desc.name, s.rc shr rcShift, desc.traceImpl) + + var j: GcEnv + init j.traceStack + s.setColor markerColor + trace(s +! sizeof(RefHeader), desc, j) + + while j.traceStack.len > 0: + let (u, desc) = j.traceStack.pop() + let p = u[] + let t = head(p) + if t.color != markerColor: + t.setColor markerColor + trace(p, desc, j) + when traceCollector: + cprintf("[BreakCycles] followed: %p RC %ld\n", t, t.rc shr rcShift) + else: + if (t.rc shr rcShift) > 0: + dec t.rc, rcIncrement + # mark as a link that the produced destructor does not have to follow: + u[] = nil + when traceCollector: + cprintf("[BreakCycles] niled out: %p RC %ld\n", t, t.rc shr rcShift) + else: + # anyhow as a link that the produced destructor does not have to follow: + u[] = nil + when traceCollector: + cprintf("[Bug] %p %s RC %ld\n", t, desc.name, t.rc shr rcShift) + deinit j.traceStack + +proc thinout*[T](x: ref T) {.inline.} = + ## turn the subgraph starting with `x` into its spanning tree by + ## `nil`'ing out any pointers that would harm the spanning tree + ## structure. Any back pointers that introduced cycles + ## and thus would keep the graph from being freed are `nil`'ed. + ## This is a form of cycle collection that works well with Nim's ARC + ## and its associated cost model. + proc getDynamicTypeInfo[T](x: T): PNimTypeV2 {.magic: "GetTypeInfoV2", noSideEffect.} + + breakCycles(head(cast[pointer](x)), getDynamicTypeInfo(x[])) + +proc thinout*[T: proc](x: T) {.inline.} = + proc rawEnv[T: proc](x: T): pointer {.noSideEffect, inline.} = + {.emit: """ + `result` = `x`.ClE_0; + """.} + + let p = rawEnv(x) + breakCycles(head(p), cast[ptr PNimTypeV2](p)[]) + +proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + # According to Lins it's correct to do nothing else here. + #cprintf("[DeCREF] %p\n", p) + +proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p %s\n", p, desc.name) + else: + dec cell.rc, rcIncrement + #cprintf("[DeCREF] %p %s %ld\n", p, desc.name, cell.rc) diff --git a/lib/system/debugger.nim b/lib/system/debugger.nim deleted file mode 100644 index b18c61755..000000000 --- a/lib/system/debugger.nim +++ /dev/null @@ -1,305 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This file implements basic features for any debugger. - -type - VarSlot* {.compilerproc, final.} = object ## a slot in a frame - address*: pointer ## the variable's address - typ*: PNimType ## the variable's type - name*: cstring ## the variable's name; for globals this is "module.name" - - PExtendedFrame = ptr ExtendedFrame - ExtendedFrame = object # If the debugger is enabled the compiler - # provides an extended frame. Of course - # only slots that are - # needed are allocated and not 10_000, - # except for the global data description. - f: TFrame - slots: array[0..10_000, VarSlot] -{.deprecated: [TVarSlot: VarSlot, TExtendedFrame: ExtendedFrame].} - -var - dbgGlobalData: ExtendedFrame # this reserves much space, but - # for now it is the most practical way - -proc dbgRegisterGlobal(name: cstring, address: pointer, - typ: PNimType) {.compilerproc.} = - let i = dbgGlobalData.f.len - if i >= high(dbgGlobalData.slots): - #debugOut("[Warning] cannot register global ") - return - dbgGlobalData.slots[i].name = name - dbgGlobalData.slots[i].typ = typ - dbgGlobalData.slots[i].address = address - inc(dbgGlobalData.f.len) - -proc getLocal*(frame: PFrame; slot: int): VarSlot {.inline.} = - ## retrieves the meta data for the local variable at `slot`. CAUTION: An - ## invalid `slot` value causes a corruption! - result = cast[PExtendedFrame](frame).slots[slot] - -proc getGlobalLen*(): int {.inline.} = - ## gets the number of registered globals. - result = dbgGlobalData.f.len - -proc getGlobal*(slot: int): VarSlot {.inline.} = - ## retrieves the meta data for the global variable at `slot`. CAUTION: An - ## invalid `slot` value causes a corruption! - result = dbgGlobalData.slots[slot] - -# ------------------- breakpoint support ------------------------------------ - -type - Breakpoint* = object ## represents a break point - low*, high*: int ## range from low to high; if disabled - ## both low and high are set to their negative values - filename*: cstring ## the filename of the breakpoint - -var - dbgBP: array[0..127, Breakpoint] # breakpoints - dbgBPlen: int - dbgBPbloom: int64 # we use a bloom filter to speed up breakpoint checking - - dbgFilenames*: array[0..300, cstring] ## registered filenames; - ## 'nil' terminated - dbgFilenameLen: int - -proc dbgRegisterFilename(filename: cstring) {.compilerproc.} = - # XXX we could check for duplicates here for DLL support - dbgFilenames[dbgFilenameLen] = filename - inc dbgFilenameLen - -proc dbgRegisterBreakpoint(line: int, - filename, name: cstring) {.compilerproc.} = - let x = dbgBPlen - if x >= high(dbgBP): - #debugOut("[Warning] cannot register breakpoint") - return - inc(dbgBPlen) - dbgBP[x].filename = filename - dbgBP[x].low = line - dbgBP[x].high = line - dbgBPbloom = dbgBPbloom or line - -proc addBreakpoint*(filename: cstring, lo, hi: int): bool = - let x = dbgBPlen - if x >= high(dbgBP): return false - inc(dbgBPlen) - result = true - dbgBP[x].filename = filename - dbgBP[x].low = lo - dbgBP[x].high = hi - for line in lo..hi: dbgBPbloom = dbgBPbloom or line - -const - FileSystemCaseInsensitive = defined(windows) or defined(dos) or defined(os2) - -proc fileMatches(c, bp: cstring): bool = - # bp = breakpoint filename - # c = current filename - # we consider it a match if bp is a suffix of c - # and the character for the suffix does not exist or - # is one of: \ / : - # depending on the OS case does not matter! - var blen: int = c_strlen(bp) - var clen: int = c_strlen(c) - if blen > clen: return false - # check for \ / : - if clen-blen-1 >= 0 and c[clen-blen-1] notin {'\\', '/', ':'}: - return false - var i = 0 - while i < blen: - var x = bp[i] - var y = c[i+clen-blen] - when FileSystemCaseInsensitive: - if x >= 'A' and x <= 'Z': x = chr(ord(x) - ord('A') + ord('a')) - if y >= 'A' and y <= 'Z': y = chr(ord(y) - ord('A') + ord('a')) - if x != y: return false - inc(i) - return true - -proc canonFilename*(filename: cstring): cstring = - ## returns 'nil' if the filename cannot be found. - for i in 0 .. <dbgFilenameLen: - result = dbgFilenames[i] - if fileMatches(result, filename): return result - result = nil - -iterator listBreakpoints*(): ptr Breakpoint = - ## lists all breakpoints. - for i in 0..dbgBPlen-1: yield addr(dbgBP[i]) - -proc isActive*(b: ptr Breakpoint): bool = b.low > 0 -proc flip*(b: ptr Breakpoint) = - ## enables or disables 'b' depending on its current state. - b.low = -b.low; b.high = -b.high - -proc checkBreakpoints*(filename: cstring, line: int): ptr Breakpoint = - ## in which breakpoint (if any) we are. - if (dbgBPbloom and line) != line: return nil - for b in listBreakpoints(): - if line >= b.low and line <= b.high and filename == b.filename: return b - -# ------------------- watchpoint support ------------------------------------ - -type - Hash = int - Watchpoint {.pure, final.} = object - name: cstring - address: pointer - typ: PNimType - oldValue: Hash -{.deprecated: [THash: Hash, TWatchpoint: Watchpoint].} - -var - watchpoints: array [0..99, Watchpoint] - watchpointsLen: int - -proc `!&`(h: Hash, val: int): Hash {.inline.} = - result = h +% val - result = result +% result shl 10 - result = result xor (result shr 6) - -proc `!$`(h: Hash): Hash {.inline.} = - result = h +% h shl 3 - result = result xor (result shr 11) - result = result +% result shl 15 - -proc hash(data: pointer, size: int): Hash = - var h: Hash = 0 - var p = cast[cstring](data) - var i = 0 - var s = size - while s > 0: - h = h !& ord(p[i]) - inc(i) - dec(s) - result = !$h - -proc hashGcHeader(data: pointer): Hash = - const headerSize = sizeof(int)*2 - result = hash(cast[pointer](cast[int](data) -% headerSize), headerSize) - -proc genericHashAux(dest: pointer, mt: PNimType, shallow: bool, - h: Hash): Hash -proc genericHashAux(dest: pointer, n: ptr TNimNode, shallow: bool, - h: Hash): Hash = - var d = cast[ByteAddress](dest) - case n.kind - of nkSlot: - result = genericHashAux(cast[pointer](d +% n.offset), n.typ, shallow, h) - of nkList: - result = h - for i in 0..n.len-1: - result = result !& genericHashAux(dest, n.sons[i], shallow, result) - of nkCase: - result = h !& hash(cast[pointer](d +% n.offset), n.typ.size) - var m = selectBranch(dest, n) - if m != nil: result = genericHashAux(dest, m, shallow, result) - of nkNone: sysAssert(false, "genericHashAux") - -proc genericHashAux(dest: pointer, mt: PNimType, shallow: bool, - h: Hash): Hash = - sysAssert(mt != nil, "genericHashAux 2") - case mt.kind - of tyString: - var x = cast[PPointer](dest)[] - result = h - if x != nil: - let s = cast[NimString](x) - when defined(trackGcHeaders): - result = result !& hashGcHeader(x) - else: - result = result !& hash(x, s.len) - of tySequence: - var x = cast[PPointer](dest) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - result = h - if dst != 0: - when defined(trackGcHeaders): - result = result !& hashGcHeader(cast[PPointer](dest)[]) - else: - for i in 0..cast[PGenericSeq](dst).len-1: - result = result !& genericHashAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - mt.base, shallow, result) - of tyObject, tyTuple: - # we don't need to copy m_type field for tyObject, as they are equal anyway - result = genericHashAux(dest, mt.node, shallow, h) - of tyArray, tyArrayConstr: - let d = cast[ByteAddress](dest) - result = h - for i in 0..(mt.size div mt.base.size)-1: - result = result !& genericHashAux(cast[pointer](d +% i*% mt.base.size), - mt.base, shallow, result) - of tyRef: - when defined(trackGcHeaders): - var s = cast[PPointer](dest)[] - if s != nil: - result = result !& hashGcHeader(s) - else: - if shallow: - result = h !& hash(dest, mt.size) - else: - result = h - var s = cast[PPointer](dest)[] - if s != nil: - result = result !& genericHashAux(s, mt.base, shallow, result) - else: - result = h !& hash(dest, mt.size) # hash raw bits - -proc genericHash(dest: pointer, mt: PNimType): int = - result = genericHashAux(dest, mt, false, 0) - -proc dbgRegisterWatchpoint(address: pointer, name: cstring, - typ: PNimType) {.compilerproc.} = - let L = watchPointsLen - for i in 0.. <L: - if watchPoints[i].name == name: - # address may have changed: - watchPoints[i].address = address - return - if L >= watchPoints.high: - #debugOut("[Warning] cannot register watchpoint") - return - watchPoints[L].name = name - watchPoints[L].address = address - watchPoints[L].typ = typ - watchPoints[L].oldValue = genericHash(address, typ) - inc watchPointsLen - -proc dbgUnregisterWatchpoints*() = - watchPointsLen = 0 - -var - dbgLineHook*: proc () {.nimcall.} - ## set this variable to provide a procedure that should be called before - ## each executed instruction. This should only be used by debuggers! - ## Only code compiled with the ``debugger:on`` switch calls this hook. - - dbgWatchpointHook*: proc (watchpointName: cstring) {.nimcall.} - -proc checkWatchpoints = - let L = watchPointsLen - for i in 0.. <L: - let newHash = genericHash(watchPoints[i].address, watchPoints[i].typ) - if newHash != watchPoints[i].oldValue: - dbgWatchpointHook(watchPoints[i].name) - watchPoints[i].oldValue = newHash - -proc endb(line: int, file: cstring) {.compilerproc, noinline.} = - # This proc is called before every Nim code line! - if framePtr == nil: return - if dbgWatchpointHook != nil: checkWatchpoints() - framePtr.line = line # this is done here for smaller code size! - framePtr.filename = file - if dbgLineHook != nil: dbgLineHook() - -include "system/endb" diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 03230e541..72d35f518 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -7,18 +7,69 @@ # distribution, for details about the copyright. # -proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) {.benign.} -proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} = +const + TableSize = when sizeof(int) <= 2: 0xff else: 0xff_ffff + +type + PtrTable = ptr object + counter, max: int + data: array[TableSize, (pointer, pointer)] + +template hashPtr(key: pointer): int = cast[int](key) shr 8 +template allocPtrTable: untyped = + cast[PtrTable](alloc0(sizeof(int)*2 + sizeof(pointer)*2*cap)) + +proc rehash(t: PtrTable): PtrTable = + let cap = (t.max+1) * 2 + result = allocPtrTable() + result.counter = t.counter + result.max = cap-1 + for i in 0..t.max: + let k = t.data[i][0] + if k != nil: + var h = hashPtr(k) + while result.data[h and result.max][0] != nil: inc h + result.data[h and result.max] = t.data[i] + dealloc t + +proc initPtrTable(): PtrTable = + const cap = 32 + result = allocPtrTable() + result.counter = 0 + result.max = cap-1 + +template deinit(t: PtrTable) = dealloc(t) + +proc get(t: PtrTable; key: pointer): pointer = + var h = hashPtr(key) + while true: + let k = t.data[h and t.max][0] + if k == nil: break + if k == key: + return t.data[h and t.max][1] + inc h + +proc put(t: var PtrTable; key, val: pointer) = + if (t.max+1) * 2 < t.counter * 3: t = rehash(t) + var h = hashPtr(key) + while t.data[h and t.max][0] != nil: inc h + t.data[h and t.max] = (key, val) + inc t.counter + +proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; + tab: var PtrTable) {.benign.} +proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; + tab: var PtrTable) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericDeepCopyAux(cast[pointer](d +% n.offset), - cast[pointer](s +% n.offset), n.typ) + cast[pointer](s +% n.offset), n.typ, tab) of nkList: for i in 0..n.len-1: - genericDeepCopyAux(dest, src, n.sons[i]) + genericDeepCopyAux(dest, src, n.sons[i], tab) of nkCase: var dd = selectBranch(dest, n) var m = selectBranch(src, n) @@ -29,90 +80,100 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} = copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), n.typ.size) if m != nil: - genericDeepCopyAux(dest, src, m) + genericDeepCopyAux(dest, src, m, tab) of nkNone: sysAssert(false, "genericDeepCopyAux") -proc copyDeepString(src: NimString): NimString {.inline.} = - if src != nil: - result = rawNewStringNoInit(src.len) - result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) - -proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = +proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericDeepCopyAux 2") case mt.kind of tyString: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil: - unsureAsgnRef(x, s2) + when defined(nimSeqsV2): + var x = cast[ptr NimStringV2](dest) + var s2 = cast[ptr NimStringV2](s)[] + nimAsgnStrV2(x[], s2) else: - unsureAsgnRef(x, copyDeepString(cast[NimString](s2))) + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + unsureAsgnRef(x, s2) + else: + unsureAsgnRef(x, copyDeepString(cast[NimString](s2))) of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil: - unsureAsgnRef(x, s2) - return - sysAssert(dest != nil, "genericDeepCopyAux 3") - unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - genericDeepCopyAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base) + when defined(nimSeqsV2): + deepSeqAssignImpl(genericDeepCopyAux, tab) + else: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + unsureAsgnRef(x, s2) + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + unsureAsgnRef(x, newSeq(mt, seq.len)) + var dst = cast[int](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux( + cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + mt.base, tab) of tyObject: # we need to copy m_type field for tyObject, as it could be empty for # sequence reallocations: - var pint = cast[ptr PNimType](dest) - pint[] = cast[ptr PNimType](src)[] if mt.base != nil: - genericDeepCopyAux(dest, src, mt.base) - genericDeepCopyAux(dest, src, mt.node) + genericDeepCopyAux(dest, src, mt.base, tab) + else: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + genericDeepCopyAux(dest, src, mt.node, tab) of tyTuple: - genericDeepCopyAux(dest, src, mt.node) + genericDeepCopyAux(dest, src, mt.node, tab) of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - genericDeepCopyAux(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) + genericDeepCopyAux(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base, tab) of tyRef: let s2 = cast[PPointer](src)[] if s2 == nil: unsureAsgnRef(cast[PPointer](dest), s2) elif mt.base.deepcopy != nil: let z = mt.base.deepcopy(s2) - unsureAsgnRef(cast[PPointer](dest), z) + when defined(nimSeqsV2): + cast[PPointer](dest)[] = z + else: + unsureAsgnRef(cast[PPointer](dest), z) else: - # we modify the header of the cell temporarily; instead of the type - # field we store a forwarding pointer. XXX This is bad when the cloning - # fails due to OOM etc. - when declared(usrToCell): - # unfortunately we only have cycle detection for our native GCs. - let x = usrToCell(s2) - let forw = cast[int](x.typ) - if (forw and 1) == 1: - # we stored a forwarding pointer, so let's use that: - let z = cast[pointer](forw and not 1) - unsureAsgnRef(cast[PPointer](dest), z) - else: + let z = tab.get(s2) + if z == nil: + when declared(usrToCell): + let x = usrToCell(s2) let realType = x.typ let z = newObj(realType, realType.base.size) - unsureAsgnRef(cast[PPointer](dest), z) - x.typ = cast[PNimType](cast[int](z) or 1) - genericDeepCopyAux(z, s2, realType.base) - x.typ = realType + tab.put(s2, z) + genericDeepCopyAux(z, s2, realType.base, tab) + else: + when false: + # addition check disabled + let x = usrToCell(s2) + let realType = x.typ + sysAssert realType == mt, " types do differ" + when defined(nimSeqsV2): + let typ = if mt.base.kind == tyObject: cast[PNimType](cast[ptr PNimTypeV2](s2)[].typeInfoV1) + else: mt.base + let z = nimNewObj(typ.size, typ.align) + cast[PPointer](dest)[] = z + else: + # this version should work for any other GC: + let typ = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[] else: mt.base + let z = newObj(mt, typ.size) + unsureAsgnRef(cast[PPointer](dest), z) + tab.put(s2, z) + genericDeepCopyAux(z, s2, typ, tab) else: - let realType = mt - let z = newObj(realType, realType.base.size) unsureAsgnRef(cast[PPointer](dest), z) - genericDeepCopyAux(z, s2, realType.base) of tyPtr: # no cycle check here, but also not really required let s2 = cast[PPointer](src)[] @@ -123,10 +184,14 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = else: copyMem(dest, src, mt.size) -proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = - genericDeepCopyAux(dest, src, mt) +proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = + when not defined(nimSeqsV2): GC_disable() + var tab = initPtrTable() + genericDeepCopyAux(dest, src, mt, tab) + deinit tab + when not defined(nimSeqsV2): GC_enable() -proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = +proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = # also invoked for 'string' var src = src genericDeepCopy(dest, addr(src), mt) @@ -134,8 +199,8 @@ proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = proc genericDeepCopyOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: - genericDeepCopy(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) + genericDeepCopy(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim new file mode 100644 index 000000000..89a739d5a --- /dev/null +++ b/lib/system/dollars.nim @@ -0,0 +1,147 @@ +## `$` is Nim's general way of spelling `toString`:idx:. +runnableExamples: + assert $0.1 == "0.1" + assert $(-2*3) == "-6" + +import std/private/[digitsutils, miscdollars] + +when not defined(nimPreviewSlimSystem): + import std/formatfloat + export addFloat + + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) + +proc `$`*(x: int): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) + +proc `$`*(x: int64): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) + +proc `$`*(x: uint64): string {.raises: [].} = + ## Outplace version of `addInt`. + addInt(result, x) + +# same as old `ctfeWhitelist` behavior, whether or not this is a good idea. +template gen(T) = + # xxx simplify this by supporting this in compiler: int{lit} | uint64{lit} | int64{lit} + func `$`*(x: T{lit}): string {.compileTime.} = result.addInt(x) +gen(int) +gen(uint64) +gen(int64) + + +proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} + ## The stringify operator for a boolean argument. Returns `x` + ## converted to the string "false" or "true". + +proc `$`*(x: char): string {.magic: "CharToStr", noSideEffect.} + ## The stringify operator for a character argument. Returns `x` + ## converted to a string. + ## ```Nim + ## assert $'c' == "c" + ## ``` + +proc `$`*(x: cstring): string {.magic: "CStrToStr", noSideEffect.} + ## The stringify operator for a CString argument. Returns `x` + ## converted to a string. + +proc `$`*(x: string): string {.magic: "StrToStr", noSideEffect.} + ## The stringify operator for a string argument. Returns `x` + ## as it is. This operator is useful for generic code, so + ## that `$expr` also works if `expr` is already a string. + +proc `$`*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} + ## The stringify operator for an enumeration argument. This works for + ## any enumeration type thanks to compiler magic. + ## + ## If a `$` operator for a concrete enumeration is provided, this is + ## used instead. (In other words: *Overwriting* is possible.) + +proc `$`*(t: typedesc): string {.magic: "TypeTrait".} + ## Returns the name of the given type. + ## + ## For more procedures dealing with `typedesc`, see + ## `typetraits module <typetraits.html>`_. + ## + ## ```Nim + ## doAssert $(typeof(42)) == "int" + ## doAssert $(typeof("Foo")) == "string" + ## static: doAssert $(typeof(@['A', 'B'])) == "seq[char]" + ## ``` + +proc `$`*[T: tuple](x: T): string = + ## Generic `$` operator for tuples that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(23, 45) == "(23, 45)" + ## $(a: 23, b: 45) == "(a: 23, b: 45)" + ## $() == "()" + ## ``` + tupleObjectDollar(result, x) + +when not defined(nimPreviewSlimSystem): + import std/objectdollar + export objectdollar + +proc collectionToString[T](x: T, prefix, separator, suffix: string): string = + result = prefix + var firstElement = true + for value in items(x): + if firstElement: + firstElement = false + else: + result.add(separator) + + when value isnot string and value isnot seq and compiles(value.isNil): + # this branch should not be necessary + if value.isNil: + result.add "nil" + else: + result.addQuoted(value) + else: + result.addQuoted(value) + result.add(suffix) + +proc `$`*[T](x: set[T]): string = + ## Generic `$` operator for sets that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## ${23, 45} == "{23, 45}" + ## ``` + collectionToString(x, "{", ", ", "}") + +proc `$`*[T](x: seq[T]): string = + ## Generic `$` operator for seqs that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(@[23, 45]) == "@[23, 45]" + ## ``` + collectionToString(x, "@[", ", ", "]") + +proc `$`*[T, U](x: HSlice[T, U]): string = + ## Generic `$` operator for slices that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(1 .. 5) == "1 .. 5" + ## ``` + result = $x.a + result.add(" .. ") + result.add($x.b) + + +when not defined(nimNoArrayToString): + proc `$`*[T, IDX](x: array[IDX, T]): string = + ## Generic `$` operator for arrays that is lifted from the components. + collectionToString(x, "[", ", ", "]") + +proc `$`*[T](x: openArray[T]): string = + ## Generic `$` operator for openarrays that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" + ## ``` + collectionToString(x, "[", ", ", "]") diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 22ac613f8..2162b234f 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -17,23 +17,44 @@ const NilLibHandle: LibHandle = nil -proc rawWrite(f: File, s: string) = - # we cannot throw an exception here! - discard writeBuffer(f, cstring(s), s.len) - proc nimLoadLibraryError(path: string) = # carefully written to avoid memory allocation: - stdout.rawWrite("could not load: ") - stdout.rawWrite(path) - stdout.rawWrite("\n") - quit(1) - -proc procAddrError(name: cstring) {.noinline.} = + const prefix = "could not load: " + cstderr.rawWrite(prefix) + cstderr.rawWrite(path) + when not defined(nimDebugDlOpen) and not defined(windows): + cstderr.rawWrite("\n(compile with -d:nimDebugDlOpen for more information)") + when defined(windows): + const badExe = "\n(bad format; library may be wrong architecture)" + let loadError = GetLastError() + if loadError == ERROR_BAD_EXE_FORMAT: + cstderr.rawWrite(badExe) + when defined(guiapp): + # Because console output is not shown in GUI apps, display the error as a + # message box instead: + var + msg: array[1000, char] + msgLeft = msg.len - 1 # leave (at least) one for nullchar + msgIdx = 0 + copyMem(msg[msgIdx].addr, prefix.cstring, prefix.len) + msgLeft -= prefix.len + msgIdx += prefix.len + let pathLen = min(path.len, msgLeft) + copyMem(msg[msgIdx].addr, path.cstring, pathLen) + msgLeft -= pathLen + msgIdx += pathLen + if loadError == ERROR_BAD_EXE_FORMAT and msgLeft >= badExe.len: + copyMem(msg[msgIdx].addr, badExe.cstring, badExe.len) + discard MessageBoxA(nil, msg[0].addr, nil, 0) + cstderr.rawWrite("\n") + rawQuit(1) + +proc procAddrError(name: cstring) {.compilerproc, nonReloadable, hcrInline.} = # carefully written to avoid memory allocation: - stdout.rawWrite("could not import: ") - stdout.write(name) - stdout.rawWrite("\n") - quit(1) + cstderr.rawWrite("could not import: ") + cstderr.rawWrite(name) + cstderr.rawWrite("\n") + rawQuit(1) # this code was inspired from Lua's source code: # Lua - An Extensible Extension Language @@ -52,11 +73,14 @@ when defined(posix): # # c stuff: - var - RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int + when defined(linux) or defined(macosx): + const RTLD_NOW = cint(2) + else: + var + RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: cint proc dlclose(lib: LibHandle) {.importc, header: "<dlfcn.h>".} - proc dlopen(path: cstring, mode: int): LibHandle {. + proc dlopen(path: cstring, mode: cint): LibHandle {. importc, header: "<dlfcn.h>".} proc dlsym(lib: LibHandle, name: cstring): ProcAddr {. importc, header: "<dlfcn.h>".} @@ -67,10 +91,15 @@ when defined(posix): dlclose(lib) proc nimLoadLibrary(path: string): LibHandle = - result = dlopen(path, RTLD_NOW) - let error = dlerror() - if error != nil: - c_fprintf(c_stdout, "%s\n", error) + let flags = + when defined(globalSymbols): RTLD_NOW or RTLD_GLOBAL + else: RTLD_NOW + result = dlopen(path, flags) + when defined(nimDebugDlOpen): + let error = dlerror() + if error != nil: + cstderr.rawWrite(error) + cstderr.rawWrite("\n") proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) @@ -108,12 +137,63 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return + const decoratedLength = 250 + var decorated: array[decoratedLength, char] + decorated[0] = '_' + var m = 1 + while m < (decoratedLength - 5): + if name[m - 1] == '\x00': break + decorated[m] = name[m - 1] + inc(m) + decorated[m] = '@' for i in countup(0, 50): - var decorated = "_" & $name & "@" & $(i * 4) - result = getProcAddress(cast[THINSTANCE](lib), cstring(decorated)) + var k = i * 4 + if k div 100 == 0: + if k div 10 == 0: + m = m + 1 + else: + m = m + 2 + else: + m = m + 3 + decorated[m + 1] = '\x00' + while true: + decorated[m] = chr(ord('0') + (k %% 10)) + dec(m) + k = k div 10 + if k == 0: break + result = getProcAddress(cast[THINSTANCE](lib), cast[cstring](addr decorated)) if result != nil: return procAddrError(name) +elif defined(genode): + + proc nimUnloadLibrary(lib: LibHandle) = + raiseAssert("nimUnloadLibrary not implemented") + + proc nimLoadLibrary(path: string): LibHandle = + raiseAssert("nimLoadLibrary not implemented") + + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = + raiseAssert("nimGetProcAddr not implemented") + +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr) or defined(nuttx): + proc nimUnloadLibrary(lib: LibHandle) = + cstderr.rawWrite("nimUnLoadLibrary not implemented") + cstderr.rawWrite("\n") + rawQuit(1) + + proc nimLoadLibrary(path: string): LibHandle = + cstderr.rawWrite("nimLoadLibrary not implemented") + cstderr.rawWrite("\n") + rawQuit(1) + + + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = + cstderr.rawWrite("nimGetProAddr not implemented") + cstderr.rawWrite(name) + cstderr.rawWrite("\n") + rawQuit(1) + else: {.error: "no implementation for dyncalls".} diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim index a14f43e7e..ea6776f58 100644 --- a/lib/system/embedded.nim +++ b/lib/system/embedded.nim @@ -19,8 +19,9 @@ proc nimFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = discard proc popFrame {.compilerRtl, inl.} = discard proc setFrame(s: PFrame) {.compilerRtl, inl.} = discard -proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = discard -proc popSafePoint {.compilerRtl, inl.} = discard +when not gotoBasedExceptions: + proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = discard + proc popSafePoint {.compilerRtl, inl.} = discard proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} = discard proc popCurrentException {.compilerRtl, inl.} = discard @@ -29,15 +30,32 @@ const nativeStackTraceSupported = false hasSomeStackTrace = false -proc quitOrDebug() {.inline.} = - quit(1) +proc quitOrDebug() {.noreturn, importc: "abort", header: "<stdlib.h>", nodecl.} proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} = - sysFatal(ReraiseError, "exception handling is not available") + sysFatal(ReraiseDefect, "exception handling is not available") + +proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, + line: int) {.compilerRtl.} = + sysFatal(ReraiseDefect, "exception handling is not available") proc reraiseException() {.compilerRtl.} = - sysFatal(ReraiseError, "no exception to reraise") + sysFatal(ReraiseDefect, "no exception to reraise") proc writeStackTrace() = discard -proc setControlCHook(hook: proc () {.noconv.} not nil) = discard +proc unsetControlCHook() = discard +proc setControlCHook(hook: proc () {.noconv.}) = discard + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + sysFatal(ReraiseDefect, "exception handling is not available") + +when gotoBasedExceptions: + var nimInErrorMode {.threadvar.}: bool + + proc nimErrorFlag(): ptr bool {.compilerRtl, inl.} = + result = addr(nimInErrorMode) + + proc nimTestErrorFlag() {.compilerRtl.} = + if nimInErrorMode: + sysFatal(ReraiseDefect, "exception handling is not available") diff --git a/lib/system/endb.nim b/lib/system/endb.nim deleted file mode 100644 index b2cc5624b..000000000 --- a/lib/system/endb.nim +++ /dev/null @@ -1,558 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This file implements the embedded debugger that can be linked -# with the application. Mostly we do not use dynamic memory here as that -# would interfere with the GC and trigger ON/OFF errors if the -# user program corrupts memory. Unfortunately, for dispaying -# variables we use the ``system.repr()`` proc which uses Nim -# strings and thus allocates memory from the heap. Pity, but -# I do not want to implement ``repr()`` twice. - -const - EndbBeg = "*** endb" - EndbEnd = "***\n" - -type - StaticStr = object - len: int - data: array[0..100, char] - - BreakpointFilename = object - b: ptr Breakpoint - filename: StaticStr - - DbgState = enum - dbOff, # debugger is turned off - dbStepInto, # debugger is in tracing mode - dbStepOver, - dbSkipCurrent, - dbQuiting, # debugger wants to quit - dbBreakpoints # debugger is only interested in breakpoints -{.deprecated: [TStaticStr: StaticStr, TBreakpointFilename: BreakpointFilename, - TDbgState: DbgState].} - -var - dbgUser: StaticStr # buffer for user input; first command is ``step_into`` - # needs to be global cause we store the last command - # in it - dbgState: DbgState # state of debugger - dbgSkipToFrame: PFrame # frame to be skipped to - - maxDisplayRecDepth: int = 5 # do not display too much data! - - brkPoints: array[0..127, BreakpointFilename] - -proc setLen(s: var StaticStr, newLen=0) = - s.len = newLen - s.data[newLen] = '\0' - -proc add(s: var StaticStr, c: char) = - if s.len < high(s.data)-1: - s.data[s.len] = c - s.data[s.len+1] = '\0' - inc s.len - -proc add(s: var StaticStr, c: cstring) = - var i = 0 - while c[i] != '\0': - add s, c[i] - inc i - -proc assign(s: var StaticStr, c: cstring) = - setLen(s) - add s, c - -proc `==`(a, b: StaticStr): bool = - if a.len == b.len: - for i in 0 .. a.len-1: - if a.data[i] != b.data[i]: return false - return true - -proc `==`(a: StaticStr, b: cstring): bool = - result = c_strcmp(a.data, b) == 0 - -proc write(f: File, s: StaticStr) = - write(f, cstring(s.data)) - -proc listBreakPoints() = - write(stdout, EndbBeg) - write(stdout, "| Breakpoints:\n") - for b in listBreakpoints(): - write(stdout, abs(b.low)) - if b.high != b.low: - write(stdout, "..") - write(stdout, abs(b.high)) - write(stdout, " ") - write(stdout, b.filename) - if b.isActive: - write(stdout, " [disabled]\n") - else: - write(stdout, "\n") - write(stdout, EndbEnd) - -proc openAppend(filename: cstring): File = - var p: pointer = fopen(filename, "ab") - if p != nil: - result = cast[File](p) - write(result, "----------------------------------------\n") - -proc dbgRepr(p: pointer, typ: PNimType): string = - var cl: ReprClosure - initReprClosure(cl) - cl.recDepth = maxDisplayRecDepth - # locks for the GC turned out to be a bad idea... - # inc(recGcLock) - result = "" - reprAux(result, p, typ, cl) - # dec(recGcLock) - deinitReprClosure(cl) - -proc writeVariable(stream: File, slot: VarSlot) = - write(stream, slot.name) - write(stream, " = ") - writeLine(stream, dbgRepr(slot.address, slot.typ)) - -proc listFrame(stream: File, f: PFrame) = - write(stream, EndbBeg) - write(stream, "| Frame (") - write(stream, f.len) - write(stream, " slots):\n") - for i in 0 .. f.len-1: - writeLine(stream, getLocal(f, i).name) - write(stream, EndbEnd) - -proc listLocals(stream: File, f: PFrame) = - write(stream, EndbBeg) - write(stream, "| Frame (") - write(stream, f.len) - write(stream, " slots):\n") - for i in 0 .. f.len-1: - writeVariable(stream, getLocal(f, i)) - write(stream, EndbEnd) - -proc listGlobals(stream: File) = - write(stream, EndbBeg) - write(stream, "| Globals:\n") - for i in 0 .. getGlobalLen()-1: - writeLine(stream, getGlobal(i).name) - write(stream, EndbEnd) - -proc debugOut(msg: cstring) = - # the *** *** markers are for easy recognition of debugger - # output for external frontends. - write(stdout, EndbBeg) - write(stdout, "| ") - write(stdout, msg) - write(stdout, EndbEnd) - -proc dbgFatal(msg: cstring) = - debugOut(msg) - dbgAborting = true # the debugger wants to abort - quit(1) - -proc dbgShowCurrentProc(dbgFramePointer: PFrame) = - if dbgFramePointer != nil: - write(stdout, "*** endb| now in proc: ") - write(stdout, dbgFramePointer.procname) - write(stdout, " ***\n") - else: - write(stdout, "*** endb| (proc name not available) ***\n") - -proc dbgShowExecutionPoint() = - write(stdout, "*** endb| ") - write(stdout, framePtr.filename) - write(stdout, "(") - write(stdout, framePtr.line) - write(stdout, ") ") - write(stdout, framePtr.procname) - write(stdout, " ***\n") - -proc scanAndAppendWord(src: cstring, a: var StaticStr, start: int): int = - result = start - # skip whitespace: - while src[result] in {'\t', ' '}: inc(result) - while true: - case src[result] - of 'a'..'z', '0'..'9': add(a, src[result]) - of '_': discard # just skip it - of 'A'..'Z': add(a, chr(ord(src[result]) - ord('A') + ord('a'))) - else: break - inc(result) - -proc scanWord(src: cstring, a: var StaticStr, start: int): int = - setlen(a) - result = scanAndAppendWord(src, a, start) - -proc scanFilename(src: cstring, a: var StaticStr, start: int): int = - result = start - setLen a - while src[result] in {'\t', ' '}: inc(result) - while src[result] notin {'\t', ' ', '\0'}: - add(a, src[result]) - inc(result) - -proc scanNumber(src: cstring, a: var int, start: int): int = - result = start - a = 0 - while src[result] in {'\t', ' '}: inc(result) - while true: - case src[result] - of '0'..'9': a = a * 10 + ord(src[result]) - ord('0') - of '_': discard # skip underscores (nice for long line numbers) - else: break - inc(result) - -proc dbgHelp() = - debugOut(""" -list of commands (see the manual for further help): - GENERAL -h, help display this help message -q, quit quit the debugger and the program -<ENTER> repeat the previous debugger command - EXECUTING -s, step single step, stepping into routine calls -n, next single step, without stepping into routine calls -f, skipcurrent continue execution until the current routine finishes -c, continue, r, run continue execution until the next breakpoint -i, ignore continue execution, ignore all breakpoints - BREAKPOINTS -b, break [fromline [toline]] [file] - set a new breakpoint for line and file - if line or file are omitted the current one is used -breakpoints display the entire breakpoint list -toggle fromline [file] enable or disable a breakpoint -filenames list all valid filenames - DATA DISPLAY -e, eval <expr> evaluate the expression <expr> -o, out <file> <expr> evaluate <expr> and write it to <file> -w, where display the current execution point -stackframe [file] display current stack frame [and write it to file] -u, up go up in the call stack -d, down go down in the call stack -bt, backtrace display the entire call stack -l, locals display available local variables -g, globals display available global variables -maxdisplay <integer> set the display's recursion maximum -""") - -proc invalidCommand() = - debugOut("[Warning] invalid command ignored (type 'h' for help) ") - -proc hasExt(s: cstring): bool = - # returns true if s has a filename extension - var i = 0 - while s[i] != '\0': - if s[i] == '.': return true - inc i - -proc parseBreakpoint(s: cstring, start: int): Breakpoint = - var dbgTemp: StaticStr - var i = scanNumber(s, result.low, start) - if result.low == 0: result.low = framePtr.line - i = scanNumber(s, result.high, i) - if result.high == 0: result.high = result.low - i = scanFilename(s, dbgTemp, i) - if dbgTemp.len != 0: - if not hasExt(dbgTemp.data): add(dbgTemp, ".nim") - result.filename = canonFilename(dbgTemp.data.cstring) - if result.filename.isNil: - debugOut("[Warning] no breakpoint could be set; unknown filename ") - return - else: - result.filename = framePtr.filename - -proc createBreakPoint(s: cstring, start: int) = - let br = parseBreakpoint(s, start) - if not br.filename.isNil: - if not addBreakpoint(br.filename, br.low, br.high): - debugOut("[Warning] no breakpoint could be set; out of breakpoint space ") - -proc breakpointToggle(s: cstring, start: int) = - var a = parseBreakpoint(s, start) - if not a.filename.isNil: - var b = checkBreakpoints(a.filename, a.low) - if not b.isNil: b.flip - else: debugOut("[Warning] unknown breakpoint ") - -proc dbgEvaluate(stream: File, s: cstring, start: int, f: PFrame) = - var dbgTemp: StaticStr - var i = scanWord(s, dbgTemp, start) - while s[i] in {' ', '\t'}: inc(i) - var v: VarSlot - if s[i] == '.': - inc(i) - add(dbgTemp, '.') - i = scanAndAppendWord(s, dbgTemp, i) - for i in 0 .. getGlobalLen()-1: - let v = getGlobal(i) - if c_strcmp(v.name, dbgTemp.data) == 0: - writeVariable(stream, v) - else: - for i in 0 .. f.len-1: - let v = getLocal(f, i) - if c_strcmp(v.name, dbgTemp.data) == 0: - writeVariable(stream, v) - -proc dbgOut(s: cstring, start: int, currFrame: PFrame) = - var dbgTemp: StaticStr - var i = scanFilename(s, dbgTemp, start) - if dbgTemp.len == 0: - invalidCommand() - return - var stream = openAppend(dbgTemp.data) - if stream == nil: - debugOut("[Warning] could not open or create file ") - return - dbgEvaluate(stream, s, i, currFrame) - close(stream) - -proc dbgStackFrame(s: cstring, start: int, currFrame: PFrame) = - var dbgTemp: StaticStr - var i = scanFilename(s, dbgTemp, start) - if dbgTemp.len == 0: - # just write it to stdout: - listFrame(stdout, currFrame) - else: - var stream = openAppend(dbgTemp.data) - if stream == nil: - debugOut("[Warning] could not open or create file ") - return - listFrame(stream, currFrame) - close(stream) - -proc readLine(f: File, line: var StaticStr): bool = - while true: - var c = fgetc(f) - if c < 0'i32: - if line.len > 0: break - else: return false - if c == 10'i32: break # LF - if c == 13'i32: # CR - c = fgetc(f) # is the next char LF? - if c != 10'i32: ungetc(c, f) # no, put the character back - break - add line, chr(int(c)) - result = true - -proc listFilenames() = - write(stdout, EndbBeg) - write(stdout, "| Files:\n") - var i = 0 - while true: - let x = dbgFilenames[i] - if x.isNil: break - write(stdout, x) - write(stdout, "\n") - inc i - write(stdout, EndbEnd) - -proc dbgWriteStackTrace(f: PFrame) -proc commandPrompt() = - # if we return from this routine, user code executes again - var - again = true - dbgFramePtr = framePtr # for going down and up the stack - dbgDown = 0 # how often we did go down - dbgTemp: StaticStr - - while again: - write(stdout, "*** endb| >>") - let oldLen = dbgUser.len - dbgUser.len = 0 - if not readLine(stdin, dbgUser): break - if dbgUser.len == 0: dbgUser.len = oldLen - # now look what we have to do: - var i = scanWord(dbgUser.data, dbgTemp, 0) - template `?`(x: expr): expr = dbgTemp == cstring(x) - if ?"s" or ?"step": - dbgState = dbStepInto - again = false - elif ?"n" or ?"next": - dbgState = dbStepOver - dbgSkipToFrame = framePtr - again = false - elif ?"f" or ?"skipcurrent": - dbgState = dbSkipCurrent - dbgSkipToFrame = framePtr.prev - again = false - elif ?"c" or ?"continue" or ?"r" or ?"run": - dbgState = dbBreakpoints - again = false - elif ?"i" or ?"ignore": - dbgState = dbOff - again = false - elif ?"h" or ?"help": - dbgHelp() - elif ?"q" or ?"quit": - dbgState = dbQuiting - dbgAborting = true - again = false - quit(1) # BUGFIX: quit with error code > 0 - elif ?"e" or ?"eval": - var - prevState = dbgState - prevSkipFrame = dbgSkipToFrame - dbgState = dbSkipCurrent - dbgEvaluate(stdout, dbgUser.data, i, dbgFramePtr) - dbgState = prevState - dbgSkipToFrame = prevSkipFrame - elif ?"o" or ?"out": - dbgOut(dbgUser.data, i, dbgFramePtr) - elif ?"stackframe": - dbgStackFrame(dbgUser.data, i, dbgFramePtr) - elif ?"w" or ?"where": - dbgShowExecutionPoint() - elif ?"l" or ?"locals": - var - prevState = dbgState - prevSkipFrame = dbgSkipToFrame - dbgState = dbSkipCurrent - listLocals(stdout, dbgFramePtr) - dbgState = prevState - dbgSkipToFrame = prevSkipFrame - elif ?"g" or ?"globals": - var - prevState = dbgState - prevSkipFrame = dbgSkipToFrame - dbgState = dbSkipCurrent - listGlobals(stdout) - dbgState = prevState - dbgSkipToFrame = prevSkipFrame - elif ?"u" or ?"up": - if dbgDown <= 0: - debugOut("[Warning] cannot go up any further ") - else: - dbgFramePtr = framePtr - for j in 0 .. dbgDown-2: # BUGFIX - dbgFramePtr = dbgFramePtr.prev - dec(dbgDown) - dbgShowCurrentProc(dbgFramePtr) - elif ?"d" or ?"down": - if dbgFramePtr != nil: - inc(dbgDown) - dbgFramePtr = dbgFramePtr.prev - dbgShowCurrentProc(dbgFramePtr) - else: - debugOut("[Warning] cannot go down any further ") - elif ?"bt" or ?"backtrace": - dbgWriteStackTrace(framePtr) - elif ?"b" or ?"break": - createBreakPoint(dbgUser.data, i) - elif ?"breakpoints": - listBreakPoints() - elif ?"toggle": - breakpointToggle(dbgUser.data, i) - elif ?"filenames": - listFilenames() - elif ?"maxdisplay": - var parsed: int - i = scanNumber(dbgUser.data, parsed, i) - if dbgUser.data[i-1] in {'0'..'9'}: - if parsed == 0: maxDisplayRecDepth = -1 - else: maxDisplayRecDepth = parsed - else: - invalidCommand() - else: invalidCommand() - -proc endbStep() = - # we get into here if an unhandled exception has been raised - # XXX: do not allow the user to run the program any further? - # XXX: BUG: the frame is lost here! - dbgShowExecutionPoint() - commandPrompt() - -proc dbgWriteStackTrace(f: PFrame) = - const - firstCalls = 32 - var - it = f - i = 0 - total = 0 - tempFrames: array [0..127, PFrame] - # setup long head: - while it != nil and i <= high(tempFrames)-firstCalls: - tempFrames[i] = it - inc(i) - inc(total) - it = it.prev - # go up the stack to count 'total': - var b = it - while it != nil: - inc(total) - it = it.prev - var skipped = 0 - if total > len(tempFrames): - # skip N - skipped = total-i-firstCalls+1 - for j in 1..skipped: - if b != nil: b = b.prev - # create '...' entry: - tempFrames[i] = nil - inc(i) - # setup short tail: - while b != nil and i <= high(tempFrames): - tempFrames[i] = b - inc(i) - b = b.prev - for j in countdown(i-1, 0): - if tempFrames[j] == nil: - write(stdout, "(") - write(stdout, skipped) - write(stdout, " calls omitted) ...") - else: - write(stdout, tempFrames[j].filename) - if tempFrames[j].line > 0: - write(stdout, '(') - write(stdout, tempFrames[j].line) - write(stdout, ')') - write(stdout, ' ') - write(stdout, tempFrames[j].procname) - write(stdout, "\n") - -proc checkForBreakpoint = - let b = checkBreakpoints(framePtr.filename, framePtr.line) - if b != nil: - write(stdout, "*** endb| reached ") - write(stdout, framePtr.filename) - write(stdout, "(") - write(stdout, framePtr.line) - write(stdout, ") ") - write(stdout, framePtr.procname) - write(stdout, " ***\n") - commandPrompt() - -proc lineHookImpl() {.nimcall.} = - case dbgState - of dbStepInto: - # we really want the command prompt here: - dbgShowExecutionPoint() - commandPrompt() - of dbSkipCurrent, dbStepOver: # skip current routine - if framePtr == dbgSkipToFrame: - dbgShowExecutionPoint() - commandPrompt() - else: - # breakpoints are wanted though (I guess) - checkForBreakpoint() - of dbBreakpoints: - # debugger is only interested in breakpoints - checkForBreakpoint() - else: discard - -proc watchpointHookImpl(name: cstring) {.nimcall.} = - dbgWriteStackTrace(framePtr) - debugOut(name) - -proc initDebugger {.inline.} = - dbgState = dbStepInto - dbgUser.len = 1 - dbgUser.data[0] = 's' - dbgWatchpointHook = watchpointHookImpl - dbgLineHook = lineHookImpl diff --git a/lib/system/exceptions.nim b/lib/system/exceptions.nim new file mode 100644 index 000000000..63588f858 --- /dev/null +++ b/lib/system/exceptions.nim @@ -0,0 +1,122 @@ +## Exception and effect types used in Nim code. + +type + TimeEffect* = object of RootEffect ## Time effect. + IOEffect* = object of RootEffect ## IO effect. + ReadIOEffect* = object of IOEffect ## Effect describing a read IO operation. + WriteIOEffect* = object of IOEffect ## Effect describing a write IO operation. + ExecIOEffect* = object of IOEffect ## Effect describing an executing IO operation. + +type + IOError* = object of CatchableError ## \ + ## Raised if an IO error occurred. + EOFError* = object of IOError ## \ + ## Raised if an IO "end of file" error occurred. + OSError* = object of CatchableError ## \ + ## Raised if an operating system service failed. + errorCode*: int32 ## OS-defined error code describing this error. + LibraryError* = object of OSError ## \ + ## Raised if a dynamic library could not be loaded. + ResourceExhaustedError* = object of CatchableError ## \ + ## Raised if a resource request could not be fulfilled. + ArithmeticDefect* = object of Defect ## \ + ## Raised if any kind of arithmetic error occurred. + DivByZeroDefect* = object of ArithmeticDefect ## \ + ## Raised for runtime integer divide-by-zero errors. + + OverflowDefect* = object of ArithmeticDefect ## \ + ## Raised for runtime integer overflows. + ## + ## This happens for calculations whose results are too large to fit in the + ## provided bits. + AccessViolationDefect* = object of Defect ## \ + ## Raised for invalid memory access errors + AssertionDefect* = object of Defect ## \ + ## Raised when assertion is proved wrong. + ## + ## Usually the result of using the `assert() template + ## <assertions.html#assert.t,untyped,string>`_. + ValueError* = object of CatchableError ## \ + ## Raised for string and object conversion errors. + KeyError* = object of ValueError ## \ + ## Raised if a key cannot be found in a table. + ## + ## Mostly used by the `tables <tables.html>`_ module, it can also be raised + ## by other collection modules like `sets <sets.html>`_ or `strtabs + ## <strtabs.html>`_. + OutOfMemDefect* = object of Defect ## \ + ## Raised for unsuccessful attempts to allocate memory. + IndexDefect* = object of Defect ## \ + ## Raised if an array index is out of bounds. + + FieldDefect* = object of Defect ## \ + ## Raised if a record field is not accessible because its discriminant's + ## value does not fit. + RangeDefect* = object of Defect ## \ + ## Raised if a range check error occurred. + StackOverflowDefect* = object of Defect ## \ + ## Raised if the hardware stack used for subroutine calls overflowed. + ReraiseDefect* = object of Defect ## \ + ## Raised if there is no exception to reraise. + ObjectAssignmentDefect* = object of Defect ## \ + ## Raised if an object gets assigned to its parent's object. + ObjectConversionDefect* = object of Defect ## \ + ## Raised if an object is converted to an incompatible object type. + ## You can use `of` operator to check if conversion will succeed. + FloatingPointDefect* = object of Defect ## \ + ## Base class for floating point exceptions. + FloatInvalidOpDefect* = object of FloatingPointDefect ## \ + ## Raised by invalid operations according to IEEE. + ## + ## Raised by `0.0/0.0`, for example. + FloatDivByZeroDefect* = object of FloatingPointDefect ## \ + ## Raised by division by zero. + ## + ## Divisor is zero and dividend is a finite nonzero number. + FloatOverflowDefect* = object of FloatingPointDefect ## \ + ## Raised for overflows. + ## + ## The operation produced a result that exceeds the range of the exponent. + FloatUnderflowDefect* = object of FloatingPointDefect ## \ + ## Raised for underflows. + ## + ## The operation produced a result that is too small to be represented as a + ## normal number. + FloatInexactDefect* = object of FloatingPointDefect ## \ + ## Raised for inexact results. + ## + ## The operation produced a result that cannot be represented with infinite + ## precision -- for example: `2.0 / 3.0, log(1.1)` + ## + ## **Note**: Nim currently does not detect these! + DeadThreadDefect* = object of Defect ## \ + ## Raised if it is attempted to send a message to a dead thread. + NilAccessDefect* = object of Defect ## \ + ## Raised on dereferences of `nil` pointers. + ## + ## This is only raised if the `segfaults module <segfaults.html>`_ was imported! + +when not defined(nimPreviewSlimSystem): + type + ArithmeticError* {.deprecated: "See corresponding Defect".} = ArithmeticDefect + DivByZeroError* {.deprecated: "See corresponding Defect".} = DivByZeroDefect + OverflowError* {.deprecated: "See corresponding Defect".} = OverflowDefect + AccessViolationError* {.deprecated: "See corresponding Defect".} = AccessViolationDefect + AssertionError* {.deprecated: "See corresponding Defect".} = AssertionDefect + OutOfMemError* {.deprecated: "See corresponding Defect".} = OutOfMemDefect + IndexError* {.deprecated: "See corresponding Defect".} = IndexDefect + + FieldError* {.deprecated: "See corresponding Defect".} = FieldDefect + RangeError* {.deprecated: "See corresponding Defect".} = RangeDefect + StackOverflowError* {.deprecated: "See corresponding Defect".} = StackOverflowDefect + ReraiseError* {.deprecated: "See corresponding Defect".} = ReraiseDefect + ObjectAssignmentError* {.deprecated: "See corresponding Defect".} = ObjectAssignmentDefect + ObjectConversionError* {.deprecated: "See corresponding Defect".} = ObjectConversionDefect + FloatingPointError* {.deprecated: "See corresponding Defect".} = FloatingPointDefect + FloatInvalidOpError* {.deprecated: "See corresponding Defect".} = FloatInvalidOpDefect + FloatDivByZeroError* {.deprecated: "See corresponding Defect".} = FloatDivByZeroDefect + FloatOverflowError* {.deprecated: "See corresponding Defect".} = FloatOverflowDefect + FloatUnderflowError* {.deprecated: "See corresponding Defect".} = FloatUnderflowDefect + FloatInexactError* {.deprecated: "See corresponding Defect".} = FloatInexactDefect + DeadThreadError* {.deprecated: "See corresponding Defect".} = DeadThreadDefect + NilAccessError* {.deprecated: "See corresponding Defect".} = NilAccessDefect diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index df28c1493..dae5c4a4a 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -10,69 +10,167 @@ # Exception handling code. Carefully coded so that tiny programs which do not # use the heap (and nor exceptions) do not include the GC or memory allocator. +import std/private/miscdollars +import stacktraces + +const noStacktraceAvailable = "No stack traceback available\n" + var - errorMessageWriter*: (proc(msg: string) {.tags: [WriteIOEffect], benign.}) + errorMessageWriter*: (proc(msg: string) {.tags: [WriteIOEffect], benign, + nimcall.}) ## Function that will be called - ## instead of stdmsg.write when printing stacktrace. + ## instead of `stdmsg.write` when printing stacktrace. ## Unstable API. -when not defined(windows) or not defined(guiapp): - proc writeToStdErr(msg: cstring) = write(stdmsg, msg) +when defined(windows): + proc GetLastError(): int32 {.header: "<windows.h>", nodecl.} + const ERROR_BAD_EXE_FORMAT = 193 +when not defined(windows) or not defined(guiapp): + proc writeToStdErr(msg: cstring) = rawWrite(cstderr, msg) + proc writeToStdErr(msg: cstring, length: int) = + rawWriteString(cstderr, msg, length) else: - proc MessageBoxA(hWnd: cint, lpText, lpCaption: cstring, uType: int): int32 {. + proc MessageBoxA(hWnd: pointer, lpText, lpCaption: cstring, uType: int): int32 {. header: "<windows.h>", nodecl.} - proc writeToStdErr(msg: cstring) = - discard MessageBoxA(0, msg, nil, 0) + discard MessageBoxA(nil, msg, nil, 0) + proc writeToStdErr(msg: cstring, length: int) = + discard MessageBoxA(nil, msg, nil, 0) + +proc writeToStdErr(msg: string) {.inline.} = + # fix bug #13115: handles correctly '\0' unlike default implicit conversion to cstring + writeToStdErr(msg.cstring, msg.len) -proc showErrorMessage(data: cstring) = +proc showErrorMessage(data: cstring, length: int) {.gcsafe, raises: [].} = + var toWrite = true if errorMessageWriter != nil: - errorMessageWriter($data) - else: - writeToStdErr(data) + try: + errorMessageWriter($data) + toWrite = false + except: + discard + if toWrite: + when defined(genode): + # stderr not available by default, use the LOG session + echo data + else: + writeToStdErr(data, length) + +proc showErrorMessage2(data: string) {.inline.} = + showErrorMessage(data.cstring, data.len) proc chckIndx(i, a, b: int): int {.inline, compilerproc, benign.} proc chckRange(i, a, b: int): int {.inline, compilerproc, benign.} proc chckRangeF(x, a, b: float): float {.inline, compilerproc, benign.} proc chckNil(p: pointer) {.noinline, compilerproc, benign.} +type + GcFrame = ptr GcFrameHeader + GcFrameHeader {.compilerproc.} = object + len: int + prev: ptr GcFrameHeader + +when NimStackTraceMsgs: + var frameMsgBuf* {.threadvar.}: string + +when not defined(nimV2): + var + framePtr {.threadvar.}: PFrame + var - framePtr {.threadvar.}: PFrame - excHandler {.threadvar.}: PSafePoint - # list of exception handlers - # a global variable for the root of all try blocks currException {.threadvar.}: ref Exception +when not gotoBasedExceptions: + var + excHandler {.threadvar.}: PSafePoint + # list of exception handlers + # a global variable for the root of all try blocks + gcFramePtr {.threadvar.}: GcFrame + +when gotoBasedExceptions: + type + FrameState = tuple[framePtr: PFrame, + currException: ref Exception] +else: + type + FrameState = tuple[gcFramePtr: GcFrame, framePtr: PFrame, + excHandler: PSafePoint, currException: ref Exception] + +proc getFrameState*(): FrameState {.compilerRtl, inl.} = + when gotoBasedExceptions: + return (framePtr, currException) + else: + return (gcFramePtr, framePtr, excHandler, currException) + +proc setFrameState*(state: FrameState) {.compilerRtl, inl.} = + when gotoBasedExceptions: + framePtr = state.framePtr + currException = state.currException + else: + gcFramePtr = state.gcFramePtr + framePtr = state.framePtr + excHandler = state.excHandler + currException = state.currException + proc getFrame*(): PFrame {.compilerRtl, inl.} = framePtr proc popFrame {.compilerRtl, inl.} = framePtr = framePtr.prev +when false: + proc popFrameOfAddr(s: PFrame) {.compilerRtl.} = + var it = framePtr + if it == s: + framePtr = framePtr.prev + else: + while it != nil: + if it == s: + framePtr = it.prev + break + it = it.prev + proc setFrame*(s: PFrame) {.compilerRtl, inl.} = framePtr = s -proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = - s.hasRaiseAction = false - s.prev = excHandler - excHandler = s +when not gotoBasedExceptions: + proc getGcFrame*(): GcFrame {.compilerRtl, inl.} = gcFramePtr + proc popGcFrame*() {.compilerRtl, inl.} = gcFramePtr = gcFramePtr.prev + proc setGcFrame*(s: GcFrame) {.compilerRtl, inl.} = gcFramePtr = s + proc pushGcFrame*(s: GcFrame) {.compilerRtl, inl.} = + s.prev = gcFramePtr + zeroMem(cast[pointer](cast[int](s)+%sizeof(GcFrameHeader)), s.len*sizeof(pointer)) + gcFramePtr = s -proc popSafePoint {.compilerRtl, inl.} = - excHandler = excHandler.prev + proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = + s.prev = excHandler + excHandler = s -proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} = - e.parent = currException + proc popSafePoint {.compilerRtl, inl.} = + excHandler = excHandler.prev + +proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = + e.up = currException currException = e + #showErrorMessage2 "A" proc popCurrentException {.compilerRtl, inl.} = - currException = nil # currException.parent + currException = currException.up + #showErrorMessage2 "B" + +proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = + discard "only for bootstrapping compatbility" + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + currException = e # some platforms have native support for stack traces: const - nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and + nativeStackTraceSupported = (defined(macosx) or defined(linux)) and not NimStackTrace - hasSomeStackTrace = NimStackTrace or - defined(nativeStackTrace) and nativeStackTraceSupported + hasSomeStackTrace = NimStackTrace or defined(nimStackTraceOverride) or + (defined(nativeStackTrace) and nativeStackTraceSupported) + when defined(nativeStacktrace) and nativeStackTraceSupported: type @@ -90,13 +188,13 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempAddresses: array [0..127, pointer] # should not be alloc'd on stack + tempAddresses: array[maxStackTraceLines, pointer] # should not be alloc'd on stack tempDlInfo: TDl_info proc auxWriteStackTraceWithBacktrace(s: var string) = when hasThreadSupport: var - tempAddresses: array [0..127, pointer] # but better than a threadvar + tempAddresses: array[maxStackTraceLines, pointer] # but better than a threadvar tempDlInfo: TDl_info # This is allowed to be expensive since it only happens during crashes # (but this way you don't need manual stack tracing) @@ -122,68 +220,121 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: # interested in enabled = true -when not hasThreadSupport: +when hasSomeStackTrace and not hasThreadSupport: var - tempFrames: array [0..127, PFrame] # should not be alloc'd on stack + tempFrames: array[maxStackTraceLines, PFrame] # should not be alloc'd on stack -proc auxWriteStackTrace(f: PFrame, s: var string) = - when hasThreadSupport: - var - tempFrames: array [0..127, PFrame] # but better than a threadvar - const - firstCalls = 32 +template reraisedFrom(z): untyped = + StackTraceEntry(procname: nil, line: z, filename: nil) + +proc auxWriteStackTrace(f: PFrame; s: var seq[StackTraceEntry]) = var it = f i = 0 - total = 0 - # setup long head: - while it != nil and i <= high(tempFrames)-firstCalls: - tempFrames[i] = it + while it != nil: inc(i) - inc(total) it = it.prev - # go up the stack to count 'total': - var b = it + var last = i-1 + when true: # not defined(gcDestructors): + if s.len == 0: + s = newSeq[StackTraceEntry](i) + else: + last = s.len + i - 1 + s.setLen(last+1) + it = f while it != nil: - inc(total) + s[last] = StackTraceEntry(procname: it.procname, + line: it.line, + filename: it.filename) + when NimStackTraceMsgs: + let first = if it.prev == nil: 0 else: it.prev.frameMsgLen + if it.frameMsgLen > first: + s[last].frameMsg.setLen(it.frameMsgLen - first) + # somehow string slicing not available here + for i in first .. it.frameMsgLen-1: + s[last].frameMsg[i-first] = frameMsgBuf[i] it = it.prev - var skipped = 0 - if total > len(tempFrames): - # skip N - skipped = total-i-firstCalls+1 - for j in 1..skipped: - if b != nil: b = b.prev - # create '...' entry: - tempFrames[i] = nil - inc(i) - # setup short tail: - while b != nil and i <= high(tempFrames): - tempFrames[i] = b - inc(i) - b = b.prev - for j in countdown(i-1, 0): - if tempFrames[j] == nil: - add(s, "(") - add(s, $skipped) - add(s, " calls omitted) ...") + dec last + +template addFrameEntry(s: var string, f: StackTraceEntry|PFrame) = + var oldLen = s.len + s.toLocation(f.filename, f.line, 0) + for k in 1..max(1, 25-(s.len-oldLen)): add(s, ' ') + add(s, f.procname) + when NimStackTraceMsgs: + when typeof(f) is StackTraceEntry: + add(s, f.frameMsg) else: - var oldLen = s.len - add(s, tempFrames[j].filename) - if tempFrames[j].line > 0: - add(s, '(') - add(s, $tempFrames[j].line) - add(s, ')') - for k in 1..max(1, 25-(s.len-oldLen)): add(s, ' ') - add(s, tempFrames[j].procname) - add(s, "\n") - -proc stackTraceAvailable*(): bool + var first = if f.prev == nil: 0 else: f.prev.frameMsgLen + for i in first..<f.frameMsgLen: add(s, frameMsgBuf[i]) + add(s, "\n") + +proc `$`(stackTraceEntries: seq[StackTraceEntry]): string = + when defined(nimStackTraceOverride): + let s = addDebuggingInfo(stackTraceEntries) + else: + let s = stackTraceEntries + + result = newStringOfCap(2000) + for i in 0 .. s.len-1: + if s[i].line == reraisedFromBegin: result.add "[[reraised from:\n" + elif s[i].line == reraisedFromEnd: result.add "]]\n" + else: addFrameEntry(result, s[i]) when hasSomeStackTrace: + + proc auxWriteStackTrace(f: PFrame, s: var string) = + when hasThreadSupport: + var + tempFrames: array[maxStackTraceLines, PFrame] # but better than a threadvar + const + firstCalls = 32 + var + it = f + i = 0 + total = 0 + # setup long head: + while it != nil and i <= high(tempFrames)-firstCalls: + tempFrames[i] = it + inc(i) + inc(total) + it = it.prev + # go up the stack to count 'total': + var b = it + while it != nil: + inc(total) + it = it.prev + var skipped = 0 + if total > len(tempFrames): + # skip N + skipped = total-i-firstCalls+1 + for j in 1..skipped: + if b != nil: b = b.prev + # create '...' entry: + tempFrames[i] = nil + inc(i) + # setup short tail: + while b != nil and i <= high(tempFrames): + tempFrames[i] = b + inc(i) + b = b.prev + for j in countdown(i-1, 0): + if tempFrames[j] == nil: + add(s, "(") + add(s, $skipped) + add(s, " calls omitted) ...\n") + else: + addFrameEntry(s, tempFrames[j]) + + proc stackTraceAvailable*(): bool + proc rawWriteStackTrace(s: var string) = - when NimStackTrace: + when defined(nimStackTraceOverride): + add(s, "Traceback (most recent call last, using override)\n") + auxWriteStackTraceWithOverride(s) + elif NimStackTrace: if framePtr == nil: - add(s, "No stack traceback available\n") + add(s, noStacktraceAvailable) else: add(s, "Traceback (most recent call last)\n") auxWriteStackTrace(framePtr, s) @@ -191,9 +342,20 @@ when hasSomeStackTrace: add(s, "Traceback from system (most recent call last)\n") auxWriteStackTraceWithBacktrace(s) else: - add(s, "No stack traceback available\n") + add(s, noStacktraceAvailable) + + proc rawWriteStackTrace(s: var seq[StackTraceEntry]) = + when defined(nimStackTraceOverride): + auxWriteStackTraceWithOverride(s) + elif NimStackTrace: + auxWriteStackTrace(framePtr, s) + else: + s = @[] + proc stackTraceAvailable(): bool = - when NimStackTrace: + when defined(nimStackTraceOverride): + result = true + elif NimStackTrace: if framePtr == nil: result = false else: @@ -205,129 +367,273 @@ when hasSomeStackTrace: else: proc stackTraceAvailable*(): bool = result = false -proc quitOrDebug() {.inline.} = - when not defined(endb): - quit(1) +var onUnhandledException*: (proc (errorMsg: string) {. + nimcall, gcsafe.}) ## Set this error \ + ## handler to override the existing behaviour on an unhandled exception. + ## + ## The default is to write a stacktrace to `stderr` and then call `quit(1)`. + ## Unstable API. + +proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy, gcsafe.} = + when hasSomeStackTrace: + var buf = newStringOfCap(2000) + if e.trace.len == 0: + rawWriteStackTrace(buf) + else: + var trace = $e.trace + add(buf, trace) + {.gcsafe.}: + `=destroy`(trace) + add(buf, "Error: unhandled exception: ") + add(buf, e.msg) + add(buf, " [") + add(buf, $e.name) + add(buf, "]\n") + + if onUnhandledException != nil: + onUnhandledException(buf) + else: + showErrorMessage2(buf) + {.gcsafe.}: + `=destroy`(buf) else: - endbStep() # call the debugger + # ugly, but avoids heap allocations :-) + template xadd(buf, s, slen) = + if L + slen < high(buf): + copyMem(addr(buf[L]), (when s is cstring: s else: cstring(s)), slen) + inc L, slen + template add(buf, s) = + xadd(buf, s, s.len) + var buf: array[0..2000, char] + var L = 0 + if e.trace.len != 0: + var trace = $e.trace + add(buf, trace) + {.gcsafe.}: + `=destroy`(trace) + add(buf, "Error: unhandled exception: ") + add(buf, e.msg) + add(buf, " [") + xadd(buf, e.name, e.name.len) + add(buf, "]\n") + if onUnhandledException != nil: + onUnhandledException($cast[cstring](buf.addr)) + else: + showErrorMessage(cast[cstring](buf.addr), L) + +proc reportUnhandledError(e: ref Exception) {.nodestroy, gcsafe.} = + if unhandledExceptionHook != nil: + unhandledExceptionHook(e) + when hostOS != "any": + reportUnhandledErrorAux(e) + +when not gotoBasedExceptions: + proc nimLeaveFinally() {.compilerRtl.} = + when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: + {.emit: "throw;".} + else: + if excHandler != nil: + c_longjmp(excHandler.context, 1) + else: + reportUnhandledError(currException) + rawQuit(1) + +when gotoBasedExceptions: + var nimInErrorMode {.threadvar.}: bool + + proc nimErrorFlag(): ptr bool {.compilerRtl, inl.} = + result = addr(nimInErrorMode) + + proc nimTestErrorFlag() {.compilerRtl.} = + ## This proc must be called before `currException` is destroyed. + ## It also must be called at the end of every thread to ensure no + ## error is swallowed. + if nimInErrorMode and currException != nil: + reportUnhandledError(currException) + currException = nil + rawQuit(1) + +proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = + when defined(nimPanics): + if e of Defect: + reportUnhandledError(e) + rawQuit(1) -proc raiseExceptionAux(e: ref Exception) = if localRaiseHook != nil: if not localRaiseHook(e): return if globalRaiseHook != nil: if not globalRaiseHook(e): return - when defined(cpp): - if e[] of OutOfMemError: - showErrorMessage(e.name) - quitOrDebug() + when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: + if e == currException: + {.emit: "throw;".} else: pushCurrentException(e) - {.emit: "throw NimException(`e`, `e`->name);".} + {.emit: "throw `e`;".} + elif quirkyExceptions or gotoBasedExceptions: + pushCurrentException(e) + when gotoBasedExceptions: + inc nimInErrorMode else: if excHandler != nil: - if not excHandler.hasRaiseAction or excHandler.raiseAction(e): - pushCurrentException(e) - c_longjmp(excHandler.context, 1) - elif e[] of OutOfMemError: - showErrorMessage(e.name) - quitOrDebug() + pushCurrentException(e) + c_longjmp(excHandler.context, 1) else: - when hasSomeStackTrace: - var buf = newStringOfCap(2000) - if isNil(e.trace): rawWriteStackTrace(buf) - else: add(buf, e.trace) - add(buf, "Error: unhandled exception: ") - if not isNil(e.msg): add(buf, e.msg) - add(buf, " [") - add(buf, $e.name) - add(buf, "]\n") - showErrorMessage(buf) - else: - # ugly, but avoids heap allocations :-) - template xadd(buf, s, slen: expr) = - if L + slen < high(buf): - copyMem(addr(buf[L]), cstring(s), slen) - inc L, slen - template add(buf, s: expr) = - xadd(buf, s, s.len) - var buf: array [0..2000, char] - var L = 0 - add(buf, "Error: unhandled exception: ") - if not isNil(e.msg): add(buf, e.msg) - add(buf, " [") - xadd(buf, e.name, c_strlen(e.name)) - add(buf, "]\n") - showErrorMessage(buf) - quitOrDebug() - -proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} = - e.name = ename + reportUnhandledError(e) + rawQuit(1) + +proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, + line: int) {.compilerRtl, nodestroy.} = + if e.name.isNil: e.name = ename when hasSomeStackTrace: - e.trace = "" - rawWriteStackTrace(e.trace) + when defined(nimStackTraceOverride): + if e.trace.len == 0: + rawWriteStackTrace(e.trace) + else: + e.trace.add reraisedFrom(reraisedFromBegin) + auxWriteStackTraceWithOverride(e.trace) + e.trace.add reraisedFrom(reraisedFromEnd) + elif NimStackTrace: + if e.trace.len == 0: + rawWriteStackTrace(e.trace) + elif framePtr != nil: + e.trace.add reraisedFrom(reraisedFromBegin) + auxWriteStackTrace(framePtr, e.trace) + e.trace.add reraisedFrom(reraisedFromEnd) + else: + if procname != nil and filename != nil: + e.trace.add StackTraceEntry(procname: procname, filename: filename, line: line) raiseExceptionAux(e) +proc raiseException(e: sink(ref Exception), ename: cstring) {.compilerRtl.} = + raiseExceptionEx(e, ename, nil, nil, 0) + proc reraiseException() {.compilerRtl.} = if currException == nil: - sysFatal(ReraiseError, "no exception to reraise") + sysFatal(ReraiseDefect, "no exception to reraise") else: - raiseExceptionAux(currException) + when gotoBasedExceptions: + inc nimInErrorMode + else: + raiseExceptionAux(currException) + +proc threadTrouble() = + # also forward declared, it is 'raises: []' hence the try-except. + try: + if currException != nil: reportUnhandledError(currException) + except: + discard + rawQuit 1 proc writeStackTrace() = when hasSomeStackTrace: var s = "" rawWriteStackTrace(s) - showErrorMessage(s) else: - showErrorMessage("No stack traceback available\n") + let s = noStacktraceAvailable + cast[proc (s: string) {.noSideEffect, tags: [], nimcall, raises: [].}](showErrorMessage2)(s) proc getStackTrace(): string = when hasSomeStackTrace: result = "" rawWriteStackTrace(result) else: - result = "No stack traceback available\n" + result = noStacktraceAvailable proc getStackTrace(e: ref Exception): string = - if not isNil(e) and not isNil(e.trace): - result = e.trace + if not isNil(e): + result = $e.trace else: result = "" -when defined(nimRequiresNimFrame): - proc stackOverflow() {.noinline.} = - writeStackTrace() - showErrorMessage("Stack overflow\n") - quitOrDebug() - - proc nimFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = - s.calldepth = if framePtr == nil: 0 else: framePtr.calldepth+1 - s.prev = framePtr - framePtr = s - if s.calldepth == 2000: stackOverflow() -else: - proc pushFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = - # XXX only for backwards compatibility - s.prev = framePtr - framePtr = s +proc getStackTraceEntries*(e: ref Exception): lent seq[StackTraceEntry] = + ## Returns the attached stack trace to the exception `e` as + ## a `seq`. This is not yet available for the JS backend. + e.trace -when defined(endb): - var - dbgAborting: bool # whether the debugger wants to abort +proc getStackTraceEntries*(): seq[StackTraceEntry] = + ## Returns the stack trace entries for the current stack trace. + ## This is not yet available for the JS backend. + when hasSomeStackTrace: + rawWriteStackTrace(result) + +const nimCallDepthLimit {.intdefine.} = 2000 + +proc callDepthLimitReached() {.noinline.} = + writeStackTrace() + let msg = "Error: call depth limit reached in a debug build (" & + $nimCallDepthLimit & " function calls). You can change it with " & + "-d:nimCallDepthLimit=<int> but really try to avoid deep " & + "recursions instead.\n" + showErrorMessage2(msg) + rawQuit(1) + +proc nimFrame(s: PFrame) {.compilerRtl, inl, raises: [].} = + if framePtr == nil: + s.calldepth = 0 + when NimStackTraceMsgs: s.frameMsgLen = 0 + else: + s.calldepth = framePtr.calldepth+1 + when NimStackTraceMsgs: s.frameMsgLen = framePtr.frameMsgLen + s.prev = framePtr + framePtr = s + if s.calldepth == nimCallDepthLimit: callDepthLimitReached() + +when defined(cpp) and appType != "lib" and not gotoBasedExceptions and + not defined(js) and not defined(nimscript) and + hostOS != "standalone" and hostOS != "any" and not defined(noCppExceptions) and + not quirkyExceptions: -when not defined(noSignalHandler): - proc signalHandler(sig: cint) {.exportc: "signalHandler", noconv.} = - template processSignal(s, action: expr) {.immediate, dirty.} = + type + StdException {.importcpp: "std::exception", header: "<exception>".} = object + + proc what(ex: StdException): cstring {.importcpp: "((char *)#.what())", nodecl.} + + proc setTerminate(handler: proc() {.noconv.}) + {.importc: "std::set_terminate", header: "<exception>".} + + setTerminate proc() {.noconv.} = + # Remove ourself as a handler, reinstalling the default handler. + setTerminate(nil) + + var msg = "Unknown error in unexpected exception handler" + try: + {.emit: "#if !defined(_MSC_VER) || (_MSC_VER >= 1923)".} + raise + {.emit: "#endif".} + except Exception: + msg = currException.getStackTrace() & "Error: unhandled exception: " & + currException.msg & " [" & $currException.name & "]" + except StdException as e: + msg = "Error: unhandled cpp exception: " & $e.what() + except: + msg = "Error: unhandled unknown cpp exception" + + {.emit: "#if defined(_MSC_VER) && (_MSC_VER < 1923)".} + msg = "Error: unhandled unknown cpp exception" + {.emit: "#endif".} + + when defined(genode): + # stderr not available by default, use the LOG session + echo msg + else: + writeToStdErr msg & "\n" + + rawQuit 1 + +when not defined(noSignalHandler) and not defined(useNimRtl): + type Sighandler = proc (a: cint) {.noconv, benign.} + # xxx factor with ansi_c.CSighandlerT, posix.Sighandler + + proc signalHandler(sign: cint) {.exportc: "signalHandler", noconv.} = + template processSignal(s, action: untyped) {.dirty.} = if s == SIGINT: action("SIGINT: Interrupted by Ctrl-C.\n") elif s == SIGSEGV: action("SIGSEGV: Illegal storage access. (Attempt to read from nil?)\n") elif s == SIGABRT: - when defined(endb): - if dbgAborting: return # the debugger wants to abort action("SIGABRT: Abnormal termination.\n") elif s == SIGFPE: action("SIGFPE: Arithmetic error.\n") elif s == SIGILL: action("SIGILL: Illegal operation.\n") - elif s == SIGBUS: + elif (when declared(SIGBUS): s == SIGBUS else: false): action("SIGBUS: Illegal storage access. (Attempt to read from nil?)\n") else: block platformSpecificSignal: @@ -338,35 +644,62 @@ when not defined(noSignalHandler): action("unknown signal\n") # print stack trace and quit + when defined(memtracker): + logPendingOps() when hasSomeStackTrace: - GC_disable() + when not usesDestructors: GC_disable() var buf = newStringOfCap(2000) rawWriteStackTrace(buf) - processSignal(sig, buf.add) # nice hu? currying a la Nim :-) - showErrorMessage(buf) - GC_enable() + processSignal(sign, buf.add) # nice hu? currying a la Nim :-) + showErrorMessage2(buf) + when not usesDestructors: GC_enable() else: var msg: cstring - template asgn(y: expr) = msg = y - processSignal(sig, asgn) - showErrorMessage(msg) - when defined(endb): dbgAborting = true - quit(1) # always quit when SIGABRT + template asgn(y) = + msg = y + processSignal(sign, asgn) + # xxx use string for msg instead of cstring, and here use showErrorMessage2(msg) + # unless there's a good reason to use cstring in signal handler to avoid + # using gc? + showErrorMessage(msg, msg.len) + + when defined(posix): + # reset the signal handler to OS default + c_signal(sign, SIG_DFL) + + # re-raise the signal, which will arrive once this handler exit. + # this lets the OS perform actions like core dumping and will + # also return the correct exit code to the shell. + discard c_raise(sign) + else: + rawQuit(1) + + var SIG_IGN {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler proc registerSignalHandler() = + # xxx `signal` is deprecated and has many caveats, we should use `sigaction` instead, e.g. + # https://stackoverflow.com/questions/231912/what-is-the-difference-between-sigaction-and-signal c_signal(SIGINT, signalHandler) c_signal(SIGSEGV, signalHandler) c_signal(SIGABRT, signalHandler) c_signal(SIGFPE, signalHandler) c_signal(SIGILL, signalHandler) - c_signal(SIGBUS, signalHandler) + when declared(SIGBUS): + c_signal(SIGBUS, signalHandler) when declared(SIGPIPE): - c_signal(SIGPIPE, signalHandler) + when defined(nimLegacySigpipeHandler): + c_signal(SIGPIPE, signalHandler) + else: + c_signal(SIGPIPE, SIG_IGN) registerSignalHandler() # call it in initialization section -proc setControlCHook(hook: proc () {.noconv.} not nil) = +proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: - type SignalHandler = proc (sig: cint) {.noconv, benign.} - {.deprecated: [TSignalHandler: SignalHandler].} - c_signal(SIGINT, cast[SignalHandler](hook)) + when declared(Sighandler): + c_signal(SIGINT, cast[Sighandler](hook)) + +when not defined(noSignalHandler) and not defined(useNimRtl): + proc unsetControlCHook() = + # proc to unset a hook set by setControlCHook + c_signal(SIGINT, signalHandler) diff --git a/lib/system/fatal.nim b/lib/system/fatal.nim new file mode 100644 index 000000000..25c05e52d --- /dev/null +++ b/lib/system/fatal.nim @@ -0,0 +1,58 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +{.push profiler: off.} + +const + gotoBasedExceptions = compileOption("exceptions", "goto") + quirkyExceptions = compileOption("exceptions", "quirky") + +when hostOS == "standalone": + include "$projectpath/panicoverride" + + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline.} = + panic(message) + + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline.} = + rawoutput(message) + panic(arg) + +elif quirkyExceptions and not defined(nimscript): + import ansi_c + + func name(t: typedesc): string {.magic: "TypeTrait".} + + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline, noreturn.} = + when nimvm: + # TODO when doAssertRaises works in CT, add a test for it + raise (ref exceptn)(msg: message & arg) + else: + {.noSideEffect.}: + writeStackTrace() + var buf = newStringOfCap(200) + add(buf, "Error: unhandled exception: ") + add(buf, message) + add(buf, arg) + add(buf, " [") + add(buf, name exceptn) + add(buf, "]\n") + cstderr.rawWrite buf + rawQuit 1 + + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline, noreturn.} = + sysFatal(exceptn, message, "") + +else: + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline, noreturn.} = + raise (ref exceptn)(msg: message) + + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline, noreturn.} = + raise (ref exceptn)(msg: message & arg) + +{.pop.} diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim new file mode 100644 index 000000000..70dd857d5 --- /dev/null +++ b/lib/system/formatfloat.nim @@ -0,0 +1,6 @@ +when not defined(nimPreviewSlimSystem): + import std/formatfloat + export formatfloat + {.deprecated: "use `std/formatfloat`".} +else: + {.error: "use `std/formatfloat`".} diff --git a/lib/system/gc.nim b/lib/system/gc.nim index c25cf4606..9289c7f55 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,36 +9,77 @@ # Garbage Collector # -# The basic algorithm is *Deferred Reference Counting* with cycle detection. -# This is achieved by combining a Deutsch-Bobrow garbage collector -# together with Christoper's partial mark-sweep garbage collector. -# -# Special care has been taken to avoid recursion as far as possible to avoid -# stack overflows when traversing deep datastructures. It is well-suited -# for soft real time applications (like games). - -when defined(nimCoroutines): - import arch +# Refcounting + Mark&Sweep. Complex algorithms avoided. +# Been there, done that, didn't work. + +#[ + +A *cell* is anything that is traced by the GC +(sequences, refs, strings, closures). + +The basic algorithm is *Deferrent Reference Counting* with cycle detection. +References on the stack are not counted for better performance and easier C +code generation. + +Each cell has a header consisting of a RC and a pointer to its type +descriptor. However the program does not know about these, so they are placed at +negative offsets. In the GC code the type `PCell` denotes a pointer +decremented by the right offset, so that the header can be accessed easily. It +is extremely important that `pointer` is not confused with a `PCell`. + +In Nim the compiler cannot always know if a reference +is stored on the stack or not. This is caused by var parameters. +Consider this example: + + ```Nim + proc setRef(r: var ref TNode) = + new(r) + + proc usage = + var + r: ref TNode + setRef(r) # here we should not update the reference counts, because + # r is on the stack + setRef(r.left) # here we should update the refcounts! + ``` + +We have to decide at runtime whether the reference is on the stack or not. +The generated code looks roughly like this: + + ```C + void setref(TNode** ref) { + unsureAsgnRef(ref, newObj(TNode_TI, sizeof(TNode))) + } + void usage(void) { + setRef(&r) + setRef(&r->left) + } + ``` + +Note that for systems with a continuous stack (which most systems have) +the check whether the ref is on the stack is very cheap (only two +comparisons). +]# {.push profiler:off.} const CycleIncrease = 2 # is a multiplicative increase - InitialCycleThreshold = 4*1024*1024 # X MB because cycle checking is slow - ZctThreshold = 500 # we collect garbage if the ZCT's size - # reaches this threshold - # this seems to be a good value + InitialCycleThreshold = when defined(nimCycleBreaker): high(int) + else: 4*1024*1024 # X MB because cycle checking is slow + InitialZctThreshold = 500 # we collect garbage if the ZCT's size + # reaches this threshold + # this seems to be a good value withRealTime = defined(useRealtimeGC) - useMarkForDebug = defined(gcGenerational) - useBackupGc = false # use a simple M&S GC to collect - # cycles instead of the complex - # algorithm when withRealTime and not declared(getTicks): include "system/timers" when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} +when hasThreadSupport: + import std/sharedlist + const rcIncrement = 0b1000 # so that lowest 3 bits are not touched rcBlack = 0b000 # cell is colored black; in use or free @@ -52,10 +93,10 @@ type WalkOp = enum waMarkGlobal, # part of the backup/debug mark&sweep waMarkPrecise, # part of the backup/debug mark&sweep - waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, - waCollectWhite #, waDebug + waZctDecRef, waPush + #, waDebug - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -68,54 +109,61 @@ type cycleTableSize: int # max entries in cycle table maxPause: int64 # max measured GC pause in nanoseconds - GcStack {.final.} = object - prev: ptr GcStack - next: ptr GcStack - starts: pointer - pos: pointer - maxStackSize: int + GcStack {.final, pure.} = object + when nimCoroutines: + prev: ptr GcStack + next: ptr GcStack + maxStackSize: int # Used to track statistics because we can not use + # GcStat.maxStackSize when multiple stacks exist. + bottom: pointer + + when withRealTime or nimCoroutines: + pos: pointer # Used with `withRealTime` only for code clarity, see GC_Step(). + when withRealTime: + bottomSaved: pointer GcHeap {.final, pure.} = object # this contains the zero count and - # non-zero count table - stack: ptr GcStack - stackBottom: pointer + # non-zero count table + stack: GcStack + when nimCoroutines: + activeStack: ptr GcStack # current executing coroutine stack. cycleThreshold: int + zctThreshold: int when useCellIds: idGenerator: int zct: CellSeq # the zero count table decStack: CellSeq # cells in the stack that are to decref again - cycleRoots: CellSet tempStack: CellSeq # temporary stack for recursion elimination recGcLock: int # prevent recursion via finalizers; no thread lock when withRealTime: maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 region: MemRegion # garbage collected region stat: GcStat - when useMarkForDebug or useBackupGc: - marked: CellSet -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, - TGcStat: GcStat].} + marked: CellSet + additionalRoots: CellSeq # dummy roots for GC_ref/unref + when hasThreadSupport: + toDispose: SharedList[pointer] + gcThreadId: int + var gch {.rtlThreadVar.}: GcHeap when not defined(useNimRtl): instantiateForRegion(gch.region) -template acquire(gch: GcHeap) = - when hasThreadSupport and hasSharedHeap: - acquireSys(HeapLock) - -template release(gch: GcHeap) = - when hasThreadSupport and hasSharedHeap: - releaseSys(HeapLock) - template gcAssert(cond: bool, msg: string) = when defined(useGcAssert): if not cond: - echo "[GCASSERT] ", msg + cstderr.rawWrite "[GCASSERT] " + cstderr.rawWrite msg + when defined(logGC): + cstderr.rawWrite "[GCASSERT] statistics:\L" + cstderr.rawWrite GC_getStatistics() GC_disable() writeStackTrace() - quit 1 + #var x: ptr int + #echo x[] + rawQuit 1 proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = if (c.refcount and ZctFlag) == 0: @@ -124,133 +172,116 @@ proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) - -proc canBeCycleRoot(c: PCell): bool {.inline.} = - result = ntfAcyclic notin c.typ.flags + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = - result = int(usrToCell(p).refcount) shr rcShift + result = usrToCell(p).refcount shr rcShift # this that has to equals zero, otherwise we have to round up UnitsPerPage: when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} -template color(c): expr = c.refCount and colorMask +template color(c): untyped = c.refCount and colorMask template setColor(c, col) = when col == rcBlack: c.refcount = c.refcount and not colorMask else: c.refcount = c.refcount and not colorMask or col -proc writeCell(msg: cstring, c: PCell) = - var kind = -1 - if c.typ != nil: kind = ord(c.typ.kind) - when leakDetector: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", - msg, c, kind, c.refcount shr rcShift, c.filename, c.line) - else: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", - msg, c, kind, c.refcount shr rcShift, c.color) +when defined(logGC): + proc writeCell(msg: cstring, c: PCell) = + var kind = -1 + var typName: cstring = "nil" + if c.typ != nil: + kind = ord(c.typ.kind) + when defined(nimTypeNames): + if not c.typ.name.isNil: + typName = c.typ.name + + when leakDetector: + c_printf("[GC] %s: %p %d %s rc=%ld from %s(%ld)\n", + msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line) + else: + c_printf("[GC] %s: %p %d %s rc=%ld; thread=%ld\n", + msg, c, kind, typName, c.refcount shr rcShift, gch.gcThreadId) + +template logCell(msg: cstring, c: PCell) = + when defined(logGC): + writeCell(msg, c) -template gcTrace(cell, state: expr): stmt {.immediate.} = +template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap) {.benign, raises: [].} +proc isOnStack(p: pointer): bool {.noinline, benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes -when hasThreadSupport and hasSharedHeap: - template `--`(x: expr): expr = atomicDec(x, rcIncrement) <% rcIncrement - template `++`(x: expr): stmt = discard atomicInc(x, rcIncrement) -else: - template `--`(x: expr): expr = - dec(x, rcIncrement) - x <% rcIncrement - template `++`(x: expr): stmt = inc(x, rcIncrement) - -proc prepareDealloc(cell: PCell) = - when useMarkForDebug: - gcAssert(cell notin gch.marked, "Cell still alive!") - if cell.typ.finalizer != nil: - # the finalizer could invoke something that - # allocates memory; this could trigger a garbage - # collection. Since we are already collecting we - # prevend recursive entering here by a lock. - # XXX: we should set the cell's children to nil! - inc(gch.recGcLock) - (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) - dec(gch.recGcLock) +proc incRef(c: PCell) {.inline.} = + gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") + c.refcount = c.refcount +% rcIncrement + # and not colorMask + logCell("incRef", c) -proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = - # we MUST access gch as a global here, because this crosses DLL boundaries! - when hasThreadSupport and hasSharedHeap: - acquireSys(HeapLock) - when cycleGC: - if c.color != rcPurple: - c.setColor(rcPurple) - incl(gch.cycleRoots, c) - when hasThreadSupport and hasSharedHeap: - releaseSys(HeapLock) +proc nimGCref(p: pointer) {.compilerproc.} = + # we keep it from being collected by pretending it's not even allocated: + let c = usrToCell(p) + add(gch.additionalRoots, c) + incRef(c) proc rtlAddZCT(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! - when hasThreadSupport and hasSharedHeap: - acquireSys(HeapLock) addZCT(gch.zct, c) - when hasThreadSupport and hasSharedHeap: - releaseSys(HeapLock) proc decRef(c: PCell) {.inline.} = gcAssert(isAllocatedPtr(gch.region, c), "decRef: interiorPtr") gcAssert(c.refcount >=% rcIncrement, "decRef") - if --c.refcount: + c.refcount = c.refcount -% rcIncrement + if c.refcount <% rcIncrement: rtlAddZCT(c) - elif canbeCycleRoot(c): - # unfortunately this is necessary here too, because a cycle might just - # have been broken up and we could recycle it. - rtlAddCycleRoot(c) - #writeCell("decRef", c) + logCell("decRef", c) + +proc nimGCunref(p: pointer) {.compilerproc.} = + let cell = usrToCell(p) + var L = gch.additionalRoots.len-1 + var i = L + let d = gch.additionalRoots.d + while i >= 0: + if d[i] == cell: + d[i] = d[L] + dec gch.additionalRoots.len + break + dec(i) + decRef(usrToCell(p)) -proc incRef(c: PCell) {.inline.} = - gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") - c.refcount = c.refcount +% rcIncrement - # and not colorMask - #writeCell("incRef", c) - if canbeCycleRoot(c): - rtlAddCycleRoot(c) - -proc nimGCref(p: pointer) {.compilerProc, inline.} = incRef(usrToCell(p)) -proc nimGCunref(p: pointer) {.compilerProc, inline.} = decRef(usrToCell(p)) +include gc_common -proc GC_addCycleRoot*[T](p: ref T) {.inline.} = - ## adds 'p' to the cycle candidate set for the cycle collector. It is - ## necessary if you used the 'acyclic' pragma for optimization - ## purposes and need to break cycles manually. - rtlAddCycleRoot(usrToCell(cast[pointer](p))) +template beforeDealloc(gch: var GcHeap; c: PCell; msg: typed) = + when false: + for i in 0..gch.decStack.len-1: + if gch.decStack.d[i] == c: + sysAssert(false, msg) -proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} = +proc nimGCunrefNoCycle(p: pointer) {.compilerproc, inline.} = sysAssert(allocInv(gch.region), "begin nimGCunrefNoCycle") - var c = usrToCell(p) - gcAssert(isAllocatedPtr(gch.region, c), "nimGCunrefNoCycle: isAllocatedPtr") - if --c.refcount: - rtlAddZCT(c) - sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 2") + decRef(usrToCell(p)) sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 5") -proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} = +proc nimGCunrefRC1(p: pointer) {.compilerproc, inline.} = + decRef(usrToCell(p)) + +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = # the code generator calls this proc! gcAssert(not isOnStack(dest), "asgnRef") # BUGFIX: first incRef then decRef! @@ -258,21 +289,12 @@ proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} = if dest[] != nil: decRef(usrToCell(dest[])) dest[] = src -proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerProc, inline.} = - # the code generator calls this proc if it is known at compile time that no - # cycle is possible. - if src != nil: - var c = usrToCell(src) - ++c.refcount - if dest[] != nil: - var c = usrToCell(dest[]) - if --c.refcount: - rtlAddZCT(c) - dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, + deprecated: "old compiler compat".} = asgnRef(dest, src) -proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} = +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc.} = # unsureAsgnRef updates the reference counters only if dest is not on the - # stack. It is used by the code generator if it cannot decide wether a + # stack. It is used by the code generator if it cannot decide whether a # reference is in the stack or not (this can happen for var parameters). if not isOnStack(dest): if src != nil: incRef(usrToCell(src)) @@ -291,6 +313,7 @@ proc initGC() = when traceGC: for i in low(CellState)..high(CellState): init(states[i]) gch.cycleThreshold = InitialCycleThreshold + gch.zctThreshold = InitialZctThreshold gch.stat.stackScans = 0 gch.stat.cycleCollections = 0 gch.stat.maxThreshold = 0 @@ -300,33 +323,22 @@ proc initGC() = # init the rt init(gch.zct) init(gch.tempStack) - init(gch.cycleRoots) init(gch.decStack) - when useMarkForDebug or useBackupGc: - init(gch.marked) - -when useMarkForDebug or useBackupGc: - type - GlobalMarkerProc = proc () {.nimcall, benign.} - {.deprecated: [TGlobalMarkerProc: GlobalMarkerProc].} - var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - - proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 + init(gch.marked) + init(gch.additionalRoots) + when hasThreadSupport: + init(gch.toDispose) + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc cellsetReset(s: var CellSet) = deinit(s) init(s) +{.push stacktrace:off.} + proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -346,7 +358,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -360,10 +372,10 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = else: discard proc forAllChildren(cell: PCell, op: WalkOp) = - gcAssert(cell != nil, "forAllChildren: 1") - gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") - gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 4" + gcAssert(cell != nil, "forAllChildren: cell is nil") + gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: pointer not part of the heap") + gcAssert(cell.typ != nil, "forAllChildren: cell.typ is nil") + gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: unknown GC'ed type" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) @@ -372,12 +384,11 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) + forAllChildrenAux(cast[pointer](d +% align(GenericSeqSize, cell.typ.base.align) +% i *% cell.typ.base.size), cell.typ.base, op) else: discard proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = @@ -395,7 +406,7 @@ proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = var d = gch.zct.d when true: # loop unrolled for performance: - template replaceZctEntry(i: expr) = + template replaceZctEntry(i: untyped) = c = d[i] if c.refcount >=% rcIncrement: c.refcount = c.refcount and not ZctFlag @@ -431,92 +442,100 @@ proc gcInvariant*() = markForDebug(gch) {.pop.} +template setFrameInfo(c: PCell) = + when leakDetector: + if framePtr != nil and framePtr.prev != nil: + c.filename = framePtr.prev.filename + c.line = framePtr.prev.line + else: + c.filename = nil + c.line = 0 + proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 + incTypeSize typ, size sysAssert(allocInv(gch.region), "rawNewObj begin") - acquire(gch) gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + #gcAssert typ.kind in {tyString, tySequence} or size >= typ.base.size, "size too small" + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + setFrameInfo(res) # refcount is zero, color is black, but mark it to be in the ZCT res.refcount = ZctFlag sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") # its refcount is zero, so add it to the ZCT: addNewObjToZCT(res, gch) - when logGC: writeCell("new cell", res) + logCell("new cell", res) + track("rawNewObj", res, size) gcTrace(res, csAllocated) - release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) sysAssert(allocInv(gch.region), "rawNewObj end") -{.pop.} +{.pop.} # .stackTrace off +{.pop.} # .profiler off proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(typ, size, gch) when defined(memProfiler): nimProfile(size) -proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = result = rawNewObj(typ, size, gch) zeroMem(result, size) when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) +{.pop.} -proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = # generates a new object and sets its reference counter to 1 + incTypeSize typ, size sysAssert(allocInv(gch.region), "newObjRC1 begin") - acquire(gch) gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) sysAssert(allocInv(gch.region), "newObjRC1 after collectCT") var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + setFrameInfo(res) res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - when logGC: writeCell("new cell", res) + logCell("new cell", res) + track("newObjRC1", res, size) gcTrace(res, csAllocated) - release(gch) when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) zeroMem(result, size) sysAssert(allocInv(gch.region), "newObjRC1 end") when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size result = newObjRC1(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) +{.pop.} proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - acquire(gch) collectCT(gch) var ol = usrToCell(old) sysAssert(ol.typ != nil, "growObj: 1") @@ -524,51 +543,31 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = sysAssert(allocInv(gch.region), "growObj begin") var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size + var elemSize,elemAlign = 1 + if ol.typ.kind != tyString: + elemSize = ol.typ.base.size + elemAlign = ol.typ.base.align + incTypeSize ol.typ, newsize - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len * elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") # This can be wrong for intermediate temps that are nevertheless on the # heap because of lambda lifting: #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") - when logGC: - writeCell("growObj old cell", ol) - writeCell("growObj new cell", res) + logCell("growObj old cell", ol) + logCell("growObj new cell", res) gcTrace(ol, csZctFreed) gcTrace(res, csAllocated) - when reallyDealloc: - sysAssert(allocInv(gch.region), "growObj before dealloc") - if ol.refcount shr rcShift <=% 1: - # free immediately to save space: - if (ol.refcount and ZctFlag) != 0: - var j = gch.zct.len-1 - var d = gch.zct.d - while j >= 0: - if d[j] == ol: - d[j] = res - break - dec(j) - if canbeCycleRoot(ol): excl(gch.cycleRoots, ol) - rawDealloc(gch.region, ol) - else: - # we split the old refcount in 2 parts. XXX This is still not entirely - # correct if the pointer that receives growObj's result is on the stack. - # A better fix would be to emit the location specific write barrier for - # 'growObj', but this is lots of more work and who knows what new problems - # this would create. - res.refcount = rcIncrement - decRef(ol) - else: - sysAssert(ol.typ != nil, "growObj: 5") - zeroMem(ol, sizeof(Cell)) - release(gch) + track("growObj old", ol, 0) + track("growObj new", res, newsize) + # since we steal the old seq's contents, we set the old length to 0. + cast[PGenericSeq](old).len = 0 when useCellIds: inc gch.idGenerator - res.id = gch.idGenerator + res.id = gch.idGenerator * 1000_000 + gch.gcThreadId result = cellToUsr(res) sysAssert(allocInv(gch.region), "growObj end") when defined(memProfiler): nimProfile(newsize-oldsize) @@ -576,95 +575,48 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = proc growObj(old: pointer, newsize: int): pointer {.rtl.} = result = growObj(old, newsize, gch) -{.push profiler:off.} +{.push profiler:off, stackTrace:off.} # ---------------- cycle collector ------------------------------------------- proc freeCyclicCell(gch: var GcHeap, c: PCell) = prepareDealloc(c) gcTrace(c, csCycFreed) - when logGC: writeCell("cycle collector dealloc cell", c) + track("cycle collector dealloc cell", c, 0) + logCell("cycle collector dealloc cell", c) when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") + beforeDealloc(gch, c, "freeCyclicCell: stack trash") rawDealloc(gch.region, c) else: gcAssert(c.typ != nil, "freeCyclicCell") zeroMem(c, sizeof(Cell)) -proc markGray(s: PCell) = - if s.color != rcGray: - setColor(s, rcGray) - forAllChildren(s, waMarkGray) - -proc scanBlack(s: PCell) = - s.setColor(rcBlack) - forAllChildren(s, waScanBlack) - -proc scan(s: PCell) = - if s.color == rcGray: - if s.refcount >=% rcIncrement: - scanBlack(s) - else: - s.setColor(rcWhite) - forAllChildren(s, waScan) - -proc collectWhite(s: PCell) = - # This is a hacky way to deal with the following problem (bug #1796) - # Consider this content in cycleRoots: - # x -> a; y -> a where 'a' is an acyclic object so not included in - # cycleRoots itself. Then 'collectWhite' used to free 'a' twice. The - # 'isAllocatedPtr' check prevents this. This also means we do not need - # to query 's notin gch.cycleRoots' at all. - if isAllocatedPtr(gch.region, s) and s.color == rcWhite: - s.setColor(rcBlack) - forAllChildren(s, waCollectWhite) - freeCyclicCell(gch, s) - -proc markRoots(gch: var GcHeap) = - var tabSize = 0 - for s in elements(gch.cycleRoots): - #writeCell("markRoot", s) - inc tabSize - if s.color == rcPurple and s.refcount >=% rcIncrement: - markGray(s) - else: - excl(gch.cycleRoots, s) - # (s.color == rcBlack and rc == 0) as 1 condition: - if s.refcount == 0: - freeCyclicCell(gch, s) - gch.stat.cycleTableSize = max(gch.stat.cycleTableSize, tabSize) - -when useBackupGc: - proc sweep(gch: var GcHeap) = - for x in allObjects(gch.region): - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - if c notin gch.marked: freeCyclicCell(gch, c) - -when useMarkForDebug or useBackupGc: - proc markS(gch: var GcHeap, c: PCell) = - incl(gch.marked, c) - gcAssert gch.tempStack.len == 0, "stack not empty!" - forAllChildren(c, waMarkPrecise) - while gch.tempStack.len > 0: - dec gch.tempStack.len - var d = gch.tempStack.d[gch.tempStack.len] - if not containsOrIncl(gch.marked, d): - forAllChildren(d, waMarkPrecise) - - proc markGlobals(gch: var GcHeap) = - for i in 0 .. < globalMarkersLen: globalMarkers[i]() - - proc stackMarkS(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[TAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - markS(gch, objStart) +proc sweep(gch: var GcHeap) = + for x in allObjects(gch.region): + if isCell(x): + # cast to PCell is correct here: + var c = cast[PCell](x) + if c notin gch.marked: freeCyclicCell(gch, c) + +proc markS(gch: var GcHeap, c: PCell) = + gcAssert isAllocatedPtr(gch.region, c), "markS: foreign heap root detected A!" + incl(gch.marked, c) + gcAssert gch.tempStack.len == 0, "stack not empty!" + forAllChildren(c, waMarkPrecise) + while gch.tempStack.len > 0: + dec gch.tempStack.len + var d = gch.tempStack.d[gch.tempStack.len] + gcAssert isAllocatedPtr(gch.region, d), "markS: foreign heap root detected B!" + if not containsOrIncl(gch.marked, d): + forAllChildren(d, waMarkPrecise) + +proc markGlobals(gch: var GcHeap) {.raises: [].} = + if gch.gcThreadId == 0: + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() + let d = gch.additionalRoots.d + for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i]) when logGC: var @@ -672,11 +624,11 @@ when logGC: cycleCheckALen = 0 proc alreadySeen(c: PCell): bool = - for i in 0 .. <cycleCheckALen: + for i in 0 .. cycleCheckALen-1: if cycleCheckA[i] == c: return true if cycleCheckALen == len(cycleCheckA): gcAssert(false, "cycle detection overflow") - quit 1 + rawQuit 1 cycleCheckA[cycleCheckALen] = c inc cycleCheckALen @@ -686,7 +638,7 @@ when logGC: else: writeCell("cell {", s) forAllChildren(s, waDebug) - c_fprintf(c_stdout, "}\n") + c_printf("}\n") proc doOperation(p: pointer, op: WalkOp) = if p == nil: return @@ -697,132 +649,69 @@ proc doOperation(p: pointer, op: WalkOp) = case op of waZctDecRef: #if not isAllocatedPtr(gch.region, c): - # c_fprintf(c_stdout, "[GC] decref bug: %p", c) + # c_printf("[GC] decref bug: %p", c) gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") gcAssert(c.refcount >=% rcIncrement, "doOperation 2") - #c.refcount = c.refcount -% rcIncrement - when logGC: writeCell("decref (from doOperation)", c) + logCell("decref (from doOperation)", c) + track("waZctDecref", p, 0) decRef(c) - #if c.refcount <% rcIncrement: addZCT(gch.zct, c) of waPush: add(gch.tempStack, c) - of waCycleDecRef: - gcAssert(c.refcount >=% rcIncrement, "doOperation 3") - c.refcount = c.refcount -% rcIncrement - of waMarkGray: - gcAssert(c.refcount >=% rcIncrement, "waMarkGray") - c.refcount = c.refcount -% rcIncrement - markGray(c) - of waScan: scan(c) - of waScanBlack: - c.refcount = c.refcount +% rcIncrement - if c.color != rcBlack: - scanBlack(c) - of waCollectWhite: collectWhite(c) of waMarkGlobal: - when useMarkForDebug or useBackupGc: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - markS(gch, c) - else: - markS(gch, c) + markS(gch, c) of waMarkPrecise: - when useMarkForDebug or useBackupGc: - add(gch.tempStack, c) + add(gch.tempStack, c) #of waDebug: debugGraph(c) proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, WalkOp(op)) -proc collectZCT(gch: var GcHeap): bool {.benign.} +proc collectZCT(gch: var GcHeap): bool {.benign, raises: [].} -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) {.noinline, cdecl, - benign.} - -proc collectRoots(gch: var GcHeap) = - for s in elements(gch.cycleRoots): - collectWhite(s) - -proc collectCycles(gch: var GcHeap) = +proc collectCycles(gch: var GcHeap) {.raises: [].} = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) - when useBackupGc: - cellsetReset(gch.marked) - markStackAndRegistersForSweep(gch) - markGlobals(gch) - sweep(gch) - else: - markRoots(gch) - # scanRoots: - for s in elements(gch.cycleRoots): scan(s) - collectRoots(gch) - - cellsetReset(gch.cycleRoots) - # alive cycles need to be kept in 'cycleRoots' if they are referenced - # from the stack; otherwise the write barrier will add the cycle root again - # anyway: - when false: - var d = gch.decStack.d - var cycleRootsLen = 0 - for i in 0..gch.decStack.len-1: - var c = d[i] - gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots" - gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell" - if canBeCycleRoot(c): - #if c notin gch.cycleRoots: - inc cycleRootsLen - incl(gch.cycleRoots, c) - gcAssert c.typ != nil, "addBackStackRoots 2" - if cycleRootsLen != 0: - cfprintf(cstdout, "cycle roots: %ld\n", cycleRootsLen) + cellsetReset(gch.marked) + var d = gch.decStack.d + for i in 0..gch.decStack.len-1: + sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" + markS(gch, d[i]) + markGlobals(gch) + sweep(gch) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: # mark the cell: - objStart.refcount = objStart.refcount +% rcIncrement + incRef(objStart) add(gch.decStack, objStart) when false: + let cell = usrToCell(p) if isAllocatedPtr(gch.region, cell): sysAssert false, "allocated pointer but not interior?" # mark the cell: - cell.refcount = cell.refcount +% rcIncrement + incRef(cell) add(gch.decStack, cell) sysAssert(allocInv(gch.region), "gcMark end") -proc markThreadStacks(gch: var GcHeap) = - when hasThreadSupport and hasSharedHeap: - {.error: "not fully implemented".} - var it = threadList - while it != nil: - # mark registers: - for i in 0 .. high(it.registers): gcMark(gch, it.registers[i]) - var sp = cast[TAddress](it.stackBottom) - var max = cast[TAddress](it.stackTop) - # XXX stack direction? - # XXX unroll this loop: - while sp <=% max: - gcMark(gch, cast[ppointer](sp)[]) - sp = sp +% sizeof(pointer) - it = it.next - -include gc_common - -proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = +#[ + This method is conditionally marked with an attribute so that it gets ignored by the LLVM ASAN + (Address SANitizer) intrumentation as it will raise false errors due to the implementation of + garbage collection that is used by Nim. For more information, please see the documentation of + `CLANG_NO_SANITIZE_ADDRESS` in `lib/nimbase.h`. + ]# +proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl, + codegenDecl: "CLANG_NO_SANITIZE_ADDRESS N_LIB_PRIVATE $# $#$#".} = forEachStackSlot(gch, gcMark) -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) = - forEachStackSlot(gch, stackMarkS) - proc collectZCT(gch: var GcHeap): bool = # Note: Freeing may add child objects to the ZCT! So essentially we do # deep freeing, which is bad for incremental operation. In order to @@ -852,9 +741,8 @@ proc collectZCT(gch: var GcHeap): bool = # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - when cycleGC: - if canbeCycleRoot(c): excl(gch.cycleRoots, c) - when logGC: writeCell("zct dealloc cell", c) + logCell("zct dealloc cell", c) + track("zct dealloc cell", c, 0) gcTrace(c, csZctFreed) # We are about to free the object, call the finalizer BEFORE its # children are deleted as well, because otherwise the finalizer may @@ -863,6 +751,7 @@ proc collectZCT(gch: var GcHeap): bool = forAllChildren(c, waZctDecRef) when reallyDealloc: sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") + beforeDealloc(gch, c, "collectZCT: stack trash") rawDealloc(gch.region, c) else: sysAssert(c.typ != nil, "collectZCT 2") @@ -884,24 +773,21 @@ proc unmarkStackAndRegisters(gch: var GcHeap) = for i in 0..gch.decStack.len-1: sysAssert isAllocatedPtr(gch.region, d[i]), "unmarkStackAndRegisters" decRef(d[i]) - #var c = d[i] - # XXX no need for an atomic dec here: - #if --c.refcount: - # addZCT(gch.zct, c) - #sysAssert c.typ != nil, "unmarkStackAndRegisters 2" gch.decStack.len = 0 -proc collectCTBody(gch: var GcHeap) = +proc collectCTBody(gch: var GcHeap) {.raises: [].} = when withRealTime: let t0 = getticks() sysAssert(allocInv(gch.region), "collectCT: begin") - when not defined(nimCoroutines): + when nimCoroutines: + for stack in gch.stack.items(): + gch.stat.maxStackSize = max(gch.stat.maxStackSize, stack.stackSize()) + else: gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) sysAssert(gch.decStack.len == 0, "collectCT") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) - markThreadStacks(gch) gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) inc(gch.stat.stackScans) if collectZCT(gch): @@ -921,34 +807,24 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxPause = max(gch.stat.maxPause, duration) when defined(reportMissedDeadlines): if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) - -when useMarkForDebug or useBackupGc: - proc markForDebug(gch: var GcHeap) = - markStackAndRegistersForSweep(gch) - markGlobals(gch) - -when defined(nimCoroutines): - proc currentStackSizes(): int = - for stack in items(gch.stack): - result = result + stackSize(stack.starts, stack.pos) + c_printf("[GC] missed deadline: %ld\n", duration) proc collectCT(gch: var GcHeap) = - # stackMarkCosts prevents some pathological behaviour: Stack marking - # becomes more expensive with large stacks and large stacks mean that - # cells with RC=0 are more likely to be kept alive by the stack. - when defined(nimCoroutines): - let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold) - else: - let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold) - if (gch.zct.len >= stackMarkCosts or (cycleGC and + if (gch.zct.len >= gch.zctThreshold or (cycleGC and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and gch.recGcLock == 0: - when useMarkForDebug: + when false: prepareForInteriorPointerChecking(gch.region) cellsetReset(gch.marked) markForDebug(gch) collectCTBody(gch) + gch.zctThreshold = max(InitialZctThreshold, gch.zct.len * CycleIncrease) + +proc GC_collectZct*() = + ## Collect the ZCT (zero count table). Unstable, experimental API for + ## testing purposes. + ## DO NOT USE! + collectCTBody(gch) when withRealTime: proc toNano(x: int): Nanos {.inline.} = @@ -958,28 +834,42 @@ when withRealTime: gch.maxPause = MaxPauseInUs.toNano proc GC_step(gch: var GcHeap, us: int, strongAdvice: bool) = - acquire(gch) gch.maxPause = us.toNano - if (gch.zct.len >= ZctThreshold or (cycleGC and + if (gch.zct.len >= gch.zctThreshold or (cycleGC and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or strongAdvice: collectCTBody(gch) - release(gch) + gch.zctThreshold = max(InitialZctThreshold, gch.zct.len * CycleIncrease) + + proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = + if stackSize >= 0: + var stackTop {.volatile.}: pointer + gch.getActiveStack().pos = addr(stackTop) + + for stack in gch.stack.items(): + stack.bottomSaved = stack.bottom + when stackIncreases: + stack.bottom = cast[pointer]( + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) + else: + stack.bottom = cast[pointer]( + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) + + GC_step(gch, us, strongAdvice) - proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) + if stackSize >= 0: + for stack in gch.stack.items(): + stack.bottom = stack.bottomSaved when not defined(useNimRtl): proc GC_disable() = - when hasThreadSupport and hasSharedHeap: - discard atomicInc(gch.recGcLock, 1) - else: - inc(gch.recGcLock) + inc(gch.recGcLock) proc GC_enable() = - if gch.recGcLock > 0: - when hasThreadSupport and hasSharedHeap: - discard atomicDec(gch.recGcLock, 1) - else: - dec(gch.recGcLock) + when defined(nimDoesntTrackDefects): + if gch.recGcLock <= 0: + raise newException(AssertionDefect, + "API usage error: GC_enable called but GC is already enabled") + dec(gch.recGcLock) proc GC_setStrategy(strategy: GC_Strategy) = discard @@ -988,19 +878,16 @@ when not defined(useNimRtl): gch.cycleThreshold = InitialCycleThreshold proc GC_disableMarkAndSweep() = - gch.cycleThreshold = high(gch.cycleThreshold)-1 + gch.cycleThreshold = high(typeof(gch.cycleThreshold))-1 # set to the max value to suppress the cycle detector proc GC_fullCollect() = - acquire(gch) var oldThreshold = gch.cycleThreshold gch.cycleThreshold = 0 # forces cycle collection collectCT(gch) gch.cycleThreshold = oldThreshold - release(gch) proc GC_getStatistics(): string = - GC_disable() result = "[GC] total memory: " & $(getTotalMem()) & "\n" & "[GC] occupied memory: " & $(getOccupiedMem()) & "\n" & "[GC] stack scans: " & $gch.stat.stackScans & "\n" & @@ -1009,13 +896,16 @@ when not defined(useNimRtl): "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & "[GC] zct capacity: " & $gch.zct.cap & "\n" & "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) - when defined(nimCoroutines): - result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" + "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n" + when nimCoroutines: + result.add "[GC] number of stacks: " & $gch.stack.len & "\n" for stack in items(gch.stack): - result = result & "[GC] stack " & stack.starts.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" + result.add "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & cast[pointer](stack.maxStackSize).repr & "\n" else: - result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" - GC_enable() + # this caused memory leaks, see #10488 ; find a way without `repr` + # maybe using a local copy of strutils.toHex or snprintf + when defined(logGC): + result.add "[GC] stack bottom: " & gch.stack.bottom.repr + result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" -{.pop.} +{.pop.} # profiler: off, stackTrace: off diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim deleted file mode 100644 index e68a8586e..000000000 --- a/lib/system/gc2.nim +++ /dev/null @@ -1,911 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Garbage Collector -# -# The basic algorithm is *Deferred Reference Counting* with an incremental mark -# and sweep GC to free cycles. It is hard realtime in that if you play -# according to its rules, no deadline will ever be missed. - -# XXX Ensure by smart color masking that the object is not in the ZCT. - -when defined(nimCoroutines): - import arch - -{.push profiler:off.} - -const - CycleIncrease = 2 # is a multiplicative increase - InitialCycleThreshold = 4*1024*1024 # X MB because cycle checking is slow - ZctThreshold = 500 # we collect garbage if the ZCT's size - # reaches this threshold - # this seems to be a good value - withRealTime = defined(useRealtimeGC) - -when withRealTime and not declared(getTicks): - include "system/timers" -when defined(memProfiler): - proc nimProfile(requestedSize: int) {.benign.} - -type - ObjectSpaceIter = object - state: range[-1..0] - -iterToProc(allObjects, ptr ObjectSpaceIter, allObjectsAsProc) - -const - rcIncrement = 0b1000 # so that lowest 3 bits are not touched - rcBlackOrig = 0b000 - rcWhiteOrig = 0b001 - rcGrey = 0b010 # traditional color for incremental mark&sweep - rcUnused = 0b011 - ZctFlag = 0b100 # in ZCT - rcShift = 3 # shift by rcShift to get the reference counter - colorMask = 0b011 -type - WalkOp = enum - waMarkGlobal, # part of the backup mark&sweep - waMarkGrey, - waZctDecRef #, waDebug - - Phase {.pure.} = enum - None, Marking, Sweeping - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} - # A ref type can have a finalizer that is called before the object's - # storage is freed. - - GcStat = object - stackScans: int # number of performed stack scans (for statistics) - cycleCollections: int # number of performed full collections - maxThreshold: int # max threshold that has been set - maxStackSize: int # max stack size - maxStackCells: int # max stack cells in ``decStack`` - cycleTableSize: int # max entries in cycle table - maxPause: int64 # max measured GC pause in nanoseconds - - GcStack = object - prev: ptr GcStack - next: ptr GcStack - starts: pointer - pos: pointer - maxStackSize: int - - GcHeap = object # this contains the zero count and - # non-zero count table - black: int # either 0 or 1. - stack: ptr GcStack - stackBottom: pointer - phase: Phase - cycleThreshold: int - when useCellIds: - idGenerator: int - zct: CellSeq # the zero count table - decStack: CellSeq # cells in the stack that are to decref again - greyStack: CellSeq - recGcLock: int # prevent recursion via finalizers; no thread lock - when withRealTime: - maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 - region: MemRegion # garbage collected region - stat: GcStat - additionalRoots: CellSeq # dummy roots for GC_ref/unref - spaceIter: ObjectSpaceIter - -var - gch {.rtlThreadVar.}: GcHeap - -when not defined(useNimRtl): - instantiateForRegion(gch.region) - -proc initGC() = - when not defined(useNimRtl): - when traceGC: - for i in low(CellState)..high(CellState): init(states[i]) - gch.cycleThreshold = InitialCycleThreshold - gch.stat.stackScans = 0 - gch.stat.cycleCollections = 0 - gch.stat.maxThreshold = 0 - gch.stat.maxStackSize = 0 - gch.stat.maxStackCells = 0 - gch.stat.cycleTableSize = 0 - # init the rt - init(gch.zct) - init(gch.decStack) - init(gch.additionalRoots) - init(gch.greyStack) - -template gcAssert(cond: bool, msg: string) = - when defined(useGcAssert): - if not cond: - echo "[GCASSERT] ", msg - GC_disable() - writeStackTrace() - quit 1 - -proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = - if (c.refcount and ZctFlag) == 0: - c.refcount = c.refcount or ZctFlag - add(s, c) - -proc cellToUsr(cell: PCell): pointer {.inline.} = - # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) - -proc usrToCell(usr: pointer): PCell {.inline.} = - # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) - -proc canBeCycleRoot(c: PCell): bool {.inline.} = - result = ntfAcyclic notin c.typ.flags - -proc extGetCellType(c: pointer): PNimType {.compilerproc.} = - # used for code generation concerning debugging - result = usrToCell(c).typ - -proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = - result = int(usrToCell(p).refcount) shr rcShift - -# this that has to equals zero, otherwise we have to round up UnitsPerPage: -when BitsPerPage mod (sizeof(int)*8) != 0: - {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} - -template color(c): expr = c.refCount and colorMask -template setColor(c, col) = - c.refcount = c.refcount and not colorMask or col - -proc writeCell(msg: cstring, c: PCell) = - var kind = -1 - if c.typ != nil: kind = ord(c.typ.kind) - when leakDetector: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", - msg, c, kind, c.refcount shr rcShift, c.filename, c.line) - else: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", - msg, c, kind, c.refcount shr rcShift, c.color) - -template gcTrace(cell, state: expr): stmt {.immediate.} = - when traceGC: traceCell(cell, state) - -# forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} -# we need the prototype here for debugging purposes - -when hasThreadSupport and hasSharedHeap: - template `--`(x: expr): expr = atomicDec(x, rcIncrement) <% rcIncrement - template `++`(x: expr): stmt = discard atomicInc(x, rcIncrement) -else: - template `--`(x: expr): expr = - dec(x, rcIncrement) - x <% rcIncrement - template `++`(x: expr): stmt = inc(x, rcIncrement) - -proc prepareDealloc(cell: PCell) = - if cell.typ.finalizer != nil: - # the finalizer could invoke something that - # allocates memory; this could trigger a garbage - # collection. Since we are already collecting we - # prevend recursive entering here by a lock. - # XXX: we should set the cell's children to nil! - inc(gch.recGcLock) - (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) - dec(gch.recGcLock) - -proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = - # we MUST access gch as a global here, because this crosses DLL boundaries! - discard - -proc rtlAddZCT(c: PCell) {.rtl, inl.} = - # we MUST access gch as a global here, because this crosses DLL boundaries! - addZCT(gch.zct, c) - -proc decRef(c: PCell) {.inline.} = - gcAssert(isAllocatedPtr(gch.region, c), "decRef: interiorPtr") - gcAssert(c.refcount >=% rcIncrement, "decRef") - if --c.refcount: - rtlAddZCT(c) - -proc incRef(c: PCell) {.inline.} = - gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") - c.refcount = c.refcount +% rcIncrement - -proc nimGCref(p: pointer) {.compilerProc.} = - let cell = usrToCell(p) - incRef(cell) - add(gch.additionalRoots, cell) - -proc nimGCunref(p: pointer) {.compilerProc.} = - let cell = usrToCell(p) - decRef(cell) - var L = gch.additionalRoots.len-1 - var i = L - let d = gch.additionalRoots.d - while i >= 0: - if d[i] == cell: - d[i] = d[L] - dec gch.additionalRoots.len - break - dec(i) - -template markGrey(x: PCell) = - if x.color == 1-gch.black and gch.phase == Phase.Marking: - x.setColor(rcGrey) - add(gch.greyStack, x) - -proc GC_addCycleRoot*[T](p: ref T) {.inline.} = - ## adds 'p' to the cycle candidate set for the cycle collector. It is - ## necessary if you used the 'acyclic' pragma for optimization - ## purposes and need to break cycles manually. - rtlAddCycleRoot(usrToCell(cast[pointer](p))) - -proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} = - sysAssert(allocInv(gch.region), "begin nimGCunrefNoCycle") - var c = usrToCell(p) - gcAssert(isAllocatedPtr(gch.region, c), "nimGCunrefNoCycle: isAllocatedPtr") - if --c.refcount: - rtlAddZCT(c) - sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 2") - sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 5") - -proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} = - # the code generator calls this proc! - gcAssert(not isOnStack(dest), "asgnRef") - # BUGFIX: first incRef then decRef! - if src != nil: - let s = usrToCell(src) - incRef(s) - markGrey(s) - if dest[] != nil: decRef(usrToCell(dest[])) - dest[] = src - -proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerProc, inline.} = - # the code generator calls this proc if it is known at compile time that no - # cycle is possible. - gcAssert(not isOnStack(dest), "asgnRefNoCycle") - if src != nil: - var c = usrToCell(src) - ++c.refcount - markGrey(c) - if dest[] != nil: - var c = usrToCell(dest[]) - if --c.refcount: - rtlAddZCT(c) - dest[] = src - -proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} = - # unsureAsgnRef updates the reference counters only if dest is not on the - # stack. It is used by the code generator if it cannot decide wether a - # reference is in the stack or not (this can happen for var parameters). - if not isOnStack(dest): - if src != nil: - let s = usrToCell(src) - incRef(s) - markGrey(s) - # XXX finally use assembler for the stack checking instead! - # the test for '!= nil' is correct, but I got tired of the segfaults - # resulting from the crappy stack checking: - if cast[int](dest[]) >=% PageSize: decRef(usrToCell(dest[])) - else: - # can't be an interior pointer if it's a stack location! - gcAssert(interiorAllocatedPtr(gch.region, dest) == nil, - "stack loc AND interior pointer") - dest[] = src - -type - GlobalMarkerProc = proc () {.nimcall, benign.} -var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - -proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 - -proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) - case n.kind - of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) - of nkList: - for i in 0..n.len-1: - forAllSlotsAux(dest, n.sons[i], op) - of nkCase: - var m = selectBranch(dest, n) - if m != nil: forAllSlotsAux(dest, m, op) - of nkNone: sysAssert(false, "forAllSlotsAux") - -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) - if dest == nil: return # nothing to do - if ntfNoRefs notin mt.flags: - case mt.kind - of tyRef, tyString, tySequence: # leaf: - doOperation(cast[PPointer](d)[], op) - of tyObject, tyTuple: - forAllSlotsAux(dest, mt.node, op) - of tyArray, tyArrayConstr, tyOpenArray: - for i in 0..(mt.size div mt.base.size)-1: - forAllChildrenAux(cast[pointer](d +% i *% mt.base.size), mt.base, op) - else: discard - -proc forAllChildren(cell: PCell, op: WalkOp) = - gcAssert(cell != nil, "forAllChildren: 1") - gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") - gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 4" - let marker = cell.typ.marker - if marker != nil: - marker(cellToUsr(cell), op.int) - else: - case cell.typ.kind - of tyRef: # common case - forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) - of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) - var s = cast[PGenericSeq](d) - if s != nil: - for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) - else: discard - -proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = - # we check the last 8 entries (cache line) for a slot that could be reused. - # In 63% of all cases we succeed here! But we have to optimize the heck - # out of this small linear search so that ``newObj`` is not slowed down. - # - # Slots to try cache hit - # 1 32% - # 4 59% - # 8 63% - # 16 66% - # all slots 68% - var L = gch.zct.len - var d = gch.zct.d - when true: - # loop unrolled for performance: - template replaceZctEntry(i: expr) = - c = d[i] - if c.refcount >=% rcIncrement: - c.refcount = c.refcount and not ZctFlag - d[i] = res - return - if L > 8: - var c: PCell - replaceZctEntry(L-1) - replaceZctEntry(L-2) - replaceZctEntry(L-3) - replaceZctEntry(L-4) - replaceZctEntry(L-5) - replaceZctEntry(L-6) - replaceZctEntry(L-7) - replaceZctEntry(L-8) - add(gch.zct, res) - else: - d[L] = res - inc(gch.zct.len) - else: - for i in countdown(L-1, max(0, L-8)): - var c = d[i] - if c.refcount >=% rcIncrement: - c.refcount = c.refcount and not ZctFlag - d[i] = res - return - add(gch.zct, res) - -{.push stackTrace: off, profiler:off.} -proc gcInvariant*() = - sysAssert(allocInv(gch.region), "injected") - when declared(markForDebug): - markForDebug(gch) -{.pop.} - -proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = - # generates a new object and sets its reference counter to 0 - sysAssert(allocInv(gch.region), "rawNewObj begin") - gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) - var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") - # now it is buffered in the ZCT - res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line - # refcount is zero, color is black, but mark it to be in the ZCT - res.refcount = ZctFlag or gch.black - sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - # its refcount is zero, so add it to the ZCT: - addNewObjToZCT(res, gch) - when logGC: writeCell("new cell", res) - gcTrace(res, csAllocated) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - sysAssert(allocInv(gch.region), "rawNewObj end") - -{.pop.} - -proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) - when defined(memProfiler): nimProfile(size) - -proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) - zeroMem(result, size) - when defined(memProfiler): nimProfile(size) - -proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) - result = newObj(typ, size) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - when defined(memProfiler): nimProfile(size) - -proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = - # generates a new object and sets its reference counter to 1 - sysAssert(allocInv(gch.region), "newObjRC1 begin") - gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) - sysAssert(allocInv(gch.region), "newObjRC1 after collectCT") - - var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") - # now it is buffered in the ZCT - res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line - res.refcount = rcIncrement or gch.black # refcount is 1 - sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - when logGC: writeCell("new cell", res) - gcTrace(res, csAllocated) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - zeroMem(result, size) - sysAssert(allocInv(gch.region), "newObjRC1 end") - when defined(memProfiler): nimProfile(size) - -proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) - result = newObjRC1(typ, size) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - when defined(memProfiler): nimProfile(size) - -proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - collectCT(gch) - var ol = usrToCell(old) - gcAssert(isAllocatedPtr(gch.region, ol), "growObj: freed pointer?") - - sysAssert(ol.typ != nil, "growObj: 1") - gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") - sysAssert(allocInv(gch.region), "growObj begin") - - var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size - - let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize - copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), - newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - # This can be wrong for intermediate temps that are nevertheless on the - # heap because of lambda lifting: - #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") - when logGC: - writeCell("growObj old cell", ol) - writeCell("growObj new cell", res) - gcTrace(ol, csZctFreed) - gcTrace(res, csAllocated) - when reallyDealloc: - sysAssert(allocInv(gch.region), "growObj before dealloc") - if ol.refcount shr rcShift <=% 1: - # free immediately to save space: - if (ol.refcount and ZctFlag) != 0: - var j = gch.zct.len-1 - var d = gch.zct.d - while j >= 0: - if d[j] == ol: - d[j] = res - break - dec(j) - rawDealloc(gch.region, ol) - else: - # we split the old refcount in 2 parts. XXX This is still not entirely - # correct if the pointer that receives growObj's result is on the stack. - # A better fix would be to emit the location specific write barrier for - # 'growObj', but this is lots of more work and who knows what new problems - # this would create. - res.refcount = rcIncrement or gch.black - decRef(ol) - else: - sysAssert(ol.typ != nil, "growObj: 5") - zeroMem(ol, sizeof(Cell)) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - sysAssert(allocInv(gch.region), "growObj end") - when defined(memProfiler): nimProfile(newsize-oldsize) - -proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - result = growObj(old, newsize, gch) - -{.push profiler:off.} - - -template takeStartTime(workPackageSize) {.dirty.} = - const workPackage = workPackageSize - var debugticker = 1000 - when withRealTime: - var steps = workPackage - var t0: Ticks - if gch.maxPause > 0: t0 = getticks() - -template takeTime {.dirty.} = - when withRealTime: dec steps - dec debugticker - -template checkTime {.dirty.} = - if debugticker <= 0: - echo "in loop" - debugticker = 1000 - when withRealTime: - if steps == 0: - steps = workPackage - if gch.maxPause > 0: - let duration = getticks() - t0 - # the GC's measuring is not accurate and needs some cleanup actions - # (stack unmarking), so subtract some short amount of time in - # order to miss deadlines less often: - if duration >= gch.maxPause - 50_000: - return false - -# ---------------- cycle collector ------------------------------------------- - -proc freeCyclicCell(gch: var GcHeap, c: PCell) = - gcAssert(isAllocatedPtr(gch.region, c), "freeCyclicCell: freed pointer?") - - var d = gch.decStack.d - for i in 0..gch.decStack.len-1: - gcAssert d[i] != c, "wtf man, freeing obviously alive stuff?!!" - - prepareDealloc(c) - gcTrace(c, csCycFreed) - when logGC: writeCell("cycle collector dealloc cell", c) - when reallyDealloc: - sysAssert(allocInv(gch.region), "free cyclic cell") - rawDealloc(gch.region, c) - else: - gcAssert(c.typ != nil, "freeCyclicCell") - zeroMem(c, sizeof(Cell)) - -proc sweep(gch: var GcHeap): bool = - takeStartTime(100) - echo "loop start" - let black = gch.black - while true: - let x = allObjectsAsProc(gch.region, addr gch.spaceIter) - if gch.spaceIter.state < 0: break - takeTime() - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - gcAssert c.color != rcGrey, "cell is still grey?" - if c.color != black: freeCyclicCell(gch, c) - # Since this is incremental, we MUST not set the object to 'white' here. - # We could set all the remaining objects to white after the 'sweep' - # completed but instead we flip the meaning of black/white to save one - # traversal over the heap! - checkTime() - # prepare for next iteration: - echo "loop end" - gch.spaceIter = ObjectSpaceIter() - result = true - -proc markRoot(gch: var GcHeap, c: PCell) = - # since we start with 'black' cells, we need to mark them here too: - if c.color != rcGrey: - c.setColor(rcGrey) - add(gch.greyStack, c) - -proc markIncremental(gch: var GcHeap): bool = - var L = addr(gch.greyStack.len) - takeStartTime(100) - while L[] > 0: - var c = gch.greyStack.d[0] - sysAssert(isAllocatedPtr(gch.region, c), "markIncremental: isAllocatedPtr") - gch.greyStack.d[0] = gch.greyStack.d[L[] - 1] - dec(L[]) - takeTime() - if c.color == rcGrey: - c.setColor(gch.black) - forAllChildren(c, waMarkGrey) - checkTime() - gcAssert gch.greyStack.len == 0, "markIncremental: greystack not empty " - result = true - -proc markGlobals(gch: var GcHeap) = - for i in 0 .. < globalMarkersLen: globalMarkers[i]() - -proc markLocals(gch: var GcHeap) = - var d = gch.decStack.d - for i in 0 .. < gch.decStack.len: - sysAssert isAllocatedPtr(gch.region, d[i]), "markLocals" - markRoot(gch, d[i]) - -when logGC: - var - cycleCheckA: array[100, PCell] - cycleCheckALen = 0 - - proc alreadySeen(c: PCell): bool = - for i in 0 .. <cycleCheckALen: - if cycleCheckA[i] == c: return true - if cycleCheckALen == len(cycleCheckA): - gcAssert(false, "cycle detection overflow") - quit 1 - cycleCheckA[cycleCheckALen] = c - inc cycleCheckALen - - proc debugGraph(s: PCell) = - if alreadySeen(s): - writeCell("child cell (already seen) ", s) - else: - writeCell("cell {", s) - forAllChildren(s, waDebug) - c_fprintf(c_stdout, "}\n") - -proc doOperation(p: pointer, op: WalkOp) = - if p == nil: return - var c: PCell = usrToCell(p) - gcAssert(c != nil, "doOperation: 1") - # the 'case' should be faster than function pointers because of easy - # prediction: - case op - of waZctDecRef: - #if not isAllocatedPtr(gch.region, c): - # c_fprintf(c_stdout, "[GC] decref bug: %p", c) - gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") - gcAssert(c.refcount >=% rcIncrement, "doOperation 2") - #c.refcount = c.refcount -% rcIncrement - when logGC: writeCell("decref (from doOperation)", c) - decRef(c) - #if c.refcount <% rcIncrement: addZCT(gch.zct, c) - of waMarkGlobal: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - markRoot(gch, c) - else: - markRoot(gch, c) - of waMarkGrey: - if c.color == 1-gch.black: - c.setColor(rcGrey) - add(gch.greyStack, c) - #of waDebug: debugGraph(c) - -proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = - doOperation(d, WalkOp(op)) - -proc collectZCT(gch: var GcHeap): bool {.benign.} - -proc collectCycles(gch: var GcHeap): bool = - # ensure the ZCT 'color' is not used: - while gch.zct.len > 0: discard collectZCT(gch) - case gch.phase - of Phase.None, Phase.Marking: - #if gch.phase == Phase.None: - gch.phase = Phase.Marking - markGlobals(gch) - markLocals(gch) - if markIncremental(gch): - gch.phase = Phase.Sweeping - of Phase.Sweeping: - gcAssert gch.greyStack.len == 0, "greystack not empty" - if sweep(gch): - gch.phase = Phase.None - # flip black/white meanings: - gch.black = 1 - gch.black - result = true - -proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - # mark the cell: - objStart.refcount = objStart.refcount +% rcIncrement - add(gch.decStack, objStart) - sysAssert(allocInv(gch.region), "gcMark end") - -include gc_common - -proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = - forEachStackSlot(gch, gcMark) - -proc collectZCT(gch: var GcHeap): bool = - # Note: Freeing may add child objects to the ZCT! So essentially we do - # deep freeing, which is bad for incremental operation. In order to - # avoid a deep stack, we move objects to keep the ZCT small. - # This is performance critical! - var L = addr(gch.zct.len) - takeStartTime(100) - - while L[] > 0: - var c = gch.zct.d[0] - sysAssert(isAllocatedPtr(gch.region, c), "CollectZCT: isAllocatedPtr") - # remove from ZCT: - gcAssert((c.refcount and ZctFlag) == ZctFlag, "collectZCT") - - c.refcount = c.refcount and not ZctFlag - gch.zct.d[0] = gch.zct.d[L[] - 1] - dec(L[]) - takeTime() - if c.refcount <% rcIncrement: - # It may have a RC > 0, if it is in the hardware stack or - # it has not been removed yet from the ZCT. This is because - # ``incref`` does not bother to remove the cell from the ZCT - # as this might be too slow. - # In any case, it should be removed from the ZCT. But not - # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - when logGC: writeCell("zct dealloc cell", c) - gcTrace(c, csZctFreed) - # We are about to free the object, call the finalizer BEFORE its - # children are deleted as well, because otherwise the finalizer may - # access invalid memory. This is done by prepareDealloc(): - prepareDealloc(c) - forAllChildren(c, waZctDecRef) - when reallyDealloc: - sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") - rawDealloc(gch.region, c) - else: - sysAssert(c.typ != nil, "collectZCT 2") - zeroMem(c, sizeof(Cell)) - checkTime() - result = true - -proc unmarkStackAndRegisters(gch: var GcHeap) = - var d = gch.decStack.d - for i in 0..gch.decStack.len-1: - sysAssert isAllocatedPtr(gch.region, d[i]), "unmarkStackAndRegisters" - decRef(d[i]) - gch.decStack.len = 0 - -proc collectCTBody(gch: var GcHeap) = - when withRealTime: - let t0 = getticks() - sysAssert(allocInv(gch.region), "collectCT: begin") - - when not defined(nimCoroutines): - gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) - sysAssert(gch.decStack.len == 0, "collectCT") - prepareForInteriorPointerChecking(gch.region) - markStackAndRegisters(gch) - gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) - inc(gch.stat.stackScans) - if collectZCT(gch): - when cycleGC: - if getOccupiedMem(gch.region) >= gch.cycleThreshold or alwaysCycleGC: - if collectCycles(gch): - inc(gch.stat.cycleCollections) - gch.cycleThreshold = max(InitialCycleThreshold, getOccupiedMem() * - CycleIncrease) - gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) - unmarkStackAndRegisters(gch) - sysAssert(allocInv(gch.region), "collectCT: end") - - when withRealTime: - let duration = getticks() - t0 - gch.stat.maxPause = max(gch.stat.maxPause, duration) - when defined(reportMissedDeadlines): - if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) - -when defined(nimCoroutines): - proc currentStackSizes(): int = - for stack in items(gch.stack): - result = result + stackSize(stack.starts, stack.pos) - -proc collectCT(gch: var GcHeap) = - # stackMarkCosts prevents some pathological behaviour: Stack marking - # becomes more expensive with large stacks and large stacks mean that - # cells with RC=0 are more likely to be kept alive by the stack. - when defined(nimCoroutines): - let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold) - else: - let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold) - if (gch.zct.len >= stackMarkCosts or (cycleGC and - getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and - gch.recGcLock == 0: - collectCTBody(gch) - -when withRealTime: - proc toNano(x: int): Nanos {.inline.} = - result = x * 1000 - - proc GC_setMaxPause*(MaxPauseInUs: int) = - gch.maxPause = MaxPauseInUs.toNano - - proc GC_step(gch: var GcHeap, us: int, strongAdvice: bool) = - gch.maxPause = us.toNano - if (gch.zct.len >= ZctThreshold or (cycleGC and - getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or - strongAdvice: - collectCTBody(gch) - - proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) - -when not defined(useNimRtl): - proc GC_disable() = - when hasThreadSupport and hasSharedHeap: - discard atomicInc(gch.recGcLock, 1) - else: - inc(gch.recGcLock) - proc GC_enable() = - if gch.recGcLock > 0: - when hasThreadSupport and hasSharedHeap: - discard atomicDec(gch.recGcLock, 1) - else: - dec(gch.recGcLock) - - proc GC_setStrategy(strategy: GC_Strategy) = - discard - - proc GC_enableMarkAndSweep() = - gch.cycleThreshold = InitialCycleThreshold - - proc GC_disableMarkAndSweep() = - gch.cycleThreshold = high(gch.cycleThreshold)-1 - # set to the max value to suppress the cycle detector - - proc GC_fullCollect() = - var oldThreshold = gch.cycleThreshold - gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) - gch.cycleThreshold = oldThreshold - - proc GC_getStatistics(): string = - GC_disable() - result = "[GC] total memory: " & $(getTotalMem()) & "\n" & - "[GC] occupied memory: " & $(getOccupiedMem()) & "\n" & - "[GC] stack scans: " & $gch.stat.stackScans & "\n" & - "[GC] stack cells: " & $gch.stat.maxStackCells & "\n" & - "[GC] cycle collections: " & $gch.stat.cycleCollections & "\n" & - "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & - "[GC] zct capacity: " & $gch.zct.cap & "\n" & - "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) - when defined(nimCoroutines): - result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" - for stack in items(gch.stack): - result = result & "[GC] stack " & stack.starts.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" - else: - result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" - GC_enable() - -{.pop.} diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 47e8b4b1f..eb0884560 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -7,95 +7,203 @@ # distribution, for details about the copyright. # -proc len(stack: ptr GcStack): int = - if stack == nil: - return 0 - - var s = stack - result = 1 - while s.next != nil: - inc(result) - s = s.next - -when defined(nimCoroutines): - proc stackSize(stackBottom: pointer, pos: pointer=nil): int {.noinline.} = - var sp: pointer - if pos == nil: - var stackTop {.volatile.}: pointer - sp = addr(stackTop) +type + ForeignCell* = object + data*: pointer + owner: ptr GcHeap + +proc protect*(x: pointer): ForeignCell = + nimGCref(x) + result.data = x + result.owner = addr(gch) + +when defined(nimTypeNames): + type InstancesInfo = array[400, (cstring, int, int)] + proc sortInstances(a: var InstancesInfo; n: int) = + # we use shellsort here; fast and simple + var h = 1 + while true: + h = 3 * h + 1 + if h > n: break + while true: + h = h div 3 + for i in countup(h, n - 1): + var v = a[i] + var j = i + while a[j - h][2] < v[2]: + a[j] = a[j - h] + j = j - h + if j < h: break + a[j] = v + if h == 1: break + + iterator dumpHeapInstances*(): tuple[name: cstring; count: int; sizes: int] = + ## Iterate over summaries of types on heaps. + ## This data may be inaccurate if allocations + ## are made by the iterator body. + if strDesc.nextType == nil: + strDesc.nextType = nimTypeRoot + strDesc.name = "string" + nimTypeRoot = addr strDesc + var it = nimTypeRoot + while it != nil: + if (it.instances > 0 or it.sizes != 0): + yield (it.name, it.instances, it.sizes) + it = it.nextType + + proc dumpNumberOfInstances* = + var a: InstancesInfo + var n = 0 + var totalAllocated = 0 + for it in dumpHeapInstances(): + a[n] = it + inc n + inc totalAllocated, it.sizes + sortInstances(a, n) + for i in 0 .. n-1: + c_fprintf(cstdout, "[Heap] %s: #%ld; bytes: %ld\n", a[i][0], a[i][1], a[i][2]) + c_fprintf(cstdout, "[Heap] total number of bytes: %ld\n", totalAllocated) + when defined(nimTypeNames): + let (allocs, deallocs) = getMemCounters() + c_fprintf(cstdout, "[Heap] allocs/deallocs: %ld/%ld\n", allocs, deallocs) + + when defined(nimGcRefLeak): + proc oomhandler() = + c_fprintf(cstdout, "[Heap] ROOTS: #%ld\n", gch.additionalRoots.len) + writeLeaks() + + outOfMemHook = oomhandler + +template decTypeSize(cell, t) = + when defined(nimTypeNames): + if t.kind in {tyString, tySequence}: + let cap = cast[PGenericSeq](cellToUsr(cell)).space + let size = + if t.kind == tyString: + cap + 1 + GenericSeqSize + else: + align(GenericSeqSize, t.base.align) + cap * t.base.size + atomicDec t.sizes, size+sizeof(Cell) else: - sp = pos - result = abs(cast[int](sp) - cast[int](stackBottom)) + atomicDec t.sizes, t.base.size+sizeof(Cell) + atomicDec t.instances - proc GC_addStack*(starts: pointer) {.cdecl, exportc.} = - var sp {.volatile.}: pointer - var stack = cast[ptr GcStack](alloc0(sizeof(GcStack))) - stack.starts = starts - stack.pos = addr sp - if gch.stack == nil: - gch.stack = stack +template incTypeSize(typ, size) = + when defined(nimTypeNames): + atomicInc typ.instances + atomicInc typ.sizes, size+sizeof(Cell) + +proc dispose*(x: ForeignCell) = + when hasThreadSupport: + # if we own it we can free it directly: + if x.owner == addr(gch): + nimGCunref(x.data) else: - stack.next = gch.stack - gch.stack.prev = stack - gch.stack = stack - # c_fprintf(c_stdout, "[GC] added stack 0x%016X\n", starts) - - proc GC_removeStack*(starts: pointer) {.cdecl, exportc.} = - var stack = gch.stack - while stack != nil: - if stack.starts == starts: - if stack.prev == nil: - if stack.next != nil: - stack.next.prev = nil - gch.stack = stack.next - else: - stack.prev.next = stack.next - if stack.next != nil: - stack.next.prev = stack.prev - dealloc(stack) - # echo "[GC] removed stack ", starts.repr + x.owner.toDispose.add(x.data) + else: + nimGCunref(x.data) + +proc isNotForeign*(x: ForeignCell): bool = + ## returns true if 'x' belongs to the calling thread. + ## No deep copy has to be performed then. + x.owner == addr(gch) + +when nimCoroutines: + iterator items(first: var GcStack): ptr GcStack = + var item = addr(first) + while true: + yield item + item = item.next + if item == addr(first): break - else: - stack = stack.next - - proc GC_setCurrentStack*(starts, pos: pointer) {.cdecl, exportc.} = - var stack = gch.stack - while stack != nil: - if stack.starts == starts: - stack.pos = pos - stack.maxStackSize = max(stack.maxStackSize, stackSize(stack.starts, pos)) - return - stack = stack.next - gcAssert(false, "Current stack position does not belong to registered stack") + + proc append(first: var GcStack, stack: ptr GcStack) = + ## Append stack to the ring of stacks. + first.prev.next = stack + stack.prev = first.prev + first.prev = stack + stack.next = addr(first) + + proc append(first: var GcStack): ptr GcStack = + ## Allocate new GcStack object, append it to the ring of stacks and return it. + result = cast[ptr GcStack](alloc0(sizeof(GcStack))) + first.append(result) + + proc remove(first: var GcStack, stack: ptr GcStack) = + ## Remove stack from ring of stacks. + gcAssert(addr(first) != stack, "Main application stack can not be removed") + if addr(first) == stack or stack == nil: + return + stack.prev.next = stack.next + stack.next.prev = stack.prev + dealloc(stack) + + proc remove(stack: ptr GcStack) = + gch.stack.remove(stack) + + proc find(first: var GcStack, bottom: pointer): ptr GcStack = + ## Find stack struct based on bottom pointer. If `bottom` is nil then main + ## thread stack is is returned. + if bottom == nil: + return addr(gch.stack) + + for stack in first.items(): + if stack.bottom == bottom: + return stack + + proc len(stack: var GcStack): int = + for _ in stack.items(): + result = result + 1 else: - proc stackSize(): int {.noinline.} = - var stackTop {.volatile.}: pointer - result = abs(cast[int](addr(stackTop)) - cast[int](gch.stackBottom)) + # This iterator gets optimized out in forEachStackSlot(). + iterator items(first: var GcStack): ptr GcStack = yield addr(first) + proc len(stack: var GcStack): int = 1 -iterator items(stack: ptr GcStack): ptr GcStack = - var s = stack - while not isNil(s): - yield s - s = s.next +when defined(nimdoc): + proc setupForeignThreadGc*() {.gcsafe.} = + ## Call this if you registered a callback that will be run from a thread not + ## under your control. This has a cheap thread-local guard, so the GC for + ## this thread will only be initialized once per thread, no matter how often + ## it is called. + ## + ## This function is available only when `--threads:on` and `--tlsEmulation:off` + ## switches are used + discard -var - localGcInitialized {.rtlThreadVar.}: bool - -proc setupForeignThreadGc*() = - ## call this if you registered a callback that will be run from a thread not - ## under your control. This has a cheap thread-local guard, so the GC for - ## this thread will only be initialized once per thread, no matter how often - ## it is called. - if not localGcInitialized: - localGcInitialized = true - var stackTop {.volatile.}: pointer - setStackBottom(addr(stackTop)) - initGC() + proc tearDownForeignThreadGc*() {.gcsafe.} = + ## Call this to tear down the GC, previously initialized by `setupForeignThreadGc`. + ## If GC has not been previously initialized, or has already been torn down, the + ## call does nothing. + ## + ## This function is available only when `--threads:on` and `--tlsEmulation:off` + ## switches are used + discard +elif declared(threadType): + proc setupForeignThreadGc*() {.gcsafe.} = + if threadType == ThreadType.None: + var stackTop {.volatile.}: pointer + nimGC_setStackBottom(addr(stackTop)) + initGC() + threadType = ThreadType.ForeignThread + + proc tearDownForeignThreadGc*() {.gcsafe.} = + if threadType != ThreadType.ForeignThread: + return + when declared(deallocOsPages): deallocOsPages() + threadType = ThreadType.None + when declared(gch): zeroMem(addr gch, sizeof(gch)) + +else: + template setupForeignThreadGc*() = + {.error: "setupForeignThreadGc is available only when ``--threads:on`` and ``--tlsEmulation:off`` are used".} + + template tearDownForeignThreadGc*() = + {.error: "tearDownForeignThreadGc is available only when ``--threads:on`` and ``--tlsEmulation:off`` are used".} # ----------------- stack management -------------------------------------- # inspired from Smart Eiffel -when defined(emscripten): +when defined(emscripten) or defined(wasm): const stackIncreases = true elif defined(sparc): const stackIncreases = false @@ -105,52 +213,131 @@ elif defined(hppa) or defined(hp9000) or defined(hp9000s300) or else: const stackIncreases = false -when not defined(useNimRtl): - {.push stack_trace: off.} - proc setStackBottom(theStackBottom: pointer) = - #c_fprintf(c_stdout, "stack bottom: %p;\n", theStackBottom) - # the first init must be the one that defines the stack bottom: - when defined(nimCoroutines): - GC_addStack(theStackBottom) +proc stackSize(stack: ptr GcStack): int {.noinline.} = + when nimCoroutines: + var pos = stack.pos + else: + var pos {.volatile, noinit.}: pointer + pos = addr(pos) + + if pos != nil: + when stackIncreases: + result = cast[int](pos) -% cast[int](stack.bottom) else: - if gch.stackBottom == nil: gch.stackBottom = theStackBottom + result = cast[int](stack.bottom) -% cast[int](pos) + else: + result = 0 + +proc stackSize(): int {.noinline.} = + result = 0 + for stack in gch.stack.items(): + result = result + stack.stackSize() + +when nimCoroutines: + proc setPosition(stack: ptr GcStack, position: pointer) = + stack.pos = position + stack.maxStackSize = max(stack.maxStackSize, stack.stackSize()) + + proc setPosition(stack: var GcStack, position: pointer) = + setPosition(addr(stack), position) + + proc getActiveStack(gch: var GcHeap): ptr GcStack = + return gch.activeStack + + proc isActiveStack(stack: ptr GcStack): bool = + return gch.activeStack == stack +else: + # Stack positions do not need to be tracked if coroutines are not used. + proc setPosition(stack: ptr GcStack, position: pointer) = discard + proc setPosition(stack: var GcStack, position: pointer) = discard + # There is just one stack - main stack of the thread. It is active always. + proc getActiveStack(gch: var GcHeap): ptr GcStack = addr(gch.stack) + proc isActiveStack(stack: ptr GcStack): bool = true + +{.push stack_trace: off.} +when nimCoroutines: + proc GC_addStack(bottom: pointer) {.cdecl, dynlib, exportc.} = + # c_fprintf(stdout, "GC_addStack: %p;\n", bottom) + var stack = gch.stack.append() + stack.bottom = bottom + stack.setPosition(bottom) + + proc GC_removeStack(bottom: pointer) {.cdecl, dynlib, exportc.} = + # c_fprintf(stdout, "GC_removeStack: %p;\n", bottom) + gch.stack.find(bottom).remove() + + proc GC_setActiveStack(bottom: pointer) {.cdecl, dynlib, exportc.} = + ## Sets active stack and updates current stack position. + # c_fprintf(stdout, "GC_setActiveStack: %p;\n", bottom) + var sp {.volatile.}: pointer + gch.activeStack = gch.stack.find(bottom) + gch.activeStack.setPosition(addr(sp)) + + proc GC_getActiveStack() : pointer {.cdecl, exportc.} = + return gch.activeStack.bottom + +when not defined(useNimRtl): + proc nimGC_setStackBottom(theStackBottom: pointer) = + # Initializes main stack of the thread. + when nimCoroutines: + if gch.stack.next == nil: + # Main stack was not initialized yet + gch.stack.next = addr(gch.stack) + gch.stack.prev = addr(gch.stack) + gch.stack.bottom = theStackBottom + gch.stack.maxStackSize = 0 + gch.activeStack = addr(gch.stack) + + if gch.stack.bottom == nil: + # This branch will not be called when -d:nimCoroutines - it is fine, + # because same thing is done just above. + #c_fprintf(stdout, "stack bottom: %p;\n", theStackBottom) + # the first init must be the one that defines the stack bottom: + gch.stack.bottom = theStackBottom + elif theStackBottom != gch.stack.bottom: + var a = cast[int](theStackBottom) # and not PageMask - PageSize*2 + var b = cast[int](gch.stack.bottom) + #c_fprintf(stdout, "old: %p new: %p;\n",gch.stack.bottom,theStackBottom) + when stackIncreases: + gch.stack.bottom = cast[pointer](min(a, b)) else: - var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 - var b = cast[ByteAddress](gch.stackBottom) - #c_fprintf(c_stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom) - when stackIncreases: - gch.stackBottom = cast[pointer](min(a, b)) - else: - gch.stackBottom = cast[pointer](max(a, b)) - {.pop.} + gch.stack.bottom = cast[pointer](max(a, b)) + + when nimCoroutines: + if theStackBottom != nil: gch.stack.bottom = theStackBottom + + gch.stack.setPosition(theStackBottom) +{.pop.} + +proc isOnStack(p: pointer): bool = + var stackTop {.volatile, noinit.}: pointer + stackTop = addr(stackTop) + var a = cast[int](gch.getActiveStack().bottom) + var b = cast[int](stackTop) + when not stackIncreases: + swap(a, b) + var x = cast[int](p) + result = a <=% x and x <=% b when defined(sparc): # For SPARC architecture. - when defined(nimCoroutines): + when nimCoroutines: {.error: "Nim coroutines are not supported on this platform."} - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - var b = cast[TAddress](gch.stackBottom) - var a = cast[TAddress](stackTop) - var x = cast[TAddress](p) - result = a <=% x and x <=% b - - template forEachStackSlot(gch, gcMark: expr) {.immediate, dirty.} = + template forEachStackSlot(gch, gcMark: untyped) {.dirty.} = when defined(sparcv9): asm """"flushw \n" """ else: asm """"ta 0x3 ! ST_FLUSH_WINDOWS\n" """ var - max = gch.stackBottom + max = gch.stack.bottom sp: PPointer stackTop: array[0..1, pointer] sp = addr(stackTop[0]) # Addresses decrease as the stack grows. while sp <= max: gcMark(gch, sp[]) - sp = cast[PPointer](cast[TAddress](sp) +% sizeof(pointer)) + sp = cast[PPointer](cast[int](sp) +% sizeof(pointer)) elif defined(ELATE): {.error: "stack marking code is to be written for this architecture".} @@ -159,101 +346,52 @@ elif stackIncreases: # --------------------------------------------------------------------------- # Generic code for architectures where addresses increase as the stack grows. # --------------------------------------------------------------------------- - when defined(nimCoroutines): - {.error: "Nim coroutines are not supported on this platform."} - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - var a = cast[ByteAddress](gch.stackBottom) - var b = cast[ByteAddress](stackTop) - var x = cast[ByteAddress](p) - result = a <=% x and x <=% b - - var - jmpbufSize {.importc: "sizeof(jmp_buf)", nodecl.}: int - # a little hack to get the size of a JmpBuf in the generated C code - # in a platform independent way - - template forEachStackSlot(gch, gcMark: expr) {.immediate, dirty.} = - var registers {.noinit.}: C_JmpBuf - if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. - var max = cast[ByteAddress](gch.stackBottom) - var sp = cast[ByteAddress](addr(registers)) +% jmpbufSize -% sizeof(pointer) - # sp will traverse the JMP_BUF as well (jmp_buf size is added, - # otherwise sp would be below the registers structure). + when defined(emscripten) or defined(wasm): + var + jmpbufSize {.importc: "sizeof(jmp_buf)", nodecl.}: int + # a little hack to get the size of a JmpBuf in the generated C code + # in a platform independent way + + template forEachStackSlotAux(gch, gcMark: untyped) {.dirty.} = + for stack in gch.stack.items(): + var max = cast[int](gch.stack.bottom) + var sp = cast[int](addr(registers)) -% sizeof(pointer) while sp >=% max: gcMark(gch, cast[PPointer](sp)[]) sp = sp -% sizeof(pointer) + template forEachStackSlot(gch, gcMark: untyped) {.dirty.} = + when defined(emscripten) or defined(wasm): + var registers: cint + forEachStackSlotAux(gch, gcMark) + else: + var registers {.noinit.}: C_JmpBuf + if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. + forEachStackSlotAux(gch, gcMark) + else: # --------------------------------------------------------------------------- # Generic code for architectures where addresses decrease as the stack grows. # --------------------------------------------------------------------------- - when defined(nimCoroutines): - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - for stack in items(gch.stack): - var b = cast[ByteAddress](stack.starts) - var a = cast[ByteAddress](stack.starts) - stack.maxStackSize - var x = cast[ByteAddress](p) - if a <=% x and x <=% b: - return true - - template forEachStackSlot(gch, gcMark: expr) {.immediate, dirty.} = - # We use a jmp_buf buffer that is in the C stack. - # Used to traverse the stack and registers assuming - # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] - var registers {.noinit.}: Registers - getRegisters(registers) - for i in registers.low .. registers.high: - gcMark(gch, cast[PPointer](registers[i])) - - for stack in items(gch.stack): - stack.maxStackSize = max(stack.maxStackSize, stackSize(stack.starts)) - var max = cast[ByteAddress](stack.starts) - var sp = cast[ByteAddress](stack.pos) - # loop unrolled: - while sp <% max - 8*sizeof(pointer): - gcMark(gch, cast[PStackSlice](sp)[0]) - gcMark(gch, cast[PStackSlice](sp)[1]) - gcMark(gch, cast[PStackSlice](sp)[2]) - gcMark(gch, cast[PStackSlice](sp)[3]) - gcMark(gch, cast[PStackSlice](sp)[4]) - gcMark(gch, cast[PStackSlice](sp)[5]) - gcMark(gch, cast[PStackSlice](sp)[6]) - gcMark(gch, cast[PStackSlice](sp)[7]) - sp = sp +% sizeof(pointer)*8 - # last few entries: - while sp <=% max: - gcMark(gch, cast[PPointer](sp)[]) - sp = sp +% sizeof(pointer) - else: - proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer - stackTop = addr(stackTop) - var b = cast[ByteAddress](gch.stackBottom) - var a = cast[ByteAddress](stackTop) - var x = cast[ByteAddress](p) - result = a <=% x and x <=% b - - template forEachStackSlot(gch, gcMark: expr) {.immediate, dirty.} = - # We use a jmp_buf buffer that is in the C stack. - # Used to traverse the stack and registers assuming - # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] - var registers {.noinit.}: C_JmpBuf - if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. - var max = cast[ByteAddress](gch.stackBottom) - var sp = cast[ByteAddress](addr(registers)) + template forEachStackSlot(gch, gcMark: untyped) {.dirty.} = + # We use a jmp_buf buffer that is in the C stack. + # Used to traverse the stack and registers assuming + # that 'setjmp' will save registers in the C stack. + type PStackSlice = ptr array[0..7, pointer] + var registers {.noinit.}: C_JmpBuf + # Update position of stack gc is executing in. + gch.getActiveStack().setPosition(addr(registers)) + if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. + for stack in gch.stack.items(): + var max = cast[int](stack.bottom) + var sp = cast[int](addr(registers)) when defined(amd64): - # words within the jmp_buf structure may not be properly aligned. - let regEnd = sp +% sizeof(registers) - while sp <% regEnd: - gcMark(gch, cast[PPointer](sp)[]) - gcMark(gch, cast[PPointer](sp +% sizeof(pointer) div 2)[]) - sp = sp +% sizeof(pointer) + if stack.isActiveStack(): + # words within the jmp_buf structure may not be properly aligned. + let regEnd = sp +% sizeof(registers) + while sp <% regEnd: + gcMark(gch, cast[PPointer](sp)[]) + sp = sp +% sizeof(pointer) # Make sure sp is word-aligned sp = sp and not (sizeof(pointer) - 1) # loop unrolled: @@ -275,3 +413,70 @@ else: # ---------------------------------------------------------------------------- # end of non-portable code # ---------------------------------------------------------------------------- + +proc prepareDealloc(cell: PCell) {.raises: [].} = + when declared(useMarkForDebug): + when useMarkForDebug: + gcAssert(cell notin gch.marked, "Cell still alive!") + let t = cell.typ + if t.finalizer != nil: + # the finalizer could invoke something that + # allocates memory; this could trigger a garbage + # collection. Since we are already collecting we + # prevent recursive entering here by a lock. + # XXX: we should set the cell's children to nil! + inc(gch.recGcLock) + (cast[Finalizer](t.finalizer))(cellToUsr(cell)) + dec(gch.recGcLock) + decTypeSize(cell, t) + +proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = + ## Frees the thread local heap. Runs every finalizer if `runFinalizers` + ## is true. If `allowGcAfterwards` is true, a minimal amount of allocation + ## happens to ensure the GC can continue to work after the call + ## to `deallocHeap`. + template deallocCell(x) = + if isCell(x): + # cast to PCell is correct here: + prepareDealloc(cast[PCell](x)) + + if runFinalizers: + when not declared(allObjectsAsProc): + for x in allObjects(gch.region): + deallocCell(x) + else: + var spaceIter: ObjectSpaceIter + while true: + let x = allObjectsAsProc(gch.region, addr spaceIter) + if spaceIter.state < 0: break + deallocCell(x) + + deallocOsPages(gch.region) + zeroMem(addr gch.region, sizeof(gch.region)) + if allowGcAfterwards: + initGC() + +type + GlobalMarkerProc = proc () {.nimcall, benign, raises: [].} +var + globalMarkersLen {.exportc.}: int + globalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] + threadLocalMarkersLen {.exportc.}: int + threadLocalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] + gHeapidGenerator: int + +proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = + if globalMarkersLen <= high(globalMarkers): + globalMarkers[globalMarkersLen] = markerProc + inc globalMarkersLen + else: + cstderr.rawWrite("[GC] cannot register global variable; too many global variables") + rawQuit 1 + +proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = + if threadLocalMarkersLen <= high(threadLocalMarkers): + threadLocalMarkers[threadLocalMarkersLen] = markerProc + inc threadLocalMarkersLen + else: + cstderr.rawWrite("[GC] cannot register thread local variable; too many thread local variables") + rawQuit 1 diff --git a/lib/system/gc_hooks.nim b/lib/system/gc_hooks.nim new file mode 100644 index 000000000..ace62eea0 --- /dev/null +++ b/lib/system/gc_hooks.nim @@ -0,0 +1,53 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Hooks for memory management. Can be used to implement custom garbage +## collectors etc. + +type + GlobalMarkerProc = proc () {.nimcall, benign, raises: [], tags: [].} +var + globalMarkersLen: int + globalMarkers: array[0..3499, GlobalMarkerProc] + threadLocalMarkersLen: int + threadLocalMarkers: array[0..3499, GlobalMarkerProc] + +proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = + if globalMarkersLen <= high(globalMarkers): + globalMarkers[globalMarkersLen] = markerProc + inc globalMarkersLen + else: + cstderr.rawWrite("[GC] cannot register global variable; too many global variables") + rawQuit 1 + +proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = + if threadLocalMarkersLen <= high(threadLocalMarkers): + threadLocalMarkers[threadLocalMarkersLen] = markerProc + inc threadLocalMarkersLen + else: + cstderr.rawWrite("[GC] cannot register thread local variable; too many thread local variables") + rawQuit 1 + +proc traverseGlobals*() = + for i in 0..globalMarkersLen-1: + globalMarkers[i]() + +proc traverseThreadLocals*() = + for i in 0..threadLocalMarkersLen-1: + threadLocalMarkers[i]() + +var + newObjHook*: proc (typ: PNimType, size: int): pointer {.nimcall, tags: [], raises: [], gcsafe.} + traverseObjHook*: proc (p: pointer, op: int) {.nimcall, tags: [], raises: [], gcsafe.} + +proc nimGCvisit(p: pointer, op: int) {.inl, compilerRtl.} = + traverseObjHook(p, op) + +proc newObj(typ: PNimType, size: int): pointer {.inl, compilerRtl.} = + result = newObjHook(typ, size) diff --git a/lib/system/gc_interface.nim b/lib/system/gc_interface.nim new file mode 100644 index 000000000..84145f33a --- /dev/null +++ b/lib/system/gc_interface.nim @@ -0,0 +1,100 @@ +# ----------------- GC interface --------------------------------------------- +const + usesDestructors = defined(gcDestructors) or defined(gcHooks) + +when not usesDestructors: + {.pragma: nodestroy.} + +when hasAlloc: + type + GC_Strategy* = enum ## The strategy the GC should use for the application. + gcThroughput, ## optimize for throughput + gcResponsiveness, ## optimize for responsiveness (default) + gcOptimizeTime, ## optimize for speed + gcOptimizeSpace ## optimize for memory footprint + +when hasAlloc and not defined(js) and not usesDestructors: + proc GC_disable*() {.rtl, inl, benign, raises: [].} + ## Disables the GC. If called `n` times, `n` calls to `GC_enable` + ## are needed to reactivate the GC. + ## + ## Note that in most circumstances one should only disable + ## the mark and sweep phase with + ## `GC_disableMarkAndSweep <#GC_disableMarkAndSweep>`_. + + proc GC_enable*() {.rtl, inl, benign, raises: [].} + ## Enables the GC again. + + proc GC_fullCollect*() {.rtl, benign.} + ## Forces a full garbage collection pass. + ## Ordinary code does not need to call this (and should not). + + proc GC_enableMarkAndSweep*() {.rtl, benign.} + proc GC_disableMarkAndSweep*() {.rtl, benign.} + ## The current implementation uses a reference counting garbage collector + ## with a seldomly run mark and sweep phase to free cycles. The mark and + ## sweep phase may take a long time and is not needed if the application + ## does not create cycles. Thus the mark and sweep phase can be deactivated + ## and activated separately from the rest of the GC. + + proc GC_getStatistics*(): string {.rtl, benign.} + ## Returns an informative string about the GC's activity. This may be useful + ## for tweaking. + + proc GC_ref*[T](x: ref T) {.magic: "GCref", benign.} + proc GC_ref*[T](x: seq[T]) {.magic: "GCref", benign.} + proc GC_ref*(x: string) {.magic: "GCref", benign.} + ## Marks the object `x` as referenced, so that it will not be freed until + ## it is unmarked via `GC_unref`. + ## If called n-times for the same object `x`, + ## n calls to `GC_unref` are needed to unmark `x`. + + proc GC_unref*[T](x: ref T) {.magic: "GCunref", benign.} + proc GC_unref*[T](x: seq[T]) {.magic: "GCunref", benign.} + proc GC_unref*(x: string) {.magic: "GCunref", benign.} + ## See the documentation of `GC_ref <#GC_ref,string>`_. + + proc nimGC_setStackBottom*(theStackBottom: pointer) {.compilerRtl, noinline, benign, raises: [].} + ## Expands operating GC stack range to `theStackBottom`. Does nothing + ## if current stack bottom is already lower than `theStackBottom`. + +when hasAlloc and defined(js): + template GC_disable* = + {.warning: "GC_disable is a no-op in JavaScript".} + + template GC_enable* = + {.warning: "GC_enable is a no-op in JavaScript".} + + template GC_fullCollect* = + {.warning: "GC_fullCollect is a no-op in JavaScript".} + + template GC_setStrategy* = + {.warning: "GC_setStrategy is a no-op in JavaScript".} + + template GC_enableMarkAndSweep* = + {.warning: "GC_enableMarkAndSweep is a no-op in JavaScript".} + + template GC_disableMarkAndSweep* = + {.warning: "GC_disableMarkAndSweep is a no-op in JavaScript".} + + template GC_ref*[T](x: ref T) = + {.warning: "GC_ref is a no-op in JavaScript".} + + template GC_ref*[T](x: seq[T]) = + {.warning: "GC_ref is a no-op in JavaScript".} + + template GC_ref*(x: string) = + {.warning: "GC_ref is a no-op in JavaScript".} + + template GC_unref*[T](x: ref T) = + {.warning: "GC_unref is a no-op in JavaScript".} + + template GC_unref*[T](x: seq[T]) = + {.warning: "GC_unref is a no-op in JavaScript".} + + template GC_unref*(x: string) = + {.warning: "GC_unref is a no-op in JavaScript".} + + template GC_getStatistics*(): string = + {.warning: "GC_getStatistics is a no-op in JavaScript".} + "" diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index d1aecb7a2..c885a6893 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -10,9 +10,6 @@ # A simple mark&sweep garbage collector for Nim. Define the # symbol ``gcUseBitvectors`` to generate a variant of this GC. -when defined(nimCoroutines): - import arch - {.push profiler:off.} const @@ -24,11 +21,14 @@ const rcGrey = 1 # unused rcBlack = 2 -template mulThreshold(x): expr {.immediate.} = x * 2 +template mulThreshold(x): untyped = x * 2 when defined(memProfiler): proc nimProfile(requestedSize: int) +when hasThreadSupport: + import std/sharedlist + type WalkOp = enum waMarkGlobal, # we need to mark conservatively for global marker procs @@ -36,29 +36,32 @@ type # local waMarkPrecise # fast precise marking - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. - GlobalMarkerProc = proc () {.nimcall, benign.} - GcStat = object collections: int # number of performed full collections maxThreshold: int # max threshold that has been set maxStackSize: int # max stack size freedObjects: int # max entries in cycle table - GcStack {.final.} = object - prev: ptr GcStack - next: ptr GcStack - starts: pointer - pos: pointer - maxStackSize: int + GcStack {.final, pure.} = object + when nimCoroutines: + prev: ptr GcStack + next: ptr GcStack + maxStackSize: int # Used to track statistics because we can not use + # GcStat.maxStackSize when multiple stacks exist. + bottom: pointer + + when nimCoroutines: + pos: pointer GcHeap = object # this contains the zero count and # non-zero count table - stack: ptr GcStack - stackBottom: pointer + stack: GcStack + when nimCoroutines: + activeStack: ptr GcStack # current executing coroutine stack. cycleThreshold: int when useCellIds: idGenerator: int @@ -68,92 +71,107 @@ type recGcLock: int # prevent recursion via finalizers; no thread lock region: MemRegion # garbage collected region stat: GcStat + when hasThreadSupport: + toDispose: SharedList[pointer] + gcThreadId: int additionalRoots: CellSeq # dummy roots for GC_ref/unref -{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, - TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} + when defined(nimTracing): + tracing: bool + indentation: int + var gch {.rtlThreadVar.}: GcHeap when not defined(useNimRtl): instantiateForRegion(gch.region) -template acquire(gch: GcHeap) = - when hasThreadSupport and hasSharedHeap: - acquireSys(HeapLock) - -template release(gch: GcHeap) = - when hasThreadSupport and hasSharedHeap: - releaseSys(HeapLock) - template gcAssert(cond: bool, msg: string) = when defined(useGcAssert): if not cond: - echo "[GCASSERT] ", msg - quit 1 + cstderr.rawWrite "[GCASSERT] " + cstderr.rawWrite msg + rawQuit 1 proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) - -proc canbeCycleRoot(c: PCell): bool {.inline.} = - result = ntfAcyclic notin c.typ.flags + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ -proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} = +proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline, compilerproc.} = dest[] = src proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = result = 0 -var - globalMarkersLen: int - globalMarkers: array[0.. 7_000, GlobalMarkerProc] - -proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = - if globalMarkersLen <= high(globalMarkers): - globalMarkers[globalMarkersLen] = markerProc - inc globalMarkersLen - else: - echo "[GC] cannot register global variable; too many global variables" - quit 1 - # this that has to equals zero, otherwise we have to round up UnitsPerPage: when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes -proc prepareDealloc(cell: PCell) = - if cell.typ.finalizer != nil: - # the finalizer could invoke something that - # allocates memory; this could trigger a garbage - # collection. Since we are already collecting we - # prevend recursive entering here by a lock. - # XXX: we should set the cell's children to nil! - inc(gch.recGcLock) - (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) - dec(gch.recGcLock) - -proc nimGCref(p: pointer) {.compilerProc.} = +when defined(nimGcRefLeak): + const + MaxTraceLen = 20 # tracking the last 20 calls is enough + + type + GcStackTrace = object + lines: array[0..MaxTraceLen-1, cstring] + files: array[0..MaxTraceLen-1, cstring] + + proc captureStackTrace(f: PFrame, st: var GcStackTrace) = + const + firstCalls = 5 + var + it = f + i = 0 + total = 0 + while it != nil and i <= high(st.lines)-(firstCalls-1): + # the (-1) is for the "..." entry + st.lines[i] = it.procname + st.files[i] = it.filename + inc(i) + inc(total) + it = it.prev + var b = it + while it != nil: + inc(total) + it = it.prev + for j in 1..total-i-(firstCalls-1): + if b != nil: b = b.prev + if total != i: + st.lines[i] = "..." + st.files[i] = "..." + inc(i) + while b != nil and i <= high(st.lines): + st.lines[i] = b.procname + st.files[i] = b.filename + inc(i) + b = b.prev + + var ax: array[10_000, GcStackTrace] + +proc nimGCref(p: pointer) {.compilerproc.} = # we keep it from being collected by pretending it's not even allocated: when false: when withBitvectors: excl(gch.allocated, usrToCell(p)) else: usrToCell(p).refcount = rcBlack + when defined(nimGcRefLeak): + captureStackTrace(framePtr, ax[gch.additionalRoots.len]) add(gch.additionalRoots, usrToCell(p)) -proc nimGCunref(p: pointer) {.compilerProc.} = +proc nimGCunref(p: pointer) {.compilerproc.} = let cell = usrToCell(p) var L = gch.additionalRoots.len-1 var i = L @@ -161,6 +179,8 @@ proc nimGCunref(p: pointer) {.compilerProc.} = while i >= 0: if d[i] == cell: d[i] = d[L] + when defined(nimGcRefLeak): + ax[i] = ax[L] dec gch.additionalRoots.len break dec(i) @@ -168,6 +188,18 @@ proc nimGCunref(p: pointer) {.compilerProc.} = when withBitvectors: incl(gch.allocated, usrToCell(p)) else: usrToCell(p).refcount = rcWhite +when defined(nimGcRefLeak): + proc writeLeaks() = + for i in 0..gch.additionalRoots.len-1: + c_fprintf(stdout, "[Heap] NEW STACK TRACE\n") + for ii in 0..MaxTraceLen-1: + let line = ax[i].lines[ii] + let file = ax[i].files[ii] + if isNil(line): break + c_fprintf(stdout, "[Heap] %s(%s)\n", file, line) + +include gc_common + proc initGC() = when not defined(useNimRtl): gch.cycleThreshold = InitialThreshold @@ -179,9 +211,13 @@ proc initGC() = when withBitvectors: init(gch.allocated) init(gch.marked) + when hasThreadSupport: + init(gch.toDispose) + gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 + gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -193,7 +229,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -218,21 +254,21 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) - var s = cast[PGenericSeq](d) - if s != nil: - for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) + when not defined(nimSeqsV2): + var d = cast[int](cellToUsr(cell)) + var s = cast[PGenericSeq](d) + if s != nil: + for i in 0..s.len-1: + forAllChildrenAux(cast[pointer](d +% align(GenericSeqSize, cell.typ.base.align) +% i *% cell.typ.base.size), cell.typ.base, op) else: discard proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 - acquire(gch) + incTypeSize typ, size gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) + collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -240,7 +276,6 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = res.filename = framePtr.prev.filename res.line = framePtr.prev.line res.refcount = 0 - release(gch) when withBitvectors: incl(gch.allocated, res) when useCellIds: inc gch.idGenerator @@ -263,64 +298,65 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(typ, size, gch) when defined(memProfiler): nimProfile(size) -proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) - result = newObj(typ, size) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - when defined(memProfiler): nimProfile(size) - proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(typ, size, gch) zeroMem(result, size) when defined(memProfiler): nimProfile(size) -proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) - result = newObj(typ, size) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - when defined(memProfiler): nimProfile(size) - -proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - acquire(gch) - collectCT(gch) - var ol = usrToCell(old) - sysAssert(ol.typ != nil, "growObj: 1") - gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") - - var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size - - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize - copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), - newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - when false: - # this is wrong since seqs can be shared via 'shallow': - when withBitvectors: excl(gch.allocated, ol) - when reallyDealloc: rawDealloc(gch.region, ol) - else: - zeroMem(ol, sizeof(Cell)) - when withBitvectors: incl(gch.allocated, res) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - release(gch) - result = cellToUsr(res) - when defined(memProfiler): nimProfile(newsize-oldsize) +when not defined(nimSeqsV2): + {.push overflowChecks: on.} + proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = + # `newObj` already uses locks, so no need for them here. + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + when defined(memProfiler): nimProfile(size) + + proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + when defined(memProfiler): nimProfile(size) + {.pop.} + + proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = + collectCT(gch, newsize + sizeof(Cell)) + var ol = usrToCell(old) + sysAssert(ol.typ != nil, "growObj: 1") + gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") + + var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) + var elemSize, elemAlign = 1 + if ol.typ.kind != tyString: + elemSize = ol.typ.base.size + elemAlign = ol.typ.base.align + incTypeSize ol.typ, newsize + + var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize + copyMem(res, ol, oldsize + sizeof(Cell)) + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), + newsize-oldsize) + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") + when withBitvectors: incl(gch.allocated, res) + when useCellIds: + inc gch.idGenerator + res.id = gch.idGenerator + result = cellToUsr(res) + when defined(memProfiler): nimProfile(newsize-oldsize) -proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - result = growObj(old, newsize, gch) + proc growObj(old: pointer, newsize: int): pointer {.rtl.} = + result = growObj(old, newsize, gch) {.push profiler:off.} # ----------------- collector ----------------------------------------------- proc mark(gch: var GcHeap, c: PCell) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) when withBitvectors: incl(gch.marked, c) gcAssert gch.tempStack.len == 0, "stack not empty!" @@ -332,29 +368,41 @@ proc mark(gch: var GcHeap, c: PCell) = forAllChildren(d, waMarkPrecise) else: # XXX no 'if c.refCount != rcBlack' here? - c.refCount = rcBlack + when defined(nimTracing): + if gch.tracing: + for i in 1..gch.indentation: c_fprintf(stdout, " ") + c_fprintf(stdout, "start marking %p of type %s ((\n", + c, c.typ.name) + inc gch.indentation, 2 + + c.refcount = rcBlack gcAssert gch.tempStack.len == 0, "stack not empty!" forAllChildren(c, waMarkPrecise) while gch.tempStack.len > 0: dec gch.tempStack.len var d = gch.tempStack.d[gch.tempStack.len] if d.refcount == rcWhite: - d.refCount = rcBlack + d.refcount = rcBlack forAllChildren(d, waMarkPrecise) + when defined(nimTracing): + if gch.tracing: + dec gch.indentation, 2 + for i in 1..gch.indentation: c_fprintf(stdout, " ") + c_fprintf(stdout, "finished marking %p of type %s))\n", + c, c.typ.name) + proc doOperation(p: pointer, op: WalkOp) = if p == nil: return var c: PCell = usrToCell(p) gcAssert(c != nil, "doOperation: 1") case op - of waMarkGlobal: - when hasThreadSupport: - # could point to a cell which we don't own and don't want to touch/trace - if isAllocatedPtr(gch.region, c): - mark(gch, c) + of waMarkGlobal: mark(gch, c) + of waMarkPrecise: + when defined(nimTracing): + if c.refcount == rcWhite: mark(gch, c) else: - mark(gch, c) - of waMarkPrecise: add(gch.tempStack, c) + add(gch.tempStack, c) proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, WalkOp(op)) @@ -380,38 +428,40 @@ proc sweep(gch: var GcHeap) = if c.refcount == rcBlack: c.refcount = rcWhite else: freeCyclicCell(gch, c) -when false: - proc newGcInvariant*() = - for x in allObjects(gch.region): - if isCell(x): - var c = cast[PCell](x) - if c.typ == nil: - writeStackTrace() - quit 1 - proc markGlobals(gch: var GcHeap) = - for i in 0 .. < globalMarkersLen: globalMarkers[i]() + if gch.gcThreadId == 0: + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- globals marking phase:\n") + for i in 0 .. globalMarkersLen-1: globalMarkers[i]() + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- thread locals marking phase:\n") + for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- additional roots marking phase:\n") let d = gch.additionalRoots.d - for i in 0 .. < gch.additionalRoots.len: mark(gch, d[i]) + for i in 0 .. gch.additionalRoots.len-1: mark(gch, d[i]) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: mark(gch, objStart) -include gc_common - proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = forEachStackSlot(gch, gcMark) proc collectCTBody(gch: var GcHeap) = - when not defined(nimCoroutines): + when not nimCoroutines: gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) + when defined(nimTracing): + if gch.tracing: + c_fprintf(stdout, "------- stack marking phase:\n") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) markGlobals(gch) @@ -425,22 +475,21 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) sysAssert(allocInv(gch.region), "collectCT: end") -proc collectCT(gch: var GcHeap) = - if getOccupiedMem(gch.region) >= gch.cycleThreshold and gch.recGcLock == 0: +proc collectCT(gch: var GcHeap; size: int) = + let fmem = getFreeMem(gch.region) + if (getOccupiedMem(gch.region) >= gch.cycleThreshold or + size > fmem and fmem > InitialThreshold) and gch.recGcLock == 0: collectCTBody(gch) when not defined(useNimRtl): proc GC_disable() = - when hasThreadSupport and hasSharedHeap: - atomicInc(gch.recGcLock, 1) - else: - inc(gch.recGcLock) + inc(gch.recGcLock) proc GC_enable() = - if gch.recGcLock > 0: - when hasThreadSupport and hasSharedHeap: - atomicDec(gch.recGcLock, 1) - else: - dec(gch.recGcLock) + when defined(nimDoesntTrackDefects): + if gch.recGcLock <= 0: + raise newException(AssertionDefect, + "API usage error: GC_enable called but GC is already enabled") + dec(gch.recGcLock) proc GC_setStrategy(strategy: GC_Strategy) = discard @@ -448,30 +497,30 @@ when not defined(useNimRtl): gch.cycleThreshold = InitialThreshold proc GC_disableMarkAndSweep() = - gch.cycleThreshold = high(gch.cycleThreshold)-1 + gch.cycleThreshold = high(typeof(gch.cycleThreshold))-1 # set to the max value to suppress the cycle detector + when defined(nimTracing): + proc GC_logTrace*() = + gch.tracing = true + proc GC_fullCollect() = - acquire(gch) - var oldThreshold = gch.cycleThreshold + let oldThreshold = gch.cycleThreshold gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) + collectCT(gch, 0) gch.cycleThreshold = oldThreshold - release(gch) proc GC_getStatistics(): string = - GC_disable() result = "[GC] total memory: " & $getTotalMem() & "\n" & "[GC] occupied memory: " & $getOccupiedMem() & "\n" & "[GC] collections: " & $gch.stat.collections & "\n" & "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & "[GC] freed objects: " & $gch.stat.freedObjects & "\n" - when defined(nimCoroutines): - result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" + when nimCoroutines: + result.add "[GC] number of stacks: " & $gch.stack.len & "\n" for stack in items(gch.stack): - result = result & "[GC] stack " & stack.starts.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" + result.add "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" else: - result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" - GC_enable() + result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" {.pop.} diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim new file mode 100644 index 000000000..d96de7eac --- /dev/null +++ b/lib/system/gc_regions.nim @@ -0,0 +1,442 @@ +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# "Stack GC" for embedded devices or ultra performance requirements. +import std/private/syslocks + +when defined(memProfiler): + proc nimProfile(requestedSize: int) {.benign.} + +when defined(useMalloc): + proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + proc emalloc(size: int): pointer {.importc: "malloc", header: "<stdlib.h>".} + proc efree(mem: pointer) {.importc: "free", header: "<stdlib.h>".} + + proc osAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osTryAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + efree(p) + +else: + include osalloc + +# We manage memory as a thread local stack. Since the allocation pointer +# is detached from the control flow pointer, this model is vastly more +# useful than the traditional programming model while almost as safe. +# Individual objects can also be deleted but no coalescing is performed. +# Stacks can also be moved from one thread to another. + +# We also support 'finalizers'. + +type + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + # A ref type can have a finalizer that is called before the object's + # storage is freed. + + AlignType = BiggestFloat + ObjHeader = object + typ: PNimType + nextFinal: ptr ObjHeader # next object with finalizer + + Chunk = ptr BaseChunk + BaseChunk = object + next: Chunk + size: int + head, tail: ptr ObjHeader # first and last object in chunk that + # has a finalizer attached to it + +const + MaxSmallObject = 128 + +type + FreeEntry = ptr object + next: FreeEntry + SizedFreeEntry = ptr object + next: SizedFreeEntry + size: int + StackPtr = object + bump: pointer + remaining: int + current: Chunk + + MemRegion* = object + remaining: int + bump: pointer + head, tail: Chunk + nextChunkSize, totalSize: int + when false: + freeLists: array[MaxSmallObject div MemAlign, FreeEntry] + holes: SizedFreeEntry + when hasThreadSupport: + lock: SysLock + + SeqHeader = object # minor hack ahead: Since we know that seqs + # and strings cannot have finalizers, we use the field + # instead for a 'region' field so that they can grow + # and shrink safely. + typ: PNimType + region: ptr MemRegion + +var + tlRegion {.threadvar.}: MemRegion +# tempStrRegion {.threadvar.}: MemRegion # not yet used + +template withRegion*(r: var MemRegion; body: untyped) = + let oldRegion = tlRegion + tlRegion = r + try: + body + finally: + r = tlRegion + tlRegion = oldRegion + +template inc(p: pointer, s: int) = + p = cast[pointer](cast[int](p) +% s) + +template dec(p: pointer, s: int) = + p = cast[pointer](cast[int](p) -% s) + +template `+!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +template `-!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) -% s) + +const nimMinHeapPages {.intdefine.} = 4 + +proc allocSlowPath(r: var MemRegion; size: int) = + # we need to ensure that the underlying linked list + # stays small. Say we want to grab 16GB of RAM with some + # exponential growth function. So we allocate 16KB, then + # 32 KB, 64 KB, 128KB, 256KB, 512KB, 1MB, 2MB, 4MB, + # 8MB, 16MB, 32MB, 64MB, 128MB, 512MB, 1GB, 2GB, 4GB, 8GB, + # 16GB --> list contains only 20 elements! That's reasonable. + if (r.totalSize and 1) == 0: + r.nextChunkSize = if r.totalSize < 64 * 1024: PageSize*nimMinHeapPages + else: r.nextChunkSize*2 + var s = roundup(size+sizeof(BaseChunk), PageSize) + var fresh: Chunk + if s > r.nextChunkSize: + fresh = cast[Chunk](osAllocPages(s)) + else: + fresh = cast[Chunk](osTryAllocPages(r.nextChunkSize)) + if fresh == nil: + fresh = cast[Chunk](osAllocPages(s)) + # lowest bit in totalSize is the "don't increase nextChunkSize" + inc r.totalSize + else: + s = r.nextChunkSize + fresh.size = s + fresh.head = nil + fresh.tail = nil + fresh.next = nil + inc r.totalSize, s + let old = r.tail + if old == nil: + r.head = fresh + else: + r.tail.next = fresh + r.bump = fresh +! sizeof(BaseChunk) + r.tail = fresh + r.remaining = s - sizeof(BaseChunk) + +proc allocFast(r: var MemRegion; size: int): pointer = + when false: + if size <= MaxSmallObject: + var it = r.freeLists[size div MemAlign] + if it != nil: + r.freeLists[size div MemAlign] = it.next + return pointer(it) + else: + var it = r.holes + var prev: SizedFreeEntry = nil + while it != nil: + if it.size >= size: + if prev != nil: prev.next = it.next + else: r.holes = it.next + return pointer(it) + prev = it + it = it.next + let size = roundup(size, MemAlign) + if size > r.remaining: + allocSlowPath(r, size) + sysAssert(size <= r.remaining, "size <= r.remaining") + dec(r.remaining, size) + result = r.bump + inc r.bump, size + +proc runFinalizers(c: Chunk) = + var it = c.head + while it != nil: + # indivually freed objects with finalizer stay in the list, but + # their typ is nil then: + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + it = it.nextFinal + +proc runFinalizers(c: Chunk; newbump: pointer) = + var it = c.head + var prev: ptr ObjHeader = nil + while it != nil: + let nxt = it.nextFinal + if it >= newbump: + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + elif prev != nil: + prev.nextFinal = nil + prev = it + it = nxt + +proc dealloc(r: var MemRegion; p: pointer; size: int) = + let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader)) + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(p) + it.typ = nil + # it is beneficial to not use the free lists here: + if r.bump -! size == p: + dec r.bump, size + when false: + if size <= MaxSmallObject: + let it = cast[FreeEntry](p) + it.next = r.freeLists[size div MemAlign] + r.freeLists[size div MemAlign] = it + else: + let it = cast[SizedFreeEntry](p) + it.size = size + it.next = r.holes + r.holes = it + +proc deallocAll(r: var MemRegion; head: Chunk) = + var it = head + while it != nil: + let nxt = it.next + runFinalizers(it) + dec r.totalSize, it.size + osDeallocPages(it, it.size) + it = nxt + +proc deallocAll*(r: var MemRegion) = + deallocAll(r, r.head) + zeroMem(addr r, sizeof r) + +proc obstackPtr*(r: MemRegion): StackPtr = + result.bump = r.bump + result.remaining = r.remaining + result.current = r.tail + +template computeRemaining(r): untyped = + r.tail.size -% (cast[int](r.bump) -% cast[int](r.tail)) + +proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = + # free everything after 'sp': + if sp.current != nil and sp.current.next != nil: + deallocAll(r, sp.current.next) + sp.current.next = nil + when false: + # better leak this memory than be sorry: + for i in 0..high(r.freeLists): r.freeLists[i] = nil + r.holes = nil + if r.tail != nil: runFinalizers(r.tail, sp.bump) + + r.bump = sp.bump + r.tail = sp.current + r.remaining = sp.remaining + +proc obstackPtr*(): StackPtr = tlRegion.obstackPtr() +proc setObstackPtr*(sp: StackPtr) = tlRegion.setObstackPtr(sp) +proc deallocAll*() = tlRegion.deallocAll() + +proc deallocOsPages(r: var MemRegion) = r.deallocAll() + +when false: + let obs = obstackPtr() + try: + body + finally: + setObstackPtr(obs) + +template withScratchRegion*(body: untyped) = + let oldRegion = tlRegion + tlRegion = MemRegion() + try: + body + finally: + deallocAll() + tlRegion = oldRegion + +when false: + proc joinRegion*(dest: var MemRegion; src: MemRegion) = + # merging is not hard. + if dest.head.isNil: + dest.head = src.head + else: + dest.tail.next = src.head + dest.tail = src.tail + dest.bump = src.bump + dest.remaining = src.remaining + dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize) + inc dest.totalSize, src.totalSize + +proc isOnHeap*(r: MemRegion; p: pointer): bool = + # the tail chunk is the largest, so check it first. It's also special + # in that contains the current bump pointer: + if r.tail >= p and p < r.bump: + return true + var it = r.head + while it != r.tail: + if it >= p and p <= it+!it.size: return true + it = it.next + +proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = + var res = cast[ptr ObjHeader](allocFast(r, size + sizeof(ObjHeader))) + res.typ = typ + if typ.finalizer != nil: + res.nextFinal = r.head.head + r.head.head = res + result = res +! sizeof(ObjHeader) + +proc rawNewSeq(r: var MemRegion, typ: PNimType, size: int): pointer = + var res = cast[ptr SeqHeader](allocFast(r, size + sizeof(SeqHeader))) + res.typ = typ + res.region = addr(r) + result = res +! sizeof(SeqHeader) + +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = + sysAssert typ.kind notin {tySequence, tyString}, "newObj cannot be used to construct seqs" + result = rawNewObj(tlRegion, typ, size) + zeroMem(result, size) + when defined(memProfiler): nimProfile(size) + +proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = + sysAssert typ.kind notin {tySequence, tyString}, "newObj cannot be used to construct seqs" + result = rawNewObj(tlRegion, typ, size) + when defined(memProfiler): nimProfile(size) + +{.push overflowChecks: on.} +proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = roundup(align(GenericSeqSize, typ.base.align) + len * typ.base.size, MemAlign) + result = rawNewSeq(tlRegion, typ, size) + zeroMem(result, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc newStr(typ: PNimType, len: int; init: bool): pointer {.compilerRtl.} = + let size = roundup(len + GenericSeqSize, MemAlign) + result = rawNewSeq(tlRegion, typ, size) + if init: zeroMem(result, size) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = len +{.pop.} + +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(tlRegion, typ, size) + zeroMem(result, size) + +proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = + result = newSeq(typ, len) + +proc growObj(regionUnused: var MemRegion; old: pointer, newsize: int): pointer = + let sh = cast[ptr SeqHeader](old -! sizeof(SeqHeader)) + let typ = sh.typ + result = rawNewSeq(sh.region[], typ, + roundup(newsize, MemAlign)) + let elemSize = if typ.kind == tyString: 1 else: typ.base.size + let elemAlign = if typ.kind == tyString: 1 else: typ.base.align + let oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize + zeroMem(result +! oldsize, newsize-oldsize) + copyMem(result, old, oldsize) + dealloc(sh.region[], old, roundup(oldsize, MemAlign)) + +proc growObj(old: pointer, newsize: int): pointer {.rtl.} = + result = growObj(tlRegion, old, newsize) + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, + deprecated: "old compiler compat".} = asgnRef(dest, src) + +proc allocImpl(size: Natural): pointer = + result = c_malloc(cast[csize_t](size)) + if result == nil: raiseOutOfMem() +proc alloc0Impl(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc reallocImpl(p: pointer, newsize: Natural): pointer = + result = c_realloc(p, cast[csize_t](newsize)) + if result == nil: raiseOutOfMem() +proc realloc0Impl(p: pointer, oldsize, newsize: Natural): pointer = + result = c_realloc(p, cast[csize_t](newsize)) + if result == nil: raiseOutOfMem() + if newsize > oldsize: + zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) +proc deallocImpl(p: pointer) = c_free(p) + +proc alloc0(r: var MemRegion; size: Natural): pointer = + # ignore the region. That is correct for the channels module + # but incorrect in general. XXX + result = alloc0(size) + +proc alloc(r: var MemRegion; size: Natural): pointer = + # ignore the region. That is correct for the channels module + # but incorrect in general. XXX + result = alloc(size) + +proc dealloc(r: var MemRegion; p: pointer) = dealloc(p) + +proc allocSharedImpl(size: Natural): pointer = + result = c_malloc(cast[csize_t](size)) + if result == nil: raiseOutOfMem() +proc allocShared0Impl(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc reallocSharedImpl(p: pointer, newsize: Natural): pointer = + result = c_realloc(p, cast[csize_t](newsize)) + if result == nil: raiseOutOfMem() +proc reallocShared0Impl(p: pointer, oldsize, newsize: Natural): pointer = + result = c_realloc(p, cast[csize_t](newsize)) + if result == nil: raiseOutOfMem() + if newsize > oldsize: + zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) +proc deallocSharedImpl(p: pointer) = c_free(p) + +when hasThreadSupport: + proc getFreeSharedMem(): int = 0 + proc getTotalSharedMem(): int = 0 + proc getOccupiedSharedMem(): int = 0 + +proc GC_disable() = discard +proc GC_enable() = discard +proc GC_fullCollect() = discard +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard +proc GC_getStatistics(): string = return "" + +proc getOccupiedMem(): int = + result = tlRegion.totalSize - tlRegion.remaining +proc getFreeMem(): int = tlRegion.remaining +proc getTotalMem(): int = + result = tlRegion.totalSize + +proc getOccupiedMem*(r: MemRegion): int = + result = r.totalSize - r.remaining +proc getFreeMem*(r: MemRegion): int = r.remaining +proc getTotalMem*(r: MemRegion): int = + result = r.totalSize + +proc nimGC_setStackBottom(theStackBottom: pointer) = discard + +proc nimGCref(x: pointer) {.compilerproc.} = discard +proc nimGCunref(x: pointer) {.compilerproc.} = discard diff --git a/lib/system/hti.nim b/lib/system/hti.nim index bfb13059e..a26aff982 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -7,12 +7,6 @@ # distribution, for details about the copyright. # -when declared(NimString): - # we are in system module: - {.pragma: codegenType, compilerproc.} -else: - {.pragma: codegenType, importc.} - type # This should be the same as ast.TTypeKind # many enum fields are not used at runtime @@ -23,8 +17,8 @@ type tyEmpty, tyArrayConstr, tyNil, - tyExpr, - tyStmt, + tyUntyped, + tyTyped, tyTypeDesc, tyGenericInvocation, # ``T[a, b]`` for types to invoke tyGenericBody, # ``T[a, b, body]`` last parameter is the body @@ -46,7 +40,7 @@ type tyPointer, tyOpenArray, tyString, - tyCString, + tyCstring, tyForward, tyInt, tyInt8, @@ -62,31 +56,68 @@ type tyUInt16, tyUInt32, tyUInt64, - tyBigNum, + tyOwned, tyUnused1, tyUnused2, + tyVarargsHidden, + tyUncheckedArray, + tyErrorHidden, + tyBuiltInTypeClassHidden, + tyUserTypeClassHidden, + tyUserTypeClassInstHidden, + tyCompositeTypeClassHidden, + tyInferredHidden, + tyAndHidden, tyOrHidden, tyNotHidden, + tyAnythingHidden, + tyStaticHidden, + tyFromExprHidden, + tyOptDeprecated, + tyVoidHidden TNimNodeKind = enum nkNone, nkSlot, nkList, nkCase - TNimNode {.codegenType.} = object + TNimNode {.compilerproc.} = object kind: TNimNodeKind offset: int typ: ptr TNimType name: cstring len: int - sons: ptr array [0..0x7fff, ptr TNimNode] + sons: ptr array[0x7fff, ptr TNimNode] TNimTypeFlag = enum ntfNoRefs = 0, # type contains no tyRef, tySequence, tyString ntfAcyclic = 1, # type cannot form a cycle ntfEnumHole = 2 # enum has holes and thus `$` for them needs the slow # version - TNimType {.codegenType.} = object - size: int + TNimType {.compilerproc.} = object + when defined(gcHooks): + head*: pointer + size*: int + align*: int kind: TNimKind flags: set[TNimTypeFlag] - base: ptr TNimType + base*: ptr TNimType node: ptr TNimNode # valid for tyRecord, tyObject, tyTuple, tyEnum - finalizer: pointer # the finalizer for the type - marker: proc (p: pointer, op: int) {.nimcall, benign.} # marker proc for GC - deepcopy: proc (p: pointer): pointer {.nimcall, benign.} - PNimType = ptr TNimType + finalizer*: pointer # the finalizer for the type + marker*: proc (p: pointer, op: int) {.nimcall, benign, tags: [], raises: [].} # marker proc for GC + deepcopy: proc (p: pointer): pointer {.nimcall, benign, tags: [], raises: [].} + when defined(nimSeqsV2): + typeInfoV2*: pointer + when defined(nimTypeNames): + name: cstring + nextType: ptr TNimType + instances: int # count the number of instances + sizes: int # sizes of all instances in bytes + +when defined(gcHooks): + type + PNimType* = ptr TNimType +else: + type + PNimType = ptr TNimType + +when defined(nimTypeNames): + # Declare this variable only once in system.nim + when declared(ThisIsSystem): + var nimTypeRoot {.compilerproc.}: PNimType + else: + var nimTypeRoot {.importc.}: PNimType # node.len may be the ``first`` element of a set diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 3caeefcbc..3bf0b9893 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -30,12 +30,13 @@ when defined(createNimRtl): {.pragma: inl.} {.pragma: compilerRtl, compilerproc, exportc: "nimrtl_$1", dynlib.} elif defined(useNimRtl): - when defined(windows): - const nimrtl* = "nimrtl.dll" - elif defined(macosx): - const nimrtl* = "libnimrtl.dylib" - else: - const nimrtl* = "libnimrtl.so" + #[ + `{.rtl.}` should only be used for non-generic procs. + ]# + const nimrtl* = + when defined(windows): "nimrtl.dll" + elif defined(macosx): "libnimrtl.dylib" + else: "libnimrtl.so" {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, importc: "nimrtl_$1", dynlib: nimrtl.} @@ -44,15 +45,6 @@ else: {.pragma: inl, inline.} {.pragma: compilerRtl, compilerproc.} -when not defined(nimsuperops): - {.pragma: operator.} +{.pragma: benign, gcsafe.} -when defined(nimlocks): - {.pragma: benign, gcsafe, locks: 0.} -else: - {.pragma: benign, gcsafe.} - -when defined(nimTableGet): - {.pragma: deprecatedGet, deprecated.} -else: - {.pragma: deprecatedGet.} +{.push sinkInference: on.} diff --git a/lib/system/indexerrors.nim b/lib/system/indexerrors.nim new file mode 100644 index 000000000..6a8cb8a0a --- /dev/null +++ b/lib/system/indexerrors.nim @@ -0,0 +1,15 @@ +# imported by other modules, unlike helpers.nim which is included +# xxx this is now included instead of imported, we should import instead + +template formatErrorIndexBound*[T](i, a, b: T): string = + when defined(standalone): + "indexOutOfBounds" + else: + if b < a: "index out of bounds, the container is empty" + else: "index " & $i & " not in " & $a & " .. " & $b + +template formatErrorIndexBound*[T](i, n: T): string = + formatErrorIndexBound(i, 0, n) + +template formatFieldDefect*(f, discVal): string = + f & discVal & "'" diff --git a/lib/system/indices.nim b/lib/system/indices.nim new file mode 100644 index 000000000..f2bad2528 --- /dev/null +++ b/lib/system/indices.nim @@ -0,0 +1,164 @@ +when not defined(nimHasSystemRaisesDefect): + {.pragma: systemRaisesDefect.} + +type + BackwardsIndex* = distinct int ## Type that is constructed by `^` for + ## reversed array accesses. + ## (See `^ template <#^.t,int>`_) + +template `^`*(x: int): BackwardsIndex = BackwardsIndex(x) + ## Builtin `roof`:idx: operator that can be used for convenient array access. + ## `a[^x]` is a shortcut for `a[a.len-x]`. + ## + ## ```nim + ## let + ## a = [1, 3, 5, 7, 9] + ## b = "abcdefgh" + ## + ## echo a[^1] # => 9 + ## echo b[^2] # => g + ## ``` + +proc `[]`*[T](s: openArray[T]; i: BackwardsIndex): T {.inline, systemRaisesDefect.} = + system.`[]`(s, s.len - int(i)) + +proc `[]`*[Idx, T](a: array[Idx, T]; i: BackwardsIndex): T {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] +proc `[]`*(s: string; i: BackwardsIndex): char {.inline, systemRaisesDefect.} = s[s.len - int(i)] + +proc `[]`*[T](s: var openArray[T]; i: BackwardsIndex): var T {.inline, systemRaisesDefect.} = + system.`[]`(s, s.len - int(i)) +proc `[]`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex): var T {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] +proc `[]`*(s: var string; i: BackwardsIndex): var char {.inline, systemRaisesDefect.} = s[s.len - int(i)] + +proc `[]=`*[T](s: var openArray[T]; i: BackwardsIndex; x: T) {.inline, systemRaisesDefect.} = + system.`[]=`(s, s.len - int(i), x) +proc `[]=`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex; x: T) {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] = x +proc `[]=`*(s: var string; i: BackwardsIndex; x: char) {.inline, systemRaisesDefect.} = + s[s.len - int(i)] = x + +template `..^`*(a, b: untyped): untyped = + ## A shortcut for `.. ^` to avoid the common gotcha that a space between + ## '..' and '^' is required. + a .. ^b + +template `..<`*(a, b: untyped): untyped = + ## A shortcut for `a .. pred(b)`. + ## ```nim + ## for i in 5 ..< 9: + ## echo i # => 5; 6; 7; 8 + ## ``` + a .. (when b is BackwardsIndex: succ(b) else: pred(b)) + +template `[]`*(s: string; i: int): char = arrGet(s, i) +template `[]=`*(s: string; i: int; val: char) = arrPut(s, i, val) + +template `^^`(s, i: untyped): untyped = + (when i is BackwardsIndex: s.len - int(i) else: int(i)) + +template spliceImpl(s, a, L, b: typed): untyped = + # make room for additional elements or cut: + var shift = b.len - max(0,L) # ignore negative slice size + var newLen = s.len + shift + if shift > 0: + # enlarge: + setLen(s, newLen) + for i in countdown(newLen-1, a+b.len): movingCopy(s[i], s[i-shift]) + else: + for i in countup(a+b.len, newLen-1): movingCopy(s[i], s[i-shift]) + # cut down: + setLen(s, newLen) + # fill the hole: + for i in 0 ..< b.len: s[a+i] = b[i] + +proc `[]`*[T, U: Ordinal](s: string, x: HSlice[T, U]): string {.inline, systemRaisesDefect.} = + ## Slice operation for strings. + ## Returns the inclusive range `[s[x.a], s[x.b]]`: + ## ```nim + ## var s = "abcdef" + ## assert s[1..3] == "bcd" + ## ``` + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + result = newString(L) + for i in 0 ..< L: result[i] = s[i + a] + +proc `[]=`*[T, U: Ordinal](s: var string, x: HSlice[T, U], b: string) {.systemRaisesDefect.} = + ## Slice assignment for strings. + ## + ## If `b.len` is not exactly the number of elements that are referred to + ## by `x`, a `splice`:idx: is performed: + ## + runnableExamples: + var s = "abcdefgh" + s[1 .. ^2] = "xyz" + assert s == "axyzh" + + var a = s ^^ x.a + var L = (s ^^ x.b) - a + 1 + if L == b.len: + for i in 0..<L: s[i+a] = b[i] + else: + spliceImpl(s, a, L, b) + +proc `[]`*[Idx, T; U, V: Ordinal](a: array[Idx, T], x: HSlice[U, V]): seq[T] {.systemRaisesDefect.} = + ## Slice operation for arrays. + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + ## ```nim + ## var a = [1, 2, 3, 4] + ## assert a[0..2] == @[1, 2, 3] + ## ``` + ## + ## See also: + ## * `toOpenArray(array[I, T];I,I) <#toOpenArray,array[I,T],I,I>`_ + let xa = a ^^ x.a + let L = (a ^^ x.b) - xa + 1 + result = newSeq[T](L) + for i in 0..<L: result[i] = a[Idx(i + xa)] + +proc `[]=`*[Idx, T; U, V: Ordinal](a: var array[Idx, T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} = + ## Slice assignment for arrays. + ## ```nim + ## var a = [10, 20, 30, 40, 50] + ## a[1..2] = @[99, 88] + ## assert a == [10, 99, 88, 40, 50] + ## ``` + let xa = a ^^ x.a + let L = (a ^^ x.b) - xa + 1 + if L == b.len: + for i in 0..<L: a[Idx(i + xa)] = b[i] + else: + sysFatal(RangeDefect, "different lengths for slice assignment") + +proc `[]`*[T; U, V: Ordinal](s: openArray[T], x: HSlice[U, V]): seq[T] {.systemRaisesDefect.} = + ## Slice operation for sequences. + ## Returns the inclusive range `[s[x.a], s[x.b]]`: + ## ```nim + ## var s = @[1, 2, 3, 4] + ## assert s[0..2] == @[1, 2, 3] + ## ``` + ## + ## See also: + ## * `toOpenArray(openArray[T];int,int) <#toOpenArray,openArray[T],int,int>`_ + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + newSeq(result, L) + for i in 0 ..< L: result[i] = s[i + a] + +proc `[]=`*[T; U, V: Ordinal](s: var seq[T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} = + ## Slice assignment for sequences. + ## + ## If `b.len` is not exactly the number of elements that are referred to + ## by `x`, a `splice`:idx: is performed. + runnableExamples: + var s = @"abcdefgh" + s[1 .. ^2] = @"xyz" + assert s == @"axyzh" + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + if L == b.len: + for i in 0 ..< L: s[i+a] = b[i] + else: + spliceImpl(s, a, L, b) diff --git a/lib/system/integerops.nim b/lib/system/integerops.nim new file mode 100644 index 000000000..4ef3594f1 --- /dev/null +++ b/lib/system/integerops.nim @@ -0,0 +1,132 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Integer arithmetic with overflow checking. Uses +# intrinsics or inline assembler. + +proc raiseOverflow {.compilerproc, noinline.} = + # a single proc to reduce code size to a minimum + sysFatal(OverflowDefect, "over- or underflow") + +proc raiseDivByZero {.compilerproc, noinline.} = + sysFatal(DivByZeroDefect, "division by zero") + +{.pragma: nimbaseH, importc, nodecl, noSideEffect, compilerproc.} + +when not defined(nimEmulateOverflowChecks): + # take the #define from nimbase.h + + proc nimAddInt(a, b: int, res: ptr int): bool {.nimbaseH.} + proc nimSubInt(a, b: int, res: ptr int): bool {.nimbaseH.} + proc nimMulInt(a, b: int, res: ptr int): bool {.nimbaseH.} + + proc nimAddInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + proc nimSubInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + proc nimMulInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + +# unary minus and 'abs' not required here anymore and are directly handled +# in the code generator. +# 'nimModInt' does exist in nimbase.h without check as we moved the +# check for 0 to the codgen. +proc nimModInt(a, b: int; res: ptr int): bool {.nimbaseH.} + +proc nimModInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + +# Platform independent versions. + +template addImplFallback(name, T, U) {.dirty.} = + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) + cast[U](b)) + if (r xor a) >= T(0) or (r xor b) >= T(0): + res[] = r + else: + result = true + +addImplFallback(nimAddInt, int, uint) +addImplFallback(nimAddInt64, int64, uint64) + +template subImplFallback(name, T, U) {.dirty.} = + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) - cast[U](b)) + if (r xor a) >= 0 or (r xor not b) >= 0: + res[] = r + else: + result = true + +subImplFallback(nimSubInt, int, uint) +subImplFallback(nimSubInt64, int64, uint64) + +template mulImplFallback(name, T, U, conv) {.dirty.} = + # + # This code has been inspired by Python's source code. + # The native int product x*y is either exactly right or *way* off, being + # just the last n bits of the true product, where n is the number of bits + # in an int (the delivered product is the true product plus i*2**n for + # some integer i). + # + # The native float64 product x*y is subject to three + # rounding errors: on a sizeof(int)==8 box, each cast to double can lose + # info, and even on a sizeof(int)==4 box, the multiplication can lose info. + # But, unlike the native int product, it's not in *range* trouble: even + # if sizeof(int)==32 (256-bit ints), the product easily fits in the + # dynamic range of a float64. So the leading 50 (or so) bits of the float64 + # product are correct. + # + # We check these two ways against each other, and declare victory if + # they're approximately the same. Else, because the native int product is + # the only one that can lose catastrophic amounts of information, it's the + # native int product that must have overflowed. + # + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) * cast[U](b)) + let floatProd = conv(a) * conv(b) + let resAsFloat = conv(r) + # Fast path for normal case: small multiplicands, and no info + # is lost in either method. + if resAsFloat == floatProd: + res[] = r + else: + # Somebody somewhere lost info. Close enough, or way off? Note + # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). + # The difference either is or isn't significant compared to the + # true value (of which floatProd is a good approximation). + + # abs(diff)/abs(prod) <= 1/32 iff + # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" + if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): + res[] = r + else: + result = true + +mulImplFallback(nimMulInt, int, uint, toFloat) +mulImplFallback(nimMulInt64, int64, uint64, toBiggestFloat) + + +template divImplFallback(name, T) {.dirty.} = + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + # we moved the b == 0 case out into the codegen. + if a == low(T) and b == T(-1): + result = true + else: + res[] = a div b + +divImplFallback(nimDivInt, int) +divImplFallback(nimDivInt64, int64) + +proc raiseFloatInvalidOp {.compilerproc, noinline.} = + sysFatal(FloatInvalidOpDefect, "FPU operation caused a NaN result") + +proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = + if x > 0.0: + sysFatal(FloatOverflowDefect, "FPU operation caused an overflow") + else: + sysFatal(FloatUnderflowDefect, "FPU operations caused an underflow") diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim new file mode 100644 index 000000000..125bee98f --- /dev/null +++ b/lib/system/iterators.nim @@ -0,0 +1,353 @@ +## Default iterators for some Nim types. + +when defined(nimPreviewSlimSystem): + import std/assertions + +when not defined(nimNoLentIterators): + template lent2(T): untyped = lent T +else: + template lent2(T): untyped = T + +template unCheckedInc(x) = + {.push overflowChecks: off.} + inc(x) + {.pop.} + +iterator items*[T: not char](a: openArray[T]): lent2 T {.inline.} = + ## Iterates over each item of `a`. + var i = 0 + while i < len(a): + yield a[i] + unCheckedInc(i) + +iterator items*[T: char](a: openArray[T]): T {.inline.} = + ## Iterates over each item of `a`. + # a VM bug currently prevents taking address of openArray[char] + # elements converted from a string (would fail in `tests/misc/thallo.nim`) + # in any case there's no performance advantage of returning char by address. + var i = 0 + while i < len(a): + yield a[i] + unCheckedInc(i) + +iterator mitems*[T](a: var openArray[T]): var T {.inline.} = + ## Iterates over each item of `a` so that you can modify the yielded value. + var i = 0 + while i < len(a): + yield a[i] + unCheckedInc(i) + +iterator items*[IX, T](a: array[IX, T]): T {.inline.} = + ## Iterates over each item of `a`. + when a.len > 0: + var i = low(IX) + while true: + yield a[i] + if i >= high(IX): break + unCheckedInc(i) + +iterator mitems*[IX, T](a: var array[IX, T]): var T {.inline.} = + ## Iterates over each item of `a` so that you can modify the yielded value. + when a.len > 0: + var i = low(IX) + while true: + yield a[i] + if i >= high(IX): break + unCheckedInc(i) + +iterator items*[T](a: set[T]): T {.inline.} = + ## Iterates over each element of `a`. `items` iterates only over the + ## elements that are really in the set (and not over the ones the set is + ## able to hold). + var i = low(T).int + while i <= high(T).int: + when T is enum and not defined(js): + if cast[T](i) in a: yield cast[T](i) + else: + if T(i) in a: yield T(i) + unCheckedInc(i) + +iterator items*(a: cstring): char {.inline.} = + ## Iterates over each item of `a`. + runnableExamples: + from std/sequtils import toSeq + assert toSeq("abc\0def".cstring) == @['a', 'b', 'c'] + assert toSeq("abc".cstring) == @['a', 'b', 'c'] + #[ + assert toSeq(nil.cstring) == @[] # xxx fails with SIGSEGV + this fails with SIGSEGV; unclear whether we want to instead yield nothing + or pay a small price to check for `nil`, a benchmark is needed. Note that + other procs support `nil`. + ]# + template impl() = + var i = 0 + let n = len(a) + while i < n: + yield a[i] + unCheckedInc(i) + when defined(js): impl() + else: + when nimvm: + # xxx `cstring` should behave like c backend instead. + impl() + else: + var i = 0 + while a[i] != '\0': + yield a[i] + unCheckedInc(i) + +iterator mitems*(a: var cstring): var char {.inline.} = + ## Iterates over each item of `a` so that you can modify the yielded value. + # xxx this should give CT error in js RT. + runnableExamples: + from std/sugar import collect + var a = "abc\0def" + prepareMutation(a) + var b = a.cstring + let s = collect: + for bi in mitems(b): + if bi == 'b': bi = 'B' + bi + assert s == @['a', 'B', 'c'] + assert b == "aBc" + assert a == "aBc\0def" + + template impl() = + var i = 0 + let n = len(a) + while i < n: + yield a[i] + unCheckedInc(i) + when defined(js): impl() + else: + when nimvm: impl() + else: + var i = 0 + while a[i] != '\0': + yield a[i] + unCheckedInc(i) + +iterator items*[T: enum and Ordinal](E: typedesc[T]): T = + ## Iterates over the values of `E`. + ## See also `enumutils.items` for enums with holes. + runnableExamples: + type Goo = enum g0 = 2, g1, g2 + from std/sequtils import toSeq + assert Goo.toSeq == [g0, g1, g2] + for v in low(E) .. high(E): + yield v + +iterator items*[T: Ordinal](s: Slice[T]): T = + ## Iterates over the slice `s`, yielding each value between `s.a` and `s.b` + ## (inclusively). + for x in s.a .. s.b: + yield x + +iterator pairs*[T](a: openArray[T]): tuple[key: int, val: T] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + var i = 0 + while i < len(a): + yield (i, a[i]) + unCheckedInc(i) + +iterator mpairs*[T](a: var openArray[T]): tuple[key: int, val: var T]{.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. + var i = 0 + while i < len(a): + yield (i, a[i]) + unCheckedInc(i) + +iterator pairs*[IX, T](a: array[IX, T]): tuple[key: IX, val: T] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + when a.len > 0: + var i = low(IX) + while true: + yield (i, a[i]) + if i >= high(IX): break + unCheckedInc(i) + +iterator mpairs*[IX, T](a: var array[IX, T]): tuple[key: IX, val: var T] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. + when a.len > 0: + var i = low(IX) + while true: + yield (i, a[i]) + if i >= high(IX): break + unCheckedInc(i) + +iterator pairs*[T](a: seq[T]): tuple[key: int, val: T] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + var i = 0 + let L = len(a) + while i < L: + yield (i, a[i]) + unCheckedInc(i) + assert(len(a) == L, "the length of the seq changed while iterating over it") + +iterator mpairs*[T](a: var seq[T]): tuple[key: int, val: var T] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. + var i = 0 + let L = len(a) + while i < L: + yield (i, a[i]) + unCheckedInc(i) + assert(len(a) == L, "the length of the seq changed while iterating over it") + +iterator pairs*(a: string): tuple[key: int, val: char] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + var i = 0 + let L = len(a) + while i < L: + yield (i, a[i]) + unCheckedInc(i) + assert(len(a) == L, "the length of the string changed while iterating over it") + +iterator mpairs*(a: var string): tuple[key: int, val: var char] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. + var i = 0 + let L = len(a) + while i < L: + yield (i, a[i]) + unCheckedInc(i) + assert(len(a) == L, "the length of the string changed while iterating over it") + +iterator pairs*(a: cstring): tuple[key: int, val: char] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + when defined(js): + var i = 0 + var L = len(a) + while i < L: + yield (i, a[i]) + unCheckedInc(i) + else: + var i = 0 + while a[i] != '\0': + yield (i, a[i]) + unCheckedInc(i) + +iterator mpairs*(a: var cstring): tuple[key: int, val: var char] {.inline.} = + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. + when defined(js): + var i = 0 + var L = len(a) + while i < L: + yield (i, a[i]) + unCheckedInc(i) + else: + var i = 0 + while a[i] != '\0': + yield (i, a[i]) + unCheckedInc(i) + +iterator items*[T](a: seq[T]): lent2 T {.inline.} = + ## Iterates over each item of `a`. + var i = 0 + let L = len(a) + while i < L: + yield a[i] + unCheckedInc(i) + assert(len(a) == L, "the length of the seq changed while iterating over it") + +iterator mitems*[T](a: var seq[T]): var T {.inline.} = + ## Iterates over each item of `a` so that you can modify the yielded value. + var i = 0 + let L = len(a) + while i < L: + yield a[i] + unCheckedInc(i) + assert(len(a) == L, "the length of the seq changed while iterating over it") + +iterator items*(a: string): char {.inline.} = + ## Iterates over each item of `a`. + var i = 0 + let L = len(a) + while i < L: + yield a[i] + unCheckedInc(i) + assert(len(a) == L, "the length of the string changed while iterating over it") + +iterator mitems*(a: var string): var char {.inline.} = + ## Iterates over each item of `a` so that you can modify the yielded value. + var i = 0 + let L = len(a) + while i < L: + yield a[i] + unCheckedInc(i) + assert(len(a) == L, "the length of the string changed while iterating over it") + + +iterator fields*[T: tuple|object](x: T): RootObj {. + magic: "Fields", noSideEffect.} = + ## Iterates over every field of `x`. + ## + ## .. warning:: This really transforms the 'for' and unrolls the loop. + ## The current implementation also has a bug + ## that affects symbol binding in the loop body. + runnableExamples: + var t = (1, "foo") + for v in fields(t): v = default(typeof(v)) + doAssert t == (0, "") + +iterator fields*[S:tuple|object, T:tuple|object](x: S, y: T): tuple[key: string, val: RootObj] {. + magic: "Fields", noSideEffect.} = + ## Iterates over every field of `x` and `y`. + ## + ## .. warning:: This really transforms the 'for' and unrolls the loop. + ## The current implementation also has a bug that affects symbol binding + ## in the loop body. + runnableExamples: + var t1 = (1, "foo") + var t2 = default(typeof(t1)) + for v1, v2 in fields(t1, t2): v2 = v1 + doAssert t1 == t2 + +iterator fieldPairs*[T: tuple|object](x: T): tuple[key: string, val: RootObj] {. + magic: "FieldPairs", noSideEffect.} = + ## Iterates over every field of `x` returning their name and value. + ## + ## When you iterate over objects with different field types you have to use + ## the compile time `when` instead of a runtime `if` to select the code + ## you want to run for each type. To perform the comparison use the `is + ## operator <manual.html#generics-is-operator>`_. + ## Another way to do the same without `when` is to leave the task of + ## picking the appropriate code to a secondary proc which you overload for + ## each field type and pass the `value` to. + ## + ## .. warning:: This really transforms the 'for' and unrolls the loop. The + ## current implementation also has a bug that affects symbol binding in the + ## loop body. + runnableExamples: + type + Custom = object + foo: string + bar: bool + proc `$`(x: Custom): string = + result = "Custom:" + for name, value in x.fieldPairs: + when value is bool: + result.add("\n\t" & name & " is " & $value) + else: + result.add("\n\t" & name & " '" & value & "'") + +iterator fieldPairs*[S: tuple|object, T: tuple|object](x: S, y: T): tuple[ + key: string, a, b: RootObj] {. + magic: "FieldPairs", noSideEffect.} = + ## Iterates over every field of `x` and `y`. + ## + ## .. warning:: This really transforms the 'for' and unrolls the loop. + ## The current implementation also has a bug that affects symbol binding + ## in the loop body. + runnableExamples: + type Foo = object + x1: int + x2: string + var a1 = Foo(x1: 12, x2: "abc") + var a2: Foo + for name, v1, v2 in fieldPairs(a1, a2): + when name == "x2": v2 = v1 + doAssert a2 == Foo(x1: 0, x2: "abc") diff --git a/lib/system/iterators_1.nim b/lib/system/iterators_1.nim new file mode 100644 index 000000000..d00e3f823 --- /dev/null +++ b/lib/system/iterators_1.nim @@ -0,0 +1,180 @@ +when sizeof(int) <= 2: + type IntLikeForCount = int|int8|int16|char|bool|uint8|enum +else: + type IntLikeForCount = int|int8|int16|int32|char|bool|uint8|uint16|enum + +iterator countdown*[T](a, b: T, step: Positive = 1): T {.inline.} = + ## Counts from ordinal value `a` down to `b` (inclusive) with the given + ## step count. + ## + ## `T` may be any ordinal type, `step` may only be positive. + ## + ## **Note**: This fails to count to `low(int)` if T = int for + ## efficiency reasons. + runnableExamples: + import std/sugar + let x = collect(newSeq): + for i in countdown(7, 3): + i + + assert x == @[7, 6, 5, 4, 3] + + let y = collect(newseq): + for i in countdown(9, 2, 3): + i + assert y == @[9, 6, 3] + when T is (uint|uint64): + var res = a + while res >= b: + yield res + if res == b: break + dec(res, step) + elif T is IntLikeForCount and T is Ordinal: + var res = int(a) + while res >= int(b): + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) + dec(res, step) + else: + var res = a + while res >= b: + yield res + dec(res, step) + +iterator countup*[T](a, b: T, step: Positive = 1): T {.inline.} = + ## Counts from ordinal value `a` to `b` (inclusive) with the given + ## step count. + ## + ## `T` may be any ordinal type, `step` may only be positive. + ## + ## **Note**: This fails to count to `high(int)` if T = int for + ## efficiency reasons. + runnableExamples: + import std/sugar + let x = collect(newSeq): + for i in countup(3, 7): + i + + assert x == @[3, 4, 5, 6, 7] + + let y = collect(newseq): + for i in countup(2, 9, 3): + i + assert y == @[2, 5, 8] + mixin inc + when T is IntLikeForCount and T is Ordinal: + var res = int(a) + while res <= int(b): + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) + inc(res, step) + else: + var res = a + while res <= b: + yield res + inc(res, step) + +iterator `..`*[T](a, b: T): T {.inline.} = + ## An alias for `countup(a, b, 1)`. + ## + ## See also: + ## * [..<](#..<.i,T,T) + runnableExamples: + import std/sugar + + let x = collect(newSeq): + for i in 3 .. 7: + i + + assert x == @[3, 4, 5, 6, 7] + mixin inc + when T is IntLikeForCount and T is Ordinal: + var res = int(a) + while res <= int(b): + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) + inc(res) + else: + var res = a + while res <= b: + yield res + inc(res) + +template dotdotImpl(t) {.dirty.} = + iterator `..`*(a, b: t): t {.inline.} = + ## A type specialized version of `..` for convenience so that + ## mixing integer types works better. + ## + ## See also: + ## * [..<](#..<.i,T,T) + var res = a + while res <= b: + yield res + inc(res) + +dotdotImpl(int64) +dotdotImpl(int32) +dotdotImpl(uint64) +dotdotImpl(uint32) + +iterator `..<`*[T](a, b: T): T {.inline.} = + mixin inc + var i = a + while i < b: + yield i + inc i + +template dotdotLessImpl(t) {.dirty.} = + iterator `..<`*(a, b: t): t {.inline.} = + ## A type specialized version of `..<` for convenience so that + ## mixing integer types works better. + var res = a + while res < b: + yield res + inc(res) + +dotdotLessImpl(int64) +dotdotLessImpl(int32) +dotdotLessImpl(uint64) +dotdotLessImpl(uint32) + +iterator `||`*[S, T](a: S, b: T, annotation: static string = "parallel for"): T {. + inline, magic: "OmpParFor", sideEffect.} = + ## OpenMP parallel loop iterator. Same as `..` but the loop may run in parallel. + ## + ## `annotation` is an additional annotation for the code generator to use. + ## The default annotation is `parallel for`. + ## Please refer to the `OpenMP Syntax Reference + ## <https://www.openmp.org/wp-content/uploads/OpenMP-4.5-1115-CPP-web.pdf>`_ + ## for further information. + ## + ## Note that the compiler maps that to + ## the `#pragma omp parallel for` construct of `OpenMP`:idx: and as + ## such isn't aware of the parallelism in your code! Be careful! Later + ## versions of `||` will get proper support by Nim's code generator + ## and GC. + discard + +iterator `||`*[S, T](a: S, b: T, step: Positive, annotation: static string = "parallel for"): T {. + inline, magic: "OmpParFor", sideEffect.} = + ## OpenMP parallel loop iterator with stepping. + ## Same as `countup` but the loop may run in parallel. + ## + ## `annotation` is an additional annotation for the code generator to use. + ## The default annotation is `parallel for`. + ## Please refer to the `OpenMP Syntax Reference + ## <https://www.openmp.org/wp-content/uploads/OpenMP-4.5-1115-CPP-web.pdf>`_ + ## for further information. + ## + ## Note that the compiler maps that to + ## the `#pragma omp parallel for` construct of `OpenMP`:idx: and as + ## such isn't aware of the parallelism in your code! Be careful! Later + ## versions of `||` will get proper support by Nim's code generator + ## and GC. + discard diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 5bac54772..5599240fd 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -1,16 +1,14 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -when defined(nodejs): - proc alert*(s: cstring) {.importc: "console.log", nodecl.} -else: - proc alert*(s: cstring) {.importc, nodecl.} +include system/indexerrors +import std/private/miscdollars proc log*(s: cstring) {.importc: "console.log", varargs, nodecl.} @@ -36,8 +34,6 @@ type JSRef = ref RootObj # Fake type. -{.deprecated: [TSafePoint: SafePoint, TCallFrame: CallFrame].} - var framePtr {.importc, nodecl, volatile.}: PCallFrame excHandler {.importc, nodecl, volatile.}: int = 0 @@ -52,24 +48,46 @@ proc nimCharToStr(x: char): string {.compilerproc.} = result = newString(1) result[0] = x +proc isNimException(): bool {.asmNoStackFrame.} = + {.emit: "return `lastJSError` && `lastJSError`.m_type;".} + +proc getCurrentException*(): ref Exception {.compilerRtl, benign.} = + if isNimException(): result = cast[ref Exception](lastJSError) + proc getCurrentExceptionMsg*(): string = if lastJSError != nil: - return $lastJSError.message - else: - return "" + if isNimException(): + return cast[Exception](lastJSError).msg + else: + var msg: cstring + {.emit: """ + if (`lastJSError`.message !== undefined) { + `msg` = `lastJSError`.message; + } + """.} + if not msg.isNil: + return $msg + return "" + +proc setCurrentException*(exc: ref Exception) = + lastJSError = cast[PJSError](exc) + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + ## Used to set up exception handling for closure iterators + setCurrentException(e) proc auxWriteStackTrace(f: PCallFrame): string = type - TempFrame = tuple[procname: cstring, line: int] - {.deprecated: [TTempFrame: TempFrame].} + TempFrame = tuple[procname: cstring, line: int, filename: cstring] var it = f i = 0 total = 0 - tempFrames: array [0..63, TempFrame] + tempFrames: array[0..63, TempFrame] while it != nil and i <= high(tempFrames): tempFrames[i].procname = it.procname tempFrames[i].line = it.line + tempFrames[i].filename = it.filename inc(i) inc(total) it = it.prev @@ -83,74 +101,86 @@ proc auxWriteStackTrace(f: PCallFrame): string = add(result, $(total-i)) add(result, " calls omitted) ...\n") for j in countdown(i-1, 0): + result.toLocation($tempFrames[j].filename, tempFrames[j].line, 0) + add(result, " at ") add(result, tempFrames[j].procname) - if tempFrames[j].line > 0: - add(result, ", line: ") - add(result, $tempFrames[j].line) add(result, "\n") proc rawWriteStackTrace(): string = if framePtr != nil: result = "Traceback (most recent call last)\n" & auxWriteStackTrace(framePtr) - framePtr = nil - elif lastJSError != nil: - result = $lastJSError.stack else: result = "No stack traceback available\n" +proc writeStackTrace() = + var trace = rawWriteStackTrace() + trace.setLen(trace.len - 1) + echo trace + +proc getStackTrace*(): string = rawWriteStackTrace() +proc getStackTrace*(e: ref Exception): string = e.trace + proc unhandledException(e: ref Exception) {. compilerproc, asmNoStackFrame.} = - when NimStackTrace: - var buf = rawWriteStackTrace() + var buf = "" + if e.msg.len != 0: + add(buf, "Error: unhandled exception: ") + add(buf, e.msg) else: - var buf = "" - if e.msg != nil and e.msg[0] != '\0': - add(buf, "Error: unhandled exception: ") - add(buf, e.msg) - else: - add(buf, "Error: unhandled exception") - add(buf, " [") - add(buf, e.name) - add(buf, "]\n") - alert(buf) + add(buf, "Error: unhandled exception") + add(buf, " [") + add(buf, e.name) + add(buf, "]\n") + when NimStackTrace: + add(buf, rawWriteStackTrace()) + let cbuf = cstring(buf) + when NimStackTrace: + framePtr = nil + {.emit: """ + if (typeof(Error) !== "undefined") { + throw new Error(`cbuf`); + } + else { + throw `cbuf`; + } + """.} proc raiseException(e: ref Exception, ename: cstring) {. compilerproc, asmNoStackFrame.} = e.name = ename - when not defined(noUnhandledHandler): - if excHandler == 0: - unhandledException(e) - asm "throw `e`;" + if excHandler == 0: + unhandledException(e) + when NimStackTrace: + e.trace = rawWriteStackTrace() + {.emit: "throw `e`;".} proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: - raise newException(ReraiseError, "no exception to reraise") + raise newException(ReraiseDefect, "no exception to reraise") else: - when not defined(noUnhandledHandler): - if excHandler == 0: - var isNimException : bool - asm "`isNimException` = lastJSError.m_type;" - if isNimException: - unhandledException(cast[ref Exception](lastJSError)) - asm "throw lastJSError;" + if excHandler == 0: + if isNimException(): + unhandledException(cast[ref Exception](lastJSError)) -proc raiseOverflow {.exportc: "raiseOverflow", noreturn.} = - raise newException(OverflowError, "over- or underflow") + {.emit: "throw lastJSError;".} -proc raiseDivByZero {.exportc: "raiseDivByZero", noreturn.} = - raise newException(DivByZeroError, "division by zero") +proc raiseOverflow {.exportc: "raiseOverflow", noreturn, compilerproc.} = + raise newException(OverflowDefect, "over- or underflow") + +proc raiseDivByZero {.exportc: "raiseDivByZero", noreturn, compilerproc.} = + raise newException(DivByZeroDefect, "division by zero") proc raiseRangeError() {.compilerproc, noreturn.} = - raise newException(RangeError, "value out of range") + raise newException(RangeDefect, "value out of range") -proc raiseIndexError() {.compilerproc, noreturn.} = - raise newException(IndexError, "index out of bounds") +proc raiseIndexError(i, a, b: int) {.compilerproc, noreturn.} = + raise newException(IndexDefect, formatErrorIndexBound(int(i), int(a), int(b))) -proc raiseFieldError(f: string) {.compilerproc, noreturn.} = - raise newException(FieldError, f & " is not accessible") +proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noreturn.} = + raise newException(FieldDefect, formatFieldDefect(f, discVal)) -proc SetConstr() {.varargs, asmNoStackFrame, compilerproc.} = - asm """ +proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = + {.emit: """ var result = {}; for (var i = 0; i < arguments.length; ++i) { var x = arguments[i]; @@ -163,149 +193,204 @@ proc SetConstr() {.varargs, asmNoStackFrame, compilerproc.} = } } return result; - """ + """.} + +proc makeNimstrLit(c: cstring): string {.asmNoStackFrame, compilerproc.} = + {.emit: """ + var result = []; + for (var i = 0; i < `c`.length; ++i) { + result[i] = `c`.charCodeAt(i); + } + return result; + """.} proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} = - asm """ - var result = []; - for (var i = 0; i < `c`.length; ++i) { - result[i] = `c`.charCodeAt(i); + {.emit: """ + var ln = `c`.length; + var result = new Array(ln); + var r = 0; + for (var i = 0; i < ln; ++i) { + var ch = `c`.charCodeAt(i); + + if (ch < 128) { + result[r] = ch; } - result[result.length] = 0; // terminating zero - return result; - """ - -proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} = - asm """ - var len = `s`.length-1; - var asciiPart = new Array(len); - var fcc = String.fromCharCode; - var nonAsciiPart = null; - var nonAsciiOffset = 0; - for (var i = 0; i < len; ++i) { - if (nonAsciiPart !== null) { - var offset = (i - nonAsciiOffset) * 2; - nonAsciiPart[offset] = "%"; - nonAsciiPart[offset + 1] = `s`[i].toString(16); + else { + if (ch < 2048) { + result[r] = (ch >> 6) | 192; } - else if (`s`[i] < 128) - asciiPart[i] = fcc(`s`[i]); else { - asciiPart.length = i; - nonAsciiOffset = i; - nonAsciiPart = new Array((len - i) * 2); - --i; + if (ch < 55296 || ch >= 57344) { + result[r] = (ch >> 12) | 224; + } + else { + ++i; + ch = 65536 + (((ch & 1023) << 10) | (`c`.charCodeAt(i) & 1023)); + result[r] = (ch >> 18) | 240; + ++r; + result[r] = ((ch >> 12) & 63) | 128; + } + ++r; + result[r] = ((ch >> 6) & 63) | 128; } + ++r; + result[r] = (ch & 63) | 128; } - asciiPart = asciiPart.join(""); - return (nonAsciiPart === null) ? - asciiPart : asciiPart + decodeURIComponent(nonAsciiPart.join("")); - """ + ++r; + } + return result; + """.} + +proc toJSStr(s: string): cstring {.compilerproc.} = + proc fromCharCode(c: char): cstring {.importc: "String.fromCharCode".} + proc join(x: openArray[cstring]; d = cstring""): cstring {. + importcpp: "#.join(@)".} + proc decodeURIComponent(x: cstring): cstring {. + importc: "decodeURIComponent".} + + proc toHexString(c: char; d = 16): cstring {.importcpp: "#.toString(@)".} + + proc log(x: cstring) {.importc: "console.log".} + + var res = newSeq[cstring](s.len) + var i = 0 + var j = 0 + while i < s.len: + var c = s[i] + if c < '\128': + res[j] = fromCharCode(c) + inc i + else: + var helper = newSeq[cstring]() + while true: + let code = toHexString(c) + if code.len == 1: + helper.add cstring"%0" + else: + helper.add cstring"%" + helper.add code + inc i + if i >= s.len or s[i] < '\128': break + c = s[i] + try: + res[j] = decodeURIComponent join(helper) + except: + res[j] = join(helper) + inc j + setLen(res, j) + result = join(res) proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} = - asm """ - var result = new Array(`len`+1); - result[0] = 0; - result[`len`] = 0; + {.emit: """ + var result = new Array(`len`); + for (var i = 0; i < `len`; i++) {result[i] = 0;} return result; - """ + """.} proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} = # argument type is a fake - asm """ + {.emit: """ var result = 0; for (var elem in `a`) { ++result; } return result; - """ + """.} proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ for (var elem in `a`) { if (!`b`[elem]) return false; } for (var elem in `b`) { if (!`a`[elem]) return false; } return true; - """ + """.} proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ for (var elem in `a`) { if (!`b`[elem]) return false; } return true; - """ + """.} proc SetLt(a, b: int): bool {.compilerproc.} = result = SetLe(a, b) and not SetEq(a, b) proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { if (`b`[elem]) { result[elem] = true; } } return result; - """ + """.} proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { result[elem] = true; } for (var elem in `b`) { result[elem] = true; } return result; - """ + """.} proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { if (!`b`[elem]) { result[elem] = true; } } return result; - """ + """.} -proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerProc.} = - asm """ +proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerproc.} = + {.emit: """ if (`a` == `b`) return 0; if (!`a`) return -1; if (!`b`) return 1; - for (var i = 0; i < `a`.length-1; ++i) { + for (var i = 0; i < `a`.length && i < `b`.length; i++) { var result = `a`[i] - `b`[i]; if (result != 0) return result; } - return 0; - """ - -proc cmp(x, y: string): int = return cmpStrings(x, y) + return `a`.length - `b`.length; + """.} + +proc cmp(x, y: string): int = + when nimvm: + if x == y: result = 0 + elif x < y: result = -1 + else: result = 1 + else: + result = cmpStrings(x, y) -proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerProc.} = - asm """ +proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerproc.} = + {.emit: """ if (`a` == `b`) return true; + if (`a` === null && `b`.length == 0) return true; + if (`b` === null && `a`.length == 0) return true; if ((!`a`) || (!`b`)) return false; var alen = `a`.length; if (alen != `b`.length) return false; for (var i = 0; i < alen; ++i) if (`a`[i] != `b`[i]) return false; return true; - """ + """.} when defined(kwin): proc rawEcho {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { buf += `toJSStr`(arguments[i]); } print(buf); - """ + """.} -elif defined(nodejs): +elif not defined(nimOldEcho): proc ewriteln(x: cstring) = log(x) proc rawEcho {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { buf += `toJSStr`(arguments[i]); } console.log(buf); - """ + """.} else: proc ewriteln(x: cstring) = @@ -332,78 +417,85 @@ else: """.} # Arithmetic: +proc checkOverflowInt(a: int) {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`a` > 2147483647 || `a` < -2147483648) `raiseOverflow`(); + """.} + proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` + `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + `checkOverflowInt`(result); return result; - """ + """.} proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` - `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + `checkOverflowInt`(result); return result; - """ + """.} proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` * `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + `checkOverflowInt`(result); return result; - """ + """.} proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); - return Math.floor(`a` / `b`); - """ + return Math.trunc(`a` / `b`); + """.} proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); - return Math.floor(`a` % `b`); - """ + return Math.trunc(`a` % `b`); + """.} + +proc checkOverflowInt64(a: int64) {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`a` > 9223372036854775807n || `a` < -9223372036854775808n) `raiseOverflow`(); + """.} -proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc addInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` + `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); + `checkOverflowInt64`(result); return result; - """ + """.} -proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc subInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` - `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); + `checkOverflowInt64`(result); return result; - """ + """.} -proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc mulInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` * `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); + `checkOverflowInt64`(result); return result; - """ - -proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.floor(`a` / `b`); - """ - -proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.floor(`a` % `b`); - """ + """.} + +proc divInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`b` == 0n) `raiseDivByZero`(); + if (`b` == -1n && `a` == 9223372036854775807n) `raiseOverflow`(); + return `a` / `b`; + """.} + +proc modInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`b` == 0n) `raiseDivByZero`(); + if (`b` == -1n && `a` == 9223372036854775807n) `raiseOverflow`(); + return `a` % `b`; + """.} proc negInt(a: int): int {.compilerproc.} = result = a*(-1) @@ -417,67 +509,13 @@ proc absInt(a: int): int {.compilerproc.} = proc absInt64(a: int64): int64 {.compilerproc.} = result = if a < 0: a*(-1) else: a -proc leU(a, b: int): bool {.compilerproc.} = - result = abs(a) <= abs(b) - -proc ltU(a, b: int): bool {.compilerproc.} = - result = abs(a) < abs(b) - -proc leU64(a, b: int64): bool {.compilerproc.} = - result = abs(a) <= abs(b) -proc ltU64(a, b: int64): bool {.compilerproc.} = - result = abs(a) < abs(b) - -proc addU(a, b: int): int {.compilerproc.} = - result = abs(a) + abs(b) -proc addU64(a, b: int64): int64 {.compilerproc.} = - result = abs(a) + abs(b) - -proc subU(a, b: int): int {.compilerproc.} = - result = abs(a) - abs(b) -proc subU64(a, b: int64): int64 {.compilerproc.} = - result = abs(a) - abs(b) - -proc mulU(a, b: int): int {.compilerproc.} = - result = abs(a) * abs(b) -proc mulU64(a, b: int64): int64 {.compilerproc.} = - result = abs(a) * abs(b) - -proc divU(a, b: int): int {.compilerproc.} = - result = abs(a) div abs(b) -proc divU64(a, b: int64): int64 {.compilerproc.} = - result = abs(a) div abs(b) - -proc modU(a, b: int): int {.compilerproc.} = - result = abs(a) mod abs(b) -proc modU64(a, b: int64): int64 {.compilerproc.} = - result = abs(a) mod abs(b) - -proc ze*(a: int): int {.compilerproc.} = - result = a - -proc ze64*(a: int64): int64 {.compilerproc.} = - result = a - -proc toU8*(a: int): int8 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - -proc toU16*(a: int): int16 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - -proc toU32*(a: int64): int32 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b -type NimString = string # hack for hti.nim +proc chckNilDisp(p: JSRef) {.compilerproc.} = + if p == nil: + sysFatal(NilAccessDefect, "cannot dispatch; dispatcher is nil") + include "system/hti" proc isFatPointer(ti: PNimType): bool = @@ -492,19 +530,22 @@ proc nimCopyAux(dest, src: JSRef, n: ptr TNimNode) {.compilerproc.} = case n.kind of nkNone: sysAssert(false, "nimCopyAux") of nkSlot: - asm """ + {.emit: """ `dest`[`n`.offset] = nimCopy(`dest`[`n`.offset], `src`[`n`.offset], `n`.typ); - """ + """.} of nkList: - for i in 0..n.len-1: - nimCopyAux(dest, src, n.sons[i]) + {.emit: """ + for (var i = 0; i < `n`.sons.length; i++) { + nimCopyAux(`dest`, `src`, `n`.sons[i]); + } + """.} of nkCase: - asm """ + {.emit: """ `dest`[`n`.offset] = nimCopy(`dest`[`n`.offset], `src`[`n`.offset], `n`.typ); for (var i = 0; i < `n`.sons.length; ++i) { nimCopyAux(`dest`, `src`, `n`.sons[i][1]); } - """ + """.} proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = case ti.kind @@ -512,9 +553,9 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = if not isFatPointer(ti): result = src else: - asm "`result` = [`src`[0], `src`[1]];" + {.emit: "`result` = [`src`[0], `src`[1]];".} of tySet: - asm """ + {.emit: """ if (`dest` === null || `dest` === undefined) { `dest` = {}; } @@ -523,84 +564,76 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = } for (var key in `src`) { `dest`[key] = `src`[key]; } `result` = `dest`; - """ + """.} of tyTuple, tyObject: if ti.base != nil: result = nimCopy(dest, src, ti.base) elif ti.kind == tyObject: - asm "`result` = (`dest` === null || `dest` === undefined) ? {m_type: `ti`} : `dest`;" + {.emit: "`result` = (`dest` === null || `dest` === undefined) ? {m_type: `ti`} : `dest`;".} else: - asm "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;" + {.emit: "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;".} nimCopyAux(result, src, ti.node) - of tySequence, tyArrayConstr, tyOpenArray, tyArray: - asm """ + of tyArrayConstr, tyArray: + # In order to prevent a type change (TypedArray -> Array) and to have better copying performance, + # arrays constructors are considered separately + {.emit: """ + if(ArrayBuffer.isView(`src`)) { + if(`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new `src`.constructor(`src`); + } else { + `dest`.set(`src`, 0); + } + `result` = `dest`; + } else { + if (`src` === null) { + `result` = null; + } + else { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new Array(`src`.length); + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } + } + } + """.} + of tySequence, tyOpenArray: + {.emit: """ if (`src` === null) { `result` = null; } else { - if (`dest` === null || `dest` === undefined) { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { `dest` = new Array(`src`.length); } - else { - `dest`.length = `src`.length; - } `result` = `dest`; for (var i = 0; i < `src`.length; ++i) { `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); } } - """ + """.} of tyString: - asm """ + {.emit: """ if (`src` !== null) { `result` = `src`.slice(0); } - """ + """.} else: result = src -proc genericReset(x: JSRef, ti: PNimType): JSRef {.compilerproc.} = - asm "`result` = null;" - case ti.kind - of tyPtr, tyRef, tyVar, tyNil: - if isFatPointer(ti): - asm """ - `result` = [null, 0]; - """ - of tySet: - asm """ - `result` = {}; - """ - of tyTuple, tyObject: - if ti.kind == tyObject: - asm "`result` = {m_type: `ti`};" - else: - asm "`result` = {};" - of tySequence, tyOpenArray: - asm """ - `result` = []; - """ - of tyArrayConstr, tyArray: - asm """ - `result` = new Array(`x`.length); - for (var i = 0; i < `x`.length; ++i) { - `result`[i] = genericReset(`x`[i], `ti`.base); - } - """ - else: - discard - proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. - asmNoStackFrame, compilerproc.} = + asmNoStackFrame, compilerproc.} = # types are fake - asm """ + {.emit: """ var result = new Array(`len`); for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); return result; - """ + """.} proc chckIndx(i, a, b: int): int {.compilerproc.} = if i >= a and i <= b: return i - else: raiseIndexError() + else: raiseIndexError(i, a, b) proc chckRange(i, a, b: int): int {.compilerproc.} = if i >= a and i <= b: return i @@ -612,7 +645,7 @@ proc chckObj(obj, subclass: PNimType) {.compilerproc.} = if x == subclass: return # optimized fast path while x != subclass: if x == nil: - raise newException(ObjectConversionError, "invalid object conversion") + raise newException(ObjectConversionDefect, "invalid object conversion") x = x.base proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = @@ -625,13 +658,12 @@ proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = return true proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} = - asm """ - `x`[`x`.length-1] = `c`; `x`.push(0); - """ + {.emit: "`x`.push(`c`);".} {.pop.} proc tenToThePowerOf(b: int): BiggestFloat = + # xxx deadcode var b = b var a = 10.0 result = 1.0 @@ -645,72 +677,92 @@ proc tenToThePowerOf(b: int): BiggestFloat = const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} -# XXX use JS's native way here -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {. - compilerProc.} = - var - esign = 1.0 - sign = 1.0 - i = start - exponent: int - flags: int - number = 0.0 + +proc parseFloatNative(a: openarray[char]): float = + var str = "" + for x in a: + str.add x + + let cstr = cstring str + + {.emit: """ + `result` = Number(`cstr`); + """.} + +proc nimParseBiggestFloat(s: openarray[char], number: var BiggestFloat): int {.compilerproc.} = + var sign: bool + var i = 0 if s[i] == '+': inc(i) elif s[i] == '-': - sign = -1.0 + sign = true inc(i) if s[i] == 'N' or s[i] == 'n': if s[i+1] == 'A' or s[i+1] == 'a': if s[i+2] == 'N' or s[i+2] == 'n': if s[i+3] notin IdentChars: number = NaN - return i+3 - start + return i+3 return 0 if s[i] == 'I' or s[i] == 'i': if s[i+1] == 'N' or s[i+1] == 'n': if s[i+2] == 'F' or s[i+2] == 'f': if s[i+3] notin IdentChars: - number = Inf*sign - return i+3 - start + number = if sign: -Inf else: Inf + return i+3 return 0 - while s[i] in {'0'..'9'}: - # Read integer part - flags = flags or 1 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) + + var buf: string + # we could also use an `array[char, N]` buffer to avoid reallocs, or + # use a 2-pass algorithm that first computes the length. + if sign: buf.add '-' + template addInc = + buf.add s[i] inc(i) + template eatUnderscores = while s[i] == '_': inc(i) - # Decimal? - if s[i] == '.': - var hd = 1.0 + while s[i] in {'0'..'9'}: # Read integer part + buf.add s[i] inc(i) - while s[i] in {'0'..'9'}: - # Read fractional part - flags = flags or 2 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - hd = hd * 10.0 - inc(i) - while s[i] == '_': inc(i) - number = number / hd # this complicated way preserves precision + eatUnderscores() + if s[i] == '.': # Decimal? + addInc() + while s[i] in {'0'..'9'}: # Read fractional part + addInc() + eatUnderscores() # Again, read integer and fractional part - if flags == 0: return 0 - # Exponent? - if s[i] in {'e', 'E'}: - inc(i) - if s[i] == '+': - inc(i) - elif s[i] == '-': - esign = -1.0 - inc(i) - if s[i] notin {'0'..'9'}: - return 0 + if buf.len == ord(sign): return 0 + if s[i] in {'e', 'E'}: # Exponent? + addInc() + if s[i] == '+': inc(i) + elif s[i] == '-': addInc() + if s[i] notin {'0'..'9'}: return 0 while s[i] in {'0'..'9'}: - exponent = exponent * 10 + ord(s[i]) - ord('0') - inc(i) - while s[i] == '_': inc(i) - # Calculate Exponent - let hd = tenToThePowerOf(exponent) - if esign > 0.0: number = number * hd - else: number = number / hd - # evaluate sign - number = number * sign - result = i - start + addInc() + eatUnderscores() + number = parseFloatNative(buf) + result = i + +# Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce +# 'Math.trunc' for Nim's ``div`` and ``mod`` operators: +when defined(nimJsMathTruncPolyfill): + {.emit: """ +if (!Math.trunc) { + Math.trunc = function(v) { + v = +v; + if (!isFinite(v)) return v; + return (v - v % 1) || (v < 0 ? -0 : v === 0 ? v : 0); + }; +} +""".} + +proc cmpClosures(a, b: JSRef): bool {.compilerproc, asmNoStackFrame.} = + # Both `a` and `b` need to be a closure + {.emit: """ + if (`a` !== null && `a`.ClP_0 !== undefined && + `b` !== null && `b`.ClP_0 !== undefined) { + return `a`.ClP_0 == `b`.ClP_0 && `a`.ClE_0 == `b`.ClE_0; + } else { + return `a` == `b`; + } + """ + .} diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim new file mode 100644 index 000000000..a94d0995c --- /dev/null +++ b/lib/system/memalloc.nim @@ -0,0 +1,449 @@ +when notJSnotNims: + proc zeroMem*(p: pointer, size: Natural) {.inline, noSideEffect, + tags: [], raises: [].} + ## Overwrites the contents of the memory at `p` with the value 0. + ## + ## Exactly `size` bytes will be overwritten. Like any procedure + ## dealing with raw memory this is **unsafe**. + + proc copyMem*(dest, source: pointer, size: Natural) {.inline, benign, + tags: [], raises: [].} + ## Copies the contents from the memory at `source` to the memory + ## at `dest`. + ## Exactly `size` bytes will be copied. The memory + ## regions may not overlap. Like any procedure dealing with raw + ## memory this is **unsafe**. + + proc moveMem*(dest, source: pointer, size: Natural) {.inline, benign, + tags: [], raises: [].} + ## Copies the contents from the memory at `source` to the memory + ## at `dest`. + ## + ## Exactly `size` bytes will be copied. The memory + ## regions may overlap, `moveMem` handles this case appropriately + ## and is thus somewhat more safe than `copyMem`. Like any procedure + ## dealing with raw memory this is still **unsafe**, though. + + proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect, + tags: [], raises: [].} + ## Compares the memory blocks `a` and `b`. `size` bytes will + ## be compared. + ## + ## If the blocks are equal, `true` is returned, `false` + ## otherwise. Like any procedure dealing with raw memory this is + ## **unsafe**. + + proc cmpMem*(a, b: pointer, size: Natural): int {.inline, noSideEffect, + tags: [], raises: [].} + ## Compares the memory blocks `a` and `b`. `size` bytes will + ## be compared. + ## + ## Returns: + ## * a value less than zero, if `a < b` + ## * a value greater than zero, if `a > b` + ## * zero, if `a == b` + ## + ## Like any procedure dealing with raw memory this is + ## **unsafe**. + +when hasAlloc and not defined(js): + + proc allocImpl*(size: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} + proc alloc0Impl*(size: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} + proc deallocImpl*(p: pointer) {.noconv, rtl, tags: [], benign, raises: [].} + proc reallocImpl*(p: pointer, newSize: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} + proc realloc0Impl*(p: pointer, oldSize, newSize: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} + + proc allocSharedImpl*(size: Natural): pointer {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} + proc allocShared0Impl*(size: Natural): pointer {.noconv, rtl, benign, raises: [], tags: [].} + proc deallocSharedImpl*(p: pointer) {.noconv, rtl, benign, raises: [], tags: [].} + proc reallocSharedImpl*(p: pointer, newSize: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} + proc reallocShared0Impl*(p: pointer, oldSize, newSize: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} + + # Allocator statistics for memory leak tests + + {.push stackTrace: off.} + + type AllocStats* = object + allocCount: int + deallocCount: int + + proc `-`*(a, b: AllocStats): AllocStats = + result.allocCount = a.allocCount - b.allocCount + result.deallocCount = a.deallocCount - b.deallocCount + + template dumpAllocstats*(code: untyped) = + let stats1 = getAllocStats() + code + let stats2 = getAllocStats() + echo $(stats2 - stats1) + + when defined(nimAllocStats): + var stats: AllocStats + template incStat(what: untyped) = atomicInc stats.what + proc getAllocStats*(): AllocStats = stats + + else: + template incStat(what: untyped) = discard + proc getAllocStats*(): AllocStats = discard + + template alloc*(size: Natural): pointer = + ## Allocates a new memory block with at least `size` bytes. + ## + ## The block has to be freed with `realloc(block, 0) <#realloc.t,pointer,Natural>`_ + ## or `dealloc(block) <#dealloc,pointer>`_. + ## The block is not initialized, so reading + ## from it before writing to it is undefined behaviour! + ## + ## The allocated memory belongs to its allocating thread! + ## Use `allocShared <#allocShared.t,Natural>`_ to allocate from a shared heap. + ## + ## See also: + ## * `alloc0 <#alloc0.t,Natural>`_ + incStat(allocCount) + allocImpl(size) + + proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline, benign, raises: [].} = + ## Allocates a new memory block with at least `T.sizeof * size` bytes. + ## + ## The block has to be freed with `resize(block, 0) <#resize,ptr.T,Natural>`_ + ## or `dealloc(block) <#dealloc,pointer>`_. + ## The block is not initialized, so reading + ## from it before writing to it is undefined behaviour! + ## + ## The allocated memory belongs to its allocating thread! + ## Use `createSharedU <#createSharedU,typedesc>`_ to allocate from a shared heap. + ## + ## See also: + ## * `create <#create,typedesc>`_ + cast[ptr T](alloc(T.sizeof * size)) + + template alloc0*(size: Natural): pointer = + ## Allocates a new memory block with at least `size` bytes. + ## + ## The block has to be freed with `realloc(block, 0) <#realloc.t,pointer,Natural>`_ + ## or `dealloc(block) <#dealloc,pointer>`_. + ## The block is initialized with all bytes containing zero, so it is + ## somewhat safer than `alloc <#alloc.t,Natural>`_. + ## + ## The allocated memory belongs to its allocating thread! + ## Use `allocShared0 <#allocShared0.t,Natural>`_ to allocate from a shared heap. + incStat(allocCount) + alloc0Impl(size) + + proc create*(T: typedesc, size = 1.Positive): ptr T {.inline, benign, raises: [].} = + ## Allocates a new memory block with at least `T.sizeof * size` bytes. + ## + ## The block has to be freed with `resize(block, 0) <#resize,ptr.T,Natural>`_ + ## or `dealloc(block) <#dealloc,pointer>`_. + ## The block is initialized with all bytes containing zero, so it is + ## somewhat safer than `createU <#createU,typedesc>`_. + ## + ## The allocated memory belongs to its allocating thread! + ## Use `createShared <#createShared,typedesc>`_ to allocate from a shared heap. + cast[ptr T](alloc0(sizeof(T) * size)) + + template realloc*(p: pointer, newSize: Natural): pointer = + ## Grows or shrinks a given memory block. + ## + ## If `p` is **nil** then a new memory block is returned. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `realloc` calls `dealloc(p)`. + ## In other cases the block has to be freed with + ## `dealloc(block) <#dealloc,pointer>`_. + ## + ## The allocated memory belongs to its allocating thread! + ## Use `reallocShared <#reallocShared.t,pointer,Natural>`_ to reallocate + ## from a shared heap. + reallocImpl(p, newSize) + + template realloc0*(p: pointer, oldSize, newSize: Natural): pointer = + ## Grows or shrinks a given memory block. + ## + ## If `p` is **nil** then a new memory block is returned. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `realloc` calls `dealloc(p)`. + ## In other cases the block has to be freed with + ## `dealloc(block) <#dealloc,pointer>`_. + ## + ## The block is initialized with all bytes containing zero, so it is + ## somewhat safer then realloc + ## + ## The allocated memory belongs to its allocating thread! + ## Use `reallocShared <#reallocShared.t,pointer,Natural>`_ to reallocate + ## from a shared heap. + realloc0Impl(p, oldSize, newSize) + + proc resize*[T](p: ptr T, newSize: Natural): ptr T {.inline, benign, raises: [].} = + ## Grows or shrinks a given memory block. + ## + ## If `p` is **nil** then a new memory block is returned. + ## In either way the block has at least `T.sizeof * newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `resize` calls `dealloc(p)`. + ## In other cases the block has to be freed with `free`. + ## + ## The allocated memory belongs to its allocating thread! + ## Use `resizeShared <#resizeShared,ptr.T,Natural>`_ to reallocate + ## from a shared heap. + cast[ptr T](realloc(p, T.sizeof * newSize)) + + proc dealloc*(p: pointer) {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} = + ## Frees the memory allocated with `alloc`, `alloc0`, + ## `realloc`, `create` or `createU`. + ## + ## **This procedure is dangerous!** + ## If one forgets to free the memory a leak occurs; if one tries to + ## access freed memory (or just freeing it twice!) a core dump may happen + ## or other memory may be corrupted. + ## + ## The freed memory must belong to its allocating thread! + ## Use `deallocShared <#deallocShared,pointer>`_ to deallocate from a shared heap. + incStat(deallocCount) + deallocImpl(p) + + template allocShared*(size: Natural): pointer = + ## Allocates a new memory block on the shared heap with at + ## least `size` bytes. + ## + ## The block has to be freed with + ## `reallocShared(block, 0) <#reallocShared.t,pointer,Natural>`_ + ## or `deallocShared(block) <#deallocShared,pointer>`_. + ## + ## The block is not initialized, so reading from it before writing + ## to it is undefined behaviour! + ## + ## See also: + ## * `allocShared0 <#allocShared0.t,Natural>`_. + incStat(allocCount) + allocSharedImpl(size) + + proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline, tags: [], + benign, raises: [].} = + ## Allocates a new memory block on the shared heap with at + ## least `T.sizeof * size` bytes. + ## + ## The block has to be freed with + ## `resizeShared(block, 0) <#resizeShared,ptr.T,Natural>`_ or + ## `freeShared(block) <#freeShared,ptr.T>`_. + ## + ## The block is not initialized, so reading from it before writing + ## to it is undefined behaviour! + ## + ## See also: + ## * `createShared <#createShared,typedesc>`_ + cast[ptr T](allocShared(T.sizeof * size)) + + template allocShared0*(size: Natural): pointer = + ## Allocates a new memory block on the shared heap with at + ## least `size` bytes. + ## + ## The block has to be freed with + ## `reallocShared(block, 0) <#reallocShared.t,pointer,Natural>`_ + ## or `deallocShared(block) <#deallocShared,pointer>`_. + ## + ## The block is initialized with all bytes + ## containing zero, so it is somewhat safer than + ## `allocShared <#allocShared.t,Natural>`_. + incStat(allocCount) + allocShared0Impl(size) + + proc createShared*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + ## Allocates a new memory block on the shared heap with at + ## least `T.sizeof * size` bytes. + ## + ## The block has to be freed with + ## `resizeShared(block, 0) <#resizeShared,ptr.T,Natural>`_ or + ## `freeShared(block) <#freeShared,ptr.T>`_. + ## + ## The block is initialized with all bytes + ## containing zero, so it is somewhat safer than + ## `createSharedU <#createSharedU,typedesc>`_. + cast[ptr T](allocShared0(T.sizeof * size)) + + template reallocShared*(p: pointer, newSize: Natural): pointer = + ## Grows or shrinks a given memory block on the heap. + ## + ## If `p` is **nil** then a new memory block is returned. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `reallocShared` calls + ## `deallocShared(p)`. + ## In other cases the block has to be freed with + ## `deallocShared <#deallocShared,pointer>`_. + reallocSharedImpl(p, newSize) + + template reallocShared0*(p: pointer, oldSize, newSize: Natural): pointer = + ## Grows or shrinks a given memory block on the heap. + ## + ## When growing, the new bytes of the block is initialized with all bytes + ## containing zero, so it is somewhat safer then reallocShared + ## + ## If `p` is **nil** then a new memory block is returned. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `reallocShared` calls + ## `deallocShared(p)`. + ## In other cases the block has to be freed with + ## `deallocShared <#deallocShared,pointer>`_. + reallocShared0Impl(p, oldSize, newSize) + + proc resizeShared*[T](p: ptr T, newSize: Natural): ptr T {.inline, raises: [].} = + ## Grows or shrinks a given memory block on the heap. + ## + ## If `p` is **nil** then a new memory block is returned. + ## In either way the block has at least `T.sizeof * newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `resizeShared` calls + ## `freeShared(p)`. + ## In other cases the block has to be freed with + ## `freeShared <#freeShared,ptr.T>`_. + cast[ptr T](reallocShared(p, T.sizeof * newSize)) + + proc deallocShared*(p: pointer) {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} = + ## Frees the memory allocated with `allocShared`, `allocShared0` or + ## `reallocShared`. + ## + ## **This procedure is dangerous!** + ## If one forgets to free the memory a leak occurs; if one tries to + ## access freed memory (or just freeing it twice!) a core dump may happen + ## or other memory may be corrupted. + incStat(deallocCount) + deallocSharedImpl(p) + + proc freeShared*[T](p: ptr T) {.inline, benign, raises: [].} = + ## Frees the memory allocated with `createShared`, `createSharedU` or + ## `resizeShared`. + ## + ## **This procedure is dangerous!** + ## If one forgets to free the memory a leak occurs; if one tries to + ## access freed memory (or just freeing it twice!) a core dump may happen + ## or other memory may be corrupted. + deallocShared(p) + + include bitmasks + + template `+!`(p: pointer, s: SomeInteger): pointer = + cast[pointer](cast[int](p) +% int(s)) + + template `-!`(p: pointer, s: SomeInteger): pointer = + cast[pointer](cast[int](p) -% int(s)) + + proc alignedAlloc(size, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = allocShared(size) + else: + result = alloc(size) + else: + # allocate (size + align - 1) necessary for alignment, + # plus 2 bytes to store offset + when compileOption("threads"): + let base = allocShared(size + align - 1 + sizeof(uint16)) + else: + let base = alloc(size + align - 1 + sizeof(uint16)) + # memory layout: padding + offset (2 bytes) + user_data + # in order to deallocate: read offset at user_data - 2 bytes, + # then deallocate user_data - offset + let offset = align - (cast[int](base) and (align - 1)) + cast[ptr uint16](base +! (offset - sizeof(uint16)))[] = uint16(offset) + result = base +! offset + + proc alignedAlloc0(size, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = allocShared0(size) + else: + result = alloc0(size) + else: + # see comments for alignedAlloc + when compileOption("threads"): + let base = allocShared0(size + align - 1 + sizeof(uint16)) + else: + let base = alloc0(size + align - 1 + sizeof(uint16)) + let offset = align - (cast[int](base) and (align - 1)) + cast[ptr uint16](base +! (offset - sizeof(uint16)))[] = uint16(offset) + result = base +! offset + + proc alignedDealloc(p: pointer, align: int) {.compilerproc.} = + if align <= MemAlign: + when compileOption("threads"): + deallocShared(p) + else: + dealloc(p) + else: + # read offset at p - 2 bytes, then deallocate (p - offset) pointer + let offset = cast[ptr uint16](p -! sizeof(uint16))[] + when compileOption("threads"): + deallocShared(p -! offset) + else: + dealloc(p -! offset) + + proc alignedRealloc(p: pointer, oldSize, newSize, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = reallocShared(p, newSize) + else: + result = realloc(p, newSize) + else: + result = alignedAlloc(newSize, align) + copyMem(result, p, oldSize) + alignedDealloc(p, align) + + proc alignedRealloc0(p: pointer, oldSize, newSize, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = reallocShared0(p, oldSize, newSize) + else: + result = realloc0(p, oldSize, newSize) + else: + result = alignedAlloc(newSize, align) + copyMem(result, p, oldSize) + zeroMem(result +! oldSize, newSize - oldSize) + alignedDealloc(p, align) + + {.pop.} + +# GC interface: + +when hasAlloc: + proc getOccupiedMem*(): int {.rtl.} + ## Returns the number of bytes that are owned by the process and hold data. + + proc getFreeMem*(): int {.rtl.} + ## Returns the number of bytes that are owned by the process, but do not + ## hold any meaningful data. + + proc getTotalMem*(): int {.rtl.} + ## Returns the number of bytes that are owned by the process. + + +when defined(js): + # Stubs: + proc getOccupiedMem(): int = return -1 + proc getFreeMem(): int = return -1 + proc getTotalMem(): int = return -1 + + proc dealloc(p: pointer) = discard + proc alloc(size: Natural): pointer = discard + proc alloc0(size: Natural): pointer = discard + proc realloc(p: pointer, newsize: Natural): pointer = discard + proc realloc0(p: pointer, oldsize, newsize: Natural): pointer = discard + + proc allocShared(size: Natural): pointer = discard + proc allocShared0(size: Natural): pointer = discard + proc deallocShared(p: pointer) = discard + proc reallocShared(p: pointer, newsize: Natural): pointer = discard + proc reallocShared0(p: pointer, oldsize, newsize: Natural): pointer = discard + + +when hasAlloc and hasThreadSupport and not defined(useMalloc): + proc getOccupiedSharedMem*(): int {.rtl.} + ## Returns the number of bytes that are owned by the process + ## on the shared heap and hold data. This is only available when + ## threads are enabled. + + proc getFreeSharedMem*(): int {.rtl.} + ## Returns the number of bytes that are owned by the + ## process on the shared heap, but do not hold any meaningful data. + ## This is only available when threads are enabled. + + proc getTotalSharedMem*(): int {.rtl.} + ## Returns the number of bytes on the shared heap that are owned by the + ## process. This is only available when threads are enabled. diff --git a/lib/system/memory.nim b/lib/system/memory.nim new file mode 100644 index 000000000..156773c48 --- /dev/null +++ b/lib/system/memory.nim @@ -0,0 +1,55 @@ +{.push stack_trace: off.} + +const useLibC = not defined(nimNoLibc) + +when useLibC: + import ansi_c + +proc nimCopyMem*(dest, source: pointer, size: Natural) {.nonReloadable, compilerproc, inline.} = + when useLibC: + c_memcpy(dest, source, cast[csize_t](size)) + else: + let d = cast[ptr UncheckedArray[byte]](dest) + let s = cast[ptr UncheckedArray[byte]](source) + var i = 0 + while i < size: + d[i] = s[i] + inc i + +proc nimSetMem*(a: pointer, v: cint, size: Natural) {.nonReloadable, inline.} = + when useLibC: + c_memset(a, v, cast[csize_t](size)) + else: + let a = cast[ptr UncheckedArray[byte]](a) + var i = 0 + let v = cast[byte](v) + while i < size: + a[i] = v + inc i + +proc nimZeroMem*(p: pointer, size: Natural) {.compilerproc, nonReloadable, inline.} = + nimSetMem(p, 0, size) + +proc nimCmpMem*(a, b: pointer, size: Natural): cint {.compilerproc, nonReloadable, inline.} = + when useLibC: + c_memcmp(a, b, cast[csize_t](size)) + else: + let a = cast[ptr UncheckedArray[byte]](a) + let b = cast[ptr UncheckedArray[byte]](b) + var i = 0 + while i < size: + let d = a[i].cint - b[i].cint + if d != 0: return d + inc i + +proc nimCStrLen*(a: cstring): int {.compilerproc, nonReloadable, inline.} = + if a.isNil: return 0 + when useLibC: + cast[int](c_strlen(a)) + else: + var a = cast[ptr byte](a) + while a[] != 0: + a = cast[ptr byte](cast[uint](a) + 1) + inc result + +{.pop.} diff --git a/lib/system/memtracker.nim b/lib/system/memtracker.nim new file mode 100644 index 000000000..289f4e024 --- /dev/null +++ b/lib/system/memtracker.nim @@ -0,0 +1,106 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Memory tracking support for Nim. + +when not defined(memTracker): + {.error: "Memory tracking support is turned off! Enable memory tracking by passing `--memtracker:on` to the compiler (see the Nim Compiler User Guide for more options).".} + +when defined(noSignalHandler): + {.error: "Memory tracking works better with the default signal handler.".} + +# We don't want to memtrack the tracking code ... +{.push memtracker: off.} + +when declared(getThreadId): + template myThreadId(): untyped = getThreadId() +else: + template myThreadId(): untyped = 0 + +type + LogEntry* = object + op*: cstring + address*: pointer + size*: int + file*: cstring + line*: int + thread*: int + TrackLog* = object + count*: int + disabled: bool + data*: array[400, LogEntry] + TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], gcsafe.} + +var + gLog*: TrackLog + gLogger*: TrackLogger = proc (log: TrackLog) = discard + ilocs: array[4000, (int, int)] + ilocn: int + +proc trackLocation*(p: pointer; size: int) = + let x = (cast[int](p), size) + for i in 0..ilocn-1: + # already known? + if ilocs[i] == x: return + ilocs[ilocn] = x + inc ilocn + +proc setTrackLogger*(logger: TrackLogger) = + gLogger = logger + +proc addEntry(entry: LogEntry) = + if not gLog.disabled: + var interesting = false + for i in 0..ilocn-1: + let p = ilocs[i] + # X..Y and C..D overlap iff (X <= D and C <= Y) + let x = p[0] + let y = p[0]+p[1]-1 + let c = cast[int](entry.address) + let d = c + entry.size-1 + if x <= d and c <= y: + interesting = myThreadId() != entry.thread # true + break + if interesting: + gLog.disabled = true + cprintf("interesting %s:%ld %s\n", entry.file, entry.line, entry.op) + let x = cast[proc() {.nimcall, tags: [], gcsafe, raises: [].}](writeStackTrace) + x() + rawQuit 1 + #if gLog.count > high(gLog.data): + # gLogger(gLog) + # gLog.count = 0 + #gLog.data[gLog.count] = entry + #inc gLog.count + #gLog.disabled = false + +proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.compilerproc.} = + addEntry LogEntry(op: "write", address: address, + size: size, file: file, line: line, thread: myThreadId()) + +proc memTrackerOp*(op: cstring; address: pointer; size: int) {.tags: [], + gcsafe.} = + addEntry LogEntry(op: op, address: address, size: size, + file: "", line: 0, thread: myThreadId()) + +proc memTrackerDisable*() = + gLog.disabled = true + +proc memTrackerEnable*() = + gLog.disabled = false + +proc logPendingOps() {.noconv.} = + # forward declared and called from Nim's signal handler. + gLogger(gLog) + gLog.count = 0 + +import std/exitprocs +addExitProc logPendingOps + +{.pop.} diff --git a/lib/system/mm/boehm.nim b/lib/system/mm/boehm.nim new file mode 100644 index 000000000..362d2d470 --- /dev/null +++ b/lib/system/mm/boehm.nim @@ -0,0 +1,140 @@ + + + +proc boehmGCinit {.importc: "GC_init", boehmGC.} +proc boehmGC_disable {.importc: "GC_disable", boehmGC.} +proc boehmGC_enable {.importc: "GC_enable", boehmGC.} +proc boehmGCincremental {. + importc: "GC_enable_incremental", boehmGC.} +proc boehmGCfullCollect {.importc: "GC_gcollect", boehmGC.} +proc boehmGC_set_all_interior_pointers(flag: cint) {. + importc: "GC_set_all_interior_pointers", boehmGC.} +proc boehmAlloc(size: int): pointer {.importc: "GC_malloc", boehmGC.} +proc boehmAllocAtomic(size: int): pointer {. + importc: "GC_malloc_atomic", boehmGC.} +proc boehmRealloc(p: pointer, size: int): pointer {. + importc: "GC_realloc", boehmGC.} +proc boehmDealloc(p: pointer) {.importc: "GC_free", boehmGC.} +when hasThreadSupport: + proc boehmGC_allow_register_threads {. + importc: "GC_allow_register_threads", boehmGC.} + +proc boehmGetHeapSize: int {.importc: "GC_get_heap_size", boehmGC.} + ## Return the number of bytes in the heap. Excludes collector private + ## data structures. Includes empty blocks and fragmentation loss. + ## Includes some pages that were allocated but never written. + +proc boehmGetFreeBytes: int {.importc: "GC_get_free_bytes", boehmGC.} + ## Return a lower bound on the number of free bytes in the heap. + +proc boehmGetBytesSinceGC: int {.importc: "GC_get_bytes_since_gc", boehmGC.} + ## Return the number of bytes allocated since the last collection. + +proc boehmGetTotalBytes: int {.importc: "GC_get_total_bytes", boehmGC.} + ## Return the total number of bytes allocated in this process. + ## Never decreases. + +proc boehmRegisterFinalizer(obj, ff, cd, off, ocd: pointer) {.importc: "GC_register_finalizer", boehmGC.} + +proc allocAtomic(size: int): pointer = + result = boehmAllocAtomic(size) + zeroMem(result, size) + +when not defined(useNimRtl): + + proc allocImpl(size: Natural): pointer = + result = boehmAlloc(size) + if result == nil: raiseOutOfMem() + proc alloc0Impl(size: Natural): pointer = + result = alloc(size) + proc reallocImpl(p: pointer, newSize: Natural): pointer = + result = boehmRealloc(p, newSize) + if result == nil: raiseOutOfMem() + proc realloc0Impl(p: pointer, oldSize, newSize: Natural): pointer = + result = boehmRealloc(p, newSize) + if result == nil: raiseOutOfMem() + if newSize > oldSize: + zeroMem(cast[pointer](cast[int](result) + oldSize), newSize - oldSize) + proc deallocImpl(p: pointer) = boehmDealloc(p) + + proc allocSharedImpl(size: Natural): pointer = allocImpl(size) + proc allocShared0Impl(size: Natural): pointer = alloc0Impl(size) + proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = reallocImpl(p, newSize) + proc reallocShared0Impl(p: pointer, oldSize, newSize: Natural): pointer = realloc0Impl(p, oldSize, newSize) + proc deallocSharedImpl(p: pointer) = deallocImpl(p) + + when hasThreadSupport: + proc getFreeSharedMem(): int = + boehmGetFreeBytes() + proc getTotalSharedMem(): int = + boehmGetHeapSize() + proc getOccupiedSharedMem(): int = + getTotalSharedMem() - getFreeSharedMem() + + #boehmGCincremental() + + proc GC_disable() = boehmGC_disable() + proc GC_enable() = boehmGC_enable() + proc GC_fullCollect() = boehmGCfullCollect() + proc GC_setStrategy(strategy: GC_Strategy) = discard + proc GC_enableMarkAndSweep() = discard + proc GC_disableMarkAndSweep() = discard + proc GC_getStatistics(): string = return "" + + proc getOccupiedMem(): int = return boehmGetHeapSize()-boehmGetFreeBytes() + proc getFreeMem(): int = return boehmGetFreeBytes() + proc getTotalMem(): int = return boehmGetHeapSize() + + proc nimGC_setStackBottom(theStackBottom: pointer) = discard + +proc initGC() = + when defined(boehmNoIntPtr): + # See #12286 + boehmGC_set_all_interior_pointers(0) + boehmGCinit() + when hasThreadSupport: + boehmGC_allow_register_threads() + +proc boehmgc_finalizer(obj: pointer, typedFinalizer: (proc(x: pointer) {.cdecl.})) = + typedFinalizer(obj) + + +proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = + if ntfNoRefs in typ.flags: result = allocAtomic(size) + else: result = alloc(size) + if typ.finalizer != nil: + boehmRegisterFinalizer(result, boehmgc_finalizer, typ.finalizer, nil, nil) +{.push overflowChecks: on.} +proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = + result = newObj(typ, align(GenericSeqSize, typ.base.align) + len * typ.base.size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len +{.pop.} + +proc growObj(old: pointer, newsize: int): pointer = + result = realloc(old, newsize) + +proc nimGCref(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, + deprecated: "old compiler compat".} = asgnRef(dest, src) + +type + MemRegion = object + +proc alloc(r: var MemRegion, size: int): pointer = + result = boehmAlloc(size) + if result == nil: raiseOutOfMem() +proc alloc0(r: var MemRegion, size: int): pointer = + result = alloc(size) + zeroMem(result, size) +proc dealloc(r: var MemRegion, p: pointer) = boehmDealloc(p) +proc deallocOsPages(r: var MemRegion) {.inline.} = discard +proc deallocOsPages() {.inline.} = discard + +include "system/cellsets" diff --git a/lib/system/mm/go.nim b/lib/system/mm/go.nim new file mode 100644 index 000000000..8f3aeb964 --- /dev/null +++ b/lib/system/mm/go.nim @@ -0,0 +1,153 @@ + +when defined(windows): + const goLib = "libgo.dll" +elif defined(macosx): + const goLib = "libgo.dylib" +else: + const goLib = "libgo.so" + +proc initGC() = discard +proc GC_disable() = discard +proc GC_enable() = discard +proc go_gc() {.importc: "go_gc", dynlib: goLib.} +proc GC_fullCollect() = go_gc() +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard + +const + goNumSizeClasses = 67 + +type + goMStats = object + alloc: uint64 # bytes allocated and still in use + total_alloc: uint64 # bytes allocated (even if freed) + sys: uint64 # bytes obtained from system + nlookup: uint64 # number of pointer lookups + nmalloc: uint64 # number of mallocs + nfree: uint64 # number of frees + heap_objects: uint64 # total number of allocated objects + pause_total_ns: uint64 # cumulative nanoseconds in GC stop-the-world pauses since the program started + numgc: uint32 # number of completed GC cycles + +proc goMemStats(): goMStats {.importc: "go_mem_stats", dynlib: goLib.} +proc goMalloc(size: uint): pointer {.importc: "go_malloc", dynlib: goLib.} +proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.} +proc writebarrierptr(dest: PPointer, src: pointer) {.importc: "writebarrierptr", codegenDecl:"$1 $2$3 __asm__ (\"main.Atomic_store_pointer\");\n$1 $2$3", dynlib: goLib.} + +proc GC_getStatistics(): string = + var mstats = goMemStats() + result = "[GC] total allocated memory: " & $(mstats.total_alloc) & "\n" & + "[GC] total memory obtained from system: " & $(mstats.sys) & "\n" & + "[GC] occupied memory: " & $(mstats.alloc) & "\n" & + "[GC] number of pointer lookups: " & $(mstats.nlookup) & "\n" & + "[GC] number of mallocs: " & $(mstats.nmalloc) & "\n" & + "[GC] number of frees: " & $(mstats.nfree) & "\n" & + "[GC] heap objects: " & $(mstats.heap_objects) & "\n" & + "[GC] number of completed GC cycles: " & $(mstats.numgc) & "\n" & + "[GC] total GC pause time [ms]: " & $(mstats.pause_total_ns div 1000_000) + +proc getOccupiedMem(): int = + var mstats = goMemStats() + result = int(mstats.alloc) + +proc getFreeMem(): int = + var mstats = goMemStats() + result = int(mstats.sys - mstats.alloc) + +proc getTotalMem(): int = + var mstats = goMemStats() + result = int(mstats.sys) + +proc nimGC_setStackBottom(theStackBottom: pointer) = discard + +proc allocImpl(size: Natural): pointer = + result = goMalloc(size.uint) + +proc alloc0Impl(size: Natural): pointer = + result = goMalloc(size.uint) + +proc reallocImpl(p: pointer, newsize: Natural): pointer = + doAssert false, "not implemented" + +proc realloc0Impl(p: pointer, oldsize, newsize: Natural): pointer = + doAssert false, "not implemented" + +proc deallocImpl(p: pointer) = + discard + +proc allocSharedImpl(size: Natural): pointer = allocImpl(size) +proc allocShared0Impl(size: Natural): pointer = alloc0Impl(size) +proc reallocSharedImpl(p: pointer, newsize: Natural): pointer = reallocImpl(p, newsize) +proc reallocShared0Impl(p: pointer, oldsize, newsize: Natural): pointer = realloc0Impl(p, oldsize, newsize) +proc deallocSharedImpl(p: pointer) = deallocImpl(p) + +when hasThreadSupport: + proc getFreeSharedMem(): int = discard + proc getTotalSharedMem(): int = discard + proc getOccupiedSharedMem(): int = discard + +proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = + writebarrierptr(addr(result), goMalloc(size.uint)) + if typ.finalizer != nil: + goSetFinalizer(result, typ.finalizer) + +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = + writebarrierptr(addr(result), newObj(typ, size)) + +proc newObjNoInit(typ: PNimType, size: int): pointer = + writebarrierptr(addr(result), newObj(typ, size)) + +proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = + writebarrierptr(addr(result), newObj(typ, align(GenericSeqSize, typ.base.align) + len * typ.base.size)) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + cast[PGenericSeq](result).elemSize = typ.base.size + cast[PGenericSeq](result).elemAlign = typ.base.align + +proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = + writebarrierptr(addr(result), newSeq(typ, len)) + +proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObjNoInit(typ, align(GenericSeqSize, typ.base.align) + cap * typ.base.size) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + cast[PGenericSeq](result).elemSize = typ.base.size + cast[PGenericSeq](result).elemAlign = typ.base.align + +proc typedMemMove(dest: pointer, src: pointer, size: uint) {.importc: "typedmemmove", dynlib: goLib.} + +proc growObj(old: pointer, newsize: int): pointer = + # the Go GC doesn't have a realloc + let old = cast[PGenericSeq](old) + var metadataOld = cast[PGenericSeq](old) + if metadataOld.elemSize == 0: + metadataOld.elemSize = 1 + + let oldsize = align(GenericSeqSize, old.elemAlign) + old.len * old.elemSize + writebarrierptr(addr(result), goMalloc(newsize.uint)) + typedMemMove(result, old, oldsize.uint) + +proc nimGCref(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunrefNoCycle(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunrefRC1(p: pointer) {.compilerproc, inline.} = discard +proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = discard + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + writebarrierptr(dest, src) +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + writebarrierptr(dest, src) +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, + deprecated: "old compiler compat".} = asgnRef(dest, src) + +type + MemRegion = object + +proc alloc(r: var MemRegion, size: int): pointer = + result = alloc(size) +proc alloc0(r: var MemRegion, size: int): pointer = + result = alloc0Impl(size) +proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) +proc deallocOsPages(r: var MemRegion) {.inline.} = discard +proc deallocOsPages() {.inline.} = discard diff --git a/lib/system/mm/malloc.nim b/lib/system/mm/malloc.nim new file mode 100644 index 000000000..47f1a95ae --- /dev/null +++ b/lib/system/mm/malloc.nim @@ -0,0 +1,97 @@ + +{.push stackTrace: off.} + +proc allocImpl(size: Natural): pointer = + result = c_malloc(size.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() + +proc alloc0Impl(size: Natural): pointer = + result = c_calloc(size.csize_t, 1) + when defined(zephyr): + if result == nil: + raiseOutOfMem() + +proc reallocImpl(p: pointer, newSize: Natural): pointer = + result = c_realloc(p, newSize.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() + +proc realloc0Impl(p: pointer, oldsize, newSize: Natural): pointer = + result = realloc(p, newSize.csize_t) + if newSize > oldSize: + zeroMem(cast[pointer](cast[uint](result) + uint(oldSize)), newSize - oldSize) + +proc deallocImpl(p: pointer) = + c_free(p) + + +# The shared allocators map on the regular ones + +proc allocSharedImpl(size: Natural): pointer = + allocImpl(size) + +proc allocShared0Impl(size: Natural): pointer = + alloc0Impl(size) + +proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = + reallocImpl(p, newSize) + +proc reallocShared0Impl(p: pointer, oldsize, newSize: Natural): pointer = + realloc0Impl(p, oldSize, newSize) + +proc deallocSharedImpl(p: pointer) = deallocImpl(p) + + +# Empty stubs for the GC + +proc GC_disable() = discard +proc GC_enable() = discard + +when not defined(gcOrc): + proc GC_fullCollect() = discard + proc GC_enableMarkAndSweep() = discard + proc GC_disableMarkAndSweep() = discard + +proc GC_setStrategy(strategy: GC_Strategy) = discard + +proc getOccupiedMem(): int = discard +proc getFreeMem(): int = discard +proc getTotalMem(): int = discard + +proc nimGC_setStackBottom(theStackBottom: pointer) = discard + +proc initGC() = discard + +proc newObjNoInit(typ: PNimType, size: int): pointer = + result = alloc(size) + +proc growObj(old: pointer, newsize: int): pointer = + result = realloc(old, newsize) + +proc nimGCref(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard + +when not defined(gcDestructors): + proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src + +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, + deprecated: "old compiler compat".} = asgnRef(dest, src) + +type + MemRegion = object + +proc alloc(r: var MemRegion, size: int): pointer = + result = alloc(size) +proc alloc0(r: var MemRegion, size: int): pointer = + result = alloc0Impl(size) +proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) +proc deallocOsPages(r: var MemRegion) = discard +proc deallocOsPages() = discard + +{.pop.} diff --git a/lib/system/mm/none.nim b/lib/system/mm/none.nim new file mode 100644 index 000000000..7818a0805 --- /dev/null +++ b/lib/system/mm/none.nim @@ -0,0 +1,46 @@ + +when appType == "lib": + {.warning: "nogc in a library context may not work".} + +include "system/alloc" + +proc initGC() = discard +proc GC_disable() = discard +proc GC_enable() = discard +proc GC_fullCollect() = discard +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard +proc GC_getStatistics(): string = return "" + +proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = + result = alloc0Impl(size) + +proc newObjNoInit(typ: PNimType, size: int): pointer = + result = alloc(size) + +{.push overflowChecks: on.} +proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = + result = newObj(typ, align(GenericSeqSize, typ.align) + len * typ.base.size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len +{.pop.} + +proc growObj(old: pointer, newsize: int): pointer = + result = realloc(old, newsize) + +proc nimGC_setStackBottom(theStackBottom: pointer) = discard +proc nimGCref(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, + deprecated: "old compiler compat".} = asgnRef(dest, src) + +var allocator {.rtlThreadVar.}: MemRegion +instantiateForRegion(allocator) + +include "system/cellsets" diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 1c13f3ff8..26f2f0bbf 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -17,11 +17,11 @@ const debugGC = false # we wish to debug the GC... logGC = false traceGC = false # extensive debugging - alwaysCycleGC = false - alwaysGC = defined(fulldebug) # collect after every memory + alwaysCycleGC = defined(nimSmokeCycles) + alwaysGC = defined(nimFulldebug) # collect after every memory # allocation (for debugging) - leakDetector = false - overwriteFree = false + leakDetector = defined(nimLeakDetector) + overwriteFree = defined(nimBurnFree) # overwrite memory with 0xFF before free trackAllocationSource = leakDetector cycleGC = true # (de)activate the cycle GC @@ -30,500 +30,91 @@ const coalescRight = true coalescLeft = true logAlloc = false - useCellIds = defined(corruption) + useCellIds = defined(nimCorruption) type PPointer = ptr pointer - ByteArray = array[0..ArrayDummySize, byte] + ByteArray = UncheckedArray[byte] PByte = ptr ByteArray PString = ptr string -{.deprecated: [TByteArray: ByteArray].} -# Page size of the system; in most cases 4096 bytes. For exotic OS or -# CPU this needs to be changed: -const - PageShift = when defined(cpu16): 8 else: 12 - PageSize = 1 shl PageShift - PageMask = PageSize-1 - - MemAlign = 8 # also minimal allocatable memory block - - BitsPerPage = PageSize div MemAlign - UnitsPerPage = BitsPerPage div (sizeof(int)*8) - # how many ints do we need to describe a page: - # on 32 bit systems this is only 16 (!) - - TrunkShift = 9 - BitsPerTrunk = 1 shl TrunkShift # needs to be power of 2 and divisible by 64 - TrunkMask = BitsPerTrunk - 1 - IntsPerTrunk = BitsPerTrunk div (sizeof(int)*8) - IntShift = 5 + ord(sizeof(int) == 8) # 5 or 6, depending on int width - IntMask = 1 shl IntShift - 1 +when declared(IntsPerTrunk): + discard +else: + include bitmasks proc raiseOutOfMem() {.noinline.} = if outOfMemHook != nil: outOfMemHook() - echo("out of memory") - quit(1) + cstderr.rawWrite("out of memory\n") + rawQuit(1) when defined(boehmgc): - proc boehmGCinit {.importc: "GC_init", boehmGC.} - proc boehmGC_disable {.importc: "GC_disable", boehmGC.} - proc boehmGC_enable {.importc: "GC_enable", boehmGC.} - proc boehmGCincremental {. - importc: "GC_enable_incremental", boehmGC.} - proc boehmGCfullCollect {.importc: "GC_gcollect", boehmGC.} - proc boehmAlloc(size: int): pointer {.importc: "GC_malloc", boehmGC.} - proc boehmAllocAtomic(size: int): pointer {. - importc: "GC_malloc_atomic", boehmGC.} - proc boehmRealloc(p: pointer, size: int): pointer {. - importc: "GC_realloc", boehmGC.} - proc boehmDealloc(p: pointer) {.importc: "GC_free", boehmGC.} - when hasThreadSupport: - proc boehmGC_allow_register_threads {. - importc: "GC_allow_register_threads", boehmGC.} - - proc boehmGetHeapSize: int {.importc: "GC_get_heap_size", boehmGC.} - ## Return the number of bytes in the heap. Excludes collector private - ## data structures. Includes empty blocks and fragmentation loss. - ## Includes some pages that were allocated but never written. - - proc boehmGetFreeBytes: int {.importc: "GC_get_free_bytes", boehmGC.} - ## Return a lower bound on the number of free bytes in the heap. - - proc boehmGetBytesSinceGC: int {.importc: "GC_get_bytes_since_gc", boehmGC.} - ## Return the number of bytes allocated since the last collection. - - proc boehmGetTotalBytes: int {.importc: "GC_get_total_bytes", boehmGC.} - ## Return the total number of bytes allocated in this process. - ## Never decreases. - - proc allocAtomic(size: int): pointer = - result = boehmAllocAtomic(size) - zeroMem(result, size) - - when not defined(useNimRtl): - - proc alloc(size: Natural): pointer = - result = boehmAlloc(size) - if result == nil: raiseOutOfMem() - proc alloc0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) - proc realloc(p: pointer, newsize: Natural): pointer = - result = boehmRealloc(p, newsize) - if result == nil: raiseOutOfMem() - proc dealloc(p: pointer) = boehmDealloc(p) - - proc allocShared(size: Natural): pointer = - result = boehmAlloc(size) - if result == nil: raiseOutOfMem() - proc allocShared0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) - proc reallocShared(p: pointer, newsize: Natural): pointer = - result = boehmRealloc(p, newsize) - if result == nil: raiseOutOfMem() - proc deallocShared(p: pointer) = boehmDealloc(p) - - when hasThreadSupport: - proc getFreeSharedMem(): int = - boehmGetFreeBytes() - proc getTotalSharedMem(): int = - boehmGetHeapSize() - proc getOccupiedSharedMem(): int = - getTotalSharedMem() - getFreeSharedMem() - - #boehmGCincremental() - - proc GC_disable() = boehmGC_disable() - proc GC_enable() = boehmGC_enable() - proc GC_fullCollect() = boehmGCfullCollect() - proc GC_setStrategy(strategy: GC_Strategy) = discard - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - proc GC_getStatistics(): string = return "" - - proc getOccupiedMem(): int = return boehmGetHeapSize()-boehmGetFreeBytes() - proc getFreeMem(): int = return boehmGetFreeBytes() - proc getTotalMem(): int = return boehmGetHeapSize() - - proc setStackBottom(theStackBottom: pointer) = discard - - proc initGC() = - boehmGCinit() - when hasThreadSupport: - boehmGC_allow_register_threads() - - proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = - if ntfNoRefs in typ.flags: result = allocAtomic(size) - else: result = alloc(size) - proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - - proc growObj(old: pointer, newsize: int): pointer = - result = realloc(old, newsize) - - proc nimGCref(p: pointer) {.compilerproc, inline.} = discard - proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard - - proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - - type - MemRegion = object {.final, pure.} - {.deprecated: [TMemRegion: MemRegion].} - - proc alloc(r: var MemRegion, size: int): pointer = - result = boehmAlloc(size) - if result == nil: raiseOutOfMem() - proc alloc0(r: var MemRegion, size: int): pointer = - result = alloc(size) - zeroMem(result, size) - proc dealloc(r: var MemRegion, p: pointer) = boehmDealloc(p) - proc deallocOsPages(r: var MemRegion) {.inline.} = discard - proc deallocOsPages() {.inline.} = discard - - include "system/cellsets" + include system / mm / boehm elif defined(gogc): - when defined(windows): - const goLib = "libgo.dll" - elif defined(macosx): - const goLib = "libgo.dylib" - else: - const goLib = "libgo.so" - - proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - - proc initGC() = discard - # runtime_setgcpercent is only available in GCC 5 - proc GC_disable() = discard - proc GC_enable() = discard - proc goRuntimeGC(force: int32) {.importc: "runtime_gc", dynlib: goLib.} - proc GC_fullCollect() = goRuntimeGC(2) - proc GC_setStrategy(strategy: GC_Strategy) = discard - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - - const - goNumSizeClasses = 67 - - type - cbool {.importc: "_Bool", nodecl.} = bool - - goMStats_inner_struct = object - size: uint32 - nmalloc: uint64 - nfree: uint64 - - goMStats = object - # General statistics. - alloc: uint64 # bytes allocated and still in use - total_alloc: uint64 # bytes allocated (even if freed) - sys: uint64 # bytes obtained from system (should be sum of xxx_sys below, no locking, approximate) - nlookup: uint64 # number of pointer lookups - nmalloc: uint64 # number of mallocs - nfree: uint64 # number of frees - # Statistics about malloc heap. - # protected by mheap.Lock - heap_alloc: uint64 # bytes allocated and still in use - heap_sys: uint64 # bytes obtained from system - heap_idle: uint64 # bytes in idle spans - heap_inuse: uint64 # bytes in non-idle spans - heap_released: uint64 # bytes released to the OS - heap_objects: uint64 # total number of allocated objects - # Statistics about allocation of low-level fixed-size structures. - # Protected by FixAlloc locks. - stacks_inuse: uint64 # bootstrap stacks - stacks_sys: uint64 - mspan_inuse: uint64 # MSpan structures - mspan_sys: uint64 - mcache_inuse: uint64 # MCache structures - mcache_sys: uint64 - buckhash_sys: uint64 # profiling bucket hash table - gc_sys: uint64 - other_sys: uint64 - # Statistics about garbage collector. - # Protected by mheap or stopping the world during GC. - next_gc: uint64 # next GC (in heap_alloc time) - last_gc: uint64 # last GC (in absolute time) - pause_total_ns: uint64 - pause_ns: array[256, uint64] - numgc: uint32 - enablegc: cbool - debuggc: cbool - # Statistics about allocation size classes. - by_size: array[goNumSizeClasses, goMStats_inner_struct] - - proc goRuntime_ReadMemStats(a2: ptr goMStats) {.cdecl, - importc: "runtime_ReadMemStats", - codegenDecl: "$1 $2$3 __asm__ (\"runtime.ReadMemStats\");\n$1 $2$3", - dynlib: goLib.} - - proc GC_getStatistics(): string = - var mstats: goMStats - goRuntime_ReadMemStats(addr mstats) - result = "[GC] total allocated memory: " & $(mstats.total_alloc) & "\n" & - "[GC] total memory obtained from system: " & $(mstats.sys) & "\n" & - "[GC] occupied memory: " & $(mstats.alloc) & "\n" & - "[GC] number of pointer lookups: " & $(mstats.nlookup) & "\n" & - "[GC] number of mallocs: " & $(mstats.nmalloc) & "\n" & - "[GC] number of frees: " & $(mstats.nfree) & "\n" & - "[GC] heap objects: " & $(mstats.heap_objects) & "\n" & - "[GC] numgc: " & $(mstats.numgc) & "\n" & - "[GC] enablegc: " & $(mstats.enablegc) & "\n" & - "[GC] debuggc: " & $(mstats.debuggc) & "\n" & - "[GC] total pause time [ms]: " & $(mstats.pause_total_ns div 1000_000) - - proc getOccupiedMem(): int = - var mstats: goMStats - goRuntime_ReadMemStats(addr mstats) - result = int(mstats.alloc) - - proc getFreeMem(): int = - var mstats: goMStats - goRuntime_ReadMemStats(addr mstats) - result = int(mstats.sys - mstats.alloc) - - proc getTotalMem(): int = - var mstats: goMStats - goRuntime_ReadMemStats(addr mstats) - result = int(mstats.sys) - - proc setStackBottom(theStackBottom: pointer) = discard - - proc alloc(size: Natural): pointer = - result = cmalloc(size) - if result == nil: raiseOutOfMem() - - proc alloc0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) + include system / mm / go - proc realloc(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) - if result == nil: raiseOutOfMem() +elif (defined(nogc) or defined(gcDestructors)) and defined(useMalloc): + include system / mm / malloc - proc dealloc(p: pointer) = cfree(p) + when defined(nogc): + proc GC_getStatistics(): string = "" + proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = + result = alloc0(size) - proc allocShared(size: Natural): pointer = - result = cmalloc(size) - if result == nil: raiseOutOfMem() - - proc allocShared0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) - - proc reallocShared(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) - if result == nil: raiseOutOfMem() - - proc deallocShared(p: pointer) = cfree(p) - - when hasThreadSupport: - proc getFreeSharedMem(): int = discard - proc getTotalSharedMem(): int = discard - proc getOccupiedSharedMem(): int = discard - - const goFlagNoZero: uint32 = 1 shl 3 - proc goRuntimeMallocGC(size: uint, typ: uint, flag: uint32): pointer {.importc: "runtime_mallocgc", dynlib: goLib.} - proc goFree(v: pointer) {.importc: "__go_free", dynlib: goLib.} - - proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.} - - proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = - result = goRuntimeMallocGC(roundup(size, sizeof(pointer)).uint, 0.uint, 0.uint32) - if typ.finalizer != nil: - goSetFinalizer(result, typ.finalizer) - - proc newObjNoInit(typ: PNimType, size: int): pointer = - result = goRuntimeMallocGC(roundup(size, sizeof(pointer)).uint, 0.uint, goFlagNoZero) - if typ.finalizer != nil: - goSetFinalizer(result, typ.finalizer) - - proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, len * typ.base.size + GenericSeqSize) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - cast[PGenericSeq](result).elemSize = typ.base.size - - proc growObj(old: pointer, newsize: int): pointer = - # the Go GC doesn't have a realloc - var - oldsize = cast[PGenericSeq](old).len * cast[PGenericSeq](old).elemSize + GenericSeqSize - result = goRuntimeMallocGC(roundup(newsize, sizeof(pointer)).uint, 0.uint, goFlagNoZero) - copyMem(result, old, oldsize) - zeroMem(cast[pointer](cast[ByteAddress](result) +% oldsize), newsize - oldsize) - goFree(old) - - proc nimGCref(p: pointer) {.compilerproc, inline.} = discard - proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard - - proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - - type - MemRegion = object {.final, pure.} - {.deprecated: [TMemRegion: MemRegion].} - - proc alloc(r: var MemRegion, size: int): pointer = - result = alloc(size) - proc alloc0(r: var MemRegion, size: int): pointer = - result = alloc0(size) - proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) - proc deallocOsPages(r: var MemRegion) {.inline.} = discard - proc deallocOsPages() {.inline.} = discard - -elif defined(nogc) and defined(useMalloc): - - when not defined(useNimRtl): - proc alloc(size: Natural): pointer = - result = cmalloc(size) - if result == nil: raiseOutOfMem() - proc alloc0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) - proc realloc(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) - if result == nil: raiseOutOfMem() - proc dealloc(p: pointer) = cfree(p) - - proc allocShared(size: Natural): pointer = - result = cmalloc(size) - if result == nil: raiseOutOfMem() - proc allocShared0(size: Natural): pointer = - result = alloc(size) - zeroMem(result, size) - proc reallocShared(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) - if result == nil: raiseOutOfMem() - proc deallocShared(p: pointer) = cfree(p) - - proc GC_disable() = discard - proc GC_enable() = discard - proc GC_fullCollect() = discard - proc GC_setStrategy(strategy: GC_Strategy) = discard - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - proc GC_getStatistics(): string = return "" - - proc getOccupiedMem(): int = discard - proc getFreeMem(): int = discard - proc getTotalMem(): int = discard - - proc setStackBottom(theStackBottom: pointer) = discard - - proc initGC() = discard - - proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = - result = alloc(size) - proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - proc newObjNoInit(typ: PNimType, size: int): pointer = - result = alloc(size) - - proc growObj(old: pointer, newsize: int): pointer = - result = realloc(old, newsize) - - proc nimGCref(p: pointer) {.compilerproc, inline.} = discard - proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard - - proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - - type - MemRegion = object {.final, pure.} - {.deprecated: [TMemRegion: MemRegion].} - - proc alloc(r: var MemRegion, size: int): pointer = - result = alloc(size) - proc alloc0(r: var MemRegion, size: int): pointer = - result = alloc0(size) - proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) - proc deallocOsPages(r: var MemRegion) {.inline.} = discard - proc deallocOsPages() {.inline.} = discard + proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = + result = newObj(typ, align(GenericSeqSize, typ.align) + len * typ.base.size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len elif defined(nogc): - # Even though we don't want the GC, we cannot simply use C's memory manager - # because Nim's runtime wants ``realloc`` to zero out the additional - # space which C's ``realloc`` does not. And we cannot get the old size of an - # object, because C does not support this operation... Even though every - # possible implementation has to have a way to determine the object's size. - # C just sucks. - when appType == "lib": - {.warning: "nogc in a library context may not work".} - - include "system/alloc" - - proc initGC() = discard - proc GC_disable() = discard - proc GC_enable() = discard - proc GC_fullCollect() = discard - proc GC_setStrategy(strategy: GC_Strategy) = discard - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - proc GC_getStatistics(): string = return "" - - proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = - result = alloc0(size) - - proc newObjNoInit(typ: PNimType, size: int): pointer = - result = alloc(size) - - proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - proc growObj(old: pointer, newsize: int): pointer = - result = realloc(old, newsize) - - proc setStackBottom(theStackBottom: pointer) = discard - proc nimGCref(p: pointer) {.compilerproc, inline.} = discard - proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard - - proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src - - var allocator {.rtlThreadVar.}: MemRegion - instantiateForRegion(allocator) - - include "system/cellsets" + include system / mm / none else: - include "system/alloc" - - include "system/cellsets" - when not leakDetector: - sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") - when compileOption("gc", "v2"): - include "system/gc2" + when not defined(gcRegions): + include "system/alloc" + + when not usesDestructors: + include "system/cellsets" + when not leakDetector and not useCellIds and not defined(nimV2): + sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") + when defined(gcRegions): + # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here + include "system/gc_regions" + elif defined(nimV2) or usesDestructors: + when not defined(useNimRtl): + var allocator {.rtlThreadVar.}: MemRegion + instantiateForRegion(allocator) + when defined(gcHooks): + include "system/gc_hooks" elif defined(gcMarkAndSweep): # XXX use 'compileOption' here include "system/gc_ms" - elif defined(gcGenerational): - include "system/gc" else: include "system/gc" +when not declared(nimNewSeqOfCap) and not defined(nimSeqsV2): + {.push overflowChecks: on.} + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + when defined(gcRegions): + let s = cap * typ.base.size # newStr already adds GenericSeqSize + result = newStr(typ, s, ntfNoRefs notin typ.base.flags) + else: + let s = align(GenericSeqSize, typ.base.align) + cap * typ.base.size + when declared(newObjNoInit): + result = if ntfNoRefs in typ.base.flags: newObjNoInit(typ, s) else: newObj(typ, s) + else: + result = newObj(typ, s) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + {.pop.} + {.pop.} + +when not declared(ForeignCell): + type ForeignCell* = object + data*: pointer + + proc protect*(x: pointer): ForeignCell = ForeignCell(data: x) + proc dispose*(x: ForeignCell) = discard + proc isNotForeign*(x: ForeignCell): bool = false diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index aaba11324..cf81f6d86 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -7,40 +7,57 @@ # distribution, for details about the copyright. # +## To learn about scripting in Nim see `NimScript<nims.html>`_ # Nim's configuration system now uses Nim for scripting. This module provides # a few things that are required for this to work. +const + buildOS* {.magic: "BuildOS".}: string = "" + ## The OS this build is running on. Can be different from `system.hostOS` + ## for cross compilations. + + buildCPU* {.magic: "BuildCPU".}: string = "" + ## The CPU this build is running on. Can be different from `system.hostCPU` + ## for cross compilations. + template builtin = discard # We know the effects better than the compiler: {.push hint[XDeclaredButNotUsed]: off.} -proc listDirs*(dir: string): seq[string] = - ## Lists all the subdirectories (non-recursively) in the directory `dir`. - builtin -proc listFiles*(dir: string): seq[string] = - ## Lists all the files (non-recursively) in the directory `dir`. - builtin - -proc removeDir(dir: string){. +proc listDirsImpl(dir: string): seq[string] {. + tags: [ReadIOEffect], raises: [OSError].} = builtin +proc listFilesImpl(dir: string): seq[string] {. + tags: [ReadIOEffect], raises: [OSError].} = builtin +proc removeDir(dir: string, checkDir = true) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc removeFile(dir: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc moveFile(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin +proc moveDir(src, dest: string) {. + tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc copyFile(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin +proc copyDir(src, dest: string) {. + tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc createDir(dir: string) {.tags: [WriteIOEffect], raises: [OSError].} = builtin -proc getOsError: string = builtin + +proc getError: string = builtin proc setCurrentDir(dir: string) = builtin -proc getCurrentDir(): string = builtin +proc getCurrentDir*(): string = + ## Retrieves the current working directory. + builtin proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} = builtin +proc warningImpl(arg, orig: string) = discard +proc hintImpl(arg, orig: string) = discard + proc paramStr*(i: int): string = - ## Retrieves the ``i``'th command line parameter. + ## Retrieves the `i`'th command line parameter. builtin proc paramCount*(): int = @@ -49,9 +66,34 @@ proc paramCount*(): int = proc switch*(key: string, val="") = ## Sets a Nim compiler command line switch, for - ## example ``switch("checks", "on")``. + ## example `switch("checks", "on")`. builtin +proc warning*(name: string; val: bool) = + ## Disables or enables a specific warning. + let v = if val: "on" else: "off" + warningImpl(name & ":" & v, "warning:" & name & ":" & v) + +proc hint*(name: string; val: bool) = + ## Disables or enables a specific hint. + let v = if val: "on" else: "off" + hintImpl(name & ":" & v, "hint:" & name & ":" & v) + +proc patchFile*(package, filename, replacement: string) = + ## Overrides the location of a given file belonging to the + ## passed package. + ## If the `replacement` is not an absolute path, the path + ## is interpreted to be local to the Nimscript file that contains + ## the call to `patchFile`, Nim's `--path` is not used at all + ## to resolve the filename! + ## The compiler also performs `path substitution <nimc.html#compiler-usage-commandminusline-switches>`_ on `replacement`. + ## + ## Example: + ## ```nim + ## patchFile("stdlib", "asyncdispatch", "patches/replacement") + ## ``` + discard + proc getCommand*(): string = ## Gets the Nim command that the compiler has been invoked with, for example ## "c", "js", "build", "help". @@ -69,12 +111,20 @@ proc cmpic*(a, b: string): int = ## Compares `a` and `b` ignoring case. cmpIgnoreCase(a, b) -proc getEnv*(key: string): string {.tags: [ReadIOEffect].} = +proc getEnv*(key: string; default = ""): string {.tags: [ReadIOEffect].} = ## Retrieves the environment variable of name `key`. builtin proc existsEnv*(key: string): bool {.tags: [ReadIOEffect].} = - ## Checks for the existance of an environment variable named `key`. + ## Checks for the existence of an environment variable named `key`. + builtin + +proc putEnv*(key, val: string) {.tags: [WriteIOEffect].} = + ## Sets the value of the environment variable named `key` to `val`. + builtin + +proc delEnv*(key: string) {.tags: [WriteIOEffect].} = + ## Deletes the environment variable named `key`. builtin proc fileExists*(filename: string): bool {.tags: [ReadIOEffect].} = @@ -86,13 +136,9 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -proc existsFile*(filename: string): bool = - ## An alias for ``fileExists``. - fileExists(filename) - -proc existsDir*(dir: string): bool = - ## An alias for ``dirExists``. - dirExists(dir) +proc selfExe*(): string {.deprecated: "Deprecated since v1.7; Use getCurrentCompilerExe".} = + ## Returns the currently running nim or nimble executable. + builtin proc toExe*(filename: string): string = ## On Windows adds ".exe" to `filename`, else returns `filename` unmodified. @@ -104,16 +150,28 @@ proc toDll*(filename: string): string = proc strip(s: string): string = var i = 0 - while s[i] in {' ', '\c', '\L'}: inc i + while s[i] in {' ', '\c', '\n'}: inc i result = s.substr(i) + if result[0] == '"' and result[^1] == '"': + result = result[1..^2] template `--`*(key, val: untyped) = - ## A shortcut for ``switch(astToStr(key), astToStr(val))``. - switch(astToStr(key), strip astToStr(val)) + ## A shortcut for `switch <#switch,string,string>`_ + ## Example: + ## ```nim + ## --path:somePath # same as switch("path", "somePath") + ## --path:"someOtherPath" # same as switch("path", "someOtherPath") + ## --hint:"[Conf]:off" # same as switch("hint", "[Conf]:off") + ## ``` + switch(strip(astToStr(key)), strip(astToStr(val))) template `--`*(key: untyped) = - ## A shortcut for ``switch(astToStr(key)``. - switch(astToStr(key), "") + ## A shortcut for `switch <#switch,string,string>`_ + ## Example: + ## ```nim + ## --listCmd # same as switch("listCmd") + ## ``` + switch(strip(astToStr(key))) type ScriptMode* {.pure.} = enum ## Controls the behaviour of the script. @@ -126,20 +184,33 @@ var mode*: ScriptMode ## Set this to influence how mkDir, rmDir, rmFile etc. ## behave +template checkError(exc: untyped): untyped = + let err = getError() + if err.len > 0: raise newException(exc, err) + template checkOsError = - let err = getOsError() - if err.len > 0: raise newException(OSError, err) + checkError(OSError) template log(msg: string, body: untyped) = - if mode == ScriptMode.Verbose or mode == ScriptMode.Whatif: + if mode in {ScriptMode.Verbose, ScriptMode.Whatif}: echo "[NimScript] ", msg - if mode != ScriptMode.WhatIf: + if mode != ScriptMode.Whatif: body -proc rmDir*(dir: string) {.raises: [OSError].} = +proc listDirs*(dir: string): seq[string] = + ## Lists all the subdirectories (non-recursively) in the directory `dir`. + result = listDirsImpl(dir) + checkOsError() + +proc listFiles*(dir: string): seq[string] = + ## Lists all the files (non-recursively) in the directory `dir`. + result = listFilesImpl(dir) + checkOsError() + +proc rmDir*(dir: string, checkDir = false) {.raises: [OSError].} = ## Removes the directory `dir`. log "rmDir: " & dir: - removeDir dir + removeDir(dir, checkDir = checkDir) checkOsError() proc rmFile*(file: string) {.raises: [OSError].} = @@ -161,24 +232,63 @@ proc mvFile*(`from`, to: string) {.raises: [OSError].} = moveFile `from`, to checkOsError() +proc mvDir*(`from`, to: string) {.raises: [OSError].} = + ## Moves the dir `from` to `to`. + log "mvDir: " & `from` & ", " & to: + moveDir `from`, to + checkOsError() + proc cpFile*(`from`, to: string) {.raises: [OSError].} = ## Copies the file `from` to `to`. log "cpFile: " & `from` & ", " & to: copyFile `from`, to checkOsError() -proc exec*(command: string) = - ## Executes an external process. +proc cpDir*(`from`, to: string) {.raises: [OSError].} = + ## Copies the dir `from` to `to`. + log "cpDir: " & `from` & ", " & to: + copyDir `from`, to + checkOsError() + +proc exec*(command: string) {. + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = + ## Executes an external process. If the external process terminates with + ## a non-zero exit code, an OSError exception is raised. The command is + ## executed relative to the current source path. + ## + ## .. note:: If you need a version of `exec` that returns the exit code + ## and text output of the command, you can use `system.gorgeEx + ## <system.html#gorgeEx,string,string,string>`_. log "exec: " & command: if rawExec(command) != 0: raise newException(OSError, "FAILED: " & command) checkOsError() proc exec*(command: string, input: string, cache = "") {. - raises: [OSError], tags: [ExecIOEffect].} = - ## Executes an external process. + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = + ## Executes an external process. If the external process terminates with + ## a non-zero exit code, an OSError exception is raised. + ## + ## .. warning:: This version of `exec` is executed relative to the nimscript + ## module path, which affects how the command resolves relative paths. Thus + ## it is generally better to use `gorgeEx` directly when you need more + ## control over the execution environment or when working with commands + ## that deal with relative paths. log "exec: " & command: - echo staticExec(command, input, cache) + let (output, exitCode) = gorgeEx(command, input, cache) + echo output + if exitCode != 0: + raise newException(OSError, "FAILED: " & command) + +proc selfExec*(command: string) {. + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = + ## Executes an external command with the current nim/nimble executable. + ## `Command` must not contain the "nim " part. + let c = selfExe() & " " & command + log "exec: " & c: + if rawExec(c) != 0: + raise newException(OSError, "FAILED: " & c) + checkOsError() proc put*(key, value: string) = ## Sets a configuration 'key' like 'gcc.options.always' to its value. @@ -189,7 +299,7 @@ proc get*(key: string): string = builtin proc exists*(key: string): bool = - ## Checks for the existance of a configuration 'key' + ## Checks for the existence of a configuration 'key' ## like 'gcc.options.always'. builtin @@ -197,81 +307,146 @@ proc nimcacheDir*(): string = ## Retrieves the location of 'nimcache'. builtin +proc projectName*(): string = + ## Retrieves the name of the current project + builtin + +proc projectDir*(): string = + ## Retrieves the absolute directory of the current project + builtin + +proc projectPath*(): string = + ## Retrieves the absolute path of the current project + builtin + proc thisDir*(): string = - ## Retrieves the location of the current ``nims`` script file. + ## Retrieves the directory of the current `nims` script file. Its path is + ## obtained via `currentSourcePath` (although, currently, + ## `currentSourcePath` resolves symlinks, unlike `thisDir`). builtin proc cd*(dir: string) {.raises: [OSError].} = ## Changes the current directory. ## ## The change is permanent for the rest of the execution, since this is just - ## a shortcut for `os.setCurrentDir() - ## <http://nim-lang.org/os.html#setCurrentDir,string>`_ . Use the `withDir() - ## <#withDir>`_ template if you want to perform a temporary change only. + ## a shortcut for `os.setCurrentDir() <os.html#setCurrentDir,string>`_ . Use + ## the `withDir() <#withDir.t,string,untyped>`_ template if you want to + ## perform a temporary change only. setCurrentDir(dir) checkOsError() +proc findExe*(bin: string): string = + ## Searches for bin in the current working directory and then in directories + ## listed in the PATH environment variable. Returns "" if the exe cannot be + ## found. + builtin + template withDir*(dir: string; body: untyped): untyped = ## Changes the current directory temporarily. ## - ## If you need a permanent change, use the `cd() <#cd>`_ proc. Usage example: - ## - ## .. code-block:: nim + ## If you need a permanent change, use the `cd() <#cd,string>`_ proc. + ## Usage example: + ## ```nim + ## # inside /some/path/ ## withDir "foo": - ## # inside foo - ## #back to last dir - var curDir = getCurrentDir() + ## # move to /some/path/foo/ + ## # back in /some/path/ + ## ``` + let curDir = getCurrentDir() try: cd(dir) body finally: cd(curDir) -template `==?`(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 - proc writeTask(name, desc: string) = if desc.len > 0: var spaces = " " for i in 0 ..< 20 - name.len: spaces.add ' ' echo name, spaces, desc -template task*(name: untyped; description: string; body: untyped): untyped = - ## Defines a task. Hidden tasks are supported via an empty description. - ## Example: - ## - ## .. code-block:: nim - ## task build, "default build is via the C backend": - ## setCommand "c" - proc `name Task`() = body - - let cmd = getCommand() - if cmd.len == 0 or cmd ==? "help": - setCommand "help" - writeTask(astToStr(name), description) - elif cmd ==? astToStr(name): - setCommand "nop" - `name Task`() +proc cppDefine*(define: string) = + ## tell Nim that `define` is a C preprocessor `#define` and so always + ## needs to be mangled. + builtin -var - packageName* = "" ## Nimble support: Set this to the package name. It - ## is usually not required to do that, nims' filename is - ## the default. - version*: string ## Nimble support: The package's version. - author*: string ## Nimble support: The package's author. - description*: string ## Nimble support: The package's description. - license*: string ## Nimble support: The package's license. - srcdir*: string ## Nimble support: The package's source directory. - binDir*: string ## Nimble support: The package's binary directory. - backend*: string ## Nimble support: The package's backend. - - skipDirs*, skipFiles*, skipExt*, installDirs*, installFiles*, - installExt*, bin*: seq[string] = @[] ## Nimble metadata. - requiresData*: seq[string] = @[] ## Exposes the list of requirements for read - ## and write accesses. - -proc requires*(deps: varargs[string]) = - ## Nimble support: Call this to set the list of requirements of your Nimble - ## package. - for d in deps: requiresData.add(d) +proc stdinReadLine(): string {. + tags: [ReadIOEffect], raises: [IOError].} = + builtin + +proc stdinReadAll(): string {. + tags: [ReadIOEffect], raises: [IOError].} = + builtin + +proc readLineFromStdin*(): string {.raises: [IOError].} = + ## Reads a line of data from stdin - blocks until \n or EOF which happens when stdin is closed + log "readLineFromStdin": + result = stdinReadLine() + checkError(EOFError) + +proc readAllFromStdin*(): string {.raises: [IOError].} = + ## Reads all data from stdin - blocks until EOF which happens when stdin is closed + log "readAllFromStdin": + result = stdinReadAll() + checkError(EOFError) + +when not defined(nimble): + template `==?`(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 + template task*(name: untyped; description: string; body: untyped): untyped = + ## Defines a task. Hidden tasks are supported via an empty description. + ## + ## Example: + ## ```nim + ## task build, "default build is via the C backend": + ## setCommand "c" + ## ``` + ## + ## For a task named `foo`, this template generates a `proc` named + ## `fooTask`. This is useful if you need to call one task in + ## another in your Nimscript. + ## + ## Example: + ## + ## ```nim + ## task foo, "foo": # > nim foo + ## echo "Running foo" # Running foo + ## + ## task bar, "bar": # > nim bar + ## echo "Running bar" # Running bar + ## fooTask() # Running foo + ## ``` + proc `name Task`*() = + setCommand "nop" + body + + let cmd = getCommand() + if cmd.len == 0 or cmd ==? "help": + setCommand "help" + writeTask(astToStr(name), description) + elif cmd ==? astToStr(name): + `name Task`() + + # nimble has its own implementation for these things. + var + packageName* = "" ## Nimble support: Set this to the package name. It + ## is usually not required to do that, nims' filename is + ## the default. + version*: string ## Nimble support: The package's version. + author*: string ## Nimble support: The package's author. + description*: string ## Nimble support: The package's description. + license*: string ## Nimble support: The package's license. + srcDir*: string ## Nimble support: The package's source directory. + binDir*: string ## Nimble support: The package's binary directory. + backend*: string ## Nimble support: The package's backend. + + skipDirs*, skipFiles*, skipExt*, installDirs*, installFiles*, + installExt*, bin*: seq[string] = @[] ## Nimble metadata. + requiresData*: seq[string] = @[] ## Exposes the list of requirements for read + ## and write accesses. + + proc requires*(deps: varargs[string]) = + ## Nimble support: Call this to set the list of requirements of your Nimble + ## package. + for d in deps: requiresData.add(d) {.pop.} diff --git a/lib/system/orc.nim b/lib/system/orc.nim new file mode 100644 index 000000000..c02a24989 --- /dev/null +++ b/lib/system/orc.nim @@ -0,0 +1,543 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Cycle collector based on +# https://www.cs.purdue.edu/homes/hosking/690M/Bacon01Concurrent.pdf +# And ideas from Lins' in 2008 by the notion of "critical links", see +# "Cyclic reference counting" by Rafael Dueire Lins +# R.D. Lins / Information Processing Letters 109 (2008) 71–78 +# + +include cellseqs_v2 + +const + colBlack = 0b000 + colGray = 0b001 + colWhite = 0b010 + maybeCycle = 0b100 # possibly part of a cycle; this has to be a "sticky" bit + jumpStackFlag = 0b1000 + colorMask = 0b011 + + logOrc = defined(nimArcIds) + +type + TraceProc = proc (p, env: pointer) {.nimcall, benign.} + DisposeProc = proc (p: pointer) {.nimcall, benign.} + +template color(c): untyped = c.rc and colorMask +template setColor(c, col) = + when col == colBlack: + c.rc = c.rc and not colorMask + else: + c.rc = c.rc and not colorMask or col + +const + optimizedOrc = false # not defined(nimOldOrc) +# XXX Still incorrect, see tests/arc/tdestroy_in_loopcond + +proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = + let h = head(p) + inc h.rc, rcIncrement + when optimizedOrc: + if cyclic: + h.rc = h.rc or maybeCycle + +proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = + when optimizedOrc: + if p != nil: + let h = head(p) + h.rc = h.rc or maybeCycle + +proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} = + # This is only used by the old RTTI mechanism and we know + # that 'dest[]' is nil and needs no destruction. Which is really handy + # as we cannot destroy the object reliably if it's an object of unknown + # compile-time type. + dest[] = src + if src != nil: nimIncRefCyclic(src, true) + +const + useJumpStack = false # for thavlak the jump stack doesn't improve the performance at all + +type + GcEnv = object + traceStack: CellSeq[ptr pointer] + when useJumpStack: + jumpStack: CellSeq[ptr pointer] # Lins' jump stack in order to speed up traversals + toFree: CellSeq[Cell] + freed, touched, edges, rcSum: int + keepThreshold: bool + +proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = + if desc.traceImpl != nil: + var p = s +! sizeof(RefHeader) + cast[TraceProc](desc.traceImpl)(p, addr(j)) + +include threadids + +when logOrc or orcLeakDetector: + proc writeCell(msg: cstring; s: Cell; desc: PNimTypeV2) = + when orcLeakDetector: + cfprintf(cstderr, "%s %s file: %s:%ld; color: %ld; thread: %ld\n", + msg, desc.name, s.filename, s.line, s.color, getThreadId()) + else: + cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld; thread: %ld\n", + msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color, getThreadId()) + +proc free(s: Cell; desc: PNimTypeV2) {.inline.} = + when traceCollector: + cprintf("[From ] %p rc %ld color %ld\n", s, s.rc shr rcShift, s.color) + let p = s +! sizeof(RefHeader) + + when logOrc: writeCell("free", s, desc) + + if desc.destructor != nil: + cast[DestructorProc](desc.destructor)(p) + + when false: + cstderr.rawWrite desc.name + cstderr.rawWrite " " + if desc.destructor == nil: + cstderr.rawWrite "lacks dispose" + if desc.traceImpl != nil: + cstderr.rawWrite ", but has trace\n" + else: + cstderr.rawWrite ", and lacks trace\n" + else: + cstderr.rawWrite "has dispose!\n" + + nimRawDispose(p, desc.align) + +template orcAssert(cond, msg) = + when logOrc: + if not cond: + cfprintf(cstderr, "[Bug!] %s\n", msg) + rawQuit 1 + +when logOrc: + proc strstr(s, sub: cstring): cstring {.header: "<string.h>", importc.} + +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + + orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!" + + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, desc) + +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) + +var + roots {.threadvar.}: CellSeq[Cell] + +proc unregisterCycle(s: Cell) = + # swap with the last element. O(1) + let idx = s.rootIdx-1 + when false: + if idx >= roots.len or idx < 0: + cprintf("[Bug!] %ld %ld\n", idx, roots.len) + rawQuit 1 + roots.d[idx] = roots.d[roots.len-1] + roots.d[idx][0].rootIdx = idx+1 + dec roots.len + s.rootIdx = 0 + +proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + #[ + proc scanBlack(s: Cell) = + setColor(s, colBlack) + for t in sons(s): + t.rc = t.rc + rcIncrement + if t.color != colBlack: + scanBlack(t) + ]# + s.setColor colBlack + let until = j.traceStack.len + trace(s, desc, j) + when logOrc: writeCell("root still alive", s, desc) + while j.traceStack.len > until: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + inc t.rc, rcIncrement + if t.color != colBlack: + t.setColor colBlack + trace(t, desc, j) + when logOrc: writeCell("child still alive", t, desc) + +proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + #[ + proc markGray(s: Cell) = + if s.color != colGray: + setColor(s, colGray) + for t in sons(s): + t.rc = t.rc - rcIncrement + if t.color != colGray: + markGray(t) + ]# + if s.color != colGray: + s.setColor colGray + inc j.touched + # keep in mind that refcounts are zero based so add 1 here: + inc j.rcSum, (s.rc shr rcShift) + 1 + orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + dec t.rc, rcIncrement + inc j.edges + when useJumpStack: + if (t.rc shr rcShift) >= 0 and (t.rc and jumpStackFlag) == 0: + t.rc = t.rc or jumpStackFlag + when traceCollector: + cprintf("[Now in jumpstack] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) + j.jumpStack.add(entry, desc) + if t.color != colGray: + t.setColor colGray + inc j.touched + # we already decremented its refcount so account for that: + inc j.rcSum, (t.rc shr rcShift) + 2 + trace(t, desc, j) + +proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + #[ + proc scan(s: Cell) = + if s.color == colGray: + if s.rc > 0: + scanBlack(s) + else: + s.setColor(colWhite) + for t in sons(s): scan(t) + ]# + if s.color == colGray: + if (s.rc shr rcShift) >= 0: + scanBlack(s, desc, j) + # XXX this should be done according to Lins' paper but currently breaks + #when useJumpStack: + # s.setColor colPurple + else: + when useJumpStack: + # first we have to repair all the nodes we have seen + # that are still alive; we also need to mark what they + # refer to as alive: + while j.jumpStack.len > 0: + let (entry, desc) = j.jumpStack.pop + let t = head entry[] + # not in jump stack anymore! + t.rc = t.rc and not jumpStackFlag + if t.color == colGray and (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + # XXX this should be done according to Lins' paper but currently breaks + #t.setColor colPurple + when traceCollector: + cprintf("[jump stack] %p %ld\n", t, t.rc shr rcShift) + + orcAssert(j.traceStack.len == 0, "scan: trace stack not empty") + s.setColor(colWhite) + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + if t.color == colGray: + if (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + else: + when useJumpStack: + # first we have to repair all the nodes we have seen + # that are still alive; we also need to mark what they + # refer to as alive: + while j.jumpStack.len > 0: + let (entry, desc) = j.jumpStack.pop + let t = head entry[] + # not in jump stack anymore! + t.rc = t.rc and not jumpStackFlag + if t.color == colGray and (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + # XXX this should be done according to Lins' paper but currently breaks + #t.setColor colPurple + when traceCollector: + cprintf("[jump stack] %p %ld\n", t, t.rc shr rcShift) + + t.setColor(colWhite) + trace(t, desc, j) + +when false: + proc writeCell(msg: cstring; s: Cell) = + cfprintf(cstderr, "%s %p root index: %ld; RC: %ld; color: %ld\n", + msg, s, s.rootIdx, s.rc shr rcShift, s.color) + +proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = + #[ + was: 'collectWhite'. + + proc collectWhite(s: Cell) = + if s.color == colWhite and not buffered(s): + s.setColor(colBlack) + for t in sons(s): + collectWhite(t) + free(s) # watch out, a bug here! + ]# + if s.color == col and s.rootIdx == 0: + orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty") + + s.setColor(colBlack) + j.toFree.add(s, desc) + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + entry[] = nil # ensure that the destructor does touch moribund objects! + if t.color == col and t.rootIdx == 0: + j.toFree.add(t, desc) + t.setColor(colBlack) + trace(t, desc, j) + +const + defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128 + +when defined(nimStressOrc): + const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8 +else: + var rootsThreshold {.threadvar.}: int + +proc collectCyclesBacon(j: var GcEnv; lowMark: int) = + # pretty direct translation from + # https://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf + # Fig. 2. Synchronous Cycle Collection + #[ + for s in roots: + markGray(s) + for s in roots: + scan(s) + for s in roots: + remove s from roots + s.buffered = false + collectWhite(s) + ]# + let last = roots.len - 1 + when logOrc: + for i in countdown(last, lowMark): + writeCell("root", roots.d[i][0], roots.d[i][1]) + + for i in countdown(last, lowMark): + markGray(roots.d[i][0], roots.d[i][1], j) + + var colToCollect = colWhite + if j.rcSum == j.edges: + # short-cut: we know everything is garbage: + colToCollect = colGray + # remember the fact that we got so lucky: + j.keepThreshold = true + else: + for i in countdown(last, lowMark): + scan(roots.d[i][0], roots.d[i][1], j) + + init j.toFree + for i in 0 ..< roots.len: + let s = roots.d[i][0] + s.rootIdx = 0 + collectColor(s, roots.d[i][1], colToCollect, j) + + # Bug #22927: `free` calls destructors which can append to `roots`. + # We protect against this here by setting `roots.len` to 0 and also + # setting the threshold so high that no cycle collection can be triggered + # until we are out of this critical section: + when not defined(nimStressOrc): + let oldThreshold = rootsThreshold + rootsThreshold = high(int) + roots.len = 0 + + for i in 0 ..< j.toFree.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) + free(j.toFree.d[i][0], j.toFree.d[i][1]) + + when not defined(nimStressOrc): + rootsThreshold = oldThreshold + + inc j.freed, j.toFree.len + deinit j.toFree + +when defined(nimOrcStats): + var freedCyclicObjects {.threadvar.}: int + +proc partialCollect(lowMark: int) = + when false: + if roots.len < 10 + lowMark: return + when logOrc: + cfprintf(cstderr, "[partialCollect] begin\n") + var j: GcEnv + init j.traceStack + collectCyclesBacon(j, lowMark) + when logOrc: + cfprintf(cstderr, "[partialCollect] end; freed %ld touched: %ld work: %ld\n", j.freed, j.touched, + roots.len - lowMark) + roots.len = lowMark + deinit j.traceStack + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed + +proc collectCycles() = + ## Collect cycles. + when logOrc: + cfprintf(cstderr, "[collectCycles] begin\n") + + var j: GcEnv + init j.traceStack + when useJumpStack: + init j.jumpStack + collectCyclesBacon(j, 0) + while j.jumpStack.len > 0: + let (t, desc) = j.jumpStack.pop + # not in jump stack anymore! + t.rc = t.rc and not jumpStackFlag + deinit j.jumpStack + else: + collectCyclesBacon(j, 0) + + deinit j.traceStack + if roots.len == 0: + deinit roots + + when not defined(nimStressOrc): + # compute the threshold based on the previous history + # of the cycle collector's effectiveness: + # we're effective when we collected 50% or more of the nodes + # we touched. If we're effective, we can reset the threshold: + if j.keepThreshold: + discard + elif j.freed * 2 >= j.touched: + when not defined(nimFixedOrc): + rootsThreshold = max(rootsThreshold div 3 * 2, 16) + else: + rootsThreshold = 0 + #cfprintf(cstderr, "[collectCycles] freed %ld, touched %ld new threshold %ld\n", j.freed, j.touched, rootsThreshold) + elif rootsThreshold < high(int) div 4: + rootsThreshold = (if rootsThreshold <= 0: defaultThreshold else: rootsThreshold) * 3 div 2 + when logOrc: + cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld touched: %ld mem: %ld rcSum: %ld edges: %ld\n", j.freed, rootsThreshold, j.touched, + getOccupiedMem(), j.rcSum, j.edges) + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed + +when defined(nimOrcStats): + type + OrcStats* = object ## Statistics of the cycle collector subsystem. + freedCyclicObjects*: int ## Number of freed cyclic objects. + proc GC_orcStats*(): OrcStats = + ## Returns the statistics of the cycle collector subsystem. + result = OrcStats(freedCyclicObjects: freedCyclicObjects) + +proc registerCycle(s: Cell; desc: PNimTypeV2) = + s.rootIdx = roots.len+1 + if roots.d == nil: init(roots) + add(roots, s, desc) + + if roots.len - defaultThreshold >= rootsThreshold: + collectCycles() + when logOrc: + writeCell("[added root]", s, desc) + + orcAssert strstr(desc.name, "TType") == nil, "added a TType as a root!" + +proc GC_runOrc* = + ## Forces a cycle collection pass. + collectCycles() + orcAssert roots.len == 0, "roots not empty!" + +proc GC_enableOrc*() = + ## Enables the cycle collector subsystem of `--mm:orc`. This is a `--mm:orc` + ## specific API. Check with `when defined(gcOrc)` for its existence. + when not defined(nimStressOrc): + rootsThreshold = 0 + +proc GC_disableOrc*() = + ## Disables the cycle collector subsystem of `--mm:orc`. This is a `--mm:orc` + ## specific API. Check with `when defined(gcOrc)` for its existence. + when not defined(nimStressOrc): + rootsThreshold = high(int) + +proc GC_prepareOrc*(): int {.inline.} = roots.len + +proc GC_partialCollect*(limit: int) = + partialCollect(limit) + +proc GC_fullCollect* = + ## Forces a full garbage collection pass. With `--mm:orc` triggers the cycle + ## collector. This is an alias for `GC_runOrc`. + collectCycles() + +proc GC_enableMarkAndSweep*() = + ## For `--mm:orc` an alias for `GC_enableOrc`. + GC_enableOrc() + +proc GC_disableMarkAndSweep*() = + ## For `--mm:orc` an alias for `GC_disableOrc`. + GC_disableOrc() + +const + acyclicFlag = 1 # see also cggtypes.nim, proc genTypeInfoV2Impl + +when optimizedOrc: + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 and (s.rc and maybeCycle) != 0 +else: + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 + +proc rememberCycle(isDestroyAction: bool; s: Cell; desc: PNimTypeV2) {.noinline.} = + if isDestroyAction: + if s.rootIdx > 0: + unregisterCycle(s) + else: + # do not call 'rememberCycle' again unless this cell + # got an 'incRef' event: + if s.rootIdx == 0 and markedAsCyclic(s, desc): + s.setColor colBlack + registerCycle(s, desc) + +proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + rememberCycle(result, cell, cast[ptr PNimTypeV2](p)[]) + +proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + if result: + if cell.rootIdx > 0: + unregisterCycle(cell) + +proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p %s\n", p, desc.name) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + rememberCycle(result, cell, desc) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim new file mode 100644 index 000000000..5509d0070 --- /dev/null +++ b/lib/system/osalloc.nim @@ -0,0 +1,218 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + sysAssert(result >= x, "roundup: result < x") + #return ((-x) and (v-1)) +% x + +sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") +sysAssert(roundup(15, 8) == 16, "roundup broken") +sysAssert(roundup(65, 8) == 72, "roundup broken 2") + +# ------------ platform specific chunk allocation code ----------- + +# some platforms have really weird unmap behaviour: +# unmap(blockStart, PageSize) +# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 +# and x86 are safe though; Windows is special because MEM_RELEASE can only be +# used with a size of 0. We also allow unmapping to be turned off with +# -d:nimAllocNoUnmap: +const doNotUnmap = not (defined(amd64) or defined(i386)) or + defined(windows) or defined(nimAllocNoUnmap) + + +when defined(nimAllocPagesViaMalloc): + when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + {.error: "-d:nimAllocPagesViaMalloc is only supported with --mm:arc or --mm:atomicArc or --mm:orc".} + + proc osTryAllocPages(size: int): pointer {.inline.} = + let base = c_malloc(csize_t size + PageSize - 1 + sizeof(uint32)) + if base == nil: raiseOutOfMem() + # memory layout: padding + offset (4 bytes) + user_data + # in order to deallocate: read offset at user_data - 4 bytes, + # then deallocate user_data - offset + let offset = PageSize - (cast[int](base) and (PageSize - 1)) + cast[ptr uint32](base +! (offset - sizeof(uint32)))[] = uint32(offset) + result = base +! offset + + proc osAllocPages(size: int): pointer {.inline.} = + result = osTryAllocPages(size) + if result == nil: raiseOutOfMem() + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # read offset at p - 4 bytes, then deallocate (p - offset) pointer + let offset = cast[ptr uint32](p -! sizeof(uint32))[] + c_free(p -! offset) + +elif defined(emscripten) and not defined(StandaloneHeapSize): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + type + PEmscriptenMMapBlock = ptr EmscriptenMMapBlock + EmscriptenMMapBlock {.pure, inheritable.} = object + realSize: int # size of previous chunk; for coalescing + realPointer: pointer # if < PageSize it is a small chunk + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: int) {.header: "<sys/mman.h>".} + + proc osAllocPages(block_size: int): pointer {.inline.} = + let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 + result = mmap(nil, realSize, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + let realPointer = result + let pos = cast[int](result) + + # Convert pointer to PageSize correct one. + var new_pos = cast[int](pos) +% (PageSize - (pos %% PageSize)) + if (new_pos-pos) < sizeof(EmscriptenMMapBlock): + new_pos = new_pos +% PageSize + result = cast[pointer](new_pos) + + var mmapDescrPos = cast[int](result) -% sizeof(EmscriptenMMapBlock) + + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + mmapDescr.realSize = realSize + mmapDescr.realPointer = realPointer + + #c_fprintf(stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + + proc osTryAllocPages(size: int): pointer = osAllocPages(size) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + var mmapDescrPos = cast[int](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + munmap(mmapDescr.realPointer, mmapDescr.realSize) + +elif defined(genode) and not defined(StandaloneHeapSize): + include genode/alloc # osAllocPages, osTryAllocPages, osDeallocPages + +elif defined(posix) and not defined(StandaloneHeapSize): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + + when defined(netbsd) or defined(openbsd): + # OpenBSD security for setjmp/longjmp coroutines + var MAP_STACK {.importc: "MAP_STACK", header: "<sys/mman.h>".}: cint + else: + const MAP_STACK = 0 # avoid sideeffects + + when defined(macosx) or defined(freebsd): + const MAP_ANONYMOUS = 0x1000 + const MAP_PRIVATE = 0x02 # Changes are private + elif defined(solaris): + const MAP_ANONYMOUS = 0x100 + const MAP_PRIVATE = 0x02 # Changes are private + elif defined(linux) and defined(amd64): + # actually, any architecture using asm-generic, but being conservative here, + # some arches like mips and alpha use different values + const MAP_ANONYMOUS = 0x20 + const MAP_PRIVATE = 0x02 # Changes are private + elif defined(haiku): + const MAP_ANONYMOUS = 0x08 + const MAP_PRIVATE = 0x02 + else: # posix including netbsd or openbsd + var + MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + MAP_PRIVATE {.importc: "MAP_PRIVATE", header: "<sys/mman.h>".}: cint + + proc mmap(adr: pointer, len: csize_t, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: csize_t): cint {.header: "<sys/mman.h>".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, cast[csize_t](size), PROT_READ or PROT_WRITE, + MAP_ANONYMOUS or MAP_PRIVATE or MAP_STACK, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, cast[csize_t](size), PROT_READ or PROT_WRITE, + MAP_ANONYMOUS or MAP_PRIVATE or MAP_STACK, -1, 0) + if result == cast[pointer](-1): result = nil + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + when reallyOsDealloc: discard munmap(p, cast[csize_t](size)) + +elif defined(windows) and not defined(StandaloneHeapSize): + const + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + MEM_TOP_DOWN = 0x100000 + PAGE_READWRITE = 0x04 + + MEM_DECOMMIT = 0x4000 + MEM_RELEASE = 0x8000 + + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "<windows.h>", stdcall, importc: "VirtualAlloc".} + + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32): cint {.header: "<windows.h>", stdcall, + importc: "VirtualFree".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + if result == nil: raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # according to Microsoft, 0 is the only correct value for MEM_RELEASE: + # This means that the OS has some different view over how big the block is + # that we want to free! So, we cannot reliably release the memory back to + # Windows :-(. We have to live with MEM_DECOMMIT instead. + # Well that used to be the case but MEM_DECOMMIT fragments the address + # space heavily, so we now treat Windows as a strange unmap target. + when reallyOsDealloc: + if virtualFree(p, 0, MEM_RELEASE) == 0: + cprintf "virtualFree failing!" + rawQuit 1 + #VirtualFree(p, size, MEM_DECOMMIT) + +elif hostOS == "standalone" or defined(StandaloneHeapSize): + const StandaloneHeapSize {.intdefine.}: int = 1024 * PageSize + var + theHeap: array[StandaloneHeapSize div sizeof(float64), float64] # 'float64' for alignment + bumpPointer = cast[int](addr theHeap) + + proc osAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + else: + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + if bumpPointer-size == cast[int](p): + dec bumpPointer, size + +else: + {.error: "Port memory manager to your platform".} diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index eeada5c51..0619f3fca 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -7,9 +7,11 @@ # distribution, for details about the copyright. # -## Platform detection for Nim. This module is included by the system module! +## Platform detection for NimScript. This module is included by the system module! ## Do not import it directly! +# CPU architectures have alias names mapped in tools/niminst/makefile.nimf + type CpuPlatform* {.pure.} = enum ## the CPU this program will run on. none, ## unknown CPU @@ -20,20 +22,30 @@ type powerpc64, ## 64 bit PowerPC powerpc64el, ## Little Endian 64 bit PowerPC sparc, ## Sparc based processor + sparc64, ## 64-bit Sparc based processor + hppa, ## HP PA-RISC ia64, ## Intel Itanium amd64, ## x86_64 (AMD64); 64 bit x86 compatible CPU mips, ## Mips based processor mipsel, ## Little Endian Mips based processor + mips64, ## 64-bit MIPS processor + mips64el, ## Little Endian 64-bit MIPS processor arm, ## ARM based processor arm64, ## ARM64 based processor vm, ## Some Virtual machine: Nim's VM or JavaScript - avr ## AVR based processor - msp430 ## TI MSP430 microcontroller + avr, ## AVR based processor + msp430, ## TI MSP430 microcontroller + riscv32, ## RISC-V 32-bit processor + riscv64, ## RISC-V 64-bit processor + wasm32, ## WASM, 32-bit + e2k, ## MCST Elbrus 2000 + loongarch64, ## LoongArch 64-bit processor + s390x ## IBM Z OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, irix, netbsd, freebsd, openbsd, aix, palmos, qnx, amiga, - atari, netware, macos, macosx, haiku, js, nimVM, standalone + atari, netware, macos, macosx, haiku, android, js, standalone, nintendoswitch const targetOS* = when defined(windows): OsPlatform.windows @@ -56,9 +68,10 @@ const elif defined(macosx): OsPlatform.macosx elif defined(macos): OsPlatform.macos elif defined(haiku): OsPlatform.haiku + elif defined(android): OsPlatform.android elif defined(js): OsPlatform.js - elif defined(nimrodVM): OsPlatform.nimVM elif defined(standalone): OsPlatform.standalone + elif defined(nintendoswitch): OsPlatform.nintendoswitch else: OsPlatform.none ## the OS this program will run on. @@ -69,14 +82,24 @@ const elif defined(powerpc64): CpuPlatform.powerpc64 elif defined(powerpc64el): CpuPlatform.powerpc64el elif defined(sparc): CpuPlatform.sparc + elif defined(sparc64): CpuPlatform.sparc64 + elif defined(hppa): CpuPlatform.hppa elif defined(ia64): CpuPlatform.ia64 elif defined(amd64): CpuPlatform.amd64 elif defined(mips): CpuPlatform.mips elif defined(mipsel): CpuPlatform.mipsel + elif defined(mips64): CpuPlatform.mips64 + elif defined(mips64el): CpuPlatform.mips64el elif defined(arm): CpuPlatform.arm elif defined(arm64): CpuPlatform.arm64 elif defined(vm): CpuPlatform.vm elif defined(avr): CpuPlatform.avr elif defined(msp430): CpuPlatform.msp430 + elif defined(riscv32): CpuPlatform.riscv32 + elif defined(riscv64): CpuPlatform.riscv64 + elif defined(wasm32): CpuPlatform.wasm32 + elif defined(e2k): CpuPlatform.e2k + elif defined(loongarch64): CpuPlatform.loongarch64 + elif defined(s390x): CpuPlatform.s390x else: CpuPlatform.none ## the CPU this program will run on. diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index 4f600417e..e7eb6ac82 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -13,15 +13,21 @@ # (except perhaps loops that have no side-effects). At every Nth call a # stack trace is taken. A stack tace is a list of cstrings. +when defined(profiler) and defined(memProfiler): + {.error: "profiler and memProfiler cannot be defined at the same time (See Embedded Stack Trace Profiler (ESTP) User Guide) for more details".} + {.push profiler: off.} const MaxTraceLen = 20 # tracking the last 20 calls is enough type - StackTrace* = array [0..MaxTraceLen-1, cstring] + StackTrace* = object + lines*: array[0..MaxTraceLen-1, cstring] + files*: array[0..MaxTraceLen-1, cstring] ProfilerHook* = proc (st: StackTrace) {.nimcall.} -{.deprecated: [TStackTrace: StackTrace, TProfilerHook: ProfilerHook].} + +proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] proc captureStackTrace(f: PFrame, st: var StackTrace) = const @@ -30,9 +36,10 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = it = f i = 0 total = 0 - while it != nil and i <= high(st)-(firstCalls-1): + while it != nil and i <= high(st.lines)-(firstCalls-1): # the (-1) is for the "..." entry - st[i] = it.procname + st.lines[i] = it.procname + st.files[i] = it.filename inc(i) inc(total) it = it.prev @@ -43,17 +50,24 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = for j in 1..total-i-(firstCalls-1): if b != nil: b = b.prev if total != i: - st[i] = "..." + st.lines[i] = "..." + st.files[i] = "..." inc(i) - while b != nil and i <= high(st): - st[i] = b.procname + while b != nil and i <= high(st.lines): + st.lines[i] = b.procname + st.files[i] = b.filename inc(i) b = b.prev +var + profilingRequestedHook*: proc (): bool {.nimcall, gcsafe.} + ## set this variable to provide a procedure that implements a profiler in + ## user space. See the `nimprof` module for a reference implementation. + when defined(memProfiler): type - MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, benign.} - {.deprecated: [TMemProfilerHook: MemProfilerHook].} + MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, gcsafe.} + var profilerHook*: MemProfilerHook ## set this variable to provide a procedure that implements a profiler in @@ -65,37 +79,25 @@ when defined(memProfiler): hook(st, requestedSize) proc nimProfile(requestedSize: int) = - if not isNil(profilerHook): + if not isNil(profilingRequestedHook) and profilingRequestedHook(): callProfilerHook(profilerHook, requestedSize) else: - const - SamplingInterval = 50_000 - # set this to change the default sampling interval var profilerHook*: ProfilerHook ## set this variable to provide a procedure that implements a profiler in ## user space. See the `nimprof` module for a reference implementation. - gTicker {.threadvar.}: int proc callProfilerHook(hook: ProfilerHook) {.noinline.} = # 'noinline' so that 'nimProfile' does not perform the stack allocation # in the common case. - var st: StackTrace - captureStackTrace(framePtr, st) - hook(st) + when not defined(nimdoc): + var st: StackTrace + captureStackTrace(framePtr, st) + hook(st) proc nimProfile() = ## This is invoked by the compiler in every loop and on every proc entry! - if gTicker == 0: - gTicker = -1 - if not isNil(profilerHook): - # disable recursive calls: XXX should use try..finally, - # but that's too expensive! - let oldHook = profilerHook - profilerHook = nil - callProfilerHook(oldHook) - profilerHook = oldHook - gTicker = SamplingInterval - dec gTicker + if not isNil(profilingRequestedHook) and profilingRequestedHook(): + callProfilerHook(profilerHook) {.pop.} diff --git a/lib/system/rawquits.nim b/lib/system/rawquits.nim new file mode 100644 index 000000000..f0ead10c6 --- /dev/null +++ b/lib/system/rawquits.nim @@ -0,0 +1,27 @@ +import system/ctypes + +when defined(nimNoQuit): + proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(genode): + import genode/env + + var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr + + type GenodeEnv = GenodeEnvPtr + ## Opaque type representing Genode environment. + + proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, + importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "<base/sleep.h>".} + + proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = + systemEnv.rawQuit(errorcode) + + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc rawQuit(errorcode: cint) {. + magic: "Exit", importc: "exit", header: "<stdlib.h>", noreturn.} \ No newline at end of file diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 1f81a0813..13118e40b 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -16,40 +16,24 @@ proc reprInt(x: int64): string {.compilerproc.} = return $x proc reprFloat(x: float): string {.compilerproc.} = return $x proc reprPointer(x: pointer): string {.compilerproc.} = - var buf: array [0..59, char] - discard c_sprintf(buf, "%p", x) - return $buf + result = newString(60) + let n = c_snprintf(cast[cstring](addr result[0]), csize_t(60), "%p", x) + setLen(result, n) -proc `$`(x: uint64): string = - if x == 0: - result = "0" - else: - var buf: array [60, char] - var i = 0 - var n = x - while n != 0: - let nn = n div 10'u64 - buf[i] = char(n - 10'u64 * nn + ord('0')) - inc i - n = nn - - let half = i div 2 - # Reverse - for t in 0 .. < half: swap(buf[t], buf[i-t-1]) - result = $buf - -proc reprStrAux(result: var string, s: string) = +proc reprStrAux(result: var string, s: cstring; len: int) = if cast[pointer](s) == nil: add result, "nil" return - add result, reprPointer(cast[pointer](s)) & "\"" - for i in 0.. <s.len: + if len > 0: + add result, reprPointer(cast[pointer](s)) + add result, "\"" + for i in 0 .. pred(len): let c = s[i] case c of '"': add result, "\\\"" of '\\': add result, "\\\\" # BUGFIX: forgotten of '\10': add result, "\\10\"\n\"" # " \n " # better readability - of '\128' .. '\255', '\0'..'\9', '\11'..'\31': + of '\127' .. '\255', '\0'..'\9', '\11'..'\31': add result, "\\" & reprInt(ord(c)) else: result.add(c) @@ -57,7 +41,7 @@ proc reprStrAux(result: var string, s: string) = proc reprStr(s: string): string {.compilerRtl.} = result = "" - reprStrAux(result, s) + reprStrAux(result, s, s.len) proc reprBool(x: bool): string {.compilerRtl.} = if x: result = "true" @@ -68,30 +52,30 @@ proc reprChar(x: char): string {.compilerRtl.} = case x of '"': add result, "\\\"" of '\\': add result, "\\\\" - of '\128' .. '\255', '\0'..'\31': add result, "\\" & reprInt(ord(x)) + of '\127' .. '\255', '\0'..'\31': add result, "\\" & reprInt(ord(x)) else: add result, x add result, "\'" proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - # we read an 'int' but this may have been too large, so mask the other bits: - let e = if typ.size == 1: e and 0xff - elif typ.size == 2: e and 0xffff - else: e - # XXX we need a proper narrowing based on signedness here - #e and ((1 shl (typ.size*8)) - 1) + ## Return string representation for enumeration values + var n = typ.node if ntfEnumHole notin typ.flags: - if e <% typ.node.len: - return $typ.node.sons[e].name + let o = e - n.sons[0].offset + if o >= 0 and o <% typ.node.len: + return $n.sons[o].name else: # ugh we need a slow linear search: - var n = typ.node var s = n.sons for i in 0 .. n.len-1: - if s[i].offset == e: return $s[i].name + if s[i].offset == e: + return $s[i].name + result = $e & " (invalid data!)" +include system/repr_impl + type - PByteArray = ptr array[0.. 0xffff, int8] + PByteArray = ptr UncheckedArray[byte] # array[0xffff, byte] proc addSetElem(result: var string, elem: int, typ: PNimType) {.benign.} = case typ.kind @@ -108,22 +92,23 @@ proc reprSetAux(result: var string, p: pointer, typ: PNimType) = var elemCounter = 0 # we need this flag for adding the comma at # the right places add result, "{" - var u: int64 + var u: uint64 case typ.size - of 1: u = ze64(cast[ptr int8](p)[]) - of 2: u = ze64(cast[ptr int16](p)[]) - of 4: u = ze64(cast[ptr int32](p)[]) - of 8: u = cast[ptr int64](p)[] + of 1: u = cast[ptr uint8](p)[] + of 2: u = cast[ptr uint16](p)[] + of 4: u = cast[ptr uint32](p)[] + of 8: u = cast[ptr uint64](p)[] else: + u = uint64(0) var a = cast[PByteArray](p) for i in 0 .. typ.size*8-1: - if (ze(a[i div 8]) and (1 shl (i mod 8))) != 0: + if (uint(a[i shr 3]) and (1'u shl (i and 7))) != 0: if elemCounter > 0: add result, ", " addSetElem(result, i+typ.node.len, typ.base) inc(elemCounter) if typ.size <= 8: for i in 0..sizeof(int64)*8-1: - if (u and (1'i64 shl int64(i))) != 0'i64: + if (u and (1'u64 shl uint64(i))) != 0'u64: if elemCounter > 0: add result, ", " addSetElem(result, i+typ.node.len, typ.base) inc(elemCounter) @@ -140,7 +125,6 @@ type marked: CellSet recdepth: int # do not recurse endlessly indent: int # indentation -{.deprecated: [TReprClosure: ReprClosure].} when not defined(useNimRtl): proc initReprClosure(cl: var ReprClosure) = @@ -171,19 +155,35 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..typ.size div bs - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), typ.base, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), typ.base, cl) add result, "]" + when defined(nimSeqsV2): + type + GenericSeq = object + len: int + p: pointer + PGenericSeq = ptr GenericSeq + const payloadOffset = sizeof(int) + sizeof(pointer) + # see seqs.nim: cap: int + # region: Allocator + + template payloadPtr(x: untyped): untyped = cast[PGenericSeq](x).p + else: + const payloadOffset = GenericSeqSize ## the payload offset always depends on the alignment of the member type. + template payloadPtr(x: untyped): untyped = x + proc reprSequence(result: var string, p: pointer, typ: PNimType, cl: var ReprClosure) = if p == nil: - add result, "nil" + add result, "[]" return - result.add(reprPointer(p) & "[") + result.add(reprPointer(p)) + result.add "@[" var bs = typ.base.size for i in 0..cast[PGenericSeq](p).len-1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + GenericSeqSize + i*bs), + reprAux(result, cast[pointer](cast[int](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), typ.base, cl) add result, "]" @@ -194,14 +194,14 @@ when not defined(useNimRtl): of nkSlot: add result, $n.name add result, " = " - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) of nkList: for i in 0..n.len-1: if i > 0: add result, ",\n" reprRecordAux(result, p, n.sons[i], cl) of nkCase: var m = selectBranch(p, n) - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) if m != nil: reprRecordAux(result, p, m, cl) proc reprRecord(result: var string, p: pointer, typ: PNimType, @@ -224,17 +224,26 @@ when not defined(useNimRtl): cl: var ReprClosure) = # we know that p is not nil here: when declared(CellSet): - when defined(boehmGC) or defined(gogc) or defined(nogc): + when defined(boehmGC) or defined(gogc) or defined(nogc) or usesDestructors: var cell = cast[PCell](p) else: var cell = usrToCell(p) - add result, "ref " & reprPointer(p) + add result, if typ.kind == tyPtr: "ptr " else: "ref " + add result, reprPointer(p) if cell notin cl.marked: # only the address is shown: incl(cl.marked, cell) add result, " --> " reprAux(result, p, typ.base, cl) + proc getInt(p: pointer, size: int): int = + case size + of 1: return int(cast[ptr uint8](p)[]) + of 2: return int(cast[ptr uint16](p)[]) + of 4: return int(cast[ptr uint32](p)[]) + of 8: return int(cast[ptr uint64](p)[]) + else: discard + proc reprAux(result: var string, p: pointer, typ: PNimType, cl: var ReprClosure) = if cl.recdepth == 0: @@ -259,40 +268,48 @@ when not defined(useNimRtl): of tyInt16: add result, $int(cast[ptr int16](p)[]) of tyInt32: add result, $int(cast[ptr int32](p)[]) of tyInt64: add result, $(cast[ptr int64](p)[]) - of tyUInt8: add result, $ze(cast[ptr int8](p)[]) - of tyUInt16: add result, $ze(cast[ptr int16](p)[]) + of tyUInt: add result, $(cast[ptr uint](p)[]) + of tyUInt8: add result, $(cast[ptr uint8](p)[]) + of tyUInt16: add result, $(cast[ptr uint16](p)[]) + of tyUInt32: add result, $(cast[ptr uint32](p)[]) + of tyUInt64: add result, $(cast[ptr uint64](p)[]) of tyFloat: add result, $(cast[ptr float](p)[]) of tyFloat32: add result, $(cast[ptr float32](p)[]) of tyFloat64: add result, $(cast[ptr float64](p)[]) - of tyEnum: add result, reprEnum(cast[ptr int](p)[], typ) + of tyEnum: add result, reprEnum(getInt(p, typ.size), typ) of tyBool: add result, reprBool(cast[ptr bool](p)[]) of tyChar: add result, reprChar(cast[ptr char](p)[]) - of tyString: reprStrAux(result, cast[ptr string](p)[]) - of tyCString: + of tyString: + let sp = cast[ptr string](p) + reprStrAux(result, sp[].cstring, sp[].len) + of tyCstring: let cs = cast[ptr cstring](p)[] if cs.isNil: add result, "nil" - else: reprStrAux(result, $cs) + else: reprStrAux(result, cs, cs.len) of tyRange: reprAux(result, p, typ.base, cl) of tyProc, tyPointer: if cast[PPointer](p)[] == nil: add result, "nil" else: add result, reprPointer(cast[PPointer](p)[]) + of tyUncheckedArray: + add result, "[...]" else: add result, "(invalid data!)" inc(cl.recdepth) -proc reprOpenArray(p: pointer, length: int, elemtyp: PNimType): string {. - compilerRtl.} = - var - cl: ReprClosure - initReprClosure(cl) - result = "[" - var bs = elemtyp.size - for i in 0..length - 1: - if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), elemtyp, cl) - add result, "]" - deinitReprClosure(cl) +when not defined(useNimRtl): + proc reprOpenArray(p: pointer, length: int, elemtyp: PNimType): string {. + compilerRtl.} = + var + cl: ReprClosure + initReprClosure(cl) + result = "[" + var bs = elemtyp.size + for i in 0..length - 1: + if i > 0: add result, ", " + reprAux(result, cast[pointer](cast[int](p) + i*bs), elemtyp, cl) + add result, "]" + deinitReprClosure(cl) when not defined(useNimRtl): proc reprAny(p: pointer, typ: PNimType): string = @@ -305,5 +322,6 @@ when not defined(useNimRtl): else: var p = p reprAux(result, addr(p), typ, cl) - add result, "\n" + when defined(nimLegacyReprWithNewline): # see PR #16034 + add result, "\n" deinitReprClosure(cl) diff --git a/lib/system/repr_impl.nim b/lib/system/repr_impl.nim new file mode 100644 index 000000000..b9ec1890f --- /dev/null +++ b/lib/system/repr_impl.nim @@ -0,0 +1,15 @@ +#[ +other APIs common to system/repr and system/reprjs could be refactored here, eg: +* reprChar +* reprBool +* reprStr + +Another possibility in future work would be to have a single include file instead +of system/repr and system/reprjs, and use `when defined(js)` inside it. +]# + +proc reprDiscriminant*(e: int, typ: PNimType): string {.compilerRtl.} = + case typ.kind + of tyEnum: reprEnum(e, typ) + of tyBool: $(e != 0) + else: $e diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim new file mode 100644 index 000000000..d2aef536c --- /dev/null +++ b/lib/system/repr_v2.nim @@ -0,0 +1,194 @@ +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/formatfloat + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + ## imported from typetraits + +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} + ## imported from typetraits + +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + +proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} + +proc repr*(x: int): string = + ## Same as $x + $x + +proc repr*(x: int64): string = + ## Same as $x + $x + +proc repr*(x: uint64): string {.noSideEffect.} = + ## Same as $x + $x + +proc repr*(x: float): string = + ## Same as $x + $x + +proc repr*(x: bool): string {.magic: "BoolToStr", noSideEffect.} + ## repr for a boolean argument. Returns `x` + ## converted to the string "false" or "true". + +proc repr*(x: char): string {.noSideEffect, raises: [].} = + ## repr for a character argument. Returns `x` + ## converted to an escaped string. + ## + ## ```Nim + ## assert repr('c') == "'c'" + ## ``` + result = "'" + # Elides string creations if not needed + if x in {'\\', '\0'..'\31', '\127'..'\255'}: + result.add '\\' + if x in {'\0'..'\31', '\127'..'\255'}: + result.add $x.uint8 + else: + result.add x + result.add '\'' + +proc repr*(x: string | cstring): string {.noSideEffect, raises: [].} = + ## repr for a string argument. Returns `x` + ## converted to a quoted and escaped string. + result = "\"" + for i in 0..<x.len: + if x[i] in {'"', '\\', '\0'..'\31', '\127'..'\255'}: + result.add '\\' + case x[i]: + of '\n': + result.add "n\n" + of '\0'..'\9', '\11'..'\31', '\127'..'\255': + result.add $x[i].uint8 + else: + result.add x[i] + result.add '\"' + +proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect, raises: [].} + ## repr for an enumeration argument. This works for + ## any enumeration type thanks to compiler magic. + ## + ## If a `repr` operator for a concrete enumeration is provided, this is + ## used instead. (In other words: *Overwriting* is possible.) + +proc reprDiscriminant*(e: int): string {.compilerproc.} = + # repr and reprjs can use `PNimType` to symbolize `e`; making this work here + # would require a way to pass the set of enum stringified values to cgen. + $e + +proc repr*(p: pointer): string = + ## repr of pointer as its hexadecimal value + if p == nil: + result = "nil" + else: + when nimvm: + result = "ptr" + else: + const HexChars = "0123456789ABCDEF" + const len = sizeof(pointer) * 2 + var n = cast[uint](p) + result = newString(len) + for j in countdown(len-1, 0): + result[j] = HexChars[n and 0xF] + n = n shr 4 + +proc repr*(p: proc | iterator {.closure.}): string = + ## repr of a proc as its address + repr(cast[ptr pointer](unsafeAddr p)[]) + +template repr*[T: distinct|(range and not enum)](x: T): string = + when T is range: # add a branch to handle range + repr(rangeBase(typeof(x))(x)) + elif T is distinct: + repr(distinctBase(typeof(x))(x)) + else: + {.error: "cannot happen".} + +template repr*(t: typedesc): string = $t + +proc reprObject[T: tuple|object](res: var string, x: T) {.noSideEffect, raises: [].} = + res.add '(' + var firstElement = true + const isNamed = T is object or isNamedTuple(T) + when not isNamed: + var count = 0 + for name, value in fieldPairs(x): + if not firstElement: res.add(", ") + when isNamed: + res.add(name) + res.add(": ") + else: + count.inc + res.add repr(value) + firstElement = false + when not isNamed: + if count == 1: + res.add(',') # $(1,) should print as the semantically legal (1,) + res.add(')') + + +proc repr*[T: tuple|object](x: T): string {.noSideEffect, raises: [].} = + ## Generic `repr` operator for tuples that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(23, 45) == "(23, 45)" + ## $(a: 23, b: 45) == "(a: 23, b: 45)" + ## $() == "()" + ## ``` + when T is object: + result = $typeof(x) + reprObject(result, x) + +proc repr*[T](x: ref T | ptr T): string {.noSideEffect, raises: [].} = + if isNil(x): return "nil" + when T is object: + result = $typeof(x) + reprObject(result, x[]) + else: + result = when typeof(x) is ref: "ref " else: "ptr " + result.add repr(x[]) + +proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string {.noSideEffect, raises: [].} = + result = prefix + var firstElement = true + for value in items(x): + if firstElement: + firstElement = false + else: + result.add(separator) + result.add repr(value) + result.add(suffix) + +proc repr*[T](x: set[T]): string = + ## Generic `repr` operator for sets that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## ${23, 45} == "{23, 45}" + ## ``` + collectionToRepr(x, "{", ", ", "}") + +proc repr*[T](x: seq[T]): string = + ## Generic `repr` operator for seqs that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(@[23, 45]) == "@[23, 45]" + ## ``` + collectionToRepr(x, "@[", ", ", "]") + +proc repr*[T, IDX](x: array[IDX, T]): string = + ## Generic `repr` operator for arrays that is lifted from the components. + collectionToRepr(x, "[", ", ", "]") + +proc repr*[T](x: openArray[T]): string = + ## Generic `repr` operator for openarrays that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" + ## ``` + collectionToRepr(x, "[", ", ", "]") + +proc repr*[T](x: UncheckedArray[T]): string = + "[...]" diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index 57237cfff..761d66aec 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -6,18 +6,246 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. # +# The generic ``repr`` procedure for the javascript backend. -proc reprInt(x: int64): string {.compilerproc.} = return $x +when defined(nimPreviewSlimSystem): + import std/formatfloat + +proc reprInt(x: int64): string {.compilerproc.} = $x +proc reprInt(x: uint64): string {.compilerproc.} = $x +proc reprInt(x: int): string {.compilerproc.} = $x +proc reprFloat(x: float): string {.compilerproc.} = $x + +proc reprPointer(p: pointer): string {.compilerproc.} = + # Do we need to generate the full 8bytes ? In js a pointer is an int anyway + var tmp: int + {.emit: "`tmp` = `p`_Idx || 0;".} + result = $tmp + +proc reprBool(x: bool): string {.compilerRtl.} = + if x: result = "true" + else: result = "false" proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - if ntfEnumHole notin typ.flags: - if e <% typ.node.len: - return $typ.node.sons[e].name + var tmp: bool + let item = typ.node.sons[e] + {.emit: "`tmp` = `item` !== undefined;".} + if tmp: + result = makeNimstrLit(item.name) + else: + result = $e & " (invalid data!)" + +include system/repr_impl + +proc reprChar(x: char): string {.compilerRtl.} = + result = "\'" + case x + of '"': add(result, "\\\"") + of '\\': add(result, "\\\\") + of '\127'..'\255', '\0'..'\31': add(result, "\\" & reprInt(ord(x))) + else: add(result, x) + add(result, "\'") + +proc reprStrAux(result: var string, s: cstring | string, len: int) = + add(result, "\"") + for i in 0 .. len-1: + let c = s[i] + case c + of '"': add(result, "\\\"") + of '\\': add(result, "\\\\") + #of '\10': add(result, "\\10\"\n\"") + of '\127'..'\255', '\0'..'\31': + add(result, "\\" & reprInt(ord(c))) + else: + add(result, c) + add(result, "\"") + +proc reprStr(s: string): string {.compilerRtl.} = + reprStrAux(result, s, s.len) + +proc addSetElem(result: var string, elem: int, typ: PNimType) = + # Dispatch each set element to the correct repr<Type> proc + case typ.kind: + of tyEnum: add(result, reprEnum(elem, typ)) + of tyBool: add(result, reprBool(bool(elem))) + of tyChar: add(result, reprChar(chr(elem))) + of tyRange: addSetElem(result, elem, typ.base) # Note the base to advance towards the element type + of tyInt..tyInt64, tyUInt8, tyUInt16: add result, reprInt(elem) + else: # data corrupt --> inform the user + add(result, " (invalid data!)") + +iterator setKeys(s: int): int {.inline.} = + # The type of s is a lie, but it's expected to be a set. + # Iterate over the JS object representing a set + # and returns the keys as int. + var len: int + var yieldRes: int + var i: int = 0 + {. emit: """ + var setObjKeys = Object.getOwnPropertyNames(`s`); + `len` = setObjKeys.length; + """ .} + while i < len: + {. emit: "`yieldRes` = parseInt(setObjKeys[`i`],10);\n" .} + yield yieldRes + inc i + +proc reprSetAux(result: var string, s: int, typ: PNimType) = + add(result, "{") + var first: bool = true + for el in setKeys(s): + if first: + first = false + else: + add(result, ", ") + addSetElem(result, el, typ.base) + add(result, "}") + +proc reprSet(e: int, typ: PNimType): string {.compilerRtl.} = + reprSetAux(result, e, typ) + +type + ReprClosure {.final.} = object + recDepth: int # do not recurse endlessly + indent: int # indentation + +proc initReprClosure(cl: var ReprClosure) = + cl.recDepth = -1 # default is to display everything! + cl.indent = 0 + +proc reprAux(result: var string, p: pointer, typ: PNimType, cl: var ReprClosure) + +proc reprArray(a: pointer, typ: PNimType, + cl: var ReprClosure): string {.compilerRtl.} = + # We prepend @ to seq, the C backend prepends the pointer to the seq. + result = if typ.kind == tySequence: "@[" else: "[" + var len: int = 0 + + {. emit: "`len` = `a`.length;\n" .} + var dereffed: pointer = a + for i in 0 .. len-1: + if i > 0 : + add(result, ", ") + # advance pointer and point to element at index + {. emit: """ + `dereffed`_Idx = `i`; + `dereffed` = `a`[`dereffed`_Idx]; + """ .} + reprAux(result, dereffed, typ.base, cl) + + add(result, "]") + +proc isPointedToNil(p: pointer): bool = + {. emit: "if (`p` === null) {`result` = true;}\n" .} + +proc reprRef(result: var string, p: pointer, typ: PNimType, + cl: var ReprClosure) = + if p.isPointedToNil: + add(result, "nil") + return + add(result, "ref " & reprPointer(p)) + add(result, " --> ") + if typ.base.kind != tyArray: + {. emit: """ + if (`p` != null && `p`.length > 0) { + `p` = `p`[`p`_Idx]; + } + """ .} + reprAux(result, p, typ.base, cl) + +proc reprRecordAux(result: var string, o: pointer, typ: PNimType, cl: var ReprClosure) = + add(result, "[") + + var first = true + var val = o + if typ.node.len == 0: + # if the object has only one field, len is 0 and sons is nil, the field is in node + let key: cstring = typ.node.name + add(result, $key & " = ") + {. emit: "`val` = `o`[`key`];\n" .} + reprAux(result, val, typ.node.typ, cl) + else: + # if the object has more than one field, sons is not nil and contains the fields. + for i in 0 .. typ.node.len-1: + if first: first = false + else: add(result, ",\n") + + let key: cstring = typ.node.sons[i].name + add(result, $key & " = ") + {. emit: "`val` = `o`[`key`];\n" .} # access the field by name + reprAux(result, val, typ.node.sons[i].typ, cl) + add(result, "]") + +proc reprRecord(o: pointer, typ: PNimType, cl: var ReprClosure): string {.compilerRtl.} = + reprRecordAux(result, o, typ, cl) + + +proc reprJsonStringify(p: int): string {.compilerRtl.} = + # As a last resort, use stringify + # We use this for tyOpenArray, tyVarargs while genTypeInfo is not implemented + var tmp: cstring + {. emit: "`tmp` = JSON.stringify(`p`);\n" .} + result = $tmp + +proc reprAux(result: var string, p: pointer, typ: PNimType, + cl: var ReprClosure) = + if cl.recDepth == 0: + add(result, "...") + return + dec(cl.recDepth) + case typ.kind + of tyInt..tyInt32, tyUInt..tyUInt32: + add(result, reprInt(cast[int](p))) + of tyInt64: + add(result, reprInt(cast[int64](p))) + of tyUInt64: + add(result, reprInt(cast[uint64](p))) + of tyChar: + add(result, reprChar(cast[char](p))) + of tyBool: + add(result, reprBool(cast[bool](p))) + of tyFloat..tyFloat128: + add(result, reprFloat(cast[float](p))) + of tyString: + var fp: int + {. emit: "`fp` = `p`;\n" .} + add(result, reprStr(cast[string](p))) + of tyCstring: + var fp: cstring + {. emit: "`fp` = `p`;\n" .} + if fp.isNil: + add(result, "nil") + else: + reprStrAux(result, fp, fp.len) + of tyEnum, tyOrdinal: + var fp: int + {. emit: "`fp` = `p`;\n" .} + add(result, reprEnum(fp, typ)) + of tySet: + var fp: int + {. emit: "`fp` = `p`;\n" .} + add(result, reprSet(fp, typ)) + of tyRange: reprAux(result, p, typ.base, cl) + of tyObject, tyTuple: + add(result, reprRecord(p, typ, cl)) + of tyArray, tyArrayConstr, tySequence: + add(result, reprArray(p, typ, cl)) + of tyPointer: + add(result, reprPointer(p)) + of tyPtr, tyRef: + reprRef(result, p, typ, cl) + of tyProc: + if p.isPointedToNil: + add(result, "nil") + else: + add(result, reprPointer(p)) else: - # ugh we need a slow linear search: - var n = typ.node - var s = n.sons - for i in 0 .. n.len-1: - if s[i].offset == e: return $s[i].name - result = $e & " (invalid data!)" + add(result, "(invalid data!)" & reprJsonStringify(cast[int](p))) + inc(cl.recDepth) +proc reprAny(p: pointer, typ: PNimType): string {.compilerRtl.} = + var cl: ReprClosure + initReprClosure(cl) + reprAux(result, p, typ, cl) + when defined(nimLegacyReprWithNewline): # see PR #16034 + add result, "\n" diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim new file mode 100644 index 000000000..572e77408 --- /dev/null +++ b/lib/system/seqs_v2.nim @@ -0,0 +1,227 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +# import std/typetraits +# strs already imported allocateds for us. + + +# Some optimizations here may be not to empty-seq-initialize some symbols, then StrictNotNil complains. +{.push warning[StrictNotNil]: off.} # See https://github.com/nim-lang/Nim/issues/21401 + + +## Default seq implementation used by Nim's core. +type + NimSeqPayloadBase = object + cap: int + + NimSeqPayload[T] = object + cap: int + data: UncheckedArray[T] + + NimSeqV2*[T] = object # \ + # if you change this implementation, also change seqs_v2_reimpl.nim! + len: int + p: ptr NimSeqPayload[T] + + NimRawSeq = object + len: int + p: pointer + +const nimSeqVersion {.core.} = 2 + +# XXX make code memory safe for overflows in '*' + +proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises: [].} = + # we have to use type erasure here as Nim does not support generic + # compilerProcs. Oh well, this will all be inlined anyway. + if cap > 0: + var p = cast[ptr NimSeqPayloadBase](alignedAlloc0(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize, elemAlign)) + p.cap = cap + result = p + else: + result = nil + +proc newSeqPayloadUninit(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises: [].} = + # Used in `newSeqOfCap()`. + if cap > 0: + var p = cast[ptr NimSeqPayloadBase](alignedAlloc(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize, elemAlign)) + p.cap = cap + result = p + else: + result = nil + +template `+!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +template `-!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) -% s) + +proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + if addlen <= 0: + result = p + elif p == nil: + result = newSeqPayload(len+addlen, elemSize, elemAlign) + else: + # Note: this means we cannot support things that have internal pointers as + # they get reallocated here. This needs to be documented clearly. + var p = cast[ptr NimSeqPayloadBase](p) + let oldCap = p.cap and not strlitFlag + let newCap = max(resize(oldCap), len+addlen) + var q: ptr NimSeqPayloadBase + if (p.cap and strlitFlag) == strlitFlag: + q = cast[ptr NimSeqPayloadBase](alignedAlloc(headerSize + elemSize * newCap, elemAlign)) + copyMem(q +! headerSize, p +! headerSize, len * elemSize) + else: + let oldSize = headerSize + elemSize * oldCap + let newSize = headerSize + elemSize * newCap + q = cast[ptr NimSeqPayloadBase](alignedRealloc(p, oldSize, newSize, elemAlign)) + + zeroMem(q +! headerSize +! len * elemSize, addlen * elemSize) + q.cap = newCap + result = q + +proc zeroNewElements(len: int; q: pointer; addlen, elemSize, elemAlign: int) {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + zeroMem(q +! headerSize +! len * elemSize, addlen * elemSize) + +proc prepareSeqAddUninit(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + if addlen <= 0: + result = p + elif p == nil: + result = newSeqPayloadUninit(len+addlen, elemSize, elemAlign) + else: + # Note: this means we cannot support things that have internal pointers as + # they get reallocated here. This needs to be documented clearly. + var p = cast[ptr NimSeqPayloadBase](p) + let oldCap = p.cap and not strlitFlag + let newCap = max(resize(oldCap), len+addlen) + if (p.cap and strlitFlag) == strlitFlag: + var q = cast[ptr NimSeqPayloadBase](alignedAlloc(headerSize + elemSize * newCap, elemAlign)) + copyMem(q +! headerSize, p +! headerSize, len * elemSize) + q.cap = newCap + result = q + else: + let oldSize = headerSize + elemSize * oldCap + let newSize = headerSize + elemSize * newCap + var q = cast[ptr NimSeqPayloadBase](alignedRealloc(p, oldSize, newSize, elemAlign)) + q.cap = newCap + result = q + +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = + when nimvm: + {.cast(tags: []).}: + setLen(x, newLen) + else: + #sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'" + when not supportsCopyMem(T): + for i in countdown(x.len - 1, newLen): + reset x[i] + # XXX This is wrong for const seqs that were moved into 'x'! + {.noSideEffect.}: + cast[ptr NimSeqV2[T]](addr x).len = newLen + +proc grow*[T](x: var seq[T]; newLen: Natural; value: T) {.nodestroy.} = + let oldLen = x.len + #sysAssert newLen >= x.len, "invalid newLen parameter for 'grow'" + if newLen <= oldLen: return + var xu = cast[ptr NimSeqV2[T]](addr x) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newLen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newLen - oldLen, sizeof(T), alignof(T))) + xu.len = newLen + for i in oldLen .. newLen-1: + wasMoved(xu.p.data[i]) + `=copy`(xu.p.data[i], value) + +proc add*[T](x: var seq[T]; y: sink T) {.magic: "AppendSeqElem", noSideEffect, nodestroy.} = + ## Generic proc for adding a data item `y` to a container `x`. + ## + ## For containers that have an order, `add` means *append*. New generic + ## containers should also call their adding proc `add` for consistency. + ## Generic code becomes much easier to write if the Nim naming scheme is + ## respected. + {.cast(noSideEffect).}: + let oldLen = x.len + var xu = cast[ptr NimSeqV2[T]](addr x) + if xu.p == nil or (xu.p.cap and not strlitFlag) < oldLen+1: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, 1, sizeof(T), alignof(T))) + xu.len = oldLen+1 + # .nodestroy means `xu.p.data[oldLen] = value` is compiled into a + # copyMem(). This is fine as know by construction that + # in `xu.p.data[oldLen]` there is nothing to destroy. + # We also save the `wasMoved + destroy` pair for the sink parameter. + xu.p.data[oldLen] = y + +proc setLen[T](s: var seq[T], newlen: Natural) {.nodestroy.} = + {.noSideEffect.}: + if newlen < s.len: + shrink(s, newlen) + else: + let oldLen = s.len + if newlen <= oldLen: return + var xu = cast[ptr NimSeqV2[T]](addr s) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newlen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) + xu.len = newlen + for i in oldLen..<newlen: + xu.p.data[i] = default(T) + +proc newSeq[T](s: var seq[T], len: Natural) = + shrink(s, 0) + setLen(s, len) + +proc sameSeqPayload(x: pointer, y: pointer): bool {.compilerRtl, inl.} = + result = cast[ptr NimRawSeq](x)[].p == cast[ptr NimRawSeq](y)[].p + + +func capacity*[T](self: seq[T]): int {.inline.} = + ## Returns the current capacity of the seq. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var lst = newSeqOfCap[string](cap = 42) + lst.add "Nim" + assert lst.capacity == 42 + + let sek = cast[ptr NimSeqV2[T]](unsafeAddr self) + result = if sek.p != nil: sek.p.cap and not strlitFlag else: 0 + +func setLenUninit*[T](s: var seq[T], newlen: Natural) {.nodestroy.} = + ## Sets the length of seq `s` to `newlen`. `T` may be any sequence type. + ## New slots will not be initialized. + ## + ## If the current length is greater than the new length, + ## `s` will be truncated. + ## ```nim + ## var x = @[10, 20] + ## x.setLenUninit(5) + ## x[4] = 50 + ## assert x[4] == 50 + ## x.setLenUninit(1) + ## assert x == @[10] + ## ``` + {.noSideEffect.}: + if newlen < s.len: + shrink(s, newlen) + else: + let oldLen = s.len + if newlen <= oldLen: return + var xu = cast[ptr NimSeqV2[T]](addr s) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newlen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) + xu.len = newlen + +{.pop.} # See https://github.com/nim-lang/Nim/issues/21401 diff --git a/lib/system/seqs_v2_reimpl.nim b/lib/system/seqs_v2_reimpl.nim new file mode 100644 index 000000000..09b7e7ac4 --- /dev/null +++ b/lib/system/seqs_v2_reimpl.nim @@ -0,0 +1,24 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +type + NimSeqPayloadReimpl = object + cap: int + data: pointer + + NimSeqV2Reimpl = object + len: int + p: ptr NimSeqPayloadReimpl + +template frees(s: NimSeqV2Reimpl) = + if s.p != nil and (s.p.cap and strlitFlag) != strlitFlag: + when compileOption("threads"): + deallocShared(s.p) + else: + dealloc(s.p) \ No newline at end of file diff --git a/lib/system/setops.nim b/lib/system/setops.nim new file mode 100644 index 000000000..67aa3097a --- /dev/null +++ b/lib/system/setops.nim @@ -0,0 +1,89 @@ +func incl*[T](x: var set[T], y: T) {.magic: "Incl".} = + ## Includes element `y` in the set `x`. + ## + ## This is the same as `x = x + {y}`, but it might be more efficient. + runnableExamples: + var a = {1, 3, 5} + a.incl(2) + assert a == {1, 2, 3, 5} + a.incl(4) + assert a == {1, 2, 3, 4, 5} + +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template incl*[T](x: var set[T], y: set[T]) {.callsite.} = + ## Includes the set `y` in the set `x`. + runnableExamples: + var a = {1, 3, 5, 7} + var b = {4, 5, 6} + a.incl(b) + assert a == {1, 3, 4, 5, 6, 7} + x = x + y + +func excl*[T](x: var set[T], y: T) {.magic: "Excl".} = + ## Excludes element `y` from the set `x`. + ## + ## This is the same as `x = x - {y}`, but it might be more efficient. + runnableExamples: + var b = {2, 3, 5, 6, 12, 54} + b.excl(5) + assert b == {2, 3, 6, 12, 54} + +template excl*[T](x: var set[T], y: set[T]) {.callsite.} = + ## Excludes the set `y` from the set `x`. + runnableExamples: + var a = {1, 3, 5, 7} + var b = {3, 4, 5} + a.excl(b) + assert a == {1, 7} + x = x - y + +func card*[T](x: set[T]): int {.magic: "Card".} = + ## Returns the cardinality of the set `x`, i.e. the number of elements + ## in the set. + runnableExamples: + var a = {1, 3, 5, 7} + assert card(a) == 4 + var b = {1, 3, 5, 7, 5} + assert card(b) == 4 # repeated 5 doesn't count + +func len*[T](x: set[T]): int {.magic: "Card".} + ## An alias for `card(x)`. + + +func `*`*[T](x, y: set[T]): set[T] {.magic: "MulSet".} = + ## This operator computes the intersection of two sets. + runnableExamples: + assert {1, 2, 3} * {2, 3, 4} == {2, 3} + +func `+`*[T](x, y: set[T]): set[T] {.magic: "PlusSet".} = + ## This operator computes the union of two sets. + runnableExamples: + assert {1, 2, 3} + {2, 3, 4} == {1, 2, 3, 4} + +func `-`*[T](x, y: set[T]): set[T] {.magic: "MinusSet".} = + ## This operator computes the difference of two sets. + runnableExamples: + assert {1, 2, 3} - {2, 3, 4} == {1} + +func contains*[T](x: set[T], y: T): bool {.magic: "InSet".} = + ## One should overload this proc if one wants to overload the `in` operator. + ## + ## The parameters are in reverse order! `a in b` is a template for + ## `contains(b, a)`. + ## This is because the unification algorithm that Nim uses for overload + ## resolution works from left to right. + ## But for the `in` operator that would be the wrong direction for this + ## piece of code: + runnableExamples: + var s: set[range['a'..'z']] = {'a'..'c'} + assert s.contains('c') + assert 'b' in s + assert 'd' notin s + assert set['a'..'z'] is set[range['a'..'z']] + ## If `in` had been declared as `[T](elem: T, s: set[T])` then `T` would + ## have been bound to `char`. But `s` is not compatible to type + ## `set[char]`! The solution is to bind `T` to `range['a'..'z']`. This + ## is achieved by reversing the parameters for `contains`; `in` then + ## passes its arguments in reverse order. diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 66877de30..97431c296 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -9,21 +9,20 @@ # set handling -type - NimSet = array [0..4*2048-1, uint8] -{.deprecated: [TNimSet: NimSet].} -proc countBits32(n: int32): int {.compilerproc.} = - var v = n - v = v -% ((v shr 1'i32) and 0x55555555'i32) - v = (v and 0x33333333'i32) +% ((v shr 2'i32) and 0x33333333'i32) - result = ((v +% (v shr 4'i32) and 0xF0F0F0F'i32) *% 0x1010101'i32) shr 24'i32 +proc cardSetImpl(s: ptr UncheckedArray[uint8], len: int): int {.inline.} = + var i = 0 + result = 0 + var num = 0'u64 + when defined(x86) or defined(amd64): + while i < len - 8: + copyMem(addr num, addr s[i], 8) + inc(result, countBits64(num)) + inc(i, 8) -proc countBits64(n: int64): int {.compilerproc.} = - result = countBits32(toU32(n and 0xffffffff'i64)) + - countBits32(toU32(n shr 32'i64)) + while i < len: + inc(result, countBits32(uint32(s[i]))) + inc(i, 1) -proc cardSet(s: NimSet, len: int): int {.compilerproc.} = - result = 0 - for i in countup(0, len-1): - inc(result, countBits32(int32(s[i]))) +proc cardSet(s: ptr UncheckedArray[uint8], len: int): int {.compilerproc, inline.} = + result = cardSetImpl(s, len) diff --git a/lib/system/stacktraces.nim b/lib/system/stacktraces.nim new file mode 100644 index 000000000..42be9d94f --- /dev/null +++ b/lib/system/stacktraces.nim @@ -0,0 +1,83 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Additional code for customizable stack traces. Unstable API, for internal +# usage only. + +const + reraisedFromBegin* = -10 + reraisedFromEnd* = -100 + maxStackTraceLines* = 128 + +when defined(nimStackTraceOverride): + ## Procedure types for overriding the default stack trace. + type + cuintptr_t* {.importc: "uintptr_t", nodecl.} = uint + ## This is the same as the type `uintptr_t` in C. + + StackTraceOverrideGetTracebackProc* = proc (): string {. + nimcall, gcsafe, raises: [], tags: [], noinline.} + StackTraceOverrideGetProgramCountersProc* = proc (maxLength: cint): seq[cuintptr_t] {. + nimcall, gcsafe, raises: [], tags: [], noinline.} + StackTraceOverrideGetDebuggingInfoProc* = + proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {. + nimcall, gcsafe, raises: [], tags: [], noinline.} + + # Default procedures (not normally used, because people opting in on this + # override are supposed to register their own versions). + var + stackTraceOverrideGetTraceback: StackTraceOverrideGetTracebackProc = + proc (): string {.nimcall, gcsafe, raises: [], tags: [], noinline.} = + discard + #result = "Stack trace override procedure not registered.\n" + stackTraceOverrideGetProgramCounters: StackTraceOverrideGetProgramCountersProc = + proc (maxLength: cint): seq[cuintptr_t] {.nimcall, gcsafe, raises: [], tags: [], noinline.} = + discard + stackTraceOverrideGetDebuggingInfo: StackTraceOverrideGetDebuggingInfoProc = + proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {. + nimcall, gcsafe, raises: [], tags: [], noinline.} = + discard + + # Custom procedure registration. + proc registerStackTraceOverride*(overrideProc: StackTraceOverrideGetTracebackProc) = + ## Override the default stack trace inside rawWriteStackTrace() with your + ## own procedure. + stackTraceOverrideGetTraceback = overrideProc + proc registerStackTraceOverrideGetProgramCounters*(overrideProc: StackTraceOverrideGetProgramCountersProc) = + stackTraceOverrideGetProgramCounters = overrideProc + proc registerStackTraceOverrideGetDebuggingInfo*(overrideProc: StackTraceOverrideGetDebuggingInfoProc) = + stackTraceOverrideGetDebuggingInfo = overrideProc + + # Custom stack trace manipulation. + proc auxWriteStackTraceWithOverride*(s: var string) = + add(s, stackTraceOverrideGetTraceback()) + + proc auxWriteStackTraceWithOverride*(s: var seq[StackTraceEntry]) = + let programCounters = stackTraceOverrideGetProgramCounters(maxStackTraceLines) + if s.len == 0: + s = newSeqOfCap[StackTraceEntry](programCounters.len) + for programCounter in programCounters: + s.add(StackTraceEntry(programCounter: cast[uint](programCounter))) + + # We may have more stack trace lines in the output, due to inlined procedures. + proc addDebuggingInfo*(s: seq[StackTraceEntry]): seq[StackTraceEntry] = + var programCounters: seq[cuintptr_t] + # We process program counters in groups from complete stack traces, because + # we have logic that keeps track of certain functions being inlined or not. + for entry in s: + if entry.procname.isNil and entry.programCounter != 0: + programCounters.add(cast[cuintptr_t](entry.programCounter)) + elif entry.procname.isNil and (entry.line == reraisedFromBegin or entry.line == reraisedFromEnd): + result.add(stackTraceOverrideGetDebuggingInfo(programCounters, maxStackTraceLines)) + programCounters = @[] + result.add(entry) + else: + result.add(entry) + if programCounters.len > 0: + result.add(stackTraceOverrideGetDebuggingInfo(programCounters, maxStackTraceLines)) diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim new file mode 100644 index 000000000..89046253b --- /dev/null +++ b/lib/system/strmantle.nim @@ -0,0 +1,263 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Compilerprocs for strings that do not depend on the string implementation. + +import std/private/digitsutils + + +proc cmpStrings(a, b: string): int {.inline, compilerproc.} = + let alen = a.len + let blen = b.len + let minlen = min(alen, blen) + if minlen > 0: + result = c_memcmp(unsafeAddr a[0], unsafeAddr b[0], cast[csize_t](minlen)).int + if result == 0: + result = alen - blen + else: + result = alen - blen + +proc leStrings(a, b: string): bool {.inline, compilerproc.} = + # required by upcoming backends (NIR). + cmpStrings(a, b) <= 0 + +proc ltStrings(a, b: string): bool {.inline, compilerproc.} = + # required by upcoming backends (NIR). + cmpStrings(a, b) < 0 + +proc eqStrings(a, b: string): bool {.inline, compilerproc.} = + let alen = a.len + let blen = b.len + if alen == blen: + if alen == 0: return true + return equalMem(unsafeAddr(a[0]), unsafeAddr(b[0]), alen) + +proc hashString(s: string): int {.compilerproc.} = + # the compiler needs exactly the same hash function! + # this used to be used for efficient generation of string case statements + var h = 0'u + for i in 0..len(s)-1: + h = h + uint(s[i]) + h = h + h shl 10 + h = h xor (h shr 6) + h = h + h shl 3 + h = h xor (h shr 11) + h = h + h shl 15 + result = cast[int](h) + +proc eqCstrings(a, b: cstring): bool {.inline, compilerproc.} = + if pointer(a) == pointer(b): result = true + elif a.isNil or b.isNil: result = false + else: result = c_strcmp(a, b) == 0 + +proc hashCstring(s: cstring): int {.compilerproc.} = + # the compiler needs exactly the same hash function! + # this used to be used for efficient generation of cstring case statements + if s.isNil: return 0 + var h : uint = 0 + var i = 0 + while true: + let c = s[i] + if c == '\0': break + h = h + uint(c) + h = h + h shl 10 + h = h xor (h shr 6) + inc i + h = h + h shl 3 + h = h xor (h shr 11) + h = h + h shl 15 + result = cast[int](h) + +proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. + importc: "strtod", header: "<stdlib.h>", noSideEffect.} + +const + IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} + powtens = [1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22] + + +{.push staticBoundChecks: off.} + +proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, + ): int {.compilerproc.} = + # This routine attempt to parse float that can parsed quickly. + # i.e. whose integer part can fit inside a 53bits integer. + # their real exponent must also be <= 22. If the float doesn't follow + # these restrictions, transform the float into this form: + # INTEGER * 10 ^ exponent and leave the work to standard `strtod()`. + # This avoid the problems of decimal character portability. + # see: http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + var + i = 0 + sign = 1.0 + kdigits, fdigits = 0 + exponent = 0 + integer = uint64(0) + fracExponent = 0 + expSign = 1 + firstDigit = -1 + hasSign = false + + # Sign? + if i < s.len and (s[i] == '+' or s[i] == '-'): + hasSign = true + if s[i] == '-': + sign = -1.0 + inc(i) + + # NaN? + if i+2 < s.len and (s[i] == 'N' or s[i] == 'n'): + if s[i+1] == 'A' or s[i+1] == 'a': + if s[i+2] == 'N' or s[i+2] == 'n': + if i+3 >= s.len or s[i+3] notin IdentChars: + number = NaN + return i+3 + return 0 + + # Inf? + if i+2 < s.len and (s[i] == 'I' or s[i] == 'i'): + if s[i+1] == 'N' or s[i+1] == 'n': + if s[i+2] == 'F' or s[i+2] == 'f': + if i+3 >= s.len or s[i+3] notin IdentChars: + number = Inf*sign + return i+3 + return 0 + + if i < s.len and s[i] in {'0'..'9'}: + firstDigit = (s[i].ord - '0'.ord) + # Integer part? + while i < s.len and s[i] in {'0'..'9'}: + inc(kdigits) + integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64 + inc(i) + while i < s.len and s[i] == '_': inc(i) + + # Fractional part? + if i < s.len and s[i] == '.': + inc(i) + # if no integer part, Skip leading zeros + if kdigits <= 0: + while i < s.len and s[i] == '0': + inc(fracExponent) + inc(i) + while i < s.len and s[i] == '_': inc(i) + + if firstDigit == -1 and i < s.len and s[i] in {'0'..'9'}: + firstDigit = (s[i].ord - '0'.ord) + # get fractional part + while i < s.len and s[i] in {'0'..'9'}: + inc(fdigits) + inc(fracExponent) + integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64 + inc(i) + while i < s.len and s[i] == '_': inc(i) + + # if has no digits: return error + if kdigits + fdigits <= 0 and + (i == 0 or # no char consumed (empty string). + (i == 1 and hasSign)): # or only '+' or '- + return 0 + + if i+1 < s.len and s[i] in {'e', 'E'}: + inc(i) + if s[i] == '+' or s[i] == '-': + if s[i] == '-': + expSign = -1 + + inc(i) + if s[i] notin {'0'..'9'}: + return 0 + while i < s.len and s[i] in {'0'..'9'}: + exponent = exponent * 10 + (ord(s[i]) - ord('0')) + inc(i) + while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored + + var realExponent = expSign*exponent - fracExponent + let expNegative = realExponent < 0 + var absExponent = abs(realExponent) + + # if exponent greater than can be represented: +/- zero or infinity + if absExponent > 999: + if integer == 0: + number = 0.0 + elif expNegative: + number = 0.0*sign + else: + number = Inf*sign + return i + + # if integer is representable in 53 bits: fast path + # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) + let digits = kdigits + fdigits + if digits <= 15 or (digits <= 16 and firstDigit <= 8): + # max float power of ten with set bits above the 53th bit is 10^22 + if absExponent <= 22: + if expNegative: + number = sign * integer.float / powtens[absExponent] + else: + number = sign * integer.float * powtens[absExponent] + return i + + # if exponent is greater try to fit extra exponent above 22 by multiplying + # integer part is there is space left. + let slop = 15 - kdigits - fdigits + if absExponent <= 22 + slop and not expNegative: + number = sign * integer.float * powtens[slop] * powtens[absExponent-slop] + return i + + # if failed: slow path with strtod. + var t: array[500, char] # flaviu says: 325 is the longest reasonable literal + var ti = 0 + let maxlen = t.high - "e+000".len # reserve enough space for exponent + + let endPos = i + result = endPos + i = 0 + # re-parse without error checking, any error should be handled by the code above. + if i < endPos and s[i] == '.': i.inc + while i < endPos and s[i] in {'0'..'9','+','-'}: + if ti < maxlen: + t[ti] = s[i]; inc(ti) + inc(i) + while i < endPos and s[i] in {'.', '_'}: # skip underscore and decimal point + inc(i) + + # insert exponent + t[ti] = 'E' + inc(ti) + t[ti] = if expNegative: '-' else: '+' + inc(ti, 4) + + # insert adjusted exponent + t[ti-1] = ('0'.ord + absExponent mod 10).char + absExponent = absExponent div 10 + t[ti-2] = ('0'.ord + absExponent mod 10).char + absExponent = absExponent div 10 + t[ti-3] = ('0'.ord + absExponent mod 10).char + number = c_strtod(cast[cstring](addr t), nil) + +{.pop.} # staticBoundChecks + +proc nimBoolToStr(x: bool): string {.compilerRtl.} = + return if x: "true" else: "false" + +proc nimCharToStr(x: char): string {.compilerRtl.} = + result = newString(1) + result[0] = x + +when defined(gcDestructors): + proc GC_getStatistics*(): string = + result = "[GC] total memory: " + result.addInt getTotalMem() + result.add "\n[GC] occupied memory: " + result.addInt getOccupiedMem() + result.add '\n' + #"[GC] cycle collections: " & $gch.stat.cycleCollections & "\n" & diff --git a/lib/system/strs_v2.nim b/lib/system/strs_v2.nim new file mode 100644 index 000000000..404b4f78d --- /dev/null +++ b/lib/system/strs_v2.nim @@ -0,0 +1,224 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Default new string implementation used by Nim's core. + +type + NimStrPayloadBase = object + cap: int + + NimStrPayload {.core.} = object + cap: int + data: UncheckedArray[char] + + NimStringV2 {.core.} = object + len: int + p: ptr NimStrPayload ## can be nil if len == 0. + +const nimStrVersion {.core.} = 2 + +template isLiteral(s): bool = (s.p == nil) or (s.p.cap and strlitFlag) == strlitFlag + +template contentSize(cap): int = cap + 1 + sizeof(NimStrPayloadBase) + +template frees(s) = + if not isLiteral(s): + when compileOption("threads"): + deallocShared(s.p) + else: + dealloc(s.p) + +template allocPayload(newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](allocShared(contentSize(newLen))) + else: + cast[ptr NimStrPayload](alloc(contentSize(newLen))) + +template allocPayload0(newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](allocShared0(contentSize(newLen))) + else: + cast[ptr NimStrPayload](alloc0(contentSize(newLen))) + +template reallocPayload(p: pointer, newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](reallocShared(p, contentSize(newLen))) + else: + cast[ptr NimStrPayload](realloc(p, contentSize(newLen))) + +template reallocPayload0(p: pointer; oldLen, newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](reallocShared0(p, contentSize(oldLen), contentSize(newLen))) + else: + cast[ptr NimStrPayload](realloc0(p, contentSize(oldLen), contentSize(newLen))) + +proc resize(old: int): int {.inline.} = + if old <= 0: result = 4 + elif old <= high(int16): result = old * 2 + else: result = old * 3 div 2 # for large arrays * 3/2 is better + +proc prepareAdd(s: var NimStringV2; addLen: int) {.compilerRtl.} = + let newLen = s.len + addLen + if isLiteral(s): + let oldP = s.p + # can't mutate a literal, so we need a fresh copy here: + s.p = allocPayload(newLen) + s.p.cap = newLen + if s.len > 0: + # we are about to append, so there is no need to copy the \0 terminator: + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], min(s.len, newLen)) + elif oldP == nil: + # In the case of `newString(0) & ""`, since `src.len == 0`, `appendString` + # will not set the `\0` terminator, so we set it here. + s.p.data[0] = '\0' + else: + let oldCap = s.p.cap and not strlitFlag + if newLen > oldCap: + let newCap = max(newLen, resize(oldCap)) + s.p = reallocPayload(s.p, newCap) + s.p.cap = newCap + if newLen < newCap: + zeroMem(cast[pointer](addr s.p.data[newLen+1]), newCap - newLen) + +proc nimAddCharV1(s: var NimStringV2; c: char) {.compilerRtl, inl.} = + #if (s.p == nil) or (s.len+1 > s.p.cap and not strlitFlag): + prepareAdd(s, 1) + s.p.data[s.len] = c + inc s.len + s.p.data[s.len] = '\0' + +proc toNimStr(str: cstring, len: int): NimStringV2 {.compilerproc.} = + if len <= 0: + result = NimStringV2(len: 0, p: nil) + else: + var p = allocPayload(len) + p.cap = len + copyMem(unsafeAddr p.data[0], str, len+1) + result = NimStringV2(len: len, p: p) + +proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = + if str == nil: toNimStr(str, 0) + else: toNimStr(str, str.len) + +proc nimToCStringConv(s: NimStringV2): cstring {.compilerproc, nonReloadable, inline.} = + if s.len == 0: result = cstring"" + else: result = cast[cstring](unsafeAddr s.p.data) + +proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inline.} = + if src.len > 0: + # also copy the \0 terminator: + copyMem(unsafeAddr dest.p.data[dest.len], unsafeAddr src.p.data[0], src.len+1) + inc dest.len, src.len + +proc appendChar(dest: var NimStringV2; c: char) {.compilerproc, inline.} = + dest.p.data[dest.len] = c + inc dest.len + dest.p.data[dest.len] = '\0' + +proc rawNewString(space: int): NimStringV2 {.compilerproc.} = + # this is also 'system.newStringOfCap'. + if space <= 0: + result = NimStringV2(len: 0, p: nil) + else: + var p = allocPayload(space) + p.cap = space + p.data[0] = '\0' + result = NimStringV2(len: 0, p: p) + +proc mnewString(len: int): NimStringV2 {.compilerproc.} = + if len <= 0: + result = NimStringV2(len: 0, p: nil) + else: + var p = allocPayload0(len) + p.cap = len + result = NimStringV2(len: len, p: p) + +proc setLengthStrV2(s: var NimStringV2, newLen: int) {.compilerRtl.} = + if newLen == 0: + discard "do not free the buffer here, pattern 's.setLen 0' is common for avoiding allocations" + else: + if isLiteral(s): + let oldP = s.p + s.p = allocPayload(newLen) + s.p.cap = newLen + if s.len > 0: + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], min(s.len, newLen)) + if newLen > s.len: + zeroMem(cast[pointer](addr s.p.data[s.len]), newLen - s.len + 1) + else: + s.p.data[newLen] = '\0' + else: + zeroMem(cast[pointer](addr s.p.data[0]), newLen + 1) + elif newLen > s.len: + let oldCap = s.p.cap and not strlitFlag + if newLen > oldCap: + let newCap = max(newLen, resize(oldCap)) + s.p = reallocPayload0(s.p, oldCap, newCap) + s.p.cap = newCap + + s.p.data[newLen] = '\0' + s.len = newLen + +proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} = + if a.p == b.p and a.len == b.len: return + if isLiteral(b): + # we can shallow copy literals: + frees(a) + a.len = b.len + a.p = b.p + else: + if isLiteral(a) or (a.p.cap and not strlitFlag) < b.len: + # we have to allocate the 'cap' here, consider + # 'let y = newStringOfCap(); var x = y' + # on the other hand... These get turned into moves now. + frees(a) + a.p = allocPayload(b.len) + a.p.cap = b.len + a.len = b.len + copyMem(unsafeAddr a.p.data[0], unsafeAddr b.p.data[0], b.len+1) + +proc nimPrepareStrMutationImpl(s: var NimStringV2) = + let oldP = s.p + # can't mutate a literal, so we need a fresh copy here: + s.p = allocPayload(s.len) + s.p.cap = s.len + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len+1) + +proc nimPrepareStrMutationV2(s: var NimStringV2) {.compilerRtl, inl.} = + if s.p != nil and (s.p.cap and strlitFlag) == strlitFlag: + nimPrepareStrMutationImpl(s) + +proc prepareMutation*(s: var string) {.inline.} = + # string literals are "copy on write", so you need to call + # `prepareMutation` before modifying the strings via `addr`. + {.cast(noSideEffect).}: + let s = unsafeAddr s + nimPrepareStrMutationV2(cast[ptr NimStringV2](s)[]) + +proc nimAddStrV1(s: var NimStringV2; src: NimStringV2) {.compilerRtl, inl.} = + #if (s.p == nil) or (s.len+1 > s.p.cap and not strlitFlag): + prepareAdd(s, src.len) + appendString s, src + +proc nimDestroyStrV1(s: NimStringV2) {.compilerRtl, inl.} = + frees(s) + +proc nimStrAtLe(s: string; idx: int; ch: char): bool {.compilerRtl, inl.} = + result = idx < s.len and s[idx] <= ch + +func capacity*(self: string): int {.inline.} = + ## Returns the current capacity of the string. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var str = newStringOfCap(cap = 42) + str.add "Nim" + assert str.capacity == 42 + + let str = cast[ptr NimStringV2](unsafeAddr self) + result = if str.p != nil: str.p.cap and not strlitFlag else: 0 diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim deleted file mode 100644 index 3d0b2aa8a..000000000 --- a/lib/system/sysio.nim +++ /dev/null @@ -1,328 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# Nim's standard IO library. It contains high-performance -# routines for reading and writing data to (buffered) files or -# TTYs. - -{.push debugger:off .} # the user does not want to trace a part - # of the standard library! - - -proc fputs(c: cstring, f: File) {.importc: "fputs", header: "<stdio.h>", - tags: [WriteIOEffect].} -proc fgets(c: cstring, n: int, f: File): cstring {. - importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} -proc fgetc(stream: File): cint {.importc: "fgetc", header: "<stdio.h>", - tags: [ReadIOEffect].} -proc ungetc(c: cint, f: File) {.importc: "ungetc", header: "<stdio.h>", - tags: [].} -proc putc(c: char, stream: File) {.importc: "putc", header: "<stdio.h>", - tags: [WriteIOEffect].} -proc fprintf(f: File, frmt: cstring) {.importc: "fprintf", - header: "<stdio.h>", varargs, tags: [WriteIOEffect].} -proc strlen(c: cstring): int {. - importc: "strlen", header: "<string.h>", tags: [].} - -# C routine that is used here: -proc fread(buf: pointer, size, n: int, f: File): int {. - importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} -proc fseek(f: File, offset: clong, whence: int): int {. - importc: "fseek", header: "<stdio.h>", tags: [].} -proc ftell(f: File): int {.importc: "ftell", header: "<stdio.h>", tags: [].} -proc ferror(f: File): int {.importc: "ferror", header: "<stdio.h>", tags: [].} -proc setvbuf(stream: File, buf: pointer, typ, size: cint): cint {. - importc, header: "<stdio.h>", tags: [].} -proc memchr(s: pointer, c: cint, n: csize): pointer {. - importc: "memchr", header: "<string.h>", tags: [].} -proc memset(s: pointer, c: cint, n: csize) {. - header: "<string.h>", importc: "memset", tags: [].} - -{.push stackTrace:off, profiler:off.} -proc write(f: File, c: cstring) = fputs(c, f) -{.pop.} - -when NoFakeVars: - when defined(windows): - const - IOFBF = cint(0) - IONBF = cint(4) - elif defined(macosx) or defined(linux): - const - IOFBF = cint(0) - IONBF = cint(2) - else: - {.error: "IOFBF not ported to your platform".} -else: - var - IOFBF {.importc: "_IOFBF", nodecl.}: cint - IONBF {.importc: "_IONBF", nodecl.}: cint - -const - BufSize = 4000 - -proc raiseEIO(msg: string) {.noinline, noreturn.} = - sysFatal(IOError, msg) - -proc readLine(f: File, line: var TaintedString): bool = - var pos = 0 - # Use the currently reserved space for a first try - when defined(nimscript): - var space = 80 - else: - var space = cast[PGenericSeq](line.string).space - line.string.setLen(space) - - while true: - # memset to \l so that we can tell how far fgets wrote, even on EOF, where - # fgets doesn't append an \l - memset(addr line.string[pos], '\l'.ord, space) - if fgets(addr line.string[pos], space, f) == nil: - line.string.setLen(0) - return false - let m = memchr(addr line.string[pos], '\l'.ord, space) - if m != nil: - # \l found: Could be our own or the one by fgets, in any case, we're done - var last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0]) - if last > 0 and line.string[last-1] == '\c': - line.string.setLen(last-1) - return true - # We have to distinguish between two possible cases: - # \0\l\0 => line ending in a null character. - # \0\l\l => last line without newline, null was put there by fgets. - elif last > 0 and line.string[last-1] == '\0': - if last < pos + space - 1 and line.string[last+1] != '\0': - dec last - line.string.setLen(last) - return true - else: - # fgets will have inserted a null byte at the end of the string. - dec space - # No \l found: Increase buffer and read more - inc pos, space - space = 128 # read in 128 bytes at a time - line.string.setLen(pos+space) - -proc readLine(f: File): TaintedString = - result = TaintedString(newStringOfCap(80)) - if not readLine(f, result): raiseEIO("EOF reached") - -proc write(f: File, i: int) = - when sizeof(int) == 8: - fprintf(f, "%lld", i) - else: - fprintf(f, "%ld", i) - -proc write(f: File, i: BiggestInt) = - when sizeof(BiggestInt) == 8: - fprintf(f, "%lld", i) - else: - fprintf(f, "%ld", i) - -proc write(f: File, b: bool) = - if b: write(f, "true") - else: write(f, "false") -proc write(f: File, r: float32) = fprintf(f, "%g", r) -proc write(f: File, r: BiggestFloat) = fprintf(f, "%g", r) - -proc write(f: File, c: char) = putc(c, f) -proc write(f: File, a: varargs[string, `$`]) = - for x in items(a): write(f, x) - -proc readAllBuffer(file: File): string = - # This proc is for File we want to read but don't know how many - # bytes we need to read before the buffer is empty. - result = "" - var buffer = newString(BufSize) - while true: - var bytesRead = readBuffer(file, addr(buffer[0]), BufSize) - if bytesRead == BufSize: - result.add(buffer) - else: - buffer.setLen(bytesRead) - result.add(buffer) - break - -proc rawFileSize(file: File): int = - # this does not raise an error opposed to `getFileSize` - var oldPos = ftell(file) - discard fseek(file, 0, 2) # seek the end of the file - result = ftell(file) - discard fseek(file, clong(oldPos), 0) - -proc readAllFile(file: File, len: int): string = - # We acquire the filesize beforehand and hope it doesn't change. - # Speeds things up. - result = newString(len) - let bytes = readBuffer(file, addr(result[0]), len) - if endOfFile(file): - if bytes < len: - result.setLen(bytes) - elif ferror(file) != 0: - raiseEIO("error while reading from file") - else: - # We read all the bytes but did not reach the EOF - # Try to read it as a buffer - result.add(readAllBuffer(file)) - -proc readAllFile(file: File): string = - var len = rawFileSize(file) - result = readAllFile(file, len) - -proc readAll(file: File): TaintedString = - # Separate handling needed because we need to buffer when we - # don't know the overall length of the File. - when declared(stdin): - let len = if file != stdin: rawFileSize(file) else: -1 - else: - let len = rawFileSize(file) - if len > 0: - result = readAllFile(file, len).TaintedString - else: - result = readAllBuffer(file).TaintedString - -proc readFile(filename: string): TaintedString = - var f = open(filename) - try: - result = readAll(f).TaintedString - finally: - close(f) - -proc writeFile(filename, content: string) = - var f = open(filename, fmWrite) - try: - f.write(content) - finally: - close(f) - -proc endOfFile(f: File): bool = - # do not blame me; blame the ANSI C standard this is so brain-damaged - var c = fgetc(f) - ungetc(c, f) - return c < 0'i32 - -proc writeLn[Ty](f: File, x: varargs[Ty, `$`]) = - for i in items(x): - write(f, i) - write(f, "\n") - -proc writeLine[Ty](f: File, x: varargs[Ty, `$`]) = - for i in items(x): - write(f, i) - write(f, "\n") - -when declared(stdout): - proc rawEcho(x: string) {.inline, compilerproc.} = write(stdout, x) - proc rawEchoNL() {.inline, compilerproc.} = write(stdout, "\n") - -# interface to the C procs: - -when (defined(windows) and not defined(useWinAnsi)) or defined(nimdoc): - include "system/widestrs" - -when defined(windows) and not defined(useWinAnsi): - when defined(cpp): - proc wfopen(filename, mode: WideCString): pointer {. - importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} - proc wfreopen(filename, mode: WideCString, stream: File): File {. - importcpp: "_wfreopen((const wchar_t*)#, (const wchar_t*)#, #)", nodecl.} - else: - proc wfopen(filename, mode: WideCString): pointer {. - importc: "_wfopen", nodecl.} - proc wfreopen(filename, mode: WideCString, stream: File): File {. - importc: "_wfreopen", nodecl.} - - proc fopen(filename, mode: cstring): pointer = - var f = newWideCString(filename) - var m = newWideCString(mode) - result = wfopen(f, m) - - proc freopen(filename, mode: cstring, stream: File): File = - var f = newWideCString(filename) - var m = newWideCString(mode) - result = wfreopen(f, m, stream) - -else: - proc fopen(filename, mode: cstring): pointer {.importc: "fopen", noDecl.} - proc freopen(filename, mode: cstring, stream: File): File {. - importc: "freopen", nodecl.} - -const - FormatOpen: array [FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] - #"rt", "wt", "w+t", "r+t", "at" - # we always use binary here as for Nim the OS line ending - # should not be translated. - - -proc open(f: var File, filename: string, - mode: FileMode = fmRead, - bufSize: int = -1): bool = - var p: pointer = fopen(filename, FormatOpen[mode]) - if p != nil: - result = true - f = cast[File](p) - if bufSize > 0 and bufSize <= high(cint).int: - discard setvbuf(f, nil, IOFBF, bufSize.cint) - elif bufSize == 0: - discard setvbuf(f, nil, IONBF, 0) - -proc reopen(f: File, filename: string, mode: FileMode = fmRead): bool = - var p: pointer = freopen(filename, FormatOpen[mode], f) - result = p != nil - -proc fdopen(filehandle: FileHandle, mode: cstring): File {. - importc: pccHack & "fdopen", header: "<stdio.h>".} - -proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = - f = fdopen(filehandle, FormatOpen[mode]) - result = f != nil - -proc fwrite(buf: pointer, size, n: int, f: File): int {. - importc: "fwrite", noDecl.} - -proc readBuffer(f: File, buffer: pointer, len: Natural): int = - result = fread(buffer, 1, len, f) - -proc readBytes(f: File, a: var openArray[int8|uint8], start, len: Natural): int = - result = readBuffer(f, addr(a[start]), len) - -proc readChars(f: File, a: var openArray[char], start, len: Natural): int = - result = readBuffer(f, addr(a[start]), len) - -{.push stackTrace:off, profiler:off.} -proc writeBytes(f: File, a: openArray[int8|uint8], start, len: Natural): int = - var x = cast[ptr array[0..1000_000_000, int8]](a) - result = writeBuffer(f, addr(x[start]), len) -proc writeChars(f: File, a: openArray[char], start, len: Natural): int = - var x = cast[ptr array[0..1000_000_000, int8]](a) - result = writeBuffer(f, addr(x[start]), len) -proc writeBuffer(f: File, buffer: pointer, len: Natural): int = - result = fwrite(buffer, 1, len, f) - -proc write(f: File, s: string) = - if writeBuffer(f, cstring(s), s.len) != s.len: - raiseEIO("cannot write string to file") -{.pop.} - -proc setFilePos(f: File, pos: int64) = - if fseek(f, clong(pos), 0) != 0: - raiseEIO("cannot set file position") - -proc getFilePos(f: File): int64 = - result = ftell(f) - if result < 0: raiseEIO("cannot retrieve file position") - -proc getFileSize(f: File): int64 = - var oldPos = getFilePos(f) - discard fseek(f, 0, 2) # seek the end of the file - result = getFilePos(f) - setFilePos(f, oldPos) - -{.pop.} diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim deleted file mode 100644 index 7a113b9d4..000000000 --- a/lib/system/syslocks.nim +++ /dev/null @@ -1,106 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Low level system locks and condition vars. - -when defined(Windows): - type - Handle = int - SysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi - DebugInfo: pointer - LockCount: int32 - RecursionCount: int32 - OwningThread: int - LockSemaphore: int - Reserved: int32 - - SysCond = Handle - - {.deprecated: [THandle: Handle, TSysLock: SysLock, TSysCond: SysCond].} - - proc initSysLock(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "InitializeCriticalSection".} - ## Initializes the lock `L`. - - proc tryAcquireSysAux(L: var SysLock): int32 {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "TryEnterCriticalSection".} - ## Tries to acquire the lock `L`. - - proc tryAcquireSys(L: var SysLock): bool {.inline.} = - result = tryAcquireSysAux(L) != 0'i32 - - proc acquireSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "EnterCriticalSection".} - ## Acquires the lock `L`. - - proc releaseSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "LeaveCriticalSection".} - ## Releases the lock `L`. - - proc deinitSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "DeleteCriticalSection".} - - proc createEvent(lpEventAttributes: pointer, - bManualReset, bInitialState: int32, - lpName: cstring): SysCond {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CreateEventA".} - - proc closeHandle(hObject: Handle) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CloseHandle".} - proc waitForSingleObject(hHandle: Handle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", noSideEffect.} - - proc signalSysCond(hEvent: SysCond) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "SetEvent".} - - proc initSysCond(cond: var SysCond) {.inline.} = - cond = createEvent(nil, 0'i32, 0'i32, nil) - proc deinitSysCond(cond: var SysCond) {.inline.} = - closeHandle(cond) - proc waitSysCond(cond: var SysCond, lock: var SysLock) = - releaseSys(lock) - discard waitForSingleObject(cond, -1'i32) - acquireSys(lock) - - proc waitSysCondWindows(cond: var SysCond) = - discard waitForSingleObject(cond, -1'i32) - -else: - type - SysLock {.importc: "pthread_mutex_t", pure, final, - header: "<sys/types.h>".} = object - SysCond {.importc: "pthread_cond_t", pure, final, - header: "<sys/types.h>".} = object - - proc initSysLock(L: var SysLock, attr: pointer = nil) {. - importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - - proc acquireSys(L: var SysLock) {.noSideEffect, - importc: "pthread_mutex_lock", header: "<pthread.h>".} - proc tryAcquireSysAux(L: var SysLock): cint {.noSideEffect, - importc: "pthread_mutex_trylock", header: "<pthread.h>".} - - proc tryAcquireSys(L: var SysLock): bool {.inline.} = - result = tryAcquireSysAux(L) == 0'i32 - - proc releaseSys(L: var SysLock) {.noSideEffect, - importc: "pthread_mutex_unlock", header: "<pthread.h>".} - proc deinitSys(L: var SysLock) {.noSideEffect, - importc: "pthread_mutex_destroy", header: "<pthread.h>".} - - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - proc deinitSysCond(cond: var SysCond) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} - diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim deleted file mode 100644 index 7aef86df9..000000000 --- a/lib/system/sysspawn.nim +++ /dev/null @@ -1,194 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements Nim's 'spawn'. - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -{.push stackTrace:off.} - -# We declare our own condition variables here to get rid of the dummy lock -# on Windows: - -type - CondVar = object - c: SysCond - when defined(posix): - stupidLock: SysLock - counter: int - -proc createCondVar(): CondVar = - initSysCond(result.c) - when defined(posix): - initSysLock(result.stupidLock) - #acquireSys(result.stupidLock) - -proc destroyCondVar(c: var CondVar) {.inline.} = - deinitSysCond(c.c) - -proc await(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - while cv.counter <= 0: - waitSysCond(cv.c, cv.stupidLock) - dec cv.counter - releaseSys(cv.stupidLock) - else: - waitSysCondWindows(cv.c) - -proc signal(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - inc cv.counter - releaseSys(cv.stupidLock) - signalSysCond(cv.c) - -type - FastCondVar = object - event, slowPath: bool - slow: CondVar - -proc createFastCondVar(): FastCondVar = - initSysCond(result.slow.c) - when defined(posix): - initSysLock(result.slow.stupidLock) - #acquireSys(result.slow.stupidLock) - result.event = false - result.slowPath = false - -proc await(cv: var FastCondVar) = - #for i in 0 .. 50: - # if cas(addr cv.event, true, false): - # # this is a HIT: Triggers > 95% in my tests. - # return - # cpuRelax() - #cv.slowPath = true - # XXX For some reason this crashes some test programs - await(cv.slow) - cv.event = false - -proc signal(cv: var FastCondVar) = - cv.event = true - #if cas(addr cv.slowPath, true, false): - signal(cv.slow) - -type - Barrier* {.compilerProc.} = object - counter: int - cv: CondVar - -proc barrierEnter*(b: ptr Barrier) {.compilerProc.} = - atomicInc b.counter - -proc barrierLeave*(b: ptr Barrier) {.compilerProc.} = - atomicDec b.counter - if b.counter <= 0: signal(b.cv) - -proc openBarrier*(b: ptr Barrier) {.compilerProc.} = - b.counter = 0 - b.cv = createCondVar() - -proc closeBarrier*(b: ptr Barrier) {.compilerProc.} = - await(b.cv) - destroyCondVar(b.cv) - -{.pop.} - -# ---------------------------------------------------------------------------- - -type - WorkerProc = proc (thread, args: pointer) {.nimcall, gcsafe.} - Worker = object - taskArrived: CondVar - taskStarted: FastCondVar #\ - # task data: - f: WorkerProc - data: pointer - ready: bool # put it here for correct alignment! - -proc nimArgsPassingDone(p: pointer) {.compilerProc.} = - let w = cast[ptr Worker](p) - signal(w.taskStarted) - -var gSomeReady = createFastCondVar() - -proc slave(w: ptr Worker) {.thread.} = - while true: - w.ready = true # If we instead signal "workerReady" we need the scheduler - # to notice this. The scheduler could then optimize the - # layout of the worker threads (e.g. keep the list sorted) - # so that no search for a "ready" thread is necessary. - # This might be implemented later, but is more tricky than - # it looks because 'spawn' itself can run concurrently. - signal(gSomeReady) - await(w.taskArrived) - assert(not w.ready) - # shield against spurious wakeups: - if w.data != nil: - w.f(w, w.data) - w.data = nil - -const NumThreads = 4 - -var - workers: array[NumThreads, TThread[ptr Worker]] - workersData: array[NumThreads, Worker] - -proc setup() = - for i in 0.. <NumThreads: - workersData[i].taskArrived = createCondVar() - workersData[i].taskStarted = createFastCondVar() - createThread(workers[i], slave, addr(workersData[i])) - -proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a 'spawn' or a direct call is - ## preferable. If it returns 'true' a 'spawn' may make sense. In general - ## it is not necessary to call this directly; use 'spawnX' instead. - result = gSomeReady.event - -proc spawn*(call: stmt) {.magic: "Spawn".} - ## always spawns a new task, so that the 'call' is never executed on - ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - -template spawnX*(call: stmt) = - ## spawns a new task if a CPU core is ready, otherwise executes the - ## call in the calling thread. Usually it is advised to - ## use 'spawn' in order to not block the producer for an unknown - ## amount of time. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - if preferSpawn(): spawn call - else: call - -proc nimSpawn(fn: WorkerProc; data: pointer) {.compilerProc.} = - # implementation of 'spawn' that is used by the code generator. - while true: - for i in 0.. high(workers): - let w = addr(workersData[i]) - if cas(addr w.ready, true, false): - w.data = data - w.f = fn - signal(w.taskArrived) - await(w.taskStarted) - return - await(gSomeReady) - -proc sync*() = - ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate - ## waiting, you have to use an explicit barrier. - while true: - var allReady = true - for i in 0 .. high(workers): - if not allReady: break - allReady = allReady and workersData[i].ready - if allReady: break - await(gSomeReady) - -setup() diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index e2137e8f4..3621c4960 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -15,77 +15,85 @@ # we don't use refcounts because that's a behaviour # the programmer may not want + +proc dataPointer(a: PGenericSeq, elemAlign: int): pointer = + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign)) + +proc dataPointer(a: PGenericSeq, elemAlign, elemSize, index: int): pointer = + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) + proc resize(old: int): int {.inline.} = if old <= 0: result = 4 elif old < 65536: result = old * 2 else: result = old * 3 div 2 # for large arrays * 3/2 is better -proc cmpStrings(a, b: NimString): int {.inline, compilerProc.} = - if a == b: return 0 - if a == nil: return -1 - if b == nil: return 1 - return c_strcmp(a.data, b.data) - -proc eqStrings(a, b: NimString): bool {.inline, compilerProc.} = - if a == b: return true - if a == nil or b == nil: return false - return a.len == b.len and - c_memcmp(a.data, b.data, a.len) == 0'i32 - when declared(allocAtomic): - template allocStr(size: expr): expr = + template allocStr(size: untyped): untyped = cast[NimString](allocAtomic(size)) - template allocStrNoInit(size: expr): expr = + template allocStrNoInit(size: untyped): untyped = cast[NimString](boehmAllocAtomic(size)) +elif defined(gcRegions): + template allocStr(size: untyped): untyped = + cast[NimString](newStr(addr(strDesc), size, true)) + + template allocStrNoInit(size: untyped): untyped = + cast[NimString](newStr(addr(strDesc), size, false)) + else: - template allocStr(size: expr): expr = + template allocStr(size: untyped): untyped = cast[NimString](newObj(addr(strDesc), size)) - template allocStrNoInit(size: expr): expr = + template allocStrNoInit(size: untyped): untyped = cast[NimString](newObjNoInit(addr(strDesc), size)) -proc rawNewStringNoInit(space: int): NimString {.compilerProc.} = - var s = space - if s < 7: s = 7 +proc rawNewStringNoInit(space: int): NimString = + let s = max(space, 7) result = allocStrNoInit(sizeof(TGenericSeq) + s + 1) result.reserved = s when defined(gogc): result.elemSize = 1 -proc rawNewString(space: int): NimString {.compilerProc.} = - var s = space - if s < 7: s = 7 - result = allocStr(sizeof(TGenericSeq) + s + 1) - result.reserved = s - when defined(gogc): - result.elemSize = 1 +proc rawNewString(space: int): NimString {.compilerproc.} = + result = rawNewStringNoInit(space) + result.len = 0 + result.data[0] = '\0' -proc mnewString(len: int): NimString {.compilerProc.} = - result = rawNewString(len) +proc mnewString(len: int): NimString {.compilerproc.} = + result = rawNewStringNoInit(len) result.len = len + zeroMem(addr result.data[0], len + 1) + +proc copyStrLast(s: NimString, start, last: int): NimString {.compilerproc.} = + # This is not used by most recent versions of the compiler anymore, but + # required for bootstrapping purposes. + let start = max(start, 0) + if s == nil: return nil + let len = min(last, s.len-1) - start + 1 + result = rawNewStringNoInit(len) + result.len = len + copyMem(addr(result.data), addr(s.data[start]), len) + result.data[len] = '\0' -proc copyStrLast(s: NimString, start, last: int): NimString {.compilerProc.} = - var start = max(start, 0) - var len = min(last, s.len-1) - start + 1 - if len > 0: - result = rawNewStringNoInit(len) - result.len = len - c_memcpy(result.data, addr(s.data[start]), len) - result.data[len] = '\0' - else: - result = rawNewString(len) - -proc copyStr(s: NimString, start: int): NimString {.compilerProc.} = +proc copyStr(s: NimString, start: int): NimString {.compilerproc.} = + # This is not used by most recent versions of the compiler anymore, but + # required for bootstrapping purposes. + if s == nil: return nil result = copyStrLast(s, start, s.len-1) -proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} = +proc nimToCStringConv(s: NimString): cstring {.compilerproc, nonReloadable, inline.} = + if s == nil or s.len == 0: result = cstring"" + else: result = cast[cstring](addr s.data) + +proc toNimStr(str: cstring, len: int): NimString {.compilerproc.} = result = rawNewStringNoInit(len) result.len = len - c_memcpy(result.data, str, len + 1) + copyMem(addr(result.data), str, len) + result.data[len] = '\0' proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} = - result = toNimStr(str, c_strlen(str)) + if str == nil: NimString(nil) + else: toNimStr(str, str.len) proc copyString(src: NimString): NimString {.compilerRtl.} = if src != nil: @@ -94,42 +102,61 @@ proc copyString(src: NimString): NimString {.compilerRtl.} = else: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) + sysAssert((seqShallowFlag and result.reserved) == 0, "copyString") + when defined(nimShallowStrings): + if (src.reserved and strlitFlag) != 0: + result.reserved = (result.reserved and not strlitFlag) or seqShallowFlag + +proc newOwnedString(src: NimString; n: int): NimString = + result = rawNewStringNoInit(n) + result.len = n + copyMem(addr(result.data), addr(src.data), n) + result.data[n] = '\0' proc copyStringRC1(src: NimString): NimString {.compilerRtl.} = if src != nil: - when declared(newObjRC1): - var s = src.len - if s < 7: s = 7 - result = cast[NimString](newObjRC1(addr(strDesc), sizeof(TGenericSeq) + - s+1)) - result.reserved = s + if (src.reserved and seqShallowFlag) != 0: + result = src + when declared(incRef): + incRef(usrToCell(result)) else: - result = rawNewStringNoInit(src.len) + when declared(newObjRC1) and not defined(gcRegions): + var s = src.len + if s < 7: s = 7 + result = cast[NimString](newObjRC1(addr(strDesc), sizeof(TGenericSeq) + + s+1)) + result.reserved = s + when defined(gogc): + result.elemSize = 1 + else: + result = rawNewStringNoInit(src.len) + result.len = src.len + copyMem(addr(result.data), addr(src.data), src.len + 1) + sysAssert((seqShallowFlag and result.reserved) == 0, "copyStringRC1") + when defined(nimShallowStrings): + if (src.reserved and strlitFlag) != 0: + result.reserved = (result.reserved and not strlitFlag) or seqShallowFlag + +proc copyDeepString(src: NimString): NimString {.inline.} = + if src != nil: + result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) - - -proc hashString(s: string): int {.compilerproc.} = - # the compiler needs exactly the same hash function! - # this used to be used for efficient generation of string case statements - var h = 0 - for i in 0..len(s)-1: - h = h +% ord(s[i]) - h = h +% h shl 10 - h = h xor (h shr 6) - h = h +% h shl 3 - h = h xor (h shr 11) - h = h +% h shl 15 - result = h + copyMem(addr(result.data), addr(src.data), src.len + 1) proc addChar(s: NimString, c: char): NimString = # is compilerproc! - result = s - if result.len >= result.space: - result.reserved = resize(result.space) - result = cast[NimString](growObj(result, - sizeof(TGenericSeq) + result.reserved + 1)) + if s == nil: + result = rawNewStringNoInit(1) + result.len = 0 + else: + result = s + if result.len >= result.space: + let r = resize(result.space) + result = rawNewStringNoInit(r) + result.len = s.len + copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) + result.reserved = r result.data[result.len] = c result.data[result.len+1] = '\0' inc(result.len) @@ -166,19 +193,24 @@ proc addChar(s: NimString, c: char): NimString = # s = rawNewString(0); proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = - if dest.len + addlen <= dest.space: + if dest == nil: + result = rawNewString(addlen) + elif dest.len + addlen <= dest.space: result = dest else: # slow path: - var sp = max(resize(dest.space), dest.len + addlen) - result = cast[NimString](growObj(dest, sizeof(TGenericSeq) + sp + 1)) + let sp = max(resize(dest.space), dest.len + addlen) + result = rawNewStringNoInit(sp) + result.len = dest.len + copyMem(addr result.data[0], unsafeAddr(dest.data[0]), dest.len+1) result.reserved = sp #result = rawNewString(sp) #copyMem(result, dest, dest.len + sizeof(TGenericSeq)) # DO NOT UPDATE LEN YET: dest.len = newLen proc appendString(dest, src: NimString) {.compilerproc, inline.} = - c_memcpy(addr(dest.data[dest.len]), src.data, src.len + 1) - inc(dest.len, src.len) + if src != nil: + copyMem(addr(dest.data[dest.len]), addr(src.data), src.len + 1) + inc(dest.len, src.len) proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = dest.data[dest.len] = c @@ -186,17 +218,27 @@ proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = inc(dest.len) proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.} = - var n = max(newLen, 0) - if n <= s.space: + let n = max(newLen, 0) + if s == nil: + if n == 0: + return s + else: + result = mnewString(n) + elif n <= s.space: result = s else: - result = resizeString(s, n) + let sp = max(resize(s.space), n) + result = rawNewStringNoInit(sp) + result.len = s.len + copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len) + zeroMem(addr result.data[s.len], n - s.len) + result.reserved = sp result.len = n result.data[n] = '\0' # ----------------- sequences ---------------------------------------------- -proc incrSeq(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} = +proc incrSeq(seq: PGenericSeq, elemSize, elemAlign: int): PGenericSeq {.compilerproc.} = # increments the length by one: # this is needed for supporting ``add``; # @@ -205,231 +247,117 @@ proc incrSeq(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} = # seq[seq->len-1] = x; result = seq if result.len >= result.space: - result.reserved = resize(result.space) - result = cast[PGenericSeq](growObj(result, elemSize * result.reserved + - GenericSeqSize)) + let r = resize(result.space) + result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, elemAlign) + elemSize * r)) + result.reserved = r inc(result.len) -proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} = +proc incrSeqV2(seq: PGenericSeq, elemSize, elemAlign: int): PGenericSeq {.compilerproc.} = # incrSeq version 2 result = seq if result.len >= result.space: - result.reserved = resize(result.space) - result = cast[PGenericSeq](growObj(result, elemSize * result.reserved + - GenericSeqSize)) - -proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. - compilerRtl.} = + let r = resize(result.space) + result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, elemAlign) + elemSize * r)) + result.reserved = r + +proc incrSeqV3(s: PGenericSeq, typ: PNimType): PGenericSeq {.compilerproc.} = + if s == nil: + result = cast[PGenericSeq](newSeq(typ, 1)) + result.len = 0 + else: + result = s + if result.len >= result.space: + let r = resize(result.space) + result = cast[PGenericSeq](newSeq(typ, r)) + result.len = s.len + copyMem(dataPointer(result, typ.base.align), dataPointer(s, typ.base.align), s.len * typ.base.size) + # since we steal the content from 's', it's crucial to set s's len to 0. + s.len = 0 + +proc setLengthSeq(seq: PGenericSeq, elemSize, elemAlign, newLen: int): PGenericSeq {. + compilerRtl, inl.} = result = seq if result.space < newLen: - result.reserved = max(resize(result.space), newLen) - result = cast[PGenericSeq](growObj(result, elemSize * result.reserved + - GenericSeqSize)) + let r = max(resize(result.space), newLen) + result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, elemAlign) + elemSize * r)) + result.reserved = r elif newLen < result.len: # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc): - when false: # compileOption("gc", "v2"): + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcRegions): + if ntfNoRefs notin extGetCellType(result).base.flags: for i in newLen..result.len-1: - let len0 = gch.tempStack.len - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), - extGetCellType(result).base, waPush) - let len1 = gch.tempStack.len - for i in len0 .. <len1: - doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) - gch.tempStack.len = len0 - else: - for i in newLen..result.len-1: - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), + forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). - # This is a tought problem, because even if we don't zeroMem here, in the + # This is a tough problem, because even if we don't zeroMem here, in the # presence of user defined destructors, the user will expect the cell to be - # "destroyed" thus creating the same problem. We can destoy the cell in the + # "destroyed" thus creating the same problem. We can destroy the cell in the # finalizer of the sequence, but this makes destruction non-deterministic. - zeroMem(cast[pointer](cast[ByteAddress](result) +% GenericSeqSize +% - (newLen*%elemSize)), (result.len-%newLen) *% elemSize) + zeroMem(dataPointer(result, elemAlign, elemSize, newLen), (result.len-%newLen) *% elemSize) result.len = newLen -# --------------- other string routines ---------------------------------- -proc nimIntToStr(x: int): string {.compilerRtl.} = - result = newString(sizeof(x)*4) - var i = 0 - var y = x - while true: - var d = y div 10 - result[i] = chr(abs(int(y - d*10)) + ord('0')) - inc(i) - y = d - if y == 0: break - if x < 0: - result[i] = '-' - inc(i) - setLen(result, i) - # mirror the string: - for j in 0..i div 2 - 1: - swap(result[j], result[i-j-1]) - -proc nimFloatToStr(f: float): string {.compilerproc.} = - var buf: array[0..64, char] - var n: int = c_sprintf(buf, "%.16g", f) - var hasDot = false - for i in 0..n-1: - if buf[i] == ',': - buf[i] = '.' - hasDot = true - elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: - hasDot = true - if not hasDot: - buf[n] = '.' - buf[n+1] = '0' - buf[n+2] = '\0' - # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' are produced. - # We want to get rid of these here: - if buf[n-1] == 'N': - result = "nan" - elif buf[n-1] == 'F': - if buf[0] == '-': - result = "-inf" +proc setLengthSeqV2(s: PGenericSeq, typ: PNimType, newLen: int): PGenericSeq {. + compilerRtl.} = + sysAssert typ.kind == tySequence, "setLengthSeqV2: type is not a seq" + if s == nil: + if newLen == 0: + result = s else: - result = "inf" + result = cast[PGenericSeq](newSeq(typ, newLen)) else: - result = $buf - -proc strtod(buf: cstring, endptr: ptr cstring): float64 {.importc, - header: "<stdlib.h>", noSideEffect.} - -var decimalPoint: char - -proc getDecimalPoint(): char = - result = decimalPoint - if result == '\0': - if strtod("0,5", nil) == 0.5: result = ',' - else: result = '.' - # yes this is threadsafe in practice, spare me: - decimalPoint = result - -const - IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} - -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, - start = 0): int {.compilerProc.} = - # This routine leverages `strtod()` for the non-trivial task of - # parsing floating point numbers correctly. Because `strtod()` is - # locale-dependent with respect to the radix character, we create - # a copy where the decimal point is replaced with the locale's - # radix character. - var - i = start - sign = 1.0 - t: array[500, char] # flaviu says: 325 is the longest reasonable literal - ti = 0 - hasdigits = false - - template addToBuf(c) = - if ti < t.high: - t[ti] = c; inc(ti) - - # Sign? - if s[i] == '+' or s[i] == '-': - if s[i] == '-': - sign = -1.0 - t[ti] = s[i] - inc(i); inc(ti) - - # NaN? - if s[i] == 'N' or s[i] == 'n': - if s[i+1] == 'A' or s[i+1] == 'a': - if s[i+2] == 'N' or s[i+2] == 'n': - if s[i+3] notin IdentChars: - number = NaN - return i+3 - start - return 0 - - # Inf? - if s[i] == 'I' or s[i] == 'i': - if s[i+1] == 'N' or s[i+1] == 'n': - if s[i+2] == 'F' or s[i+2] == 'f': - if s[i+3] notin IdentChars: - number = Inf*sign - return i+3 - start - return 0 - - # Integer part? - while s[i] in {'0'..'9'}: - hasdigits = true - addToBuf(s[i]) - inc(i); - while s[i] == '_': inc(i) - - # Fractional part? - if s[i] == '.': - addToBuf(getDecimalPoint()) - inc(i) - while s[i] in {'0'..'9'}: - hasdigits = true - addToBuf(s[i]) - inc(i) - while s[i] == '_': inc(i) - if not hasdigits: - return 0 - - # Exponent? - if s[i] in {'e', 'E'}: - addToBuf(s[i]) - inc(i) - if s[i] in {'+', '-'}: - addToBuf(s[i]) - inc(i) - if s[i] notin {'0'..'9'}: - return 0 - while s[i] in {'0'..'9'}: - addToBuf(s[i]) - inc(i) - while s[i] == '_': inc(i) - number = strtod(t, nil) - result = i - start - -proc nimInt64ToStr(x: int64): string {.compilerRtl.} = - result = newString(sizeof(x)*4) - var i = 0 - var y = x - while true: - var d = y div 10 - result[i] = chr(abs(int(y - d*10)) + ord('0')) - inc(i) - y = d - if y == 0: break - if x < 0: - result[i] = '-' - inc(i) - setLen(result, i) - # mirror the string: - for j in 0..i div 2 - 1: - swap(result[j], result[i-j-1]) - -proc nimBoolToStr(x: bool): string {.compilerRtl.} = - return if x: "true" else: "false" - -proc nimCharToStr(x: char): string {.compilerRtl.} = - result = newString(1) - result[0] = x - -proc binaryStrSearch(x: openArray[string], y: string): int {.compilerproc.} = - var - a = 0 - b = len(x) - while a < b: - var mid = (a + b) div 2 - if x[mid] < y: - a = mid + 1 + let elemSize = typ.base.size + let elemAlign = typ.base.align + if s.space < newLen: + let r = max(resize(s.space), newLen) + result = cast[PGenericSeq](newSeq(typ, r)) + copyMem(dataPointer(result, elemAlign), dataPointer(s, elemAlign), s.len * elemSize) + # since we steal the content from 's', it's crucial to set s's len to 0. + s.len = 0 + elif newLen < s.len: + result = s + # we need to decref here, otherwise the GC leaks! + when not defined(boehmGC) and not defined(nogc) and + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcRegions): + if ntfNoRefs notin typ.base.flags: + for i in newLen..result.len-1: + forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), + extGetCellType(result).base, waZctDecRef) + + # XXX: zeroing out the memory can still result in crashes if a wiped-out + # cell is aliased by another pointer (ie proc parameter or a let variable). + # This is a tough problem, because even if we don't zeroMem here, in the + # presence of user defined destructors, the user will expect the cell to be + # "destroyed" thus creating the same problem. We can destroy the cell in the + # finalizer of the sequence, but this makes destruction non-deterministic. + zeroMem(dataPointer(result, elemAlign, elemSize, newLen), (result.len-%newLen) *% elemSize) else: - b = mid - if a < len(x) and x[a] == y: - result = a - else: - result = -1 + result = s + zeroMem(dataPointer(result, elemAlign, elemSize, result.len), (newLen-%result.len) *% elemSize) + result.len = newLen + +func capacity*(self: string): int {.inline.} = + ## Returns the current capacity of the string. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var str = newStringOfCap(cap = 42) + str.add "Nim" + assert str.capacity == 42 + + let str = cast[NimString](self) + result = if str != nil: str.space else: 0 + +func capacity*[T](self: seq[T]): int {.inline.} = + ## Returns the current capacity of the seq. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var lst = newSeqOfCap[string](cap = 42) + lst.add "Nim" + assert lst.capacity == 42 + + let sek = cast[PGenericSeq](self) + result = if sek != nil: sek.space else: 0 diff --git a/lib/system/threadids.nim b/lib/system/threadids.nim new file mode 100644 index 000000000..3a6eadcbb --- /dev/null +++ b/lib/system/threadids.nim @@ -0,0 +1,103 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# we need to cache current threadId to not perform syscall all the time +var threadId {.threadvar.}: int + +when defined(windows): + proc getCurrentThreadId(): int32 {. + stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getCurrentThreadId()) + result = threadId + +elif defined(linux): + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} + when defined(amd64): + const NR_gettid = clong(186) + else: + var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: clong + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(NR_gettid)) + result = threadId + +elif defined(dragonfly): + proc lwp_gettid(): int32 {.importc, header: "unistd.h".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_gettid()) + result = threadId + +elif defined(openbsd): + proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getthrid()) + result = threadId + +elif defined(netbsd): + proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_self()) + result = threadId + +elif defined(freebsd): + proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} + var SYS_thr_self {.importc:"SYS_thr_self", header:"<sys/syscall.h>".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + var tid = 0.cint + if threadId == 0: + discard syscall(SYS_thr_self, addr tid) + threadId = tid + result = threadId + +elif defined(macosx): + proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} + var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"<sys/syscall.h>".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(SYS_thread_selfid)) + result = threadId + +elif defined(solaris): + type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int + proc thr_self(): thread_t {.importc, header: "<thread.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(thr_self()) + result = threadId + +elif defined(haiku): + type thr_id {.importc: "thread_id", header: "<OS.h>".} = distinct int32 + proc find_thread(name: cstring): thr_id {.importc, header: "<OS.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(find_thread(nil)) + result = threadId diff --git a/lib/system/threadimpl.nim b/lib/system/threadimpl.nim new file mode 100644 index 000000000..285b8f5e7 --- /dev/null +++ b/lib/system/threadimpl.nim @@ -0,0 +1,111 @@ +var + nimThreadDestructionHandlers* {.rtlThreadVar.}: seq[proc () {.closure, gcsafe, raises: [].}] +when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcRegions): + proc deallocOsPages() {.rtl, raises: [].} +proc threadTrouble() {.raises: [], gcsafe.} +# create for the main thread. Note: do not insert this data into the list +# of all threads; it's not to be stopped etc. +when not defined(useNimRtl): + #when not defined(createNimRtl): initStackBottom() + when declared(initGC): + initGC() + when not emulatedThreadVars: + type ThreadType {.pure.} = enum + None = 0, + NimThread = 1, + ForeignThread = 2 + var + threadType {.rtlThreadVar.}: ThreadType + + threadType = ThreadType.NimThread + +when defined(gcDestructors): + proc deallocThreadStorage(p: pointer) = c_free(p) +else: + template deallocThreadStorage(p: pointer) = deallocShared(p) + +template afterThreadRuns() = + for i in countdown(nimThreadDestructionHandlers.len-1, 0): + nimThreadDestructionHandlers[i]() + +proc onThreadDestruction*(handler: proc () {.closure, gcsafe, raises: [].}) = + ## Registers a *thread local* handler that is called at the thread's + ## destruction. + ## + ## A thread is destructed when the `.thread` proc returns + ## normally or when it raises an exception. Note that unhandled exceptions + ## in a thread nevertheless cause the whole process to die. + nimThreadDestructionHandlers.add handler + +when defined(boehmgc): + type GCStackBaseProc = proc(sb: pointer, t: pointer) {.noconv.} + proc boehmGC_call_with_stack_base(sbp: GCStackBaseProc, p: pointer) + {.importc: "GC_call_with_stack_base", boehmGC.} + proc boehmGC_register_my_thread(sb: pointer) + {.importc: "GC_register_my_thread", boehmGC.} + proc boehmGC_unregister_my_thread() + {.importc: "GC_unregister_my_thread", boehmGC.} + + proc threadProcWrapDispatch[TArg](sb: pointer, thrd: pointer) {.noconv, raises: [].} = + boehmGC_register_my_thread(sb) + try: + let thrd = cast[ptr Thread[TArg]](thrd) + when TArg is void: + thrd.dataFn() + else: + thrd.dataFn(thrd.data) + except: + threadTrouble() + finally: + afterThreadRuns() + boehmGC_unregister_my_thread() +else: + proc threadProcWrapDispatch[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + try: + when TArg is void: + thrd.dataFn() + else: + when defined(nimV2): + thrd.dataFn(thrd.data) + else: + var x: TArg + deepCopy(x, thrd.data) + thrd.dataFn(x) + except: + threadTrouble() + finally: + afterThreadRuns() + when hasAllocStack: + deallocThreadStorage(thrd.rawStack) + +proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + when defined(boehmgc): + boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) + elif not defined(nogc) and not defined(gogc) and not defined(gcRegions) and not usesDestructors: + var p {.volatile.}: pointer + # init the GC for refc/markandsweep + nimGC_setStackBottom(addr(p)) + when declared(initGC): + initGC() + when declared(threadType): + threadType = ThreadType.NimThread + threadProcWrapDispatch[TArg](thrd) + when declared(deallocOsPages): deallocOsPages() + else: + threadProcWrapDispatch(thrd) + +template nimThreadProcWrapperBody*(closure: untyped): untyped = + var thrd = cast[ptr Thread[TArg]](closure) + var core = thrd.core + when declared(globalsSlot): threadVarSetValue(globalsSlot, thrd.core) + threadProcWrapStackFrame(thrd) + # Since an unhandled exception terminates the whole process (!), there is + # no need for a ``try finally`` here, nor would it be correct: The current + # exception is tried to be re-raised by the code-gen after the ``finally``! + # However this is doomed to fail, because we already unmapped every heap + # page! + + # mark as not running anymore: + thrd.core = nil + thrd.dataFn = nil + deallocThreadStorage(cast[pointer](core)) diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim new file mode 100644 index 000000000..e6ad9dca5 --- /dev/null +++ b/lib/system/threadlocalstorage.nim @@ -0,0 +1,125 @@ +import std/private/threadtypes + +when defined(windows): + type + ThreadVarSlot = distinct int32 + + proc threadVarAlloc(): ThreadVarSlot {. + importc: "TlsAlloc", stdcall, header: "<windows.h>".} + proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. + importc: "TlsSetValue", stdcall, header: "<windows.h>".} + proc tlsGetValue(dwTlsIndex: ThreadVarSlot): pointer {. + importc: "TlsGetValue", stdcall, header: "<windows.h>".} + + proc getLastError(): uint32 {. + importc: "GetLastError", stdcall, header: "<windows.h>".} + proc setLastError(x: uint32) {. + importc: "SetLastError", stdcall, header: "<windows.h>".} + + proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer = + let realLastError = getLastError() + result = tlsGetValue(dwTlsIndex) + setLastError(realLastError) + +elif defined(genode): + const + GenodeHeader = "genode_cpp/threads.h" + + type + ThreadVarSlot = int + + proc threadVarAlloc(): ThreadVarSlot = 0 + + proc offMainThread(): bool {. + importcpp: "Nim::SysThread::offMainThread", + header: GenodeHeader.} + + proc threadVarSetValue(value: pointer) {. + importcpp: "Nim::SysThread::threadVarSetValue(@)", + header: GenodeHeader.} + + proc threadVarGetValue(): pointer {. + importcpp: "Nim::SysThread::threadVarGetValue()", + header: GenodeHeader.} + + var mainTls: pointer + + proc threadVarSetValue(s: ThreadVarSlot, value: pointer) {.inline.} = + if offMainThread(): + threadVarSetValue(value); + else: + mainTls = value + + proc threadVarGetValue(s: ThreadVarSlot): pointer {.inline.} = + if offMainThread(): + threadVarGetValue(); + else: + mainTls + +else: + when not (defined(macosx) or defined(haiku)): + {.passl: "-pthread".} + + when not defined(haiku): + {.passc: "-pthread".} + + when (defined(linux) or defined(nintendoswitch)) and defined(amd64): + type + ThreadVarSlot {.importc: "pthread_key_t", + header: "<sys/types.h>".} = distinct cuint + elif defined(openbsd) and defined(amd64): + type + ThreadVarSlot {.importc: "pthread_key_t", + header: "<pthread.h>".} = cint + else: + type + ThreadVarSlot {.importc: "pthread_key_t", + header: "<sys/types.h>".} = object + + proc pthread_getspecific(a1: ThreadVarSlot): pointer {. + importc: "pthread_getspecific", header: pthreadh.} + proc pthread_key_create(a1: ptr ThreadVarSlot, + destruct: proc (x: pointer) {.noconv.}): int32 {. + importc: "pthread_key_create", header: pthreadh.} + proc pthread_key_delete(a1: ThreadVarSlot): int32 {. + importc: "pthread_key_delete", header: pthreadh.} + + proc pthread_setspecific(a1: ThreadVarSlot, a2: pointer): int32 {. + importc: "pthread_setspecific", header: pthreadh.} + + proc threadVarAlloc(): ThreadVarSlot {.inline.} = + discard pthread_key_create(addr(result), nil) + proc threadVarSetValue(s: ThreadVarSlot, value: pointer) {.inline.} = + discard pthread_setspecific(s, value) + proc threadVarGetValue(s: ThreadVarSlot): pointer {.inline.} = + result = pthread_getspecific(s) + + +when emulatedThreadVars: + # the compiler generates this proc for us, so that we can get the size of + # the thread local var block; we use this only for sanity checking though + proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".} + + + +when emulatedThreadVars: + var globalsSlot: ThreadVarSlot + + when not defined(useNimRtl): + var mainThread: GcThread + + proc GetThreadLocalVars(): pointer {.compilerRtl, inl.} = + result = addr(cast[PGcThread](threadVarGetValue(globalsSlot)).tls) + + proc initThreadVarsEmulation() {.compilerproc, inline.} = + when not defined(useNimRtl): + globalsSlot = threadVarAlloc() + when declared(mainThread): + threadVarSetValue(globalsSlot, addr(mainThread)) + +when not defined(useNimRtl): + when emulatedThreadVars: + if nimThreadVarsSize() > sizeof(ThreadLocalStorage): + c_fprintf(cstderr, """too large thread local storage size requested, +use -d:\"nimTlsSize=X\" to setup even more or stop using unittest.nim""") + rawQuit 1 diff --git a/lib/system/threads.nim b/lib/system/threads.nim deleted file mode 100644 index bdb737e35..000000000 --- a/lib/system/threads.nim +++ /dev/null @@ -1,474 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Thread support for Nim. **Note**: This is part of the system module. -## Do not import it directly. To activate thread support you need to compile -## with the ``--threads:on`` command line switch. -## -## Nim's memory model for threads is quite different from other common -## programming languages (C, Pascal): Each thread has its own -## (garbage collected) heap and sharing of memory is restricted. This helps -## to prevent race conditions and improves efficiency. See `the manual for -## details of this memory model <manual.html#threads>`_. -## -## Example: -## -## .. code-block:: Nim -## -## import locks -## -## var -## thr: array [0..4, Thread[tuple[a,b: int]]] -## L: Lock -## -## proc threadFunc(interval: tuple[a,b: int]) {.thread.} = -## for i in interval.a..interval.b: -## acquire(L) # lock stdout -## echo i -## release(L) -## -## initLock(L) -## -## for i in 0..high(thr): -## createThread(thr[i], threadFunc, (i*10, i*10+5)) -## joinThreads(thr) - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -const - maxRegisters = 256 # don't think there is an arch with more registers - useStackMaskHack = false ## use the stack mask hack for better performance - StackGuardSize = 4096 - ThreadStackMask = 1024*256*sizeof(int)-1 - ThreadStackSize = ThreadStackMask+1 - StackGuardSize - -when defined(windows): - type - SysThread = Handle - WinThreadProc = proc (x: pointer): int32 {.stdcall.} - {.deprecated: [TSysThread: SysThread, TWinThreadProc: WinThreadProc].} - - proc createThread(lpThreadAttributes: pointer, dwStackSize: int32, - lpStartAddress: WinThreadProc, - lpParameter: pointer, - dwCreationFlags: int32, - lpThreadId: var int32): SysThread {. - stdcall, dynlib: "kernel32", importc: "CreateThread".} - - proc winSuspendThread(hThread: SysThread): int32 {. - stdcall, dynlib: "kernel32", importc: "SuspendThread".} - - proc winResumeThread(hThread: SysThread): int32 {. - stdcall, dynlib: "kernel32", importc: "ResumeThread".} - - proc waitForMultipleObjects(nCount: int32, - lpHandles: ptr SysThread, - bWaitAll: int32, - dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} - - proc terminateThread(hThread: SysThread, dwExitCode: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "TerminateThread".} - - type - ThreadVarSlot = distinct int32 - - when true: - proc threadVarAlloc(): ThreadVarSlot {. - importc: "TlsAlloc", stdcall, header: "<windows.h>".} - proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. - importc: "TlsSetValue", stdcall, header: "<windows.h>".} - proc tlsGetValue(dwTlsIndex: ThreadVarSlot): pointer {. - importc: "TlsGetValue", stdcall, header: "<windows.h>".} - - proc getLastError(): uint32 {. - importc: "GetLastError", stdcall, header: "<windows.h>".} - proc setLastError(x: uint32) {. - importc: "SetLastError", stdcall, header: "<windows.h>".} - - proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer = - let realLastError = getLastError() - result = tlsGetValue(dwTlsIndex) - setLastError(realLastError) - else: - proc threadVarAlloc(): ThreadVarSlot {. - importc: "TlsAlloc", stdcall, dynlib: "kernel32".} - proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. - importc: "TlsSetValue", stdcall, dynlib: "kernel32".} - proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer {. - importc: "TlsGetValue", stdcall, dynlib: "kernel32".} - - proc setThreadAffinityMask(hThread: SysThread, dwThreadAffinityMask: uint) {. - importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} - -else: - when not defined(macosx): - {.passL: "-pthread".} - - {.passC: "-pthread".} - const - schedh = "#define _GNU_SOURCE\n#include <sched.h>" - pthreadh = "#define _GNU_SOURCE\n#include <pthread.h>" - - type - SysThread {.importc: "pthread_t", header: "<sys/types.h>", - final, pure.} = object - Pthread_attr {.importc: "pthread_attr_t", - header: "<sys/types.h>", final, pure.} = object - - Timespec {.importc: "struct timespec", - header: "<time.h>", final, pure.} = object - tv_sec: int - tv_nsec: int - {.deprecated: [TSysThread: SysThread, Tpthread_attr: PThreadAttr, - Ttimespec: Timespec].} - - proc pthread_attr_init(a1: var PthreadAttr) {. - importc, header: pthreadh.} - proc pthread_attr_setstacksize(a1: var PthreadAttr, a2: int) {. - importc, header: pthreadh.} - - proc pthread_create(a1: var SysThread, a2: var PthreadAttr, - a3: proc (x: pointer): pointer {.noconv.}, - a4: pointer): cint {.importc: "pthread_create", - header: pthreadh.} - proc pthread_join(a1: SysThread, a2: ptr pointer): cint {. - importc, header: pthreadh.} - - proc pthread_cancel(a1: SysThread): cint {. - importc: "pthread_cancel", header: pthreadh.} - - type - ThreadVarSlot {.importc: "pthread_key_t", pure, final, - header: "<sys/types.h>".} = object - {.deprecated: [TThreadVarSlot: ThreadVarSlot].} - - proc pthread_getspecific(a1: ThreadVarSlot): pointer {. - importc: "pthread_getspecific", header: pthreadh.} - proc pthread_key_create(a1: ptr ThreadVarSlot, - destruct: proc (x: pointer) {.noconv.}): int32 {. - importc: "pthread_key_create", header: pthreadh.} - proc pthread_key_delete(a1: ThreadVarSlot): int32 {. - importc: "pthread_key_delete", header: pthreadh.} - - proc pthread_setspecific(a1: ThreadVarSlot, a2: pointer): int32 {. - importc: "pthread_setspecific", header: pthreadh.} - - proc threadVarAlloc(): ThreadVarSlot {.inline.} = - discard pthread_key_create(addr(result), nil) - proc threadVarSetValue(s: ThreadVarSlot, value: pointer) {.inline.} = - discard pthread_setspecific(s, value) - proc threadVarGetValue(s: ThreadVarSlot): pointer {.inline.} = - result = pthread_getspecific(s) - - when useStackMaskHack: - proc pthread_attr_setstack(attr: var PthreadAttr, stackaddr: pointer, - size: int): cint {. - importc: "pthread_attr_setstack", header: pthreadh.} - - type CpuSet {.importc: "cpu_set_t", header: schedh.} = object - proc cpusetZero(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} - proc cpusetIncl(cpu: cint; s: var CpuSet) {. - importc: "CPU_SET", header: schedh.} - - proc setAffinity(thread: SysThread; setsize: csize; s: var CpuSet) {. - importc: "pthread_setaffinity_np", header: pthreadh.} - -const - emulatedThreadVars = compileOption("tlsEmulation") - -when emulatedThreadVars: - # the compiler generates this proc for us, so that we can get the size of - # the thread local var block; we use this only for sanity checking though - proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".} - -# we preallocate a fixed size for thread local storage, so that no heap -# allocations are needed. Currently less than 7K are used on a 64bit machine. -# We use ``float`` for proper alignment: -type - ThreadLocalStorage = array [0..1_000, float] - - PGcThread = ptr GcThread - GcThread {.pure, inheritable.} = object - sys: SysThread - when emulatedThreadVars and not useStackMaskHack: - tls: ThreadLocalStorage - else: - nil - when hasSharedHeap: - next, prev: PGcThread - stackBottom, stackTop: pointer - stackSize: int - else: - nil -{.deprecated: [TThreadLocalStorage: ThreadLocalStorage, TGcThread: GcThread].} - -when not defined(useNimRtl): - when not useStackMaskHack: - var mainThread: GcThread - -#const globalsSlot = ThreadVarSlot(0) -#sysAssert checkSlot.int == globalsSlot.int - -when emulatedThreadVars: - # XXX it'd be more efficient to not use a global variable for the - # thread storage slot, but to rely on the implementation to assign slot X - # for us... ;-) - var globalsSlot: ThreadVarSlot - - proc GetThreadLocalVars(): pointer {.compilerRtl, inl.} = - result = addr(cast[PGcThread](threadVarGetValue(globalsSlot)).tls) - - proc initThreadVarsEmulation() {.compilerProc, inline.} = - when not defined(useNimRtl): - globalsSlot = threadVarAlloc() - when declared(mainThread): - threadVarSetValue(globalsSlot, addr(mainThread)) - -when useStackMaskHack: - proc maskStackPointer(offset: int): pointer {.compilerRtl, inl.} = - var x {.volatile.}: pointer - x = addr(x) - result = cast[pointer]((cast[int](x) and not ThreadStackMask) +% - (0) +% offset) - -# create for the main thread. Note: do not insert this data into the list -# of all threads; it's not to be stopped etc. -when not defined(useNimRtl): - when not useStackMaskHack: - #when not defined(createNimRtl): initStackBottom() - when declared(initGC): initGC() - - when emulatedThreadVars: - if nimThreadVarsSize() > sizeof(ThreadLocalStorage): - echo "too large thread local storage size requested" - quit 1 - - when hasSharedHeap and not defined(boehmgc) and not defined(gogc) and not defined(nogc): - var - threadList: PGcThread - - proc registerThread(t: PGcThread) = - # we need to use the GC global lock here! - acquireSys(HeapLock) - t.prev = nil - t.next = threadList - if threadList != nil: - sysAssert(threadList.prev == nil, "threadList.prev == nil") - threadList.prev = t - threadList = t - releaseSys(HeapLock) - - proc unregisterThread(t: PGcThread) = - # we need to use the GC global lock here! - acquireSys(HeapLock) - if t == threadList: threadList = t.next - if t.next != nil: t.next.prev = t.prev - if t.prev != nil: t.prev.next = t.next - # so that a thread can be unregistered twice which might happen if the - # code executes `destroyThread`: - t.next = nil - t.prev = nil - releaseSys(HeapLock) - - # on UNIX, the GC uses ``SIGFREEZE`` to tell every thread to stop so that - # the GC can examine the stacks? - proc stopTheWord() = discard - -# We jump through some hops here to ensure that Nim thread procs can have -# the Nim calling convention. This is needed because thread procs are -# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just -# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. - -type - Thread* {.pure, final.}[TArg] = - object of GcThread ## Nim thread. A thread is a heavy object (~14K) - ## that **must not** be part of a message! Use - ## a ``ThreadId`` for that. - when TArg is void: - dataFn: proc () {.nimcall, gcsafe.} - else: - dataFn: proc (m: TArg) {.nimcall, gcsafe.} - data: TArg - ThreadId*[TArg] = ptr Thread[TArg] ## the current implementation uses - ## a pointer as a thread ID. -{.deprecated: [TThread: Thread, TThreadId: ThreadId].} - -when not defined(boehmgc) and not hasSharedHeap and not defined(gogc): - proc deallocOsPages() - -when defined(boehmgc): - type GCStackBaseProc = proc(sb: pointer, t: pointer) {.noconv.} - proc boehmGC_call_with_stack_base(sbp: GCStackBaseProc, p: pointer) - {.importc: "GC_call_with_stack_base", boehmGC.} - proc boehmGC_register_my_thread(sb: pointer) - {.importc: "GC_register_my_thread", boehmGC.} - proc boehmGC_unregister_my_thread() - {.importc: "GC_unregister_my_thread", boehmGC.} - - proc threadProcWrapDispatch[TArg](sb: pointer, thrd: pointer) {.noconv.} = - boehmGC_register_my_thread(sb) - let thrd = cast[ptr Thread[TArg]](thrd) - when TArg is void: - thrd.dataFn() - else: - thrd.dataFn(thrd.data) - boehmGC_unregister_my_thread() -else: - proc threadProcWrapDispatch[TArg](thrd: ptr Thread[TArg]) = - when TArg is void: - thrd.dataFn() - else: - thrd.dataFn(thrd.data) - -proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = - when defined(boehmgc): - boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) - elif not defined(nogc) and not defined(gogc): - var p {.volatile.}: proc(a: ptr Thread[TArg]) {.nimcall.} = - threadProcWrapDispatch[TArg] - when not hasSharedHeap: - # init the GC for refc/markandsweep - setStackBottom(addr(p)) - initGC() - when declared(registerThread): - thrd.stackBottom = addr(thrd) - registerThread(thrd) - p(thrd) - when declared(registerThread): unregisterThread(thrd) - when declared(deallocOsPages): deallocOsPages() - else: - threadProcWrapDispatch(thrd) - -template threadProcWrapperBody(closure: expr) {.immediate.} = - when declared(globalsSlot): threadVarSetValue(globalsSlot, closure) - var thrd = cast[ptr Thread[TArg]](closure) - threadProcWrapStackFrame(thrd) - # Since an unhandled exception terminates the whole process (!), there is - # no need for a ``try finally`` here, nor would it be correct: The current - # exception is tried to be re-raised by the code-gen after the ``finally``! - # However this is doomed to fail, because we already unmapped every heap - # page! - - # mark as not running anymore: - thrd.dataFn = nil - -{.push stack_trace:off.} -when defined(windows): - proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = - threadProcWrapperBody(closure) - # implicitly return 0 -else: - proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = - threadProcWrapperBody(closure) -{.pop.} - -proc running*[TArg](t: Thread[TArg]): bool {.inline.} = - ## returns true if `t` is running. - result = t.dataFn != nil - -when hostOS == "windows": - proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = - ## waits for the thread `t` to finish. - discard waitForSingleObject(t.sys, -1'i32) - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## waits for every thread in `t` to finish. - var a: array[0..255, SysThread] - sysAssert a.len >= t.len, "a.len >= t.len" - for i in 0..t.high: a[i] = t[i].sys - discard waitForMultipleObjects(t.len.int32, - cast[ptr SysThread](addr(a)), 1, -1) - -else: - proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = - ## waits for the thread `t` to finish. - discard pthread_join(t.sys, nil) - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## waits for every thread in `t` to finish. - for i in 0..t.high: joinThread(t[i]) - -when false: - # XXX a thread should really release its heap here somehow: - proc destroyThread*[TArg](t: var Thread[TArg]) = - ## forces the thread `t` to terminate. This is potentially dangerous if - ## you don't have full control over `t` and its acquired resources. - when hostOS == "windows": - discard TerminateThread(t.sys, 1'i32) - else: - discard pthread_cancel(t.sys) - when declared(registerThread): unregisterThread(addr(t)) - t.dataFn = nil - -when hostOS == "windows": - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread.}, - param: TArg) = - ## creates a new thread `t` and starts its execution. Entry point is the - ## proc `tp`. `param` is passed to `tp`. `TArg` can be ``void`` if you - ## don't need to pass any data to the thread. - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.stackSize = ThreadStackSize - var dummyThreadId: int32 - t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], - addr(t), 0'i32, dummyThreadId) - if t.sys <= 0: - raise newException(ResourceExhaustedError, "cannot create thread") - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - ## pins a thread to a `CPU`:idx:. In other words sets a - ## thread's `affinity`:idx:. If you don't know what this means, you - ## shouldn't use this proc. - setThreadAffinityMask(t.sys, uint(1 shl cpu)) - -else: - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread.}, - param: TArg) = - ## creates a new thread `t` and starts its execution. Entry point is the - ## proc `tp`. `param` is passed to `tp`. `TArg` can be ``void`` if you - ## don't need to pass any data to the thread. - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.stackSize = ThreadStackSize - var a {.noinit.}: PthreadAttr - pthread_attr_init(a) - pthread_attr_setstacksize(a, ThreadStackSize) - if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: - raise newException(ResourceExhaustedError, "cannot create thread") - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - ## pins a thread to a `CPU`:idx:. In other words sets a - ## thread's `affinity`:idx:. If you don't know what this means, you - ## shouldn't use this proc. - var s {.noinit.}: CpuSet - cpusetZero(s) - cpusetIncl(cpu.cint, s) - setAffinity(t.sys, sizeof(s), s) - -proc createThread*(t: var Thread[void], tp: proc () {.thread.}) = - createThread[void](t, tp) - -proc threadId*[TArg](t: var Thread[TArg]): ThreadId[TArg] {.inline.} = - ## returns the thread ID of `t`. - result = addr(t) - -when false: - proc mainThreadId*[TArg](): ThreadId[TArg] = - ## returns the thread ID of the main thread. - result = cast[ThreadId[TArg]](addr(mainThread)) - -when useStackMaskHack: - proc runMain(tp: proc () {.thread.}) {.compilerproc.} = - var mainThread: Thread[pointer] - createThread(mainThread, tp) - joinThread(mainThread) diff --git a/lib/system/timers.nim b/lib/system/timers.nim index ac8418824..ffb0f7716 100644 --- a/lib/system/timers.nim +++ b/lib/system/timers.nim @@ -8,12 +8,11 @@ # ## Timer support for the realtime GC. Based on -## `<https://github.com/jckarter/clay/blob/master/compiler/src/hirestimer.cpp>`_ +## `<https://github.com/jckarter/clay/blob/master/compiler/hirestimer.cpp>`_ type Ticks = distinct int64 Nanos = int64 -{.deprecated: [TTicks: Ticks, TNanos: Nanos].} when defined(windows): @@ -32,15 +31,14 @@ when defined(windows): result = Nanos(float64(a.int64 - b.int64) * performanceCounterRate) -elif defined(macosx): +elif defined(macosx) and not defined(emscripten): type MachTimebaseInfoData {.pure, final, importc: "mach_timebase_info_data_t", header: "<mach/mach_time.h>".} = object - numer, denom: int32 - {.deprecated: [TMachTimebaseInfoData: MachTimebaseInfoData].} + numer, denom: int32 # note: `uint32` in sources - proc mach_absolute_time(): int64 {.importc, header: "<mach/mach.h>".} + proc mach_absolute_time(): uint64 {.importc, header: "<mach/mach_time.h>".} proc mach_timebase_info(info: var MachTimebaseInfoData) {.importc, header: "<mach/mach_time.h>".} @@ -51,7 +49,7 @@ elif defined(macosx): mach_timebase_info(timeBaseInfo) proc `-`(a, b: Ticks): Nanos = - result = (a.int64 - b.int64) * timeBaseInfo.numer div timeBaseInfo.denom + result = (a.int64 - b.int64) * timeBaseInfo.numer div timeBaseInfo.denom elif defined(posixRealtime): type @@ -61,7 +59,6 @@ elif defined(posixRealtime): final, pure.} = object ## struct timespec tv_sec: int ## Seconds. tv_nsec: int ## Nanoseconds. - {.deprecated: [TClockid: Clickid, TTimeSpec: TimeSpec].} var CLOCK_REALTIME {.importc: "CLOCK_REALTIME", header: "<time.h>".}: Clockid @@ -78,12 +75,18 @@ elif defined(posixRealtime): else: # fallback Posix implementation: + when not declared(Time): + when defined(linux): + type Time = clong + else: + type Time = int + type Timeval {.importc: "struct timeval", header: "<sys/select.h>", final, pure.} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. - {.deprecated: [Ttimeval: Timeval].} + tv_sec: Time ## Seconds. + tv_usec: clong ## Microseconds. + proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim deleted file mode 100644 index 5a30a7c0f..000000000 --- a/lib/system/widestrs.nim +++ /dev/null @@ -1,160 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Nim support for C/C++'s `wide strings`:idx:. This is part of the system -# module! Do not import it directly! - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -type - Utf16Char* = distinct int16 - WideCString* = ref array[0.. 1_000_000, Utf16Char] -{.deprecated: [TUtf16Char: Utf16Char].} - -proc len*(w: WideCString): int = - ## returns the length of a widestring. This traverses the whole string to - ## find the binary zero end marker! - while int16(w[result]) != 0'i16: inc result - -const - UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) - UNI_MAX_BMP = 0x0000FFFF - UNI_MAX_UTF16 = 0x0010FFFF - UNI_MAX_UTF32 = 0x7FFFFFFF - UNI_MAX_LEGAL_UTF32 = 0x0010FFFF - - halfShift = 10 - halfBase = 0x0010000 - halfMask = 0x3FF - - UNI_SUR_HIGH_START = 0xD800 - UNI_SUR_HIGH_END = 0xDBFF - UNI_SUR_LOW_START = 0xDC00 - UNI_SUR_LOW_END = 0xDFFF - -template ones(n: expr): expr = ((1 shl n)-1) - -template fastRuneAt(s: cstring, i: int, result: expr, doInc = true) = - ## Returns the unicode character ``s[i]`` in `result`. If ``doInc == true`` - ## `i` is incremented by the number of bytes that have been processed. - bind ones - - if ord(s[i]) <=% 127: - result = ord(s[i]) - when doInc: inc(i) - elif ord(s[i]) shr 5 == 0b110: - #assert(ord(s[i+1]) shr 6 == 0b10) - result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) - when doInc: inc(i, 2) - elif ord(s[i]) shr 4 == 0b1110: - #assert(ord(s[i+1]) shr 6 == 0b10) - #assert(ord(s[i+2]) shr 6 == 0b10) - result = (ord(s[i]) and ones(4)) shl 12 or - (ord(s[i+1]) and ones(6)) shl 6 or - (ord(s[i+2]) and ones(6)) - when doInc: inc(i, 3) - elif ord(s[i]) shr 3 == 0b11110: - #assert(ord(s[i+1]) shr 6 == 0b10) - #assert(ord(s[i+2]) shr 6 == 0b10) - #assert(ord(s[i+3]) shr 6 == 0b10) - result = (ord(s[i]) and ones(3)) shl 18 or - (ord(s[i+1]) and ones(6)) shl 12 or - (ord(s[i+2]) and ones(6)) shl 6 or - (ord(s[i+3]) and ones(6)) - when doInc: inc(i, 4) - else: - result = 0xFFFD - when doInc: inc(i) - -iterator runes(s: cstring): int = - var - i = 0 - result: int - while s[i] != '\0': - fastRuneAt(s, i, result, true) - yield result - -proc newWideCString*(source: cstring, L: int): WideCString = - unsafeNew(result, L * 4 + 2) - #result = cast[wideCString](alloc(L * 4 + 2)) - var d = 0 - for ch in runes(source): - if ch <=% UNI_MAX_BMP: - if ch >=% UNI_SUR_HIGH_START and ch <=% UNI_SUR_LOW_END: - result[d] = UNI_REPLACEMENT_CHAR - else: - result[d] = Utf16Char(toU16(ch)) - elif ch >% UNI_MAX_UTF16: - result[d] = UNI_REPLACEMENT_CHAR - else: - let ch = ch -% halfBase - result[d] = Utf16Char(toU16((ch shr halfShift) +% UNI_SUR_HIGH_START)) - inc d - result[d] = Utf16Char(toU16((ch and halfMask) +% UNI_SUR_LOW_START)) - inc d - result[d] = Utf16Char(0'i16) - -proc newWideCString*(s: cstring): WideCString = - if s.isNil: return nil - - when not declared(c_strlen): - proc c_strlen(a: cstring): int {. - header: "<string.h>", noSideEffect, importc: "strlen".} - - let L = c_strlen(s) - result = newWideCString(s, L) - -proc newWideCString*(s: string): WideCString = - result = newWideCString(s, s.len) - -proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = - result = newStringOfCap(estimate + estimate shr 2) - - var i = 0 - while w[i].int16 != 0'i16: - var ch = int(cast[uint16](w[i])) - inc i - if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: - # If the 16 bits following the high surrogate are in the source buffer... - let ch2 = int(cast[uint16](w[i])) - - # If it's a low surrogate, convert to UTF32: - if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: - ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase - inc i - else: - #invalid UTF-16 - ch = replacement - elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: - #invalid UTF-16 - ch = replacement - - if ch < 0x80: - result.add chr(ch) - elif ch < 0x800: - result.add chr((ch shr 6) or 0xc0) - result.add chr((ch and 0x3f) or 0x80) - elif ch < 0x10000: - result.add chr((ch shr 12) or 0xe0) - result.add chr(((ch shr 6) and 0x3f) or 0x80) - result.add chr((ch and 0x3f) or 0x80) - elif ch <= 0x10FFFF: - result.add chr((ch shr 18) or 0xf0) - result.add chr(((ch shr 12) and 0x3f) or 0x80) - result.add chr(((ch shr 6) and 0x3f) or 0x80) - result.add chr((ch and 0x3f) or 0x80) - else: - # replacement char(in case user give very large number): - result.add chr(0xFFFD shr 12 or 0b1110_0000) - result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) - result.add chr(0xFFFD and ones(6) or 0b10_0000_00) - -proc `$`*(s: WideCString): string = - result = s $ 80 diff --git a/lib/system_overview.rst b/lib/system_overview.rst new file mode 100644 index 000000000..cc0643bf1 --- /dev/null +++ b/lib/system_overview.rst @@ -0,0 +1,177 @@ +.. default-role:: code +.. include:: ../doc/rstcommon.rst + +The System module imports several separate modules, and their documentation +is in separate files: + +* `iterators <iterators.html>`_ +* `exceptions <exceptions.html>`_ +* `assertions <assertions.html>`_ +* `dollars <dollars.html>`_ +* `ctypes <ctypes.html>`_ + + +Here is a short overview of the most commonly used functions from the +`system` module. Function names in the tables below are clickable and +will take you to the full documentation of the function. + +There are many more functions available than the ones listed in this overview. +Use the table of contents on the left-hand side and/or `Ctrl+F` to navigate +through this module. + + +Strings and characters +---------------------- + +============================= ======================================= +Proc Usage +============================= ======================================= +`len(s)<#len,string>`_ Return the length of a string +`chr(i)<#chr,range[]>`_ Convert an `int` in the range `0..255` + to a character +`ord(c)<#ord,T>`_ Return `int` value of a character +`a & b<#&,string,string>`_ Concatenate two strings +`s.add(c)<#add,string,char>`_ Add character to the string +`$<dollars.html>`_ Convert various types to string +============================= ======================================= + +**See also:** +* `strutils module <strutils.html>`_ for common string functions +* `strformat module <strformat.html>`_ for string interpolation and formatting +* `unicode module <unicode.html>`_ for Unicode UTF-8 handling +* `strscans <strscans.html>`_ for `scanf` and `scanp` macros, which offer + easier substring extraction than regular expressions +* `strtabs module <strtabs.html>`_ for efficient hash tables + (dictionaries, in some programming languages) mapping from strings to strings + + + +Seqs +---- + +============================================================= ========================================== +Proc Usage +============================================================= ========================================== +`newSeq<#newSeq>`_ Create a new sequence of a given length +`newSeqOfCap<#newSeqOfCap,Natural>`_ Create a new sequence with zero length + and a given capacity +`setLen<#setLen,seq[T],Natural>`_ Set the length of a sequence +`len<#len,seq[T]>`_ Return the length of a sequence +`@<#@,openArray[T]>`_ Turn an array into a sequence +`add<#add,seq[T],sinkT>`_ Add an item to the sequence +`insert<#insert,seq[T],sinkT>`_ Insert an item at a specific position +`delete<#delete,seq[T],Natural>`_ Delete an item while preserving the + order of elements (`O(n)` operation) +`del<#del,seq[T],Natural>`_ `O(1)` removal, doesn't preserve the order +`pop<#pop,seq[T]>`_ Remove and return last item of a sequence +`x & y<#&,seq[T],seq[T]>`_ Concatenate two sequences +`x[a .. b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence (both ends included) +`x[a .. ^b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence but `b` is a + reversed index (both ends included) +`x[a ..< b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence (excluded upper bound) +============================================================= ========================================== + +**See also:** +* `sequtils module <sequtils.html>`_ for operations on container + types (including strings) +* `json module <json.html>`_ for a structure which allows heterogeneous members +* `lists module <lists.html>`_ for linked lists + + + +Sets +---- + +Built-in bit sets. + +=============================== ====================================== +Proc Usage +=============================== ====================================== +`incl<#incl,set[T],T>`_ Include element `y` in the set `x` +`excl<#excl,set[T],T>`_ Exclude element `y` from the set `x` +`card<#card,set[T]>`_ Return the cardinality of the set, + i.e. the number of elements +`a * b<#*,set[T],set[T]>`_ Intersection +`a + b<#+,set[T],set[T]>`_ Union +`a - b<#-,set[T],set[T]>`_ Difference +`contains<#contains,set[T],T>`_ Check if an element is in the set +[a < b](#<,set[T],set[T]) Check if `a` is a subset of `b` +=============================== ====================================== + +**See also:** +* `setutils module <setutils.html>`_ for bit set convenience functions +* `sets module <sets.html>`_ for hash sets +* `intsets module <intsets.html>`_ for efficient int sets + + + +Numbers +------- + +============================== ================================== ===================== +Proc Usage Also known as + (in other languages) +============================== ================================== ===================== +`div<#div,int,int>`_ Integer division `//` +`mod<#mod,int,int>`_ Integer modulo (remainder) `%` +`shl<#shl,int,SomeInteger>`_ Shift left `<<` +`shr<#shr,int,SomeInteger>`_ Shift right `>>` +`ashr<#ashr,int,SomeInteger>`_ Arithmetic shift right +`and<#and,int,int>`_ Bitwise `and` `&` +`or<#or,int,int>`_ Bitwise `or` `|` +`xor<#xor,int,int>`_ Bitwise `xor` `^` +`not<#not,int>`_ Bitwise `not` (complement) `~` +`toInt<#toInt,float>`_ Convert floating-point number + into an `int` +`toFloat<#toFloat,int>`_ Convert an integer into a `float` +============================== ================================== ===================== + +**See also:** +* `math module <math.html>`_ for mathematical operations like trigonometric + functions, logarithms, square and cubic roots, etc. +* `complex module <complex.html>`_ for operations on complex numbers +* `rationals module <rationals.html>`_ for rational numbers + + + +Ordinals +-------- + +`Ordinal type <#Ordinal>`_ includes integer, bool, character, and enumeration +types, as well as their subtypes. + +===================== ======================================= +Proc Usage +===================== ======================================= +`succ<#succ,T,int>`_ Successor of the value +`pred<#pred,T,int>`_ Predecessor of the value +`inc<#inc,T,int>`_ Increment the ordinal +`dec<#dec,T,int>`_ Decrement the ordinal +`high<#high,T>`_ Return the highest possible value +`low<#low,T>`_ Return the lowest possible value +`ord<#ord,T>`_ Return `int` value of an ordinal value +===================== ======================================= + + + +Misc +---- + +==================================================== ============================================ +Proc Usage +==================================================== ============================================ +`is<#is,T,S>`_ Check if two arguments are of the same type +`isnot<#isnot.t,untyped,untyped>`_ Negated version of `is` +`!=<#!%3D.t,untyped,untyped>`_ Not equals +`addr<#addr,T>`_ Take the address of a memory location +`T and F<#and,bool,bool>`_ Boolean `and` +`T or F<#or,bool,bool>`_ Boolean `or` +`T xor F<#xor,bool,bool>`_ Boolean `xor` (exclusive or) +`not T<#not,bool>`_ Boolean `not` +`a[^x]<#^.t,int>`_ Take the element at the reversed index `x` +`a .. b<#..,sinkT,sinkU>`_ Binary slice that constructs an interval + `[a, b]` +`a ..^ b<#..^.t,untyped,untyped>`_ Interval `[a, b]` but `b` as reversed index +[a ..< b](#..<.t,untyped,untyped) Interval `[a, b)` (excluded upper bound) +[runnableExamples](#runnableExamples,string,untyped) Create testable documentation +==================================================== ============================================ diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim new file mode 100644 index 000000000..207172f8c --- /dev/null +++ b/lib/windows/registry.nim @@ -0,0 +1,82 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module is experimental and its interface may change. + +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/widestrs + +type + HKEY* = uint + +const + HKEY_LOCAL_MACHINE* = HKEY(0x80000002u) + HKEY_CURRENT_USER* = HKEY(2147483649) + + RRF_RT_ANY = 0x0000ffff + KEY_WOW64_64KEY = 0x0100 + KEY_WOW64_32KEY = 0x0200 + KEY_READ = 0x00020019 + REG_SZ = 1 + +proc regOpenKeyEx(hKey: HKEY, lpSubKey: WideCString, ulOptions: int32, + samDesired: int32, + phkResult: var HKEY): int32 {. + importc: "RegOpenKeyExW", dynlib: "Advapi32.dll", stdcall.} + +proc regCloseKey(hkey: HKEY): int32 {. + importc: "RegCloseKey", dynlib: "Advapi32.dll", stdcall.} + +proc regGetValue(key: HKEY, lpSubKey, lpValue: WideCString; + dwFlags: int32 = RRF_RT_ANY, pdwType: ptr int32, + pvData: pointer, + pcbData: ptr int32): int32 {. + importc: "RegGetValueW", dynlib: "Advapi32.dll", stdcall.} + +template call(f) = + let err = f + if err != 0: + raiseOSError(err.OSErrorCode, astToStr(f)) + +proc getUnicodeValue*(path, key: string; handle: HKEY): string = + let hh = newWideCString path + let kk = newWideCString key + var bufSize: int32 + # try a couple of different flag settings: + var flags: int32 = RRF_RT_ANY + let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufSize) + if err != 0: + var newHandle: HKEY + call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) + call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufSize) + if bufSize > 0: + var res = newWideCString(bufSize) + call regGetValue(newHandle, nil, kk, flags, nil, addr res[0], + addr bufSize) + result = res $ bufSize + call regCloseKey(newHandle) + else: + if bufSize > 0: + var res = newWideCString(bufSize) + call regGetValue(handle, hh, kk, flags, nil, addr res[0], + addr bufSize) + result = res $ bufSize + +proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, + dwType: int32; lpData: WideCString; cbData: int32): int32 {. + importc: "RegSetKeyValueW", dynlib: "Advapi32.dll", stdcall.} + +proc setUnicodeValue*(path, key, val: string; handle: HKEY) = + let hh = newWideCString path + let kk = newWideCString key + let vv = newWideCString val + call regSetValue(handle, hh, kk, REG_SZ, vv, (vv.len.int32+1)*2) + diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 4962186fb..79681376b 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -10,29 +10,44 @@ ## This module implements a small wrapper for some needed Win API procedures, ## so that the Nim compiler does not depend on the huge Windows module. -{.deadCodeElim:on.} +import std/dynlib -const - useWinUnicode* = not defined(useWinAnsi) +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +{.passc: "-DWIN32_LEAN_AND_MEAN".} + +when defined(nimPreviewSlimSystem): + from std/syncio import FileHandle + import std/widestrs +type WinChar* = Utf16Char + +# See https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types type Handle* = int LONG* = int32 ULONG* = int32 PULONG* = ptr int WINBOOL* = int32 + ## `WINBOOL` uses opposite convention as posix, !=0 meaning success. + # xxx this should be distinct int32, distinct would make code less error prone + PBOOL* = ptr WINBOOL DWORD* = int32 PDWORD* = ptr DWORD LPINT* = ptr int32 + ULONG_PTR* = uint + PULONG_PTR* = ptr uint HDC* = Handle HGLRC* = Handle + BYTE* = uint8 - SECURITY_ATTRIBUTES* {.final, pure.} = object + SECURITY_ATTRIBUTES* = object nLength*: int32 lpSecurityDescriptor*: pointer bInheritHandle*: WINBOOL - STARTUPINFO* {.final, pure.} = object + STARTUPINFO* = object cb*: int32 lpReserved*: cstring lpDesktop*: cstring @@ -52,17 +67,17 @@ type hStdOutput*: Handle hStdError*: Handle - PROCESS_INFORMATION* {.final, pure.} = object + PROCESS_INFORMATION* = object hProcess*: Handle hThread*: Handle dwProcessId*: int32 dwThreadId*: int32 - FILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT + FILETIME* = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT dwLowDateTime*: DWORD dwHighDateTime*: DWORD - BY_HANDLE_FILE_INFORMATION* {.final, pure.} = object + BY_HANDLE_FILE_INFORMATION* = object dwFileAttributes*: DWORD ftCreationTime*: FILETIME ftLastAccessTime*: FILETIME @@ -74,16 +89,19 @@ type nFileIndexHigh*: DWORD nFileIndexLow*: DWORD -{.deprecated: [THandle: Handle, TSECURITY_ATTRIBUTES: SECURITY_ATTRIBUTES, - TSTARTUPINFO: STARTUPINFO, TPROCESS_INFORMATION: PROCESS_INFORMATION, - TFILETIME: FILETIME, TBY_HANDLE_FILE_INFORMATION: BY_HANDLE_FILE_INFORMATION].} + OSVERSIONINFO* = object + dwOSVersionInfoSize*: DWORD + dwMajorVersion*: DWORD + dwMinorVersion*: DWORD + dwBuildNumber*: DWORD + dwPlatformId*: DWORD + szCSDVersion*: array[0..127, WinChar] + + Protoent* = object + p_name*: cstring + p_aliases*: cstringArray + p_proto*: cshort -when useWinUnicode: - type WinChar* = Utf16Char - {.deprecated: [TWinChar: WinChar].} -else: - type WinChar* = char - {.deprecated: [TWinChar: WinChar].} const STARTF_USESHOWWINDOW* = 1'i32 @@ -96,6 +114,7 @@ const WAIT_TIMEOUT* = 0x00000102'i32 WAIT_FAILED* = 0xFFFFFFFF'i32 INFINITE* = -1'i32 + STILL_ACTIVE* = 0x00000103'i32 STD_INPUT_HANDLE* = -10'i32 STD_OUTPUT_HANDLE* = -11'i32 @@ -113,32 +132,44 @@ const PIPE_ACCESS_OUTBOUND* = 2'i32 PIPE_NOWAIT* = 0x00000001'i32 SYNCHRONIZE* = 0x00100000'i32 - FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32 CREATE_NO_WINDOW* = 0x08000000'i32 + HANDLE_FLAG_INHERIT* = 0x00000001'i32 + +proc isSuccess*(a: WINBOOL): bool {.inline.} = + ## Returns true if `a != 0`. Windows uses a different convention than POSIX, + ## where `a == 0` is commonly used on success. + a != 0 +proc getVersionExW*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetVersionExW", sideEffect.} +proc getVersionExA*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetVersionExA", sideEffect.} + +proc getVersion*(): DWORD {.stdcall, dynlib: "kernel32", importc: "GetVersion", sideEffect.} + proc closeHandle*(hObject: Handle): WINBOOL {.stdcall, dynlib: "kernel32", importc: "CloseHandle".} -proc readFile*(hFile: Handle, Buffer: pointer, nNumberOfBytesToRead: int32, +proc readFile*(hFile: Handle, buffer: pointer, nNumberOfBytesToRead: int32, lpNumberOfBytesRead: ptr int32, lpOverlapped: pointer): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "ReadFile".} + stdcall, dynlib: "kernel32", importc: "ReadFile", sideEffect.} -proc writeFile*(hFile: Handle, Buffer: pointer, nNumberOfBytesToWrite: int32, +proc writeFile*(hFile: Handle, buffer: pointer, nNumberOfBytesToWrite: int32, lpNumberOfBytesWritten: ptr int32, lpOverlapped: pointer): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "WriteFile".} + stdcall, dynlib: "kernel32", importc: "WriteFile", sideEffect.} proc createPipe*(hReadPipe, hWritePipe: var Handle, lpPipeAttributes: var SECURITY_ATTRIBUTES, nSize: int32): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreatePipe".} + stdcall, dynlib: "kernel32", importc: "CreatePipe", sideEffect.} proc createNamedPipe*(lpName: WideCString, dwOpenMode, dwPipeMode, nMaxInstances, nOutBufferSize, nInBufferSize, nDefaultTimeOut: int32, lpSecurityAttributes: ptr SECURITY_ATTRIBUTES): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateNamedPipeW".} + stdcall, dynlib: "kernel32", importc: "CreateNamedPipeW", sideEffect.} proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, nBufferSize: int32 = 0, @@ -147,37 +178,25 @@ proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, lpBytesLeftThisMessage: ptr int32 = nil): bool {. stdcall, dynlib: "kernel32", importc: "PeekNamedPipe".} -when useWinUnicode: - proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment, lpCurrentDirectory: WideCString, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessW".} - -else: - proc createProcessA*(lpApplicationName, lpCommandLine: cstring, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment: pointer, lpCurrentDirectory: cstring, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessA".} - +proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, + lpProcessAttributes: ptr SECURITY_ATTRIBUTES, + lpThreadAttributes: ptr SECURITY_ATTRIBUTES, + bInheritHandles: WINBOOL, dwCreationFlags: int32, + lpEnvironment, lpCurrentDirectory: WideCString, + lpStartupInfo: var STARTUPINFO, + lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} proc suspendThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", - importc: "SuspendThread".} + importc: "SuspendThread", sideEffect.} proc resumeThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", - importc: "ResumeThread".} + importc: "ResumeThread", sideEffect.} proc waitForSingleObject*(hHandle: Handle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", sideEffect.} proc terminateProcess*(hProcess: Handle, uExitCode: int): WINBOOL {.stdcall, - dynlib: "kernel32", importc: "TerminateProcess".} + dynlib: "kernel32", importc: "TerminateProcess", sideEffect.} proc getExitCodeProcess*(hProcess: Handle, lpExitCode: var int32): WINBOOL {. stdcall, dynlib: "kernel32", importc: "GetExitCodeProcess".} @@ -185,87 +204,84 @@ proc getExitCodeProcess*(hProcess: Handle, lpExitCode: var int32): WINBOOL {. proc getStdHandle*(nStdHandle: int32): Handle {.stdcall, dynlib: "kernel32", importc: "GetStdHandle".} proc setStdHandle*(nStdHandle: int32, hHandle: Handle): WINBOOL {.stdcall, - dynlib: "kernel32", importc: "SetStdHandle".} + dynlib: "kernel32", importc: "SetStdHandle", sideEffect.} proc flushFileBuffers*(hFile: Handle): WINBOOL {.stdcall, dynlib: "kernel32", - importc: "FlushFileBuffers".} + importc: "FlushFileBuffers", sideEffect.} proc getLastError*(): int32 {.importc: "GetLastError", - stdcall, dynlib: "kernel32".} - -when useWinUnicode: - proc formatMessageW*(dwFlags: int32, lpSource: pointer, - dwMessageId, dwLanguageId: int32, - lpBuffer: pointer, nSize: int32, - Arguments: pointer): int32 {. - importc: "FormatMessageW", stdcall, dynlib: "kernel32".} -else: - proc formatMessageA*(dwFlags: int32, lpSource: pointer, + stdcall, dynlib: "kernel32", sideEffect.} + +proc setLastError*(error: int32) {.importc: "SetLastError", + stdcall, dynlib: "kernel32", sideEffect.} + +proc formatMessageW*(dwFlags: int32, lpSource: pointer, dwMessageId, dwLanguageId: int32, lpBuffer: pointer, nSize: int32, - Arguments: pointer): int32 {. - importc: "FormatMessageA", stdcall, dynlib: "kernel32".} + arguments: pointer): int32 {. + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} proc localFree*(p: pointer) {. importc: "LocalFree", stdcall, dynlib: "kernel32".} -when useWinUnicode: - proc getCurrentDirectoryW*(nBufferLength: int32, - lpBuffer: WideCString): int32 {. - importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall.} - proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. - importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall.} - proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. - importc: "CreateDirectoryW", dynlib: "kernel32", stdcall.} - proc removeDirectoryW*(lpPathName: WideCString): int32 {. - importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall.} - proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW".} - - proc getModuleFileNameW*(handle: Handle, buf: WideCString, - size: int32): int32 {.importc: "GetModuleFileNameW", - dynlib: "kernel32", stdcall.} -else: - proc getCurrentDirectoryA*(nBufferLength: int32, lpBuffer: cstring): int32 {. - importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall.} - proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. - importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall.} - proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. - importc: "CreateDirectoryA", dynlib: "kernel32", stdcall.} - proc removeDirectoryA*(lpPathName: cstring): int32 {. - importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall.} - proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA".} - - proc getModuleFileNameA*(handle: Handle, buf: cstring, size: int32): int32 {. - importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} - -when useWinUnicode: - proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall.} - proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, - security: pointer=nil): int32 {. - importc:"CreateHardLinkW", dynlib: "kernel32", stdcall.} -else: - proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall.} - proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, - security: pointer=nil): int32 {. - importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.} +proc getCurrentDirectoryW*(nBufferLength: int32, + lpBuffer: WideCString): int32 {. + importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. + importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. + importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc removeDirectoryW*(lpPathName: WideCString): int32 {. + importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} + +proc getModuleFileNameW*(handle: Handle, buf: WideCString, + size: int32): int32 {.importc: "GetModuleFileNameW", + dynlib: "kernel32", stdcall.} + +proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} +proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, + security: pointer=nil): int32 {. + importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} const - FILE_ATTRIBUTE_ARCHIVE* = 32'i32 - FILE_ATTRIBUTE_COMPRESSED* = 2048'i32 - FILE_ATTRIBUTE_NORMAL* = 128'i32 - FILE_ATTRIBUTE_DIRECTORY* = 16'i32 - FILE_ATTRIBUTE_HIDDEN* = 2'i32 - FILE_ATTRIBUTE_READONLY* = 1'i32 - FILE_ATTRIBUTE_REPARSE_POINT* = 1024'i32 - FILE_ATTRIBUTE_SYSTEM* = 4'i32 - FILE_ATTRIBUTE_TEMPORARY* = 256'i32 + FILE_ATTRIBUTE_READONLY* = 0x00000001'i32 + FILE_ATTRIBUTE_HIDDEN* = 0x00000002'i32 + FILE_ATTRIBUTE_SYSTEM* = 0x00000004'i32 + FILE_ATTRIBUTE_DIRECTORY* = 0x00000010'i32 + FILE_ATTRIBUTE_ARCHIVE* = 0x00000020'i32 + FILE_ATTRIBUTE_DEVICE* = 0x00000040'i32 + FILE_ATTRIBUTE_NORMAL* = 0x00000080'i32 + FILE_ATTRIBUTE_TEMPORARY* = 0x00000100'i32 + FILE_ATTRIBUTE_SPARSE_FILE* = 0x00000200'i32 + FILE_ATTRIBUTE_REPARSE_POINT* = 0x00000400'i32 + FILE_ATTRIBUTE_COMPRESSED* = 0x00000800'i32 + FILE_ATTRIBUTE_OFFLINE* = 0x00001000'i32 + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED* = 0x00002000'i32 + + FILE_FLAG_FIRST_PIPE_INSTANCE* = 0x00080000'i32 + FILE_FLAG_OPEN_NO_RECALL* = 0x00100000'i32 + FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32 + FILE_FLAG_POSIX_SEMANTICS* = 0x01000000'i32 + FILE_FLAG_BACKUP_SEMANTICS* = 0x02000000'i32 + FILE_FLAG_DELETE_ON_CLOSE* = 0x04000000'i32 + FILE_FLAG_SEQUENTIAL_SCAN* = 0x08000000'i32 + FILE_FLAG_RANDOM_ACCESS* = 0x10000000'i32 + FILE_FLAG_NO_BUFFERING* = 0x20000000'i32 + FILE_FLAG_OVERLAPPED* = 0x40000000'i32 + FILE_FLAG_WRITE_THROUGH* = 0x80000000'i32 MAX_PATH* = 260 + + MOVEFILE_COPY_ALLOWED* = 0x2'i32 + MOVEFILE_CREATE_HARDLINK* = 0x10'i32 + MOVEFILE_DELAY_UNTIL_REBOOT* = 0x4'i32 + MOVEFILE_FAIL_IF_NOT_TRACKABLE* = 0x20'i32 + MOVEFILE_REPLACE_EXISTING* = 0x1'i32 + MOVEFILE_WRITE_THROUGH* = 0x8'i32 + type WIN32_FIND_DATA* {.pure.} = object dwFileAttributes*: int32 @@ -278,110 +294,67 @@ type dwReserved1: int32 cFileName*: array[0..(MAX_PATH) - 1, WinChar] cAlternateFileName*: array[0..13, WinChar] -{.deprecated: [TWIN32_FIND_DATA: WIN32_FIND_DATA].} - -when useWinUnicode: - proc findFirstFileW*(lpFileName: WideCString, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileW".} - proc findNextFileW*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileW".} -else: - proc findFirstFileA*(lpFileName: cstring, - lpFindFileData: var WIN32_FIND_DATA): THANDLE {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileA".} - proc findNextFileA*(hFindFile: THANDLE, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileA".} + +proc findFirstFileW*(lpFileName: WideCString, + lpFindFileData: var WIN32_FIND_DATA): Handle {. + stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} +proc findNextFileW*(hFindFile: Handle, + lpFindFileData: var WIN32_FIND_DATA): int32 {. + stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} proc findClose*(hFindFile: Handle) {.stdcall, dynlib: "kernel32", importc: "FindClose".} -when useWinUnicode: - proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, - lpBuffer: WideCString, - lpFilePart: var WideCString): int32 {. +proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, + lpBuffer: WideCString, + lpFilePart: var WideCString): int32 {. + stdcall, dynlib: "kernel32", + importc: "GetFullPathNameW", sideEffect.} +proc getFileAttributesW*(lpFileName: WideCString): int32 {. stdcall, dynlib: "kernel32", - importc: "GetFullPathNameW".} - proc getFileAttributesW*(lpFileName: WideCString): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesW".} - proc setFileAttributesW*(lpFileName: WideCString, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesW".} - - proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: cint): cint {. - importc: "CopyFileW", stdcall, dynlib: "kernel32".} - - proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: cint): cint {. - importc: "MoveFileW", stdcall, dynlib: "kernel32".} - - proc getEnvironmentStringsW*(): WideCString {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW".} - proc freeEnvironmentStringsW*(para1: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW".} - - proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", - stdcall, dynlib: "kernel32".} + importc: "GetFileAttributesW", sideEffect.} +proc setFileAttributesW*(lpFileName: WideCString, + dwFileAttributes: int32): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} -else: - proc getFullPathNameA*(lpFileName: cstring, nBufferLength: int32, - lpBuffer: cstring, lpFilePart: var cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFullPathNameA".} - proc getFileAttributesA*(lpFileName: cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesA".} - proc setFileAttributesA*(lpFileName: cstring, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesA".} - - proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. - importc: "CopyFileA", stdcall, dynlib: "kernel32".} - - proc moveFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. - importc: "MoveFileA", stdcall, dynlib: "kernel32".} - - proc getEnvironmentStringsA*(): cstring {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsA".} - proc freeEnvironmentStringsA*(para1: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsA".} - - proc getCommandLineA*(): cstring {. - importc: "GetCommandLineA", stdcall, dynlib: "kernel32".} +proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, + bFailIfExists: WINBOOL): WINBOOL {. + importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} + +proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. + importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, + flags: DWORD): WINBOOL {. + importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} + +proc getEnvironmentStringsW*(): WideCString {. + stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} +proc freeEnvironmentStringsW*(para1: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} + +proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", + stdcall, dynlib: "kernel32", sideEffect.} proc rdFileTime*(f: FILETIME): int64 = - result = ze64(f.dwLowDateTime) or (ze64(f.dwHighDateTime) shl 32) + result = int64(cast[uint32](f.dwLowDateTime)) or (int64(cast[uint32](f.dwHighDateTime)) shl 32) proc rdFileSize*(f: WIN32_FIND_DATA): int64 = - result = ze64(f.nFileSizeLow) or (ze64(f.nFileSizeHigh) shl 32) + result = int64(cast[uint32](f.nFileSizeLow)) or (int64(cast[uint32](f.nFileSizeHigh)) shl 32) proc getSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. - importc: "GetSystemTimeAsFileTime", dynlib: "kernel32", stdcall.} + importc: "GetSystemTimeAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} proc sleep*(dwMilliseconds: int32){.stdcall, dynlib: "kernel32", - importc: "Sleep".} + importc: "Sleep", sideEffect.} -when useWinUnicode: - proc shellExecuteW*(HWND: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: WideCString, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW".} - -else: - proc shellExecuteA*(HWND: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: cstring, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteA".} +proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, + lpParameters, lpDirectory: WideCString, + nShowCmd: int32): Handle{. + stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} proc getFileInformationByHandle*(hFile: Handle, lpFileInformation: ptr BY_HANDLE_FILE_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "GetFileInformationByHandle".} + stdcall, dynlib: "kernel32", importc: "GetFileInformationByHandle", sideEffect.} const WSADESCRIPTION_LEN* = 256 @@ -389,21 +362,17 @@ const FD_SETSIZE* = 64 MSG_PEEK* = 2 - INADDR_ANY* = 0 + INADDR_ANY* = 0'u32 INADDR_LOOPBACK* = 0x7F000001 INADDR_BROADCAST* = -1 INADDR_NONE* = -1 ws2dll = "Ws2_32.dll" - WSAEWOULDBLOCK* = 10035 - WSAEINPROGRESS* = 10036 - -proc wsaGetLastError*(): cint {.importc: "WSAGetLastError", dynlib: ws2dll.} +proc wsaGetLastError*(): cint {.importc: "WSAGetLastError", dynlib: ws2dll, sideEffect.} type SocketHandle* = distinct int -{.deprecated: [TSocketHandle: SocketHandle].} type WSAData* {.importc: "WSADATA", header: "winsock2.h".} = object @@ -414,35 +383,38 @@ type lpVendorInfo: cstring SockAddr* {.importc: "SOCKADDR", header: "winsock2.h".} = object - sa_family*: int16 # unsigned - sa_data: array[0..13, char] + sa_family*: uint16 + sa_data*: array[0..13, char] + + PSockAddr = ptr SockAddr - InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object - s_addr*: int32 # IP address + InAddr* {.importc: "IN_ADDR", header: "winsock2.h", union.} = object + s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", header: "winsock2.h".} = object - sin_family*: int16 - sin_port*: int16 # unsigned + sin_family*: uint16 + sin_port*: uint16 sin_addr*: InAddr sin_zero*: array[0..7, char] In6_addr* {.importc: "IN6_ADDR", header: "winsock2.h".} = object - bytes*: array[0..15, char] + bytes* {.importc: "u.Byte".}: array[0..15, char] Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object - sin6_family*: int16 - sin6_port*: int16 # unsigned + sin6_family*: uint16 + sin6_port*: uint16 sin6_flowinfo*: int32 # unsigned sin6_addr*: In6_addr sin6_scope_id*: int32 # unsigned - Sockaddr_in6_old* = object - sin6_family*: int16 - sin6_port*: int16 # unsigned - sin6_flowinfo*: int32 # unsigned - sin6_addr*: In6_addr + Sockaddr_storage* {.importc: "SOCKADDR_STORAGE", + header: "winsock2.h".} = object + ss_family*: uint16 + ss_pad1 {.importc: "__ss_pad1".}: array[6, byte] + ss_align {.importc: "__ss_align".}: int64 + ss_pad2 {.importc: "__ss_pad2".}: array[112, byte] Servent* = object s_name*: cstring @@ -465,26 +437,26 @@ type fd_count*: cint # unsigned fd_array*: array[0..FD_SETSIZE-1, SocketHandle] - Timeval* = object - tv_sec*, tv_usec*: int32 - AddrInfo* = object ai_flags*: cint ## Input flags. ai_family*: cint ## Address family of socket. ai_socktype*: cint ## Socket type. ai_protocol*: cint ## Protocol of socket. - ai_addrlen*: int ## Length of socket address. + ai_addrlen*: csize_t ## Length of socket address. ai_canonname*: cstring ## Canonical name of service location. ai_addr*: ptr SockAddr ## Socket address of socket. ai_next*: ptr AddrInfo ## Pointer to next in list. SockLen* = cuint -{.deprecated: [TSockaddr_in: Sockaddr_in, TAddrinfo: AddrInfo, - TSockAddr: SockAddr, TSockLen: SockLen, TTimeval: Timeval, - TWSADATA: WSADATA, Thostent: Hostent, TServent: Servent, - TInAddr: InAddr, Tin6_addr: In6_addr, Tsockaddr_in6: Sockaddr_in6, - Tsockaddr_in6_old: Sockaddr_in6_old].} +when defined(cpp): + type + Timeval* {.importc: "timeval", header: "<time.h>".} = object + tv_sec*, tv_usec*: int32 +else: + type + Timeval* = object + tv_sec*, tv_usec*: int32 var SOMAXCONN* {.importc, header: "winsock2.h".}: cint @@ -493,6 +465,8 @@ var SO_DEBUG* {.importc, header: "winsock2.h".}: cint ## turn on debugging info recording SO_ACCEPTCONN* {.importc, header: "winsock2.h".}: cint # socket has had listen() SO_REUSEADDR* {.importc, header: "winsock2.h".}: cint # allow local address reuse + SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "winsock2.h".}: cint # allow port reuse. Since Windows does not really support it, mapped to SO_REUSEADDR. This shouldn't cause problems. + SO_KEEPALIVE* {.importc, header: "winsock2.h".}: cint # keep connections alive SO_DONTROUTE* {.importc, header: "winsock2.h".}: cint # just use interface addresses SO_BROADCAST* {.importc, header: "winsock2.h".}: cint # permit sending of broadcast msgs @@ -503,20 +477,32 @@ var SO_DONTLINGER* {.importc, header: "winsock2.h".}: cint SO_EXCLUSIVEADDRUSE* {.importc, header: "winsock2.h".}: cint # disallow local address reuse SO_ERROR* {.importc, header: "winsock2.h".}: cint + TCP_NODELAY* {.importc, header: "winsock2.h".}: cint proc `==`*(x, y: SocketHandle): bool {.borrow.} proc getservbyname*(name, proto: cstring): ptr Servent {. - stdcall, importc: "getservbyname", dynlib: ws2dll.} + stdcall, importc: "getservbyname", dynlib: ws2dll, sideEffect.} proc getservbyport*(port: cint, proto: cstring): ptr Servent {. - stdcall, importc: "getservbyport", dynlib: ws2dll.} + stdcall, importc: "getservbyport", dynlib: ws2dll, sideEffect.} proc gethostbyaddr*(ip: ptr InAddr, len: cuint, theType: cint): ptr Hostent {. - stdcall, importc: "gethostbyaddr", dynlib: ws2dll.} + stdcall, importc: "gethostbyaddr", dynlib: ws2dll, sideEffect.} proc gethostbyname*(name: cstring): ptr Hostent {. - stdcall, importc: "gethostbyname", dynlib: ws2dll.} + stdcall, importc: "gethostbyname", dynlib: ws2dll, sideEffect.} + +proc gethostname*(hostname: cstring, len: cint): cint {. + stdcall, importc: "gethostname", dynlib: ws2dll, sideEffect.} + +proc getprotobyname*( + name: cstring +): ptr Protoent {.stdcall, importc: "getprotobyname", dynlib: ws2dll, sideEffect.} + +proc getprotobynumber*( + proto: cint +): ptr Protoent {.stdcall, importc: "getprotobynumber", dynlib: ws2dll, sideEffect.} proc socket*(af, typ, protocol: cint): SocketHandle {. stdcall, importc: "socket", dynlib: ws2dll.} @@ -567,7 +553,7 @@ proc getnameinfo*(a1: ptr SockAddr, a2: SockLen, a6: SockLen, a7: cint): cint {. stdcall, importc: "getnameinfo", dynlib: ws2dll.} -proc inet_addr*(cp: cstring): int32 {. +proc inet_addr*(cp: cstring): uint32 {. stdcall, importc: "inet_addr", dynlib: ws2dll.} proc WSAFDIsSet(s: SocketHandle, set: var TFdSet): bool {. @@ -591,22 +577,18 @@ proc getaddrinfo*(nodename, servname: cstring, hints: ptr AddrInfo, res: var ptr AddrInfo): cint {. stdcall, importc: "getaddrinfo", dynlib: ws2dll.} -proc freeaddrinfo*(ai: ptr AddrInfo) {. +proc freeAddrInfo*(ai: ptr AddrInfo) {. stdcall, importc: "freeaddrinfo", dynlib: ws2dll.} proc inet_ntoa*(i: InAddr): cstring {. stdcall, importc, dynlib: ws2dll.} -proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, - stringBufSize: int32): cstring {.stdcall, importc, dynlib: ws2dll.} - const MAXIMUM_WAIT_OBJECTS* = 0x00000040 type WOHandleArray* = array[0..MAXIMUM_WAIT_OBJECTS - 1, Handle] PWOHandleArray* = ptr WOHandleArray -{.deprecated: [TWOHandleArray: WOHandleArray].} proc waitForMultipleObjects*(nCount: DWORD, lpHandles: PWOHandleArray, bWaitAll: WINBOOL, dwMilliseconds: DWORD): DWORD{. @@ -626,51 +608,66 @@ const CREATE_ALWAYS* = 2'i32 CREATE_NEW* = 1'i32 OPEN_EXISTING* = 3'i32 + OPEN_ALWAYS* = 4'i32 FILE_BEGIN* = 0'i32 INVALID_SET_FILE_POINTER* = -1'i32 NO_ERROR* = 0'i32 + PAGE_NOACCESS* = 0x01'i32 + PAGE_EXECUTE* = 0x10'i32 + PAGE_EXECUTE_READ* = 0x20'i32 + PAGE_EXECUTE_READWRITE* = 0x40'i32 PAGE_READONLY* = 2'i32 PAGE_READWRITE* = 4'i32 FILE_MAP_READ* = 4'i32 FILE_MAP_WRITE* = 2'i32 INVALID_FILE_SIZE* = -1'i32 - FILE_FLAG_BACKUP_SEMANTICS* = 33554432'i32 - FILE_FLAG_OPEN_REPARSE_POINT* = 0x00200000'i32 DUPLICATE_SAME_ACCESS* = 2 FILE_READ_DATA* = 0x00000001 # file & pipe FILE_WRITE_DATA* = 0x00000002 # file & pipe # Error Constants const + ERROR_FILE_NOT_FOUND* = 2 ## https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499- + ERROR_PATH_NOT_FOUND* = 3 ERROR_ACCESS_DENIED* = 5 + ERROR_NO_MORE_FILES* = 18 + ERROR_LOCK_VIOLATION* = 33 ERROR_HANDLE_EOF* = 38 + ERROR_FILE_EXISTS* = 80 + ERROR_BAD_ARGUMENTS* = 165 -proc duplicateHandle*(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, - hTargetProcessHandle: HANDLE, - lpTargetHandle: ptr HANDLE, +proc duplicateHandle*(hSourceProcessHandle: Handle, hSourceHandle: Handle, + hTargetProcessHandle: Handle, + lpTargetHandle: ptr Handle, dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwOptions: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "DuplicateHandle".} -proc getCurrentProcess*(): HANDLE{.stdcall, dynlib: "kernel32", + +proc getHandleInformation*(hObject: Handle, lpdwFlags: ptr DWORD): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetHandleInformation".} + +proc setHandleInformation*(hObject: Handle, dwMask: DWORD, + dwFlags: DWORD): WINBOOL {.stdcall, + dynlib: "kernel32", importc: "SetHandleInformation".} + +proc getCurrentProcess*(): Handle{.stdcall, dynlib: "kernel32", importc: "GetCurrentProcess".} -when useWinUnicode: - proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD, - lpSecurityAttributes: pointer, - dwCreationDisposition, dwFlagsAndAttributes: DWORD, - hTemplateFile: Handle): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileW".} - proc deleteFileW*(pathName: WideCString): int32 {. - importc: "DeleteFileW", dynlib: "kernel32", stdcall.} -else: - proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD, - lpSecurityAttributes: pointer, - dwCreationDisposition, dwFlagsAndAttributes: DWORD, - hTemplateFile: THANDLE): THANDLE {. - stdcall, dynlib: "kernel32", importc: "CreateFileA".} - proc deleteFileA*(pathName: cstring): int32 {. - importc: "DeleteFileA", dynlib: "kernel32", stdcall.} +proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD, + lpSecurityAttributes: pointer, + dwCreationDisposition, dwFlagsAndAttributes: DWORD, + hTemplateFile: Handle): Handle {. + stdcall, dynlib: "kernel32", importc: "CreateFileW".} +proc deleteFileW*(pathName: WideCString): int32 {. + importc: "DeleteFileW", dynlib: "kernel32", stdcall.} +proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD, + lpSecurityAttributes: pointer, + dwCreationDisposition, dwFlagsAndAttributes: DWORD, + hTemplateFile: Handle): Handle {. + stdcall, dynlib: "kernel32", importc: "CreateFileA".} +proc deleteFileA*(pathName: cstring): int32 {. + importc: "DeleteFileA", dynlib: "kernel32", stdcall.} proc setEndOfFile*(hFile: Handle): WINBOOL {.stdcall, dynlib: "kernel32", importc: "SetEndOfFile".} @@ -683,9 +680,16 @@ proc setFilePointer*(hFile: Handle, lDistanceToMove: LONG, proc getFileSize*(hFile: Handle, lpFileSizeHigh: ptr DWORD): DWORD{.stdcall, dynlib: "kernel32", importc: "GetFileSize".} +when defined(cpu32): + type + WinSizeT* = uint32 +else: + type + WinSizeT* = uint64 + proc mapViewOfFileEx*(hFileMappingObject: Handle, dwDesiredAccess: DWORD, dwFileOffsetHigh, dwFileOffsetLow: DWORD, - dwNumberOfBytesToMap: DWORD, + dwNumberOfBytesToMap: WinSizeT, lpBaseAddress: pointer): pointer{. stdcall, dynlib: "kernel32", importc: "MapViewOfFileEx".} @@ -696,16 +700,12 @@ proc createFileMappingW*(hFile: Handle, lpName: pointer): Handle {. stdcall, dynlib: "kernel32", importc: "CreateFileMappingW".} -when not useWinUnicode: - proc createFileMappingA*(hFile: THANDLE, - lpFileMappingAttributes: pointer, - flProtect, dwMaximumSizeHigh: DWORD, - dwMaximumSizeLow: DWORD, lpName: cstring): THANDLE {. - stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".} - proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, dynlib: "kernel32", importc: "UnmapViewOfFile".} +proc flushViewOfFile*(lpBaseAddress: pointer, dwNumberOfBytesToFlush: DWORD): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "FlushViewOfFile".} + type OVERLAPPED* {.pure, inheritable.} = object internal*: PULONG @@ -719,51 +719,63 @@ type POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD, para3: POVERLAPPED){.stdcall.} - GUID* {.final, pure.} = object + GUID* = object D1*: int32 D2*: int16 D3*: int16 - D4*: array [0..7, int8] -{.deprecated: [TOVERLAPPED: OVERLAPPED, TGUID: GUID].} + D4*: array[0..7, int8] const ERROR_IO_PENDING* = 997 # a.k.a WSA_IO_PENDING - FILE_FLAG_OVERLAPPED* = 1073741824 WSAECONNABORTED* = 10053 WSAEADDRINUSE* = 10048 WSAECONNRESET* = 10054 WSAEDISCON* = 10101 WSAENETRESET* = 10052 WSAETIMEDOUT* = 10060 + WSANOTINITIALISED* = 10093 + WSAENOTSOCK* = 10038 + WSAEINPROGRESS* = 10036 + WSAEINTR* = 10004 + WSAEWOULDBLOCK* = 10035 + WSAESHUTDOWN* = 10058 ERROR_NETNAME_DELETED* = 64 + STATUS_PENDING* = 0x103 proc createIoCompletionPort*(FileHandle: Handle, ExistingCompletionPort: Handle, - CompletionKey: DWORD, + CompletionKey: ULONG_PTR, NumberOfConcurrentThreads: DWORD): Handle{.stdcall, dynlib: "kernel32", importc: "CreateIoCompletionPort".} proc getQueuedCompletionStatus*(CompletionPort: Handle, - lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG, + lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG_PTR, lpOverlapped: ptr POVERLAPPED, dwMilliseconds: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} -proc getOverlappedResult*(hFile: Handle, lpOverlapped: OVERLAPPED, +proc getOverlappedResult*(hFile: Handle, lpOverlapped: POVERLAPPED, lpNumberOfBytesTransferred: var DWORD, bWait: WINBOOL): WINBOOL{. stdcall, dynlib: "kernel32", importc: "GetOverlappedResult".} +# this is copy of HasOverlappedIoCompleted() macro from <winbase.h> +# because we have declared own OVERLAPPED structure with member names not +# compatible with original names. +template hasOverlappedIoCompleted*(lpOverlapped): bool = + (cast[uint](lpOverlapped.internal) != STATUS_PENDING) + const - IOC_OUT* = 0x40000000 - IOC_IN* = 0x80000000 - IOC_WS2* = 0x08000000 + IOC_OUT* = 0x40000000'i32 + IOC_IN* = 0x80000000'i32 + IOC_WS2* = 0x08000000'i32 IOC_INOUT* = IOC_IN or IOC_OUT -template WSAIORW*(x,y): expr = (IOC_INOUT or x or y) +template WSAIORW*(x,y): untyped = (IOC_INOUT or x or y) const SIO_GET_EXTENSION_FUNCTION_POINTER* = WSAIORW(IOC_WS2,6).DWORD SO_UPDATE_ACCEPT_CONTEXT* = 0x700B AI_V4MAPPED* = 0x0008 + AF_UNSPEC* = 0 AF_INET* = 2 AF_INET6* = 23 @@ -791,11 +803,23 @@ proc WSARecv*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSARecv", dynlib: "Ws2_32.dll".} +proc WSARecvFrom*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesReceived: PDWORD, flags: PDWORD, name: ptr SockAddr, + namelen: ptr cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSARecvFrom", dynlib: "Ws2_32.dll".} + proc WSASend*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, bytesSent: PDWORD, flags: DWORD, lpOverlapped: POVERLAPPED, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".} +proc WSASendTo*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesSent: PDWORD, flags: DWORD, name: ptr SockAddr, + namelen: cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSASendTo", dynlib: "Ws2_32.dll".} + proc get_osfhandle*(fd:FileHandle): Handle {. importc: "_get_osfhandle", header:"<io.h>".} @@ -806,3 +830,244 @@ proc getSystemTimes*(lpIdleTime, lpKernelTime, proc getProcessTimes*(hProcess: Handle; lpCreationTime, lpExitTime, lpKernelTime, lpUserTime: var FILETIME): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetProcessTimes".} + +proc getSystemTimePreciseAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. + importc: "GetSystemTimePreciseAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} + +type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, + stringBufSize: int32): cstring {.gcsafe, stdcall, tags: [].} + +var inet_ntop_real: inet_ntop_proc = nil + +let ws2 = loadLib(ws2dll) +if ws2 != nil: + inet_ntop_real = cast[inet_ntop_proc](symAddr(ws2, "inet_ntop")) + +proc WSAAddressToStringA(pAddr: ptr SockAddr, addrSize: DWORD, unused: pointer, pBuff: cstring, pBuffSize: ptr DWORD): cint {.stdcall, importc, dynlib: ws2dll.} +proc inet_ntop_emulated(family: cint, paddr: pointer, pStringBuffer: cstring, + stringBufSize: int32): cstring {.stdcall.} = + case family + of AF_INET: + var sa: Sockaddr_in + sa.sin_family = AF_INET + sa.sin_addr = cast[ptr InAddr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: + result = nil + else: + result = pStringBuffer + of AF_INET6: + var sa: Sockaddr_in6 + sa.sin6_family = AF_INET6 + sa.sin6_addr = cast[ptr In6_addr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: + result = nil + else: + result = pStringBuffer + else: + setLastError(ERROR_BAD_ARGUMENTS) + result = nil + +proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, + stringBufSize: int32): cstring {.stdcall.} = + var ver: OSVERSIONINFO + ver.dwOSVersionInfoSize = sizeof(ver).DWORD + let res = getVersionExW(ver.addr) + if res == 0: + result = nil + elif ver.dwMajorVersion >= 6: + if inet_ntop_real == nil: + quit("Can't load inet_ntop proc from " & ws2dll) + result = inet_ntop_real(family, paddr, pStringBuffer, stringBufSize) + else: + result = inet_ntop_emulated(family, paddr, pStringBuffer, stringBufSize) + +type + WSAPROC_ACCEPTEX* = proc (sListenSocket: SocketHandle, + sAcceptSocket: SocketHandle, + lpOutputBuffer: pointer, dwReceiveDataLength: DWORD, + dwLocalAddressLength: DWORD, + dwRemoteAddressLength: DWORD, + lpdwBytesReceived: ptr DWORD, + lpOverlapped: POVERLAPPED): bool {. + stdcall, gcsafe, raises: [].} + + WSAPROC_CONNECTEX* = proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: DWORD, + lpdwBytesSent: ptr DWORD, + lpOverlapped: POVERLAPPED): bool {. + stdcall, gcsafe, raises: [].} + + WSAPROC_GETACCEPTEXSOCKADDRS* = proc(lpOutputBuffer: pointer, + dwReceiveDataLength: DWORD, + dwLocalAddressLength: DWORD, + dwRemoteAddressLength: DWORD, + LocalSockaddr: ptr PSockAddr, + LocalSockaddrLength: ptr cint, + RemoteSockaddr: ptr PSockAddr, + RemoteSockaddrLength: ptr cint) {. + stdcall, gcsafe, raises: [].} + +const + WT_EXECUTEDEFAULT* = 0x00000000'i32 + WT_EXECUTEINIOTHREAD* = 0x00000001'i32 + WT_EXECUTEINUITHREAD* = 0x00000002'i32 + WT_EXECUTEINWAITTHREAD* = 0x00000004'i32 + WT_EXECUTEONLYONCE* = 0x00000008'i32 + WT_EXECUTELONGFUNCTION* = 0x00000010'i32 + WT_EXECUTEINTIMERTHREAD* = 0x00000020'i32 + WT_EXECUTEINPERSISTENTIOTHREAD* = 0x00000040'i32 + WT_EXECUTEINPERSISTENTTHREAD* = 0x00000080'i32 + WT_TRANSFER_IMPERSONATION* = 0x00000100'i32 + PROCESS_TERMINATE* = 0x00000001'i32 + PROCESS_CREATE_THREAD* = 0x00000002'i32 + PROCESS_SET_SESSIONID* = 0x00000004'i32 + PROCESS_VM_OPERATION* = 0x00000008'i32 + PROCESS_VM_READ* = 0x00000010'i32 + PROCESS_VM_WRITE* = 0x00000020'i32 + PROCESS_DUP_HANDLE* = 0x00000040'i32 + PROCESS_CREATE_PROCESS* = 0x00000080'i32 + PROCESS_SET_QUOTA* = 0x00000100'i32 + PROCESS_SET_INFORMATION* = 0x00000200'i32 + PROCESS_QUERY_INFORMATION* = 0x00000400'i32 + PROCESS_SUSPEND_RESUME* = 0x00000800'i32 + PROCESS_QUERY_LIMITED_INFORMATION* = 0x00001000'i32 + PROCESS_SET_LIMITED_INFORMATION* = 0x00002000'i32 +type + WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32) {.stdcall.} + +proc postQueuedCompletionStatus*(CompletionPort: Handle, + dwNumberOfBytesTransferred: DWORD, + dwCompletionKey: ULONG_PTR, + lpOverlapped: pointer): bool + {.stdcall, dynlib: "kernel32", importc: "PostQueuedCompletionStatus".} + +proc registerWaitForSingleObject*(phNewWaitObject: ptr Handle, hObject: Handle, + Callback: WAITORTIMERCALLBACK, + Context: pointer, + dwMilliseconds: ULONG, + dwFlags: ULONG): bool + {.stdcall, dynlib: "kernel32", importc: "RegisterWaitForSingleObject".} + +proc unregisterWait*(WaitHandle: Handle): DWORD + {.stdcall, dynlib: "kernel32", importc: "UnregisterWait".} + +proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, + dwProcessId: DWORD): Handle + {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} + +proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} + +proc setEvent*(hEvent: Handle): cint + {.stdcall, dynlib: "kernel32", importc: "SetEvent".} + +const + FD_READ* = 0x00000001'i32 + FD_WRITE* = 0x00000002'i32 + FD_OOB* = 0x00000004'i32 + FD_ACCEPT* = 0x00000008'i32 + FD_CONNECT* = 0x00000010'i32 + FD_CLOSE* = 0x00000020'i32 + FD_QQS* = 0x00000040'i32 + FD_GROUP_QQS* = 0x00000080'i32 + FD_ROUTING_INTERFACE_CHANGE* = 0x00000100'i32 + FD_ADDRESS_LIST_CHANGE* = 0x00000200'i32 + FD_ALL_EVENTS* = 0x000003FF'i32 + +proc wsaEventSelect*(s: SocketHandle, hEventObject: Handle, + lNetworkEvents: clong): cint + {.stdcall, importc: "WSAEventSelect", dynlib: "ws2_32.dll".} + +proc wsaCreateEvent*(): Handle + {.stdcall, importc: "WSACreateEvent", dynlib: "ws2_32.dll".} + +proc wsaCloseEvent*(hEvent: Handle): bool + {.stdcall, importc: "WSACloseEvent", dynlib: "ws2_32.dll".} + +proc wsaResetEvent*(hEvent: Handle): bool + {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} + +type + KEY_EVENT_RECORD* = object + eventType*: int16 + bKeyDown*: WINBOOL + wRepeatCount*: int16 + wVirtualKeyCode*: int16 + wVirtualScanCode*: int16 + uChar*: int16 + dwControlKeyState*: DWORD + +proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} + +type + LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} + +const + FIBER_FLAG_FLOAT_SWITCH* = 0x01 + +proc CreateFiber*(stackSize: int, fn: LPFIBER_START_ROUTINE, param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} +proc CreateFiberEx*(stkCommit: int, stkReserve: int, flags: int32, fn: LPFIBER_START_ROUTINE, param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} +proc ConvertThreadToFiber*(param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} +proc ConvertThreadToFiberEx*(param: pointer, flags: int32): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} +proc DeleteFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} +proc SwitchToFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} +proc GetCurrentFiber*(): pointer {.stdcall, importc, header: "windows.h".} + +proc toFILETIME*(t: int64): FILETIME = + ## Convert the Windows file time timestamp `t` to `FILETIME`. + result = FILETIME(dwLowDateTime: cast[DWORD](t), dwHighDateTime: DWORD(t shr 32)) + +type + LPFILETIME* = ptr FILETIME + +proc setFileTime*(hFile: Handle, lpCreationTime: LPFILETIME, + lpLastAccessTime: LPFILETIME, lpLastWriteTime: LPFILETIME): WINBOOL + {.stdcall, dynlib: "kernel32", importc: "SetFileTime".} + +type + # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority + SID_IDENTIFIER_AUTHORITY* {.importc, header: "<windows.h>".} = object + value* {.importc: "Value".}: array[6, BYTE] + # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid + SID* {.importc, header: "<windows.h>".} = object + Revision: BYTE + SubAuthorityCount: BYTE + IdentifierAuthority: SID_IDENTIFIER_AUTHORITY + SubAuthority: ptr ptr DWORD + PSID* = ptr SID + +const + # https://docs.microsoft.com/en-us/windows/win32/secauthz/sid-components + # https://github.com/mirror/mingw-w64/blob/84c950bdab7c999ace49fe8383856be77f88c4a8/mingw-w64-headers/include/winnt.h#L2994 + SECURITY_NT_AUTHORITY* = [BYTE(0), BYTE(0), BYTE(0), BYTE(0), BYTE(0), BYTE(5)] + SECURITY_BUILTIN_DOMAIN_RID* = 32 + DOMAIN_ALIAS_RID_ADMINS* = 544 + +proc allocateAndInitializeSid*(pIdentifierAuthority: ptr SID_IDENTIFIER_AUTHORITY, + nSubAuthorityCount: BYTE, + nSubAuthority0: DWORD, + nSubAuthority1: DWORD, + nSubAuthority2: DWORD, + nSubAuthority3: DWORD, + nSubAuthority4: DWORD, + nSubAuthority5: DWORD, + nSubAuthority6: DWORD, + nSubAuthority7: DWORD, + pSid: ptr PSID): WINBOOL + {.stdcall, dynlib: "Advapi32", importc: "AllocateAndInitializeSid".} +proc checkTokenMembership*(tokenHandle: Handle, sidToCheck: PSID, + isMember: PBOOL): WINBOOL + {.stdcall, dynlib: "Advapi32", importc: "CheckTokenMembership".} +proc freeSid*(pSid: PSID): PSID + {.stdcall, dynlib: "Advapi32", importc: "FreeSid".} + +when defined(nimHasStyleChecks): + {.pop.} # {.push styleChecks: off.} diff --git a/lib/wrappers/iup.nim b/lib/wrappers/iup.nim deleted file mode 100644 index d910173ca..000000000 --- a/lib/wrappers/iup.nim +++ /dev/null @@ -1,957 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# C header files translated by hand -# Licence of IUP follows: - - -# **************************************************************************** -# Copyright (C) 1994-2009 Tecgraf, PUC-Rio. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# **************************************************************************** - -{.deadCodeElim: on.} - -when defined(windows): - const dllname = "iup(|30|27|26|25|24).dll" -elif defined(macosx): - const dllname = "libiup(|3.0|2.7|2.6|2.5|2.4).dylib" -else: - const dllname = "libiup(|3.0|2.7|2.6|2.5|2.4).so(|.1)" - -const - IUP_NAME* = "IUP - Portable User Interface" - IUP_COPYRIGHT* = "Copyright (C) 1994-2009 Tecgraf, PUC-Rio." - IUP_DESCRIPTION* = "Portable toolkit for building graphical user interfaces." - constIUP_VERSION* = "3.0" - constIUP_VERSION_NUMBER* = 300000 - constIUP_VERSION_DATE* = "2009/07/18" - -type - Ihandle = object - PIhandle* = ptr Ihandle - - Icallback* = proc (arg: PIhandle): cint {.cdecl.} - -# pre-defineded dialogs -proc fileDlg*: PIhandle {.importc: "IupFileDlg", dynlib: dllname, cdecl.} -proc messageDlg*: PIhandle {.importc: "IupMessageDlg", dynlib: dllname, cdecl.} -proc colorDlg*: PIhandle {.importc: "IupColorDlg", dynlib: dllname, cdecl.} -proc fontDlg*: PIhandle {.importc: "IupFontDlg", dynlib: dllname, cdecl.} - -proc getFile*(arq: cstring): cint {. - importc: "IupGetFile", dynlib: dllname, cdecl.} -proc message*(title, msg: cstring) {. - importc: "IupMessage", dynlib: dllname, cdecl.} -proc messagef*(title, format: cstring) {. - importc: "IupMessagef", dynlib: dllname, cdecl, varargs.} -proc alarm*(title, msg, b1, b2, b3: cstring): cint {. - importc: "IupAlarm", dynlib: dllname, cdecl.} -proc scanf*(format: cstring): cint {. - importc: "IupScanf", dynlib: dllname, cdecl, varargs.} -proc listDialog*(theType: cint, title: cstring, size: cint, - list: cstringArray, op, maxCol, maxLin: cint, - marks: ptr cint): cint {. - importc: "IupListDialog", dynlib: dllname, cdecl.} -proc getText*(title, text: cstring): cint {. - importc: "IupGetText", dynlib: dllname, cdecl.} -proc getColor*(x, y: cint, r, g, b: var byte): cint {. - importc: "IupGetColor", dynlib: dllname, cdecl.} - -type - Iparamcb* = proc (dialog: PIhandle, paramIndex: cint, - userData: pointer): cint {.cdecl.} - -proc getParam*(title: cstring, action: Iparamcb, userData: pointer, - format: cstring): cint {. - importc: "IupGetParam", cdecl, varargs, dynlib: dllname.} -proc getParamv*(title: cstring, action: Iparamcb, userData: pointer, - format: cstring, paramCount, paramExtra: cint, - paramData: pointer): cint {. - importc: "IupGetParamv", cdecl, dynlib: dllname.} - - -# Functions - -proc open*(argc: ptr cint, argv: ptr cstringArray): cint {. - importc: "IupOpen", cdecl, dynlib: dllname.} -proc close*() {.importc: "IupClose", cdecl, dynlib: dllname.} -proc imageLibOpen*() {.importc: "IupImageLibOpen", cdecl, dynlib: dllname.} - -proc mainLoop*(): cint {.importc: "IupMainLoop", cdecl, dynlib: dllname, - discardable.} -proc loopStep*(): cint {.importc: "IupLoopStep", cdecl, dynlib: dllname, - discardable.} -proc mainLoopLevel*(): cint {.importc: "IupMainLoopLevel", cdecl, - dynlib: dllname, discardable.} -proc flush*() {.importc: "IupFlush", cdecl, dynlib: dllname.} -proc exitLoop*() {.importc: "IupExitLoop", cdecl, dynlib: dllname.} - -proc update*(ih: PIhandle) {.importc: "IupUpdate", cdecl, dynlib: dllname.} -proc updateChildren*(ih: PIhandle) {.importc: "IupUpdateChildren", cdecl, dynlib: dllname.} -proc redraw*(ih: PIhandle, children: cint) {.importc: "IupRedraw", cdecl, dynlib: dllname.} -proc refresh*(ih: PIhandle) {.importc: "IupRefresh", cdecl, dynlib: dllname.} - -proc mapFont*(iupfont: cstring): cstring {.importc: "IupMapFont", cdecl, dynlib: dllname.} -proc unMapFont*(driverfont: cstring): cstring {.importc: "IupUnMapFont", cdecl, dynlib: dllname.} -proc help*(url: cstring): cint {.importc: "IupHelp", cdecl, dynlib: dllname.} -proc load*(filename: cstring): cstring {.importc: "IupLoad", cdecl, dynlib: dllname.} - -proc iupVersion*(): cstring {.importc: "IupVersion", cdecl, dynlib: dllname.} -proc iupVersionDate*(): cstring {.importc: "IupVersionDate", cdecl, dynlib: dllname.} -proc iupVersionNumber*(): cint {.importc: "IupVersionNumber", cdecl, dynlib: dllname.} -proc setLanguage*(lng: cstring) {.importc: "IupSetLanguage", cdecl, dynlib: dllname.} -proc getLanguage*(): cstring {.importc: "IupGetLanguage", cdecl, dynlib: dllname.} - -proc destroy*(ih: PIhandle) {.importc: "IupDestroy", cdecl, dynlib: dllname.} -proc detach*(child: PIhandle) {.importc: "IupDetach", cdecl, dynlib: dllname.} -proc append*(ih, child: PIhandle): PIhandle {. - importc: "IupAppend", cdecl, dynlib: dllname, discardable.} -proc insert*(ih, refChild, child: PIhandle): PIhandle {. - importc: "IupInsert", cdecl, dynlib: dllname, discardable.} -proc getChild*(ih: PIhandle, pos: cint): PIhandle {. - importc: "IupGetChild", cdecl, dynlib: dllname.} -proc getChildPos*(ih, child: PIhandle): cint {. - importc: "IupGetChildPos", cdecl, dynlib: dllname.} -proc getChildCount*(ih: PIhandle): cint {. - importc: "IupGetChildCount", cdecl, dynlib: dllname.} -proc getNextChild*(ih, child: PIhandle): PIhandle {. - importc: "IupGetNextChild", cdecl, dynlib: dllname.} -proc getBrother*(ih: PIhandle): PIhandle {. - importc: "IupGetBrother", cdecl, dynlib: dllname.} -proc getParent*(ih: PIhandle): PIhandle {. - importc: "IupGetParent", cdecl, dynlib: dllname.} -proc getDialog*(ih: PIhandle): PIhandle {. - importc: "IupGetDialog", cdecl, dynlib: dllname.} -proc getDialogChild*(ih: PIhandle, name: cstring): PIhandle {. - importc: "IupGetDialogChild", cdecl, dynlib: dllname.} -proc reparent*(ih, newParent: PIhandle): cint {. - importc: "IupReparent", cdecl, dynlib: dllname.} - -proc popup*(ih: PIhandle, x, y: cint): cint {. - importc: "IupPopup", cdecl, dynlib: dllname, discardable.} -proc show*(ih: PIhandle): cint {. - importc: "IupShow", cdecl, dynlib: dllname, discardable.} -proc showXY*(ih: PIhandle, x, y: cint): cint {. - importc: "IupShowXY", cdecl, dynlib: dllname, discardable.} -proc hide*(ih: PIhandle): cint {. - importc: "IupHide", cdecl, dynlib: dllname, discardable.} -proc map*(ih: PIhandle): cint {. - importc: "IupMap", cdecl, dynlib: dllname, discardable.} -proc unmap*(ih: PIhandle) {. - importc: "IupUnmap", cdecl, dynlib: dllname, discardable.} - -proc setAttribute*(ih: PIhandle, name, value: cstring) {. - importc: "IupSetAttribute", cdecl, dynlib: dllname.} -proc storeAttribute*(ih: PIhandle, name, value: cstring) {. - importc: "IupStoreAttribute", cdecl, dynlib: dllname.} -proc setAttributes*(ih: PIhandle, str: cstring): PIhandle {. - importc: "IupSetAttributes", cdecl, dynlib: dllname.} -proc getAttribute*(ih: PIhandle, name: cstring): cstring {. - importc: "IupGetAttribute", cdecl, dynlib: dllname.} -proc getAttributes*(ih: PIhandle): cstring {. - importc: "IupGetAttributes", cdecl, dynlib: dllname.} -proc getInt*(ih: PIhandle, name: cstring): cint {. - importc: "IupGetInt", cdecl, dynlib: dllname.} -proc getInt2*(ih: PIhandle, name: cstring): cint {. - importc: "IupGetInt2", cdecl, dynlib: dllname.} -proc getIntInt*(ih: PIhandle, name: cstring, i1, i2: var cint): cint {. - importc: "IupGetIntInt", cdecl, dynlib: dllname.} -proc getFloat*(ih: PIhandle, name: cstring): cfloat {. - importc: "IupGetFloat", cdecl, dynlib: dllname.} -proc setfAttribute*(ih: PIhandle, name, format: cstring) {. - importc: "IupSetfAttribute", cdecl, dynlib: dllname, varargs.} -proc getAllAttributes*(ih: PIhandle, names: cstringArray, n: cint): cint {. - importc: "IupGetAllAttributes", cdecl, dynlib: dllname.} -proc setAtt*(handleName: cstring, ih: PIhandle, name: cstring): PIhandle {. - importc: "IupSetAtt", cdecl, dynlib: dllname, varargs, discardable.} - -proc setGlobal*(name, value: cstring) {. - importc: "IupSetGlobal", cdecl, dynlib: dllname.} -proc storeGlobal*(name, value: cstring) {. - importc: "IupStoreGlobal", cdecl, dynlib: dllname.} -proc getGlobal*(name: cstring): cstring {. - importc: "IupGetGlobal", cdecl, dynlib: dllname.} - -proc setFocus*(ih: PIhandle): PIhandle {. - importc: "IupSetFocus", cdecl, dynlib: dllname.} -proc getFocus*(): PIhandle {. - importc: "IupGetFocus", cdecl, dynlib: dllname.} -proc previousField*(ih: PIhandle): PIhandle {. - importc: "IupPreviousField", cdecl, dynlib: dllname.} -proc nextField*(ih: PIhandle): PIhandle {. - importc: "IupNextField", cdecl, dynlib: dllname.} - -proc getCallback*(ih: PIhandle, name: cstring): Icallback {. - importc: "IupGetCallback", cdecl, dynlib: dllname.} -proc setCallback*(ih: PIhandle, name: cstring, fn: Icallback): Icallback {. - importc: "IupSetCallback", cdecl, dynlib: dllname, discardable.} - -proc setCallbacks*(ih: PIhandle, name: cstring, fn: Icallback): PIhandle {. - importc: "IupSetCallbacks", cdecl, dynlib: dllname, varargs, discardable.} - -proc getFunction*(name: cstring): Icallback {. - importc: "IupGetFunction", cdecl, dynlib: dllname.} -proc setFunction*(name: cstring, fn: Icallback): Icallback {. - importc: "IupSetFunction", cdecl, dynlib: dllname, discardable.} -proc getActionName*(): cstring {. - importc: "IupGetActionName", cdecl, dynlib: dllname.} - -proc getHandle*(name: cstring): PIhandle {. - importc: "IupGetHandle", cdecl, dynlib: dllname.} -proc setHandle*(name: cstring, ih: PIhandle): PIhandle {. - importc: "IupSetHandle", cdecl, dynlib: dllname.} -proc getAllNames*(names: cstringArray, n: cint): cint {. - importc: "IupGetAllNames", cdecl, dynlib: dllname.} -proc getAllDialogs*(names: cstringArray, n: cint): cint {. - importc: "IupGetAllDialogs", cdecl, dynlib: dllname.} -proc getName*(ih: PIhandle): cstring {. - importc: "IupGetName", cdecl, dynlib: dllname.} - -proc setAttributeHandle*(ih: PIhandle, name: cstring, ihNamed: PIhandle) {. - importc: "IupSetAttributeHandle", cdecl, dynlib: dllname.} -proc getAttributeHandle*(ih: PIhandle, name: cstring): PIhandle {. - importc: "IupGetAttributeHandle", cdecl, dynlib: dllname.} - -proc getClassName*(ih: PIhandle): cstring {. - importc: "IupGetClassName", cdecl, dynlib: dllname.} -proc getClassType*(ih: PIhandle): cstring {. - importc: "IupGetClassType", cdecl, dynlib: dllname.} -proc getClassAttributes*(classname: cstring, names: cstringArray, - n: cint): cint {. - importc: "IupGetClassAttributes", cdecl, dynlib: dllname.} -proc saveClassAttributes*(ih: PIhandle) {. - importc: "IupSaveClassAttributes", cdecl, dynlib: dllname.} -proc setClassDefaultAttribute*(classname, name, value: cstring) {. - importc: "IupSetClassDefaultAttribute", cdecl, dynlib: dllname.} - -proc create*(classname: cstring): PIhandle {. - importc: "IupCreate", cdecl, dynlib: dllname.} -proc createv*(classname: cstring, params: pointer): PIhandle {. - importc: "IupCreatev", cdecl, dynlib: dllname.} -proc createp*(classname: cstring, first: pointer): PIhandle {. - importc: "IupCreatep", cdecl, dynlib: dllname, varargs.} - -proc fill*(): PIhandle {.importc: "IupFill", cdecl, dynlib: dllname.} -proc radio*(child: PIhandle): PIhandle {. - importc: "IupRadio", cdecl, dynlib: dllname.} -proc vbox*(child: PIhandle): PIhandle {. - importc: "IupVbox", cdecl, dynlib: dllname, varargs.} -proc vboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupVboxv", cdecl, dynlib: dllname.} -proc zbox*(child: PIhandle): PIhandle {. - importc: "IupZbox", cdecl, dynlib: dllname, varargs.} -proc zboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupZboxv", cdecl, dynlib: dllname.} -proc hbox*(child: PIhandle): PIhandle {. - importc: "IupHbox", cdecl, dynlib: dllname, varargs.} -proc hboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupHboxv", cdecl, dynlib: dllname.} - -proc normalizer*(ihFirst: PIhandle): PIhandle {. - importc: "IupNormalizer", cdecl, dynlib: dllname, varargs.} -proc normalizerv*(ihList: ptr PIhandle): PIhandle {. - importc: "IupNormalizerv", cdecl, dynlib: dllname.} - -proc cbox*(child: PIhandle): PIhandle {. - importc: "IupCbox", cdecl, dynlib: dllname, varargs.} -proc cboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupCboxv", cdecl, dynlib: dllname.} -proc sbox*(child: PIhandle): PIhandle {. - importc: "IupSbox", cdecl, dynlib: dllname.} - -proc frame*(child: PIhandle): PIhandle {. - importc: "IupFrame", cdecl, dynlib: dllname.} - -proc image*(width, height: cint, pixmap: pointer): PIhandle {. - importc: "IupImage", cdecl, dynlib: dllname.} -proc imageRGB*(width, height: cint, pixmap: pointer): PIhandle {. - importc: "IupImageRGB", cdecl, dynlib: dllname.} -proc imageRGBA*(width, height: cint, pixmap: pointer): PIhandle {. - importc: "IupImageRGBA", cdecl, dynlib: dllname.} - -proc item*(title, action: cstring): PIhandle {. - importc: "IupItem", cdecl, dynlib: dllname.} -proc submenu*(title: cstring, child: PIhandle): PIhandle {. - importc: "IupSubmenu", cdecl, dynlib: dllname.} -proc separator*(): PIhandle {. - importc: "IupSeparator", cdecl, dynlib: dllname.} -proc menu*(child: PIhandle): PIhandle {. - importc: "IupMenu", cdecl, dynlib: dllname, varargs.} -proc menuv*(children: ptr PIhandle): PIhandle {. - importc: "IupMenuv", cdecl, dynlib: dllname.} - -proc button*(title, action: cstring): PIhandle {. - importc: "IupButton", cdecl, dynlib: dllname.} -proc link*(url, title: cstring): PIhandle {. - importc: "IupLink", cdecl, dynlib: dllname.} -proc canvas*(action: cstring): PIhandle {. - importc: "IupCanvas", cdecl, dynlib: dllname.} -proc dialog*(child: PIhandle): PIhandle {. - importc: "IupDialog", cdecl, dynlib: dllname.} -proc user*(): PIhandle {. - importc: "IupUser", cdecl, dynlib: dllname.} -proc label*(title: cstring): PIhandle {. - importc: "IupLabel", cdecl, dynlib: dllname.} -proc list*(action: cstring): PIhandle {. - importc: "IupList", cdecl, dynlib: dllname.} -proc text*(action: cstring): PIhandle {. - importc: "IupText", cdecl, dynlib: dllname.} -proc multiLine*(action: cstring): PIhandle {. - importc: "IupMultiLine", cdecl, dynlib: dllname.} -proc toggle*(title, action: cstring): PIhandle {. - importc: "IupToggle", cdecl, dynlib: dllname.} -proc timer*(): PIhandle {. - importc: "IupTimer", cdecl, dynlib: dllname.} -proc progressBar*(): PIhandle {. - importc: "IupProgressBar", cdecl, dynlib: dllname.} -proc val*(theType: cstring): PIhandle {. - importc: "IupVal", cdecl, dynlib: dllname.} -proc tabs*(child: PIhandle): PIhandle {. - importc: "IupTabs", cdecl, dynlib: dllname, varargs.} -proc tabsv*(children: ptr PIhandle): PIhandle {. - importc: "IupTabsv", cdecl, dynlib: dllname.} -proc tree*(): PIhandle {.importc: "IupTree", cdecl, dynlib: dllname.} - -proc spin*(): PIhandle {.importc: "IupSpin", cdecl, dynlib: dllname.} -proc spinbox*(child: PIhandle): PIhandle {. - importc: "IupSpinbox", cdecl, dynlib: dllname.} - -# IupText utilities -proc textConvertLinColToPos*(ih: PIhandle, lin, col: cint, pos: var cint) {. - importc: "IupTextConvertLinColToPos", cdecl, dynlib: dllname.} -proc textConvertPosToLinCol*(ih: PIhandle, pos: cint, lin, col: var cint) {. - importc: "IupTextConvertPosToLinCol", cdecl, dynlib: dllname.} - -proc convertXYToPos*(ih: PIhandle, x, y: cint): cint {. - importc: "IupConvertXYToPos", cdecl, dynlib: dllname.} - -# IupTree utilities -proc treeSetUserId*(ih: PIhandle, id: cint, userid: pointer): cint {. - importc: "IupTreeSetUserId", cdecl, dynlib: dllname, discardable.} -proc treeGetUserId*(ih: PIhandle, id: cint): pointer {. - importc: "IupTreeGetUserId", cdecl, dynlib: dllname.} -proc treeGetId*(ih: PIhandle, userid: pointer): cint {. - importc: "IupTreeGetId", cdecl, dynlib: dllname.} - -proc treeSetAttribute*(ih: PIhandle, name: cstring, id: cint, value: cstring) {. - importc: "IupTreeSetAttribute", cdecl, dynlib: dllname.} -proc treeStoreAttribute*(ih: PIhandle, name: cstring, id: cint, value: cstring) {. - importc: "IupTreeStoreAttribute", cdecl, dynlib: dllname.} -proc treeGetAttribute*(ih: PIhandle, name: cstring, id: cint): cstring {. - importc: "IupTreeGetAttribute", cdecl, dynlib: dllname.} -proc treeGetInt*(ih: PIhandle, name: cstring, id: cint): cint {. - importc: "IupTreeGetInt", cdecl, dynlib: dllname.} -proc treeGetFloat*(ih: PIhandle, name: cstring, id: cint): cfloat {. - importc: "IupTreeGetFloat", cdecl, dynlib: dllname.} -proc treeSetfAttribute*(ih: PIhandle, name: cstring, id: cint, format: cstring) {. - importc: "IupTreeSetfAttribute", cdecl, dynlib: dllname, varargs.} - - -# Common Return Values -const - IUP_ERROR* = cint(1) - IUP_NOERROR* = cint(0) - IUP_OPENED* = cint(-1) - IUP_INVALID* = cint(-1) - - # Callback Return Values - IUP_IGNORE* = cint(-1) - IUP_DEFAULT* = cint(-2) - IUP_CLOSE* = cint(-3) - IUP_CONTINUE* = cint(-4) - - # IupPopup and IupShowXY Parameter Values - IUP_CENTER* = cint(0xFFFF) - IUP_LEFT* = cint(0xFFFE) - IUP_RIGHT* = cint(0xFFFD) - IUP_MOUSEPOS* = cint(0xFFFC) - IUP_CURRENT* = cint(0xFFFB) - IUP_CENTERPARENT* = cint(0xFFFA) - IUP_TOP* = IUP_LEFT - IUP_BOTTOM* = IUP_RIGHT - - # SHOW_CB Callback Values - IUP_SHOW* = cint(0) - IUP_RESTORE* = cint(1) - IUP_MINIMIZE* = cint(2) - IUP_MAXIMIZE* = cint(3) - IUP_HIDE* = cint(4) - - # SCROLL_CB Callback Values - IUP_SBUP* = cint(0) - IUP_SBDN* = cint(1) - IUP_SBPGUP* = cint(2) - IUP_SBPGDN* = cint(3) - IUP_SBPOSV* = cint(4) - IUP_SBDRAGV* = cint(5) - IUP_SBLEFT* = cint(6) - IUP_SBRIGHT* = cint(7) - IUP_SBPGLEFT* = cint(8) - IUP_SBPGRIGHT* = cint(9) - IUP_SBPOSH* = cint(10) - IUP_SBDRAGH* = cint(11) - - # Mouse Button Values and Macros - IUP_BUTTON1* = cint(ord('1')) - IUP_BUTTON2* = cint(ord('2')) - IUP_BUTTON3* = cint(ord('3')) - IUP_BUTTON4* = cint(ord('4')) - IUP_BUTTON5* = cint(ord('5')) - -proc isShift*(s: cstring): bool = return s[0] == 'S' -proc isControl*(s: cstring): bool = return s[1] == 'C' -proc isButton1*(s: cstring): bool = return s[2] == '1' -proc isButton2*(s: cstring): bool = return s[3] == '2' -proc isbutton3*(s: cstring): bool = return s[4] == '3' -proc isDouble*(s: cstring): bool = return s[5] == 'D' -proc isAlt*(s: cstring): bool = return s[6] == 'A' -proc isSys*(s: cstring): bool = return s[7] == 'Y' -proc isButton4*(s: cstring): bool = return s[8] == '4' -proc isButton5*(s: cstring): bool = return s[9] == '5' - -# Pre-Defined Masks -const - IUP_MASK_FLOAT* = "[+/-]?(/d+/.?/d*|/./d+)" - IUP_MASK_UFLOAT* = "(/d+/.?/d*|/./d+)" - IUP_MASK_EFLOAT* = "[+/-]?(/d+/.?/d*|/./d+)([eE][+/-]?/d+)?" - IUP_MASK_INT* = "[+/-]?/d+" - IUP_MASK_UINT* = "/d+" - -# from 32 to 126, all character sets are equal, -# the key code i the same as the character code. -const - K_SP* = cint(ord(' ')) - K_exclam* = cint(ord('!')) - K_quotedbl* = cint(ord('\"')) - K_numbersign* = cint(ord('#')) - K_dollar* = cint(ord('$')) - K_percent* = cint(ord('%')) - K_ampersand* = cint(ord('&')) - K_apostrophe* = cint(ord('\'')) - K_parentleft* = cint(ord('(')) - K_parentright* = cint(ord(')')) - K_asterisk* = cint(ord('*')) - K_plus* = cint(ord('+')) - K_comma* = cint(ord(',')) - K_minus* = cint(ord('-')) - K_period* = cint(ord('.')) - K_slash* = cint(ord('/')) - K_0* = cint(ord('0')) - K_1* = cint(ord('1')) - K_2* = cint(ord('2')) - K_3* = cint(ord('3')) - K_4* = cint(ord('4')) - K_5* = cint(ord('5')) - K_6* = cint(ord('6')) - K_7* = cint(ord('7')) - K_8* = cint(ord('8')) - K_9* = cint(ord('9')) - K_colon* = cint(ord(':')) - K_semicolon* = cint(ord(';')) - K_less* = cint(ord('<')) - K_equal* = cint(ord('=')) - K_greater* = cint(ord('>')) - K_question* = cint(ord('?')) - K_at* = cint(ord('@')) - K_upperA* = cint(ord('A')) - K_upperB* = cint(ord('B')) - K_upperC* = cint(ord('C')) - K_upperD* = cint(ord('D')) - K_upperE* = cint(ord('E')) - K_upperF* = cint(ord('F')) - K_upperG* = cint(ord('G')) - K_upperH* = cint(ord('H')) - K_upperI* = cint(ord('I')) - K_upperJ* = cint(ord('J')) - K_upperK* = cint(ord('K')) - K_upperL* = cint(ord('L')) - K_upperM* = cint(ord('M')) - K_upperN* = cint(ord('N')) - K_upperO* = cint(ord('O')) - K_upperP* = cint(ord('P')) - K_upperQ* = cint(ord('Q')) - K_upperR* = cint(ord('R')) - K_upperS* = cint(ord('S')) - K_upperT* = cint(ord('T')) - K_upperU* = cint(ord('U')) - K_upperV* = cint(ord('V')) - K_upperW* = cint(ord('W')) - K_upperX* = cint(ord('X')) - K_upperY* = cint(ord('Y')) - K_upperZ* = cint(ord('Z')) - K_bracketleft* = cint(ord('[')) - K_backslash* = cint(ord('\\')) - K_bracketright* = cint(ord(']')) - K_circum* = cint(ord('^')) - K_underscore* = cint(ord('_')) - K_grave* = cint(ord('`')) - K_lowera* = cint(ord('a')) - K_lowerb* = cint(ord('b')) - K_lowerc* = cint(ord('c')) - K_lowerd* = cint(ord('d')) - K_lowere* = cint(ord('e')) - K_lowerf* = cint(ord('f')) - K_lowerg* = cint(ord('g')) - K_lowerh* = cint(ord('h')) - K_loweri* = cint(ord('i')) - K_lowerj* = cint(ord('j')) - K_lowerk* = cint(ord('k')) - K_lowerl* = cint(ord('l')) - K_lowerm* = cint(ord('m')) - K_lowern* = cint(ord('n')) - K_lowero* = cint(ord('o')) - K_lowerp* = cint(ord('p')) - K_lowerq* = cint(ord('q')) - K_lowerr* = cint(ord('r')) - K_lowers* = cint(ord('s')) - K_lowert* = cint(ord('t')) - K_loweru* = cint(ord('u')) - K_lowerv* = cint(ord('v')) - K_lowerw* = cint(ord('w')) - K_lowerx* = cint(ord('x')) - K_lowery* = cint(ord('y')) - K_lowerz* = cint(ord('z')) - K_braceleft* = cint(ord('{')) - K_bar* = cint(ord('|')) - K_braceright* = cint(ord('}')) - K_tilde* = cint(ord('~')) - -proc isPrint*(c: cint): bool = return c > 31 and c < 127 - -# also define the escape sequences that have keys associated -const - K_BS* = cint(ord('\b')) - K_TAB* = cint(ord('\t')) - K_LF* = cint(10) - K_CR* = cint(13) - -# IUP Extended Key Codes, range start at 128 -# Modifiers use 256 interval -# These key code definitions are specific to IUP - -proc isXkey*(c: cint): bool = return c > 128 -proc isShiftXkey*(c: cint): bool = return c > 256 and c < 512 -proc isCtrlXkey*(c: cint): bool = return c > 512 and c < 768 -proc isAltXkey*(c: cint): bool = return c > 768 and c < 1024 -proc isSysXkey*(c: cint): bool = return c > 1024 and c < 1280 - -proc iUPxCODE*(c: cint): cint = return c + cint(128) # Normal (must be above 128) -proc iUPsxCODE*(c: cint): cint = - return c + cint(256) - # Shift (must have range to include the standard keys and the normal - # extended keys, so must be above 256 - -proc iUPcxCODE*(c: cint): cint = return c + cint(512) # Ctrl -proc iUPmxCODE*(c: cint): cint = return c + cint(768) # Alt -proc iUPyxCODE*(c: cint): cint = return c + cint(1024) # Sys (Win or Apple) - -const - IUP_NUMMAXCODES* = 1280 ## 5*256=1280 Normal+Shift+Ctrl+Alt+Sys - - K_HOME* = iUPxCODE(1) - K_UP* = iUPxCODE(2) - K_PGUP* = iUPxCODE(3) - K_LEFT* = iUPxCODE(4) - K_MIDDLE* = iUPxCODE(5) - K_RIGHT* = iUPxCODE(6) - K_END* = iUPxCODE(7) - K_DOWN* = iUPxCODE(8) - K_PGDN* = iUPxCODE(9) - K_INS* = iUPxCODE(10) - K_DEL* = iUPxCODE(11) - K_PAUSE* = iUPxCODE(12) - K_ESC* = iUPxCODE(13) - K_ccedilla* = iUPxCODE(14) - K_F1* = iUPxCODE(15) - K_F2* = iUPxCODE(16) - K_F3* = iUPxCODE(17) - K_F4* = iUPxCODE(18) - K_F5* = iUPxCODE(19) - K_F6* = iUPxCODE(20) - K_F7* = iUPxCODE(21) - K_F8* = iUPxCODE(22) - K_F9* = iUPxCODE(23) - K_F10* = iUPxCODE(24) - K_F11* = iUPxCODE(25) - K_F12* = iUPxCODE(26) - K_Print* = iUPxCODE(27) - K_Menu* = iUPxCODE(28) - - K_acute* = iUPxCODE(29) # no Shift/Ctrl/Alt - - K_sHOME* = iUPsxCODE(K_HOME) - K_sUP* = iUPsxCODE(K_UP) - K_sPGUP* = iUPsxCODE(K_PGUP) - K_sLEFT* = iUPsxCODE(K_LEFT) - K_sMIDDLE* = iUPsxCODE(K_MIDDLE) - K_sRIGHT* = iUPsxCODE(K_RIGHT) - K_sEND* = iUPsxCODE(K_END) - K_sDOWN* = iUPsxCODE(K_DOWN) - K_sPGDN* = iUPsxCODE(K_PGDN) - K_sINS* = iUPsxCODE(K_INS) - K_sDEL* = iUPsxCODE(K_DEL) - K_sSP* = iUPsxCODE(K_SP) - K_sTAB* = iUPsxCODE(K_TAB) - K_sCR* = iUPsxCODE(K_CR) - K_sBS* = iUPsxCODE(K_BS) - K_sPAUSE* = iUPsxCODE(K_PAUSE) - K_sESC* = iUPsxCODE(K_ESC) - K_sCcedilla* = iUPsxCODE(K_ccedilla) - K_sF1* = iUPsxCODE(K_F1) - K_sF2* = iUPsxCODE(K_F2) - K_sF3* = iUPsxCODE(K_F3) - K_sF4* = iUPsxCODE(K_F4) - K_sF5* = iUPsxCODE(K_F5) - K_sF6* = iUPsxCODE(K_F6) - K_sF7* = iUPsxCODE(K_F7) - K_sF8* = iUPsxCODE(K_F8) - K_sF9* = iUPsxCODE(K_F9) - K_sF10* = iUPsxCODE(K_F10) - K_sF11* = iUPsxCODE(K_F11) - K_sF12* = iUPsxCODE(K_F12) - K_sPrint* = iUPsxCODE(K_Print) - K_sMenu* = iUPsxCODE(K_Menu) - - K_cHOME* = iUPcxCODE(K_HOME) - K_cUP* = iUPcxCODE(K_UP) - K_cPGUP* = iUPcxCODE(K_PGUP) - K_cLEFT* = iUPcxCODE(K_LEFT) - K_cMIDDLE* = iUPcxCODE(K_MIDDLE) - K_cRIGHT* = iUPcxCODE(K_RIGHT) - K_cEND* = iUPcxCODE(K_END) - K_cDOWN* = iUPcxCODE(K_DOWN) - K_cPGDN* = iUPcxCODE(K_PGDN) - K_cINS* = iUPcxCODE(K_INS) - K_cDEL* = iUPcxCODE(K_DEL) - K_cSP* = iUPcxCODE(K_SP) - K_cTAB* = iUPcxCODE(K_TAB) - K_cCR* = iUPcxCODE(K_CR) - K_cBS* = iUPcxCODE(K_BS) - K_cPAUSE* = iUPcxCODE(K_PAUSE) - K_cESC* = iUPcxCODE(K_ESC) - K_cCcedilla* = iUPcxCODE(K_ccedilla) - K_cF1* = iUPcxCODE(K_F1) - K_cF2* = iUPcxCODE(K_F2) - K_cF3* = iUPcxCODE(K_F3) - K_cF4* = iUPcxCODE(K_F4) - K_cF5* = iUPcxCODE(K_F5) - K_cF6* = iUPcxCODE(K_F6) - K_cF7* = iUPcxCODE(K_F7) - K_cF8* = iUPcxCODE(K_F8) - K_cF9* = iUPcxCODE(K_F9) - K_cF10* = iUPcxCODE(K_F10) - K_cF11* = iUPcxCODE(K_F11) - K_cF12* = iUPcxCODE(K_F12) - K_cPrint* = iUPcxCODE(K_Print) - K_cMenu* = iUPcxCODE(K_Menu) - - K_mHOME* = iUPmxCODE(K_HOME) - K_mUP* = iUPmxCODE(K_UP) - K_mPGUP* = iUPmxCODE(K_PGUP) - K_mLEFT* = iUPmxCODE(K_LEFT) - K_mMIDDLE* = iUPmxCODE(K_MIDDLE) - K_mRIGHT* = iUPmxCODE(K_RIGHT) - K_mEND* = iUPmxCODE(K_END) - K_mDOWN* = iUPmxCODE(K_DOWN) - K_mPGDN* = iUPmxCODE(K_PGDN) - K_mINS* = iUPmxCODE(K_INS) - K_mDEL* = iUPmxCODE(K_DEL) - K_mSP* = iUPmxCODE(K_SP) - K_mTAB* = iUPmxCODE(K_TAB) - K_mCR* = iUPmxCODE(K_CR) - K_mBS* = iUPmxCODE(K_BS) - K_mPAUSE* = iUPmxCODE(K_PAUSE) - K_mESC* = iUPmxCODE(K_ESC) - K_mCcedilla* = iUPmxCODE(K_ccedilla) - K_mF1* = iUPmxCODE(K_F1) - K_mF2* = iUPmxCODE(K_F2) - K_mF3* = iUPmxCODE(K_F3) - K_mF4* = iUPmxCODE(K_F4) - K_mF5* = iUPmxCODE(K_F5) - K_mF6* = iUPmxCODE(K_F6) - K_mF7* = iUPmxCODE(K_F7) - K_mF8* = iUPmxCODE(K_F8) - K_mF9* = iUPmxCODE(K_F9) - K_mF10* = iUPmxCODE(K_F10) - K_mF11* = iUPmxCODE(K_F11) - K_mF12* = iUPmxCODE(K_F12) - K_mPrint* = iUPmxCODE(K_Print) - K_mMenu* = iUPmxCODE(K_Menu) - - K_yHOME* = iUPyxCODE(K_HOME) - K_yUP* = iUPyxCODE(K_UP) - K_yPGUP* = iUPyxCODE(K_PGUP) - K_yLEFT* = iUPyxCODE(K_LEFT) - K_yMIDDLE* = iUPyxCODE(K_MIDDLE) - K_yRIGHT* = iUPyxCODE(K_RIGHT) - K_yEND* = iUPyxCODE(K_END) - K_yDOWN* = iUPyxCODE(K_DOWN) - K_yPGDN* = iUPyxCODE(K_PGDN) - K_yINS* = iUPyxCODE(K_INS) - K_yDEL* = iUPyxCODE(K_DEL) - K_ySP* = iUPyxCODE(K_SP) - K_yTAB* = iUPyxCODE(K_TAB) - K_yCR* = iUPyxCODE(K_CR) - K_yBS* = iUPyxCODE(K_BS) - K_yPAUSE* = iUPyxCODE(K_PAUSE) - K_yESC* = iUPyxCODE(K_ESC) - K_yCcedilla* = iUPyxCODE(K_ccedilla) - K_yF1* = iUPyxCODE(K_F1) - K_yF2* = iUPyxCODE(K_F2) - K_yF3* = iUPyxCODE(K_F3) - K_yF4* = iUPyxCODE(K_F4) - K_yF5* = iUPyxCODE(K_F5) - K_yF6* = iUPyxCODE(K_F6) - K_yF7* = iUPyxCODE(K_F7) - K_yF8* = iUPyxCODE(K_F8) - K_yF9* = iUPyxCODE(K_F9) - K_yF10* = iUPyxCODE(K_F10) - K_yF11* = iUPyxCODE(K_F11) - K_yF12* = iUPyxCODE(K_F12) - K_yPrint* = iUPyxCODE(K_Print) - K_yMenu* = iUPyxCODE(K_Menu) - - K_sPlus* = iUPsxCODE(K_plus) - K_sComma* = iUPsxCODE(K_comma) - K_sMinus* = iUPsxCODE(K_minus) - K_sPeriod* = iUPsxCODE(K_period) - K_sSlash* = iUPsxCODE(K_slash) - K_sAsterisk* = iUPsxCODE(K_asterisk) - - K_cupperA* = iUPcxCODE(K_upperA) - K_cupperB* = iUPcxCODE(K_upperB) - K_cupperC* = iUPcxCODE(K_upperC) - K_cupperD* = iUPcxCODE(K_upperD) - K_cupperE* = iUPcxCODE(K_upperE) - K_cupperF* = iUPcxCODE(K_upperF) - K_cupperG* = iUPcxCODE(K_upperG) - K_cupperH* = iUPcxCODE(K_upperH) - K_cupperI* = iUPcxCODE(K_upperI) - K_cupperJ* = iUPcxCODE(K_upperJ) - K_cupperK* = iUPcxCODE(K_upperK) - K_cupperL* = iUPcxCODE(K_upperL) - K_cupperM* = iUPcxCODE(K_upperM) - K_cupperN* = iUPcxCODE(K_upperN) - K_cupperO* = iUPcxCODE(K_upperO) - K_cupperP* = iUPcxCODE(K_upperP) - K_cupperQ* = iUPcxCODE(K_upperQ) - K_cupperR* = iUPcxCODE(K_upperR) - K_cupperS* = iUPcxCODE(K_upperS) - K_cupperT* = iUPcxCODE(K_upperT) - K_cupperU* = iUPcxCODE(K_upperU) - K_cupperV* = iUPcxCODE(K_upperV) - K_cupperW* = iUPcxCODE(K_upperW) - K_cupperX* = iUPcxCODE(K_upperX) - K_cupperY* = iUPcxCODE(K_upperY) - K_cupperZ* = iUPcxCODE(K_upperZ) - K_c1* = iUPcxCODE(K_1) - K_c2* = iUPcxCODE(K_2) - K_c3* = iUPcxCODE(K_3) - K_c4* = iUPcxCODE(K_4) - K_c5* = iUPcxCODE(K_5) - K_c6* = iUPcxCODE(K_6) - K_c7* = iUPcxCODE(K_7) - K_c8* = iUPcxCODE(K_8) - K_c9* = iUPcxCODE(K_9) - K_c0* = iUPcxCODE(K_0) - K_cPlus* = iUPcxCODE(K_plus) - K_cComma* = iUPcxCODE(K_comma) - K_cMinus* = iUPcxCODE(K_minus) - K_cPeriod* = iUPcxCODE(K_period) - K_cSlash* = iUPcxCODE(K_slash) - K_cSemicolon* = iUPcxCODE(K_semicolon) - K_cEqual* = iUPcxCODE(K_equal) - K_cBracketleft* = iUPcxCODE(K_bracketleft) - K_cBracketright* = iUPcxCODE(K_bracketright) - K_cBackslash* = iUPcxCODE(K_backslash) - K_cAsterisk* = iUPcxCODE(K_asterisk) - - K_mupperA* = iUPmxCODE(K_upperA) - K_mupperB* = iUPmxCODE(K_upperB) - K_mupperC* = iUPmxCODE(K_upperC) - K_mupperD* = iUPmxCODE(K_upperD) - K_mupperE* = iUPmxCODE(K_upperE) - K_mupperF* = iUPmxCODE(K_upperF) - K_mupperG* = iUPmxCODE(K_upperG) - K_mupperH* = iUPmxCODE(K_upperH) - K_mupperI* = iUPmxCODE(K_upperI) - K_mupperJ* = iUPmxCODE(K_upperJ) - K_mupperK* = iUPmxCODE(K_upperK) - K_mupperL* = iUPmxCODE(K_upperL) - K_mupperM* = iUPmxCODE(K_upperM) - K_mupperN* = iUPmxCODE(K_upperN) - K_mupperO* = iUPmxCODE(K_upperO) - K_mupperP* = iUPmxCODE(K_upperP) - K_mupperQ* = iUPmxCODE(K_upperQ) - K_mupperR* = iUPmxCODE(K_upperR) - K_mupperS* = iUPmxCODE(K_upperS) - K_mupperT* = iUPmxCODE(K_upperT) - K_mupperU* = iUPmxCODE(K_upperU) - K_mupperV* = iUPmxCODE(K_upperV) - K_mupperW* = iUPmxCODE(K_upperW) - K_mupperX* = iUPmxCODE(K_upperX) - K_mupperY* = iUPmxCODE(K_upperY) - K_mupperZ* = iUPmxCODE(K_upperZ) - K_m1* = iUPmxCODE(K_1) - K_m2* = iUPmxCODE(K_2) - K_m3* = iUPmxCODE(K_3) - K_m4* = iUPmxCODE(K_4) - K_m5* = iUPmxCODE(K_5) - K_m6* = iUPmxCODE(K_6) - K_m7* = iUPmxCODE(K_7) - K_m8* = iUPmxCODE(K_8) - K_m9* = iUPmxCODE(K_9) - K_m0* = iUPmxCODE(K_0) - K_mPlus* = iUPmxCODE(K_plus) - K_mComma* = iUPmxCODE(K_comma) - K_mMinus* = iUPmxCODE(K_minus) - K_mPeriod* = iUPmxCODE(K_period) - K_mSlash* = iUPmxCODE(K_slash) - K_mSemicolon* = iUPmxCODE(K_semicolon) - K_mEqual* = iUPmxCODE(K_equal) - K_mBracketleft* = iUPmxCODE(K_bracketleft) - K_mBracketright* = iUPmxCODE(K_bracketright) - K_mBackslash* = iUPmxCODE(K_backslash) - K_mAsterisk* = iUPmxCODE(K_asterisk) - - K_yA* = iUPyxCODE(K_upperA) - K_yB* = iUPyxCODE(K_upperB) - K_yC* = iUPyxCODE(K_upperC) - K_yD* = iUPyxCODE(K_upperD) - K_yE* = iUPyxCODE(K_upperE) - K_yF* = iUPyxCODE(K_upperF) - K_yG* = iUPyxCODE(K_upperG) - K_yH* = iUPyxCODE(K_upperH) - K_yI* = iUPyxCODE(K_upperI) - K_yJ* = iUPyxCODE(K_upperJ) - K_yK* = iUPyxCODE(K_upperK) - K_yL* = iUPyxCODE(K_upperL) - K_yM* = iUPyxCODE(K_upperM) - K_yN* = iUPyxCODE(K_upperN) - K_yO* = iUPyxCODE(K_upperO) - K_yP* = iUPyxCODE(K_upperP) - K_yQ* = iUPyxCODE(K_upperQ) - K_yR* = iUPyxCODE(K_upperR) - K_yS* = iUPyxCODE(K_upperS) - K_yT* = iUPyxCODE(K_upperT) - K_yU* = iUPyxCODE(K_upperU) - K_yV* = iUPyxCODE(K_upperV) - K_yW* = iUPyxCODE(K_upperW) - K_yX* = iUPyxCODE(K_upperX) - K_yY* = iUPyxCODE(K_upperY) - K_yZ* = iUPyxCODE(K_upperZ) - K_y1* = iUPyxCODE(K_1) - K_y2* = iUPyxCODE(K_2) - K_y3* = iUPyxCODE(K_3) - K_y4* = iUPyxCODE(K_4) - K_y5* = iUPyxCODE(K_5) - K_y6* = iUPyxCODE(K_6) - K_y7* = iUPyxCODE(K_7) - K_y8* = iUPyxCODE(K_8) - K_y9* = iUPyxCODE(K_9) - K_y0* = iUPyxCODE(K_0) - K_yPlus* = iUPyxCODE(K_plus) - K_yComma* = iUPyxCODE(K_comma) - K_yMinus* = iUPyxCODE(K_minus) - K_yPeriod* = iUPyxCODE(K_period) - K_ySlash* = iUPyxCODE(K_slash) - K_ySemicolon* = iUPyxCODE(K_semicolon) - K_yEqual* = iUPyxCODE(K_equal) - K_yBracketleft* = iUPyxCODE(K_bracketleft) - K_yBracketright* = iUPyxCODE(K_bracketright) - K_yBackslash* = iUPyxCODE(K_backslash) - K_yAsterisk* = iUPyxCODE(K_asterisk) - -proc controlsOpen*(): cint {.cdecl, importc: "IupControlsOpen", dynlib: dllname.} -proc controlsClose*() {.cdecl, importc: "IupControlsClose", dynlib: dllname.} - -proc oldValOpen*() {.cdecl, importc: "IupOldValOpen", dynlib: dllname.} -proc oldTabsOpen*() {.cdecl, importc: "IupOldTabsOpen", dynlib: dllname.} - -proc colorbar*(): PIhandle {.cdecl, importc: "IupColorbar", dynlib: dllname.} -proc cells*(): PIhandle {.cdecl, importc: "IupCells", dynlib: dllname.} -proc colorBrowser*(): PIhandle {.cdecl, importc: "IupColorBrowser", dynlib: dllname.} -proc gauge*(): PIhandle {.cdecl, importc: "IupGauge", dynlib: dllname.} -proc dial*(theType: cstring): PIhandle {.cdecl, importc: "IupDial", dynlib: dllname.} -proc matrix*(action: cstring): PIhandle {.cdecl, importc: "IupMatrix", dynlib: dllname.} - -# IupMatrix utilities -proc matSetAttribute*(ih: PIhandle, name: cstring, lin, col: cint, - value: cstring) {. - cdecl, importc: "IupMatSetAttribute", dynlib: dllname.} -proc matStoreAttribute*(ih: PIhandle, name: cstring, lin, col: cint, - value: cstring) {.cdecl, - importc: "IupMatStoreAttribute", dynlib: dllname.} -proc matGetAttribute*(ih: PIhandle, name: cstring, lin, col: cint): cstring {. - cdecl, importc: "IupMatGetAttribute", dynlib: dllname.} -proc matGetInt*(ih: PIhandle, name: cstring, lin, col: cint): cint {. - cdecl, importc: "IupMatGetInt", dynlib: dllname.} -proc matGetFloat*(ih: PIhandle, name: cstring, lin, col: cint): cfloat {. - cdecl, importc: "IupMatGetFloat", dynlib: dllname.} -proc matSetfAttribute*(ih: PIhandle, name: cstring, lin, col: cint, - format: cstring) {.cdecl, - importc: "IupMatSetfAttribute", - dynlib: dllname, varargs.} - -# Used by IupColorbar -const - IUP_PRIMARY* = -1 - IUP_SECONDARY* = -2 - -# Initialize PPlot widget class -proc pPlotOpen*() {.cdecl, importc: "IupPPlotOpen", dynlib: dllname.} - -# Create an PPlot widget instance -proc pPlot*: PIhandle {.cdecl, importc: "IupPPlot", dynlib: dllname.} - -# Add dataset to plot -proc pPlotBegin*(ih: PIhandle, strXdata: cint) {. - cdecl, importc: "IupPPlotBegin", dynlib: dllname.} -proc pPlotAdd*(ih: PIhandle, x, y: cfloat) {. - cdecl, importc: "IupPPlotAdd", dynlib: dllname.} -proc pPlotAddStr*(ih: PIhandle, x: cstring, y: cfloat) {. - cdecl, importc: "IupPPlotAddStr", dynlib: dllname.} -proc pPlotEnd*(ih: PIhandle): cint {. - cdecl, importc: "IupPPlotEnd", dynlib: dllname.} - -proc pPlotInsertStr*(ih: PIhandle, index, sampleIndex: cint, x: cstring, - y: cfloat) {.cdecl, importc: "IupPPlotInsertStr", - dynlib: dllname.} -proc pPlotInsert*(ih: PIhandle, index, sampleIndex: cint, - x, y: cfloat) {. - cdecl, importc: "IupPPlotInsert", dynlib: dllname.} - -# convert from plot coordinates to pixels -proc pPlotTransform*(ih: PIhandle, x, y: cfloat, ix, iy: var cint) {. - cdecl, importc: "IupPPlotTransform", dynlib: dllname.} - -# Plot on the given device. Uses a "cdCanvas*". -proc pPlotPaintTo*(ih: PIhandle, cnv: pointer) {. - cdecl, importc: "IupPPlotPaintTo", dynlib: dllname.} - - diff --git a/lib/wrappers/joyent_http_parser.nim b/lib/wrappers/joyent_http_parser.nim deleted file mode 100644 index f7412d2b8..000000000 --- a/lib/wrappers/joyent_http_parser.nim +++ /dev/null @@ -1,93 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -type - csize = int - - HttpDataProc* = proc (a2: ptr HttpParser, at: cstring, length: csize): cint {.cdecl.} - HttpProc* = proc (a2: ptr HttpParser): cint {.cdecl.} - - HttpMethod* = enum - HTTP_DELETE = 0, HTTP_GET, HTTP_HEAD, HTTP_POST, HTTP_PUT, HTTP_CONNECT, - HTTP_OPTIONS, HTTP_TRACE, HTTP_COPY, HTTP_LOCK, HTTP_MKCOL, HTTP_MOVE, - HTTP_PROPFIND, HTTP_PROPPATCH, HTTP_UNLOCK, HTTP_REPORT, HTTP_MKACTIVITY, - HTTP_CHECKOUT, HTTP_MERGE, HTTP_MSEARCH, HTTP_NOTIFY, HTTP_SUBSCRIBE, - HTTP_UNSUBSCRIBE, HTTP_PATCH - - HttpParserType* = enum - HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH - - ParserFlag* = enum - F_CHUNKED = 1 shl 0, - F_CONNECTION_KEEP_ALIVE = 1 shl 1, - F_CONNECTION_CLOSE = 1 shl 2, - F_TRAILING = 1 shl 3, - F_UPGRADE = 1 shl 4, - F_SKIPBODY = 1 shl 5 - - HttpErrNo* = enum - HPE_OK, HPE_CB_message_begin, HPE_CB_path, HPE_CB_query_string, HPE_CB_url, - HPE_CB_fragment, HPE_CB_header_field, HPE_CB_header_value, - HPE_CB_headers_complete, HPE_CB_body, HPE_CB_message_complete, - HPE_INVALID_EOF_STATE, HPE_HEADER_OVERFLOW, HPE_CLOSED_CONNECTION, - HPE_INVALID_VERSION, HPE_INVALID_STATUS, HPE_INVALID_METHOD, - HPE_INVALID_URL, HPE_INVALID_HOST, HPE_INVALID_PORT, HPE_INVALID_PATH, - HPE_INVALID_QUERY_STRING, HPE_INVALID_FRAGMENT, HPE_LF_EXPECTED, - HPE_INVALID_HEADER_TOKEN, HPE_INVALID_CONTENT_LENGTH, - HPE_INVALID_CHUNK_SIZE, HPE_INVALID_CONSTANT, HPE_INVALID_INTERNAL_STATE, - HPE_STRICT, HPE_UNKNOWN - - HttpParser*{.pure, final, importc: "http_parser", header: "http_parser.h".} = object - typ {.importc: "type".}: char - flags {.importc: "flags".}: char - state*{.importc: "state".}: char - header_state*{.importc: "header_state".}: char - index*{.importc: "index".}: char - nread*{.importc: "nread".}: cint - content_length*{.importc: "content_length".}: int64 - http_major*{.importc: "http_major".}: cshort - http_minor*{.importc: "http_minor".}: cshort - status_code*{.importc: "status_code".}: cshort - http_method*{.importc: "method".}: cshort - http_errno_bits {.importc: "http_errno".}: char - upgrade {.importc: "upgrade".}: bool - data*{.importc: "data".}: pointer - - HttpParserSettings*{.pure, final, importc: "http_parser_settings", header: "http_parser.h".} = object - on_message_begin*{.importc: "on_message_begin".}: HttpProc - on_url*{.importc: "on_url".}: HttpDataProc - on_header_field*{.importc: "on_header_field".}: HttpDataProc - on_header_value*{.importc: "on_header_value".}: HttpDataProc - on_headers_complete*{.importc: "on_headers_complete".}: HttpProc - on_body*{.importc: "on_body".}: HttpDataProc - on_message_complete*{.importc: "on_message_complete".}: HttpProc -{.deprecated: [THttpMethod: HttpMethod, THttpParserType: HttpParserType, - TParserFlag: ParserFlag, THttpErrNo: HttpErrNo, - THttpParser: HttpParser, THttpParserSettings: HttpParserSettings].} - -proc http_parser_init*(parser: var HttpParser, typ: HttpParserType){. - importc: "http_parser_init", header: "http_parser.h".} - -proc http_parser_execute*(parser: var HttpParser, - settings: var HttpParserSettings, data: cstring, - len: csize): csize {. - importc: "http_parser_execute", header: "http_parser.h".} - -proc http_should_keep_alive*(parser: var HttpParser): cint{. - importc: "http_should_keep_alive", header: "http_parser.h".} - -proc http_method_str*(m: HttpMethod): cstring{. - importc: "http_method_str", header: "http_parser.h".} - -proc http_errno_name*(err: HttpErrNo): cstring{. - importc: "http_errno_name", header: "http_parser.h".} - -proc http_errno_description*(err: HttpErrNo): cstring{. - importc: "http_errno_description", header: "http_parser.h".} - diff --git a/lib/wrappers/libsvm.nim b/lib/wrappers/libsvm.nim deleted file mode 100644 index ac5889410..000000000 --- a/lib/wrappers/libsvm.nim +++ /dev/null @@ -1,117 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module is a low level wrapper for `libsvm`:idx:. - -{.deadCodeElim: on.} -const - LIBSVM_VERSION* = 312 - -when defined(windows): - const svmdll* = "libsvm.dll" -elif defined(macosx): - const svmdll* = "libsvm.dylib" -else: - const svmdll* = "libsvm.so" - -type - Node*{.pure, final.} = object - index*: cint - value*: cdouble - - Problem*{.pure, final.} = object - L*: cint - y*: ptr cdouble - x*: ptr ptr Node - - Type*{.size: sizeof(cint).} = enum - C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR - - KernelType*{.size: sizeof(cint).} = enum - LINEAR, POLY, RBF, SIGMOID, PRECOMPUTED - - Parameter*{.pure, final.} = object - typ*: Type - kernelType*: KernelType - degree*: cint # for poly - gamma*: cdouble # for poly/rbf/sigmoid - coef0*: cdouble # for poly/sigmoid - # these are for training only - cache_size*: cdouble # in MB - eps*: cdouble # stopping criteria - C*: cdouble # for C_SVC, EPSILON_SVR and NU_SVR - nr_weight*: cint # for C_SVC - weight_label*: ptr cint # for C_SVC - weight*: ptr cdouble # for C_SVC - nu*: cdouble # for NU_SVC, ONE_CLASS, and NU_SVR - p*: cdouble # for EPSILON_SVR - shrinking*: cint # use the shrinking heuristics - probability*: cint # do probability estimates -{.deprecated: [Tnode: Node, Tproblem: Problem, Ttype: Type, - TKernelType: KernelType, Tparameter: Parameter].} - -# -# svm_model -# - -type - Model*{.pure, final.} = object - param*: Parameter # parameter - nr_class*: cint # number of classes, = 2 in regression/one class svm - L*: cint # total #SV - SV*: ptr ptr Node # SVs (SV[l]) - sv_coef*: ptr ptr cdouble # coefficients for SVs in decision functions (sv_coef[k-1][l]) - rho*: ptr cdouble # constants in decision functions (rho[k*(k-1)/2]) - probA*: ptr cdouble # pariwise probability information - probB*: ptr cdouble # for classification only - label*: ptr cint # label of each class (label[k]) - nSV*: ptr cint # number of SVs for each class (nSV[k]) - # nSV[0] + nSV[1] + ... + nSV[k-1] = l - # XXX - free_sv*: cint # 1 if svm_model is created by svm_load_model - # 0 if svm_model is created by svm_train -{.deprecated: [TModel: Model].} - -proc train*(prob: ptr Problem, param: ptr Parameter): ptr Model{.cdecl, - importc: "svm_train", dynlib: svmdll.} -proc cross_validation*(prob: ptr Problem, param: ptr Parameter, nr_fold: cint, - target: ptr cdouble){.cdecl, - importc: "svm_cross_validation", dynlib: svmdll.} -proc save_model*(model_file_name: cstring, model: ptr Model): cint{.cdecl, - importc: "svm_save_model", dynlib: svmdll.} -proc load_model*(model_file_name: cstring): ptr Model{.cdecl, - importc: "svm_load_model", dynlib: svmdll.} -proc get_svm_type*(model: ptr Model): cint{.cdecl, importc: "svm_get_svm_type", - dynlib: svmdll.} -proc get_nr_class*(model: ptr Model): cint{.cdecl, importc: "svm_get_nr_class", - dynlib: svmdll.} -proc get_labels*(model: ptr Model, label: ptr cint){.cdecl, - importc: "svm_get_labels", dynlib: svmdll.} -proc get_svr_probability*(model: ptr Model): cdouble{.cdecl, - importc: "svm_get_svr_probability", dynlib: svmdll.} -proc predict_values*(model: ptr Model, x: ptr Node, dec_values: ptr cdouble): cdouble{. - cdecl, importc: "svm_predict_values", dynlib: svmdll.} -proc predict*(model: ptr Model, x: ptr Node): cdouble{.cdecl, - importc: "svm_predict", dynlib: svmdll.} -proc predict_probability*(model: ptr Model, x: ptr Node, - prob_estimates: ptr cdouble): cdouble{.cdecl, - importc: "svm_predict_probability", dynlib: svmdll.} -proc free_model_content*(model_ptr: ptr Model){.cdecl, - importc: "svm_free_model_content", dynlib: svmdll.} -proc free_and_destroy_model*(model_ptr_ptr: ptr ptr Model){.cdecl, - importc: "svm_free_and_destroy_model", dynlib: svmdll.} -proc destroy_param*(param: ptr Parameter){.cdecl, importc: "svm_destroy_param", - dynlib: svmdll.} -proc check_parameter*(prob: ptr Problem, param: ptr Parameter): cstring{. - cdecl, importc: "svm_check_parameter", dynlib: svmdll.} -proc check_probability_model*(model: ptr Model): cint{.cdecl, - importc: "svm_check_probability_model", dynlib: svmdll.} - -proc set_print_string_function*(print_func: proc (arg: cstring) {.cdecl.}){. - cdecl, importc: "svm_set_print_string_function", dynlib: svmdll.} diff --git a/lib/wrappers/libuv.nim b/lib/wrappers/libuv.nim deleted file mode 100644 index 3e28815ad..000000000 --- a/lib/wrappers/libuv.nim +++ /dev/null @@ -1,732 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## libuv is still fast moving target -## This file was last updated against a development HEAD revision of https://github.com/joyent/libuv/ - -## Use the following link to see changes (in uv.h) since then and don't forget to update the information here. -## https://github.com/joyent/libuv/compare/9f6024a6fa9d254527b4b59af724257df870288b...master - -when defined(Windows): - import winlean -else: - import posix - -type - Port* = distinct int16 ## port type - - cssize = int - coff = int - csize = int - - AllocProc* = proc (handle: PHandle, suggested_size: csize): Buf {.cdecl.} - ReadProc* = proc (stream: PStream, nread: cssize, buf: Buf) {.cdecl.} - ReadProc2* = proc (stream: PPipe, nread: cssize, buf: Buf, pending: HandleType) {.cdecl.} - WriteProc* = proc (req: PWrite, status: cint) {.cdecl.} - ConnectProc* = proc (req: PConnect, status: cint) {.cdecl.} - ShutdownProc* = proc (req: PShutdown, status: cint) {.cdecl.} - ConnectionProc* = proc (server: PStream, status: cint) {.cdecl.} - CloseProc* = proc (handle: PHandle) {.cdecl.} - TimerProc* = proc (handle: PTimer, status: cint) {.cdecl.} - AsyncProc* = proc (handle: PAsync, status: cint) {.cdecl.} - PrepareProc* = proc (handle: PPrepare, status: cint) {.cdecl.} - CheckProc* = proc (handle: PCheck, status: cint) {.cdecl.} - IdleProc* = proc (handle: PIdle, status: cint) {.cdecl.} - - PSockAddr* = ptr SockAddr - - GetAddrInfoProc* = proc (handle: PGetAddrInfo, status: cint, res: ptr AddrInfo) - - ExitProc* = proc (a2: PProcess, exit_status: cint, term_signal: cint) - FsProc* = proc (req: PFS) - WorkProc* = proc (req: PWork) - AfterWorkProc* = proc (req: PWork) - - FsEventProc* = proc (handle: PFsEvent, filename: cstring, events: cint, status: cint) - - ErrorCode* {.size: sizeof(cint).} = enum - UNKNOWN = - 1, OK = 0, EOF, EACCESS, EAGAIN, EADDRINUSE, EADDRNOTAVAIL, - EAFNOSUPPORT, EALREADY, EBADF, EBUSY, ECONNABORTED, ECONNREFUSED, - ECONNRESET, EDESTADDRREQ, EFAULT, EHOSTUNREACH, EINTR, EINVAL, EISCONN, - EMFILE, EMSGSIZE, ENETDOWN, ENETUNREACH, ENFILE, ENOBUFS, ENOMEM, ENONET, - ENOPROTOOPT, ENOTCONN, ENOTSOCK, ENOTSUP, ENOENT, EPIPE, EPROTO, - EPROTONOSUPPORT, EPROTOTYPE, ETIMEDOUT, ECHARSET, EAIFAMNOSUPPORT, - EAINONAME, EAISERVICE, EAISOCKTYPE, ESHUTDOWN, EEXIST - - HandleType* {.size: sizeof(cint).} = enum - UNKNOWN_HANDLE = 0, TCP, UDP, NAMED_PIPE, TTY, FILE, TIMER, PREPARE, CHECK, - IDLE, ASYNC, ARES_TASK, ARES_EVENT, PROCESS, FS_EVENT - - ReqType* {.size: sizeof(cint).} = enum - rUNKNOWN_REQ = 0, - rCONNECT, - rACCEPT, - rREAD, - rWRITE, - rSHUTDOWN, - rWAKEUP, - rUDP_SEND, - rFS, - rWORK, - rGETADDRINFO, - rREQ_TYPE_PRIVATE - - Err* {.pure, final, importc: "uv_err_t", header: "uv.h".} = object - code* {.importc: "code".}: ErrorCode - sys_errno* {.importc: "sys_errno_".}: cint - - FsEventType* = enum - evRENAME = 1, - evCHANGE = 2 - - TFsEvent* {.pure, final, importc: "uv_fs_event_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - filename {.importc: "filename".}: cstring - - PFsEvent* = ptr TFsEvent - - FsEvents* {.pure, final, importc: "uv_fs_event_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - filename* {.importc: "filename".}: cstring - - Buf* {.pure, final, importc: "uv_buf_t", header: "uv.h"} = object - base* {.importc: "base".}: cstring - len* {.importc: "len".}: csize - - AnyHandle* {.pure, final, importc: "uv_any_handle", header: "uv.h".} = object - tcp* {.importc: "tcp".}: TTcp - pipe* {.importc: "pipe".}: Pipe - prepare* {.importc: "prepare".}: TPrepare - check* {.importc: "check".}: TCheck - idle* {.importc: "idle".}: TIdle - async* {.importc: "async".}: TAsync - timer* {.importc: "timer".}: TTimer - getaddrinfo* {.importc: "getaddrinfo".}: Getaddrinfo - fs_event* {.importc: "fs_event".}: FsEvents - - AnyReq* {.pure, final, importc: "uv_any_req", header: "uv.h".} = object - req* {.importc: "req".}: Req - write* {.importc: "write".}: Write - connect* {.importc: "connect".}: Connect - shutdown* {.importc: "shutdown".}: Shutdown - fs_req* {.importc: "fs_req".}: Fs - work_req* {.importc: "work_req".}: Work - - ## better import this - uint64 = int64 - - Counters* {.pure, final, importc: "uv_counters_t", header: "uv.h".} = object - eio_init* {.importc: "eio_init".}: uint64 - req_init* {.importc: "req_init".}: uint64 - handle_init* {.importc: "handle_init".}: uint64 - stream_init* {.importc: "stream_init".}: uint64 - tcp_init* {.importc: "tcp_init".}: uint64 - udp_init* {.importc: "udp_init".}: uint64 - pipe_init* {.importc: "pipe_init".}: uint64 - tty_init* {.importc: "tty_init".}: uint64 - prepare_init* {.importc: "prepare_init".}: uint64 - check_init* {.importc: "check_init".}: uint64 - idle_init* {.importc: "idle_init".}: uint64 - async_init* {.importc: "async_init".}: uint64 - timer_init* {.importc: "timer_init".}: uint64 - process_init* {.importc: "process_init".}: uint64 - fs_event_init* {.importc: "fs_event_init".}: uint64 - - Loop* {.pure, final, importc: "uv_loop_t", header: "uv.h".} = object - # ares_handles_* {.importc: "uv_ares_handles_".}: pointer # XXX: This seems to be a private field? - eio_want_poll_notifier* {.importc: "uv_eio_want_poll_notifier".}: TAsync - eio_done_poll_notifier* {.importc: "uv_eio_done_poll_notifier".}: TAsync - eio_poller* {.importc: "uv_eio_poller".}: TIdle - counters* {.importc: "counters".}: Counters - last_err* {.importc: "last_err".}: Err - data* {.importc: "data".}: pointer - - PLoop* = ptr Loop - - Shutdown* {.pure, final, importc: "uv_shutdown_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - handle* {.importc: "handle".}: PStream - cb* {.importc: "cb".}: ShutdownProc - - PShutdown* = ptr Shutdown - - Handle* {.pure, final, importc: "uv_handle_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PHandle* = ptr Handle - - Stream* {.pure, final, importc: "uv_stream_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - alloc_cb* {.importc: "alloc_cb".}: AllocProc - read_cb* {.importc: "read_cb".}: ReadProc - read2_cb* {.importc: "read2_cb".}: ReadProc2 - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - write_queue_size* {.importc: "write_queue_size".}: csize - - PStream* = ptr Stream - - Write* {.pure, final, importc: "uv_write_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - cb* {.importc: "cb".}: WriteProc - send_handle* {.importc: "send_handle".}: PStream - handle* {.importc: "handle".}: PStream - - PWrite* = ptr Write - - TTcp* {.pure, final, importc: "uv_tcp_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - alloc_cb* {.importc: "alloc_cb".}: AllocProc - read_cb* {.importc: "read_cb".}: ReadProc - read2_cb* {.importc: "read2_cb".}: ReadProc2 - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - write_queue_size* {.importc: "write_queue_size".}: csize - - PTcp* = ptr TTcp - - Connect* {.pure, final, importc: "uv_connect_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - cb* {.importc: "cb".}: ConnectProc - handle* {.importc: "handle".}: PStream - - PConnect* = ptr Connect - - UdpFlags* = enum - UDP_IPV6ONLY = 1, UDP_PARTIAL = 2 - - ## XXX: better import this - cunsigned = int - - UdpSendProc* = proc (req: PUdpSend, status: cint) - UdpRecvProc* = proc (handle: PUdp, nread: cssize, buf: Buf, adr: ptr SockAddr, flags: cunsigned) - - TUdp* {.pure, final, importc: "uv_udp_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PUdp* = ptr TUdp - - UdpSend* {.pure, final, importc: "uv_udp_send_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - handle* {.importc: "handle".}: PUdp - cb* {.importc: "cb".}: UdpSendProc - - PUdpSend* = ptr UdpSend - - tTTy* {.pure, final, importc: "uv_tty_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - alloc_cb* {.importc: "alloc_cb".}: AllocProc - read_cb* {.importc: "read_cb".}: ReadProc - read2_cb* {.importc: "read2_cb".}: ReadProc2 - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - write_queue_size* {.importc: "write_queue_size".}: csize - - pTTy* = ptr tTTy - - Pipe* {.pure, final, importc: "uv_pipe_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - alloc_cb* {.importc: "alloc_cb".}: AllocProc - read_cb* {.importc: "read_cb".}: ReadProc - read2_cb* {.importc: "read2_cb".}: ReadProc2 - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - write_queue_size* {.importc: "write_queue_size".}: csize - ipc {.importc: "ipc".}: int - - PPipe* = ptr Pipe - - TPrepare* {.pure, final, importc: "uv_prepare_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PPrepare* = ptr TPrepare - - TCheck* {.pure, final, importc: "uv_check_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PCheck* = ptr TCheck - - TIdle* {.pure, final, importc: "uv_idle_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PIdle* = ptr TIdle - - TAsync* {.pure, final, importc: "uv_async_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PAsync* = ptr TAsync - - TTimer* {.pure, final, importc: "uv_timer_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - - PTimer* = ptr TTimer - - GetAddrInfo* {.pure, final, importc: "uv_getaddrinfo_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - loop* {.importc: "loop".}: PLoop - - PGetAddrInfo* = ptr GetAddrInfo - - ProcessOptions* {.pure, final, importc: "uv_process_options_t", header: "uv.h".} = object - exit_cb* {.importc: "exit_cb".}: ExitProc - file* {.importc: "file".}: cstring - args* {.importc: "args".}: cstringArray - env* {.importc: "env".}: cstringArray - cwd* {.importc: "cwd".}: cstring - windows_verbatim_arguments* {.importc: "windows_verbatim_arguments".}: cint - stdin_stream* {.importc: "stdin_stream".}: PPipe - stdout_stream* {.importc: "stdout_stream".}: PPipe - stderr_stream* {.importc: "stderr_stream".}: PPipe - - PProcessOptions* = ptr ProcessOptions - - TProcess* {.pure, final, importc: "uv_process_t", header: "uv.h".} = object - loop* {.importc: "loop".}: PLoop - typ* {.importc: "type".}: HandleType - close_cb* {.importc: "close_cb".}: CloseProc - data* {.importc: "data".}: pointer - exit_cb* {.importc: "exit_cb".}: ExitProc - pid* {.importc: "pid".}: cint - - PProcess* = ptr TProcess - - Work* {.pure, final, importc: "uv_work_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - loop* {.importc: "loop".}: PLoop - work_cb* {.importc: "work_cb".}: WorkProc - after_work_cb* {.importc: "after_work_cb".}: AfterWorkProc - - PWork* = ptr Work - - FsType* {.size: sizeof(cint).} = enum - FS_UNKNOWN = - 1, FS_CUSTOM, FS_OPEN, FS_CLOSE, FS_READ, FS_WRITE, - FS_SENDFILE, FS_STAT, FS_LSTAT, FS_FSTAT, FS_FTRUNCATE, FS_UTIME, FS_FUTIME, - FS_CHMOD, FS_FCHMOD, FS_FSYNC, FS_FDATASYNC, FS_UNLINK, FS_RMDIR, FS_MKDIR, - FS_RENAME, FS_READDIR, FS_LINK, FS_SYMLINK, FS_READLINK, FS_CHOWN, FS_FCHOWN - - FS* {.pure, final, importc: "uv_fs_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - loop* {.importc: "loop".}: PLoop - fs_type* {.importc: "fs_type".}: FsType - cb* {.importc: "cb".}: FsProc - result* {.importc: "result".}: cssize - fsPtr* {.importc: "ptr".}: pointer - path* {.importc: "path".}: cstring - errorno* {.importc: "errorno".}: cint - - PFS* = ptr FS - - Req* {.pure, final, importc: "uv_req_t", header: "uv.h".} = object - typ* {.importc: "type".}: ReqType - data* {.importc: "data".}: pointer - - PReq* = ptr Req - - AresOptions* {.pure, final, importc: "ares_options", header: "uv.h".} = object - flags* {.importc: "flags".}: int - timeout* {.importc: "timeout".}: int - tries* {.importc: "tries".}: int - ndots* {.importc: "ndots".}: int - udp_port* {.importc: "udp_port".}: Port - tcp_port* {.importc: "tcp_port".}: Port - socket_send_buffer_size* {.importc: "socket_send_buffer_size".}: int - socket_recv_buffer_size* {.importc: "socket_receive_buffer_size".}: int - servers* {.importc: "servers".}: ptr InAddr - nservers* {.importc: "nservers".}: int - domains* {.importc: "domains".}: ptr cstring - ndomains* {.importc: "ndomains".}: int - lookups* {.importc: "lookups".}: cstring - - #XXX: not yet exported - #ares_sock_state_cb sock_state_cb; - #void *sock_state_cb_data; - #struct apattern *sortlist; - #int nsort; - - PAresOptions* = ptr AresOptions - PAresChannel* = pointer -{.deprecated: [THandle: Handle, THandleType: HandleType, TAnyHandle: AnyHandle, - TAnyReq: AnyReq, TPort: Port, TErrorCode: ErrorCode, TReqType: ReqType, - TErr: Err, TFsEventType: FsEventType, - # TFsEvent: FsEvent, # Name conflict if we drop `T` - TFsEvents: FsEvents, TBuf: Buf, TCounters: Counters, TLoop: Loop, - TShutdown: Shutdown, TStream: Stream, TWrite: Write, - # TTcp: Tcp, # Name conflict if we drop `T` - TConnect: Connect, - TUdpFlags: UdpFlags, - # TUdp: Udp, # Name conflict if we drop `T` - TUdpSend: UdpSend, - # tTTy: TTy, # Name conflict if we drop `T` - TPipe: Pipe, - # TPrepare: Prepare, # Name conflict if we drop `T` - # TCheck: Check, # Name conflict if we drop `T` - # TIdle: Idle, # Name conflict if we drop `T` - # TAsync: Async, # Name conflict if we drop `T` - # TTimer: Timer, # Name conflict if we drop `T` - TGetAddrInfo: GetAddrInfo, TProcessOptions: ProcessOptions, - # TProcess: Process, # Name conflict if we drop `T` - TWork: Work, - TFsType: FsType, TFS: FS, TReq: Req, TAresOptions: AresOptions].} - -proc loop_new*(): PLoop{. - importc: "uv_loop_new", header: "uv.h".} - -proc loop_delete*(a2: PLoop){. - importc: "uv_loop_delete", header: "uv.h".} - -proc default_loop*(): PLoop{. - importc: "uv_default_loop", header: "uv.h".} - -proc run*(a2: PLoop): cint{. - importc: "uv_run", header: "uv.h".} - -proc addref*(a2: PLoop){. - importc: "uv_ref", header: "uv.h".} - -proc unref*(a2: PLoop){. - importc: "uv_unref", header: "uv.h".} - -proc update_time*(a2: PLoop){. - importc: "uv_update_time", header: "uv.h".} - -proc now*(a2: PLoop): int64{. - importc: "uv_now", header: "uv.h".} - -proc last_error*(a2: PLoop): Err{. - importc: "uv_last_error", header: "uv.h".} - -proc strerror*(err: Err): cstring{. - importc: "uv_strerror", header: "uv.h".} - -proc err_name*(err: Err): cstring{. - importc: "uv_err_name", header: "uv.h".} - -proc shutdown*(req: PShutdown, handle: PStream, cb: ShutdownProc): cint{. - importc: "uv_shutdown", header: "uv.h".} - -proc is_active*(handle: PHandle): cint{. - importc: "uv_is_active", header: "uv.h".} - -proc close*(handle: PHandle, close_cb: CloseProc){. - importc: "uv_close", header: "uv.h".} - -proc buf_init*(base: cstring, len: csize): Buf{. - importc: "uv_buf_init", header: "uv.h".} - -proc listen*(stream: PStream, backlog: cint, cb: ConnectionProc): cint{. - importc: "uv_listen", header: "uv.h".} - -proc accept*(server: PStream, client: PStream): cint{. - importc: "uv_accept", header: "uv.h".} - -proc read_start*(a2: PStream, alloc_cb: AllocProc, read_cb: ReadProc): cint{. - importc: "uv_read_start", header: "uv.h".} - -proc read_start*(a2: PStream, alloc_cb: AllocProc, read_cb: ReadProc2): cint{. - importc: "uv_read2_start", header: "uv.h".} - -proc read_stop*(a2: PStream): cint{. - importc: "uv_read_stop", header: "uv.h".} - -proc write*(req: PWrite, handle: PStream, bufs: ptr Buf, bufcnt: cint, cb: WriteProc): cint{. - importc: "uv_write", header: "uv.h".} - -proc write*(req: PWrite, handle: PStream, bufs: ptr Buf, bufcnt: cint, send_handle: PStream, cb: WriteProc): cint{. - importc: "uv_write2", header: "uv.h".} - -proc tcp_init*(a2: PLoop, handle: PTcp): cint{. - importc: "uv_tcp_init", header: "uv.h".} - -proc tcp_bind*(handle: PTcp, a3: SockAddrIn): cint{. - importc: "uv_tcp_bind", header: "uv.h".} - -proc tcp_bind6*(handle: PTcp, a3: TSockAddrIn6): cint{. - importc: "uv_tcp_bind6", header: "uv.h".} - -proc tcp_getsockname*(handle: PTcp, name: ptr SockAddr, namelen: var cint): cint{. - importc: "uv_tcp_getsockname", header: "uv.h".} - -proc tcp_getpeername*(handle: PTcp, name: ptr SockAddr, namelen: var cint): cint{. - importc: "uv_tcp_getpeername", header: "uv.h".} - -proc tcp_connect*(req: PConnect, handle: PTcp, address: SockAddrIn, cb: ConnectProc): cint{. - importc: "uv_tcp_connect", header: "uv.h".} - -proc tcp_connect6*(req: PConnect, handle: PTcp, address: TSockAddrIn6, cb: ConnectProc): cint{. - importc: "uv_tcp_connect6", header: "uv.h".} - -proc udp_init*(a2: PLoop, handle: PUdp): cint{. - importc: "uv_udp_init", header: "uv.h".} - -proc udp_bind*(handle: PUdp, adr: SockAddrIn, flags: cunsigned): cint{. - importc: "uv_udp_bind", header: "uv.h".} - -proc udp_bind6*(handle: PUdp, adr: TSockAddrIn6, flags: cunsigned): cint{. - importc: "uv_udp_bind6", header: "uv.h".} - -proc udp_getsockname*(handle: PUdp, name: ptr SockAddr, namelen: var cint): cint{. - importc: "uv_udp_getsockname", header: "uv.h".} - -proc udp_send*(req: PUdpSend, handle: PUdp, bufs: ptr Buf, bufcnt: cint, adr: SockAddrIn, send_cb: UdpSendProc): cint{. - importc: "uv_udp_send", header: "uv.h".} - -proc udp_send6*(req: PUdpSend, handle: PUdp, bufs: ptr Buf, bufcnt: cint, adr: TSockAddrIn6, send_cb: UdpSendProc): cint{. - importc: "uv_udp_send6", header: "uv.h".} - -proc udp_recv_start*(handle: PUdp, alloc_cb: AllocProc, recv_cb: UdpRecvProc): cint{. - importc: "uv_udp_recv_start", header: "uv.h".} - -proc udp_recv_stop*(handle: PUdp): cint{. - importc: "uv_udp_recv_stop", header: "uv.h".} - -proc tty_init*(a2: PLoop, a3: pTTy, fd: File): cint{. - importc: "uv_tty_init", header: "uv.h".} - -proc tty_set_mode*(a2: pTTy, mode: cint): cint{. - importc: "uv_tty_set_mode", header: "uv.h".} - -proc tty_get_winsize*(a2: pTTy, width: var cint, height: var cint): cint{. - importc: "uv_tty_get_winsize", header: "uv.h".} - -proc tty_reset_mode*() {. - importc: "uv_tty_reset_mode", header: "uv.h".} - -proc guess_handle*(file: File): HandleType{. - importc: "uv_guess_handle", header: "uv.h".} - -proc pipe_init*(a2: PLoop, handle: PPipe, ipc: int): cint{. - importc: "uv_pipe_init", header: "uv.h".} - -proc pipe_open*(a2: PPipe, file: File){. - importc: "uv_pipe_open", header: "uv.h".} - -proc pipe_bind*(handle: PPipe, name: cstring): cint{. - importc: "uv_pipe_bind", header: "uv.h".} - -proc pipe_connect*(req: PConnect, handle: PPipe, name: cstring, cb: ConnectProc): cint{. - importc: "uv_pipe_connect", header: "uv.h".} - -proc prepare_init*(a2: PLoop, prepare: PPrepare): cint{. - importc: "uv_prepare_init", header: "uv.h".} - -proc prepare_start*(prepare: PPrepare, cb: PrepareProc): cint{. - importc: "uv_prepare_start", header: "uv.h".} - -proc prepare_stop*(prepare: PPrepare): cint{. - importc: "uv_prepare_stop", header: "uv.h".} - -proc check_init*(a2: PLoop, check: PCheck): cint{. - importc: "uv_check_init", header: "uv.h".} - -proc check_start*(check: PCheck, cb: CheckProc): cint{. - importc: "uv_check_start", header: "uv.h".} - -proc check_stop*(check: PCheck): cint{. - importc: "uv_check_stop", header: "uv.h".} - -proc idle_init*(a2: PLoop, idle: PIdle): cint{. - importc: "uv_idle_init", header: "uv.h".} - -proc idle_start*(idle: PIdle, cb: IdleProc): cint{. - importc: "uv_idle_start", header: "uv.h".} - -proc idle_stop*(idle: PIdle): cint{. - importc: "uv_idle_stop", header: "uv.h".} - -proc async_init*(a2: PLoop, async: PAsync, async_cb: AsyncProc): cint{. - importc: "uv_async_init", header: "uv.h".} - -proc async_send*(async: PAsync): cint{. - importc: "uv_async_send", header: "uv.h".} - -proc timer_init*(a2: PLoop, timer: PTimer): cint{. - importc: "uv_timer_init", header: "uv.h".} - -proc timer_start*(timer: PTimer, cb: TimerProc, timeout: int64, repeat: int64): cint{. - importc: "uv_timer_start", header: "uv.h".} - -proc timer_stop*(timer: PTimer): cint{. - importc: "uv_timer_stop", header: "uv.h".} - -proc timer_again*(timer: PTimer): cint{. - importc: "uv_timer_again", header: "uv.h".} - -proc timer_set_repeat*(timer: PTimer, repeat: int64){. - importc: "uv_timer_set_repeat", header: "uv.h".} - -proc timer_get_repeat*(timer: PTimer): int64{. - importc: "uv_timer_get_repeat", header: "uv.h".} - -proc ares_init_options*(a2: PLoop, channel: PAresChannel, options: PAresOptions, optmask: cint): cint{. - importc: "uv_ares_init_options", header: "uv.h".} - -proc ares_destroy*(a2: PLoop, channel: PAresChannel){. - importc: "uv_ares_destroy", header: "uv.h".} - -proc getaddrinfo*(a2: PLoop, handle: PGetAddrInfo,getaddrinfo_cb: GetAddrInfoProc, node: cstring, service: cstring, hints: ptr AddrInfo): cint{. - importc: "uv_getaddrinfo", header: "uv.h".} - -proc freeaddrinfo*(ai: ptr AddrInfo){. - importc: "uv_freeaddrinfo", header: "uv.h".} - -proc spawn*(a2: PLoop, a3: PProcess, options: ProcessOptions): cint{. - importc: "uv_spawn", header: "uv.h".} - -proc process_kill*(a2: PProcess, signum: cint): cint{. - importc: "uv_process_kill", header: "uv.h".} - -proc queue_work*(loop: PLoop, req: PWork, work_cb: WorkProc, after_work_cb: AfterWorkProc): cint{. - importc: "uv_queue_work", header: "uv.h".} - -proc req_cleanup*(req: PFS){. - importc: "uv_fs_req_cleanup", header: "uv.h".} - -proc close*(loop: PLoop, req: PFS, file: File, cb: FsProc): cint{. - importc: "uv_fs_close", header: "uv.h".} - -proc open*(loop: PLoop, req: PFS, path: cstring, flags: cint, mode: cint, cb: FsProc): cint{. - importc: "uv_fs_open", header: "uv.h".} - -proc read*(loop: PLoop, req: PFS, file: File, buf: pointer, length: csize, offset: coff, cb: FsProc): cint{. - importc: "uv_fs_read", header: "uv.h".} - -proc unlink*(loop: PLoop, req: PFS, path: cstring, cb: FsProc): cint{. - importc: "uv_fs_unlink", header: "uv.h".} - -proc write*(loop: PLoop, req: PFS, file: File, buf: pointer, length: csize, offset: coff, cb: FsProc): cint{. - importc: "uv_fs_write", header: "uv.h".} - -proc mkdir*(loop: PLoop, req: PFS, path: cstring, mode: cint, cb: FsProc): cint{. - importc: "uv_fs_mkdir", header: "uv.h".} - -proc rmdir*(loop: PLoop, req: PFS, path: cstring, cb: FsProc): cint{. - importc: "uv_fs_rmdir", header: "uv.h".} - -proc readdir*(loop: PLoop, req: PFS, path: cstring, flags: cint, cb: FsProc): cint{. - importc: "uv_fs_readdir", header: "uv.h".} - -proc stat*(loop: PLoop, req: PFS, path: cstring, cb: FsProc): cint{. - importc: "uv_fs_stat", header: "uv.h".} - -proc fstat*(loop: PLoop, req: PFS, file: File, cb: FsProc): cint{. - importc: "uv_fs_fstat", header: "uv.h".} - -proc rename*(loop: PLoop, req: PFS, path: cstring, new_path: cstring, cb: FsProc): cint{. - importc: "uv_fs_rename", header: "uv.h".} - -proc fsync*(loop: PLoop, req: PFS, file: File, cb: FsProc): cint{. - importc: "uv_fs_fsync", header: "uv.h".} - -proc fdatasync*(loop: PLoop, req: PFS, file: File, cb: FsProc): cint{. - importc: "uv_fs_fdatasync", header: "uv.h".} - -proc ftruncate*(loop: PLoop, req: PFS, file: File, offset: coff, cb: FsProc): cint{. - importc: "uv_fs_ftruncate", header: "uv.h".} - -proc sendfile*(loop: PLoop, req: PFS, out_fd: File, in_fd: File, in_offset: coff, length: csize, cb: FsProc): cint{. - importc: "uv_fs_sendfile", header: "uv.h".} - -proc chmod*(loop: PLoop, req: PFS, path: cstring, mode: cint, cb: FsProc): cint{. - importc: "uv_fs_chmod", header: "uv.h".} - -proc utime*(loop: PLoop, req: PFS, path: cstring, atime: cdouble, mtime: cdouble, cb: FsProc): cint{. - importc: "uv_fs_utime", header: "uv.h".} - -proc futime*(loop: PLoop, req: PFS, file: File, atime: cdouble, mtime: cdouble, cb: FsProc): cint{. - importc: "uv_fs_futime", header: "uv.h".} - -proc lstat*(loop: PLoop, req: PFS, path: cstring, cb: FsProc): cint{. - importc: "uv_fs_lstat", header: "uv.h".} - -proc link*(loop: PLoop, req: PFS, path: cstring, new_path: cstring, cb: FsProc): cint{. - importc: "uv_fs_link", header: "uv.h".} - -proc symlink*(loop: PLoop, req: PFS, path: cstring, new_path: cstring, flags: cint, cb: FsProc): cint{. - importc: "uv_fs_symlink", header: "uv.h".} - -proc readlink*(loop: PLoop, req: PFS, path: cstring, cb: FsProc): cint{. - importc: "uv_fs_readlink", header: "uv.h".} - -proc fchmod*(loop: PLoop, req: PFS, file: File, mode: cint, cb: FsProc): cint{. - importc: "uv_fs_fchmod", header: "uv.h".} - -proc chown*(loop: PLoop, req: PFS, path: cstring, uid: cint, gid: cint, cb: FsProc): cint{. - importc: "uv_fs_chown", header: "uv.h".} - -proc fchown*(loop: PLoop, req: PFS, file: File, uid: cint, gid: cint, cb: FsProc): cint{. - importc: "uv_fs_fchown", header: "uv.h".} - -proc event_init*(loop: PLoop, handle: PFSEvent, filename: cstring, cb: FsEventProc): cint{. - importc: "uv_fs_event_init", header: "uv.h".} - -proc ip4_addr*(ip: cstring, port: cint): SockAddrIn{. - importc: "uv_ip4_addr", header: "uv.h".} - -proc ip6_addr*(ip: cstring, port: cint): TSockAddrIn6{. - importc: "uv_ip6_addr", header: "uv.h".} - -proc ip4_name*(src: ptr SockAddrIn, dst: cstring, size: csize): cint{. - importc: "uv_ip4_name", header: "uv.h".} - -proc ip6_name*(src: ptr TSockAddrIn6, dst: cstring, size: csize): cint{. - importc: "uv_ip6_name", header: "uv.h".} - -proc exepath*(buffer: cstring, size: var csize): cint{. - importc: "uv_exepath", header: "uv.h".} - -proc hrtime*(): uint64{. - importc: "uv_hrtime", header: "uv.h".} - -proc loadavg*(load: var array[0..2, cdouble]) {. - importc: "uv_loadavg", header: "uv.h"} - -proc get_free_memory*(): cdouble {. - importc: "uv_get_free_memory", header: "uv.h".} - -proc get_total_memory*(): cdouble {. - importc: "uv_get_total_memory", header: "uv.h".} - diff --git a/lib/wrappers/linenoise/clinenoise.c b/lib/wrappers/linenoise/linenoise.c index b4ae32472..be792b96b 100644 --- a/lib/wrappers/linenoise/clinenoise.c +++ b/lib/wrappers/linenoise/linenoise.c @@ -1,7 +1,5 @@ -/* linenoise.c -- VERSION 1.0 - * - * Guerrilla line editing library against the idea that a line editing lib - * needs to be 20,000 lines of C code. +/* linenoise.c -- guerrilla line editing library against the idea that a + * line editing lib needs to be 20,000 lines of C code. * * You can find the latest source code at: * @@ -12,7 +10,7 @@ * * ------------------------------------------------------------------------ * - * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com> + * Copyright (c) 2010-2016, Salvatore Sanfilippo <antirez at gmail dot com> * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> * * All rights reserved. @@ -113,15 +111,18 @@ #include <string.h> #include <stdlib.h> #include <ctype.h> +#include <sys/stat.h> #include <sys/types.h> #include <sys/ioctl.h> #include <unistd.h> -#include "clinenoise.h" +#include "linenoise.h" #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 #define LINENOISE_MAX_LINE 4096 static char *unsupported_term[] = {"dumb","cons25","emacs",NULL}; static linenoiseCompletionCallback *completionCallback = NULL; +static linenoiseHintsCallback *hintsCallback = NULL; +static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ @@ -409,6 +410,18 @@ void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { completionCallback = fn; } +/* Register a hits function to be called to show hits to the user at the + * right of the prompt. */ +void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) { + hintsCallback = fn; +} + +/* Register a function to free the hints returned by the hints callback + * registered with linenoiseSetHintsCallback(). */ +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) { + freeHintsCallback = fn; +} + /* This function is used by the callback function registered by the user * in order to add completion options given the input string when the * user typed <tab>. See the example.c source code for a very easy to @@ -458,6 +471,30 @@ static void abFree(struct abuf *ab) { free(ab->b); } +/* Helper of refreshSingleLine() and refreshMultiLine() to show hints + * to the right of the prompt. */ +void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { + char seq[64]; + if (hintsCallback && plen+l->len < l->cols) { + int color = -1, bold = 0; + char *hint = hintsCallback(l->buf,&color,&bold); + if (hint) { + int hintlen = strlen(hint); + int hintmaxlen = l->cols-(plen+l->len); + if (hintlen > hintmaxlen) hintlen = hintmaxlen; + if (bold == 1 && color == -1) color = 37; + if (color != -1 || bold != 0) + snprintf(seq,64,"\033[%d;%d;49m",bold,color); + abAppend(ab,seq,strlen(seq)); + abAppend(ab,hint,hintlen); + if (color != -1 || bold != 0) + abAppend(ab,"\033[0m",4); + /* Call the function to free the hint returned. */ + if (freeHintsCallback) freeHintsCallback(hint); + } + } +} + /* Single line low level line refresh. * * Rewrite the currently edited line accordingly to the buffer content, @@ -487,6 +524,8 @@ static void refreshSingleLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); abAppend(&ab,buf,len); + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); /* Erase to right */ snprintf(seq,64,"\x1b[0K"); abAppend(&ab,seq,strlen(seq)); @@ -540,6 +579,9 @@ static void refreshMultiLine(struct linenoiseState *l) { abAppend(&ab,l->prompt,strlen(l->prompt)); abAppend(&ab,l->buf,l->len); + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + /* If we are at the very end of the screen with our prompt, we need to * emit a newline and move the prompt to the first column. */ if (l->pos && @@ -558,7 +600,7 @@ static void refreshMultiLine(struct linenoiseState *l) { rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */ lndebug("rpos2 %d", rpos2); - /* Go up till we reach the expected positon. */ + /* Go up till we reach the expected position. */ if (rows-rpos2 > 0) { lndebug("go-up %d", rows-rpos2); snprintf(seq,64,"\x1b[%dA", rows-rpos2); @@ -600,7 +642,7 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { l->pos++; l->len++; l->buf[l->len] = '\0'; - if ((!mlmode && l->plen+l->len < l->cols) /* || mlmode */) { + if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ if (write(l->ofd,&c,1) == -1) return -1; @@ -723,7 +765,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) { * when ctrl+d is typed. * * The function returns the length of the current buffer. */ -static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt, linenoiseData* data) { struct linenoiseState l; @@ -774,9 +816,18 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, history_len--; free(history[history_len]); if (mlmode) linenoiseEditMoveEnd(&l); + if (hintsCallback) { + /* Force a refresh without hints to leave the previous + * line as the user typed it after a newline. */ + linenoiseHintsCallback *hc = hintsCallback; + hintsCallback = NULL; + refreshLine(&l); + hintsCallback = hc; + } return (int)l.len; case CTRL_C: /* ctrl-c */ errno = EAGAIN; + data->status = linenoiseStatus_ctrl_C; return -1; case BACKSPACE: /* backspace */ case 8: /* ctrl-h */ @@ -789,6 +840,7 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, } else { history_len--; free(history[history_len]); + data->status = linenoiseStatus_ctrl_D; return -1; } break; @@ -929,29 +981,55 @@ void linenoisePrintKeyCodes(void) { /* This function calls the line editing function linenoiseEdit() using * the STDIN file descriptor set in raw mode. */ -static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { +static int linenoiseRaw(char *buf, size_t buflen, const char *prompt, linenoiseData* data) { int count; if (buflen == 0) { errno = EINVAL; return -1; } - if (!isatty(STDIN_FILENO)) { - /* Not a tty: read from file / pipe. */ - if (fgets(buf, buflen, stdin) == NULL) return -1; - count = strlen(buf); - if (count && buf[count-1] == '\n') { - count--; - buf[count] = '\0'; + + if (enableRawMode(STDIN_FILENO) == -1) return -1; + count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt, data); + disableRawMode(STDIN_FILENO); + printf("\n"); + return count; +} + +/* This function is called when linenoise() is called with the standard + * input file descriptor not attached to a TTY. So for example when the + * program using linenoise is called in pipe or with a file redirected + * to its standard input. In this case, we want to be able to return the + * line regardless of its length (by default we are limited to 4k). */ +static char *linenoiseNoTTY(void) { + char *line = NULL; + size_t len = 0, maxlen = 0; + + while(1) { + if (len == maxlen) { + if (maxlen == 0) maxlen = 16; + maxlen *= 2; + char *oldval = line; + line = realloc(line,maxlen); + if (line == NULL) { + if (oldval) free(oldval); + return NULL; + } + } + int c = fgetc(stdin); + if (c == EOF || c == '\n') { + if (c == EOF && len == 0) { + free(line); + return NULL; + } else { + line[len] = '\0'; + return line; + } + } else { + line[len] = c; + len++; } - } else { - /* Interactive editing. */ - if (enableRawMode(STDIN_FILENO) == -1) return -1; - count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); - disableRawMode(STDIN_FILENO); - printf("\n"); } - return count; } /* The high level function that is the main API of the linenoise library. @@ -959,11 +1037,15 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { * for a blacklist of stupid terminals, and later either calls the line * editing function or uses dummy fgets() so that you will be able to type * something even in the most desperate of the conditions. */ -char *linenoise(const char *prompt) { +char *linenoiseExtra(const char *prompt, linenoiseData* data) { char buf[LINENOISE_MAX_LINE]; int count; - if (isUnsupportedTerm()) { + if (!isatty(STDIN_FILENO)) { + /* Not a tty: read from file / pipe. In this mode we don't want any + * limit to the line size, so we call a function to handle that. */ + return linenoiseNoTTY(); + } else if (isUnsupportedTerm()) { size_t len; printf("%s",prompt); @@ -976,12 +1058,26 @@ char *linenoise(const char *prompt) { } return strdup(buf); } else { - count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt); + count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt, data); if (count == -1) return NULL; return strdup(buf); } } +char *linenoise(const char *prompt) { + linenoiseData data; + data.status = linenoiseStatus_ctrl_unknown; + return linenoiseExtra(prompt, &data); +} + +/* This is just a wrapper the user may want to call in order to make sure + * the linenoise returned buffer is freed with the same allocator it was + * created with. Useful when the main program is using an alternative + * allocator. */ +void linenoiseFree(void *ptr) { + free(ptr); +} + /* ================================ History ================================= */ /* Free the history, but does not reset it. Only used when we have to @@ -1073,10 +1169,14 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ int linenoiseHistorySave(const char *filename) { - FILE *fp = fopen(filename,"w"); + mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO); + FILE *fp; int j; + fp = fopen(filename,"w"); + umask(old_umask); if (fp == NULL) return -1; + chmod(filename,S_IRUSR|S_IWUSR); for (j = 0; j < history_len; j++) fprintf(fp,"%s\n",history[j]); fclose(fp); diff --git a/lib/wrappers/linenoise/clinenoise.h b/lib/wrappers/linenoise/linenoise.h index fbb01cfaa..aa86ccb78 100644 --- a/lib/wrappers/linenoise/clinenoise.h +++ b/lib/wrappers/linenoise/linenoise.h @@ -48,11 +48,27 @@ typedef struct linenoiseCompletions { char **cvec; } linenoiseCompletions; +typedef enum linenoiseStatus { + linenoiseStatus_ctrl_unknown, + linenoiseStatus_ctrl_C, + linenoiseStatus_ctrl_D +} linenoiseStatus; + +typedef struct linenoiseData { + linenoiseStatus status; +} linenoiseData; + typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); +typedef void(linenoiseFreeHintsCallback)(void *); void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); +void linenoiseSetHintsCallback(linenoiseHintsCallback *); +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); void linenoiseAddCompletion(linenoiseCompletions *, const char *); char *linenoise(const char *prompt); +char *linenoiseExtra(const char *prompt, linenoiseData* data); +void linenoiseFree(void *ptr); int linenoiseHistoryAdd(const char *line); int linenoiseHistorySetMaxLen(int len); int linenoiseHistorySave(const char *filename); diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index b7004c6e2..186b3b252 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -9,14 +9,14 @@ type Completions* = object - len*: csize + len*: csize_t cvec*: cstringArray CompletionCallback* = proc (a2: cstring; a3: ptr Completions) {.cdecl.} -{.compile: "clinenoise.c".} +{.compile: "linenoise.c".} -proc setCompletionCallback*(a2: ptr CompletionCallback) {. +proc setCompletionCallback*(a2: CompletionCallback) {. importc: "linenoiseSetCompletionCallback".} proc addCompletion*(a2: ptr Completions; a3: cstring) {. importc: "linenoiseAddCompletion".} @@ -32,3 +32,41 @@ proc printKeyCodes*() {.importc: "linenoisePrintKeyCodes".} proc free*(s: cstring) {.importc: "free", header: "<stdlib.h>".} +when defined(nimExperimentalLinenoiseExtra) and not defined(windows): + # C interface + type LinenoiseStatus = enum + linenoiseStatus_ctrl_unknown + linenoiseStatus_ctrl_C + linenoiseStatus_ctrl_D + + type LinenoiseData* = object + status: LinenoiseStatus + + proc linenoiseExtra(prompt: cstring, data: ptr LinenoiseData): cstring {.importc.} + + # stable nim interface + type Status* = enum + lnCtrlUnkown + lnCtrlC + lnCtrlD + + type ReadLineResult* = object + line*: string + status*: Status + + proc readLineStatus*(prompt: string, result: var ReadLineResult) = + ## line editing API that allows returning the line entered and an indicator + ## of which control key was entered, allowing user to distinguish between + ## for example ctrl-C vs ctrl-D. + runnableExamples("-d:nimExperimentalLinenoiseExtra -r:off"): + var ret: ReadLineResult + while true: + readLineStatus("name: ", ret) # ctrl-D will exit, ctrl-C will go to next prompt + if ret.line.len > 0: echo ret.line + if ret.status == lnCtrlD: break + echo "exiting" + var data: LinenoiseData + let buf = linenoiseExtra(prompt, data.addr) + result.line = $buf + free(buf) + result.status = data.status.ord.Status diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim deleted file mode 100644 index af504864d..000000000 --- a/lib/wrappers/mysql.nim +++ /dev/null @@ -1,1112 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} -{.push, callconv: cdecl.} - -when defined(Unix): - when defined(macosx): - const - lib = "libmysqlclient.(15|16|17|18).dylib" - else: - const - lib = "libmysqlclient.so.(15|16|17|18)" -when defined(Windows): - const - lib = "libmysql.dll" -type - my_bool* = bool - Pmy_bool* = ptr my_bool - PVIO* = pointer - Pgptr* = ptr gptr - gptr* = cstring - Pmy_socket* = ptr my_socket - my_socket* = cint - PPByte* = pointer - cuint* = cint - -# ------------ Start of declaration in "mysql_com.h" --------------------- -# -# ** Common definition between mysql server & client -# -# Field/table name length - -const - NAME_LEN* = 64 - HOSTNAME_LENGTH* = 60 - USERNAME_LENGTH* = 16 - SERVER_VERSION_LENGTH* = 60 - SQLSTATE_LENGTH* = 5 - LOCAL_HOST* = "localhost" - LOCAL_HOST_NAMEDPIPE* = '.' - -const - NAMEDPIPE* = "MySQL" - SERVICENAME* = "MySQL" - -type - Enum_server_command* = enum - COM_SLEEP, COM_QUIT, COM_INIT_DB, COM_QUERY, COM_FIELD_LIST, COM_CREATE_DB, - COM_DROP_DB, COM_REFRESH, COM_SHUTDOWN, COM_STATISTICS, COM_PROCESS_INFO, - COM_CONNECT, COM_PROCESS_KILL, COM_DEBUG, COM_PING, COM_TIME, - COM_DELAYED_INSERT, COM_CHANGE_USER, COM_BINLOG_DUMP, COM_TABLE_DUMP, - COM_CONNECT_OUT, COM_REGISTER_SLAVE, COM_STMT_PREPARE, COM_STMT_EXECUTE, - COM_STMT_SEND_LONG_DATA, COM_STMT_CLOSE, COM_STMT_RESET, COM_SET_OPTION, - COM_STMT_FETCH, COM_END -{.deprecated: [Tenum_server_command: Enum_server_command].} - -const - SCRAMBLE_LENGTH* = 20 # Length of random string sent by server on handshake; - # this is also length of obfuscated password, - # received from client - SCRAMBLE_LENGTH_323* = 8 # length of password stored in the db: - # new passwords are preceded with '*' - SCRAMBLED_PASSWORD_CHAR_LENGTH* = SCRAMBLE_LENGTH * 2 + 1 - SCRAMBLED_PASSWORD_CHAR_LENGTH_323* = SCRAMBLE_LENGTH_323 * 2 - NOT_NULL_FLAG* = 1 # Field can't be NULL - PRI_KEY_FLAG* = 2 # Field is part of a primary key - UNIQUE_KEY_FLAG* = 4 # Field is part of a unique key - MULTIPLE_KEY_FLAG* = 8 # Field is part of a key - BLOB_FLAG* = 16 # Field is a blob - UNSIGNED_FLAG* = 32 # Field is unsigned - ZEROFILL_FLAG* = 64 # Field is zerofill - BINARY_FLAG* = 128 # Field is binary - # The following are only sent to new clients - ENUM_FLAG* = 256 # field is an enum - AUTO_INCREMENT_FLAG* = 512 # field is a autoincrement field - TIMESTAMP_FLAG* = 1024 # Field is a timestamp - SET_FLAG* = 2048 # field is a set - NO_DEFAULT_VALUE_FLAG* = 4096 # Field doesn't have default value - NUM_FLAG* = 32768 # Field is num (for clients) - PART_KEY_FLAG* = 16384 # Intern; Part of some key - GROUP_FLAG* = 32768 # Intern: Group field - UNIQUE_FLAG* = 65536 # Intern: Used by sql_yacc - BINCMP_FLAG* = 131072 # Intern: Used by sql_yacc - REFRESH_GRANT* = 1 # Refresh grant tables - REFRESH_LOG* = 2 # Start on new log file - REFRESH_TABLES* = 4 # close all tables - REFRESH_HOSTS* = 8 # Flush host cache - REFRESH_STATUS* = 16 # Flush status variables - REFRESH_THREADS* = 32 # Flush thread cache - REFRESH_SLAVE* = 64 # Reset master info and restart slave thread - REFRESH_MASTER* = 128 # Remove all bin logs in the index and truncate the index - # The following can't be set with mysql_refresh() - REFRESH_READ_LOCK* = 16384 # Lock tables for read - REFRESH_FAST* = 32768 # Intern flag - REFRESH_QUERY_CACHE* = 65536 # RESET (remove all queries) from query cache - REFRESH_QUERY_CACHE_FREE* = 0x00020000 # pack query cache - REFRESH_DES_KEY_FILE* = 0x00040000 - REFRESH_USER_RESOURCES* = 0x00080000 - CLIENT_LONG_PASSWORD* = 1 # new more secure passwords - CLIENT_FOUND_ROWS* = 2 # Found instead of affected rows - CLIENT_LONG_FLAG* = 4 # Get all column flags - CLIENT_CONNECT_WITH_DB* = 8 # One can specify db on connect - CLIENT_NO_SCHEMA* = 16 # Don't allow database.table.column - CLIENT_COMPRESS* = 32 # Can use compression protocol - CLIENT_ODBC* = 64 # Odbc client - CLIENT_LOCAL_FILES* = 128 # Can use LOAD DATA LOCAL - CLIENT_IGNORE_SPACE* = 256 # Ignore spaces before '(' - CLIENT_PROTOCOL_41* = 512 # New 4.1 protocol - CLIENT_INTERACTIVE* = 1024 # This is an interactive client - CLIENT_SSL* = 2048 # Switch to SSL after handshake - CLIENT_IGNORE_SIGPIPE* = 4096 # IGNORE sigpipes - CLIENT_TRANSACTIONS* = 8192 # Client knows about transactions - CLIENT_RESERVED* = 16384 # Old flag for 4.1 protocol - CLIENT_SECURE_CONNECTION* = 32768 # New 4.1 authentication - CLIENT_MULTI_STATEMENTS* = 65536 # Enable/disable multi-stmt support - CLIENT_MULTI_RESULTS* = 131072 # Enable/disable multi-results - CLIENT_REMEMBER_OPTIONS*: int = 1 shl 31 - SERVER_STATUS_IN_TRANS* = 1 # Transaction has started - SERVER_STATUS_AUTOCOMMIT* = 2 # Server in auto_commit mode - SERVER_STATUS_MORE_RESULTS* = 4 # More results on server - SERVER_MORE_RESULTS_EXISTS* = 8 # Multi query - next query exists - SERVER_QUERY_NO_GOOD_INDEX_USED* = 16 - SERVER_QUERY_NO_INDEX_USED* = 32 # The server was able to fulfill the clients request and opened a - # read-only non-scrollable cursor for a query. This flag comes - # in reply to COM_STMT_EXECUTE and COM_STMT_FETCH commands. - SERVER_STATUS_CURSOR_EXISTS* = 64 # This flag is sent when a read-only cursor is exhausted, in reply to - # COM_STMT_FETCH command. - SERVER_STATUS_LAST_ROW_SENT* = 128 - SERVER_STATUS_DB_DROPPED* = 256 # A database was dropped - SERVER_STATUS_NO_BACKSLASH_ESCAPES* = 512 - ERRMSG_SIZE* = 200 - NET_READ_TIMEOUT* = 30 # Timeout on read - NET_WRITE_TIMEOUT* = 60 # Timeout on write - NET_WAIT_TIMEOUT* = 8 * 60 * 60 # Wait for new query - ONLY_KILL_QUERY* = 1 - -const - MAX_TINYINT_WIDTH* = 3 # Max width for a TINY w.o. sign - MAX_SMALLINT_WIDTH* = 5 # Max width for a SHORT w.o. sign - MAX_MEDIUMINT_WIDTH* = 8 # Max width for a INT24 w.o. sign - MAX_INT_WIDTH* = 10 # Max width for a LONG w.o. sign - MAX_BIGINT_WIDTH* = 20 # Max width for a LONGLONG - MAX_CHAR_WIDTH* = 255 # Max length for a CHAR column - MAX_BLOB_WIDTH* = 8192 # Default width for blob - -type - Pst_net* = ptr St_net - St_net*{.final.} = object - vio*: PVio - buff*: cstring - buff_end*: cstring - write_pos*: cstring - read_pos*: cstring - fd*: my_socket # For Perl DBI/dbd - max_packet*: int - max_packet_size*: int - pkt_nr*: cuint - compress_pkt_nr*: cuint - write_timeout*: cuint - read_timeout*: cuint - retry_count*: cuint - fcntl*: cint - compress*: my_bool # The following variable is set if we are doing several queries in one - # command ( as in LOAD TABLE ... FROM MASTER ), - # and do not want to confuse the client with OK at the wrong time - remain_in_buf*: int - len*: int - buf_length*: int - where_b*: int - return_status*: ptr cint - reading_or_writing*: char - save_char*: cchar - no_send_ok*: my_bool # For SPs and other things that do multiple stmts - no_send_eof*: my_bool # For SPs' first version read-only cursors - no_send_error*: my_bool # Set if OK packet is already sent, and - # we do not need to send error messages - # Pointer to query object in query cache, do not equal NULL (0) for - # queries in cache that have not stored its results yet - # $endif - last_error*: array[0..(ERRMSG_SIZE) - 1, char] - sqlstate*: array[0..(SQLSTATE_LENGTH + 1) - 1, char] - last_errno*: cuint - error*: char - query_cache_query*: gptr - report_error*: my_bool # We should report error (we have unreported error) - return_errno*: my_bool - - NET* = St_net - PNET* = ptr NET -{.deprecated: [Tst_net: St_net, TNET: NET].} - -const - packet_error* = - 1 - -type - Enum_field_types* = enum # For backward compatibility - TYPE_DECIMAL, TYPE_TINY, TYPE_SHORT, TYPE_LONG, TYPE_FLOAT, TYPE_DOUBLE, - TYPE_NULL, TYPE_TIMESTAMP, TYPE_LONGLONG, TYPE_INT24, TYPE_DATE, TYPE_TIME, - TYPE_DATETIME, TYPE_YEAR, TYPE_NEWDATE, TYPE_VARCHAR, TYPE_BIT, - TYPE_NEWDECIMAL = 246, TYPE_ENUM = 247, TYPE_SET = 248, - TYPE_TINY_BLOB = 249, TYPE_MEDIUM_BLOB = 250, TYPE_LONG_BLOB = 251, - TYPE_BLOB = 252, TYPE_VAR_STRING = 253, TYPE_STRING = 254, - TYPE_GEOMETRY = 255 -{.deprecated: [Tenum_field_types: Enum_field_types].} - -const - CLIENT_MULTI_QUERIES* = CLIENT_MULTI_STATEMENTS - FIELD_TYPE_DECIMAL* = TYPE_DECIMAL - FIELD_TYPE_NEWDECIMAL* = TYPE_NEWDECIMAL - FIELD_TYPE_TINY* = TYPE_TINY - FIELD_TYPE_SHORT* = TYPE_SHORT - FIELD_TYPE_LONG* = TYPE_LONG - FIELD_TYPE_FLOAT* = TYPE_FLOAT - FIELD_TYPE_DOUBLE* = TYPE_DOUBLE - FIELD_TYPE_NULL* = TYPE_NULL - FIELD_TYPE_TIMESTAMP* = TYPE_TIMESTAMP - FIELD_TYPE_LONGLONG* = TYPE_LONGLONG - FIELD_TYPE_INT24* = TYPE_INT24 - FIELD_TYPE_DATE* = TYPE_DATE - FIELD_TYPE_TIME* = TYPE_TIME - FIELD_TYPE_DATETIME* = TYPE_DATETIME - FIELD_TYPE_YEAR* = TYPE_YEAR - FIELD_TYPE_NEWDATE* = TYPE_NEWDATE - FIELD_TYPE_ENUM* = TYPE_ENUM - FIELD_TYPE_SET* = TYPE_SET - FIELD_TYPE_TINY_BLOB* = TYPE_TINY_BLOB - FIELD_TYPE_MEDIUM_BLOB* = TYPE_MEDIUM_BLOB - FIELD_TYPE_LONG_BLOB* = TYPE_LONG_BLOB - FIELD_TYPE_BLOB* = TYPE_BLOB - FIELD_TYPE_VAR_STRING* = TYPE_VAR_STRING - FIELD_TYPE_STRING* = TYPE_STRING - FIELD_TYPE_CHAR* = TYPE_TINY - FIELD_TYPE_INTERVAL* = TYPE_ENUM - FIELD_TYPE_GEOMETRY* = TYPE_GEOMETRY - FIELD_TYPE_BIT* = TYPE_BIT # Shutdown/kill enums and constants - # Bits for THD::killable. - SHUTDOWN_KILLABLE_CONNECT* = chr(1 shl 0) - SHUTDOWN_KILLABLE_TRANS* = chr(1 shl 1) - SHUTDOWN_KILLABLE_LOCK_TABLE* = chr(1 shl 2) - SHUTDOWN_KILLABLE_UPDATE* = chr(1 shl 3) - -type - Enum_shutdown_level* = enum - SHUTDOWN_DEFAULT = 0, SHUTDOWN_WAIT_CONNECTIONS = 1, - SHUTDOWN_WAIT_TRANSACTIONS = 2, SHUTDOWN_WAIT_UPDATES = 8, - SHUTDOWN_WAIT_ALL_BUFFERS = 16, SHUTDOWN_WAIT_CRITICAL_BUFFERS = 17, - KILL_QUERY = 254, KILL_CONNECTION = 255 - Enum_cursor_type* = enum # options for mysql_set_option - CURSOR_TYPE_NO_CURSOR = 0, CURSOR_TYPE_READ_ONLY = 1, - CURSOR_TYPE_FOR_UPDATE = 2, CURSOR_TYPE_SCROLLABLE = 4 - Enum_mysql_set_option* = enum - OPTION_MULTI_STATEMENTS_ON, OPTION_MULTI_STATEMENTS_OFF -{.deprecated: [Tenum_shutdown_level: Enum_shutdown_level, - Tenum_cursor_type: Enum_cursor_type, - Tenum_mysql_set_option: Enum_mysql_set_option].} - -proc my_net_init*(net: PNET, vio: PVio): my_bool{.cdecl, dynlib: lib, - importc: "my_net_init".} -proc my_net_local_init*(net: PNET){.cdecl, dynlib: lib, - importc: "my_net_local_init".} -proc net_end*(net: PNET){.cdecl, dynlib: lib, importc: "net_end".} -proc net_clear*(net: PNET){.cdecl, dynlib: lib, importc: "net_clear".} -proc net_realloc*(net: PNET, len: int): my_bool{.cdecl, dynlib: lib, - importc: "net_realloc".} -proc net_flush*(net: PNET): my_bool{.cdecl, dynlib: lib, importc: "net_flush".} -proc my_net_write*(net: PNET, packet: cstring, length: int): my_bool{.cdecl, - dynlib: lib, importc: "my_net_write".} -proc net_write_command*(net: PNET, command: char, header: cstring, - head_len: int, packet: cstring, length: int): my_bool{. - cdecl, dynlib: lib, importc: "net_write_command".} -proc net_real_write*(net: PNET, packet: cstring, length: int): cint{.cdecl, - dynlib: lib, importc: "net_real_write".} -proc my_net_read*(net: PNET): int{.cdecl, dynlib: lib, importc: "my_net_read".} - # The following function is not meant for normal usage - # Currently it's used internally by manager.c -type - Psockaddr* = ptr Sockaddr - Sockaddr*{.final.} = object # undefined structure -{.deprecated: [Tsockaddr: Sockaddr].} - -proc my_connect*(s: my_socket, name: Psockaddr, namelen: cuint, timeout: cuint): cint{. - cdecl, dynlib: lib, importc: "my_connect".} -type - Prand_struct* = ptr Rand_struct - Rand_struct*{.final.} = object # The following is for user defined functions - seed1*: int - seed2*: int - max_value*: int - max_value_dbl*: cdouble - - Item_result* = enum - STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT, DECIMAL_RESULT - PItem_result* = ptr Item_result - Pst_udf_args* = ptr St_udf_args - St_udf_args*{.final.} = object - arg_count*: cuint # Number of arguments - arg_type*: PItem_result # Pointer to item_results - args*: cstringArray # Pointer to item_results - lengths*: ptr int # Length of string arguments - maybe_null*: cstring # Length of string arguments - attributes*: cstringArray # Pointer to attribute name - attribute_lengths*: ptr int # Length of attribute arguments - - UDF_ARGS* = St_udf_args - PUDF_ARGS* = ptr UDF_ARGS # This holds information about the result - Pst_udf_init* = ptr St_udf_init - St_udf_init*{.final.} = object - maybe_null*: my_bool # 1 if function can return NULL - decimals*: cuint # for real functions - max_length*: int # For string functions - theptr*: cstring # free pointer for function data - const_item*: my_bool # free pointer for function data - - UDF_INIT* = St_udf_init - PUDF_INIT* = ptr UDF_INIT # Constants when using compression -{.deprecated: [Trand_stuct: Rand_struct, TItem_result: Item_result, - Tst_udf_args: St_udf_args, TUDF_ARGS: UDF_ARGS, - Tst_udf_init: St_udf_init, TUDF_INIT: UDF_INIT].} - -const - NET_HEADER_SIZE* = 4 # standard header size - COMP_HEADER_SIZE* = 3 # compression header extra size - # Prototypes to password functions - # These functions are used for authentication by client and server and - # implemented in sql/password.c - -proc randominit*(para1: Prand_struct, seed1: int, seed2: int){.cdecl, - dynlib: lib, importc: "randominit".} -proc my_rnd*(para1: Prand_struct): cdouble{.cdecl, dynlib: lib, - importc: "my_rnd".} -proc create_random_string*(fto: cstring, len: cuint, rand_st: Prand_struct){. - cdecl, dynlib: lib, importc: "create_random_string".} -proc hash_password*(fto: int, password: cstring, password_len: cuint){.cdecl, - dynlib: lib, importc: "hash_password".} -proc make_scrambled_password_323*(fto: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "make_scrambled_password_323".} -proc scramble_323*(fto: cstring, message: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "scramble_323".} -proc check_scramble_323*(para1: cstring, message: cstring, salt: int): my_bool{. - cdecl, dynlib: lib, importc: "check_scramble_323".} -proc get_salt_from_password_323*(res: ptr int, password: cstring){.cdecl, - dynlib: lib, importc: "get_salt_from_password_323".} -proc make_password_from_salt_323*(fto: cstring, salt: ptr int){.cdecl, - dynlib: lib, importc: "make_password_from_salt_323".} -proc octet2hex*(fto: cstring, str: cstring, length: cuint): cstring{.cdecl, - dynlib: lib, importc: "octet2hex".} -proc make_scrambled_password*(fto: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "make_scrambled_password".} -proc scramble*(fto: cstring, message: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "scramble".} -proc check_scramble*(reply: cstring, message: cstring, hash_stage2: pointer): my_bool{. - cdecl, dynlib: lib, importc: "check_scramble".} -proc get_salt_from_password*(res: pointer, password: cstring){.cdecl, - dynlib: lib, importc: "get_salt_from_password".} -proc make_password_from_salt*(fto: cstring, hash_stage2: pointer){.cdecl, - dynlib: lib, importc: "make_password_from_salt".} - # end of password.c -proc get_tty_password*(opt_message: cstring): cstring{.cdecl, dynlib: lib, - importc: "get_tty_password".} -proc errno_to_sqlstate*(errno: cuint): cstring{.cdecl, dynlib: lib, - importc: "mysql_errno_to_sqlstate".} - # Some other useful functions -proc modify_defaults_file*(file_location: cstring, option: cstring, - option_value: cstring, section_name: cstring, - remove_option: cint): cint{.cdecl, dynlib: lib, - importc: "load_defaults".} -proc load_defaults*(conf_file: cstring, groups: cstringArray, argc: ptr cint, - argv: ptr cstringArray): cint{.cdecl, dynlib: lib, - importc: "load_defaults".} -proc my_init*(): my_bool{.cdecl, dynlib: lib, importc: "my_init".} -proc my_thread_init*(): my_bool{.cdecl, dynlib: lib, importc: "my_thread_init".} -proc my_thread_end*(){.cdecl, dynlib: lib, importc: "my_thread_end".} -const - NULL_LENGTH*: int = int(not (0)) # For net_store_length - -const - STMT_HEADER* = 4 - LONG_DATA_HEADER* = 6 # ------------ Stop of declaration in "mysql_com.h" ----------------------- - # $include "mysql_time.h" - # $include "mysql_version.h" - # $include "typelib.h" - # $include "my_list.h" /* for LISTs used in 'MYSQL' and 'MYSQL_STMT' */ - # var - # mysql_port : cuint;cvar;external; - # mysql_unix_port : Pchar;cvar;external; - -const - CLIENT_NET_READ_TIMEOUT* = 365 * 24 * 3600 # Timeout on read - CLIENT_NET_WRITE_TIMEOUT* = 365 * 24 * 3600 # Timeout on write - -type - Pst_mysql_field* = ptr St_mysql_field - St_mysql_field*{.final.} = object - name*: cstring # Name of column - org_name*: cstring # Original column name, if an alias - table*: cstring # Table of column if column was a field - org_table*: cstring # Org table name, if table was an alias - db*: cstring # Database for table - catalog*: cstring # Catalog for table - def*: cstring # Default value (set by mysql_list_fields) - len*: int # Width of column (create length) - max_length*: int # Max width for selected set - name_length*: cuint - org_name_length*: cuint - table_length*: cuint - org_table_length*: cuint - db_length*: cuint - catalog_length*: cuint - def_length*: cuint - flags*: cuint # Div flags - decimals*: cuint # Number of decimals in field - charsetnr*: cuint # Character set - ftype*: Enum_field_types # Type of field. See mysql_com.h for types - extension*: pointer - - FIELD* = St_mysql_field - PFIELD* = ptr FIELD - PROW* = ptr ROW # return data as array of strings - ROW* = cstringArray - PFIELD_OFFSET* = ptr FIELD_OFFSET # offset to current field - FIELD_OFFSET* = cuint -{.deprecated: [Tst_mysql_field: St_mysql_field, TFIELD: FIELD, TROW: ROW, - TFIELD_OFFSET: FIELD_OFFSET].} - -proc IS_PRI_KEY*(n: int32): bool -proc IS_NOT_NULL*(n: int32): bool -proc IS_BLOB*(n: int32): bool -proc IS_NUM*(t: Enum_field_types): bool -proc INTERNAL_NUM_FIELD*(f: Pst_mysql_field): bool -proc IS_NUM_FIELD*(f: Pst_mysql_field): bool -type - my_ulonglong* = int64 - Pmy_ulonglong* = ptr my_ulonglong - -const - COUNT_ERROR* = not (my_ulonglong(0)) - -type - Pst_mysql_rows* = ptr St_mysql_rows - St_mysql_rows*{.final.} = object - next*: Pst_mysql_rows # list of rows - data*: ROW - len*: int - - ROWS* = St_mysql_rows - PROWS* = ptr ROWS - PROW_OFFSET* = ptr ROW_OFFSET # offset to current row - ROW_OFFSET* = ROWS -{.deprecated: [Tst_mysql_rows: St_mysql_rows, TROWS: ROWS, - TROW_OFFSET: ROW_OFFSET].} - -const - ALLOC_MAX_BLOCK_TO_DROP* = 4096 - ALLOC_MAX_BLOCK_USAGE_BEFORE_DROP* = 10 # struct for once_alloc (block) - -type - Pst_used_mem* = ptr St_used_mem - St_used_mem*{.final.} = object - next*: Pst_used_mem # Next block in use - left*: cuint # memory left in block - size*: cuint # size of block - - USED_MEM* = St_used_mem - PUSED_MEM* = ptr USED_MEM - Pst_mem_root* = ptr St_mem_root - St_mem_root*{.final.} = object - free*: PUSED_MEM # blocks with free memory in it - used*: PUSED_MEM # blocks almost without free memory - pre_alloc*: PUSED_MEM # preallocated block - min_malloc*: cuint # if block have less memory it will be put in 'used' list - block_size*: cuint # initial block size - block_num*: cuint # allocated blocks counter - # first free block in queue test counter (if it exceed - # MAX_BLOCK_USAGE_BEFORE_DROP block will be dropped in 'used' list) - first_block_usage*: cuint - error_handler*: proc (){.cdecl.} - - MEM_ROOT* = St_mem_root - PMEM_ROOT* = ptr MEM_ROOT # ------------ Stop of declaration in "my_alloc.h" ---------------------- -{.deprecated: [Tst_used_mem: St_used_mem, TUSED_MEM: USED_MEM, - Tst_mem_root: St_mem_root, TMEM_ROOT: MEM_ROOT].} - -type - Pst_mysql_data* = ptr St_mysql_data - St_mysql_data*{.final.} = object - rows*: my_ulonglong - fields*: cuint - data*: PROWS - alloc*: MEM_ROOT - prev_ptr*: ptr PROWS - - DATA* = St_mysql_data - PDATA* = ptr DATA - Option* = enum - OPT_CONNECT_TIMEOUT, OPT_COMPRESS, OPT_NAMED_PIPE, INIT_COMMAND, - READ_DEFAULT_FILE, READ_DEFAULT_GROUP, SET_CHARSET_DIR, SET_CHARSET_NAME, - OPT_LOCAL_INFILE, OPT_PROTOCOL, SHARED_MEMORY_BASE_NAME, OPT_READ_TIMEOUT, - OPT_WRITE_TIMEOUT, OPT_USE_RESULT, OPT_USE_REMOTE_CONNECTION, - OPT_USE_EMBEDDED_CONNECTION, OPT_GUESS_CONNECTION, SET_CLIENT_IP, - SECURE_AUTH, REPORT_DATA_TRUNCATION, OPT_RECONNECT -{.deprecated: [Tst_mysql_data: St_mysql_data, TDATA: DATA, Toption: Option].} - -const - MAX_MYSQL_MANAGER_ERR* = 256 - MAX_MYSQL_MANAGER_MSG* = 256 - MANAGER_OK* = 200 - MANAGER_INFO* = 250 - MANAGER_ACCESS* = 401 - MANAGER_CLIENT_ERR* = 450 - MANAGER_INTERNAL_ERR* = 500 - -type - St_dynamic_array*{.final.} = object - buffer*: cstring - elements*: cuint - max_element*: cuint - alloc_increment*: cuint - size_of_element*: cuint - - DYNAMIC_ARRAY* = St_dynamic_array - Pst_dynamic_array* = ptr St_dynamic_array - Pst_mysql_options* = ptr St_mysql_options - St_mysql_options*{.final.} = object - connect_timeout*: cuint - read_timeout*: cuint - write_timeout*: cuint - port*: cuint - protocol*: cuint - client_flag*: int - host*: cstring - user*: cstring - password*: cstring - unix_socket*: cstring - db*: cstring - init_commands*: Pst_dynamic_array - my_cnf_file*: cstring - my_cnf_group*: cstring - charset_dir*: cstring - charset_name*: cstring - ssl_key*: cstring # PEM key file - ssl_cert*: cstring # PEM cert file - ssl_ca*: cstring # PEM CA file - ssl_capath*: cstring # PEM directory of CA-s? - ssl_cipher*: cstring # cipher to use - shared_memory_base_name*: cstring - max_allowed_packet*: int - use_ssl*: my_bool # if to use SSL or not - compress*: my_bool - named_pipe*: my_bool # On connect, find out the replication role of the server, and - # establish connections to all the peers - rpl_probe*: my_bool # Each call to mysql_real_query() will parse it to tell if it is a read - # or a write, and direct it to the slave or the master - rpl_parse*: my_bool # If set, never read from a master, only from slave, when doing - # a read that is replication-aware - no_master_reads*: my_bool - separate_thread*: my_bool - methods_to_use*: Option - client_ip*: cstring - secure_auth*: my_bool # Refuse client connecting to server if it uses old (pre-4.1.1) protocol - report_data_truncation*: my_bool # 0 - never report, 1 - always report (default) - # function pointers for local infile support - local_infile_init*: proc (para1: var pointer, para2: cstring, para3: pointer): cint{. - cdecl.} - local_infile_read*: proc (para1: pointer, para2: cstring, para3: cuint): cint - local_infile_end*: proc (para1: pointer) - local_infile_error*: proc (para1: pointer, para2: cstring, para3: cuint): cint - local_infile_userdata*: pointer - - Status* = enum - STATUS_READY, STATUS_GET_RESULT, STATUS_USE_RESULT - Protocol_type* = enum # There are three types of queries - the ones that have to go to - # the master, the ones that go to a slave, and the administrative - # type which must happen on the pivot connectioin - PROTOCOL_DEFAULT, PROTOCOL_TCP, PROTOCOL_SOCKET, PROTOCOL_PIPE, - PROTOCOL_MEMORY - Rpl_type* = enum - RPL_MASTER, RPL_SLAVE, RPL_ADMIN - Charset_info_st*{.final.} = object - number*: cuint - primary_number*: cuint - binary_number*: cuint - state*: cuint - csname*: cstring - name*: cstring - comment*: cstring - tailoring*: cstring - ftype*: cstring - to_lower*: cstring - to_upper*: cstring - sort_order*: cstring - contractions*: ptr int16 - sort_order_big*: ptr ptr int16 - tab_to_uni*: ptr int16 - tab_from_uni*: pointer # was ^MY_UNI_IDX - state_map*: cstring - ident_map*: cstring - strxfrm_multiply*: cuint - mbminlen*: cuint - mbmaxlen*: cuint - min_sort_char*: int16 - max_sort_char*: int16 - escape_with_backslash_is_dangerous*: my_bool - cset*: pointer # was ^MY_CHARSET_HANDLER - coll*: pointer # was ^MY_COLLATION_HANDLER; - - CHARSET_INFO* = Charset_info_st - Pcharset_info_st* = ptr Charset_info_st - Pcharacter_set* = ptr Character_set - Character_set*{.final.} = object - number*: cuint - state*: cuint - csname*: cstring - name*: cstring - comment*: cstring - dir*: cstring - mbminlen*: cuint - mbmaxlen*: cuint - - MY_CHARSET_INFO* = Character_set - PMY_CHARSET_INFO* = ptr MY_CHARSET_INFO - Pst_mysql_methods* = ptr St_mysql_methods - Pst_mysql* = ptr St_mysql - St_mysql*{.final.} = object - net*: NET # Communication parameters - connector_fd*: gptr # ConnectorFd for SSL - host*: cstring - user*: cstring - passwd*: cstring - unix_socket*: cstring - server_version*: cstring - host_info*: cstring - info*: cstring - db*: cstring - charset*: Pcharset_info_st - fields*: PFIELD - field_alloc*: MEM_ROOT - affected_rows*: my_ulonglong - insert_id*: my_ulonglong # id if insert on table with NEXTNR - extra_info*: my_ulonglong # Used by mysqlshow, not used by mysql 5.0 and up - thread_id*: int # Id for connection in server - packet_length*: int - port*: cuint - client_flag*: int - server_capabilities*: int - protocol_version*: cuint - field_count*: cuint - server_status*: cuint - server_language*: cuint - warning_count*: cuint - options*: St_mysql_options - status*: Status - free_me*: my_bool # If free in mysql_close - reconnect*: my_bool # set to 1 if automatic reconnect - scramble*: array[0..(SCRAMBLE_LENGTH + 1) - 1, char] # session-wide random string - # Set if this is the original connection, not a master or a slave we have - # added though mysql_rpl_probe() or mysql_set_master()/ mysql_add_slave() - rpl_pivot*: my_bool # Pointers to the master, and the next slave connections, points to - # itself if lone connection. - master*: Pst_mysql - next_slave*: Pst_mysql - last_used_slave*: Pst_mysql # needed for round-robin slave pick - last_used_con*: Pst_mysql # needed for send/read/store/use result to work correctly with replication - stmts*: pointer # was PList, list of all statements - methods*: Pst_mysql_methods - thd*: pointer # Points to boolean flag in MYSQL_RES or MYSQL_STMT. We set this flag - # from mysql_stmt_close if close had to cancel result set of this object. - unbuffered_fetch_owner*: Pmy_bool - - MySQL* = St_mysql - PMySQL* = ptr MySQL - Pst_mysql_res* = ptr St_mysql_res - St_mysql_res*{.final.} = object - row_count*: my_ulonglong - fields*: PFIELD - data*: PDATA - data_cursor*: PROWS - lengths*: ptr int # column lengths of current row - handle*: PMySQL # for unbuffered reads - field_alloc*: MEM_ROOT - field_count*: cuint - current_field*: cuint - row*: ROW # If unbuffered read - current_row*: ROW # buffer to current row - eof*: my_bool # Used by mysql_fetch_row - unbuffered_fetch_cancelled*: my_bool # mysql_stmt_close() had to cancel this result - methods*: Pst_mysql_methods - - RES* = St_mysql_res - PRES* = ptr RES - Pst_mysql_stmt* = ptr St_mysql_stmt - PSTMT* = ptr STMT - St_mysql_methods*{.final.} = object - read_query_result*: proc (MySQL: PMySQL): my_bool{.cdecl.} - advanced_command*: proc (MySQL: PMySQL, command: Enum_server_command, header: cstring, - header_length: int, arg: cstring, arg_length: int, - skip_check: my_bool): my_bool - read_rows*: proc (MySQL: PMySQL, fields: PFIELD, fields_count: cuint): PDATA - use_result*: proc (MySQL: PMySQL): PRES - fetch_lengths*: proc (fto: ptr int, column: ROW, field_count: cuint) - flush_use_result*: proc (MySQL: PMySQL) - list_fields*: proc (MySQL: PMySQL): PFIELD - read_prepare_result*: proc (MySQL: PMySQL, stmt: PSTMT): my_bool - stmt_execute*: proc (stmt: PSTMT): cint - read_binary_rows*: proc (stmt: PSTMT): cint - unbuffered_fetch*: proc (MySQL: PMySQL, row: cstringArray): cint - free_embedded_thd*: proc (MySQL: PMySQL) - read_statistics*: proc (MySQL: PMySQL): cstring - next_result*: proc (MySQL: PMySQL): my_bool - read_change_user_result*: proc (MySQL: PMySQL, buff: cstring, passwd: cstring): cint - read_rowsfrom_cursor*: proc (stmt: PSTMT): cint - - METHODS* = St_mysql_methods - PMETHODS* = ptr METHODS - Pst_mysql_manager* = ptr St_mysql_manager - St_mysql_manager*{.final.} = object - net*: NET - host*: cstring - user*: cstring - passwd*: cstring - port*: cuint - free_me*: my_bool - eof*: my_bool - cmd_status*: cint - last_errno*: cint - net_buf*: cstring - net_buf_pos*: cstring - net_data_end*: cstring - net_buf_size*: cint - last_error*: array[0..(MAX_MYSQL_MANAGER_ERR) - 1, char] - - MANAGER* = St_mysql_manager - PMANAGER* = ptr MANAGER - Pst_mysql_parameters* = ptr St_mysql_parameters - St_mysql_parameters*{.final.} = object - p_max_allowed_packet*: ptr int - p_net_buffer_length*: ptr int - - PARAMETERS* = St_mysql_parameters - PPARAMETERS* = ptr PARAMETERS - Enum_mysql_stmt_state* = enum - STMT_INIT_DONE = 1, STMT_PREPARE_DONE, STMT_EXECUTE_DONE, STMT_FETCH_DONE - Pst_mysql_bind* = ptr St_mysql_bind - St_mysql_bind*{.final.} = object - len*: int # output length pointer - is_null*: Pmy_bool # Pointer to null indicator - buffer*: pointer # buffer to get/put data - error*: PMy_bool # set this if you want to track data truncations happened during fetch - buffer_type*: Enum_field_types # buffer type - buffer_length*: int # buffer length, must be set for str/binary - # Following are for internal use. Set by mysql_stmt_bind_param - row_ptr*: ptr byte # for the current data position - offset*: int # offset position for char/binary fetch - length_value*: int # Used if length is 0 - param_number*: cuint # For null count and error messages - pack_length*: cuint # Internal length for packed data - error_value*: my_bool # used if error is 0 - is_unsigned*: my_bool # set if integer type is unsigned - long_data_used*: my_bool # If used with mysql_send_long_data - is_null_value*: my_bool # Used if is_null is 0 - store_param_func*: proc (net: PNET, param: Pst_mysql_bind){.cdecl.} - fetch_result*: proc (para1: Pst_mysql_bind, para2: PFIELD, row: PPbyte) - skip_result*: proc (para1: Pst_mysql_bind, para2: PFIELD, row: PPbyte) - - BIND* = St_mysql_bind - PBIND* = ptr BIND # statement handler - St_mysql_stmt*{.final.} = object - mem_root*: MEM_ROOT # root allocations - mysql*: PMySQL # connection handle - params*: PBIND # input parameters - `bind`*: PBIND # input parameters - fields*: PFIELD # result set metadata - result*: DATA # cached result set - data_cursor*: PROWS # current row in cached result - affected_rows*: my_ulonglong # copy of mysql->affected_rows after statement execution - insert_id*: my_ulonglong - read_row_func*: proc (stmt: Pst_mysql_stmt, row: PPbyte): cint{.cdecl.} - stmt_id*: int # Id for prepared statement - flags*: int # i.e. type of cursor to open - prefetch_rows*: int # number of rows per one COM_FETCH - server_status*: cuint # Copied from mysql->server_status after execute/fetch to know - # server-side cursor status for this statement. - last_errno*: cuint # error code - param_count*: cuint # input parameter count - field_count*: cuint # number of columns in result set - state*: Enum_mysql_stmt_state # statement state - last_error*: array[0..(ERRMSG_SIZE) - 1, char] # error message - sqlstate*: array[0..(SQLSTATE_LENGTH + 1) - 1, char] - send_types_to_server*: my_bool # Types of input parameters should be sent to server - bind_param_done*: my_bool # input buffers were supplied - bind_result_done*: char # output buffers were supplied - unbuffered_fetch_cancelled*: my_bool - update_max_length*: my_bool - - STMT* = St_mysql_stmt - - Enum_stmt_attr_type* = enum - STMT_ATTR_UPDATE_MAX_LENGTH, STMT_ATTR_CURSOR_TYPE, STMT_ATTR_PREFETCH_ROWS -{.deprecated: [Tst_dynamic_array: St_dynamic_array, Tst_mysql_options: St_mysql_options, - TDYNAMIC_ARRAY: DYNAMIC_ARRAY, Tprotocol_type: Protocol_type, - Trpl_type: Rpl_type, Tcharset_info_st: Charset_info_st, - TCHARSET_INFO: CHARSET_INFO, Tcharacter_set: Character_set, - TMY_CHARSET_INFO: MY_CHARSET_INFO, Tst_mysql: St_mysql, - Tst_mysql_methods: St_mysql_methods, TMySql: MySql, - Tst_mysql_res: St_mysql_res, TMETHODS: METHODS, TRES: RES, - Tst_mysql_manager: St_mysql_manager, TMANAGER: MANAGER, - Tst_mysql_parameters: St_mysql_parameters, TPARAMETERS: PARAMETERS, - Tenum_mysql_stmt_state: Enum_mysql_stmt_state, - Tst_mysql_bind: St_mysql_bind, TBIND: BIND, Tst_mysql_stmt: St_mysql_stmt, - TSTMT: STMT, Tenum_stmt_attr_type: Enum_stmt_attr_type, - Tstatus: Status].} - -proc server_init*(argc: cint, argv: cstringArray, groups: cstringArray): cint{. - cdecl, dynlib: lib, importc: "mysql_server_init".} -proc server_end*(){.cdecl, dynlib: lib, importc: "mysql_server_end".} - # mysql_server_init/end need to be called when using libmysqld or - # libmysqlclient (exactly, mysql_server_init() is called by mysql_init() so - # you don't need to call it explicitly; but you need to call - # mysql_server_end() to free memory). The names are a bit misleading - # (mysql_SERVER* to be used when using libmysqlCLIENT). So we add more general - # names which suit well whether you're using libmysqld or libmysqlclient. We - # intend to promote these aliases over the mysql_server* ones. -proc library_init*(argc: cint, argv: cstringArray, groups: cstringArray): cint{. - cdecl, dynlib: lib, importc: "mysql_server_init".} -proc library_end*(){.cdecl, dynlib: lib, importc: "mysql_server_end".} -proc get_parameters*(): PPARAMETERS{.stdcall, dynlib: lib, - importc: "mysql_get_parameters".} - # Set up and bring down a thread; these function should be called - # for each thread in an application which opens at least one MySQL - # connection. All uses of the connection(s) should be between these - # function calls. -proc thread_init*(): my_bool{.stdcall, dynlib: lib, importc: "mysql_thread_init".} -proc thread_end*(){.stdcall, dynlib: lib, importc: "mysql_thread_end".} - # Functions to get information from the MYSQL and MYSQL_RES structures - # Should definitely be used if one uses shared libraries. -proc num_rows*(res: PRES): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_num_rows".} -proc num_fields*(res: PRES): cuint{.stdcall, dynlib: lib, - importc: "mysql_num_fields".} -proc eof*(res: PRES): my_bool{.stdcall, dynlib: lib, importc: "mysql_eof".} -proc fetch_field_direct*(res: PRES, fieldnr: cuint): PFIELD{.stdcall, - dynlib: lib, importc: "mysql_fetch_field_direct".} -proc fetch_fields*(res: PRES): PFIELD{.stdcall, dynlib: lib, - importc: "mysql_fetch_fields".} -proc row_tell*(res: PRES): ROW_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_row_tell".} -proc field_tell*(res: PRES): FIELD_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_field_tell".} -proc field_count*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_field_count".} -proc affected_rows*(MySQL: PMySQL): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_affected_rows".} -proc insert_id*(MySQL: PMySQL): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_insert_id".} -proc errno*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, importc: "mysql_errno".} -proc error*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_error".} -proc sqlstate*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_sqlstate".} -proc warning_count*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_warning_count".} -proc info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_info".} -proc thread_id*(MySQL: PMySQL): int{.stdcall, dynlib: lib, importc: "mysql_thread_id".} -proc character_set_name*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_character_set_name".} -proc set_character_set*(MySQL: PMySQL, csname: cstring): int32{.stdcall, dynlib: lib, - importc: "mysql_set_character_set".} -proc init*(MySQL: PMySQL): PMySQL{.stdcall, dynlib: lib, importc: "mysql_init".} -proc ssl_set*(MySQL: PMySQL, key: cstring, cert: cstring, ca: cstring, capath: cstring, - cipher: cstring): my_bool{.stdcall, dynlib: lib, - importc: "mysql_ssl_set".} -proc change_user*(MySQL: PMySQL, user: cstring, passwd: cstring, db: cstring): my_bool{. - stdcall, dynlib: lib, importc: "mysql_change_user".} -proc real_connect*(MySQL: PMySQL, host: cstring, user: cstring, passwd: cstring, - db: cstring, port: cuint, unix_socket: cstring, - clientflag: int): PMySQL{.stdcall, dynlib: lib, - importc: "mysql_real_connect".} -proc select_db*(MySQL: PMySQL, db: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_select_db".} -proc query*(MySQL: PMySQL, q: cstring): cint{.stdcall, dynlib: lib, importc: "mysql_query".} -proc send_query*(MySQL: PMySQL, q: cstring, len: int): cint{.stdcall, dynlib: lib, - importc: "mysql_send_query".} -proc real_query*(MySQL: PMySQL, q: cstring, len: int): cint{.stdcall, dynlib: lib, - importc: "mysql_real_query".} -proc store_result*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, - importc: "mysql_store_result".} -proc use_result*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, importc: "mysql_use_result".} - # perform query on master -proc master_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_master_query".} -proc master_send_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, - dynlib: lib, importc: "mysql_master_send_query".} - # perform query on slave -proc slave_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_slave_query".} -proc slave_send_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, - dynlib: lib, importc: "mysql_slave_send_query".} -proc get_character_set_info*(MySQL: PMySQL, charset: PMY_CHARSET_INFO){.stdcall, - dynlib: lib, importc: "mysql_get_character_set_info".} - # local infile support -const - LOCAL_INFILE_ERROR_LEN* = 512 - -# procedure mysql_set_local_infile_handler(mysql:PMYSQL; local_infile_init:function (para1:Ppointer; para2:Pchar; para3:pointer):longint; local_infile_read:function (para1:pointer; para2:Pchar; para3:dword):longint; local_infile_end:procedure (_pa -# para6:pointer);cdecl;external mysqllib name 'mysql_set_local_infile_handler'; - -proc set_local_infile_default*(MySQL: PMySQL){.cdecl, dynlib: lib, - importc: "mysql_set_local_infile_default".} - # enable/disable parsing of all queries to decide if they go on master or - # slave -proc enable_rpl_parse*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_enable_rpl_parse".} -proc disable_rpl_parse*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_disable_rpl_parse".} - # get the value of the parse flag -proc rpl_parse_enabled*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, - importc: "mysql_rpl_parse_enabled".} - # enable/disable reads from master -proc enable_reads_from_master*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_enable_reads_from_master".} -proc disable_reads_from_master*(MySQL: PMySQL){.stdcall, dynlib: lib, importc: "mysql_disable_reads_from_master".} - # get the value of the master read flag -proc reads_from_master_enabled*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_reads_from_master_enabled".} -proc rpl_query_type*(q: cstring, length: cint): Rpl_type{.stdcall, dynlib: lib, - importc: "mysql_rpl_query_type".} - # discover the master and its slaves -proc rpl_probe*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_rpl_probe".} - # set the master, close/free the old one, if it is not a pivot -proc set_master*(MySQL: PMySQL, host: cstring, port: cuint, user: cstring, passwd: cstring): cint{. - stdcall, dynlib: lib, importc: "mysql_set_master".} -proc add_slave*(MySQL: PMySQL, host: cstring, port: cuint, user: cstring, passwd: cstring): cint{. - stdcall, dynlib: lib, importc: "mysql_add_slave".} -proc shutdown*(MySQL: PMySQL, shutdown_level: Enum_shutdown_level): cint{.stdcall, - dynlib: lib, importc: "mysql_shutdown".} -proc dump_debug_info*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, - importc: "mysql_dump_debug_info".} -proc refresh*(sql: PMySQL, refresh_options: cuint): cint{.stdcall, dynlib: lib, - importc: "mysql_refresh".} -proc kill*(MySQL: PMySQL, pid: int): cint{.stdcall, dynlib: lib, importc: "mysql_kill".} -proc set_server_option*(MySQL: PMySQL, option: Enum_mysql_set_option): cint{.stdcall, - dynlib: lib, importc: "mysql_set_server_option".} -proc ping*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, importc: "mysql_ping".} -proc stat*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_stat".} -proc get_server_info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_server_info".} -proc get_client_info*(): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_client_info".} -proc get_client_version*(): int{.stdcall, dynlib: lib, - importc: "mysql_get_client_version".} -proc get_host_info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_host_info".} -proc get_server_version*(MySQL: PMySQL): int{.stdcall, dynlib: lib, - importc: "mysql_get_server_version".} -proc get_proto_info*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_get_proto_info".} -proc list_dbs*(MySQL: PMySQL, wild: cstring): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_dbs".} -proc list_tables*(MySQL: PMySQL, wild: cstring): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_tables".} -proc list_processes*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_processes".} -proc options*(MySQL: PMySQL, option: Option, arg: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_options".} -proc free_result*(result: PRES){.stdcall, dynlib: lib, - importc: "mysql_free_result".} -proc data_seek*(result: PRES, offset: my_ulonglong){.stdcall, dynlib: lib, - importc: "mysql_data_seek".} -proc row_seek*(result: PRES, offset: ROW_OFFSET): ROW_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_row_seek".} -proc field_seek*(result: PRES, offset: FIELD_OFFSET): FIELD_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_field_seek".} -proc fetch_row*(result: PRES): ROW{.stdcall, dynlib: lib, - importc: "mysql_fetch_row".} -proc fetch_lengths*(result: PRES): ptr int{.stdcall, dynlib: lib, - importc: "mysql_fetch_lengths".} -proc fetch_field*(result: PRES): PFIELD{.stdcall, dynlib: lib, - importc: "mysql_fetch_field".} -proc list_fields*(MySQL: PMySQL, table: cstring, wild: cstring): PRES{.stdcall, - dynlib: lib, importc: "mysql_list_fields".} -proc escape_string*(fto: cstring, `from`: cstring, from_length: int): int{. - stdcall, dynlib: lib, importc: "mysql_escape_string".} -proc hex_string*(fto: cstring, `from`: cstring, from_length: int): int{.stdcall, - dynlib: lib, importc: "mysql_hex_string".} -proc real_escape_string*(MySQL: PMySQL, fto: cstring, `from`: cstring, len: int): int{. - stdcall, dynlib: lib, importc: "mysql_real_escape_string".} -proc debug*(debug: cstring){.stdcall, dynlib: lib, importc: "mysql_debug".} - # function mysql_odbc_escape_string(mysql:PMYSQL; fto:Pchar; to_length:dword; from:Pchar; from_length:dword; - # param:pointer; extend_buffer:function (para1:pointer; to:Pchar; length:Pdword):Pchar):Pchar;stdcall;external mysqllib name 'mysql_odbc_escape_string'; -proc myodbc_remove_escape*(MySQL: PMySQL, name: cstring){.stdcall, dynlib: lib, - importc: "myodbc_remove_escape".} -proc thread_safe*(): cuint{.stdcall, dynlib: lib, importc: "mysql_thread_safe".} -proc embedded*(): my_bool{.stdcall, dynlib: lib, importc: "mysql_embedded".} -proc manager_init*(con: PMANAGER): PMANAGER{.stdcall, dynlib: lib, - importc: "mysql_manager_init".} -proc manager_connect*(con: PMANAGER, host: cstring, user: cstring, - passwd: cstring, port: cuint): PMANAGER{.stdcall, - dynlib: lib, importc: "mysql_manager_connect".} -proc manager_close*(con: PMANAGER){.stdcall, dynlib: lib, - importc: "mysql_manager_close".} -proc manager_command*(con: PMANAGER, cmd: cstring, cmd_len: cint): cint{. - stdcall, dynlib: lib, importc: "mysql_manager_command".} -proc manager_fetch_line*(con: PMANAGER, res_buf: cstring, res_buf_size: cint): cint{. - stdcall, dynlib: lib, importc: "mysql_manager_fetch_line".} -proc read_query_result*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_read_query_result".} -proc stmt_init*(MySQL: PMySQL): PSTMT{.stdcall, dynlib: lib, importc: "mysql_stmt_init".} -proc stmt_prepare*(stmt: PSTMT, query: cstring, len: int): cint{.stdcall, - dynlib: lib, importc: "mysql_stmt_prepare".} -proc stmt_execute*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_execute".} -proc stmt_fetch*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_fetch".} -proc stmt_fetch_column*(stmt: PSTMT, `bind`: PBIND, column: cuint, offset: int): cint{. - stdcall, dynlib: lib, importc: "mysql_stmt_fetch_column".} -proc stmt_store_result*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_store_result".} -proc stmt_param_count*(stmt: PSTMT): int{.stdcall, dynlib: lib, - importc: "mysql_stmt_param_count".} -proc stmt_attr_set*(stmt: PSTMT, attr_type: Enum_stmt_attr_type, attr: pointer): my_bool{. - stdcall, dynlib: lib, importc: "mysql_stmt_attr_set".} -proc stmt_attr_get*(stmt: PSTMT, attr_type: Enum_stmt_attr_type, attr: pointer): my_bool{. - stdcall, dynlib: lib, importc: "mysql_stmt_attr_get".} -proc stmt_bind_param*(stmt: PSTMT, bnd: PBIND): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_bind_param".} -proc stmt_bind_result*(stmt: PSTMT, bnd: PBIND): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_bind_result".} -proc stmt_close*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_close".} -proc stmt_reset*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_reset".} -proc stmt_free_result*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_free_result".} -proc stmt_send_long_data*(stmt: PSTMT, param_number: cuint, data: cstring, - len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_send_long_data".} -proc stmt_result_metadata*(stmt: PSTMT): PRES{.stdcall, dynlib: lib, - importc: "mysql_stmt_result_metadata".} -proc stmt_param_metadata*(stmt: PSTMT): PRES{.stdcall, dynlib: lib, - importc: "mysql_stmt_param_metadata".} -proc stmt_errno*(stmt: PSTMT): cuint{.stdcall, dynlib: lib, - importc: "mysql_stmt_errno".} -proc stmt_error*(stmt: PSTMT): cstring{.stdcall, dynlib: lib, - importc: "mysql_stmt_error".} -proc stmt_sqlstate*(stmt: PSTMT): cstring{.stdcall, dynlib: lib, - importc: "mysql_stmt_sqlstate".} -proc stmt_row_seek*(stmt: PSTMT, offset: ROW_OFFSET): ROW_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_stmt_row_seek".} -proc stmt_row_tell*(stmt: PSTMT): ROW_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_stmt_row_tell".} -proc stmt_data_seek*(stmt: PSTMT, offset: my_ulonglong){.stdcall, dynlib: lib, - importc: "mysql_stmt_data_seek".} -proc stmt_num_rows*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_num_rows".} -proc stmt_affected_rows*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_affected_rows".} -proc stmt_insert_id*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_insert_id".} -proc stmt_field_count*(stmt: PSTMT): cuint{.stdcall, dynlib: lib, - importc: "mysql_stmt_field_count".} -proc commit*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_commit".} -proc rollback*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_rollback".} -proc autocommit*(MySQL: PMySQL, auto_mode: my_bool): my_bool{.stdcall, dynlib: lib, - importc: "mysql_autocommit".} -proc more_results*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_more_results".} -proc next_result*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, importc: "mysql_next_result".} -proc close*(sock: PMySQL){.stdcall, dynlib: lib, importc: "mysql_close".} - # status return codes -const - NO_DATA* = 100 - DATA_TRUNCATED* = 101 - -proc reload*(x: PMySQL): cint -when defined(USE_OLD_FUNCTIONS): - proc connect*(MySQL: PMySQL, host: cstring, user: cstring, passwd: cstring): PMySQL{.stdcall, - dynlib: lib, importc: "mysql_connect".} - proc create_db*(MySQL: PMySQL, DB: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_create_db".} - proc drop_db*(MySQL: PMySQL, DB: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_drop_db".} -proc net_safe_read*(MySQL: PMySQL): cuint{.cdecl, dynlib: lib, importc: "net_safe_read".} - -proc IS_PRI_KEY(n: int32): bool = - result = (n and PRI_KEY_FLAG) != 0 - -proc IS_NOT_NULL(n: int32): bool = - result = (n and NOT_NULL_FLAG) != 0 - -proc IS_BLOB(n: int32): bool = - result = (n and BLOB_FLAG) != 0 - -proc IS_NUM_FIELD(f: Pst_mysql_field): bool = - result = (f.flags and NUM_FLAG) != 0 - -proc IS_NUM(t: Enum_field_types): bool = - result = (t <= FIELD_TYPE_INT24) or (t == FIELD_TYPE_YEAR) or - (t == FIELD_TYPE_NEWDECIMAL) - -proc INTERNAL_NUM_FIELD(f: Pst_mysql_field): bool = - result = (f.ftype <= FIELD_TYPE_INT24) and - ((f.ftype != FIELD_TYPE_TIMESTAMP) or (f.len == 14) or (f.len == 8)) or - (f.ftype == FIELD_TYPE_YEAR) - -proc reload(x: PMySQL): cint = - result = refresh(x, REFRESH_GRANT) - -{.pop.} diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim deleted file mode 100644 index 1b2544ec0..000000000 --- a/lib/wrappers/odbcsql.nim +++ /dev/null @@ -1,847 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} - -when not defined(ODBCVER): - const - ODBCVER = 0x0351 ## define ODBC version 3.51 by default - -when defined(windows): - {.push callconv: stdcall.} - const odbclib = "odbc32.dll" -else: - {.push callconv: cdecl.} - const odbclib = "libodbc.so" - -# DATA TYPES CORRESPONDENCE -# BDE fields ODBC types -# ---------- ------------------ -# ftBlob SQL_BINARY -# ftBoolean SQL_BIT -# ftDate SQL_TYPE_DATE -# ftTime SQL_TYPE_TIME -# ftDateTime SQL_TYPE_TIMESTAMP -# ftInteger SQL_INTEGER -# ftSmallint SQL_SMALLINT -# ftFloat SQL_DOUBLE -# ftString SQL_CHAR -# ftMemo SQL_BINARY // SQL_VARCHAR -# - -type - TSqlChar* = char - TSqlSmallInt* = int16 - SqlUSmallInt* = int16 - SqlHandle* = pointer - SqlHEnv* = SqlHandle - SqlHDBC* = SqlHandle - SqlHStmt* = SqlHandle - SqlHDesc* = SqlHandle - TSqlInteger* = int - SqlUInteger* = int - SqlPointer* = pointer - TSqlReal* = cfloat - TSqlDouble* = cdouble - TSqlFloat* = cdouble - SqlHWND* = pointer - PSQLCHAR* = cstring - PSQLINTEGER* = ptr TSqlInteger - PSQLUINTEGER* = ptr SqlUInteger - PSQLSMALLINT* = ptr TSqlSmallInt - PSQLUSMALLINT* = ptr SqlUSmallInt - PSQLREAL* = ptr TSqlReal - PSQLDOUBLE* = ptr TSqlDouble - PSQLFLOAT* = ptr TSqlFloat - PSQLHANDLE* = ptr SqlHandle -{.deprecated: [ - # TSqlChar: TSqlChar, # Name conflict if we drop`T` - # TSqlSmallInt: TSqlSmallInt, # Name conflict if we drop`T` - TSqlUSmallInt: SqlUSmallInt, TSqlHandle: SqlHandle, TSqlHEnv: SqlHEnv, - TSqlHDBC: SqlHDBC, TSqlHStmt: SqlHStmt, TSqlHDesc: SqlHDesc, - # TSqlInteger: TSqlInteger, # Name conflict if we drop `T` - TSqlUInteger: SqlUInteger, TSqlPointer: SqlPointer, - # TSqlReal: TSqlReal, # Name conflict if we drop`T` - # TSqlDouble: TSqlDouble, # Name conflict if we drop`T` - # TSqlFloat: TSqlFloat, # Name conflict if we drop `T` - TSqlHWND: SqlHWND].} - -const # SQL data type codes - SQL_UNKNOWN_TYPE* = 0 - SQL_LONGVARCHAR* = (- 1) - SQL_BINARY* = (- 2) - SQL_VARBINARY* = (- 3) - SQL_LONGVARBINARY* = (- 4) - SQL_BIGINT* = (- 5) - SQL_TINYINT* = (- 6) - SQL_BIT* = (- 7) - SQL_WCHAR* = (- 8) - SQL_WVARCHAR* = (- 9) - SQL_WLONGVARCHAR* = (- 10) - SQL_CHAR* = 1 - SQL_NUMERIC* = 2 - SQL_DECIMAL* = 3 - SQL_INTEGER* = 4 - SQL_SMALLINT* = 5 - SQL_FLOAT* = 6 - SQL_REAL* = 7 - SQL_DOUBLE* = 8 - SQL_DATETIME* = 9 - SQL_VARCHAR* = 12 - SQL_TYPE_DATE* = 91 - SQL_TYPE_TIME* = 92 - SQL_TYPE_TIMESTAMP* = 93 - SQL_DATE* = 9 - SQL_TIME* = 10 - SQL_TIMESTAMP* = 11 - SQL_INTERVAL* = 10 - SQL_GUID* = - 11 # interval codes - -when ODBCVER >= 0x0300: - const - SQL_CODE_YEAR* = 1 - SQL_CODE_MONTH* = 2 - SQL_CODE_DAY* = 3 - SQL_CODE_HOUR* = 4 - SQL_CODE_MINUTE* = 5 - SQL_CODE_SECOND* = 6 - SQL_CODE_YEAR_TO_MONTH* = 7 - SQL_CODE_DAY_TO_HOUR* = 8 - SQL_CODE_DAY_TO_MINUTE* = 9 - SQL_CODE_DAY_TO_SECOND* = 10 - SQL_CODE_HOUR_TO_MINUTE* = 11 - SQL_CODE_HOUR_TO_SECOND* = 12 - SQL_CODE_MINUTE_TO_SECOND* = 13 - SQL_INTERVAL_YEAR* = 100 + SQL_CODE_YEAR - SQL_INTERVAL_MONTH* = 100 + SQL_CODE_MONTH - SQL_INTERVAL_DAY* = 100 + SQL_CODE_DAY - SQL_INTERVAL_HOUR* = 100 + SQL_CODE_HOUR - SQL_INTERVAL_MINUTE* = 100 + SQL_CODE_MINUTE - SQL_INTERVAL_SECOND* = 100 + SQL_CODE_SECOND - SQL_INTERVAL_YEAR_TO_MONTH* = 100 + SQL_CODE_YEAR_TO_MONTH - SQL_INTERVAL_DAY_TO_HOUR* = 100 + SQL_CODE_DAY_TO_HOUR - SQL_INTERVAL_DAY_TO_MINUTE* = 100 + SQL_CODE_DAY_TO_MINUTE - SQL_INTERVAL_DAY_TO_SECOND* = 100 + SQL_CODE_DAY_TO_SECOND - SQL_INTERVAL_HOUR_TO_MINUTE* = 100 + SQL_CODE_HOUR_TO_MINUTE - SQL_INTERVAL_HOUR_TO_SECOND* = 100 + SQL_CODE_HOUR_TO_SECOND - SQL_INTERVAL_MINUTE_TO_SECOND* = 100 + SQL_CODE_MINUTE_TO_SECOND -else: - const - SQL_INTERVAL_YEAR* = - 80 - SQL_INTERVAL_MONTH* = - 81 - SQL_INTERVAL_YEAR_TO_MONTH* = - 82 - SQL_INTERVAL_DAY* = - 83 - SQL_INTERVAL_HOUR* = - 84 - SQL_INTERVAL_MINUTE* = - 85 - SQL_INTERVAL_SECOND* = - 86 - SQL_INTERVAL_DAY_TO_HOUR* = - 87 - SQL_INTERVAL_DAY_TO_MINUTE* = - 88 - SQL_INTERVAL_DAY_TO_SECOND* = - 89 - SQL_INTERVAL_HOUR_TO_MINUTE* = - 90 - SQL_INTERVAL_HOUR_TO_SECOND* = - 91 - SQL_INTERVAL_MINUTE_TO_SECOND* = - 92 - - -when ODBCVER < 0x0300: - const - SQL_UNICODE* = - 95 - SQL_UNICODE_VARCHAR* = - 96 - SQL_UNICODE_LONGVARCHAR* = - 97 - SQL_UNICODE_CHAR* = SQL_UNICODE -else: - # The previous definitions for SQL_UNICODE_ are historical and obsolete - const - SQL_UNICODE* = SQL_WCHAR - SQL_UNICODE_VARCHAR* = SQL_WVARCHAR - SQL_UNICODE_LONGVARCHAR* = SQL_WLONGVARCHAR - SQL_UNICODE_CHAR* = SQL_WCHAR -const # C datatype to SQL datatype mapping - SQL_C_CHAR* = SQL_CHAR - SQL_C_LONG* = SQL_INTEGER - SQL_C_SHORT* = SQL_SMALLINT - SQL_C_FLOAT* = SQL_REAL - SQL_C_DOUBLE* = SQL_DOUBLE - SQL_C_NUMERIC* = SQL_NUMERIC - SQL_C_DEFAULT* = 99 - SQL_SIGNED_OFFSET* = - 20 - SQL_UNSIGNED_OFFSET* = - 22 - SQL_C_DATE* = SQL_DATE - SQL_C_TIME* = SQL_TIME - SQL_C_TIMESTAMP* = SQL_TIMESTAMP - SQL_C_TYPE_DATE* = SQL_TYPE_DATE - SQL_C_TYPE_TIME* = SQL_TYPE_TIME - SQL_C_TYPE_TIMESTAMP* = SQL_TYPE_TIMESTAMP - SQL_C_INTERVAL_YEAR* = SQL_INTERVAL_YEAR - SQL_C_INTERVAL_MONTH* = SQL_INTERVAL_MONTH - SQL_C_INTERVAL_DAY* = SQL_INTERVAL_DAY - SQL_C_INTERVAL_HOUR* = SQL_INTERVAL_HOUR - SQL_C_INTERVAL_MINUTE* = SQL_INTERVAL_MINUTE - SQL_C_INTERVAL_SECOND* = SQL_INTERVAL_SECOND - SQL_C_INTERVAL_YEAR_TO_MONTH* = SQL_INTERVAL_YEAR_TO_MONTH - SQL_C_INTERVAL_DAY_TO_HOUR* = SQL_INTERVAL_DAY_TO_HOUR - SQL_C_INTERVAL_DAY_TO_MINUTE* = SQL_INTERVAL_DAY_TO_MINUTE - SQL_C_INTERVAL_DAY_TO_SECOND* = SQL_INTERVAL_DAY_TO_SECOND - SQL_C_INTERVAL_HOUR_TO_MINUTE* = SQL_INTERVAL_HOUR_TO_MINUTE - SQL_C_INTERVAL_HOUR_TO_SECOND* = SQL_INTERVAL_HOUR_TO_SECOND - SQL_C_INTERVAL_MINUTE_TO_SECOND* = SQL_INTERVAL_MINUTE_TO_SECOND - SQL_C_BINARY* = SQL_BINARY - SQL_C_BIT* = SQL_BIT - SQL_C_SBIGINT* = SQL_BIGINT + SQL_SIGNED_OFFSET # SIGNED BIGINT - SQL_C_UBIGINT* = SQL_BIGINT + SQL_UNSIGNED_OFFSET # UNSIGNED BIGINT - SQL_C_TINYINT* = SQL_TINYINT - SQL_C_SLONG* = SQL_C_LONG + SQL_SIGNED_OFFSET # SIGNED INTEGER - SQL_C_SSHORT* = SQL_C_SHORT + SQL_SIGNED_OFFSET # SIGNED SMALLINT - SQL_C_STINYINT* = SQL_TINYINT + SQL_SIGNED_OFFSET # SIGNED TINYINT - SQL_C_ULONG* = SQL_C_LONG + SQL_UNSIGNED_OFFSET # UNSIGNED INTEGER - SQL_C_USHORT* = SQL_C_SHORT + SQL_UNSIGNED_OFFSET # UNSIGNED SMALLINT - SQL_C_UTINYINT* = SQL_TINYINT + SQL_UNSIGNED_OFFSET # UNSIGNED TINYINT - SQL_C_BOOKMARK* = SQL_C_ULONG # BOOKMARK - SQL_C_GUID* = SQL_GUID - SQL_TYPE_NULL* = 0 - -when ODBCVER < 0x0300: - const - SQL_TYPE_MIN* = SQL_BIT - SQL_TYPE_MAX* = SQL_VARCHAR - -const - SQL_C_VARBOOKMARK* = SQL_C_BINARY - SQL_API_SQLDESCRIBEPARAM* = 58 - SQL_NO_TOTAL* = - 4 - -type - SQL_DATE_STRUCT* {.final, pure.} = object - Year*: TSqlSmallInt - Month*: SqlUSmallInt - Day*: SqlUSmallInt - - PSQL_DATE_STRUCT* = ptr SQL_DATE_STRUCT - SQL_TIME_STRUCT* {.final, pure.} = object - Hour*: SqlUSmallInt - Minute*: SqlUSmallInt - Second*: SqlUSmallInt - - PSQL_TIME_STRUCT* = ptr SQL_TIME_STRUCT - SQL_TIMESTAMP_STRUCT* {.final, pure.} = object - Year*: SqlUSmallInt - Month*: SqlUSmallInt - Day*: SqlUSmallInt - Hour*: SqlUSmallInt - Minute*: SqlUSmallInt - Second*: SqlUSmallInt - Fraction*: SqlUInteger - - PSQL_TIMESTAMP_STRUCT* = ptr SQL_TIMESTAMP_STRUCT - -const - SQL_NAME_LEN* = 128 - SQL_OV_ODBC3* = 3 - SQL_OV_ODBC2* = 2 - SQL_ATTR_ODBC_VERSION* = 200 # Options for SQLDriverConnect - SQL_DRIVER_NOPROMPT* = 0 - SQL_DRIVER_COMPLETE* = 1 - SQL_DRIVER_PROMPT* = 2 - SQL_DRIVER_COMPLETE_REQUIRED* = 3 - SQL_IS_POINTER* = (- 4) # whether an attribute is a pointer or not - SQL_IS_UINTEGER* = (- 5) - SQL_IS_INTEGER* = (- 6) - SQL_IS_USMALLINT* = (- 7) - SQL_IS_SMALLINT* = (- 8) # SQLExtendedFetch "fFetchType" values - SQL_FETCH_BOOKMARK* = 8 - SQL_SCROLL_OPTIONS* = 44 # SQL_USE_BOOKMARKS options - SQL_UB_OFF* = 0 - SQL_UB_ON* = 1 - SQL_UB_DEFAULT* = SQL_UB_OFF - SQL_UB_FIXED* = SQL_UB_ON - SQL_UB_VARIABLE* = 2 # SQL_SCROLL_OPTIONS masks - SQL_SO_FORWARD_ONLY* = 0x00000001 - SQL_SO_KEYSET_DRIVEN* = 0x00000002 - SQL_SO_DYNAMIC* = 0x00000004 - SQL_SO_MIXED* = 0x00000008 - SQL_SO_STATIC* = 0x00000010 - SQL_BOOKMARK_PERSISTENCE* = 82 - SQL_STATIC_SENSITIVITY* = 83 # SQL_BOOKMARK_PERSISTENCE values - SQL_BP_CLOSE* = 0x00000001 - SQL_BP_DELETE* = 0x00000002 - SQL_BP_DROP* = 0x00000004 - SQL_BP_TRANSACTION* = 0x00000008 - SQL_BP_UPDATE* = 0x00000010 - SQL_BP_OTHER_HSTMT* = 0x00000020 - SQL_BP_SCROLL* = 0x00000040 - SQL_DYNAMIC_CURSOR_ATTRIBUTES1* = 144 - SQL_DYNAMIC_CURSOR_ATTRIBUTES2* = 145 - SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES1* = 146 - SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES2* = 147 - SQL_INDEX_KEYWORDS* = 148 - SQL_INFO_SCHEMA_VIEWS* = 149 - SQL_KEYSET_CURSOR_ATTRIBUTES1* = 150 - SQL_KEYSET_CURSOR_ATTRIBUTES2* = 151 - SQL_STATIC_CURSOR_ATTRIBUTES1* = 167 - SQL_STATIC_CURSOR_ATTRIBUTES2* = 168 # supported SQLFetchScroll FetchOrientation's - SQL_CA1_NEXT* = 1 - SQL_CA1_ABSOLUTE* = 2 - SQL_CA1_RELATIVE* = 4 - SQL_CA1_BOOKMARK* = 8 # supported SQLSetPos LockType's - SQL_CA1_LOCK_NO_CHANGE* = 0x00000040 - SQL_CA1_LOCK_EXCLUSIVE* = 0x00000080 - SQL_CA1_LOCK_UNLOCK* = 0x00000100 # supported SQLSetPos Operations - SQL_CA1_POS_POSITION* = 0x00000200 - SQL_CA1_POS_UPDATE* = 0x00000400 - SQL_CA1_POS_DELETE* = 0x00000800 - SQL_CA1_POS_REFRESH* = 0x00001000 # positioned updates and deletes - SQL_CA1_POSITIONED_UPDATE* = 0x00002000 - SQL_CA1_POSITIONED_DELETE* = 0x00004000 - SQL_CA1_SELECT_FOR_UPDATE* = 0x00008000 # supported SQLBulkOperations operations - SQL_CA1_BULK_ADD* = 0x00010000 - SQL_CA1_BULK_UPDATE_BY_BOOKMARK* = 0x00020000 - SQL_CA1_BULK_DELETE_BY_BOOKMARK* = 0x00040000 - SQL_CA1_BULK_FETCH_BY_BOOKMARK* = 0x00080000 # supported values for SQL_ATTR_SCROLL_CONCURRENCY - SQL_CA2_READ_ONLY_CONCURRENCY* = 1 - SQL_CA2_LOCK_CONCURRENCY* = 2 - SQL_CA2_OPT_ROWVER_CONCURRENCY* = 4 - SQL_CA2_OPT_VALUES_CONCURRENCY* = 8 # sensitivity of the cursor to its own inserts, deletes, and updates - SQL_CA2_SENSITIVITY_ADDITIONS* = 0x00000010 - SQL_CA2_SENSITIVITY_DELETIONS* = 0x00000020 - SQL_CA2_SENSITIVITY_UPDATES* = 0x00000040 # semantics of SQL_ATTR_MAX_ROWS - SQL_CA2_MAX_ROWS_SELECT* = 0x00000080 - SQL_CA2_MAX_ROWS_INSERT* = 0x00000100 - SQL_CA2_MAX_ROWS_DELETE* = 0x00000200 - SQL_CA2_MAX_ROWS_UPDATE* = 0x00000400 - SQL_CA2_MAX_ROWS_CATALOG* = 0x00000800 - SQL_CA2_MAX_ROWS_AFFECTS_ALL* = (SQL_CA2_MAX_ROWS_SELECT or - SQL_CA2_MAX_ROWS_INSERT or SQL_CA2_MAX_ROWS_DELETE or - SQL_CA2_MAX_ROWS_UPDATE or SQL_CA2_MAX_ROWS_CATALOG) # semantics of - # SQL_DIAG_CURSOR_ROW_COUNT - SQL_CA2_CRC_EXACT* = 0x00001000 - SQL_CA2_CRC_APPROXIMATE* = 0x00002000 # the kinds of positioned statements that can be simulated - SQL_CA2_SIMULATE_NON_UNIQUE* = 0x00004000 - SQL_CA2_SIMULATE_TRY_UNIQUE* = 0x00008000 - SQL_CA2_SIMULATE_UNIQUE* = 0x00010000 # Operations in SQLBulkOperations - SQL_ADD* = 4 - SQL_SETPOS_MAX_OPTION_VALUE* = SQL_ADD - SQL_UPDATE_BY_BOOKMARK* = 5 - SQL_DELETE_BY_BOOKMARK* = 6 - SQL_FETCH_BY_BOOKMARK* = 7 # Operations in SQLSetPos - SQL_POSITION* = 0 - SQL_REFRESH* = 1 - SQL_UPDATE* = 2 - SQL_DELETE* = 3 # Lock options in SQLSetPos - SQL_LOCK_NO_CHANGE* = 0 - SQL_LOCK_EXCLUSIVE* = 1 - SQL_LOCK_UNLOCK* = 2 # SQLExtendedFetch "rgfRowStatus" element values - SQL_ROW_SUCCESS* = 0 - SQL_ROW_DELETED* = 1 - SQL_ROW_UPDATED* = 2 - SQL_ROW_NOROW* = 3 - SQL_ROW_ADDED* = 4 - SQL_ROW_ERROR* = 5 - SQL_ROW_SUCCESS_WITH_INFO* = 6 - SQL_ROW_PROCEED* = 0 - SQL_ROW_IGNORE* = 1 - SQL_MAX_DSN_LENGTH* = 32 # maximum data source name size - SQL_MAX_OPTION_STRING_LENGTH* = 256 - SQL_ODBC_CURSORS* = 110 - SQL_ATTR_ODBC_CURSORS* = SQL_ODBC_CURSORS # SQL_ODBC_CURSORS options - SQL_CUR_USE_IF_NEEDED* = 0 - SQL_CUR_USE_ODBC* = 1 - SQL_CUR_USE_DRIVER* = 2 - SQL_CUR_DEFAULT* = SQL_CUR_USE_DRIVER - SQL_PARAM_TYPE_UNKNOWN* = 0 - SQL_PARAM_INPUT* = 1 - SQL_PARAM_INPUT_OUTPUT* = 2 - SQL_RESULT_COL* = 3 - SQL_PARAM_OUTPUT* = 4 - SQL_RETURN_VALUE* = 5 # special length/indicator values - SQL_NULL_DATA* = (- 1) - SQL_DATA_AT_EXEC* = (- 2) - SQL_SUCCESS* = 0 - SQL_SUCCESS_WITH_INFO* = 1 - SQL_NO_DATA* = 100 - SQL_ERROR* = (- 1) - SQL_INVALID_HANDLE* = (- 2) - SQL_STILL_EXECUTING* = 2 - SQL_NEED_DATA* = 99 # flags for null-terminated string - SQL_NTS* = (- 3) # maximum message length - SQL_MAX_MESSAGE_LENGTH* = 512 # date/time length constants - SQL_DATE_LEN* = 10 - SQL_TIME_LEN* = 8 # add P+1 if precision is nonzero - SQL_TIMESTAMP_LEN* = 19 # add P+1 if precision is nonzero - # handle type identifiers - SQL_HANDLE_ENV* = 1 - SQL_HANDLE_DBC* = 2 - SQL_HANDLE_STMT* = 3 - SQL_HANDLE_DESC* = 4 # environment attribute - SQL_ATTR_OUTPUT_NTS* = 10001 # connection attributes - SQL_ATTR_AUTO_IPD* = 10001 - SQL_ATTR_METADATA_ID* = 10014 # statement attributes - SQL_ATTR_APP_ROW_DESC* = 10010 - SQL_ATTR_APP_PARAM_DESC* = 10011 - SQL_ATTR_IMP_ROW_DESC* = 10012 - SQL_ATTR_IMP_PARAM_DESC* = 10013 - SQL_ATTR_CURSOR_SCROLLABLE* = (- 1) - SQL_ATTR_CURSOR_SENSITIVITY* = (- 2) - SQL_QUERY_TIMEOUT* = 0 - SQL_MAX_ROWS* = 1 - SQL_NOSCAN* = 2 - SQL_MAX_LENGTH* = 3 - SQL_ASYNC_ENABLE* = 4 # same as SQL_ATTR_ASYNC_ENABLE */ - SQL_BIND_TYPE* = 5 - SQL_CURSOR_TYPE* = 6 - SQL_CONCURRENCY* = 7 - SQL_KEYSET_SIZE* = 8 - SQL_ROWSET_SIZE* = 9 - SQL_SIMULATE_CURSOR* = 10 - SQL_RETRIEVE_DATA* = 11 - SQL_USE_BOOKMARKS* = 12 - SQL_GET_BOOKMARK* = 13 # GetStmtOption Only */ - SQL_ROW_NUMBER* = 14 # GetStmtOption Only */ - SQL_ATTR_CURSOR_TYPE* = SQL_CURSOR_TYPE - SQL_ATTR_CONCURRENCY* = SQL_CONCURRENCY - SQL_ATTR_FETCH_BOOKMARK_PTR* = 16 - SQL_ATTR_ROW_STATUS_PTR* = 25 - SQL_ATTR_ROWS_FETCHED_PTR* = 26 - SQL_AUTOCOMMIT* = 102 - SQL_ATTR_AUTOCOMMIT* = SQL_AUTOCOMMIT - SQL_ATTR_ROW_NUMBER* = SQL_ROW_NUMBER - SQL_TXN_ISOLATION* = 108 - SQL_ATTR_TXN_ISOLATION* = SQL_TXN_ISOLATION - SQL_ATTR_MAX_ROWS* = SQL_MAX_ROWS - SQL_ATTR_USE_BOOKMARKS* = SQL_USE_BOOKMARKS #* connection attributes */ - SQL_ACCESS_MODE* = 101 # SQL_AUTOCOMMIT =102; - SQL_LOGIN_TIMEOUT* = 103 - SQL_OPT_TRACE* = 104 - SQL_OPT_TRACEFILE* = 105 - SQL_TRANSLATE_DLL* = 106 - SQL_TRANSLATE_OPTION* = 107 # SQL_TXN_ISOLATION =108; - SQL_CURRENT_QUALIFIER* = 109 # SQL_ODBC_CURSORS =110; - SQL_QUIET_MODE* = 111 - SQL_PACKET_SIZE* = 112 #* connection attributes with new names */ - SQL_ATTR_ACCESS_MODE* = SQL_ACCESS_MODE # SQL_ATTR_AUTOCOMMIT =SQL_AUTOCOMMIT; - SQL_ATTR_CONNECTION_DEAD* = 1209 #* GetConnectAttr only */ - SQL_ATTR_CONNECTION_TIMEOUT* = 113 - SQL_ATTR_CURRENT_CATALOG* = SQL_CURRENT_QUALIFIER - SQL_ATTR_DISCONNECT_BEHAVIOR* = 114 - SQL_ATTR_ENLIST_IN_DTC* = 1207 - SQL_ATTR_ENLIST_IN_XA* = 1208 - SQL_ATTR_LOGIN_TIMEOUT* = SQL_LOGIN_TIMEOUT # SQL_ATTR_ODBC_CURSORS =SQL_ODBC_CURSORS; - SQL_ATTR_PACKET_SIZE* = SQL_PACKET_SIZE - SQL_ATTR_QUIET_MODE* = SQL_QUIET_MODE - SQL_ATTR_TRACE* = SQL_OPT_TRACE - SQL_ATTR_TRACEFILE* = SQL_OPT_TRACEFILE - SQL_ATTR_TRANSLATE_LIB* = SQL_TRANSLATE_DLL - SQL_ATTR_TRANSLATE_OPTION* = SQL_TRANSLATE_OPTION # SQL_ATTR_TXN_ISOLATION =SQL_TXN_ISOLATION; - #* SQL_ACCESS_MODE options */ - SQL_MODE_READ_WRITE* = 0 - SQL_MODE_READ_ONLY* = 1 - SQL_MODE_DEFAULT* = SQL_MODE_READ_WRITE #* SQL_AUTOCOMMIT options */ - SQL_AUTOCOMMIT_OFF* = 0 - SQL_AUTOCOMMIT_ON* = 1 - SQL_AUTOCOMMIT_DEFAULT* = SQL_AUTOCOMMIT_ON # SQL_ATTR_CURSOR_SCROLLABLE values - SQL_NONSCROLLABLE* = 0 - SQL_SCROLLABLE* = 1 # SQL_CURSOR_TYPE options - SQL_CURSOR_FORWARD_ONLY* = 0 - SQL_CURSOR_KEYSET_DRIVEN* = 1 - SQL_CURSOR_DYNAMIC* = 2 - SQL_CURSOR_STATIC* = 3 - SQL_CURSOR_TYPE_DEFAULT* = SQL_CURSOR_FORWARD_ONLY # Default value - # SQL_CONCURRENCY options - SQL_CONCUR_READ_ONLY* = 1 - SQL_CONCUR_LOCK* = 2 - SQL_CONCUR_ROWVER* = 3 - SQL_CONCUR_VALUES* = 4 - SQL_CONCUR_DEFAULT* = SQL_CONCUR_READ_ONLY # Default value - # identifiers of fields in the SQL descriptor - SQL_DESC_COUNT* = 1001 - SQL_DESC_TYPE* = 1002 - SQL_DESC_LENGTH* = 1003 - SQL_DESC_OCTET_LENGTH_PTR* = 1004 - SQL_DESC_PRECISION* = 1005 - SQL_DESC_SCALE* = 1006 - SQL_DESC_DATETIME_INTERVAL_CODE* = 1007 - SQL_DESC_NULLABLE* = 1008 - SQL_DESC_INDICATOR_PTR* = 1009 - SQL_DESC_DATA_PTR* = 1010 - SQL_DESC_NAME* = 1011 - SQL_DESC_UNNAMED* = 1012 - SQL_DESC_OCTET_LENGTH* = 1013 - SQL_DESC_ALLOC_TYPE* = 1099 # identifiers of fields in the diagnostics area - SQL_DIAG_RETURNCODE* = 1 - SQL_DIAG_NUMBER* = 2 - SQL_DIAG_ROW_COUNT* = 3 - SQL_DIAG_SQLSTATE* = 4 - SQL_DIAG_NATIVE* = 5 - SQL_DIAG_MESSAGE_TEXT* = 6 - SQL_DIAG_DYNAMIC_FUNCTION* = 7 - SQL_DIAG_CLASS_ORIGIN* = 8 - SQL_DIAG_SUBCLASS_ORIGIN* = 9 - SQL_DIAG_CONNECTION_NAME* = 10 - SQL_DIAG_SERVER_NAME* = 11 - SQL_DIAG_DYNAMIC_FUNCTION_CODE* = 12 # dynamic function codes - SQL_DIAG_ALTER_TABLE* = 4 - SQL_DIAG_CREATE_INDEX* = (- 1) - SQL_DIAG_CREATE_TABLE* = 77 - SQL_DIAG_CREATE_VIEW* = 84 - SQL_DIAG_DELETE_WHERE* = 19 - SQL_DIAG_DROP_INDEX* = (- 2) - SQL_DIAG_DROP_TABLE* = 32 - SQL_DIAG_DROP_VIEW* = 36 - SQL_DIAG_DYNAMIC_DELETE_CURSOR* = 38 - SQL_DIAG_DYNAMIC_UPDATE_CURSOR* = 81 - SQL_DIAG_GRANT* = 48 - SQL_DIAG_INSERT* = 50 - SQL_DIAG_REVOKE* = 59 - SQL_DIAG_SELECT_CURSOR* = 85 - SQL_DIAG_UNKNOWN_STATEMENT* = 0 - SQL_DIAG_UPDATE_WHERE* = 82 # Statement attribute values for cursor sensitivity - SQL_UNSPECIFIED* = 0 - SQL_INSENSITIVE* = 1 - SQL_SENSITIVE* = 2 # GetTypeInfo() request for all data types - SQL_ALL_TYPES* = 0 # Default conversion code for SQLBindCol(), SQLBindParam() and SQLGetData() - SQL_DEFAULT* = 99 # SQLGetData() code indicating that the application row descriptor - # specifies the data type - SQL_ARD_TYPE* = (- 99) # SQL date/time type subcodes - SQL_CODE_DATE* = 1 - SQL_CODE_TIME* = 2 - SQL_CODE_TIMESTAMP* = 3 # CLI option values - SQL_FALSE* = 0 - SQL_TRUE* = 1 # values of NULLABLE field in descriptor - SQL_NO_NULLS* = 0 - SQL_NULLABLE* = 1 # Value returned by SQLGetTypeInfo() to denote that it is - # not known whether or not a data type supports null values. - SQL_NULLABLE_UNKNOWN* = 2 - SQL_CLOSE* = 0 - SQL_DROP* = 1 - SQL_UNBIND* = 2 - SQL_RESET_PARAMS* = 3 # Codes used for FetchOrientation in SQLFetchScroll(), - # and in SQLDataSources() - SQL_FETCH_NEXT* = 1 - SQL_FETCH_FIRST* = 2 - SQL_FETCH_FIRST_USER* = 31 - SQL_FETCH_FIRST_SYSTEM* = 32 # Other codes used for FetchOrientation in SQLFetchScroll() - SQL_FETCH_LAST* = 3 - SQL_FETCH_PRIOR* = 4 - SQL_FETCH_ABSOLUTE* = 5 - SQL_FETCH_RELATIVE* = 6 - SQL_NULL_HENV* = SqlHEnv(nil) - SQL_NULL_HDBC* = SqlHDBC(nil) - SQL_NULL_HSTMT* = SqlHStmt(nil) - SQL_NULL_HDESC* = SqlHDesc(nil) #* null handle used in place of parent handle when allocating HENV */ - SQL_NULL_HANDLE* = SqlHandle(nil) #* Values that may appear in the result set of SQLSpecialColumns() */ - SQL_SCOPE_CURROW* = 0 - SQL_SCOPE_TRANSACTION* = 1 - SQL_SCOPE_SESSION* = 2 #* Column types and scopes in SQLSpecialColumns. */ - SQL_BEST_ROWID* = 1 - SQL_ROWVER* = 2 - SQL_ROW_IDENTIFIER* = 1 #* Reserved values for UNIQUE argument of SQLStatistics() */ - SQL_INDEX_UNIQUE* = 0 - SQL_INDEX_ALL* = 1 #* Reserved values for RESERVED argument of SQLStatistics() */ - SQL_QUICK* = 0 - SQL_ENSURE* = 1 #* Values that may appear in the result set of SQLStatistics() */ - SQL_TABLE_STAT* = 0 - SQL_INDEX_CLUSTERED* = 1 - SQL_INDEX_HASHED* = 2 - SQL_INDEX_OTHER* = 3 - SQL_SCROLL_CONCURRENCY* = 43 - SQL_TXN_CAPABLE* = 46 - SQL_TRANSACTION_CAPABLE* = SQL_TXN_CAPABLE - SQL_USER_NAME* = 47 - SQL_TXN_ISOLATION_OPTION* = 72 - SQL_TRANSACTION_ISOLATION_OPTION* = SQL_TXN_ISOLATION_OPTION - SQL_OJ_CAPABILITIES* = 115 - SQL_OUTER_JOIN_CAPABILITIES* = SQL_OJ_CAPABILITIES - SQL_XOPEN_CLI_YEAR* = 10000 - SQL_CURSOR_SENSITIVITY* = 10001 - SQL_DESCRIBE_PARAMETER* = 10002 - SQL_CATALOG_NAME* = 10003 - SQL_COLLATION_SEQ* = 10004 - SQL_MAX_IDENTIFIER_LEN* = 10005 - SQL_MAXIMUM_IDENTIFIER_LENGTH* = SQL_MAX_IDENTIFIER_LEN - SQL_SCCO_READ_ONLY* = 1 - SQL_SCCO_LOCK* = 2 - SQL_SCCO_OPT_ROWVER* = 4 - SQL_SCCO_OPT_VALUES* = 8 #* SQL_TXN_CAPABLE values */ - SQL_TC_NONE* = 0 - SQL_TC_DML* = 1 - SQL_TC_ALL* = 2 - SQL_TC_DDL_COMMIT* = 3 - SQL_TC_DDL_IGNORE* = 4 #* SQL_TXN_ISOLATION_OPTION bitmasks */ - SQL_TXN_READ_UNCOMMITTED* = 1 - SQL_TRANSACTION_READ_UNCOMMITTED* = SQL_TXN_READ_UNCOMMITTED - SQL_TXN_READ_COMMITTED* = 2 - SQL_TRANSACTION_READ_COMMITTED* = SQL_TXN_READ_COMMITTED - SQL_TXN_REPEATABLE_READ* = 4 - SQL_TRANSACTION_REPEATABLE_READ* = SQL_TXN_REPEATABLE_READ - SQL_TXN_SERIALIZABLE* = 8 - SQL_TRANSACTION_SERIALIZABLE* = SQL_TXN_SERIALIZABLE - SQL_SS_ADDITIONS* = 1 - SQL_SS_DELETIONS* = 2 - SQL_SS_UPDATES* = 4 # SQLColAttributes defines - SQL_COLUMN_COUNT* = 0 - SQL_COLUMN_NAME* = 1 - SQL_COLUMN_TYPE* = 2 - SQL_COLUMN_LENGTH* = 3 - SQL_COLUMN_PRECISION* = 4 - SQL_COLUMN_SCALE* = 5 - SQL_COLUMN_DISPLAY_SIZE* = 6 - SQL_COLUMN_NULLABLE* = 7 - SQL_COLUMN_UNSIGNED* = 8 - SQL_COLUMN_MONEY* = 9 - SQL_COLUMN_UPDATABLE* = 10 - SQL_COLUMN_AUTO_INCREMENT* = 11 - SQL_COLUMN_CASE_SENSITIVE* = 12 - SQL_COLUMN_SEARCHABLE* = 13 - SQL_COLUMN_TYPE_NAME* = 14 - SQL_COLUMN_TABLE_NAME* = 15 - SQL_COLUMN_OWNER_NAME* = 16 - SQL_COLUMN_QUALIFIER_NAME* = 17 - SQL_COLUMN_LABEL* = 18 - SQL_COLATT_OPT_MAX* = SQL_COLUMN_LABEL - SQL_COLUMN_DRIVER_START* = 1000 - SQL_DESC_ARRAY_SIZE* = 20 - SQL_DESC_ARRAY_STATUS_PTR* = 21 - SQL_DESC_AUTO_UNIQUE_VALUE* = SQL_COLUMN_AUTO_INCREMENT - SQL_DESC_BASE_COLUMN_NAME* = 22 - SQL_DESC_BASE_TABLE_NAME* = 23 - SQL_DESC_BIND_OFFSET_PTR* = 24 - SQL_DESC_BIND_TYPE* = 25 - SQL_DESC_CASE_SENSITIVE* = SQL_COLUMN_CASE_SENSITIVE - SQL_DESC_CATALOG_NAME* = SQL_COLUMN_QUALIFIER_NAME - SQL_DESC_CONCISE_TYPE* = SQL_COLUMN_TYPE - SQL_DESC_DATETIME_INTERVAL_PRECISION* = 26 - SQL_DESC_DISPLAY_SIZE* = SQL_COLUMN_DISPLAY_SIZE - SQL_DESC_FIXED_PREC_SCALE* = SQL_COLUMN_MONEY - SQL_DESC_LABEL* = SQL_COLUMN_LABEL - SQL_DESC_LITERAL_PREFIX* = 27 - SQL_DESC_LITERAL_SUFFIX* = 28 - SQL_DESC_LOCAL_TYPE_NAME* = 29 - SQL_DESC_MAXIMUM_SCALE* = 30 - SQL_DESC_MINIMUM_SCALE* = 31 - SQL_DESC_NUM_PREC_RADIX* = 32 - SQL_DESC_PARAMETER_TYPE* = 33 - SQL_DESC_ROWS_PROCESSED_PTR* = 34 - SQL_DESC_SCHEMA_NAME* = SQL_COLUMN_OWNER_NAME - SQL_DESC_SEARCHABLE* = SQL_COLUMN_SEARCHABLE - SQL_DESC_TYPE_NAME* = SQL_COLUMN_TYPE_NAME - SQL_DESC_TABLE_NAME* = SQL_COLUMN_TABLE_NAME - SQL_DESC_UNSIGNED* = SQL_COLUMN_UNSIGNED - SQL_DESC_UPDATABLE* = SQL_COLUMN_UPDATABLE #* SQLEndTran() options */ - SQL_COMMIT* = 0 - SQL_ROLLBACK* = 1 - SQL_ATTR_ROW_ARRAY_SIZE* = 27 #* SQLConfigDataSource() options */ - ODBC_ADD_DSN* = 1 - ODBC_CONFIG_DSN* = 2 - ODBC_REMOVE_DSN* = 3 - ODBC_ADD_SYS_DSN* = 4 - ODBC_CONFIG_SYS_DSN* = 5 - ODBC_REMOVE_SYS_DSN* = 6 - - SQL_ACTIVE_CONNECTIONS* = 0 # SQLGetInfo - SQL_DATA_SOURCE_NAME* = 2 - SQL_DATA_SOURCE_READ_ONLY* = 25 - SQL_DATABASE_NAME* = 2 - SQL_DBMS_NAME* = 17 - SQL_DBMS_VERSION* = 18 - SQL_DRIVER_HDBC* = 3 - SQL_DRIVER_HENV* = 4 - SQL_DRIVER_HSTMT* = 5 - SQL_DRIVER_NAME* = 6 - SQL_DRIVER_VER* = 7 - SQL_FETCH_DIRECTION* = 8 - SQL_ODBC_VER* = 10 - SQL_DRIVER_ODBC_VER* = 77 - SQL_SERVER_NAME* = 13 - SQL_ACTIVE_ENVIRONMENTS* = 116 - SQL_ACTIVE_STATEMENTS* = 1 - SQL_SQL_CONFORMANCE* = 118 - SQL_DATETIME_LITERALS* = 119 - SQL_ASYNC_MODE* = 10021 - SQL_BATCH_ROW_COUNT* = 120 - SQL_BATCH_SUPPORT* = 121 - SQL_CATALOG_LOCATION* = 114 - #SQL_CATALOG_NAME* = 10003 - SQL_CATALOG_NAME_SEPARATOR* = 41 - SQL_CATALOG_TERM* = 42 - SQL_CATALOG_USAGE* = 92 - #SQL_COLLATION_SEQ* = 10004 - SQL_COLUMN_ALIAS* = 87 - #SQL_USER_NAME* = 47 - -proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle, - OutputHandlePtr: var SqlHandle): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: TSqlInteger, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: SqlPointer, BufferLength: TSqlInteger, - StringLength: PSQLINTEGER): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLFreeHandle*(HandleType: TSqlSmallInt, Handle: SqlHandle): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetDiagRec*(HandleType: TSqlSmallInt, Handle: SqlHandle, - RecNumber: TSqlSmallInt, Sqlstate: PSQLCHAR, - NativeError: var TSqlInteger, MessageText: PSQLCHAR, - BufferLength: TSqlSmallInt, TextLength: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetDiagField*(HandleType: TSqlSmallInt, Handle: SqlHandle, - RecNumber: TSqlSmallInt, DiagIdentifier: TSqlSmallInt, - DiagInfoPtr: SqlPointer, BufferLength: TSqlSmallInt, - StringLengthPtr: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLConnect*(ConnectionHandle: SqlHDBC, ServerName: PSQLCHAR, - NameLength1: TSqlSmallInt, UserName: PSQLCHAR, - NameLength2: TSqlSmallInt, Authentication: PSQLCHAR, - NameLength3: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLDisconnect*(ConnectionHandle: SqlHDBC): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLDriverConnect*(hdbc: SqlHDBC, hwnd: SqlHWND, szCsin: cstring, - szCLen: TSqlSmallInt, szCsout: cstring, - cbCSMax: TSqlSmallInt, cbCsOut: var TSqlSmallInt, - f: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLBrowseConnect*(hdbc: SqlHDBC, szConnStrIn: PSQLCHAR, - cbConnStrIn: TSqlSmallInt, szConnStrOut: PSQLCHAR, - cbConnStrOutMax: TSqlSmallInt, - cbConnStrOut: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLExecDirect*(StatementHandle: SqlHStmt, StatementText: PSQLCHAR, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLExecDirectW*(StatementHandle: SqlHStmt, StatementText: WideCString, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLPrepare*(StatementHandle: SqlHStmt, StatementText: PSQLCHAR, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLPrepareW*(StatementHandle: SqlHStmt, StatementText: WideCString, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLCloseCursor*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLExecute*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLFetch*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLNumResultCols*(StatementHandle: SqlHStmt, ColumnCount: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLDescribeCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - ColumnName: PSQLCHAR, BufferLength: TSqlSmallInt, - NameLength: var TSqlSmallInt, DataType: var TSqlSmallInt, - ColumnSize: var SqlUInteger, - DecimalDigits: var TSqlSmallInt, Nullable: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLFetchScroll*(StatementHandle: SqlHStmt, FetchOrientation: TSqlSmallInt, - FetchOffset: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLExtendedFetch*(hstmt: SqlHStmt, fFetchType: SqlUSmallInt, - irow: TSqlInteger, pcrow: PSQLUINTEGER, - rgfRowStatus: PSQLUSMALLINT): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLGetData*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - TargetType: TSqlSmallInt, TargetValue: SqlPointer, - BufferLength: TSqlInteger, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetStmtAttr*(StatementHandle: SqlHStmt, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetStmtAttr*(StatementHandle: SqlHStmt, Attribute: TSqlInteger, - Value: SqlPointer, BufferLength: TSqlInteger, - StringLength: PSQLINTEGER): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLGetInfo*(ConnectionHandle: SqlHDBC, InfoType: SqlUSmallInt, - InfoValue: SqlPointer, BufferLength: TSqlSmallInt, - StringLength: PSQLSMALLINT): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLBulkOperations*(StatementHandle: SqlHStmt, Operation: TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLPutData*(StatementHandle: SqlHStmt, Data: SqlPointer, - StrLen_or_Ind: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLBindCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - TargetType: TSqlSmallInt, TargetValue: SqlPointer, - BufferLength: TSqlInteger, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetPos*(hstmt: SqlHStmt, irow: SqlUSmallInt, fOption: SqlUSmallInt, - fLock: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLDataSources*(EnvironmentHandle: SqlHEnv, Direction: SqlUSmallInt, - ServerName: PSQLCHAR, BufferLength1: TSqlSmallInt, - NameLength1: PSQLSMALLINT, Description: PSQLCHAR, - BufferLength2: TSqlSmallInt, NameLength2: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLDrivers*(EnvironmentHandle: SqlHEnv, Direction: SqlUSmallInt, - DriverDescription: PSQLCHAR, BufferLength1: TSqlSmallInt, - DescriptionLength1: PSQLSMALLINT, DriverAttributes: PSQLCHAR, - BufferLength2: TSqlSmallInt, AttributesLength2: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetConnectAttr*(ConnectionHandle: SqlHDBC, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR, - BufferLength: TSqlSmallInt, NameLength: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR, - NameLength: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLRowCount*(StatementHandle: SqlHStmt, RowCount: var TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLBindParameter*(hstmt: SqlHStmt, ipar: SqlUSmallInt, - fParamType: TSqlSmallInt, fCType: TSqlSmallInt, - fSqlType: TSqlSmallInt, cbColDef: SqlUInteger, - ibScale: TSqlSmallInt, rgbValue: SqlPointer, - cbValueMax: TSqlInteger, pcbValue: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLFreeStmt*(StatementHandle: SqlHStmt, Option: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLColAttribute*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - FieldIdentifier: SqlUSmallInt, - CharacterAttribute: PSQLCHAR, BufferLength: TSqlSmallInt, - StringLength: PSQLSMALLINT, - NumericAttribute: SqlPointer): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLEndTran*(HandleType: TSqlSmallInt, Handle: SqlHandle, - CompletionType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLTables*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt, szTableType: PSQLCHAR, - cbTableType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLColumns*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt, szColumnName: PSQLCHAR, - cbColumnName: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLSpecialColumns*(StatementHandle: SqlHStmt, IdentifierType: SqlUSmallInt, - CatalogName: PSQLCHAR, NameLength1: TSqlSmallInt, - SchemaName: PSQLCHAR, NameLength2: TSqlSmallInt, - TableName: PSQLCHAR, NameLength3: TSqlSmallInt, - Scope: SqlUSmallInt, - Nullable: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLProcedures*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLPrimaryKeys*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, TableName: PSQLCHAR, - NameLength3: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLProcedureColumns*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, ProcName: PSQLCHAR, - NameLength3: TSqlSmallInt, ColumnName: PSQLCHAR, - NameLength4: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, TableName: PSQLCHAR, - NameLength3: TSqlSmallInt, Unique: SqlUSmallInt, - Reserved: SqlUSmallInt): TSqlSmallInt {. - dynlib: odbclib, importc.} -proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt, - szSqlState, pfNativeError, szErrorMsg: PSQLCHAR, - cbErrorMsgMax: TSqlSmallInt, - pcbErrorMsg: PSQLINTEGER): TSqlSmallInt {. - dynlib: odbclib, importc: "SQLError".} - -{.pop.} diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index a227ac98c..9921b7ffd 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -7,35 +7,107 @@ # distribution, for details about the copyright. # -## OpenSSL support +## OpenSSL wrapper. Supports OpenSSL >= 1.1.0 dynamically (as default) or statically linked +## using `--dynlibOverride:ssl`. +## +## `-d:sslVersion=1.2.3` can be used to force an SSL version. +## This version must be included in the library name. +## `-d:useOpenssl3` may be set for OpenSSL 3 instead. +## +## There is also limited support for OpenSSL 1.0.x which may require `-d:openssl10`. +## +## Build and test examples: +## +## ```cmd +## ./bin/nim c -d:ssl -p:. -r tests/stdlib/tssl.nim +## ./bin/nim c -d:ssl --threads:on -p:. -r tests/stdlib/thttpclient_ssl.nim +## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim +## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passl:-lcrypto --passl:-lssl -r tests/untestable/tssl.nim +## ./bin/nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:ssl -p:testament/lib --threads:on tests/untestable/thttpclient_ssl_remotenetwork.nim +## ``` + +# https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html +# +from std/strutils import startsWith + +when defined(nimPreviewSlimSystem): + import std/syncio -{.deadCodeElim: on.} +when defined(nimHasStyleChecks): + {.push styleChecks: off.} -const useWinVersion = defined(Windows) or defined(nimdoc) +const useWinVersion = defined(windows) or defined(nimdoc) -when useWinVersion: - when not defined(nimOldDlls) and defined(cpu64): +# To force openSSL version use -d:sslVersion=1.2.3 +# See: #10281, #10230 +# General issue: +# Other dynamic libraries (like libpg) load different openSSL version then what nim loads. +# Having two different openSSL loaded version causes a crash. +# Use this compile time define to force the openSSL version that your other dynamic libraries want. +const sslVersion {.strdefine.}: string = "" +const useOpenssl3* {.booldefine.} = sslVersion.startsWith('3') +when sslVersion != "": + when defined(macosx): const - DLLSSLName = "(ssleay64|libssl64).dll" - DLLUtilName = "libeay64.dll" + DLLSSLName* = "libssl." & sslVersion & ".dylib" + DLLUtilName* = "libcrypto." & sslVersion & ".dylib" + from std/posix import SocketHandle + elif defined(windows): + const + DLLSSLName* = "libssl-" & sslVersion & ".dll" + DLLUtilName* = "libcrypto-" & sslVersion & ".dll" + from std/winlean import SocketHandle else: const - DLLSSLName = "(ssleay32|libssl32).dll" - DLLUtilName = "libeay32.dll" + DLLSSLName* = "libssl.so." & sslVersion + DLLUtilName* = "libcrypto.so." & sslVersion + from std/posix import SocketHandle + +elif useWinVersion: + when defined(openssl10) or defined(nimOldDlls): + when defined(cpu64): + const + DLLSSLName* = "(ssleay32|ssleay64).dll" + DLLUtilName* = "(libeay32|libeay64).dll" + else: + const + DLLSSLName* = "ssleay32.dll" + DLLUtilName* = "libeay32.dll" + elif defined(cpu64): + const + DLLSSLName* = "(libssl-1_1-x64|ssleay64|libssl64).dll" + DLLUtilName* = "(libcrypto-1_1-x64|libeay64).dll" + else: + const + DLLSSLName* = "(libssl-1_1|ssleay32|libssl32).dll" + DLLUtilName* = "(libcrypto-1_1|libeay32).dll" - from winlean import SocketHandle + from std/winlean import SocketHandle else: - const - versions = "(|.10|.1.0.1|.1.0.0|.0.9.9|.0.9.8)" + # same list of versions but ordered differently? + when defined(osx): + const versions = "(.3|.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.48|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + else: + const versions = "(.3|.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.48|.47|.46|.45|.44|.43|.41|.39|.38|.10|)" + when defined(macosx): const - DLLSSLName = "libssl" & versions & ".dylib" - DLLUtilName = "libcrypto" & versions & ".dylib" + DLLSSLName* = "libssl" & versions & ".dylib" + DLLUtilName* = "libcrypto" & versions & ".dylib" + elif defined(genode): + const + DLLSSLName* = "libssl.lib.so" + DLLUtilName* = "libcrypto.lib.so" else: const - DLLSSLName = "libssl.so" & versions - DLLUtilName = "libcrypto.so" & versions - from posix import SocketHandle + DLLSSLName* = "libssl.so" & versions + DLLUtilName* = "libcrypto.so" & versions + from std/posix import SocketHandle + +import std/dynlib + +{.pragma: lcrypto, cdecl, dynlib: DLLUtilName, importc.} +{.pragma: lssl, cdecl, dynlib: DLLSSLName, importc.} type SslStruct {.final, pure.} = object @@ -43,9 +115,9 @@ type PSslPtr* = ptr SslPtr SslCtx* = SslPtr PSSL_METHOD* = SslPtr + PSTACK* = SslPtr PX509* = SslPtr PX509_NAME* = SslPtr - PEVP_MD* = SslPtr PBIO_METHOD* = SslPtr BIO* = SslPtr EVP_PKEY* = SslPtr @@ -53,16 +125,29 @@ type PASN1_UTCTIME* = SslPtr PASN1_cInt* = SslPtr PPasswdCb* = SslPtr + EVP_MD* = SslPtr + EVP_MD_CTX* = SslPtr + EVP_PKEY_CTX* = SslPtr + ENGINE* = SslPtr PFunction* = proc () {.cdecl.} DES_cblock* = array[0..7, int8] PDES_cblock* = ptr DES_cblock des_ks_struct*{.final.} = object ks*: DES_cblock - weak_key*: cInt + weak_key*: cint des_key_schedule* = array[1..16, des_ks_struct] -{.deprecated: [PSSL: SslPtr, PSSL_CTX: SslCtx, PBIO: BIO].} + pem_password_cb* = proc(buf: cstring, size, rwflag: cint, userdata: pointer): cint {.cdecl.} + + PaddingType* = enum + RSA_PKCS1_PADDING = 1.cint, + RSA_SSLV23_PADDING = 2.cint, + RSA_NO_PADDING = 3.cint, + RSA_PKCS1_OAEP_PADDING = 4.cint, + RSA_X931_PADDING = 5.cint, + RSA_PKCS1_PSS_PADDING = 6.cint + const SSL_SENT_SHUTDOWN* = 1 @@ -120,6 +205,7 @@ const SSL_CTRL_SET_TLSEXT_SERVERNAME_CB = 53 SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG = 54 SSL_CTRL_SET_TLSEXT_HOSTNAME = 55 + SSL_CTRL_SET_ECDH_AUTO* = 94 TLSEXT_NAMETYPE_host_name* = 0 SSL_TLSEXT_ERR_OK* = 0 SSL_TLSEXT_ERR_ALERT_WARNING* = 1 @@ -136,9 +222,13 @@ const SSL_OP_NO_SSLv2* = 0x01000000 SSL_OP_NO_SSLv3* = 0x02000000 SSL_OP_NO_TLSv1* = 0x04000000 + SSL_OP_NO_TLSv1_1* = 0x08000000 SSL_OP_ALL* = 0x000FFFFF SSL_VERIFY_NONE* = 0x00000000 SSL_VERIFY_PEER* = 0x00000001 + SSL_ST_CONNECT* = 0x1000 + SSL_ST_ACCEPT* = 0x2000 + SSL_ST_INIT* = SSL_ST_CONNECT or SSL_ST_ACCEPT OPENSSL_DES_DECRYPT* = 0 OPENSSL_DES_ENCRYPT* = 1 X509_V_OK* = 0 @@ -185,53 +275,252 @@ const BIO_C_DO_STATE_MACHINE = 101 BIO_C_GET_SSL = 110 -proc SSL_library_init*(): cInt{.cdecl, dynlib: DLLSSLName, importc, discardable.} -proc SSL_load_error_strings*(){.cdecl, dynlib: DLLSSLName, importc.} -proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} - -proc SSLv23_client_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv23_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv2_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} -proc SSLv3_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} +# TLS_method(), TLS_server_method(), TLS_client_method() are introduced in 1.1.0 +# and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2 +# SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0 + +const useStaticLink = compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks) + +when useStaticLink: + # Static linking + + when defined(openssl10): + proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} + proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLeay(): culong {.cdecl, dynlib: DLLUtilName, importc.} + + proc getOpenSSLVersion*(): culong = + SSLeay() + + proc ERR_load_BIO_strings*() {.cdecl, dynlib: DLLUtilName, importc.} + else: + proc OPENSSL_init_ssl*(opts: uint64, settings: uint8): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} + proc SSL_library_init*(): cint {.discardable.} = + ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 + return OPENSSL_init_ssl(0.uint64, 0.uint8) + + proc TLS_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv23_method*(): PSSL_METHOD = + TLS_method() + + proc OpenSSL_version_num(): culong {.cdecl, dynlib: DLLUtilName, importc.} + + proc getOpenSSLVersion*(): culong = + ## Return OpenSSL version as unsigned long + OpenSSL_version_num() + + proc SSL_load_error_strings*() = + ## Removed from OpenSSL 1.1.0 + # This proc prevents breaking existing code calling SslLoadErrorStrings + # Static linking against OpenSSL < 1.1.0 is not supported + discard + + proc ERR_load_BIO_strings*() = + discard + + when defined(libressl) or defined(openssl10): + proc SSL_state(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_in_init*(ssl: SslPtr): cint {.inline.} = + SSL_state(ssl) and SSL_ST_INIT + else: + proc SSL_in_init*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + + template OpenSSL_add_all_algorithms*() = discard + + proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} + +else: + # Here we're trying to stay compatible between openssl versions. Some + # symbols are loaded dynamically and we don't use them if not found. + proc thisModule(): LibHandle {.inline.} = + var thisMod {.global.}: LibHandle + if thisMod.isNil: thisMod = loadLib() + + result = thisMod + + proc sslModule(): LibHandle {.inline, raises: [LibraryError], tags:[RootEffect].} = + var sslMod {.global.}: LibHandle + try: + if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + except: + raise newException(LibraryError, "Could not load SSL using " & DLLSSLName) + + result = sslMod + + proc utilModule(): LibHandle {.inline.} = + var utilMod {.global.}: LibHandle + if utilMod.isNil: utilMod = loadLibPattern(DLLUtilName) + + result = utilMod + + proc symNullable(dll: LibHandle, name: string, alternativeName = ""): pointer = + # Load from DLL. + if not dll.isNil: + result = symAddr(dll, name) + if result.isNil and alternativeName.len > 0: + result = symAddr(dll, alternativeName) + + # Attempt to load from current exe. + if result.isNil: + let thisDynlib = thisModule() + if thisDynlib.isNil: return nil + result = symAddr(thisDynlib, name) + if result.isNil and alternativeName.len > 0: + result = symAddr(thisDynlib, alternativeName) + + proc sslSymNullable(name: string, alternativeName = ""): pointer {.raises: [LibraryError], tags:[RootEffect].} = + sslModule().symNullable(name, alternativeName) + + proc sslSymThrows(name: string, alternativeName = ""): pointer {.raises: [LibraryError].} = + result = sslSymNullable(name, alternativeName) + if result.isNil: raiseInvalidLibrary(name) + + proc utilSymNullable(name: string, alternativeName = ""): pointer = + utilModule().symNullable(name, alternativeName) + + proc loadPSSLMethod(method1, method2: string): PSSL_METHOD {.raises: [LibraryError], tags:[RootEffect].} = + ## Load <method1> from OpenSSL if available, otherwise <method2> + ## + let methodSym = sslSymNullable(method1, method2) + if methodSym.isNil: + raise newException(LibraryError, "Could not load " & method1 & " nor " & method2) + + let method2Proc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe, raises: [].}](methodSym) + return method2Proc() + + proc CRYPTO_set_mem_functions(a,b,c: pointer) = + let theProc = cast[proc(a,b,c: pointer) {.cdecl.}](utilModule().symNullable("CRYPTO_set_mem_functions")) + if not theProc.isNil: theProc(a, b, c) + + proc SSL_library_init*(): cint {.discardable.} = + ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise + ## SSL_library_init + let newInitSym = sslSymNullable("OPENSSL_init_ssl") + if not newInitSym.isNil: + let newInitProc = + cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](newInitSym) + return newInitProc(0, 0) + let olderProc = cast[proc(): cint {.cdecl.}](sslSymThrows("SSL_library_init")) + if not olderProc.isNil: result = olderProc() + + proc SSL_load_error_strings*() = + # TODO: Are we ignoring this on purpose? SSL GitHub CI fails otherwise. + let theProc = cast[proc() {.cdecl.}](sslSymNullable("SSL_load_error_strings")) + if not theProc.isNil: theProc() + + proc ERR_load_BIO_strings*() = + let theProc = cast[proc() {.cdecl.}](utilModule().symNullable("ERR_load_BIO_strings")) + if not theProc.isNil: theProc() + + proc SSLv23_client_method*(): PSSL_METHOD = + loadPSSLMethod("SSLv23_client_method", "TLS_client_method") + + proc SSLv23_method*(): PSSL_METHOD = + loadPSSLMethod("SSLv23_method", "TLS_method") + + proc SSLv2_method*(): PSSL_METHOD = + loadPSSLMethod("SSLv2_method", "TLS_method") + + proc SSLv3_method*(): PSSL_METHOD = + loadPSSLMethod("SSLv3_method", "TLS_method") + + proc TLS_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_method", "SSLv23_method") + + proc TLS_client_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_client_method", "SSLv23_client_method") + + proc TLS_server_method*(): PSSL_METHOD = + loadPSSLMethod("TLS_server_method", "SSLv23_server_method") + + proc OpenSSL_add_all_algorithms*() = + # TODO: Are we ignoring this on purpose? SSL GitHub CI fails otherwise. + let theProc = cast[proc() {.cdecl.}](sslSymNullable("OPENSSL_add_all_algorithms_conf")) + if not theProc.isNil: theProc() + + proc getOpenSSLVersion*(): culong = + ## Return OpenSSL version as unsigned long or 0 if not available + let theProc = cast[proc(): culong {.cdecl, gcsafe.}](utilSymNullable("OpenSSL_version_num", "SSLeay")) + result = + if theProc.isNil: 0.culong + else: theProc() + + proc SSL_in_init*(ssl: SslPtr): cint = + # A compatibility wrapper for `SSL_in_init()` for OpenSSL 1.0, 1.1 and LibreSSL + const MainProc = "SSL_in_init" + let + theProc {.global.} = cast[proc(ssl: SslPtr): cint {.cdecl, gcsafe.}](sslSymNullable(MainProc)) + # Fallback + sslState {.global.} = cast[proc(ssl: SslPtr): cint {.cdecl, gcsafe.}](sslSymNullable("SSL_state")) + + if not theProc.isNil: + result = theProc(ssl) + elif not sslState.isNil: + result = sslState(ssl) and SSL_ST_INIT + else: + raiseInvalidLibrary MainProc + + proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint = + var theProc {.global.}: proc(ctx: SslCtx, str: cstring): cint {.cdecl, gcsafe.} + if theProc.isNil: + theProc = cast[typeof(theProc)](sslSymThrows("SSL_CTX_set_ciphersuites")) + result = theProc(ctx, str) + proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get_SSL_CTX*(ssl: SslPtr): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_SSL_CTX*(ssl: SslPtr, ctx: SslCtx): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_session_id_context*(context: SslCtx, sid_ctx: string, sid_ctx_len: int){.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_verified_chain*(ssl: SslPtr): PSTACK {.cdecl, dynlib: DLLSSLName, + importc.} proc SSL_CTX_new*(meth: PSSL_METHOD): SslCtx{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_load_verify_locations*(ctx: SslCtx, CAfile: cstring, - CApath: cstring): cInt{.cdecl, dynlib: DLLSSLName, importc.} + CApath: cstring): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_free*(arg0: SslCtx){.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_set_verify*(s: SslCtx, mode: int, cb: proc (a: int, b: pointer): int {.cdecl.}){.cdecl, dynlib: DLLSSLName, importc.} proc SSL_get_verify_result*(ssl: SslPtr): int{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_set_cipher_list*(s: SslCtx, ciphers: cstring): cint{.cdecl, dynlib: DLLSSLName, importc.} -proc SSL_CTX_use_certificate_file*(ctx: SslCtx, filename: cstring, typ: cInt): cInt{. +proc SSL_CTX_use_certificate_file*(ctx: SslCtx, filename: cstring, typ: cint): cint{. stdcall, dynlib: DLLSSLName, importc.} -proc SSL_CTX_use_certificate_chain_file*(ctx: SslCtx, filename: cstring): cInt{. +proc SSL_CTX_use_certificate_chain_file*(ctx: SslCtx, filename: cstring): cint{. stdcall, dynlib: DLLSSLName, importc.} proc SSL_CTX_use_PrivateKey_file*(ctx: SslCtx, - filename: cstring, typ: cInt): cInt{.cdecl, dynlib: DLLSSLName, importc.} -proc SSL_CTX_check_private_key*(ctx: SslCtx): cInt{.cdecl, dynlib: DLLSSLName, + filename: cstring, typ: cint): cint{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_check_private_key*(ctx: SslCtx): cint{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_get_ex_new_index*(argl: clong, argp: pointer, new_func: pointer, dup_func: pointer, free_func: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_ex_data*(ssl: SslCtx, idx: cint, arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_get_ex_data*(ssl: SslCtx, idx: cint): pointer {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_set_fd*(ssl: SslPtr, fd: SocketHandle): cint{.cdecl, dynlib: DLLSSLName, importc.} -proc SSL_shutdown*(ssl: SslPtr): cInt{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_shutdown*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_set_shutdown*(ssl: SslPtr, mode: cint) {.cdecl, dynlib: DLLSSLName, importc: "SSL_set_shutdown".} proc SSL_get_shutdown*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc: "SSL_get_shutdown".} proc SSL_connect*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_read*(ssl: SslPtr, buf: pointer, num: int): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_write*(ssl: SslPtr, buf: cstring, num: int): cint{.cdecl, dynlib: DLLSSLName, importc.} -proc SSL_get_error*(s: SslPtr, ret_code: cInt): cInt{.cdecl, dynlib: DLLSSLName, importc.} -proc SSL_accept*(ssl: SslPtr): cInt{.cdecl, dynlib: DLLSSLName, importc.} -proc SSL_pending*(ssl: SslPtr): cInt{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get_error*(s: SslPtr, ret_code: cint): cint{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_accept*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_pending*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} +proc BIO_new_mem_buf*(data: pointer, len: cint): BIO{.cdecl, + dynlib: DLLUtilName, importc.} proc BIO_new_ssl_connect*(ctx: SslCtx): BIO{.cdecl, dynlib: DLLSSLName, importc.} proc BIO_ctrl*(bio: BIO, cmd: cint, larg: int, arg: cstring): int{.cdecl, - dynlib: DLLSSLName, importc.} + dynlib: DLLUtilName, importc.} proc BIO_get_ssl*(bio: BIO, ssl: ptr SslPtr): int = return BIO_ctrl(bio, BIO_C_GET_SSL, 0, cast[cstring](ssl)) proc BIO_set_conn_hostname*(bio: BIO, name: cstring): int = @@ -241,61 +530,86 @@ proc BIO_do_handshake*(bio: BIO): int = proc BIO_do_connect*(bio: BIO): int = return BIO_do_handshake(bio) -when not defined(nimfix): - proc BIO_read*(b: BIO, data: cstring, length: cInt): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - proc BIO_write*(b: BIO, data: cstring, length: cInt): cInt{.cdecl, - dynlib: DLLUtilName, importc.} +proc BIO_read*(b: BIO, data: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc.} +proc BIO_write*(b: BIO, data: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc.} -proc BIO_free*(b: BIO): cInt{.cdecl, dynlib: DLLUtilName, importc.} +proc BIO_free*(b: BIO): cint{.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_print_errors_fp*(fp: File){.cdecl, dynlib: DLLSSLName, importc.} +proc ERR_print_errors_fp*(fp: File){.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl, +proc ERR_error_string*(e: culong, buf: cstring): cstring{.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} +proc ERR_get_error*(): culong{.cdecl, dynlib: DLLUtilName, importc.} +proc ERR_peek_last_error*(): culong{.cdecl, dynlib: DLLUtilName, importc.} -proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".} +proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLUtilName, importc.} -proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} +proc OPENSSL_sk_num*(stack: PSTACK): int {.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion: - proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, +proc OPENSSL_sk_value*(stack: PSTACK, index: int): pointer {.cdecl, + dynlib: DLLSSLName, importc.} + +proc d2i_X509*(px: ptr PX509, i: ptr ptr uint8, len: cint): PX509 {.cdecl, dynlib: DLLUtilName, importc.} - proc allocWrapper(size: int): pointer {.cdecl.} = alloc(size) - proc reallocWrapper(p: pointer; newsize: int): pointer {.cdecl.} = +proc i2d_X509*(cert: PX509; o: ptr ptr uint8): cint {.cdecl, + dynlib: DLLUtilName, importc.} + +proc d2i_X509*(b: string): PX509 = + ## decode DER/BER bytestring into X.509 certificate struct + var bb = b.cstring + let i = cast[ptr ptr uint8](addr bb) + let ret = d2i_X509(addr result, i, b.len.cint) + if ret.isNil: + raise newException(Exception, "X.509 certificate decoding failed") + +proc i2d_X509*(cert: PX509): string = + ## encode `cert` to DER string + let encoded_length = i2d_X509(cert, nil) + result = newString(encoded_length) + var q = result.cstring + let o = cast[ptr ptr uint8](addr q) + let length = i2d_X509(cert, o) + if length.int <= 0: + raise newException(Exception, "X.509 certificate encoding failed") + +const + useNimsAlloc = not defined(nimNoAllocForSSL) and not defined(gcDestructors) + +when not useWinVersion and not defined(macosx) and not defined(android) and useNimsAlloc: + proc allocWrapper(size: int): pointer {.cdecl.} = allocShared(size) + proc reallocWrapper(p: pointer; newSize: int): pointer {.cdecl.} = if p == nil: - if newSize > 0: result = alloc(newsize) - elif newsize == 0: dealloc(p) - else: result = realloc(p, newsize) + if newSize > 0: result = allocShared(newSize) + elif newSize == 0: deallocShared(p) + else: result = reallocShared(p, newSize) proc deallocWrapper(p: pointer) {.cdecl.} = - if p != nil: dealloc(p) + if p != nil: deallocShared(p) -proc CRYPTO_malloc_init*() = - when not useWinVersion and not defined(macosx): - CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper) + proc CRYPTO_malloc_init*() = + CRYPTO_set_mem_functions(cast[pointer](allocWrapper), cast[pointer](reallocWrapper), cast[pointer](deallocWrapper)) +else: + proc CRYPTO_malloc_init*() = + discard -proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cInt, larg: int, parg: pointer): int{. +proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cint, larg: clong, parg: pointer): clong{. cdecl, dynlib: DLLSSLName, importc.} -proc SSL_CTX_callback_ctrl(ctx: SslCtx, typ: cInt, fp: PFunction): int{. +proc SSL_CTX_callback_ctrl(ctx: SslCtx, typ: cint, fp: PFunction): int{. cdecl, dynlib: DLLSSLName, importc.} proc SSLCTXSetMode*(ctx: SslCtx, mode: int): int = - result = SSL_CTX_ctrl(ctx, SSL_CTRL_MODE, mode, nil) + result = SSL_CTX_ctrl(ctx, SSL_CTRL_MODE, clong mode, nil) -proc SSL_ctrl*(ssl: SslPtr, cmd: cInt, larg: int, parg: pointer): int{. +proc SSL_ctrl*(ssl: SslPtr, cmd: cint, larg: int, parg: pointer): int{. cdecl, dynlib: DLLSSLName, importc.} proc SSL_set_tlsext_host_name*(ssl: SslPtr, name: cstring): int = - result = SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, name) ## Set the SNI server name extension to be used in a client hello. ## Returns 1 if SNI was set, 0 if current SSL configuration doesn't support SNI. + result = SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, name) - -proc SSL_get_servername*(ssl: SslPtr, typ: cInt = TLSEXT_NAMETYPE_host_name): cstring {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get_servername*(ssl: SslPtr, typ: cint = TLSEXT_NAMETYPE_host_name): cstring {.cdecl, dynlib: DLLSSLName, importc.} ## Retrieve the server name requested in the client hello. This can be used ## in the callback set in `SSL_CTX_set_tlsext_servername_callback` to ## implement virtual hosting. May return `nil`. @@ -311,17 +625,45 @@ proc SSL_CTX_set_tlsext_servername_callback*(ctx: SslCtx, cb: proc(ssl: SslPtr, result = SSL_CTX_callback_ctrl(ctx, SSL_CTRL_SET_TLSEXT_SERVERNAME_CB, cast[PFunction](cb)) proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int = - ## Set the pointer to be used in the callback registered to ``SSL_CTX_set_tlsext_servername_callback``. + ## Set the pointer to be used in the callback registered to `SSL_CTX_set_tlsext_servername_callback`. result = SSL_CTX_ctrl(ctx, SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG, 0, arg) +type + PskClientCallback* = proc (ssl: SslPtr; + hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr uint8; + max_psk_len: cuint): cuint {.cdecl.} + + PskServerCallback* = proc (ssl: SslPtr; + identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.} + +proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.} + ## Set callback called when OpenSSL needs PSK (for client). + +proc SSL_CTX_set_psk_server_callback*(ctx: SslCtx; callback: PskServerCallback) {.cdecl, dynlib: DLLSSLName, importc.} + ## Set callback called when OpenSSL needs PSK (for server). + +proc SSL_CTX_use_psk_identity_hint*(ctx: SslCtx; hint: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + ## Set PSK identity hint to use. + +proc SSL_get_psk_identity*(ssl: SslPtr): cstring {.cdecl, dynlib: DLLSSLName, importc.} + ## Get PSK identity. + +proc SSL_CTX_set_ecdh_auto*(ctx: SslCtx, onoff: cint): cint {.inline.} = + ## Set automatic curve selection. + ## + ## On OpenSSL >= 1.1.0 this is on by default and cannot be disabled. + if getOpenSSLVersion() < 0x010100000 or getOpenSSLVersion() == 0x020000000: + result = cint SSL_CTX_ctrl(ctx, SSL_CTRL_SET_ECDH_AUTO, onoff, nil) + else: + result = 1 proc bioNew*(b: PBIO_METHOD): BIO{.cdecl, dynlib: DLLUtilName, importc: "BIO_new".} proc bioFreeAll*(b: BIO){.cdecl, dynlib: DLLUtilName, importc: "BIO_free_all".} proc bioSMem*(): PBIO_METHOD{.cdecl, dynlib: DLLUtilName, importc: "BIO_s_mem".} -proc bioCtrlPending*(b: BIO): cInt{.cdecl, dynlib: DLLUtilName, importc: "BIO_ctrl_pending".} -proc bioRead*(b: BIO, Buf: cstring, length: cInt): cInt{.cdecl, +proc bioCtrlPending*(b: BIO): cint{.cdecl, dynlib: DLLUtilName, importc: "BIO_ctrl_pending".} +proc bioRead*(b: BIO, Buf: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc: "BIO_read".} -proc bioWrite*(b: BIO, Buf: cstring, length: cInt): cInt{.cdecl, +proc bioWrite*(b: BIO, Buf: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc: "BIO_write".} proc sslSetConnectState*(s: SslPtr) {.cdecl, @@ -329,11 +671,11 @@ proc sslSetConnectState*(s: SslPtr) {.cdecl, proc sslSetAcceptState*(s: SslPtr) {.cdecl, dynlib: DLLSSLName, importc: "SSL_set_accept_state".} -proc sslRead*(ssl: SslPtr, buf: cstring, num: cInt): cInt{.cdecl, +proc sslRead*(ssl: SslPtr, buf: cstring, num: cint): cint{.cdecl, dynlib: DLLSSLName, importc: "SSL_read".} -proc sslPeek*(ssl: SslPtr, buf: cstring, num: cInt): cInt{.cdecl, +proc sslPeek*(ssl: SslPtr, buf: cstring, num: cint): cint{.cdecl, dynlib: DLLSSLName, importc: "SSL_peek".} -proc sslWrite*(ssl: SslPtr, buf: cstring, num: cInt): cInt{.cdecl, +proc sslWrite*(ssl: SslPtr, buf: cstring, num: cint): cint{.cdecl, dynlib: DLLSSLName, importc: "SSL_write".} proc sslSetBio*(ssl: SslPtr, rbio, wbio: BIO) {.cdecl, @@ -343,145 +685,78 @@ proc sslDoHandshake*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc: "SSL_do_handshake".} - proc ErrClearError*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_clear_error".} proc ErrFreeStrings*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_free_strings".} -proc ErrRemoveState*(pid: cInt){.cdecl, dynlib: DLLUtilName, importc: "ERR_remove_state".} +proc ErrRemoveState*(pid: cint){.cdecl, dynlib: DLLUtilName, importc: "ERR_remove_state".} -when true: - discard +proc PEM_read_bio_RSA_PUBKEY*(bp: BIO, x: ptr PRSA, pw: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_RSA_PUBKEY*(fp: pointer; x: ptr PRSA; cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_verify*(kind: cint, origMsg: pointer, origMsgLen: cuint, signature: pointer, + signatureLen: cuint, rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} +proc PEM_read_RSAPrivateKey*(fp: pointer; x: ptr PRSA; cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_RSAPublicKey*(fp: pointer; x: ptr PRSA; cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_bio_RSAPublicKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_bio_RSAPrivateKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_private_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_public_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_private_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_public_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.} +proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} + +# sha types +proc EVP_md_null*(): EVP_MD {.lcrypto.} +proc EVP_md2*(): EVP_MD {.lcrypto.} +proc EVP_md4*(): EVP_MD {.lcrypto.} +proc EVP_md5*(): EVP_MD {.lcrypto.} +proc EVP_sha*(): EVP_MD {.lcrypto.} +proc EVP_sha1*(): EVP_MD {.lcrypto.} +proc EVP_dss*(): EVP_MD {.lcrypto.} +proc EVP_dss1*(): EVP_MD {.lcrypto.} +proc EVP_ecdsa*(): EVP_MD {.lcrypto.} +proc EVP_sha224*(): EVP_MD {.lcrypto.} +proc EVP_sha256*(): EVP_MD {.lcrypto.} +proc EVP_sha384*(): EVP_MD {.lcrypto.} +proc EVP_sha512*(): EVP_MD {.lcrypto.} +proc EVP_mdc2*(): EVP_MD {.lcrypto.} +proc EVP_ripemd160*(): EVP_MD {.lcrypto.} +proc EVP_whirlpool*(): EVP_MD {.lcrypto.} +proc EVP_MD_size*(md: EVP_MD): cint {.lcrypto.} + +# hmac functions +proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.lcrypto.} + +# RSA key functions +proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.lcrypto.} +proc EVP_PKEY_free*(p: EVP_PKEY) {.lcrypto.} +proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.lcrypto.} +proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: EVP_MD, engine: SslPtr = nil): cint {.lcrypto.} +proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.lcrypto.} +proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.lcrypto.} +proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.lcrypto.} +proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.lcrypto.} +proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.lcrypto.} +proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.lcrypto.} + +when defined(macosx) or defined(windows): + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.lcrypto.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.lcrypto.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.lcrypto.} else: - proc SslCtxSetCipherList*(arg0: PSSL_CTX, str: cstring): cInt{.cdecl, - dynlib: DLLSSLName, importc.} - proc SslCtxNew*(meth: PSSL_METHOD): PSSL_CTX{.cdecl, - dynlib: DLLSSLName, importc.} - - proc SslSetFd*(s: PSSL, fd: cInt): cInt{.cdecl, dynlib: DLLSSLName, importc.} - proc SslCTXCtrl*(ctx: PSSL_CTX, cmd: cInt, larg: int, parg: Pointer): int{. - cdecl, dynlib: DLLSSLName, importc.} - - proc SSLSetMode*(s: PSSL, mode: int): int - proc SSLCTXGetMode*(ctx: PSSL_CTX): int - proc SSLGetMode*(s: PSSL): int - proc SslMethodV2*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} - proc SslMethodV3*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} - proc SslMethodTLSV1*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} - proc SslMethodV23*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} - proc SslCtxUsePrivateKey*(ctx: PSSL_CTX, pkey: SslPtr): cInt{.cdecl, - dynlib: DLLSSLName, importc.} - proc SslCtxUsePrivateKeyASN1*(pk: cInt, ctx: PSSL_CTX, - d: cstring, length: int): cInt{.cdecl, dynlib: DLLSSLName, importc.} - - proc SslCtxUseCertificate*(ctx: PSSL_CTX, x: SslPtr): cInt{.cdecl, - dynlib: DLLSSLName, importc.} - proc SslCtxUseCertificateASN1*(ctx: PSSL_CTX, length: int, d: cstring): cInt{. - cdecl, dynlib: DLLSSLName, importc.} - - # function SslCtxUseCertificateChainFile(ctx: PSSL_CTX; const filename: PChar):cInt; - proc SslCtxUseCertificateChainFile*(ctx: PSSL_CTX, filename: cstring): cInt{. - cdecl, dynlib: DLLSSLName, importc.} - proc SslCtxSetDefaultPasswdCb*(ctx: PSSL_CTX, cb: PPasswdCb){.cdecl, - dynlib: DLLSSLName, importc.} - proc SslCtxSetDefaultPasswdCbUserdata*(ctx: PSSL_CTX, u: SslPtr){.cdecl, - dynlib: DLLSSLName, importc.} - # function SslCtxLoadVerifyLocations(ctx: PSSL_CTX; const CAfile: PChar; const CApath: PChar):cInt; - proc SslCtxLoadVerifyLocations*(ctx: PSSL_CTX, CAfile: cstring, CApath: cstring): cInt{. - cdecl, dynlib: DLLSSLName, importc.} - proc SslNew*(ctx: PSSL_CTX): PSSL{.cdecl, dynlib: DLLSSLName, importc.} - - - proc SslConnect*(ssl: PSSL): cInt{.cdecl, dynlib: DLLSSLName, importc.} - - - proc SslGetVersion*(ssl: PSSL): cstring{.cdecl, dynlib: DLLSSLName, importc.} - proc SslGetPeerCertificate*(ssl: PSSL): PX509{.cdecl, dynlib: DLLSSLName, - importc.} - proc SslCtxSetVerify*(ctx: PSSL_CTX, mode: cInt, arg2: PFunction){.cdecl, - dynlib: DLLSSLName, importc.} - proc SSLGetCurrentCipher*(s: PSSL): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} - proc SSLCipherGetName*(c: SslPtr): cstring{.cdecl, dynlib: DLLSSLName, importc.} - proc SSLCipherGetBits*(c: SslPtr, alg_bits: var cInt): cInt{.cdecl, - dynlib: DLLSSLName, importc.} - proc SSLGetVerifyResult*(ssl: PSSL): int{.cdecl, dynlib: DLLSSLName, importc.} - # libeay.dll - proc X509New*(): PX509{.cdecl, dynlib: DLLUtilName, importc.} - proc X509Free*(x: PX509){.cdecl, dynlib: DLLUtilName, importc.} - proc X509NameOneline*(a: PX509_NAME, buf: cstring, size: cInt): cstring{. - cdecl, dynlib: DLLUtilName, importc.} - proc X509GetSubjectName*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLUtilName, - importc.} - proc X509GetIssuerName*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLUtilName, - importc.} - proc X509NameHash*(x: PX509_NAME): int{.cdecl, dynlib: DLLUtilName, importc.} - # function SslX509Digest(data: PX509; typ: PEVP_MD; md: PChar; len: PcInt):cInt; - proc X509Digest*(data: PX509, typ: PEVP_MD, md: cstring, length: var cInt): cInt{. - cdecl, dynlib: DLLUtilName, importc.} - proc X509print*(b: PBIO, a: PX509): cInt{.cdecl, dynlib: DLLUtilName, importc.} - proc X509SetVersion*(x: PX509, version: cInt): cInt{.cdecl, dynlib: DLLUtilName, - importc.} - proc X509SetPubkey*(x: PX509, pkey: EVP_PKEY): cInt{.cdecl, dynlib: DLLUtilName, - importc.} - proc X509SetIssuerName*(x: PX509, name: PX509_NAME): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - proc X509NameAddEntryByTxt*(name: PX509_NAME, field: cstring, typ: cInt, - bytes: cstring, length, loc, theSet: cInt): cInt{. - cdecl, dynlib: DLLUtilName, importc.} - proc X509Sign*(x: PX509, pkey: EVP_PKEY, md: PEVP_MD): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - proc X509GmtimeAdj*(s: PASN1_UTCTIME, adj: cInt): PASN1_UTCTIME{.cdecl, - dynlib: DLLUtilName, importc.} - proc X509SetNotBefore*(x: PX509, tm: PASN1_UTCTIME): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - proc X509SetNotAfter*(x: PX509, tm: PASN1_UTCTIME): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - proc X509GetSerialNumber*(x: PX509): PASN1_cInt{.cdecl, dynlib: DLLUtilName, - importc.} - proc EvpPkeyNew*(): EVP_PKEY{.cdecl, dynlib: DLLUtilName, importc.} - proc EvpPkeyFree*(pk: EVP_PKEY){.cdecl, dynlib: DLLUtilName, importc.} - proc EvpPkeyAssign*(pkey: EVP_PKEY, typ: cInt, key: Prsa): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - proc EvpGetDigestByName*(Name: cstring): PEVP_MD{.cdecl, dynlib: DLLUtilName, - importc.} - proc EVPcleanup*(){.cdecl, dynlib: DLLUtilName, importc.} - # function ErrErrorString(e: cInt; buf: PChar): PChar; - proc SSLeayversion*(t: cInt): cstring{.cdecl, dynlib: DLLUtilName, importc.} - - - proc OPENSSLaddallalgorithms*(){.cdecl, dynlib: DLLUtilName, importc.} - proc CRYPTOcleanupAllExData*(){.cdecl, dynlib: DLLUtilName, importc.} - proc RandScreen*(){.cdecl, dynlib: DLLUtilName, importc.} - - proc d2iPKCS12bio*(b: PBIO, Pkcs12: SslPtr): SslPtr{.cdecl, dynlib: DLLUtilName, - importc.} - proc PKCS12parse*(p12: SslPtr, pass: cstring, pkey, cert, ca: var SslPtr): cint{. - dynlib: DLLUtilName, importc, cdecl.} - - proc PKCS12free*(p12: SslPtr){.cdecl, dynlib: DLLUtilName, importc.} - proc RsaGenerateKey*(bits, e: cInt, callback: PFunction, cb_arg: SslPtr): PRSA{. - cdecl, dynlib: DLLUtilName, importc.} - proc Asn1UtctimeNew*(): PASN1_UTCTIME{.cdecl, dynlib: DLLUtilName, importc.} - proc Asn1UtctimeFree*(a: PASN1_UTCTIME){.cdecl, dynlib: DLLUtilName, importc.} - proc Asn1cIntSet*(a: PASN1_cInt, v: cInt): cInt{.cdecl, dynlib: DLLUtilName, - importc.} - proc i2dX509bio*(b: PBIO, x: PX509): cInt{.cdecl, dynlib: DLLUtilName, importc.} - proc i2dPrivateKeyBio*(b: PBIO, pkey: EVP_PKEY): cInt{.cdecl, - dynlib: DLLUtilName, importc.} - # 3DES functions - proc DESsetoddparity*(Key: des_cblock){.cdecl, dynlib: DLLUtilName, importc.} - proc DESsetkeychecked*(key: des_cblock, schedule: des_key_schedule): cInt{. - cdecl, dynlib: DLLUtilName, importc.} - proc DESecbencrypt*(Input: des_cblock, output: des_cblock, ks: des_key_schedule, - enc: cInt){.cdecl, dynlib: DLLUtilName, importc.} - # implementation - - proc SSLSetMode(s: PSSL, mode: int): int = - result = SSLctrl(s, SSL_CTRL_MODE, mode, nil) - - proc SSLCTXGetMode(ctx: PSSL_CTX): int = - result = SSLCTXctrl(ctx, SSL_CTRL_MODE, 0, nil) - - proc SSLGetMode(s: PSSL): int = - result = SSLctrl(s, SSL_CTRL_MODE, 0, nil) + # some times you will need this instead: + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new", dynlib: DLLUtilName.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free", dynlib: DLLUtilName.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup", dynlib: DLLUtilName.} # <openssl/md5.h> type @@ -496,24 +771,23 @@ type data: array[MD5_LBLOCK, MD5_LONG] num: cuint -{.pragma: ic, importc: "$1".} {.push callconv:cdecl, dynlib:DLLUtilName.} -proc md5_Init*(c: var MD5_CTX): cint{.ic.} -proc md5_Update*(c: var MD5_CTX; data: pointer; len: csize): cint{.ic.} -proc md5_Final*(md: cstring; c: var MD5_CTX): cint{.ic.} -proc md5*(d: ptr cuchar; n: csize; md: ptr cuchar): ptr cuchar{.ic.} -proc md5_Transform*(c: var MD5_CTX; b: ptr cuchar){.ic.} +proc md5_Init*(c: var MD5_CTX): cint{.importc: "MD5_Init".} +proc md5_Update*(c: var MD5_CTX; data: pointer; len: csize_t): cint{.importc: "MD5_Update".} +proc md5_Final*(md: cstring; c: var MD5_CTX): cint{.importc: "MD5_Final".} +proc md5*(d: ptr uint8; n: csize_t; md: ptr uint8): ptr uint8{.importc: "MD5".} +proc md5_Transform*(c: var MD5_CTX; b: ptr uint8){.importc: "MD5_Transform".} {.pop.} -from strutils import toHex,toLower +from std/strutils import toHex, toLowerAscii -proc hexStr (buf:cstring): string = +proc hexStr(buf: cstring): string = # turn md5s output into a nice hex str result = newStringOfCap(32) - for i in 0 .. <16: - result.add toHex(buf[i].ord, 2).toLower + for i in 0 ..< 16: + result.add toHex(buf[i].ord, 2).toLowerAscii -proc md5_File* (file: string): string {.raises: [IOError,Exception].} = +proc md5_File*(file: string): string {.raises: [IOError,Exception].} = ## Generate MD5 hash for a file. Result is a 32 character # hex string with lowercase characters (like the output # of `md5sum` @@ -524,29 +798,112 @@ proc md5_File* (file: string): string {.raises: [IOError,Exception].} = buf: array[sz,char] ctx: MD5_CTX - discard md5_init(ctx) - while(let bytes = f.readChars(buf, 0, sz); bytes > 0): - discard md5_update(ctx, buf[0].addr, bytes) + discard md5_Init(ctx) + while (let bytes = f.readChars(buf); bytes > 0): + discard md5_Update(ctx, buf[0].addr, cast[csize_t](bytes)) - discard md5_final( buf[0].addr, ctx ) + discard md5_Final(cast[cstring](buf[0].addr), ctx) f.close - result = hexStr(buf) + result = hexStr(cast[cstring](addr buf)) -proc md5_Str* (str:string): string {.raises:[IOError].} = - ##Generate MD5 hash for a string. Result is a 32 character - #hex string with lowercase characters +proc md5_Str*(str: string): string = + ## Generate MD5 hash for a string. Result is a 32 character + ## hex string with lowercase characters var ctx: MD5_CTX res: array[MD5_DIGEST_LENGTH,char] input = str.cstring - discard md5_init(ctx) + discard md5_Init(ctx) var i = 0 while i < str.len: let L = min(str.len - i, 512) - discard md5_update(ctx, input[i].addr, L) + discard md5_Update(ctx, input[i].addr, cast[csize_t](L)) i += L - discard md5_final(res,ctx) - result = hexStr(res) + discard md5_Final(cast[cstring](addr res), ctx) + result = hexStr(cast[cstring](addr res)) + +when defined(nimHasStyleChecks): + {.pop.} + + +# Certificate validation +# On old openSSL version some of these symbols are not available +when not defined(nimDisableCertificateValidation) and not defined(windows): + + # SSL_get_peer_certificate removed in 3.0 + # SSL_get1_peer_certificate added in 3.0 + when useOpenssl3: + proc SSL_get1_peer_certificate*(ssl: SslCtx): PX509 {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 = + SSL_get1_peer_certificate(ssl) + elif useStaticLink: + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 {.cdecl, dynlib: DLLSSLName, importc.} + else: + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 = + let methodSym = sslSymNullable("SSL_get_peer_certificate", "SSL_get1_peer_certificate") + if methodSym.isNil: + raise newException(LibraryError, "Could not load SSL_get_peer_certificate or SSL_get1_peer_certificate") + let method2Proc = cast[proc(ssl: SslCtx): PX509 {.cdecl, gcsafe, raises: [].}](methodSym) + return method2Proc(ssl) + + proc X509_get_subject_name*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLSSLName, importc.} + + proc X509_get_issuer_name*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLUtilName, importc.} + + proc X509_NAME_oneline*(a: PX509_NAME, buf: cstring, size: cint): cstring {. + cdecl, dynlib:DLLSSLName, importc.} + + proc X509_NAME_get_text_by_NID*(subject:cstring, NID: cint, buf: cstring, size: cint): cint{. + cdecl, dynlib:DLLSSLName, importc.} + + proc X509_check_host*(cert: PX509, name: cstring, namelen: cint, flags:cuint, peername: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + + proc X509_free*(cert: PX509) {.cdecl, dynlib: DLLSSLName, importc.} + + # Certificates store + + type PX509_STORE* = SslPtr + type PX509_OBJECT* = SslPtr + + {.push callconv:cdecl, dynlib:DLLUtilName, importc.} + + proc X509_OBJECT_new*(): PX509_OBJECT + proc X509_OBJECT_free*(a: PX509_OBJECT) + + proc X509_STORE_new*(): PX509_STORE + proc X509_STORE_free*(v: PX509_STORE) + proc X509_STORE_lock*(ctx: PX509_STORE): cint + proc X509_STORE_unlock*(ctx: PX509_STORE): cint + proc X509_STORE_up_ref*(v: PX509_STORE): cint + proc X509_STORE_set_flags*(ctx: PX509_STORE; flags: culong): cint + proc X509_STORE_set_purpose*(ctx: PX509_STORE; purpose: cint): cint + proc X509_STORE_set_trust*(ctx: PX509_STORE; trust: cint): cint + proc X509_STORE_add_cert*(ctx: PX509_STORE; x: PX509): cint + + {.pop.} + + when isMainModule: + when defined(nimPreviewSlimSystem): + import std/assertions + # A simple certificate test + let certbytes = readFile("certificate.der") + let cert = d2i_X509(certbytes) + let encoded = cert.i2d_X509() + assert encoded == certbytes + +# Application Layer Protocol Negociation extension (TLS-ALPN, RFC7301) +# Available in at least OpenSSL 1.1.1 and later, not sure if earlier +# --Iced Quinn + +proc SSL_CTX_set_alpn_protos*(ctx: SslCtx; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_alpn_protos*(ssl: SslPtr; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_alpn_select_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_alpn_selected*(ssl: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_protos_advertised_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: ptr cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_proto_select_cb*(ctx: SslCtx; cb: proc(s: SslPtr; out_proto: cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_select_next_proto*(out_proto: ptr cstring; outlen: cstring; server: cstring; server_len: cuint; client: cstring; client_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_next_proto_negotiated*(s: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} + diff --git a/lib/wrappers/pcre.nim b/lib/wrappers/pcre.nim index 6c7088bbf..9144f6784 100644 --- a/lib/wrappers/pcre.nim +++ b/lib/wrappers/pcre.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} - # The current PCRE version information. const @@ -105,7 +103,7 @@ const NOTEMPTY_ATSTART* = 0x10000000 # E D J UCP* = 0x20000000 # C3 -## Exec-time and get/set-time error codes +# Exec-time and get/set-time error codes const ERROR_NOMATCH* = -1 ERROR_NULL* = -2 @@ -146,7 +144,7 @@ const ERROR_BADLENGTH* = -32 ERROR_UNSET* = -33 -## Specific error codes for UTF-8 validity checks +# Specific error codes for UTF-8 validity checks const UTF8_ERR0* = 0 UTF8_ERR1* = 1 @@ -172,7 +170,7 @@ const UTF8_ERR21* = 21 UTF8_ERR22* = 22 # Unused (was non-character) -## Specific error codes for UTF-16 validity checks +# Specific error codes for UTF-16 validity checks const UTF16_ERR0* = 0 UTF16_ERR1* = 1 @@ -180,14 +178,14 @@ const UTF16_ERR3* = 3 UTF16_ERR4* = 4 # Unused (was non-character) -## Specific error codes for UTF-32 validity checks +# Specific error codes for UTF-32 validity checks const UTF32_ERR0* = 0 UTF32_ERR1* = 1 UTF32_ERR2* = 2 # Unused (was non-character) UTF32_ERR3* = 3 -## Request types for pcre_fullinfo() +# Request types for pcre_fullinfo() const INFO_OPTIONS* = 0 INFO_SIZE* = 1 @@ -217,8 +215,8 @@ const INFO_RECURSIONLIMIT* = 24 INFO_MATCH_EMPTY* = 25 -## Request types for pcre_config(). Do not re-arrange, in order to remain -## compatible. +# Request types for pcre_config(). Do not re-arrange, in order to remain +# compatible. const CONFIG_UTF8* = 0 CONFIG_NEWLINE* = 1 @@ -235,16 +233,16 @@ const CONFIG_UTF32* = 12 CONFIG_PARENS_LIMIT* = 13 -## Request types for pcre_study(). Do not re-arrange, in order to remain -## compatible. +# Request types for pcre_study(). Do not re-arrange, in order to remain +# compatible. const STUDY_JIT_COMPILE* = 0x0001 STUDY_JIT_PARTIAL_SOFT_COMPILE* = 0x0002 STUDY_JIT_PARTIAL_HARD_COMPILE* = 0x0004 STUDY_EXTRA_NEEDED* = 0x0008 -## Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine -## these bits, just add new ones on the end, in order to remain compatible. +# Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine +# these bits, just add new ones on the end, in order to remain compatible. const EXTRA_STUDY_DATA* = 0x0001 EXTRA_MATCH_LIMIT* = 0x0002 @@ -254,7 +252,7 @@ const EXTRA_MARK* = 0x0020 EXTRA_EXECUTABLE_JIT* = 0x0040 -## Types +# Types type Pcre* = object Pcre16* = object @@ -263,10 +261,12 @@ type JitStack16* = object JitStack32* = object +when defined(nimHasStyleChecks): + {.push styleChecks: off.} -## The structure for passing additional data to pcre_exec(). This is defined in -## such as way as to be extensible. Always add new fields at the end, in order -## to remain compatible. +# The structure for passing additional data to pcre_exec(). This is defined in +# such as way as to be extensible. Always add new fields at the end, in order +# to remain compatible. type ExtraData* = object flags*: clong ## Bits for which fields are set @@ -278,10 +278,10 @@ type mark*: pointer ## For passing back a mark pointer executable_jit*: pointer ## Contains a pointer to a compiled jit code -## The structure for passing out data via the pcre_callout_function. We use a -## structure so that new fields can be added on the end in future versions, -## without changing the API of the function, thereby allowing old clients to -## work without modification. +# The structure for passing out data via the pcre_callout_function. We use a +# structure so that new fields can be added on the end in future versions, +# without changing the API of the function, thereby allowing old clients to +# work without modification. type CalloutBlock* = object version* : cint ## Identifies version of block @@ -302,8 +302,10 @@ type mark* : pointer ## Pointer to current mark or NULL # ------------------------------------------------------------------ +when defined(nimHasStyleChecks): + {.pop.} -## User defined callback which provides a stack just before the match starts. +# User defined callback which provides a stack just before the match starts. type JitCallback* = proc (a: pointer): ptr JitStack {.cdecl.} @@ -466,14 +468,6 @@ proc study*(code: ptr Pcre, {.pop.} -{.deprecated: [MAJOR: PCRE_MAJOR, MINOR: PCRE_MINOR, - PRERELEASE: PCRE_PRERELEASE, DATE: PCRE_DATE].} - -{.deprecated: [TPcre: Pcre, TJitStack: JitStack].} type PPcre* {.deprecated.} = ptr Pcre PJitStack* {.deprecated.} = ptr JitStack - -{.deprecated: [TExtra: ExtraData].} -{.deprecated: [TCalloutBlock: CalloutBlock].} -{.deprecated: [TJitCallback: JitCallback].} diff --git a/lib/wrappers/pdcurses.nim b/lib/wrappers/pdcurses.nim deleted file mode 100644 index 2d64ac97a..000000000 --- a/lib/wrappers/pdcurses.nim +++ /dev/null @@ -1,1560 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} - -discard """ - -curses.h: -#ifdef C2NIM -#dynlib pdcursesdll -#skipinclude -#prefix PDC_ -#def FALSE -#def TRUE -#def NULL -#def bool unsigned char -#def chtype unsigned long -#def cchar_t unsigned long -#def attr_t unsigned long -#def mmask_t unsigned long -#def wchar_t char -#def PDCEX -#cdecl -#endif - -pdcwin.h: -#ifdef C2NIM -#dynlib pdcursesdll -#skipinclude -#prefix pdc_ -#prefix PDC_ -#stdcall -#endif -""" - -when defined(windows): - import windows - - when defined(nimOldDlls): - const pdcursesdll = "pdcurses.dll" - elif defined(cpu64): - const pdcursesdll = "pdcurses64.dll" - else: - const pdcursesdll = "pdcurses32.dll" - - const - unixOS = false - {.pragma: extdecl, stdcall.} - -when not defined(windows): - const - unixOS = true - {.pragma: extdecl, cdecl.} - -type - cunsignedchar = char - cunsignedlong = uint32 - -const - BUILD* = 3401 - PDCURSES* = 1 # PDCurses-only routines - XOPEN* = 1 # X/Open Curses routines - SYSVcurses* = 1 # System V Curses routines - BSDcurses* = 1 # BSD Curses routines - CHTYPE_LONG* = 1 # size of chtype; long - ERR* = (- 1) - OK* = 0 - BUTTON_RELEASED* = 0x00000000 - BUTTON_PRESSED* = 0x00000001 - BUTTON_CLICKED* = 0x00000002 - BUTTON_DOUBLE_CLICKED* = 0x00000003 - BUTTON_TRIPLE_CLICKED* = 0x00000004 - BUTTON_MOVED* = 0x00000005 # PDCurses - WHEEL_SCROLLED* = 0x00000006 # PDCurses - BUTTON_ACTION_MASK* = 0x00000007 # PDCurses - BUTTON_MODIFIER_MASK* = 0x00000038 # PDCurses - MOUSE_MOVED* = 0x00000008 - MOUSE_POSITION* = 0x00000010 - MOUSE_WHEEL_UP* = 0x00000020 - MOUSE_WHEEL_DOWN* = 0x00000040 - BUTTON1_RELEASED* = 0x00000001 - BUTTON1_PRESSED* = 0x00000002 - BUTTON1_CLICKED* = 0x00000004 - BUTTON1_DOUBLE_CLICKED* = 0x00000008 - BUTTON1_TRIPLE_CLICKED* = 0x00000010 - BUTTON1_MOVED* = 0x00000010 # PDCurses - BUTTON2_RELEASED* = 0x00000020 - BUTTON2_PRESSED* = 0x00000040 - BUTTON2_CLICKED* = 0x00000080 - BUTTON2_DOUBLE_CLICKED* = 0x00000100 - BUTTON2_TRIPLE_CLICKED* = 0x00000200 - BUTTON2_MOVED* = 0x00000200 # PDCurses - BUTTON3_RELEASED* = 0x00000400 - BUTTON3_PRESSED* = 0x00000800 - BUTTON3_CLICKED* = 0x00001000 - BUTTON3_DOUBLE_CLICKED* = 0x00002000 - BUTTON3_TRIPLE_CLICKED* = 0x00004000 - BUTTON3_MOVED* = 0x00004000 # PDCurses - BUTTON4_RELEASED* = 0x00008000 - BUTTON4_PRESSED* = 0x00010000 - BUTTON4_CLICKED* = 0x00020000 - BUTTON4_DOUBLE_CLICKED* = 0x00040000 - BUTTON4_TRIPLE_CLICKED* = 0x00080000 - BUTTON5_RELEASED* = 0x00100000 - BUTTON5_PRESSED* = 0x00200000 - BUTTON5_CLICKED* = 0x00400000 - BUTTON5_DOUBLE_CLICKED* = 0x00800000 - BUTTON5_TRIPLE_CLICKED* = 0x01000000 - MOUSE_WHEEL_SCROLL* = 0x02000000 # PDCurses - BUTTON_MODIFIER_SHIFT* = 0x04000000 # PDCurses - BUTTON_MODIFIER_CONTROL* = 0x08000000 # PDCurses - BUTTON_MODIFIER_ALT* = 0x10000000 # PDCurses - ALL_MOUSE_EVENTS* = 0x1FFFFFFF - REPORT_MOUSE_POSITION* = 0x20000000 - A_NORMAL* = 0 - A_ALTCHARSET* = 0x00010000 - A_RIGHTLINE* = 0x00020000 - A_LEFTLINE* = 0x00040000 - A_INVIS* = 0x00080000 - A_UNDERLINE* = 0x00100000 - A_REVERSE* = 0x00200000 - A_BLINK* = 0x00400000 - A_BOLD* = 0x00800000 - A_ATTRIBUTES* = 0xFFFF0000 - A_CHARTEXT* = 0x0000FFFF - A_COLOR* = 0xFF000000 - A_ITALIC* = A_INVIS - A_PROTECT* = (A_UNDERLINE or A_LEFTLINE or A_RIGHTLINE) - ATTR_SHIFT* = 19 - COLOR_SHIFT* = 24 - A_STANDOUT* = (A_REVERSE or A_BOLD) # X/Open - A_DIM* = A_NORMAL - CHR_MSK* = A_CHARTEXT # Obsolete - ATR_MSK* = A_ATTRIBUTES # Obsolete - ATR_NRM* = A_NORMAL # Obsolete - WA_ALTCHARSET* = A_ALTCHARSET - WA_BLINK* = A_BLINK - WA_BOLD* = A_BOLD - WA_DIM* = A_DIM - WA_INVIS* = A_INVIS - WA_LEFT* = A_LEFTLINE - WA_PROTECT* = A_PROTECT - WA_REVERSE* = A_REVERSE - WA_RIGHT* = A_RIGHTLINE - WA_STANDOUT* = A_STANDOUT - WA_UNDERLINE* = A_UNDERLINE - WA_HORIZONTAL* = A_NORMAL - WA_LOW* = A_NORMAL - WA_TOP* = A_NORMAL - WA_VERTICAL* = A_NORMAL - COLOR_BLACK* = 0 - COLOR_RED* = 1 - COLOR_GREEN* = 2 - COLOR_BLUE* = 4 - COLOR_CYAN* = (COLOR_BLUE or COLOR_GREEN) - COLOR_MAGENTA* = (COLOR_RED or COLOR_BLUE) - COLOR_YELLOW* = (COLOR_RED or COLOR_GREEN) - COLOR_WHITE* = 7 - KEY_CODE_YES* = 0x00000100 # If get_wch() gives a key code - KEY_BREAK* = 0x00000101 # Not on PC KBD - KEY_DOWN* = 0x00000102 # Down arrow key - KEY_UP* = 0x00000103 # Up arrow key - KEY_LEFT* = 0x00000104 # Left arrow key - KEY_RIGHT* = 0x00000105 # Right arrow key - KEY_HOME* = 0x00000106 # home key - KEY_BACKSPACE* = 0x00000107 # not on pc - KEY_F0* = 0x00000108 # function keys; 64 reserved - KEY_DL* = 0x00000148 # delete line - KEY_IL* = 0x00000149 # insert line - KEY_DC* = 0x0000014A # delete character - KEY_IC* = 0x0000014B # insert char or enter ins mode - KEY_EIC* = 0x0000014C # exit insert char mode - KEY_CLEAR* = 0x0000014D # clear screen - KEY_EOS* = 0x0000014E # clear to end of screen - KEY_EOL* = 0x0000014F # clear to end of line - KEY_SF* = 0x00000150 # scroll 1 line forward - KEY_SR* = 0x00000151 # scroll 1 line back (reverse) - KEY_NPAGE* = 0x00000152 # next page - KEY_PPAGE* = 0x00000153 # previous page - KEY_STAB* = 0x00000154 # set tab - KEY_CTAB* = 0x00000155 # clear tab - KEY_CATAB* = 0x00000156 # clear all tabs - KEY_ENTER* = 0x00000157 # enter or send (unreliable) - KEY_SRESET* = 0x00000158 # soft/reset (partial/unreliable) - KEY_RESET* = 0x00000159 # reset/hard reset (unreliable) - KEY_PRINT* = 0x0000015A # print/copy - KEY_LL* = 0x0000015B # home down/bottom (lower left) - KEY_ABORT* = 0x0000015C # abort/terminate key (any) - KEY_SHELP* = 0x0000015D # short help - KEY_LHELP* = 0x0000015E # long help - KEY_BTAB* = 0x0000015F # Back tab key - KEY_BEG* = 0x00000160 # beg(inning) key - KEY_CANCEL* = 0x00000161 # cancel key - KEY_CLOSE* = 0x00000162 # close key - KEY_COMMAND* = 0x00000163 # cmd (command) key - KEY_COPY* = 0x00000164 # copy key - KEY_CREATE* = 0x00000165 # create key - KEY_END* = 0x00000166 # end key - KEY_EXIT* = 0x00000167 # exit key - KEY_FIND* = 0x00000168 # find key - KEY_HELP* = 0x00000169 # help key - KEY_MARK* = 0x0000016A # mark key - KEY_MESSAGE* = 0x0000016B # message key - KEY_MOVE* = 0x0000016C # move key - KEY_NEXT* = 0x0000016D # next object key - KEY_OPEN* = 0x0000016E # open key - KEY_OPTIONS* = 0x0000016F # options key - KEY_PREVIOUS* = 0x00000170 # previous object key - KEY_REDO* = 0x00000171 # redo key - KEY_REFERENCE* = 0x00000172 # ref(erence) key - KEY_REFRESH* = 0x00000173 # refresh key - KEY_REPLACE* = 0x00000174 # replace key - KEY_RESTART* = 0x00000175 # restart key - KEY_RESUME* = 0x00000176 # resume key - KEY_SAVE* = 0x00000177 # save key - KEY_SBEG* = 0x00000178 # shifted beginning key - KEY_SCANCEL* = 0x00000179 # shifted cancel key - KEY_SCOMMAND* = 0x0000017A # shifted command key - KEY_SCOPY* = 0x0000017B # shifted copy key - KEY_SCREATE* = 0x0000017C # shifted create key - KEY_SDC* = 0x0000017D # shifted delete char key - KEY_SDL* = 0x0000017E # shifted delete line key - KEY_SELECT* = 0x0000017F # select key - KEY_SEND* = 0x00000180 # shifted end key - KEY_SEOL* = 0x00000181 # shifted clear line key - KEY_SEXIT* = 0x00000182 # shifted exit key - KEY_SFIND* = 0x00000183 # shifted find key - KEY_SHOME* = 0x00000184 # shifted home key - KEY_SIC* = 0x00000185 # shifted input key - KEY_SLEFT* = 0x00000187 # shifted left arrow key - KEY_SMESSAGE* = 0x00000188 # shifted message key - KEY_SMOVE* = 0x00000189 # shifted move key - KEY_SNEXT* = 0x0000018A # shifted next key - KEY_SOPTIONS* = 0x0000018B # shifted options key - KEY_SPREVIOUS* = 0x0000018C # shifted prev key - KEY_SPRINT* = 0x0000018D # shifted print key - KEY_SREDO* = 0x0000018E # shifted redo key - KEY_SREPLACE* = 0x0000018F # shifted replace key - KEY_SRIGHT* = 0x00000190 # shifted right arrow - KEY_SRSUME* = 0x00000191 # shifted resume key - KEY_SSAVE* = 0x00000192 # shifted save key - KEY_SSUSPEND* = 0x00000193 # shifted suspend key - KEY_SUNDO* = 0x00000194 # shifted undo key - KEY_SUSPEND* = 0x00000195 # suspend key - KEY_UNDO* = 0x00000196 # undo key - ALT_0* = 0x00000197 - ALT_1* = 0x00000198 - ALT_2* = 0x00000199 - ALT_3* = 0x0000019A - ALT_4* = 0x0000019B - ALT_5* = 0x0000019C - ALT_6* = 0x0000019D - ALT_7* = 0x0000019E - ALT_8* = 0x0000019F - ALT_9* = 0x000001A0 - ALT_A* = 0x000001A1 - ALT_B* = 0x000001A2 - ALT_C* = 0x000001A3 - ALT_D* = 0x000001A4 - ALT_E* = 0x000001A5 - ALT_F* = 0x000001A6 - ALT_G* = 0x000001A7 - ALT_H* = 0x000001A8 - ALT_I* = 0x000001A9 - ALT_J* = 0x000001AA - ALT_K* = 0x000001AB - ALT_L* = 0x000001AC - ALT_M* = 0x000001AD - ALT_N* = 0x000001AE - ALT_O* = 0x000001AF - ALT_P* = 0x000001B0 - ALT_Q* = 0x000001B1 - ALT_R* = 0x000001B2 - ALT_S* = 0x000001B3 - ALT_T* = 0x000001B4 - ALT_U* = 0x000001B5 - ALT_V* = 0x000001B6 - ALT_W* = 0x000001B7 - ALT_X* = 0x000001B8 - ALT_Y* = 0x000001B9 - ALT_Z* = 0x000001BA - CTL_LEFT* = 0x000001BB # Control-Left-Arrow - CTL_RIGHT* = 0x000001BC - CTL_PGUP* = 0x000001BD - CTL_PGDN* = 0x000001BE - CTL_HOME* = 0x000001BF - CTL_END* = 0x000001C0 - KEY_A1* = 0x000001C1 # upper left on Virtual keypad - KEY_A2* = 0x000001C2 # upper middle on Virt. keypad - KEY_A3* = 0x000001C3 # upper right on Vir. keypad - KEY_B1* = 0x000001C4 # middle left on Virt. keypad - KEY_B2* = 0x000001C5 # center on Virt. keypad - KEY_B3* = 0x000001C6 # middle right on Vir. keypad - KEY_C1* = 0x000001C7 # lower left on Virt. keypad - KEY_C2* = 0x000001C8 # lower middle on Virt. keypad - KEY_C3* = 0x000001C9 # lower right on Vir. keypad - PADSLASH* = 0x000001CA # slash on keypad - PADENTER* = 0x000001CB # enter on keypad - CTL_PADENTER* = 0x000001CC # ctl-enter on keypad - ALT_PADENTER* = 0x000001CD # alt-enter on keypad - PADSTOP* = 0x000001CE # stop on keypad - PADSTAR* = 0x000001CF # star on keypad - PADMINUS* = 0x000001D0 # minus on keypad - PADPLUS* = 0x000001D1 # plus on keypad - CTL_PADSTOP* = 0x000001D2 # ctl-stop on keypad - CTL_PADCENTER* = 0x000001D3 # ctl-enter on keypad - CTL_PADPLUS* = 0x000001D4 # ctl-plus on keypad - CTL_PADMINUS* = 0x000001D5 # ctl-minus on keypad - CTL_PADSLASH* = 0x000001D6 # ctl-slash on keypad - CTL_PADSTAR* = 0x000001D7 # ctl-star on keypad - ALT_PADPLUS* = 0x000001D8 # alt-plus on keypad - ALT_PADMINUS* = 0x000001D9 # alt-minus on keypad - ALT_PADSLASH* = 0x000001DA # alt-slash on keypad - ALT_PADSTAR* = 0x000001DB # alt-star on keypad - ALT_PADSTOP* = 0x000001DC # alt-stop on keypad - CTL_INS* = 0x000001DD # ctl-insert - ALT_DEL* = 0x000001DE # alt-delete - ALT_INS* = 0x000001DF # alt-insert - CTL_UP* = 0x000001E0 # ctl-up arrow - CTL_DOWN* = 0x000001E1 # ctl-down arrow - CTL_TAB* = 0x000001E2 # ctl-tab - ALT_TAB* = 0x000001E3 - ALT_MINUS* = 0x000001E4 - ALT_EQUAL* = 0x000001E5 - ALT_HOME* = 0x000001E6 - ALT_PGUP* = 0x000001E7 - ALT_PGDN* = 0x000001E8 - ALT_END* = 0x000001E9 - ALT_UP* = 0x000001EA # alt-up arrow - ALT_DOWN* = 0x000001EB # alt-down arrow - ALT_RIGHT* = 0x000001EC # alt-right arrow - ALT_LEFT* = 0x000001ED # alt-left arrow - ALT_ENTER* = 0x000001EE # alt-enter - ALT_ESC* = 0x000001EF # alt-escape - ALT_BQUOTE* = 0x000001F0 # alt-back quote - ALT_LBRACKET* = 0x000001F1 # alt-left bracket - ALT_RBRACKET* = 0x000001F2 # alt-right bracket - ALT_SEMICOLON* = 0x000001F3 # alt-semi-colon - ALT_FQUOTE* = 0x000001F4 # alt-forward quote - ALT_COMMA* = 0x000001F5 # alt-comma - ALT_STOP* = 0x000001F6 # alt-stop - ALT_FSLASH* = 0x000001F7 # alt-forward slash - ALT_BKSP* = 0x000001F8 # alt-backspace - CTL_BKSP* = 0x000001F9 # ctl-backspace - PAD0* = 0x000001FA # keypad 0 - CTL_PAD0* = 0x000001FB # ctl-keypad 0 - CTL_PAD1* = 0x000001FC - CTL_PAD2* = 0x000001FD - CTL_PAD3* = 0x000001FE - CTL_PAD4* = 0x000001FF - CTL_PAD5* = 0x00000200 - CTL_PAD6* = 0x00000201 - CTL_PAD7* = 0x00000202 - CTL_PAD8* = 0x00000203 - CTL_PAD9* = 0x00000204 - ALT_PAD0* = 0x00000205 # alt-keypad 0 - ALT_PAD1* = 0x00000206 - ALT_PAD2* = 0x00000207 - ALT_PAD3* = 0x00000208 - ALT_PAD4* = 0x00000209 - ALT_PAD5* = 0x0000020A - ALT_PAD6* = 0x0000020B - ALT_PAD7* = 0x0000020C - ALT_PAD8* = 0x0000020D - ALT_PAD9* = 0x0000020E - CTL_DEL* = 0x0000020F # clt-delete - ALT_BSLASH* = 0x00000210 # alt-back slash - CTL_ENTER* = 0x00000211 # ctl-enter - SHF_PADENTER* = 0x00000212 # shift-enter on keypad - SHF_PADSLASH* = 0x00000213 # shift-slash on keypad - SHF_PADSTAR* = 0x00000214 # shift-star on keypad - SHF_PADPLUS* = 0x00000215 # shift-plus on keypad - SHF_PADMINUS* = 0x00000216 # shift-minus on keypad - SHF_UP* = 0x00000217 # shift-up on keypad - SHF_DOWN* = 0x00000218 # shift-down on keypad - SHF_IC* = 0x00000219 # shift-insert on keypad - SHF_DC* = 0x0000021A # shift-delete on keypad - KEY_MOUSE* = 0x0000021B # "mouse" key - KEY_SHIFT_L* = 0x0000021C # Left-shift - KEY_SHIFT_R* = 0x0000021D # Right-shift - KEY_CONTROL_L* = 0x0000021E # Left-control - KEY_CONTROL_R* = 0x0000021F # Right-control - KEY_ALT_L* = 0x00000220 # Left-alt - KEY_ALT_R* = 0x00000221 # Right-alt - KEY_RESIZE* = 0x00000222 # Window resize - KEY_SUP* = 0x00000223 # Shifted up arrow - KEY_SDOWN* = 0x00000224 # Shifted down arrow - KEY_MIN* = KEY_BREAK # Minimum curses key value - KEY_MAX* = KEY_SDOWN # Maximum curses key - CLIP_SUCCESS* = 0 - CLIP_ACCESS_ERROR* = 1 - CLIP_EMPTY* = 2 - CLIP_MEMORY_ERROR* = 3 - KEY_MODIFIER_SHIFT* = 1 - KEY_MODIFIER_CONTROL* = 2 - KEY_MODIFIER_ALT* = 4 - KEY_MODIFIER_NUMLOCK* = 8 - -when appType == "gui": - const - BUTTON_SHIFT* = BUTTON_MODIFIER_SHIFT - BUTTON_CONTROL* = BUTTON_MODIFIER_CONTROL - BUTTON_CTRL* = BUTTON_MODIFIER_CONTROL - BUTTON_ALT* = BUTTON_MODIFIER_ALT -else: - const - BUTTON_SHIFT* = 0x00000008 - BUTTON_CONTROL* = 0x00000010 - BUTTON_ALT* = 0x00000020 - -type - TMOUSE_STATUS*{.pure, final.} = object - x*: cint # absolute column, 0 based, measured in characters - y*: cint # absolute row, 0 based, measured in characters - button*: array[0..3 - 1, cshort] # state of each button - changes*: cint # flags indicating what has changed with the mouse - - MEVENT*{.pure, final.} = object - id*: cshort # unused, always 0 - x*: cint - y*: cint - z*: cint # x, y same as TMOUSE_STATUS; z unused - bstate*: cunsignedlong # equivalent to changes + button[], but - # in the same format as used for mousemask() - - WINDOW*{.pure, final.} = object - cury*: cint # current pseudo-cursor - curx*: cint - maxy*: cint # max window coordinates - maxx*: cint - begy*: cint # origin on screen - begx*: cint - flags*: cint # window properties - attrs*: cunsignedlong # standard attributes and colors - bkgd*: cunsignedlong # background, normally blank - clear*: cunsignedchar # causes clear at next refresh - leaveit*: cunsignedchar # leaves cursor where it is - scroll*: cunsignedchar # allows window scrolling - nodelay*: cunsignedchar # input character wait flag - immed*: cunsignedchar # immediate update flag - sync*: cunsignedchar # synchronise window ancestors - use_keypad*: cunsignedchar # flags keypad key mode active - y*: ptr ptr cunsignedlong # pointer to line pointer array - firstch*: ptr cint # first changed character in line - lastch*: ptr cint # last changed character in line - tmarg*: cint # top of scrolling region - bmarg*: cint # bottom of scrolling region - delayms*: cint # milliseconds of delay for getch() - parx*: cint - pary*: cint # coords relative to parent (0,0) - parent*: ptr WINDOW # subwin's pointer to parent win - - PANELOBS*{.pure, final.} = object - above*: ptr PANELOBS - pan*: ptr PANEL - - PANEL*{.pure, final.} = object - win*: ptr WINDOW - wstarty*: cint - wendy*: cint - wstartx*: cint - wendx*: cint - below*: ptr PANEL - above*: ptr PANEL - user*: pointer - obscure*: ptr PANELOBS -{.deprecated: [ - #TMOUSE_STATUS: MOUSE_STATUS, # Name conflict when we drop the `T` - TMEVENT: MEVENT, TWINDOW: WINDOW, - TPANELOBS: PANELOBS, TPANEL:PANEL].} - -when unixOS: - type - SCREEN*{.pure, final.} = object - alive*: cunsignedchar # if initscr() called, and not endwin() - autocr*: cunsignedchar # if cr -> lf - cbreak*: cunsignedchar # if terminal unbuffered - echo*: cunsignedchar # if terminal echo - raw_inp*: cunsignedchar # raw input mode (v. cooked input) - raw_out*: cunsignedchar # raw output mode (7 v. 8 bits) - audible*: cunsignedchar # FALSE if the bell is visual - mono*: cunsignedchar # TRUE if current screen is mono - resized*: cunsignedchar # TRUE if TERM has been resized - orig_attr*: cunsignedchar # TRUE if we have the original colors - orig_fore*: cshort # original screen foreground color - orig_back*: cshort # original screen foreground color - cursrow*: cint # position of physical cursor - curscol*: cint # position of physical cursor - visibility*: cint # visibility of cursor - orig_cursor*: cint # original cursor size - lines*: cint # new value for LINES - cols*: cint # new value for COLS - trap_mbe*: cunsignedlong # trap these mouse button events - map_mbe_to_key*: cunsignedlong # map mouse buttons to slk - mouse_wait*: cint # time to wait (in ms) for a button release after a press - slklines*: cint # lines in use by slk_init() - slk_winptr*: ptr WINDOW # window for slk - linesrippedoff*: cint # lines ripped off via ripoffline() - linesrippedoffontop*: cint # lines ripped off on top via ripoffline() - delaytenths*: cint # 1/10ths second to wait block getch() for - preserve*: cunsignedchar # TRUE if screen background to be preserved - restore*: cint # specifies if screen background to be restored, and how - save_key_modifiers*: cunsignedchar # TRUE if each key modifiers saved with each key press - return_key_modifiers*: cunsignedchar # TRUE if modifier keys are returned as "real" keys - key_code*: cunsignedchar # TRUE if last key is a special key; - XcurscrSize*: cint # size of Xcurscr shared memory block - sb_on*: cunsignedchar - sb_viewport_y*: cint - sb_viewport_x*: cint - sb_total_y*: cint - sb_total_x*: cint - sb_cur_y*: cint - sb_cur_x*: cint - line_color*: cshort # color of line attributes - default -1 - {.deprecated: [TSCREEN: SCREEN].} -else: - type - SCREEN*{.pure, final.} = object - alive*: cunsignedchar # if initscr() called, and not endwin() - autocr*: cunsignedchar # if cr -> lf - cbreak*: cunsignedchar # if terminal unbuffered - echo*: cunsignedchar # if terminal echo - raw_inp*: cunsignedchar # raw input mode (v. cooked input) - raw_out*: cunsignedchar # raw output mode (7 v. 8 bits) - audible*: cunsignedchar # FALSE if the bell is visual - mono*: cunsignedchar # TRUE if current screen is mono - resized*: cunsignedchar # TRUE if TERM has been resized - orig_attr*: cunsignedchar # TRUE if we have the original colors - orig_fore*: cshort # original screen foreground color - orig_back*: cshort # original screen foreground color - cursrow*: cint # position of physical cursor - curscol*: cint # position of physical cursor - visibility*: cint # visibility of cursor - orig_cursor*: cint # original cursor size - lines*: cint # new value for LINES - cols*: cint # new value for COLS - trap_mbe*: cunsignedlong # trap these mouse button events - map_mbe_to_key*: cunsignedlong # map mouse buttons to slk - mouse_wait*: cint # time to wait (in ms) for a button release after a press - slklines*: cint # lines in use by slk_init() - slk_winptr*: ptr WINDOW # window for slk - linesrippedoff*: cint # lines ripped off via ripoffline() - linesrippedoffontop*: cint # lines ripped off on top via ripoffline() - delaytenths*: cint # 1/10ths second to wait block getch() for - preserve*: cunsignedchar # TRUE if screen background to be preserved - restore*: cint # specifies if screen background to be restored, and how - save_key_modifiers*: cunsignedchar # TRUE if each key modifiers saved with each key press - return_key_modifiers*: cunsignedchar # TRUE if modifier keys are returned as "real" keys - key_code*: cunsignedchar # TRUE if last key is a special key; - line_color*: cshort # color of line attributes - default -1 - {.deprecated: [TSCREEN: SCREEN].} - -var - LINES*{.importc: "LINES", dynlib: pdcursesdll.}: cint - COLS*{.importc: "COLS", dynlib: pdcursesdll.}: cint - stdscr*{.importc: "stdscr", dynlib: pdcursesdll.}: ptr WINDOW - curscr*{.importc: "curscr", dynlib: pdcursesdll.}: ptr WINDOW - SP*{.importc: "SP", dynlib: pdcursesdll.}: ptr SCREEN - Mouse_status*{.importc: "Mouse_status", dynlib: pdcursesdll.}: MOUSE_STATUS - COLORS*{.importc: "COLORS", dynlib: pdcursesdll.}: cint - COLOR_PAIRS*{.importc: "COLOR_PAIRS", dynlib: pdcursesdll.}: cint - TABSIZE*{.importc: "TABSIZE", dynlib: pdcursesdll.}: cint - acs_map*{.importc: "acs_map", dynlib: pdcursesdll.}: ptr cunsignedlong - ttytype*{.importc: "ttytype", dynlib: pdcursesdll.}: cstring - -template BUTTON_CHANGED*(x: expr): expr = - (Mouse_status.changes and (1 shl ((x) - 1))) - -template BUTTON_STATUS*(x: expr): expr = - (Mouse_status.button[(x) - 1]) - -template ACS_PICK*(w, n: expr): expr = int32(w) or A_ALTCHARSET - -template KEY_F*(n: expr): expr = KEY_F0 + n - -template COLOR_PAIR*(n: expr): expr = - ((cunsignedlong(n) shl COLOR_SHIFT) and A_COLOR) - -template PAIR_NUMBER*(n: expr): expr = - (((n) and A_COLOR) shr COLOR_SHIFT) - -const - #MOUSE_X_POS* = (Mouse_status.x) - #MOUSE_Y_POS* = (Mouse_status.y) - #A_BUTTON_CHANGED* = (Mouse_status.changes and 7) - #MOUSE_MOVED* = (Mouse_status.changes and MOUSE_MOVED) - #MOUSE_POS_REPORT* = (Mouse_status.changes and MOUSE_POSITION) - #MOUSE_WHEEL_UP* = (Mouse_status.changes and MOUSE_WHEEL_UP) - #MOUSE_WHEEL_DOWN* = (Mouse_status.changes and MOUSE_WHEEL_DOWN) - ACS_ULCORNER* = ACS_PICK('l', '+') - ACS_LLCORNER* = ACS_PICK('m', '+') - ACS_URCORNER* = ACS_PICK('k', '+') - ACS_LRCORNER* = ACS_PICK('j', '+') - ACS_RTEE* = ACS_PICK('u', '+') - ACS_LTEE* = ACS_PICK('t', '+') - ACS_BTEE* = ACS_PICK('v', '+') - ACS_TTEE* = ACS_PICK('w', '+') - ACS_HLINE* = ACS_PICK('q', '-') - ACS_VLINE* = ACS_PICK('x', '|') - ACS_PLUS* = ACS_PICK('n', '+') - ACS_S1* = ACS_PICK('o', '-') - ACS_S9* = ACS_PICK('s', '_') - ACS_DIAMOND* = ACS_PICK('`', '+') - ACS_CKBOARD* = ACS_PICK('a', ':') - ACS_DEGREE* = ACS_PICK('f', '\'') - ACS_PLMINUS* = ACS_PICK('g', '#') - ACS_BULLET* = ACS_PICK('~', 'o') - ACS_LARROW* = ACS_PICK(',', '<') - ACS_RARROW* = ACS_PICK('+', '>') - ACS_DARROW* = ACS_PICK('.', 'v') - ACS_UARROW* = ACS_PICK('-', '^') - ACS_BOARD* = ACS_PICK('h', '#') - ACS_LANTERN* = ACS_PICK('i', '*') - ACS_BLOCK* = ACS_PICK('0', '#') - ACS_S3* = ACS_PICK('p', '-') - ACS_S7* = ACS_PICK('r', '-') - ACS_LEQUAL* = ACS_PICK('y', '<') - ACS_GEQUAL* = ACS_PICK('z', '>') - ACS_PI* = ACS_PICK('{', 'n') - ACS_NEQUAL* = ACS_PICK('|', '+') - ACS_STERLING* = ACS_PICK('}', 'L') - ACS_BSSB* = ACS_ULCORNER - ACS_SSBB* = ACS_LLCORNER - ACS_BBSS* = ACS_URCORNER - ACS_SBBS* = ACS_LRCORNER - ACS_SBSS* = ACS_RTEE - ACS_SSSB* = ACS_LTEE - ACS_SSBS* = ACS_BTEE - ACS_BSSS* = ACS_TTEE - ACS_BSBS* = ACS_HLINE - ACS_SBSB* = ACS_VLINE - ACS_SSSS* = ACS_PLUS -discard """WACS_ULCORNER* = (addr((acs_map['l']))) - WACS_LLCORNER* = (addr((acs_map['m']))) - WACS_URCORNER* = (addr((acs_map['k']))) - WACS_LRCORNER* = (addr((acs_map['j']))) - WACS_RTEE* = (addr((acs_map['u']))) - WACS_LTEE* = (addr((acs_map['t']))) - WACS_BTEE* = (addr((acs_map['v']))) - WACS_TTEE* = (addr((acs_map['w']))) - WACS_HLINE* = (addr((acs_map['q']))) - WACS_VLINE* = (addr((acs_map['x']))) - WACS_PLUS* = (addr((acs_map['n']))) - WACS_S1* = (addr((acs_map['o']))) - WACS_S9* = (addr((acs_map['s']))) - WACS_DIAMOND* = (addr((acs_map['`']))) - WACS_CKBOARD* = (addr((acs_map['a']))) - WACS_DEGREE* = (addr((acs_map['f']))) - WACS_PLMINUS* = (addr((acs_map['g']))) - WACS_BULLET* = (addr((acs_map['~']))) - WACS_LARROW* = (addr((acs_map[',']))) - WACS_RARROW* = (addr((acs_map['+']))) - WACS_DARROW* = (addr((acs_map['.']))) - WACS_UARROW* = (addr((acs_map['-']))) - WACS_BOARD* = (addr((acs_map['h']))) - WACS_LANTERN* = (addr((acs_map['i']))) - WACS_BLOCK* = (addr((acs_map['0']))) - WACS_S3* = (addr((acs_map['p']))) - WACS_S7* = (addr((acs_map['r']))) - WACS_LEQUAL* = (addr((acs_map['y']))) - WACS_GEQUAL* = (addr((acs_map['z']))) - WACS_PI* = (addr((acs_map['{']))) - WACS_NEQUAL* = (addr((acs_map['|']))) - WACS_STERLING* = (addr((acs_map['}']))) - WACS_BSSB* = WACS_ULCORNER - WACS_SSBB* = WACS_LLCORNER - WACS_BBSS* = WACS_URCORNER - WACS_SBBS* = WACS_LRCORNER - WACS_SBSS* = WACS_RTEE - WACS_SSSB* = WACS_LTEE - WACS_SSBS* = WACS_BTEE - WACS_BSSS* = WACS_TTEE - WACS_BSBS* = WACS_HLINE - WACS_SBSB* = WACS_VLINE - WACS_SSSS* = WACS_PLUS""" - -proc addch*(a2: cunsignedlong): cint{.extdecl, importc: "addch", - dynlib: pdcursesdll.} -proc addchnstr*(a2: ptr cunsignedlong; a3: cint): cint{.extdecl, - importc: "addchnstr", dynlib: pdcursesdll.} -proc addchstr*(a2: ptr cunsignedlong): cint{.extdecl, importc: "addchstr", - dynlib: pdcursesdll.} -proc addnstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "addnstr", - dynlib: pdcursesdll.} -proc addstr*(a2: cstring): cint{.extdecl, importc: "addstr", dynlib: pdcursesdll.} -proc attroff*(a2: cunsignedlong): cint{.extdecl, importc: "attroff", - dynlib: pdcursesdll.} -proc attron*(a2: cunsignedlong): cint{.extdecl, importc: "attron", - dynlib: pdcursesdll.} -proc attrset*(a2: cunsignedlong): cint{.extdecl, importc: "attrset", - dynlib: pdcursesdll.} -proc attr_get*(a2: ptr cunsignedlong; a3: ptr cshort; a4: pointer): cint{.extdecl, - importc: "attr_get", dynlib: pdcursesdll.} -proc attr_off*(a2: cunsignedlong; a3: pointer): cint{.extdecl, - importc: "attr_off", dynlib: pdcursesdll.} -proc attr_on*(a2: cunsignedlong; a3: pointer): cint{.extdecl, importc: "attr_on", - dynlib: pdcursesdll.} -proc attr_set*(a2: cunsignedlong; a3: cshort; a4: pointer): cint{.extdecl, - importc: "attr_set", dynlib: pdcursesdll.} -proc baudrate*(): cint{.extdecl, importc: "baudrate", dynlib: pdcursesdll.} -proc beep*(): cint{.extdecl, importc: "beep", dynlib: pdcursesdll.} -proc bkgd*(a2: cunsignedlong): cint{.extdecl, importc: "bkgd", dynlib: pdcursesdll.} -proc bkgdset*(a2: cunsignedlong){.extdecl, importc: "bkgdset", dynlib: pdcursesdll.} -proc border*(a2: cunsignedlong; a3: cunsignedlong; a4: cunsignedlong; - a5: cunsignedlong; a6: cunsignedlong; a7: cunsignedlong; - a8: cunsignedlong; a9: cunsignedlong): cint{.extdecl, - importc: "border", dynlib: pdcursesdll.} -proc box*(a2: ptr WINDOW; a3: cunsignedlong; a4: cunsignedlong): cint{.extdecl, - importc: "box", dynlib: pdcursesdll.} -proc can_change_color*(): cunsignedchar{.extdecl, importc: "can_change_color", - dynlib: pdcursesdll.} -proc cbreak*(): cint{.extdecl, importc: "cbreak", dynlib: pdcursesdll.} -proc chgat*(a2: cint; a3: cunsignedlong; a4: cshort; a5: pointer): cint{.extdecl, - importc: "chgat", dynlib: pdcursesdll.} -proc clearok*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, - importc: "clearok", dynlib: pdcursesdll.} -proc clear*(): cint{.extdecl, importc: "clear", dynlib: pdcursesdll.} -proc clrtobot*(): cint{.extdecl, importc: "clrtobot", dynlib: pdcursesdll.} -proc clrtoeol*(): cint{.extdecl, importc: "clrtoeol", dynlib: pdcursesdll.} -proc color_content*(a2: cshort; a3: ptr cshort; a4: ptr cshort; a5: ptr cshort): cint{. - extdecl, importc: "color_content", dynlib: pdcursesdll.} -proc color_set*(a2: cshort; a3: pointer): cint{.extdecl, importc: "color_set", - dynlib: pdcursesdll.} -proc copywin*(a2: ptr WINDOW; a3: ptr WINDOW; a4: cint; a5: cint; a6: cint; - a7: cint; a8: cint; a9: cint; a10: cint): cint{.extdecl, - importc: "copywin", dynlib: pdcursesdll.} -proc curs_set*(a2: cint): cint{.extdecl, importc: "curs_set", dynlib: pdcursesdll.} -proc def_prog_mode*(): cint{.extdecl, importc: "def_prog_mode", - dynlib: pdcursesdll.} -proc def_shell_mode*(): cint{.extdecl, importc: "def_shell_mode", - dynlib: pdcursesdll.} -proc delay_output*(a2: cint): cint{.extdecl, importc: "delay_output", - dynlib: pdcursesdll.} -proc delch*(): cint{.extdecl, importc: "delch", dynlib: pdcursesdll.} -proc deleteln*(): cint{.extdecl, importc: "deleteln", dynlib: pdcursesdll.} -proc delscreen*(a2: ptr SCREEN){.extdecl, importc: "delscreen", - dynlib: pdcursesdll.} -proc delwin*(a2: ptr WINDOW): cint{.extdecl, importc: "delwin", - dynlib: pdcursesdll.} -proc derwin*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint; a6: cint): ptr WINDOW{. - extdecl, importc: "derwin", dynlib: pdcursesdll.} -proc doupdate*(): cint{.extdecl, importc: "doupdate", dynlib: pdcursesdll.} -proc dupwin*(a2: ptr WINDOW): ptr WINDOW{.extdecl, importc: "dupwin", - dynlib: pdcursesdll.} -proc echochar*(a2: cunsignedlong): cint{.extdecl, importc: "echochar", - dynlib: pdcursesdll.} -proc echo*(): cint{.extdecl, importc: "echo", dynlib: pdcursesdll.} -proc endwin*(): cint{.extdecl, importc: "endwin", dynlib: pdcursesdll.} -proc erasechar*(): char{.extdecl, importc: "erasechar", dynlib: pdcursesdll.} -proc erase*(): cint{.extdecl, importc: "erase", dynlib: pdcursesdll.} -proc filter*(){.extdecl, importc: "filter", dynlib: pdcursesdll.} -proc flash*(): cint{.extdecl, importc: "flash", dynlib: pdcursesdll.} -proc flushinp*(): cint{.extdecl, importc: "flushinp", dynlib: pdcursesdll.} -proc getbkgd*(a2: ptr WINDOW): cunsignedlong{.extdecl, importc: "getbkgd", - dynlib: pdcursesdll.} -proc getnstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "getnstr", - dynlib: pdcursesdll.} -proc getstr*(a2: cstring): cint{.extdecl, importc: "getstr", dynlib: pdcursesdll.} -proc getwin*(a2: File): ptr WINDOW{.extdecl, importc: "getwin", - dynlib: pdcursesdll.} -proc halfdelay*(a2: cint): cint{.extdecl, importc: "halfdelay", - dynlib: pdcursesdll.} -proc has_colors*(): cunsignedchar{.extdecl, importc: "has_colors", - dynlib: pdcursesdll.} -proc has_ic*(): cunsignedchar{.extdecl, importc: "has_ic", dynlib: pdcursesdll.} -proc has_il*(): cunsignedchar{.extdecl, importc: "has_il", dynlib: pdcursesdll.} -proc hline*(a2: cunsignedlong; a3: cint): cint{.extdecl, importc: "hline", - dynlib: pdcursesdll.} -proc idcok*(a2: ptr WINDOW; a3: cunsignedchar){.extdecl, importc: "idcok", - dynlib: pdcursesdll.} -proc idlok*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, importc: "idlok", - dynlib: pdcursesdll.} -proc immedok*(a2: ptr WINDOW; a3: cunsignedchar){.extdecl, importc: "immedok", - dynlib: pdcursesdll.} -proc inchnstr*(a2: ptr cunsignedlong; a3: cint): cint{.extdecl, - importc: "inchnstr", dynlib: pdcursesdll.} -proc inchstr*(a2: ptr cunsignedlong): cint{.extdecl, importc: "inchstr", - dynlib: pdcursesdll.} -proc inch*(): cunsignedlong{.extdecl, importc: "inch", dynlib: pdcursesdll.} -proc init_color*(a2: cshort; a3: cshort; a4: cshort; a5: cshort): cint{.extdecl, - importc: "init_color", dynlib: pdcursesdll.} -proc init_pair*(a2: cshort; a3: cshort; a4: cshort): cint{.extdecl, - importc: "init_pair", dynlib: pdcursesdll.} -proc initscr*(): ptr WINDOW{.extdecl, importc: "initscr", dynlib: pdcursesdll.} -proc innstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "innstr", - dynlib: pdcursesdll.} -proc insch*(a2: cunsignedlong): cint{.extdecl, importc: "insch", - dynlib: pdcursesdll.} -proc insdelln*(a2: cint): cint{.extdecl, importc: "insdelln", dynlib: pdcursesdll.} -proc insertln*(): cint{.extdecl, importc: "insertln", dynlib: pdcursesdll.} -proc insnstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "insnstr", - dynlib: pdcursesdll.} -proc insstr*(a2: cstring): cint{.extdecl, importc: "insstr", dynlib: pdcursesdll.} -proc instr*(a2: cstring): cint{.extdecl, importc: "instr", dynlib: pdcursesdll.} -proc intrflush*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, - importc: "intrflush", dynlib: pdcursesdll.} -proc isendwin*(): cunsignedchar{.extdecl, importc: "isendwin", dynlib: pdcursesdll.} -proc is_linetouched*(a2: ptr WINDOW; a3: cint): cunsignedchar{.extdecl, - importc: "is_linetouched", dynlib: pdcursesdll.} -proc is_wintouched*(a2: ptr WINDOW): cunsignedchar{.extdecl, - importc: "is_wintouched", dynlib: pdcursesdll.} -proc keyname*(a2: cint): cstring{.extdecl, importc: "keyname", dynlib: pdcursesdll.} -proc keypad*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, importc: "keypad", - dynlib: pdcursesdll.} -proc killchar*(): char{.extdecl, importc: "killchar", dynlib: pdcursesdll.} -proc leaveok*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, - importc: "leaveok", dynlib: pdcursesdll.} -proc longname*(): cstring{.extdecl, importc: "longname", dynlib: pdcursesdll.} -proc meta*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, importc: "meta", - dynlib: pdcursesdll.} -proc move*(a2: cint; a3: cint): cint{.extdecl, importc: "move", - dynlib: pdcursesdll.} -proc mvaddch*(a2: cint; a3: cint; a4: cunsignedlong): cint{.extdecl, - importc: "mvaddch", dynlib: pdcursesdll.} -proc mvaddchnstr*(a2: cint; a3: cint; a4: ptr cunsignedlong; a5: cint): cint{. - extdecl, importc: "mvaddchnstr", dynlib: pdcursesdll.} -proc mvaddchstr*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvaddchstr", dynlib: pdcursesdll.} -proc mvaddnstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvaddnstr", dynlib: pdcursesdll.} -proc mvaddstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, - importc: "mvaddstr", dynlib: pdcursesdll.} -proc mvchgat*(a2: cint; a3: cint; a4: cint; a5: cunsignedlong; a6: cshort; - a7: pointer): cint{.extdecl, importc: "mvchgat", dynlib: pdcursesdll.} -proc mvcur*(a2: cint; a3: cint; a4: cint; a5: cint): cint{.extdecl, - importc: "mvcur", dynlib: pdcursesdll.} -proc mvdelch*(a2: cint; a3: cint): cint{.extdecl, importc: "mvdelch", - dynlib: pdcursesdll.} -proc mvderwin*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "mvderwin", dynlib: pdcursesdll.} -proc mvgetch*(a2: cint; a3: cint): cint{.extdecl, importc: "mvgetch", - dynlib: pdcursesdll.} -proc mvgetnstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvgetnstr", dynlib: pdcursesdll.} -proc mvgetstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, - importc: "mvgetstr", dynlib: pdcursesdll.} -proc mvhline*(a2: cint; a3: cint; a4: cunsignedlong; a5: cint): cint{.extdecl, - importc: "mvhline", dynlib: pdcursesdll.} -proc mvinch*(a2: cint; a3: cint): cunsignedlong{.extdecl, importc: "mvinch", - dynlib: pdcursesdll.} -proc mvinchnstr*(a2: cint; a3: cint; a4: ptr cunsignedlong; a5: cint): cint{. - extdecl, importc: "mvinchnstr", dynlib: pdcursesdll.} -proc mvinchstr*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvinchstr", dynlib: pdcursesdll.} -proc mvinnstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvinnstr", dynlib: pdcursesdll.} -proc mvinsch*(a2: cint; a3: cint; a4: cunsignedlong): cint{.extdecl, - importc: "mvinsch", dynlib: pdcursesdll.} -proc mvinsnstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvinsnstr", dynlib: pdcursesdll.} -proc mvinsstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, - importc: "mvinsstr", dynlib: pdcursesdll.} -proc mvinstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, importc: "mvinstr", - dynlib: pdcursesdll.} -proc mvprintw*(a2: cint; a3: cint; a4: cstring): cint{.varargs, extdecl, - importc: "mvprintw", dynlib: pdcursesdll.} -proc mvscanw*(a2: cint; a3: cint; a4: cstring): cint{.varargs, extdecl, - importc: "mvscanw", dynlib: pdcursesdll.} -proc mvvline*(a2: cint; a3: cint; a4: cunsignedlong; a5: cint): cint{.extdecl, - importc: "mvvline", dynlib: pdcursesdll.} -proc mvwaddchnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong; - a6: cint): cint{.extdecl, importc: "mvwaddchnstr", - dynlib: pdcursesdll.} -proc mvwaddchstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwaddchstr", dynlib: pdcursesdll.} -proc mvwaddch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cunsignedlong): cint{. - extdecl, importc: "mvwaddch", dynlib: pdcursesdll.} -proc mvwaddnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwaddnstr", dynlib: pdcursesdll.} -proc mvwaddstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{.extdecl, - importc: "mvwaddstr", dynlib: pdcursesdll.} -proc mvwchgat*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint; a6: cunsignedlong; - a7: cshort; a8: pointer): cint{.extdecl, importc: "mvwchgat", - dynlib: pdcursesdll.} -proc mvwdelch*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "mvwdelch", dynlib: pdcursesdll.} -proc mvwgetch*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "mvwgetch", dynlib: pdcursesdll.} -proc mvwgetnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwgetnstr", dynlib: pdcursesdll.} -proc mvwgetstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{.extdecl, - importc: "mvwgetstr", dynlib: pdcursesdll.} -proc mvwhline*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cunsignedlong; a6: cint): cint{. - extdecl, importc: "mvwhline", dynlib: pdcursesdll.} -proc mvwinchnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong; - a6: cint): cint{.extdecl, importc: "mvwinchnstr", - dynlib: pdcursesdll.} -proc mvwinchstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwinchstr", dynlib: pdcursesdll.} -proc mvwinch*(a2: ptr WINDOW; a3: cint; a4: cint): cunsignedlong{.extdecl, - importc: "mvwinch", dynlib: pdcursesdll.} -proc mvwinnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwinnstr", dynlib: pdcursesdll.} -proc mvwinsch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cunsignedlong): cint{. - extdecl, importc: "mvwinsch", dynlib: pdcursesdll.} -proc mvwinsnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwinsnstr", dynlib: pdcursesdll.} -proc mvwinsstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{.extdecl, - importc: "mvwinsstr", dynlib: pdcursesdll.} -proc mvwinstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{.extdecl, - importc: "mvwinstr", dynlib: pdcursesdll.} -proc mvwin*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, importc: "mvwin", - dynlib: pdcursesdll.} -proc mvwprintw*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{.varargs, - extdecl, importc: "mvwprintw", dynlib: pdcursesdll.} -proc mvwscanw*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{.varargs, - extdecl, importc: "mvwscanw", dynlib: pdcursesdll.} -proc mvwvline*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cunsignedlong; a6: cint): cint{. - extdecl, importc: "mvwvline", dynlib: pdcursesdll.} -proc napms*(a2: cint): cint{.extdecl, importc: "napms", dynlib: pdcursesdll.} -proc newpad*(a2: cint; a3: cint): ptr WINDOW{.extdecl, importc: "newpad", - dynlib: pdcursesdll.} -proc newterm*(a2: cstring; a3: File; a4: File): ptr SCREEN{.extdecl, - importc: "newterm", dynlib: pdcursesdll.} -proc newwin*(a2: cint; a3: cint; a4: cint; a5: cint): ptr WINDOW{.extdecl, - importc: "newwin", dynlib: pdcursesdll.} -proc nl*(): cint{.extdecl, importc: "nl", dynlib: pdcursesdll.} -proc nocbreak*(): cint{.extdecl, importc: "nocbreak", dynlib: pdcursesdll.} -proc nodelay*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, - importc: "nodelay", dynlib: pdcursesdll.} -proc noecho*(): cint{.extdecl, importc: "noecho", dynlib: pdcursesdll.} -proc nonl*(): cint{.extdecl, importc: "nonl", dynlib: pdcursesdll.} -proc noqiflush*(){.extdecl, importc: "noqiflush", dynlib: pdcursesdll.} -proc noraw*(): cint{.extdecl, importc: "noraw", dynlib: pdcursesdll.} -proc notimeout*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, - importc: "notimeout", dynlib: pdcursesdll.} -proc overlay*(a2: ptr WINDOW; a3: ptr WINDOW): cint{.extdecl, importc: "overlay", - dynlib: pdcursesdll.} -proc overwrite*(a2: ptr WINDOW; a3: ptr WINDOW): cint{.extdecl, - importc: "overwrite", dynlib: pdcursesdll.} -proc pair_content*(a2: cshort; a3: ptr cshort; a4: ptr cshort): cint{.extdecl, - importc: "pair_content", dynlib: pdcursesdll.} -proc pechochar*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "pechochar", dynlib: pdcursesdll.} -proc pnoutrefresh*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint; a6: cint; - a7: cint; a8: cint): cint{.extdecl, importc: "pnoutrefresh", - dynlib: pdcursesdll.} -proc prefresh*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint; a6: cint; a7: cint; - a8: cint): cint{.extdecl, importc: "prefresh", dynlib: pdcursesdll.} -proc printw*(a2: cstring): cint{.varargs, extdecl, importc: "printw", - dynlib: pdcursesdll.} -proc putwin*(a2: ptr WINDOW; a3: File): cint{.extdecl, importc: "putwin", - dynlib: pdcursesdll.} -proc qiflush*(){.extdecl, importc: "qiflush", dynlib: pdcursesdll.} -proc raw*(): cint{.extdecl, importc: "raw", dynlib: pdcursesdll.} -proc redrawwin*(a2: ptr WINDOW): cint{.extdecl, importc: "redrawwin", - dynlib: pdcursesdll.} -proc refresh*(): cint{.extdecl, importc: "refresh", dynlib: pdcursesdll.} -proc reset_prog_mode*(): cint{.extdecl, importc: "reset_prog_mode", - dynlib: pdcursesdll.} -proc reset_shell_mode*(): cint{.extdecl, importc: "reset_shell_mode", - dynlib: pdcursesdll.} -proc resetty*(): cint{.extdecl, importc: "resetty", dynlib: pdcursesdll.} -#int ripoffline(int, int (*)(WINDOW *, int)); -proc savetty*(): cint{.extdecl, importc: "savetty", dynlib: pdcursesdll.} -proc scanw*(a2: cstring): cint{.varargs, extdecl, importc: "scanw", - dynlib: pdcursesdll.} -proc scr_dump*(a2: cstring): cint{.extdecl, importc: "scr_dump", - dynlib: pdcursesdll.} -proc scr_init*(a2: cstring): cint{.extdecl, importc: "scr_init", - dynlib: pdcursesdll.} -proc scr_restore*(a2: cstring): cint{.extdecl, importc: "scr_restore", - dynlib: pdcursesdll.} -proc scr_set*(a2: cstring): cint{.extdecl, importc: "scr_set", dynlib: pdcursesdll.} -proc scrl*(a2: cint): cint{.extdecl, importc: "scrl", dynlib: pdcursesdll.} -proc scroll*(a2: ptr WINDOW): cint{.extdecl, importc: "scroll", - dynlib: pdcursesdll.} -proc scrollok*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, - importc: "scrollok", dynlib: pdcursesdll.} -proc set_term*(a2: ptr SCREEN): ptr SCREEN{.extdecl, importc: "set_term", - dynlib: pdcursesdll.} -proc setscrreg*(a2: cint; a3: cint): cint{.extdecl, importc: "setscrreg", - dynlib: pdcursesdll.} -proc slk_attroff*(a2: cunsignedlong): cint{.extdecl, importc: "slk_attroff", - dynlib: pdcursesdll.} -proc slk_attr_off*(a2: cunsignedlong; a3: pointer): cint{.extdecl, - importc: "slk_attr_off", dynlib: pdcursesdll.} -proc slk_attron*(a2: cunsignedlong): cint{.extdecl, importc: "slk_attron", - dynlib: pdcursesdll.} -proc slk_attr_on*(a2: cunsignedlong; a3: pointer): cint{.extdecl, - importc: "slk_attr_on", dynlib: pdcursesdll.} -proc slk_attrset*(a2: cunsignedlong): cint{.extdecl, importc: "slk_attrset", - dynlib: pdcursesdll.} -proc slk_attr_set*(a2: cunsignedlong; a3: cshort; a4: pointer): cint{.extdecl, - importc: "slk_attr_set", dynlib: pdcursesdll.} -proc slk_clear*(): cint{.extdecl, importc: "slk_clear", dynlib: pdcursesdll.} -proc slk_color*(a2: cshort): cint{.extdecl, importc: "slk_color", - dynlib: pdcursesdll.} -proc slk_init*(a2: cint): cint{.extdecl, importc: "slk_init", dynlib: pdcursesdll.} -proc slk_label*(a2: cint): cstring{.extdecl, importc: "slk_label", - dynlib: pdcursesdll.} -proc slk_noutrefresh*(): cint{.extdecl, importc: "slk_noutrefresh", - dynlib: pdcursesdll.} -proc slk_refresh*(): cint{.extdecl, importc: "slk_refresh", dynlib: pdcursesdll.} -proc slk_restore*(): cint{.extdecl, importc: "slk_restore", dynlib: pdcursesdll.} -proc slk_set*(a2: cint; a3: cstring; a4: cint): cint{.extdecl, importc: "slk_set", - dynlib: pdcursesdll.} -proc slk_touch*(): cint{.extdecl, importc: "slk_touch", dynlib: pdcursesdll.} -proc standend*(): cint{.extdecl, importc: "standend", dynlib: pdcursesdll.} -proc standout*(): cint{.extdecl, importc: "standout", dynlib: pdcursesdll.} -proc start_color*(): cint{.extdecl, importc: "start_color", dynlib: pdcursesdll.} -proc subpad*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint; a6: cint): ptr WINDOW{. - extdecl, importc: "subpad", dynlib: pdcursesdll.} -proc subwin*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint; a6: cint): ptr WINDOW{. - extdecl, importc: "subwin", dynlib: pdcursesdll.} -proc syncok*(a2: ptr WINDOW; a3: cunsignedchar): cint{.extdecl, importc: "syncok", - dynlib: pdcursesdll.} -proc termattrs*(): cunsignedlong{.extdecl, importc: "termattrs", - dynlib: pdcursesdll.} -proc termattrs2*(): cunsignedlong{.extdecl, importc: "term_attrs", - dynlib: pdcursesdll.} -proc termname*(): cstring{.extdecl, importc: "termname", dynlib: pdcursesdll.} -proc timeout*(a2: cint){.extdecl, importc: "timeout", dynlib: pdcursesdll.} -proc touchline*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "touchline", dynlib: pdcursesdll.} -proc touchwin*(a2: ptr WINDOW): cint{.extdecl, importc: "touchwin", - dynlib: pdcursesdll.} -proc typeahead*(a2: cint): cint{.extdecl, importc: "typeahead", - dynlib: pdcursesdll.} -proc untouchwin*(a2: ptr WINDOW): cint{.extdecl, importc: "untouchwin", - dynlib: pdcursesdll.} -proc use_env*(a2: cunsignedchar){.extdecl, importc: "use_env", dynlib: pdcursesdll.} -proc vidattr*(a2: cunsignedlong): cint{.extdecl, importc: "vidattr", - dynlib: pdcursesdll.} -proc vid_attr*(a2: cunsignedlong; a3: cshort; a4: pointer): cint{.extdecl, - importc: "vid_attr", dynlib: pdcursesdll.} -#int vidputs(chtype, int (*)(int)); -#int vid_puts(attr_t, short, void *, int (*)(int)); -proc vline*(a2: cunsignedlong; a3: cint): cint{.extdecl, importc: "vline", - dynlib: pdcursesdll.} -proc vwprintw*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, varargs, - importc: "vw_printw", dynlib: pdcursesdll.} -proc vwprintw2*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, varargs, - importc: "vwprintw", dynlib: pdcursesdll.} -proc vwscanw*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, varargs, - importc: "vw_scanw", dynlib: pdcursesdll.} -proc vwscanw2*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, varargs, - importc: "vwscanw", dynlib: pdcursesdll.} -proc waddchnstr*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: cint): cint{.extdecl, - importc: "waddchnstr", dynlib: pdcursesdll.} -proc waddchstr*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "waddchstr", dynlib: pdcursesdll.} -proc waddch*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, importc: "waddch", - dynlib: pdcursesdll.} -proc waddnstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "waddnstr", dynlib: pdcursesdll.} -proc waddstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, importc: "waddstr", - dynlib: pdcursesdll.} -proc wattroff*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "wattroff", dynlib: pdcursesdll.} -proc wattron*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "wattron", dynlib: pdcursesdll.} -proc wattrset*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "wattrset", dynlib: pdcursesdll.} -proc wattr_get*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: ptr cshort; - a5: pointer): cint{.extdecl, importc: "wattr_get", - dynlib: pdcursesdll.} -proc wattr_off*(a2: ptr WINDOW; a3: cunsignedlong; a4: pointer): cint{.extdecl, - importc: "wattr_off", dynlib: pdcursesdll.} -proc wattr_on*(a2: ptr WINDOW; a3: cunsignedlong; a4: pointer): cint{.extdecl, - importc: "wattr_on", dynlib: pdcursesdll.} -proc wattr_set*(a2: ptr WINDOW; a3: cunsignedlong; a4: cshort; a5: pointer): cint{. - extdecl, importc: "wattr_set", dynlib: pdcursesdll.} -proc wbkgdset*(a2: ptr WINDOW; a3: cunsignedlong){.extdecl, importc: "wbkgdset", - dynlib: pdcursesdll.} -proc wbkgd*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, importc: "wbkgd", - dynlib: pdcursesdll.} -proc wborder*(a2: ptr WINDOW; a3: cunsignedlong; a4: cunsignedlong; - a5: cunsignedlong; a6: cunsignedlong; a7: cunsignedlong; - a8: cunsignedlong; a9: cunsignedlong; a10: cunsignedlong): cint{. - extdecl, importc: "wborder", dynlib: pdcursesdll.} -proc wchgat*(a2: ptr WINDOW; a3: cint; a4: cunsignedlong; a5: cshort; - a6: pointer): cint{.extdecl, importc: "wchgat", dynlib: pdcursesdll.} -proc wclear*(a2: ptr WINDOW): cint{.extdecl, importc: "wclear", - dynlib: pdcursesdll.} -proc wclrtobot*(a2: ptr WINDOW): cint{.extdecl, importc: "wclrtobot", - dynlib: pdcursesdll.} -proc wclrtoeol*(a2: ptr WINDOW): cint{.extdecl, importc: "wclrtoeol", - dynlib: pdcursesdll.} -proc wcolor_set*(a2: ptr WINDOW; a3: cshort; a4: pointer): cint{.extdecl, - importc: "wcolor_set", dynlib: pdcursesdll.} -proc wcursyncup*(a2: ptr WINDOW){.extdecl, importc: "wcursyncup", - dynlib: pdcursesdll.} -proc wdelch*(a2: ptr WINDOW): cint{.extdecl, importc: "wdelch", - dynlib: pdcursesdll.} -proc wdeleteln*(a2: ptr WINDOW): cint{.extdecl, importc: "wdeleteln", - dynlib: pdcursesdll.} -proc wechochar*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "wechochar", dynlib: pdcursesdll.} -proc werase*(a2: ptr WINDOW): cint{.extdecl, importc: "werase", - dynlib: pdcursesdll.} -proc wgetch*(a2: ptr WINDOW): cint{.extdecl, importc: "wgetch", - dynlib: pdcursesdll.} -proc wgetnstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "wgetnstr", dynlib: pdcursesdll.} -proc wgetstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, importc: "wgetstr", - dynlib: pdcursesdll.} -proc whline*(a2: ptr WINDOW; a3: cunsignedlong; a4: cint): cint{.extdecl, - importc: "whline", dynlib: pdcursesdll.} -proc winchnstr*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: cint): cint{.extdecl, - importc: "winchnstr", dynlib: pdcursesdll.} -proc winchstr*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "winchstr", dynlib: pdcursesdll.} -proc winch*(a2: ptr WINDOW): cunsignedlong{.extdecl, importc: "winch", - dynlib: pdcursesdll.} -proc winnstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "winnstr", dynlib: pdcursesdll.} -proc winsch*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, importc: "winsch", - dynlib: pdcursesdll.} -proc winsdelln*(a2: ptr WINDOW; a3: cint): cint{.extdecl, importc: "winsdelln", - dynlib: pdcursesdll.} -proc winsertln*(a2: ptr WINDOW): cint{.extdecl, importc: "winsertln", - dynlib: pdcursesdll.} -proc winsnstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "winsnstr", dynlib: pdcursesdll.} -proc winsstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, importc: "winsstr", - dynlib: pdcursesdll.} -proc winstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, importc: "winstr", - dynlib: pdcursesdll.} -proc wmove*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, importc: "wmove", - dynlib: pdcursesdll.} -proc wnoutrefresh*(a2: ptr WINDOW): cint{.extdecl, importc: "wnoutrefresh", - dynlib: pdcursesdll.} -proc wprintw*(a2: ptr WINDOW; a3: cstring): cint{.varargs, extdecl, - importc: "wprintw", dynlib: pdcursesdll.} -proc wredrawln*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "wredrawln", dynlib: pdcursesdll.} -proc wrefresh*(a2: ptr WINDOW): cint{.extdecl, importc: "wrefresh", - dynlib: pdcursesdll.} -proc wscanw*(a2: ptr WINDOW; a3: cstring): cint{.varargs, extdecl, - importc: "wscanw", dynlib: pdcursesdll.} -proc wscrl*(a2: ptr WINDOW; a3: cint): cint{.extdecl, importc: "wscrl", - dynlib: pdcursesdll.} -proc wsetscrreg*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "wsetscrreg", dynlib: pdcursesdll.} -proc wstandend*(a2: ptr WINDOW): cint{.extdecl, importc: "wstandend", - dynlib: pdcursesdll.} -proc wstandout*(a2: ptr WINDOW): cint{.extdecl, importc: "wstandout", - dynlib: pdcursesdll.} -proc wsyncdown*(a2: ptr WINDOW){.extdecl, importc: "wsyncdown", - dynlib: pdcursesdll.} -proc wsyncup*(a2: ptr WINDOW){.extdecl, importc: "wsyncup", dynlib: pdcursesdll.} -proc wtimeout*(a2: ptr WINDOW; a3: cint){.extdecl, importc: "wtimeout", - dynlib: pdcursesdll.} -proc wtouchln*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cint): cint{.extdecl, - importc: "wtouchln", dynlib: pdcursesdll.} -proc wvline*(a2: ptr WINDOW; a3: cunsignedlong; a4: cint): cint{.extdecl, - importc: "wvline", dynlib: pdcursesdll.} -proc addnwstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "addnwstr", - dynlib: pdcursesdll.} -proc addwstr*(a2: cstring): cint{.extdecl, importc: "addwstr", - dynlib: pdcursesdll.} -proc add_wch*(a2: ptr cunsignedlong): cint{.extdecl, importc: "add_wch", - dynlib: pdcursesdll.} -proc add_wchnstr*(a2: ptr cunsignedlong; a3: cint): cint{.extdecl, - importc: "add_wchnstr", dynlib: pdcursesdll.} -proc add_wchstr*(a2: ptr cunsignedlong): cint{.extdecl, importc: "add_wchstr", - dynlib: pdcursesdll.} -proc border_set*(a2: ptr cunsignedlong; a3: ptr cunsignedlong; - a4: ptr cunsignedlong; a5: ptr cunsignedlong; - a6: ptr cunsignedlong; a7: ptr cunsignedlong; - a8: ptr cunsignedlong; a9: ptr cunsignedlong): cint{.extdecl, - importc: "border_set", dynlib: pdcursesdll.} -proc box_set*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: ptr cunsignedlong): cint{. - extdecl, importc: "box_set", dynlib: pdcursesdll.} -proc echo_wchar*(a2: ptr cunsignedlong): cint{.extdecl, importc: "echo_wchar", - dynlib: pdcursesdll.} -proc erasewchar*(a2: cstring): cint{.extdecl, importc: "erasewchar", - dynlib: pdcursesdll.} -proc getbkgrnd*(a2: ptr cunsignedlong): cint{.extdecl, importc: "getbkgrnd", - dynlib: pdcursesdll.} -proc getcchar*(a2: ptr cunsignedlong; a3: cstring; a4: ptr cunsignedlong; - a5: ptr cshort; a6: pointer): cint{.extdecl, importc: "getcchar", - dynlib: pdcursesdll.} -proc getn_wstr*(a2: ptr cint; a3: cint): cint{.extdecl, importc: "getn_wstr", - dynlib: pdcursesdll.} -proc get_wch*(a2: ptr cint): cint{.extdecl, importc: "get_wch", - dynlib: pdcursesdll.} -proc get_wstr*(a2: ptr cint): cint{.extdecl, importc: "get_wstr", - dynlib: pdcursesdll.} -proc hline_set*(a2: ptr cunsignedlong; a3: cint): cint{.extdecl, - importc: "hline_set", dynlib: pdcursesdll.} -proc innwstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "innwstr", - dynlib: pdcursesdll.} -proc ins_nwstr*(a2: cstring; a3: cint): cint{.extdecl, importc: "ins_nwstr", - dynlib: pdcursesdll.} -proc ins_wch*(a2: ptr cunsignedlong): cint{.extdecl, importc: "ins_wch", - dynlib: pdcursesdll.} -proc ins_wstr*(a2: cstring): cint{.extdecl, importc: "ins_wstr", - dynlib: pdcursesdll.} -proc inwstr*(a2: cstring): cint{.extdecl, importc: "inwstr", - dynlib: pdcursesdll.} -proc in_wch*(a2: ptr cunsignedlong): cint{.extdecl, importc: "in_wch", - dynlib: pdcursesdll.} -proc in_wchnstr*(a2: ptr cunsignedlong; a3: cint): cint{.extdecl, - importc: "in_wchnstr", dynlib: pdcursesdll.} -proc in_wchstr*(a2: ptr cunsignedlong): cint{.extdecl, importc: "in_wchstr", - dynlib: pdcursesdll.} -proc key_name*(a2: char): cstring{.extdecl, importc: "key_name", - dynlib: pdcursesdll.} -proc killwchar*(a2: cstring): cint{.extdecl, importc: "killwchar", - dynlib: pdcursesdll.} -proc mvaddnwstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvaddnwstr", dynlib: pdcursesdll.} -proc mvaddwstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, - importc: "mvaddwstr", dynlib: pdcursesdll.} -proc mvadd_wch*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvadd_wch", dynlib: pdcursesdll.} -proc mvadd_wchnstr*(a2: cint; a3: cint; a4: ptr cunsignedlong; a5: cint): cint{. - extdecl, importc: "mvadd_wchnstr", dynlib: pdcursesdll.} -proc mvadd_wchstr*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvadd_wchstr", dynlib: pdcursesdll.} -proc mvgetn_wstr*(a2: cint; a3: cint; a4: ptr cint; a5: cint): cint{.extdecl, - importc: "mvgetn_wstr", dynlib: pdcursesdll.} -proc mvget_wch*(a2: cint; a3: cint; a4: ptr cint): cint{.extdecl, - importc: "mvget_wch", dynlib: pdcursesdll.} -proc mvget_wstr*(a2: cint; a3: cint; a4: ptr cint): cint{.extdecl, - importc: "mvget_wstr", dynlib: pdcursesdll.} -proc mvhline_set*(a2: cint; a3: cint; a4: ptr cunsignedlong; a5: cint): cint{. - extdecl, importc: "mvhline_set", dynlib: pdcursesdll.} -proc mvinnwstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvinnwstr", dynlib: pdcursesdll.} -proc mvins_nwstr*(a2: cint; a3: cint; a4: cstring; a5: cint): cint{.extdecl, - importc: "mvins_nwstr", dynlib: pdcursesdll.} -proc mvins_wch*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvins_wch", dynlib: pdcursesdll.} -proc mvins_wstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, - importc: "mvins_wstr", dynlib: pdcursesdll.} -proc mvinwstr*(a2: cint; a3: cint; a4: cstring): cint{.extdecl, - importc: "mvinwstr", dynlib: pdcursesdll.} -proc mvin_wch*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvin_wch", dynlib: pdcursesdll.} -proc mvin_wchnstr*(a2: cint; a3: cint; a4: ptr cunsignedlong; a5: cint): cint{. - extdecl, importc: "mvin_wchnstr", dynlib: pdcursesdll.} -proc mvin_wchstr*(a2: cint; a3: cint; a4: ptr cunsignedlong): cint{.extdecl, - importc: "mvin_wchstr", dynlib: pdcursesdll.} -proc mvvline_set*(a2: cint; a3: cint; a4: ptr cunsignedlong; a5: cint): cint{. - extdecl, importc: "mvvline_set", dynlib: pdcursesdll.} -proc mvwaddnwstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwaddnwstr", dynlib: pdcursesdll.} -proc mvwaddwstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{. - extdecl, importc: "mvwaddwstr", dynlib: pdcursesdll.} -proc mvwadd_wch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwadd_wch", dynlib: pdcursesdll.} -proc mvwadd_wchnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong; - a6: cint): cint{.extdecl, importc: "mvwadd_wchnstr", - dynlib: pdcursesdll.} -proc mvwadd_wchstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwadd_wchstr", dynlib: pdcursesdll.} -proc mvwgetn_wstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cint; a6: cint): cint{. - extdecl, importc: "mvwgetn_wstr", dynlib: pdcursesdll.} -proc mvwget_wch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cint): cint{. - extdecl, importc: "mvwget_wch", dynlib: pdcursesdll.} -proc mvwget_wstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cint): cint{. - extdecl, importc: "mvwget_wstr", dynlib: pdcursesdll.} -proc mvwhline_set*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong; - a6: cint): cint{.extdecl, importc: "mvwhline_set", - dynlib: pdcursesdll.} -proc mvwinnwstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwinnwstr", dynlib: pdcursesdll.} -proc mvwins_nwstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring; a6: cint): cint{. - extdecl, importc: "mvwins_nwstr", dynlib: pdcursesdll.} -proc mvwins_wch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwins_wch", dynlib: pdcursesdll.} -proc mvwins_wstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{. - extdecl, importc: "mvwins_wstr", dynlib: pdcursesdll.} -proc mvwin_wch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwin_wch", dynlib: pdcursesdll.} -proc mvwin_wchnstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong; - a6: cint): cint{.extdecl, importc: "mvwin_wchnstr", - dynlib: pdcursesdll.} -proc mvwin_wchstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong): cint{. - extdecl, importc: "mvwin_wchstr", dynlib: pdcursesdll.} -proc mvwinwstr*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cstring): cint{. - extdecl, importc: "mvwinwstr", dynlib: pdcursesdll.} -proc mvwvline_set*(a2: ptr WINDOW; a3: cint; a4: cint; a5: ptr cunsignedlong; - a6: cint): cint{.extdecl, importc: "mvwvline_set", - dynlib: pdcursesdll.} -proc pecho_wchar*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "pecho_wchar", dynlib: pdcursesdll.} -proc setcchar*(a2: ptr cunsignedlong; a3: cstring; a4: cunsignedlong; - a5: cshort; a6: pointer): cint{.extdecl, importc: "setcchar", - dynlib: pdcursesdll.} -proc slk_wset*(a2: cint; a3: cstring; a4: cint): cint{.extdecl, - importc: "slk_wset", dynlib: pdcursesdll.} -proc unget_wch*(a2: char): cint{.extdecl, importc: "unget_wch", - dynlib: pdcursesdll.} -proc vline_set*(a2: ptr cunsignedlong; a3: cint): cint{.extdecl, - importc: "vline_set", dynlib: pdcursesdll.} -proc waddnwstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "waddnwstr", dynlib: pdcursesdll.} -proc waddwstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, - importc: "waddwstr", dynlib: pdcursesdll.} -proc wadd_wch*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "wadd_wch", dynlib: pdcursesdll.} -proc wadd_wchnstr*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: cint): cint{. - extdecl, importc: "wadd_wchnstr", dynlib: pdcursesdll.} -proc wadd_wchstr*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "wadd_wchstr", dynlib: pdcursesdll.} -proc wbkgrnd*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "wbkgrnd", dynlib: pdcursesdll.} -proc wbkgrndset*(a2: ptr WINDOW; a3: ptr cunsignedlong){.extdecl, - importc: "wbkgrndset", dynlib: pdcursesdll.} -proc wborder_set*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: ptr cunsignedlong; - a5: ptr cunsignedlong; a6: ptr cunsignedlong; - a7: ptr cunsignedlong; a8: ptr cunsignedlong; - a9: ptr cunsignedlong; a10: ptr cunsignedlong): cint{.extdecl, - importc: "wborder_set", dynlib: pdcursesdll.} -proc wecho_wchar*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "wecho_wchar", dynlib: pdcursesdll.} -proc wgetbkgrnd*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "wgetbkgrnd", dynlib: pdcursesdll.} -proc wgetn_wstr*(a2: ptr WINDOW; a3: ptr cint; a4: cint): cint{.extdecl, - importc: "wgetn_wstr", dynlib: pdcursesdll.} -proc wget_wch*(a2: ptr WINDOW; a3: ptr cint): cint{.extdecl, - importc: "wget_wch", dynlib: pdcursesdll.} -proc wget_wstr*(a2: ptr WINDOW; a3: ptr cint): cint{.extdecl, - importc: "wget_wstr", dynlib: pdcursesdll.} -proc whline_set*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: cint): cint{.extdecl, - importc: "whline_set", dynlib: pdcursesdll.} -proc winnwstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "winnwstr", dynlib: pdcursesdll.} -proc wins_nwstr*(a2: ptr WINDOW; a3: cstring; a4: cint): cint{.extdecl, - importc: "wins_nwstr", dynlib: pdcursesdll.} -proc wins_wch*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "wins_wch", dynlib: pdcursesdll.} -proc wins_wstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, - importc: "wins_wstr", dynlib: pdcursesdll.} -proc winwstr*(a2: ptr WINDOW; a3: cstring): cint{.extdecl, importc: "winwstr", - dynlib: pdcursesdll.} -proc win_wch*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "win_wch", dynlib: pdcursesdll.} -proc win_wchnstr*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: cint): cint{.extdecl, - importc: "win_wchnstr", dynlib: pdcursesdll.} -proc win_wchstr*(a2: ptr WINDOW; a3: ptr cunsignedlong): cint{.extdecl, - importc: "win_wchstr", dynlib: pdcursesdll.} -proc wunctrl*(a2: ptr cunsignedlong): cstring{.extdecl, importc: "wunctrl", - dynlib: pdcursesdll.} -proc wvline_set*(a2: ptr WINDOW; a3: ptr cunsignedlong; a4: cint): cint{.extdecl, - importc: "wvline_set", dynlib: pdcursesdll.} -proc getattrs*(a2: ptr WINDOW): cunsignedlong{.extdecl, importc: "getattrs", - dynlib: pdcursesdll.} -proc getbegx*(a2: ptr WINDOW): cint{.extdecl, importc: "getbegx", - dynlib: pdcursesdll.} -proc getbegy*(a2: ptr WINDOW): cint{.extdecl, importc: "getbegy", - dynlib: pdcursesdll.} -proc getmaxx*(a2: ptr WINDOW): cint{.extdecl, importc: "getmaxx", - dynlib: pdcursesdll.} -proc getmaxy*(a2: ptr WINDOW): cint{.extdecl, importc: "getmaxy", - dynlib: pdcursesdll.} -proc getparx*(a2: ptr WINDOW): cint{.extdecl, importc: "getparx", - dynlib: pdcursesdll.} -proc getpary*(a2: ptr WINDOW): cint{.extdecl, importc: "getpary", - dynlib: pdcursesdll.} -proc getcurx*(a2: ptr WINDOW): cint{.extdecl, importc: "getcurx", - dynlib: pdcursesdll.} -proc getcury*(a2: ptr WINDOW): cint{.extdecl, importc: "getcury", - dynlib: pdcursesdll.} -proc traceoff*(){.extdecl, importc: "traceoff", dynlib: pdcursesdll.} -proc traceon*(){.extdecl, importc: "traceon", dynlib: pdcursesdll.} -proc unctrl*(a2: cunsignedlong): cstring{.extdecl, importc: "unctrl", - dynlib: pdcursesdll.} -proc crmode*(): cint{.extdecl, importc: "crmode", dynlib: pdcursesdll.} -proc nocrmode*(): cint{.extdecl, importc: "nocrmode", dynlib: pdcursesdll.} -proc draino*(a2: cint): cint{.extdecl, importc: "draino", dynlib: pdcursesdll.} -proc resetterm*(): cint{.extdecl, importc: "resetterm", dynlib: pdcursesdll.} -proc fixterm*(): cint{.extdecl, importc: "fixterm", dynlib: pdcursesdll.} -proc saveterm*(): cint{.extdecl, importc: "saveterm", dynlib: pdcursesdll.} -proc setsyx*(a2: cint; a3: cint): cint{.extdecl, importc: "setsyx", - dynlib: pdcursesdll.} -proc mouse_set*(a2: cunsignedlong): cint{.extdecl, importc: "mouse_set", - dynlib: pdcursesdll.} -proc mouse_on*(a2: cunsignedlong): cint{.extdecl, importc: "mouse_on", - dynlib: pdcursesdll.} -proc mouse_off*(a2: cunsignedlong): cint{.extdecl, importc: "mouse_off", - dynlib: pdcursesdll.} -proc request_mouse_pos*(): cint{.extdecl, importc: "request_mouse_pos", - dynlib: pdcursesdll.} -proc map_button*(a2: cunsignedlong): cint{.extdecl, importc: "map_button", - dynlib: pdcursesdll.} -proc wmouse_position*(a2: ptr WINDOW; a3: ptr cint; a4: ptr cint){.extdecl, - importc: "wmouse_position", dynlib: pdcursesdll.} -proc getmouse*(): cunsignedlong{.extdecl, importc: "getmouse", dynlib: pdcursesdll.} -proc getbmap*(): cunsignedlong{.extdecl, importc: "getbmap", dynlib: pdcursesdll.} -proc assume_default_colors*(a2: cint; a3: cint): cint{.extdecl, - importc: "assume_default_colors", dynlib: pdcursesdll.} -proc curses_version*(): cstring{.extdecl, importc: "curses_version", - dynlib: pdcursesdll.} -proc has_key*(a2: cint): cunsignedchar{.extdecl, importc: "has_key", - dynlib: pdcursesdll.} -proc use_default_colors*(): cint{.extdecl, importc: "use_default_colors", - dynlib: pdcursesdll.} -proc wresize*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "wresize", dynlib: pdcursesdll.} -proc mouseinterval*(a2: cint): cint{.extdecl, importc: "mouseinterval", - dynlib: pdcursesdll.} -proc mousemask*(a2: cunsignedlong; a3: ptr cunsignedlong): cunsignedlong{.extdecl, - importc: "mousemask", dynlib: pdcursesdll.} -proc mouse_trafo*(a2: ptr cint; a3: ptr cint; a4: cunsignedchar): cunsignedchar{. - extdecl, importc: "mouse_trafo", dynlib: pdcursesdll.} -proc nc_getmouse*(a2: ptr MEVENT): cint{.extdecl, importc: "nc_getmouse", - dynlib: pdcursesdll.} -proc ungetmouse*(a2: ptr MEVENT): cint{.extdecl, importc: "ungetmouse", - dynlib: pdcursesdll.} -proc wenclose*(a2: ptr WINDOW; a3: cint; a4: cint): cunsignedchar{.extdecl, - importc: "wenclose", dynlib: pdcursesdll.} -proc wmouse_trafo*(a2: ptr WINDOW; a3: ptr cint; a4: ptr cint; a5: cunsignedchar): cunsignedchar{. - extdecl, importc: "wmouse_trafo", dynlib: pdcursesdll.} -proc addrawch*(a2: cunsignedlong): cint{.extdecl, importc: "addrawch", - dynlib: pdcursesdll.} -proc insrawch*(a2: cunsignedlong): cint{.extdecl, importc: "insrawch", - dynlib: pdcursesdll.} -proc is_termresized*(): cunsignedchar{.extdecl, importc: "is_termresized", - dynlib: pdcursesdll.} -proc mvaddrawch*(a2: cint; a3: cint; a4: cunsignedlong): cint{.extdecl, - importc: "mvaddrawch", dynlib: pdcursesdll.} -proc mvdeleteln*(a2: cint; a3: cint): cint{.extdecl, importc: "mvdeleteln", - dynlib: pdcursesdll.} -proc mvinsertln*(a2: cint; a3: cint): cint{.extdecl, importc: "mvinsertln", - dynlib: pdcursesdll.} -proc mvinsrawch*(a2: cint; a3: cint; a4: cunsignedlong): cint{.extdecl, - importc: "mvinsrawch", dynlib: pdcursesdll.} -proc mvwaddrawch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cunsignedlong): cint{. - extdecl, importc: "mvwaddrawch", dynlib: pdcursesdll.} -proc mvwdeleteln*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "mvwdeleteln", dynlib: pdcursesdll.} -proc mvwinsertln*(a2: ptr WINDOW; a3: cint; a4: cint): cint{.extdecl, - importc: "mvwinsertln", dynlib: pdcursesdll.} -proc mvwinsrawch*(a2: ptr WINDOW; a3: cint; a4: cint; a5: cunsignedlong): cint{. - extdecl, importc: "mvwinsrawch", dynlib: pdcursesdll.} -proc raw_output*(a2: cunsignedchar): cint{.extdecl, importc: "raw_output", - dynlib: pdcursesdll.} -proc resize_term*(a2: cint; a3: cint): cint{.extdecl, importc: "resize_term", - dynlib: pdcursesdll.} -proc resize_window*(a2: ptr WINDOW; a3: cint; a4: cint): ptr WINDOW{.extdecl, - importc: "resize_window", dynlib: pdcursesdll.} -proc waddrawch*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "waddrawch", dynlib: pdcursesdll.} -proc winsrawch*(a2: ptr WINDOW; a3: cunsignedlong): cint{.extdecl, - importc: "winsrawch", dynlib: pdcursesdll.} -proc wordchar*(): char{.extdecl, importc: "wordchar", dynlib: pdcursesdll.} -proc slk_wlabel*(a2: cint): cstring{.extdecl, importc: "slk_wlabel", - dynlib: pdcursesdll.} -proc debug*(a2: cstring){.varargs, extdecl, importc: "PDC_debug", - dynlib: pdcursesdll.} -proc ungetch*(a2: cint): cint{.extdecl, importc: "PDC_ungetch", - dynlib: pdcursesdll.} -proc set_blink*(a2: cunsignedchar): cint{.extdecl, importc: "PDC_set_blink", - dynlib: pdcursesdll.} -proc set_line_color*(a2: cshort): cint{.extdecl, importc: "PDC_set_line_color", - dynlib: pdcursesdll.} -proc set_title*(a2: cstring){.extdecl, importc: "PDC_set_title", - dynlib: pdcursesdll.} -proc clearclipboard*(): cint{.extdecl, importc: "PDC_clearclipboard", - dynlib: pdcursesdll.} -proc freeclipboard*(a2: cstring): cint{.extdecl, importc: "PDC_freeclipboard", - dynlib: pdcursesdll.} -proc getclipboard*(a2: cstringArray; a3: ptr clong): cint{.extdecl, - importc: "PDC_getclipboard", dynlib: pdcursesdll.} -proc setclipboard*(a2: cstring; a3: clong): cint{.extdecl, - importc: "PDC_setclipboard", dynlib: pdcursesdll.} -proc get_input_fd*(): cunsignedlong{.extdecl, importc: "PDC_get_input_fd", - dynlib: pdcursesdll.} -proc get_key_modifiers*(): cunsignedlong{.extdecl, - importc: "PDC_get_key_modifiers", dynlib: pdcursesdll.} -proc return_key_modifiers*(a2: cunsignedchar): cint{.extdecl, - importc: "PDC_return_key_modifiers", dynlib: pdcursesdll.} -proc save_key_modifiers*(a2: cunsignedchar): cint{.extdecl, - importc: "PDC_save_key_modifiers", dynlib: pdcursesdll.} -proc bottom_panel*(pan: ptr PANEL): cint{.extdecl, importc: "bottom_panel", - dynlib: pdcursesdll.} -proc del_panel*(pan: ptr PANEL): cint{.extdecl, importc: "del_panel", - dynlib: pdcursesdll.} -proc hide_panel*(pan: ptr PANEL): cint{.extdecl, importc: "hide_panel", - dynlib: pdcursesdll.} -proc move_panel*(pan: ptr PANEL; starty: cint; startx: cint): cint{.extdecl, - importc: "move_panel", dynlib: pdcursesdll.} -proc new_panel*(win: ptr WINDOW): ptr PANEL{.extdecl, importc: "new_panel", - dynlib: pdcursesdll.} -proc panel_above*(pan: ptr PANEL): ptr PANEL{.extdecl, importc: "panel_above", - dynlib: pdcursesdll.} -proc panel_below*(pan: ptr PANEL): ptr PANEL{.extdecl, importc: "panel_below", - dynlib: pdcursesdll.} -proc panel_hidden*(pan: ptr PANEL): cint{.extdecl, importc: "panel_hidden", - dynlib: pdcursesdll.} -proc panel_userptr*(pan: ptr PANEL): pointer{.extdecl, importc: "panel_userptr", - dynlib: pdcursesdll.} -proc panel_window*(pan: ptr PANEL): ptr WINDOW{.extdecl, importc: "panel_window", - dynlib: pdcursesdll.} -proc replace_panel*(pan: ptr PANEL; win: ptr WINDOW): cint{.extdecl, - importc: "replace_panel", dynlib: pdcursesdll.} -proc set_panel_userptr*(pan: ptr PANEL; uptr: pointer): cint{.extdecl, - importc: "set_panel_userptr", dynlib: pdcursesdll.} -proc show_panel*(pan: ptr PANEL): cint{.extdecl, importc: "show_panel", - dynlib: pdcursesdll.} -proc top_panel*(pan: ptr PANEL): cint{.extdecl, importc: "top_panel", - dynlib: pdcursesdll.} -proc update_panels*(){.extdecl, importc: "update_panels", dynlib: pdcursesdll.} - -when unixOS: - proc Xinitscr*(a2: cint; a3: cstringArray): ptr WINDOW{.extdecl, - importc: "Xinitscr", dynlib: pdcursesdll.} - proc XCursesExit*(){.extdecl, importc: "XCursesExit", dynlib: pdcursesdll.} - proc sb_init*(): cint{.extdecl, importc: "sb_init", dynlib: pdcursesdll.} - proc sb_set_horz*(a2: cint; a3: cint; a4: cint): cint{.extdecl, - importc: "sb_set_horz", dynlib: pdcursesdll.} - proc sb_set_vert*(a2: cint; a3: cint; a4: cint): cint{.extdecl, - importc: "sb_set_vert", dynlib: pdcursesdll.} - proc sb_get_horz*(a2: ptr cint; a3: ptr cint; a4: ptr cint): cint{.extdecl, - importc: "sb_get_horz", dynlib: pdcursesdll.} - proc sb_get_vert*(a2: ptr cint; a3: ptr cint; a4: ptr cint): cint{.extdecl, - importc: "sb_get_vert", dynlib: pdcursesdll.} - proc sb_refresh*(): cint{.extdecl, importc: "sb_refresh", dynlib: pdcursesdll.} - -template getch*(): expr = - wgetch(stdscr) - -template ungetch*(ch: expr): expr = - ungetch(ch) - -template getbegyx*(w, y, x: expr): expr = - y = getbegy(w) - x = getbegx(w) - -template getmaxyx*(w, y, x: expr): expr = - y = getmaxy(w) - x = getmaxx(w) - -template getparyx*(w, y, x: expr): expr = - y = getpary(w) - x = getparx(w) - -template getyx*(w, y, x: expr): expr = - y = getcury(w) - x = getcurx(w) - -template getsyx*(y, x: expr): stmt = - if curscr.leaveit: - (x) = - 1 - (y) = (x) - else: getyx(curscr, (y), (x)) - -template getmouse*(x: expr): expr = - nc_getmouse(x) - -when defined(windows): - var - atrtab*{.importc: "pdc_atrtab", dynlib: pdcursesdll.}: cstring - con_out*{.importc: "pdc_con_out", dynlib: pdcursesdll.}: HANDLE - con_in*{.importc: "pdc_con_in", dynlib: pdcursesdll.}: HANDLE - quick_edit*{.importc: "pdc_quick_edit", dynlib: pdcursesdll.}: DWORD - - proc get_buffer_rows*(): cint{.extdecl, importc: "PDC_get_buffer_rows", - dynlib: pdcursesdll.} diff --git a/lib/wrappers/postgres.nim b/lib/wrappers/postgres.nim deleted file mode 100644 index f9a10dccb..000000000 --- a/lib/wrappers/postgres.nim +++ /dev/null @@ -1,364 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module contains the definitions for structures and externs for -# functions used by frontend postgres applications. It is based on -# Postgresql's libpq-fe.h. -# -# It is for postgreSQL version 7.4 and higher with support for the v3.0 -# connection-protocol. -# - -{.deadCodeElim: on.} - -when defined(windows): - const - dllName = "libpq.dll" -elif defined(macosx): - const - dllName = "libpq.dylib" -else: - const - dllName = "libpq.so(.5|)" -type - POid* = ptr Oid - Oid* = int32 - -const - ERROR_MSG_LENGTH* = 4096 - CMDSTATUS_LEN* = 40 - -type - SockAddr* = array[1..112, int8] - PGresAttDesc*{.pure, final.} = object - name*: cstring - adtid*: Oid - adtsize*: int - - PPGresAttDesc* = ptr PGresAttDesc - PPPGresAttDesc* = ptr PPGresAttDesc - PGresAttValue*{.pure, final.} = object - length*: int32 - value*: cstring - - PPGresAttValue* = ptr PGresAttValue - PPPGresAttValue* = ptr PPGresAttValue - PExecStatusType* = ptr ExecStatusType - ExecStatusType* = enum - PGRES_EMPTY_QUERY = 0, PGRES_COMMAND_OK, PGRES_TUPLES_OK, PGRES_COPY_OUT, - PGRES_COPY_IN, PGRES_BAD_RESPONSE, PGRES_NONFATAL_ERROR, PGRES_FATAL_ERROR - PGlobjfuncs*{.pure, final.} = object - fn_lo_open*: Oid - fn_lo_close*: Oid - fn_lo_creat*: Oid - fn_lo_unlink*: Oid - fn_lo_lseek*: Oid - fn_lo_tell*: Oid - fn_lo_read*: Oid - fn_lo_write*: Oid - - PPGlobjfuncs* = ptr PGlobjfuncs - PConnStatusType* = ptr ConnStatusType - ConnStatusType* = enum - CONNECTION_OK, CONNECTION_BAD, CONNECTION_STARTED, CONNECTION_MADE, - CONNECTION_AWAITING_RESPONSE, CONNECTION_AUTH_OK, CONNECTION_SETENV, - CONNECTION_SSL_STARTUP, CONNECTION_NEEDED - PGconn*{.pure, final.} = object - pghost*: cstring - pgtty*: cstring - pgport*: cstring - pgoptions*: cstring - dbName*: cstring - status*: ConnStatusType - errorMessage*: array[0..(ERROR_MSG_LENGTH) - 1, char] - Pfin*: File - Pfout*: File - Pfdebug*: File - sock*: int32 - laddr*: SockAddr - raddr*: SockAddr - salt*: array[0..(2) - 1, char] - asyncNotifyWaiting*: int32 - notifyList*: pointer - pguser*: cstring - pgpass*: cstring - lobjfuncs*: PPGlobjfuncs - - PPGconn* = ptr PGconn - PGresult*{.pure, final.} = object - ntups*: int32 - numAttributes*: int32 - attDescs*: PPGresAttDesc - tuples*: PPPGresAttValue - tupArrSize*: int32 - resultStatus*: ExecStatusType - cmdStatus*: array[0..(CMDSTATUS_LEN) - 1, char] - binary*: int32 - conn*: PPGconn - - PPGresult* = ptr PGresult - PPostgresPollingStatusType* = ptr PostgresPollingStatusType - PostgresPollingStatusType* = enum - PGRES_POLLING_FAILED = 0, PGRES_POLLING_READING, PGRES_POLLING_WRITING, - PGRES_POLLING_OK, PGRES_POLLING_ACTIVE - PPGTransactionStatusType* = ptr PGTransactionStatusType - PGTransactionStatusType* = enum - PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, - PQTRANS_UNKNOWN - PPGVerbosity* = ptr PGVerbosity - PGVerbosity* = enum - PQERRORS_TERSE, PQERRORS_DEFAULT, PQERRORS_VERBOSE - PpgNotify* = ptr pgNotify - pgNotify*{.pure, final.} = object - relname*: cstring - be_pid*: int32 - extra*: cstring - - PQnoticeReceiver* = proc (arg: pointer, res: PPGresult){.cdecl.} - PQnoticeProcessor* = proc (arg: pointer, message: cstring){.cdecl.} - Ppqbool* = ptr pqbool - pqbool* = char - P_PQprintOpt* = ptr PQprintOpt - PQprintOpt*{.pure, final.} = object - header*: pqbool - align*: pqbool - standard*: pqbool - html3*: pqbool - expanded*: pqbool - pager*: pqbool - fieldSep*: cstring - tableOpt*: cstring - caption*: cstring - fieldName*: ptr cstring - - P_PQconninfoOption* = ptr PQconninfoOption - PQconninfoOption*{.pure, final.} = object - keyword*: cstring - envvar*: cstring - compiled*: cstring - val*: cstring - label*: cstring - dispchar*: cstring - dispsize*: int32 - - PPQArgBlock* = ptr PQArgBlock - PQArgBlock*{.pure, final.} = object - length*: int32 - isint*: int32 - p*: pointer -{.deprecated: [TSockAddr: SockAddr, TPGresAttDesc: PgresAttDesc, - TPGresAttValue: PgresAttValue, TExecStatusType: ExecStatusType, - TPGlobjfuncs: Pglobjfuncs, TConnStatusType: ConnStatusType, TPGconn: Pgconn, - TPGresult: PGresult].} - -proc pqconnectStart*(conninfo: cstring): PPGconn{.cdecl, dynlib: dllName, - importc: "PQconnectStart".} -proc pqconnectPoll*(conn: PPGconn): PostgresPollingStatusType{.cdecl, - dynlib: dllName, importc: "PQconnectPoll".} -proc pqconnectdb*(conninfo: cstring): PPGconn{.cdecl, dynlib: dllName, - importc: "PQconnectdb".} -proc pqsetdbLogin*(pghost: cstring, pgport: cstring, pgoptions: cstring, - pgtty: cstring, dbName: cstring, login: cstring, pwd: cstring): PPGconn{. - cdecl, dynlib: dllName, importc: "PQsetdbLogin".} -proc pqsetdb*(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME: cstring): Ppgconn -proc pqfinish*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQfinish".} -proc pqconndefaults*(): PPQconninfoOption{.cdecl, dynlib: dllName, - importc: "PQconndefaults".} -proc pqconninfoFree*(connOptions: PPQconninfoOption){.cdecl, dynlib: dllName, - importc: "PQconninfoFree".} -proc pqresetStart*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQresetStart".} -proc pqresetPoll*(conn: PPGconn): PostgresPollingStatusType{.cdecl, - dynlib: dllName, importc: "PQresetPoll".} -proc pqreset*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQreset".} -proc pqrequestCancel*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQrequestCancel".} -proc pqdb*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQdb".} -proc pquser*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQuser".} -proc pqpass*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQpass".} -proc pqhost*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQhost".} -proc pqport*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQport".} -proc pqtty*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQtty".} -proc pqoptions*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, - importc: "PQoptions".} -proc pqstatus*(conn: PPGconn): ConnStatusType{.cdecl, dynlib: dllName, - importc: "PQstatus".} -proc pqtransactionStatus*(conn: PPGconn): PGTransactionStatusType{.cdecl, - dynlib: dllName, importc: "PQtransactionStatus".} -proc pqparameterStatus*(conn: PPGconn, paramName: cstring): cstring{.cdecl, - dynlib: dllName, importc: "PQparameterStatus".} -proc pqprotocolVersion*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQprotocolVersion".} -proc pqerrorMessage*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, - importc: "PQerrorMessage".} -proc pqsocket*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQsocket".} -proc pqbackendPID*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQbackendPID".} -proc pqclientEncoding*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQclientEncoding".} -proc pqsetClientEncoding*(conn: PPGconn, encoding: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQsetClientEncoding".} -when defined(USE_SSL): - # Get the SSL structure associated with a connection - proc pqgetssl*(conn: PPGconn): PSSL{.cdecl, dynlib: dllName, - importc: "PQgetssl".} -proc pqsetErrorVerbosity*(conn: PPGconn, verbosity: PGVerbosity): PGVerbosity{. - cdecl, dynlib: dllName, importc: "PQsetErrorVerbosity".} -proc pqtrace*(conn: PPGconn, debug_port: File){.cdecl, dynlib: dllName, - importc: "PQtrace".} -proc pquntrace*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQuntrace".} -proc pqsetNoticeReceiver*(conn: PPGconn, theProc: PQnoticeReceiver, arg: pointer): PQnoticeReceiver{. - cdecl, dynlib: dllName, importc: "PQsetNoticeReceiver".} -proc pqsetNoticeProcessor*(conn: PPGconn, theProc: PQnoticeProcessor, - arg: pointer): PQnoticeProcessor{.cdecl, - dynlib: dllName, importc: "PQsetNoticeProcessor".} -proc pqexec*(conn: PPGconn, query: cstring): PPGresult{.cdecl, dynlib: dllName, - importc: "PQexec".} -proc pqexecParams*(conn: PPGconn, command: cstring, nParams: int32, - paramTypes: POid, paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQexecParams".} -proc pqprepare*(conn: PPGconn, stmtName, query: cstring, nParams: int32, - paramTypes: POid): PPGresult{.cdecl, dynlib: dllName, importc: "PQprepare".} -proc pqexecPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, - paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQexecPrepared".} -proc pqsendQuery*(conn: PPGconn, query: cstring): int32{.cdecl, dynlib: dllName, - importc: "PQsendQuery".} -proc pqsendQueryParams*(conn: PPGconn, command: cstring, nParams: int32, - paramTypes: POid, paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, - resultFormat: int32): int32{.cdecl, dynlib: dllName, - importc: "PQsendQueryParams".} -proc pqsendQueryPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, - paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, - resultFormat: int32): int32{.cdecl, dynlib: dllName, - importc: "PQsendQueryPrepared".} -proc pqgetResult*(conn: PPGconn): PPGresult{.cdecl, dynlib: dllName, - importc: "PQgetResult".} -proc pqisBusy*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQisBusy".} -proc pqconsumeInput*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconsumeInput".} -proc pqnotifies*(conn: PPGconn): PPGnotify{.cdecl, dynlib: dllName, - importc: "PQnotifies".} -proc pqputCopyData*(conn: PPGconn, buffer: cstring, nbytes: int32): int32{. - cdecl, dynlib: dllName, importc: "PQputCopyData".} -proc pqputCopyEnd*(conn: PPGconn, errormsg: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQputCopyEnd".} -proc pqgetCopyData*(conn: PPGconn, buffer: cstringArray, async: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetCopyData".} -proc pqgetline*(conn: PPGconn, str: cstring, len: int32): int32{.cdecl, - dynlib: dllName, importc: "PQgetline".} -proc pqputline*(conn: PPGconn, str: cstring): int32{.cdecl, dynlib: dllName, - importc: "PQputline".} -proc pqgetlineAsync*(conn: PPGconn, buffer: cstring, bufsize: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetlineAsync".} -proc pqputnbytes*(conn: PPGconn, buffer: cstring, nbytes: int32): int32{.cdecl, - dynlib: dllName, importc: "PQputnbytes".} -proc pqendcopy*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQendcopy".} -proc pqsetnonblocking*(conn: PPGconn, arg: int32): int32{.cdecl, - dynlib: dllName, importc: "PQsetnonblocking".} -proc pqisnonblocking*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQisnonblocking".} -proc pqflush*(conn: PPGconn): int32{.cdecl, dynlib: dllName, importc: "PQflush".} -proc pqfn*(conn: PPGconn, fnid: int32, result_buf, result_len: ptr int32, - result_is_int: int32, args: PPQArgBlock, nargs: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQfn".} -proc pqresultStatus*(res: PPGresult): ExecStatusType{.cdecl, dynlib: dllName, - importc: "PQresultStatus".} -proc pqresStatus*(status: ExecStatusType): cstring{.cdecl, dynlib: dllName, - importc: "PQresStatus".} -proc pqresultErrorMessage*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQresultErrorMessage".} -proc pqresultErrorField*(res: PPGresult, fieldcode: int32): cstring{.cdecl, - dynlib: dllName, importc: "PQresultErrorField".} -proc pqntuples*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQntuples".} -proc pqnfields*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQnfields".} -proc pqbinaryTuples*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQbinaryTuples".} -proc pqfname*(res: PPGresult, field_num: int32): cstring{.cdecl, - dynlib: dllName, importc: "PQfname".} -proc pqfnumber*(res: PPGresult, field_name: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQfnumber".} -proc pqftable*(res: PPGresult, field_num: int32): Oid{.cdecl, dynlib: dllName, - importc: "PQftable".} -proc pqftablecol*(res: PPGresult, field_num: int32): int32{.cdecl, - dynlib: dllName, importc: "PQftablecol".} -proc pqfformat*(res: PPGresult, field_num: int32): int32{.cdecl, - dynlib: dllName, importc: "PQfformat".} -proc pqftype*(res: PPGresult, field_num: int32): Oid{.cdecl, dynlib: dllName, - importc: "PQftype".} -proc pqfsize*(res: PPGresult, field_num: int32): int32{.cdecl, dynlib: dllName, - importc: "PQfsize".} -proc pqfmod*(res: PPGresult, field_num: int32): int32{.cdecl, dynlib: dllName, - importc: "PQfmod".} -proc pqcmdStatus*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQcmdStatus".} -proc pqoidStatus*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQoidStatus".} -proc pqoidValue*(res: PPGresult): Oid{.cdecl, dynlib: dllName, - importc: "PQoidValue".} -proc pqcmdTuples*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQcmdTuples".} -proc pqgetvalue*(res: PPGresult, tup_num: int32, field_num: int32): cstring{. - cdecl, dynlib: dllName, importc: "PQgetvalue".} -proc pqgetlength*(res: PPGresult, tup_num: int32, field_num: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetlength".} -proc pqgetisnull*(res: PPGresult, tup_num: int32, field_num: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetisnull".} -proc pqclear*(res: PPGresult){.cdecl, dynlib: dllName, importc: "PQclear".} -proc pqfreemem*(p: pointer){.cdecl, dynlib: dllName, importc: "PQfreemem".} -proc pqmakeEmptyPGresult*(conn: PPGconn, status: ExecStatusType): PPGresult{. - cdecl, dynlib: dllName, importc: "PQmakeEmptyPGresult".} -proc pqescapeString*(till, `from`: cstring, len: int): int{.cdecl, - dynlib: dllName, importc: "PQescapeString".} -proc pqescapeBytea*(bintext: cstring, binlen: int, bytealen: var int): cstring{. - cdecl, dynlib: dllName, importc: "PQescapeBytea".} -proc pqunescapeBytea*(strtext: cstring, retbuflen: var int): cstring{.cdecl, - dynlib: dllName, importc: "PQunescapeBytea".} -proc pqprint*(fout: File, res: PPGresult, ps: PPQprintOpt){.cdecl, - dynlib: dllName, importc: "PQprint".} -proc pqdisplayTuples*(res: PPGresult, fp: File, fillAlign: int32, - fieldSep: cstring, printHeader: int32, quiet: int32){. - cdecl, dynlib: dllName, importc: "PQdisplayTuples".} -proc pqprintTuples*(res: PPGresult, fout: File, printAttName: int32, - terseOutput: int32, width: int32){.cdecl, dynlib: dllName, - importc: "PQprintTuples".} -proc lo_open*(conn: PPGconn, lobjId: Oid, mode: int32): int32{.cdecl, - dynlib: dllName, importc: "lo_open".} -proc lo_close*(conn: PPGconn, fd: int32): int32{.cdecl, dynlib: dllName, - importc: "lo_close".} -proc lo_read*(conn: PPGconn, fd: int32, buf: cstring, length: int): int32{. - cdecl, dynlib: dllName, importc: "lo_read".} -proc lo_write*(conn: PPGconn, fd: int32, buf: cstring, length: int): int32{. - cdecl, dynlib: dllName, importc: "lo_write".} -proc lo_lseek*(conn: PPGconn, fd: int32, offset: int32, whence: int32): int32{. - cdecl, dynlib: dllName, importc: "lo_lseek".} -proc lo_creat*(conn: PPGconn, mode: int32): Oid{.cdecl, dynlib: dllName, - importc: "lo_creat".} -proc lo_tell*(conn: PPGconn, fd: int32): int32{.cdecl, dynlib: dllName, - importc: "lo_tell".} -proc lo_unlink*(conn: PPGconn, lobjId: Oid): int32{.cdecl, dynlib: dllName, - importc: "lo_unlink".} -proc lo_import*(conn: PPGconn, filename: cstring): Oid{.cdecl, dynlib: dllName, - importc: "lo_import".} -proc lo_export*(conn: PPGconn, lobjId: Oid, filename: cstring): int32{.cdecl, - dynlib: dllName, importc: "lo_export".} -proc pqmblen*(s: cstring, encoding: int32): int32{.cdecl, dynlib: dllName, - importc: "PQmblen".} -proc pqenv2encoding*(): int32{.cdecl, dynlib: dllName, importc: "PQenv2encoding".} -proc pqsetdb(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME: cstring): PPgConn = - result = pqSetdbLogin(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME, "", "") diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim deleted file mode 100644 index e7fd2bc36..000000000 --- a/lib/wrappers/sqlite3.nim +++ /dev/null @@ -1,369 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} -when defined(windows): - when defined(nimOldDlls): - const Lib = "sqlite3.dll" - elif defined(cpu64): - const Lib = "sqlite3_64.dll" - else: - const Lib = "sqlite3_32.dll" -elif defined(macosx): - const - Lib = "libsqlite3(|.0).dylib" -else: - const - Lib = "libsqlite3.so(|.0)" - -const - SQLITE_INTEGER* = 1 - SQLITE_FLOAT* = 2 - SQLITE_BLOB* = 4 - SQLITE_NULL* = 5 - SQLITE_TEXT* = 3 - SQLITE_UTF8* = 1 - SQLITE_UTF16LE* = 2 - SQLITE_UTF16BE* = 3 # Use native byte order - SQLITE_UTF16* = 4 # sqlite3_create_function only - SQLITE_ANY* = 5 #sqlite_exec return values - SQLITE_OK* = 0 - SQLITE_ERROR* = 1 # SQL error or missing database - SQLITE_INTERNAL* = 2 # An internal logic error in SQLite - SQLITE_PERM* = 3 # Access permission denied - SQLITE_ABORT* = 4 # Callback routine requested an abort - SQLITE_BUSY* = 5 # The database file is locked - SQLITE_LOCKED* = 6 # A table in the database is locked - SQLITE_NOMEM* = 7 # A malloc() failed - SQLITE_READONLY* = 8 # Attempt to write a readonly database - SQLITE_INTERRUPT* = 9 # Operation terminated by sqlite3_interrupt() - SQLITE_IOERR* = 10 # Some kind of disk I/O error occurred - SQLITE_CORRUPT* = 11 # The database disk image is malformed - SQLITE_NOTFOUND* = 12 # (Internal Only) Table or record not found - SQLITE_FULL* = 13 # Insertion failed because database is full - SQLITE_CANTOPEN* = 14 # Unable to open the database file - SQLITE_PROTOCOL* = 15 # Database lock protocol error - SQLITE_EMPTY* = 16 # Database is empty - SQLITE_SCHEMA* = 17 # The database schema changed - SQLITE_TOOBIG* = 18 # Too much data for one row of a table - SQLITE_CONSTRAINT* = 19 # Abort due to contraint violation - SQLITE_MISMATCH* = 20 # Data type mismatch - SQLITE_MISUSE* = 21 # Library used incorrectly - SQLITE_NOLFS* = 22 # Uses OS features not supported on host - SQLITE_AUTH* = 23 # Authorization denied - SQLITE_FORMAT* = 24 # Auxiliary database format error - SQLITE_RANGE* = 25 # 2nd parameter to sqlite3_bind out of range - SQLITE_NOTADB* = 26 # File opened that is not a database file - SQLITE_ROW* = 100 # sqlite3_step() has another row ready - SQLITE_DONE* = 101 # sqlite3_step() has finished executing - SQLITE_COPY* = 0 - SQLITE_CREATE_INDEX* = 1 - SQLITE_CREATE_TABLE* = 2 - SQLITE_CREATE_TEMP_INDEX* = 3 - SQLITE_CREATE_TEMP_TABLE* = 4 - SQLITE_CREATE_TEMP_TRIGGER* = 5 - SQLITE_CREATE_TEMP_VIEW* = 6 - SQLITE_CREATE_TRIGGER* = 7 - SQLITE_CREATE_VIEW* = 8 - SQLITE_DELETE* = 9 - SQLITE_DROP_INDEX* = 10 - SQLITE_DROP_TABLE* = 11 - SQLITE_DROP_TEMP_INDEX* = 12 - SQLITE_DROP_TEMP_TABLE* = 13 - SQLITE_DROP_TEMP_TRIGGER* = 14 - SQLITE_DROP_TEMP_VIEW* = 15 - SQLITE_DROP_TRIGGER* = 16 - SQLITE_DROP_VIEW* = 17 - SQLITE_INSERT* = 18 - SQLITE_PRAGMA* = 19 - SQLITE_READ* = 20 - SQLITE_SELECT* = 21 - SQLITE_TRANSACTION* = 22 - SQLITE_UPDATE* = 23 - SQLITE_ATTACH* = 24 - SQLITE_DETACH* = 25 - SQLITE_ALTER_TABLE* = 26 - SQLITE_REINDEX* = 27 - SQLITE_DENY* = 1 - SQLITE_IGNORE* = 2 # Original from sqlite3.h: - #define SQLITE_STATIC ((void(*)(void *))0) - #define SQLITE_TRANSIENT ((void(*)(void *))-1) - SQLITE_DETERMINISTIC* = 0x800 - -const - SQLITE_STATIC* = nil - SQLITE_TRANSIENT* = cast[pointer](- 1) - -type - Sqlite3 {.pure, final.} = object - PSqlite3* = ptr Sqlite3 - PPSqlite3* = ptr PSqlite3 - Context{.pure, final.} = object - Pcontext* = ptr Context - Tstmt{.pure, final.} = object - Pstmt* = ptr Tstmt - Value{.pure, final.} = object - Pvalue* = ptr Value - PValueArg* = array[0..127, Pvalue] - - Callback* = proc (para1: pointer, para2: int32, para3, - para4: cstringArray): int32{.cdecl.} - Tbind_destructor_func* = proc (para1: pointer){.cdecl.} - Create_function_step_func* = proc (para1: Pcontext, para2: int32, - para3: PValueArg){.cdecl.} - Create_function_func_func* = proc (para1: Pcontext, para2: int32, - para3: PValueArg){.cdecl.} - Create_function_final_func* = proc (para1: Pcontext){.cdecl.} - Result_func* = proc (para1: pointer){.cdecl.} - Create_collation_func* = proc (para1: pointer, para2: int32, para3: pointer, - para4: int32, para5: pointer): int32{.cdecl.} - Collation_needed_func* = proc (para1: pointer, para2: PSqlite3, eTextRep: int32, - para4: cstring){.cdecl.} -{.deprecated: [TSqlite3: Sqlite3, TContext: Context, Tvalue: Value, - Tcallback: Callback, Tcreate_function_step_func: Create_function_step_func, - Tcreate_function_func_func: Create_function_func_func, - Tcreate_function_final_func: Create_function_final_func, - Tresult_func: Result_func, Tcreate_collation_func: Create_collation_func, - Tcollation_needed_func: Collation_needed_func].} - -proc close*(para1: PSqlite3): int32{.cdecl, dynlib: Lib, importc: "sqlite3_close".} -proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer, - errmsg: var cstring): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_exec".} -proc last_insert_rowid*(para1: PSqlite3): int64{.cdecl, dynlib: Lib, - importc: "sqlite3_last_insert_rowid".} -proc changes*(para1: PSqlite3): int32{.cdecl, dynlib: Lib, importc: "sqlite3_changes".} -proc total_changes*(para1: PSqlite3): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_total_changes".} -proc interrupt*(para1: PSqlite3){.cdecl, dynlib: Lib, importc: "sqlite3_interrupt".} -proc complete*(sql: cstring): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_complete".} -proc complete16*(sql: pointer): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_complete16".} -proc busy_handler*(para1: PSqlite3, - para2: proc (para1: pointer, para2: int32): int32{.cdecl.}, - para3: pointer): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_busy_handler".} -proc busy_timeout*(para1: PSqlite3, ms: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_busy_timeout".} -proc get_table*(para1: PSqlite3, sql: cstring, resultp: var cstringArray, - nrow, ncolumn: var cint, errmsg: ptr cstring): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_get_table".} -proc free_table*(result: cstringArray){.cdecl, dynlib: Lib, - importc: "sqlite3_free_table".} - # Todo: see how translate sqlite3_mprintf, sqlite3_vmprintf, sqlite3_snprintf - # function sqlite3_mprintf(_para1:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_mprintf'; -proc mprintf*(para1: cstring): cstring{.cdecl, varargs, dynlib: Lib, - importc: "sqlite3_mprintf".} - #function sqlite3_vmprintf(_para1:Pchar; _para2:va_list):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_vmprintf'; -proc free*(z: cstring){.cdecl, dynlib: Lib, importc: "sqlite3_free".} - #function sqlite3_snprintf(_para1:longint; _para2:Pchar; _para3:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_snprintf'; -proc snprintf*(para1: int32, para2: cstring, para3: cstring): cstring{.cdecl, - dynlib: Lib, varargs, importc: "sqlite3_snprintf".} -proc set_authorizer*(para1: PSqlite3, xAuth: proc (para1: pointer, para2: int32, - para3: cstring, para4: cstring, para5: cstring, para6: cstring): int32{. - cdecl.}, pUserData: pointer): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_set_authorizer".} -proc trace*(para1: PSqlite3, xTrace: proc (para1: pointer, para2: cstring){.cdecl.}, - para3: pointer): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_trace".} -proc progress_handler*(para1: PSqlite3, para2: int32, - para3: proc (para1: pointer): int32{.cdecl.}, - para4: pointer){.cdecl, dynlib: Lib, - importc: "sqlite3_progress_handler".} -proc commit_hook*(para1: PSqlite3, para2: proc (para1: pointer): int32{.cdecl.}, - para3: pointer): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_commit_hook".} -proc open*(filename: cstring, ppDb: var PSqlite3): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_open".} -proc open16*(filename: pointer, ppDb: var PSqlite3): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_open16".} -proc errcode*(db: PSqlite3): int32{.cdecl, dynlib: Lib, importc: "sqlite3_errcode".} -proc errmsg*(para1: PSqlite3): cstring{.cdecl, dynlib: Lib, importc: "sqlite3_errmsg".} -proc errmsg16*(para1: PSqlite3): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_errmsg16".} -proc prepare*(db: PSqlite3, zSql: cstring, nBytes: int32, ppStmt: var Pstmt, - pzTail: ptr cstring): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_prepare".} - -proc prepare_v2*(db: PSqlite3, zSql: cstring, nByte: cint, ppStmt: var Pstmt, - pzTail: ptr cstring): cint {. - importc: "sqlite3_prepare_v2", cdecl, dynlib: Lib.} - -proc prepare16*(db: PSqlite3, zSql: pointer, nBytes: int32, ppStmt: var Pstmt, - pzTail: var pointer): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_prepare16".} -proc bind_blob*(para1: Pstmt, para2: int32, para3: pointer, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_blob".} -proc bind_double*(para1: Pstmt, para2: int32, para3: float64): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_bind_double".} -proc bind_int*(para1: Pstmt, para2: int32, para3: int32): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_bind_int".} -proc bind_int64*(para1: Pstmt, para2: int32, para3: int64): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_bind_int64".} -proc bind_null*(para1: Pstmt, para2: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_null".} -proc bind_text*(para1: Pstmt, para2: int32, para3: cstring, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_text".} -proc bind_text16*(para1: Pstmt, para2: int32, para3: pointer, para4: int32, - para5: Tbind_destructor_func): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_text16".} - #function sqlite3_bind_value(_para1:Psqlite3_stmt; _para2:longint; _para3:Psqlite3_value):longint;cdecl; external Sqlite3Lib name 'sqlite3_bind_value'; - #These overloaded functions were introduced to allow the use of SQLITE_STATIC and SQLITE_TRANSIENT - #It's the c world man ;-) -proc bind_blob*(para1: Pstmt, para2: int32, para3: pointer, n: int32, - para5: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_blob".} -proc bind_text*(para1: Pstmt, para2: int32, para3: cstring, n: int32, - para5: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_text".} -proc bind_text16*(para1: Pstmt, para2: int32, para3: pointer, para4: int32, - para5: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_text16".} -proc bind_parameter_count*(para1: Pstmt): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_bind_parameter_count".} -proc bind_parameter_name*(para1: Pstmt, para2: int32): cstring{.cdecl, - dynlib: Lib, importc: "sqlite3_bind_parameter_name".} -proc bind_parameter_index*(para1: Pstmt, zName: cstring): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_bind_parameter_index".} - #function sqlite3_clear_bindings(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_clear_bindings'; -proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_column_count".} -proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib, - importc: "sqlite3_column_name".} -proc column_table_name*(para1: Pstmt; para2: int32): cstring{.cdecl, dynlib: Lib, - importc: "sqlite3_column_table_name".} -proc column_name16*(para1: Pstmt, para2: int32): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_column_name16".} -proc column_decltype*(para1: Pstmt, i: int32): cstring{.cdecl, dynlib: Lib, - importc: "sqlite3_column_decltype".} -proc column_decltype16*(para1: Pstmt, para2: int32): pointer{.cdecl, - dynlib: Lib, importc: "sqlite3_column_decltype16".} -proc step*(para1: Pstmt): int32{.cdecl, dynlib: Lib, importc: "sqlite3_step".} -proc data_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_data_count".} -proc column_blob*(para1: Pstmt, iCol: int32): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_column_blob".} -proc column_bytes*(para1: Pstmt, iCol: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_column_bytes".} -proc column_bytes16*(para1: Pstmt, iCol: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_column_bytes16".} -proc column_double*(para1: Pstmt, iCol: int32): float64{.cdecl, dynlib: Lib, - importc: "sqlite3_column_double".} -proc column_int*(para1: Pstmt, iCol: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_column_int".} -proc column_int64*(para1: Pstmt, iCol: int32): int64{.cdecl, dynlib: Lib, - importc: "sqlite3_column_int64".} -proc column_text*(para1: Pstmt, iCol: int32): cstring{.cdecl, dynlib: Lib, - importc: "sqlite3_column_text".} -proc column_text16*(para1: Pstmt, iCol: int32): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_column_text16".} -proc column_type*(para1: Pstmt, iCol: int32): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_column_type".} -proc finalize*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_finalize".} -proc reset*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, importc: "sqlite3_reset".} -proc create_function*(para1: PSqlite3, zFunctionName: cstring, nArg: int32, - eTextRep: int32, para5: pointer, - xFunc: Create_function_func_func, - xStep: Create_function_step_func, - xFinal: Create_function_final_func): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_create_function".} -proc create_function16*(para1: PSqlite3, zFunctionName: pointer, nArg: int32, - eTextRep: int32, para5: pointer, - xFunc: Create_function_func_func, - xStep: Create_function_step_func, - xFinal: Create_function_final_func): int32{.cdecl, - dynlib: Lib, importc: "sqlite3_create_function16".} -proc aggregate_count*(para1: Pcontext): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_aggregate_count".} -proc value_blob*(para1: Pvalue): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_value_blob".} -proc value_bytes*(para1: Pvalue): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_value_bytes".} -proc value_bytes16*(para1: Pvalue): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_value_bytes16".} -proc value_double*(para1: Pvalue): float64{.cdecl, dynlib: Lib, - importc: "sqlite3_value_double".} -proc value_int*(para1: Pvalue): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_value_int".} -proc value_int64*(para1: Pvalue): int64{.cdecl, dynlib: Lib, - importc: "sqlite3_value_int64".} -proc value_text*(para1: Pvalue): cstring{.cdecl, dynlib: Lib, - importc: "sqlite3_value_text".} -proc value_text16*(para1: Pvalue): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_value_text16".} -proc value_text16le*(para1: Pvalue): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_value_text16le".} -proc value_text16be*(para1: Pvalue): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_value_text16be".} -proc value_type*(para1: Pvalue): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_value_type".} -proc aggregate_context*(para1: Pcontext, nBytes: int32): pointer{.cdecl, - dynlib: Lib, importc: "sqlite3_aggregate_context".} -proc user_data*(para1: Pcontext): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_user_data".} -proc get_auxdata*(para1: Pcontext, para2: int32): pointer{.cdecl, dynlib: Lib, - importc: "sqlite3_get_auxdata".} -proc set_auxdata*(para1: Pcontext, para2: int32, para3: pointer, - para4: proc (para1: pointer){.cdecl.}){.cdecl, dynlib: Lib, - importc: "sqlite3_set_auxdata".} -proc result_blob*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, dynlib: Lib, - importc: "sqlite3_result_blob".} -proc result_double*(para1: Pcontext, para2: float64){.cdecl, dynlib: Lib, - importc: "sqlite3_result_double".} -proc result_error*(para1: Pcontext, para2: cstring, para3: int32){.cdecl, - dynlib: Lib, importc: "sqlite3_result_error".} -proc result_error16*(para1: Pcontext, para2: pointer, para3: int32){.cdecl, - dynlib: Lib, importc: "sqlite3_result_error16".} -proc result_int*(para1: Pcontext, para2: int32){.cdecl, dynlib: Lib, - importc: "sqlite3_result_int".} -proc result_int64*(para1: Pcontext, para2: int64){.cdecl, dynlib: Lib, - importc: "sqlite3_result_int64".} -proc result_null*(para1: Pcontext){.cdecl, dynlib: Lib, - importc: "sqlite3_result_null".} -proc result_text*(para1: Pcontext, para2: cstring, para3: int32, - para4: Result_func){.cdecl, dynlib: Lib, - importc: "sqlite3_result_text".} -proc result_text16*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, dynlib: Lib, - importc: "sqlite3_result_text16".} -proc result_text16le*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, dynlib: Lib, - importc: "sqlite3_result_text16le".} -proc result_text16be*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, dynlib: Lib, - importc: "sqlite3_result_text16be".} -proc result_value*(para1: Pcontext, para2: Pvalue){.cdecl, dynlib: Lib, - importc: "sqlite3_result_value".} -proc create_collation*(para1: PSqlite3, zName: cstring, eTextRep: int32, - para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, dynlib: Lib, importc: "sqlite3_create_collation".} -proc create_collation16*(para1: PSqlite3, zName: cstring, eTextRep: int32, - para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, dynlib: Lib, importc: "sqlite3_create_collation16".} -proc collation_needed*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, dynlib: Lib, importc: "sqlite3_collation_needed".} -proc collation_needed16*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, dynlib: Lib, importc: "sqlite3_collation_needed16".} -proc libversion*(): cstring{.cdecl, dynlib: Lib, importc: "sqlite3_libversion".} - #Alias for allowing better code portability (win32 is not working with external variables) -proc version*(): cstring{.cdecl, dynlib: Lib, importc: "sqlite3_libversion".} - # Not published functions -proc libversion_number*(): int32{.cdecl, dynlib: Lib, - importc: "sqlite3_libversion_number".} - #function sqlite3_key(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_key'; - #function sqlite3_rekey(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_rekey'; - #function sqlite3_sleep(_para1:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_sleep'; - #function sqlite3_expired(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_expired'; - #function sqlite3_global_recover:longint;cdecl; external Sqlite3Lib name 'sqlite3_global_recover'; -# implementation diff --git a/lib/wrappers/tinyc.nim b/lib/wrappers/tinyc.nim index 47b505abc..8d2ba3802 100644 --- a/lib/wrappers/tinyc.nim +++ b/lib/wrappers/tinyc.nim @@ -12,7 +12,6 @@ type PccState* = ptr CcState ErrorFunc* = proc (opaque: pointer, msg: cstring) {.cdecl.} -{.deprecated: [TccState: CcState, TErrorFunc: ErrorFunc].} proc openCCState*(): PccState {.importc: "tcc_new", cdecl.} ## create a new TCC compilation context @@ -20,16 +19,12 @@ proc openCCState*(): PccState {.importc: "tcc_new", cdecl.} proc closeCCState*(s: PccState) {.importc: "tcc_delete", cdecl.} ## free a TCC compilation context -proc enableDebug*(s: PccState) {.importc: "tcc_enable_debug", cdecl.} - ## add debug information in the generated code - proc setErrorFunc*(s: PccState, errorOpaque: pointer, errorFun: ErrorFunc) {. cdecl, importc: "tcc_set_error_func".} ## set error/warning display callback -proc setWarning*(s: PccState, warningName: cstring, value: int) {.cdecl, - importc: "tcc_set_warning".} - ## set/reset a warning +proc setOptions*(s: PccState, options: cstring) {.cdecl, importc: "tcc_set_options".} + ## set a options # preprocessor @@ -41,7 +36,6 @@ proc addSysincludePath*(s: PccState, pathname: cstring) {.cdecl, importc: "tcc_add_sysinclude_path".} ## add in system include path - proc defineSymbol*(s: PccState, sym, value: cstring) {.cdecl, importc: "tcc_define_symbol".} ## define preprocessor symbol 'sym'. Can put optional value @@ -65,18 +59,14 @@ proc compileString*(s: PccState, buf: cstring): cint {.cdecl, const - OutputMemory*: cint = 0 ## output will be ran in memory (no + OutputMemory*: cint = 1 ## output will be ran in memory (no ## output file) (default) - OutputExe*: cint = 1 ## executable file - OutputDll*: cint = 2 ## dynamic library - OutputObj*: cint = 3 ## object file - OutputPreprocess*: cint = 4 ## preprocessed file (used internally) - - OutputFormatElf*: cint = 0 ## default output format: ELF - OutputFormatBinary*: cint = 1 ## binary image output - OutputFormatCoff*: cint = 2 ## COFF + OutputExe*: cint = 2 ## executable file + OutputDll*: cint = 3 ## dynamic library + OutputObj*: cint = 4 ## object file + OutputPreprocess*: cint = 5 ## preprocessed file (used internally) -proc setOutputType*(s: PCCState, outputType: cint): cint {.cdecl, +proc setOutputType*(s: PccState, outputType: cint): cint {.cdecl, importc: "tcc_set_output_type".} ## set output type. MUST BE CALLED before any compilation @@ -84,7 +74,7 @@ proc addLibraryPath*(s: PccState, pathname: cstring): cint {.cdecl, importc: "tcc_add_library_path".} ## equivalent to -Lpath option -proc addLibrary*(s: PCCState, libraryname: cstring): cint {.cdecl, +proc addLibrary*(s: PccState, libraryname: cstring): cint {.cdecl, importc: "tcc_add_library".} ## the library name is the same as the argument of the '-l' option @@ -115,5 +105,3 @@ proc getSymbol*(s: PccState, name: cstring): pointer {.cdecl, proc setLibPath*(s: PccState, path: cstring) {.cdecl, importc: "tcc_set_lib_path".} ## set CONFIG_TCCDIR at runtime - - |