Nim - This repository contains the Nim compiler, Nim's stdlib, tools, and documentation. (mirror)

	Commit message (Collapse)	Author	Age	Files	Lines
*	[js] add testcase for array indexDefect and remove todo (#19838)	flywind	2022-06-10	1	-0/+9
	* remove unused opcSubstr * [js] add testcase for array indexDefect * Revert "remove unused opcSubstr" This reverts commit cb461f2545234d62c1e0b83318f3e5495c97de52.

<!DOCTYPE html> <html> <head> <meta charset="UTF-8"> <title>Mu - linux/403unicode.mu</title> <meta name="Generator" content="Vim/8.2"> <meta name="plugin-version" content="vim8.1_v2"> <meta name="syntax" content="none"> <meta name="settings" content="number_lines,use_css,pre_wrap,no_foldcolumn,expand_tabs,line_ids,prevent_copy=,use_input_for_pc=fallback"> <meta name="colorscheme" content="minimal-light"> <style>  </style> <script>  </script> </head> <body onload='JumpToLine();'> <a href='https://github.com/akkartik/mu/blob/main/linux/403unicode.mu'>https://github.com/akkartik/mu/blob/main/linux/403unicode.mu</a> <pre id='vimCodeElement'> 1 # Helpers for Unicode. 2 # 3 # Mu has no characters, only code points and code-point-utf8s. 4 # Code points are the indivisible atoms of text streams. 5 # <a href="https://en.wikipedia.org/wiki/Code_point">https://en.wikipedia.org/wiki/Code_point</a> 6 # Graphemes are the smallest self-contained unit of text. 7 # Graphemes may consist of multiple code points. 8 # 9 # Mu code-point-utf8s are always represented in utf-8, and they are required to fit 10 # in 4 bytes. 11 # 12 # Mu doesn't currently support combining code points, or code-point-utf8s made of 13 # multiple code points. One day we will. 14 # On Linux, we also don't currently support code points that translate into 15 # multiple or wide code-point-utf8s. (In particular, Tab will never be supported.) 16 17 # transliterated from tb_utf8_unicode_to_char in <a href="https://github.com/nsf/termbox">https://github.com/nsf/termbox</a> 18 # <a href="https://wiki.tcl-lang.org/page/UTF%2D8+bit+by+bit">https://wiki.tcl-lang.org/page/UTF%2D8+bit+by+bit</a> explains the algorithm 19 fn <a href='403unicode.mu.html#L19'>to-utf8</a> in: code-point -> _/eax: code-point-utf8 { 20 var c/eax: int <- copy in 21 var num-trailers/ecx: int <- copy 0 22 var first/edx: int <- copy 0 23 $to-utf8:compute-length: { 24 # single byte: just return it 25 compare c, 0x7f 26 { 27 break-if-> 28 var g/eax: code-point-utf8 <- copy c 29 return g 30 } 31 # 2 bytes 32 compare c, 0x7ff 33 { 34 break-if-> 35 num-trailers <- copy 1 36 first <- copy 0xc0 37 break $to-utf8:compute-length 38 } 39 # 3 bytes 40 compare c, 0xffff 41 { 42 break-if-> 43 num-trailers <- copy 2 44 first <- copy 0xe0 45 break $to-utf8:compute-length 46 } 47 # 4 bytes 48 compare c, 0x1fffff 49 { 50 break-if-> 51 num-trailers <- copy 3 52 first <- copy 0xf0 53 break $to-utf8:compute-length 54 } 55 # more than 4 bytes: unsupported 56 # TODO: print to stderr 57 compare c, 0x1fffff 58 { 59 break-if-> 60 <a href='304screen.subx.html#L122'>print-string-to-real-screen</a> "unsupported code point " 61 <a href='304screen.subx.html#L226'>print-int32-hex-to-real-screen</a> c 62 <a href='304screen.subx.html#L122'>print-string-to-real-screen</a> "\n" 63 var exit-status/ebx: int <- copy 1 64 <a href='000init.subx.html#L18'>syscall_exit</a> 65 } 66 } 67 # emit trailer bytes, 6 bits from 'in', first two bits '10' 68 var result/edi: code-point-utf8 <- copy 0 69 { 70 compare num-trailers, 0 71 break-if-<= 72 var tmp/esi: int <- copy c 73 tmp <- and 0x3f 74 tmp <- or 0x80 75 result <- shift-left 8 76 result <- or tmp 77 # update loop state 78 c <- shift-right 6 79 num-trailers <- decrement 80 loop 81 } 82 # emit engine 83 result <- shift-left 8 84 result <- or c 85 result <- or first 86 # 87 return result 88 } 89 90 # single-byte code point have identical code-point-utf8s 91 fn <a href='403unicode.mu.html#L91'>test-to-utf8-single-byte</a> { 92 var in-int/ecx: int <- copy 0 93 { 94 compare in-int, 0x7f 95 break-if-> 96 var in/eax: code-point <- copy in-int 97 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 98 var out-int/eax: int <- copy out 99 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, in-int, "F - test-to-utf8-single-byte" 100 in-int <- increment 101 loop 102 } 103 } 104 105 # byte | byte | byte | byte 106 # smallest 2-byte utf-8 107 fn <a href='403unicode.mu.html#L107'>test-to-utf8-two-bytes-min</a> { 108 var in/eax: code-point <- copy 0x80 # 10 00-0000 109 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 110 var out-int/eax: int <- copy out 111 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, 0x80c2, "F - <a href='403unicode.mu.html#L19'>to-utf8</a>/2a" # 110 0-0010 10 00-0000 112 } 113 114 # largest 2-byte utf-8 115 fn <a href='403unicode.mu.html#L115'>test-to-utf8-two-bytes-max</a> { 116 var in/eax: code-point <- copy 0x7ff # 1-1111 11-1111 117 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 118 var out-int/eax: int <- copy out 119 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, 0xbfdf, "F - <a href='403unicode.mu.html#L19'>to-utf8</a>/2b" # 110 1-1111 10 11-1111 120 } 121 122 # smallest 3-byte utf-8 123 fn <a href='403unicode.mu.html#L123'>test-to-utf8-three-bytes-min</a> { 124 var in/eax: code-point <- copy 0x800 # 10-0000 00-0000 125 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 126 var out-int/eax: int <- copy out 127 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, 0x80a0e0, "F - <a href='403unicode.mu.html#L19'>to-utf8</a>/3a" # 1110 0000 10 10-0000 10 00-0000 128 } 129 130 # largest 3-byte utf-8 131 fn <a href='403unicode.mu.html#L131'>test-to-utf8-three-bytes-max</a> { 132 var in/eax: code-point <- copy 0xffff # 1111 11-1111 11-1111 133 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 134 var out-int/eax: int <- copy out 135 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, 0xbfbfef, "F - <a href='403unicode.mu.html#L19'>to-utf8</a>/3b" # 1110 1111 10 11-1111 10 11-1111 136 } 137 138 # smallest 4-byte utf-8 139 fn <a href='403unicode.mu.html#L139'>test-to-utf8-four-bytes-min</a> { 140 var in/eax: code-point <- copy 0x10000 # 1-0000 00-0000 00-0000 141 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 142 var out-int/eax: int <- copy out 143 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, 0x808090f0, "F - <a href='403unicode.mu.html#L19'>to-utf8</a>/4a" # 1111-0 000 10 01-0000 10 00-0000 10 00-0000 144 } 145 146 # largest 4-byte utf-8 147 fn <a href='403unicode.mu.html#L147'>test-to-utf8-four-bytes-max</a> { 148 var in/eax: code-point <- copy 0x1fffff # 111 11-1111 11-1111 11-1111 149 var out/eax: code-point-utf8 <- <a href='403unicode.mu.html#L19'>to-utf8</a> in 150 var out-int/eax: int <- copy out 151 <a href='102test.subx.html#L23'>check-ints-equal</a> out-int, 0xbfbfbff7, "F - <a href='403unicode.mu.html#L19'>to-utf8</a>/4b" # 1111-0 111 10 11-1111 10 11-1111 10 11-1111 152 } 153 154 # read the next code-point-utf8 from a stream of bytes 155 fn <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> in: (addr stream byte) -> _/eax: code-point-utf8 { 156 # if at eof, return EOF 157 { 158 var eof?/eax: boolean <- <a href='309stream.subx.html#L6'>stream-empty?</a> in 159 compare eof?, 0/false 160 break-if-= 161 return 0xffffffff 162 } 163 var c/eax: byte <- <a href='112read-byte.subx.html#L273'>read-byte</a> in 164 var num-trailers/ecx: int <- copy 0 165 $read-code-point-utf8:compute-length: { 166 # single byte: just return it 167 compare c, 0xc0 168 { 169 break-if->= 170 var g/eax: code-point-utf8 <- copy c 171 return g 172 } 173 compare c, 0xfe 174 { 175 break-if-< 176 var g/eax: code-point-utf8 <- copy c 177 return g 178 } 179 # 2 bytes 180 compare c, 0xe0 181 { 182 break-if->= 183 num-trailers <- copy 1 184 break $read-code-point-utf8:compute-length 185 } 186 # 3 bytes 187 compare c, 0xf0 188 { 189 break-if->= 190 num-trailers <- copy 2 191 break $read-code-point-utf8:compute-length 192 } 193 # 4 bytes 194 compare c, 0xf8 195 { 196 break-if->= 197 num-trailers <- copy 3 198 break $read-code-point-utf8:compute-length 199 } 200 $read-code-point-utf8:abort: { 201 # TODO: print to stderr 202 <a href='304screen.subx.html#L122'>print-string-to-real-screen</a> "utf-8 encodings larger than 4 bytes are not yet supported. First byte seen: " 203 var n/eax: int <- copy c 204 <a href='304screen.subx.html#L226'>print-int32-hex-to-real-screen</a> n 205 <a href='304screen.subx.html#L122'>print-string-to-real-screen</a> "\n" 206 var exit-status/ebx: int <- copy 1 207 <a href='000init.subx.html#L18'>syscall_exit</a> 208 } 209 } 210 # prepend trailer bytes 211 var result/edi: code-point-utf8 <- copy c 212 var num-byte-shifts/edx: int <- copy 1 213 { 214 compare num-trailers, 0 215 break-if-<= 216 var tmp/eax: byte <- <a href='112read-byte.subx.html#L273'>read-byte</a> in 217 var tmp2/eax: int <- copy tmp 218 tmp2 <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> tmp2, num-byte-shifts 219 result <- or tmp2 220 # update loop state 221 num-byte-shifts <- increment 222 num-trailers <- decrement 223 loop 224 } 225 return result 226 } 227 228 fn <a href='403unicode.mu.html#L228'>test-read-code-point-utf8</a> { 229 var s: (stream byte 0x30) 230 var s2/ecx: (addr stream byte) <- address s 231 <a href='108write.subx.html#L24'>write</a> s2, "aΒc世d界e" 232 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 233 var n/eax: int <- copy c 234 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x61, "F - test code-point-utf8/0" 235 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 236 var n/eax: int <- copy c 237 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x92ce/greek-capital-letter-beta, "F - test code-point-utf8/1" 238 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 239 var n/eax: int <- copy c 240 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x63, "F - test code-point-utf8/2" 241 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 242 var n/eax: int <- copy c 243 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x96b8e4, "F - test code-point-utf8/3" 244 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 245 var n/eax: int <- copy c 246 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x64, "F - test code-point-utf8/4" 247 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 248 var n/eax: int <- copy c 249 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x8c95e7, "F - test code-point-utf8/5" 250 var c/eax: code-point-utf8 <- <a href='403unicode.mu.html#L155'>read-code-point-utf8</a> s2 251 var n/eax: int <- copy c 252 <a href='102test.subx.html#L23'>check-ints-equal</a> n, 0x65, "F - test code-point-utf8/6" 253 } 254 255 fn <a href='403unicode.mu.html#L255'>read-code-point-utf8-buffered</a> in: (addr buffered-file) -> _/eax: code-point-utf8 { 256 var c/eax: byte <- <a href='112read-byte.subx.html#L38'>read-byte-buffered</a> in 257 var num-trailers/ecx: int <- copy 0 258 $read-code-point-utf8-buffered:compute-length: { 259 # single byte: just return it 260 compare c, 0xc0 261 { 262 break-if->= 263 var g/eax: code-point-utf8 <- copy c 264 return g 265 } 266 compare c, 0xfe 267 { 268 break-if-< 269 var g/eax: code-point-utf8 <- copy c 270 return g 271 } 272 # 2 bytes 273 compare c, 0xe0 274 { 275 break-if->= 276 num-trailers <- copy 1 277 break $read-code-point-utf8-buffered:compute-length 278 } 279 # 3 bytes 280 compare c, 0xf0 281 { 282 break-if->= 283 num-trailers <- copy 2 284 break $read-code-point-utf8-buffered:compute-length 285 } 286 # 4 bytes 287 compare c, 0xf8 288 { 289 break-if->= 290 num-trailers <- copy 3 291 break $read-code-point-utf8-buffered:compute-length 292 } 293 $read-code-point-utf8-buffered:abort: { 294 # TODO: print to stderr 295 <a href='304screen.subx.html#L122'>print-string-to-real-screen</a> "utf-8 encodings larger than 4 bytes are not supported. First byte seen: " 296 var n/eax: int <- copy c 297 <a href='304screen.subx.html#L226'>print-int32-hex-to-real-screen</a> n 298 <a href='304screen.subx.html#L122'>print-string-to-real-screen</a> "\n" 299 var exit-status/ebx: int <- copy 1 300 <a href='000init.subx.html#L18'>syscall_exit</a> 301 } 302 } 303 # prepend trailer bytes 304 var result/edi: code-point-utf8 <- copy c 305 var num-byte-shifts/edx: int <- copy 1 306 { 307 compare num-trailers, 0 308 break-if-<= 309 var tmp/eax: byte <- <a href='112read-byte.subx.html#L38'>read-byte-buffered</a> in 310 var tmp2/eax: int <- copy tmp 311 tmp2 <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> tmp2, num-byte-shifts 312 result <- or tmp2 313 # update loop state 314 num-byte-shifts <- increment 315 num-trailers <- decrement 316 loop 317 } 318 return result 319 } 320 321 # needed because available primitives only shift by a literal/constant number of bits 322 fn <a href='403unicode.mu.html#L322'>shift-left-bytes</a> n: int, k: int -> _/eax: int { 323 var i/ecx: int <- copy 0 324 var result/eax: int <- copy n 325 { 326 compare i, k 327 break-if->= 328 compare i, 4 # only 4 bytes in 32 bits 329 break-if->= 330 result <- shift-left 8 331 i <- increment 332 loop 333 } 334 return result 335 } 336 337 fn <a href='403unicode.mu.html#L337'>test-shift-left-bytes-0</a> { 338 var result/eax: int <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1, 0 339 <a href='102test.subx.html#L23'>check-ints-equal</a> result, 1, "F - <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 0" 340 } 341 342 fn <a href='403unicode.mu.html#L342'>test-shift-left-bytes-1</a> { 343 var result/eax: int <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1, 1 344 <a href='102test.subx.html#L23'>check-ints-equal</a> result, 0x100, "F - <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1" 345 } 346 347 fn <a href='403unicode.mu.html#L347'>test-shift-left-bytes-2</a> { 348 var result/eax: int <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1, 2 349 <a href='102test.subx.html#L23'>check-ints-equal</a> result, 0x10000, "F - <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 2" 350 } 351 352 fn <a href='403unicode.mu.html#L352'>test-shift-left-bytes-3</a> { 353 var result/eax: int <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1, 3 354 <a href='102test.subx.html#L23'>check-ints-equal</a> result, 0x1000000, "F - <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 3" 355 } 356 357 fn <a href='403unicode.mu.html#L357'>test-shift-left-bytes-4</a> { 358 var result/eax: int <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1, 4 359 <a href='102test.subx.html#L23'>check-ints-equal</a> result, 0, "F - <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 4" 360 } 361 362 fn <a href='403unicode.mu.html#L362'>test-shift-left-bytes-5</a> { 363 var result/eax: int <- <a href='403unicode.mu.html#L322'>shift-left-bytes</a> 1, 5 364 <a href='102test.subx.html#L23'>check-ints-equal</a> result, 0, "F - <a href='403unicode.mu.html#L322'>shift-left-bytes</a> >4" 365 } 366 367 # write a code-point-utf8 to a stream of bytes 368 # this is like write-to-stream, except we skip leading 0 bytes 369 fn <a href='403unicode.mu.html#L369'>write-code-point-utf8</a> out: (addr stream byte), g: code-point-utf8 { 370 $write-code-point-utf8:body: { 371 var c/eax: int <- copy g 372 <a href='115write-byte.subx.html#L208'>append-byte</a> out, c # first byte is always written 373 c <- shift-right 8 374 compare c, 0 375 break-if-= $write-code-point-utf8:body 376 <a href='115write-byte.subx.html#L208'>append-byte</a> out, c 377 c <- shift-right 8 378 compare c, 0 379 break-if-= $write-code-point-utf8:body 380 <a href='115write-byte.subx.html#L208'>append-byte</a> out, c 381 c <- shift-right 8 382 compare c, 0 383 break-if-= $write-code-point-utf8:body 384 <a href='115write-byte.subx.html#L208'>append-byte</a> out, c 385 } 386 } </pre> </body> </html>