summary refs log tree commit diff stats
path: root/tests/stdlib/tunidecode.nim
blob: 689453c76f75b3980f9a1f6157aee3d7c9a4c5cf (plain) (blame) 
1
2
3
4
5
6
7
8
9
10
11
12

discard """
  cmd: "nim $target --hints:on -d:embedUnidecodeTable $options $file"
  output: "Ausserst"
"""

import unidecode

loadUnidecodeTable("lib/pure/unidecode/unidecode.dat")

#assert unidecode("\x53\x17\x4E\xB0") == "Bei Jing"
echo unidecode("Äußerst")
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-06-29 01:18:16 +0000
'n24' href='#n24'>24 25 26 27 
28

29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

51

52

53
54
55
56
57

                   


                                            
                              
                                                           














                                                               

 



                                                       
                              





















                                                               
                     
                           




                                            
void test_parse() {
  compile("recipe main [\n"
          "  1:integer <- copy 23:literal\n"
          "]\n");
  cout << '\n'; DUMP("parse");
  CHECK(Recipe_number.find("main") != Recipe_number.end());
  recipe r = Recipe[Recipe_number["main"]];
  vector<instruction>::iterator i = r.step.begin();
  CHECK_EQ(i->is_label, false);
  CHECK_EQ(i->label, "");
  CHECK_EQ(i->operation, Recipe_number["copy"]);
  CHECK_EQ(i->ingredients.size(), 1);
  CHECK_EQ(i->ingredients[0].name, string("23"));
  CHECK_EQ(i->ingredients[0].types.size(), 1);
  CHECK_EQ(i->ingredients[0].types[0], Type_number["literal"]);
  CHECK_EQ(i->ingredients[0].properties.size(), 0);
  CHECK_EQ(i->products.size(), 1);
  CHECK_EQ(i->products[0].name, string("1"));
  CHECK_EQ(i->products[0].types.size(), 1);
  CHECK_EQ(i->products[0].types[0], Type_number["integer"]);
  CHECK_EQ(i->products[0].properties.size(), 0);
}

void test_parse2() {
  compile("recipe main [\n"
          "  1:integer, 2:integer <- copy 23:literal\n"
          "]\n");
  cout << '\n'; DUMP("parse");
  CHECK(Recipe_number.find("main") != Recipe_number.end());
  recipe r = Recipe[Recipe_number["main"]];
  vector<instruction>::iterator i = r.step.begin();
  CHECK_EQ(i->is_label, false);
  CHECK_EQ(i->label, "");
  CHECK_EQ(i->operation, Recipe_number["copy"]);
  CHECK_EQ(i->ingredients.size(), 1);
  CHECK_EQ(i->ingredients[0].name, string("23"));
  CHECK_EQ(i->ingredients[0].types.size(), 1);
  CHECK_EQ(i->ingredients[0].types[0], Type_number["literal"]);
  CHECK_EQ(i->ingredients[0].properties.size(), 0);
  CHECK_EQ(i->products.size(), 2);
  CHECK_EQ(i->products[0].name, string("1"));
  CHECK_EQ(i->products[0].types.size(), 1);
  CHECK_EQ(i->products[0].types[0], Type_number["integer"]);
  CHECK_EQ(i->products[0].properties.size(), 0);
  CHECK_EQ(i->products[1].name, string("2"));
  CHECK_EQ(i->products[1].types.size(), 1);
  CHECK_EQ(i->products[1].types[0], Type_number["integer"]);
  CHECK_EQ(i->products[1].properties.size(), 0);
}

void test_literal() {
  compile("recipe main [\n"
          "  1:integer <- copy 23:literal\n"
          "]\n");
  run("main");
  CHECK_EQ(Memory[1], 23);
}
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-06-29 01:18:09 +0000
s="k">else: s.add(' ') encodeVInt(x, s) inc i s.add('}') proc genMergeInfo*(m: BModule): PRope = if optSymbolFiles notin gGlobalOptions: return nil var s = "/*\tNIM_merge_INFO:" s.add(tnl) s.add("typeCache:{") writeTypeCache(m.typeCache, s) s.add("declared:{") writeIntSet(m.declaredThings, s) s.add("typeInfo:{") writeIntSet(m.typeInfoMarker, s) s.add("labels:") encodeVInt(m.labels, s) s.add(" hasframe:") encodeVInt(ord(m.FrameDeclared), s) s.add(tnl) s.add("*/") result = s.toRope template `^`(pos: expr): expr = L.buf[pos] proc skipWhite(L: var TBaseLexer) = var pos = L.bufpos while true: case ^pos of CR: pos = nimlexbase.HandleCR(L, pos) of LF: pos = nimlexbase.HandleLF(L, pos) of ' ': inc pos else: break L.bufpos = pos proc skipUntilCmd(L: var TBaseLexer) = var pos = L.bufpos while true: case ^pos of CR: pos = nimlexbase.HandleCR(L, pos) of LF: pos = nimlexbase.HandleLF(L, pos) of '\0': break of '/': if ^(pos+1) == '*' and ^(pos+2) == '\t': inc pos, 3 break inc pos else: inc pos L.bufpos = pos proc atEndMark(buf: cstring, pos: int): bool = var s = 0 while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s result = s == NimMergeEndMark.len when false: proc readVerbatimSection(L: var TBaseLexer): PRope = var pos = L.bufpos var buf = L.buf result = newMutableRope(30_000) while true: case buf[pos] of CR: pos = nimlexbase.HandleCR(L, pos) buf = L.buf result.data.add(tnl) of LF: pos = nimlexbase.HandleLF(L, pos) buf = L.buf result.data.add(tnl) of '\0': InternalError("ccgmerge: expected: " & NimMergeEndMark) break else: if atEndMark(buf, pos): inc pos, NimMergeEndMark.len break result.data.add(buf[pos]) inc pos L.bufpos = pos freezeMutableRope(result) proc readVerbatimSection(L: var TBaseLexer): PRope = var pos = L.bufpos var buf = L.buf var r = newStringOfCap(30_000) while true: case buf[pos] of CR: pos = nimlexbase.HandleCR(L, pos) buf = L.buf r.add(tnl) of LF: pos = nimlexbase.HandleLF(L, pos) buf = L.buf r.add(tnl) of '\0': InternalError("ccgmerge: expected: " & NimMergeEndMark) break else: if atEndMark(buf, pos): inc pos, NimMergeEndMark.len break r.add(buf[pos]) inc pos L.bufpos = pos result = r.toRope proc readKey(L: var TBaseLexer, result: var string) = var pos = L.bufpos var buf = L.buf setLen(result, 0) while buf[pos] in IdentChars: result.add(buf[pos]) inc pos if buf[pos] != ':': internalError("ccgmerge: ':' expected") L.bufpos = pos + 1 # skip ':' proc NewFakeType(id: int): PType = new(result) result.id = id proc readTypeCache(L: var TBaseLexer, result: var TIdTable) = if ^L.bufpos != '{': internalError("ccgmerge: '{' expected") inc L.bufpos while ^L.bufpos != '}': skipWhite(L) var key = decodeVInt(L.buf, L.bufpos) if ^L.bufpos != ':': internalError("ccgmerge: ':' expected") inc L.bufpos var value = decodeStr(L.buf, L.bufpos) # XXX little hack: we create a "fake" type object with the correct Id # better would be to adapt the data structure to not even store the # object as key, but only the Id IdTablePut(result, newFakeType(key), value.toRope) inc L.bufpos proc readIntSet(L: var TBaseLexer, result: var TIntSet) = if ^L.bufpos != '{': internalError("ccgmerge: '{' expected") inc L.bufpos while ^L.bufpos != '}': skipWhite(L) var key = decodeVInt(L.buf, L.bufpos) result.incl(key) inc L.bufpos proc processMergeInfo(L: var TBaseLexer, m: BModule) = var k = newStringOfCap("typeCache".len) while true: skipWhite(L) if ^L.bufpos == '*' and ^(L.bufpos+1) == '/': inc(L.bufpos, 2) break readKey(L, k) case k of "typeCache": readTypeCache(L, m.typeCache) of "declared": readIntSet(L, m.declaredThings) of "typeInfo": readIntSet(L, m.typeInfoMarker) of "labels": m.labels = decodeVInt(L.buf, L.bufpos) of "hasframe": m.FrameDeclared = decodeVInt(L.buf, L.bufpos) != 0 else: InternalError("ccgmerge: unkown key: " & k) when not defined(nimhygiene): {.pragma: inject.} template withCFile(cfilename: string, body: stmt) {.immediate.} = var s = LLStreamOpen(cfilename, fmRead) if s == nil: return var L {.inject.}: TBaseLexer openBaseLexer(L, s) var k {.inject.} = newStringOfCap("NIM_merge_FORWARD_TYPES".len) while true: skipUntilCmd(L) if ^L.bufpos == '\0': break body closeBaseLexer(L) proc readMergeInfo*(cfilename: string, m: BModule) = ## reads the merge meta information into `m`. withCFile(cfilename): readKey(L, k) if k == "NIM_merge_INFO": processMergeInfo(L, m) break type TMergeSections = object {.pure.} f: TCFileSections p: TCProcSections proc readMergeSections(cfilename: string, m: var TMergeSections) = ## reads the merge sections into `m`. withCFile(cfilename): readKey(L, k) if k == "NIM_merge_INFO": nil elif ^L.bufpos == '*' and ^(L.bufpos+1) == '/': inc(L.bufpos, 2) # read back into section skipWhite(L) var verbatim = readVerbatimSection(L) skipWhite(L) var sectionA = CFileSectionNames.find(k) if sectionA > 0 and sectionA <= high(TCFileSection).int: m.f[TCFileSection(sectionA)] = verbatim else: var sectionB = CProcSectionNames.find(k) if sectionB >= 0 and sectionB <= high(TCProcSection).int: m.p[TCProcSection(sectionB)] = verbatim else: InternalError("ccgmerge: unknown section: " & k) else: InternalError("ccgmerge: '*/' expected") proc mergeRequired*(m: BModule): bool = for i in cfsHeaders..cfsProcs: if m.s[i] != nil: #echo "not empty: ", i, " ", ropeToStr(m.s[i]) return true for i in low(TCProcSection)..high(TCProcSection): if m.initProc.s(i) != nil: #echo "not empty: ", i, " ", ropeToStr(m.initProc.s[i]) return true proc mergeFiles*(cfilename: string, m: BModule) = ## merges the C file with the old version on hard disc. var old: TMergeSections readMergeSections(cfilename, old) # do the merge; old section before new section: for i in low(TCFileSection)..high(TCFileSection): m.s[i] = con(old.f[i], m.s[i]) for i in low(TCProcSection)..high(TCProcSection): m.initProc.s(i) = con(old.p[i], m.initProc.s(i))
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-06-29 01:18:09 +0000