import std/streams import std/strutils import utils/charcategory import utils/proptable var DoubleWidthRanges: seq[(uint32, uint32)] var DoubleWidthAmbiguousRanges: seq[(uint32, uint32)] #var HalfWidthRanges: seq[(uint32, uint32)] # Combining chars from https://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c # # The following two functions define the column width of an ISO 10646 # character as follows: # # - The null character (U+0000) has a column width of 0. # # - Other C0/C1 control characters and DEL will lead to a return value of 0 # # - Non-spacing and enclosing combining characters (general category code Mn # or Me in the Unicode database) have a column width of 0. # # - SOFT HYPHEN (U+00AD) has a column width of 1. # # - Other format characters (general category code Cf in the Unicode # database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. # # - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) have a # column width of 0. # # - Spacing characters in the East Asian Wide (W) or East Asian Full-width # (F) category as defined in Unicode Technical Report #11 have a column # width of 2. # # - All remaining characters (including all printable ISO 8859-1 and WGL4 # characters, Unicode control characters, etc.) have a column width of 1. # # sorted list of non-overlapping intervals of non-spacing characters generated # by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" const Combining = [ (0x0300u32, 0x036Fu32), (0x0483u32, 0x0486u32), (0x0488u32, 0x0489u32), (0x0591u32, 0x05BDu32), (0x05BFu32, 0x05BFu32), (0x05C1u32, 0x05C2u32), (0x05C4u32, 0x05C5u32), (0x05C7u32, 0x05C7u32), (0x0600u32, 0x0603u32), (0x0610u32, 0x0615u32), (0x064Bu32, 0x065Eu32), (0x0670u32, 0x0670u32), (0x06D6u32, 0x06E4u32), (0x06E7u32, 0x06E8u32), (0x06EAu32, 0x06EDu32), (0x070Fu32, 0x070Fu32), (0x0711u32, 0x0711u32), (0x0730u32, 0x074Au32), (0x07A6u32, 0x07B0u32), (0x07EBu32, 0x07F3u32), (0x0901u32, 0x0902u32), (0x093Cu32, 0x093Cu32), (0x0941u32, 0x0948u32), (0x094Du32, 0x094Du32), (0x0951u32, 0x0954u32), (0x0962u32, 0x0963u32), (0x0981u32, 0x0981u32), (0x09BCu32, 0x09BCu32), (0x09C1u32, 0x09C4u32), (0x09CDu32, 0x09CDu32), (0x09E2u32, 0x09E3u32), (0x0A01u32, 0x0A02u32), (0x0A3Cu32, 0x0A3Cu32), (0x0A41u32, 0x0A42u32), (0x0A47u32, 0x0A48u32), (0x0A4Bu32, 0x0A4Du32), (0x0A70u32, 0x0A71u32), (0x0A81u32, 0x0A82u32), (0x0ABCu32, 0x0ABCu32), (0x0AC1u32, 0x0AC5u32), (0x0AC7u32, 0x0AC8u32), (0x0ACDu32, 0x0ACDu32), (0x0AE2u32, 0x0AE3u32), (0x0B01u32, 0x0B01u32), (0x0B3Cu32, 0x0B3Cu32), (0x0B3Fu32, 0x0B3Fu32), (0x0B41u32, 0x0B43u32), (0x0B4Du32, 0x0B4Du32), (0x0B56u32, 0x0B56u32), (0x0B82u32, 0x0B82u32), (0x0BC0u32, 0x0BC0u32), (0x0BCDu32, 0x0BCDu32), (0x0C3Eu32, 0x0C40u32), (0x0C46u32, 0x0C48u32), (0x0C4Au32, 0x0C4Du32), (0x0C55u32, 0x0C56u32), (0x0CBCu32, 0x0CBCu32), (0x0CBFu32, 0x0CBFu32), (0x0CC6u32, 0x0CC6u32), (0x0CCCu32, 0x0CCDu32), (0x0CE2u32, 0x0CE3u32), (0x0D41u32, 0x0D43u32), (0x0D4Du32, 0x0D4Du32), (0x0DCAu32, 0x0DCAu32), (0x0DD2u32, 0x0DD4u32), (0x0DD6u32, 0x0DD6u32), (0x0E31u32, 0x0E31u32), (0x0E34u32, 0x0E3Au32), (0x0E47u32, 0x0E4Eu32), (0x0EB1u32, 0x0EB1u32), (0x0EB4u32, 0x0EB9u32), (0x0EBBu32, 0x0EBCu32), (0x0EC8u32, 0x0ECDu32), (0x0F18u32, 0x0F19u32), (0x0F35u32, 0x0F35u32), (0x0F37u32, 0x0F37u32), (0x0F39u32, 0x0F39u32), (0x0F71u32, 0x0F7Eu32), (0x0F80u32, 0x0F84u32), (0x0F86u32, 0x0F87u32), (0x0F90u32, 0x0F97u32), (0x0F99u32, 0x0FBCu32), (0x0FC6u32, 0x0FC6u32), (0x102Du32, 0x1030u32), (0x1032u32, 0x1032u32), (0x1036u32, 0x1037u32), (0x1039u32, 0x1039u32), (0x1058u32, 0x1059u32), (0x1160u32, 0x11FFu32), (0x135Fu32, 0x135Fu32), (0x1712u32, 0x1714u32), (0x1732u32, 0x1734u32), (0x1752u32, 0x1753u32), (0x1772u32, 0x1773u32), (0x17B4u32, 0x17B5u32), (0x17B7u32, 0x17BDu32), (0x17C6u32, 0x17C6u32), (0x17C9u32, 0x17D3u32), (0x17DDu32, 0x17DDu32), (0x180Bu32, 0x180Du32), (0x18A9u32, 0x18A9u32), (0x1920u32, 0x1922u32), (0x1927u32, 0x1928u32), (0x1932u32, 0x1932u32), (0x1939u32, 0x193Bu32), (0x1A17u32, 0x1A18u32), (0x1B00u32, 0x1B03u32), (0x1B34u32, 0x1B34u32), (0x1B36u32, 0x1B3Au32), (0x1B3Cu32, 0x1B3Cu32), (0x1B42u32, 0x1B42u32), (0x1B6Bu32, 0x1B73u32), (0x1DC0u32, 0x1DCAu32), (0x1DFEu32, 0x1DFFu32), (0x200Bu32, 0x200Fu32), (0x202Au32, 0x202Eu32), (0x2060u32, 0x2063u32), (0x206Au32, 0x206Fu32), (0x20D0u32, 0x20EFu32), (0x302Au32, 0x302Fu32), (0x3099u32, 0x309Au32), (0xA806u32, 0xA806u32), (0xA80Bu32, 0xA80Bu32), (0xA825u32, 0xA826u32), (0xFB1Eu32, 0xFB1Eu32), (0xFE00u32, 0xFE0Fu32), (0xFE20u32, 0xFE23u32), (0xFEFFu32, 0xFEFFu32), (0xFFF9u32, 0xFFFBu32), (0x10A01u32, 0x10A03u32), (0x10A05u32, 0x10A06u32), (0x10A0Cu32, 0x10A0Fu32), (0x10A38u32, 0x10A3Au32), (0x10A3Fu32, 0x10A3Fu32), (0x1D167u32, 0x1D169u32), (0x1D173u32, 0x1D182u32), (0x1D185u32, 0x1D18Bu32), (0x1D1AAu32, 0x1D1ADu32), (0x1D242u32, 0x1D244u32), (0xE0001u32, 0xE0001u32), (0xE0020u32, 0xE007Fu32), (0xE0100u32, 0xE01EFu32) ] proc loadRanges() = template add(firstcol: string, res: var seq[(uint32, uint32)]) = if firstcol.contains(".."): let fcs = firstcol.split("..") let rstart = uint32(parseHexInt(fcs[0])) let rend = uint32(parseHexInt(fcs[1])) res.add((rstart, rend)) else: let cp = uint32(parseHexInt(firstcol)) res.add((cp, cp)) var f: File if not open(f, "res/map/EastAsianWidth.txt"): stderr.write("res/map/EastAsianWidth.txt not found\n") quit(1) let s = f.readAll() f.close() for line in s.split('\n'): if line.len == 0 or line[0] == '#': continue var i = 0 var firstcol = "" var status = "" while i < line.len and line[i] notin {'#', ';'}: if line[i] != ' ': firstcol &= line[i] inc i if line[i] != '#': inc i while i < line.len and line[i] notin {'#', ';'}: if line[i] != ' ': status &= line[i] inc i case status of "W", "F": add(firstcol, DoubleWidthRanges) of "A": add(firstcol, DoubleWidthAmbiguousRanges) #of "H": add(firstcol, HalfWidthRanges) type LineWriter = object s: Stream line: string proc write(writer: var LineWriter, s: string) = if s.len + writer.line.len > 80: writer.s.writeLine(writer.line) writer.line = "" writer.line &= s proc flush(writer: var LineWriter) = writer.s.writeLine(writer.line) writer.line = "" proc main() = loadRanges() var DoubleWidthTable = makePropertyTable(DoubleWidthRanges, Combining) # Control chars return a width of 2, and are displayed as ^{letter}. for c in Controls: let u = ptint(c) let i = u div (sizeof(ptint) * 8) let m = u mod (sizeof(ptint) * 8) DoubleWidthTable[i] = DoubleWidthTable[i] or ptint(1 shl m) let CombiningTable = makePropertyTable(Combining) var dwrLen = 0 for (ucs, mapped) in DoubleWidthRanges: if ucs > uint16.high: # lower ranges are added to DoubleWidthTable inc dwrLen echo "const DoubleWidthRanges: array[" & $dwrLen & ", tuple[ucs, mapped: uint32]] = [" var writer = LineWriter(s: newFileStream(stdout)) for (ucs, mapped) in DoubleWidthRanges: if ucs > uint16.high: # lower ranges are added to DoubleWidthTable writer.write("(" & $ucs & "," & $mapped & "),") writer.flush() echo "]" echo "" echo "const DoubleWidthAmbiguousRanges: array[" & $DoubleWidthAmbiguousRanges.len & ", tuple[ucs, mapped: uint32]] = [" for (ucs, mapped) in DoubleWidthAmbiguousRanges: writer.write("(" & $ucs & "," & $mapped & "),") writer.flush() echo "]" echo "" var cmbLen = 0 for (ucs, mapped) in Combining: if ucs > uint16.high: # lower ranges are added to CombiningTable inc cmbLen echo "const Combining: array[" & $cmbLen & ", tuple[ucs, mapped: uint32]] = [" for (ucs, mapped) in Combining: if ucs > uint16.high: writer.write("(" & $ucs & "," & $mapped & "),") writer.flush() echo "]" echo "" echo "const DoubleWidthTable: PropertyTable = [" for u in DoubleWidthTable: writer.write($u & ",") writer.flush() echo "]" echo "" echo "const CombiningTable: PropertyTable = [" for u in CombiningTable: writer.write($u & ",") writer.flush() echo "]" echo "" main()