From d253a3182859c7c989449122a60d5f362f19ded0 Mon Sep 17 00:00:00 2001 From: "Kartik K. Agaram" Date: Tue, 9 Nov 2021 08:12:11 -0800 Subject: rename grapheme to code-point-utf8 Longer name, but it doesn't lie. We have no data structure right now for combining multiple code points. And it makes no sense for the notion of a grapheme to conflate its Unicode encoding. --- shell/infix.mu | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'shell/infix.mu') diff --git a/shell/infix.mu b/shell/infix.mu index 41e8fa5d..2ca1b875 100644 --- a/shell/infix.mu +++ b/shell/infix.mu @@ -312,8 +312,8 @@ fn tokenize-infix _sym-ah: (addr handle cell), trace: (addr trace) { var buffer/edi: (addr gap-buffer) <- address buffer-storage initialize-gap-buffer buffer, 0x40/max-symbol-size # scan for first non-$ - var g/eax: grapheme <- read-grapheme sym-data - add-grapheme-at-gap buffer, g + var g/eax: code-point-utf8 <- read-code-point-utf8 sym-data + add-code-point-utf8-at-gap buffer, g { compare g, 0x24/dollar break-if-!= @@ -323,28 +323,28 @@ fn tokenize-infix _sym-ah: (addr handle cell), trace: (addr trace) { break-if-= return # symbol is all '$'s; do nothing } - g <- read-grapheme sym-data - add-grapheme-at-gap buffer, g + g <- read-code-point-utf8 sym-data + add-code-point-utf8-at-gap buffer, g loop } var tokenization-needed?: boolean - var _operator-so-far?/eax: boolean <- operator-grapheme? g + var _operator-so-far?/eax: boolean <- operator-code-point-utf8? g var operator-so-far?/ecx: boolean <- copy _operator-so-far? { var done?/eax: boolean <- stream-empty? sym-data compare done?, 0/false break-if-!= - var g/eax: grapheme <- read-grapheme sym-data + var g/eax: code-point-utf8 <- read-code-point-utf8 sym-data { - var curr-operator?/eax: boolean <- operator-grapheme? g + var curr-operator?/eax: boolean <- operator-code-point-utf8? g compare curr-operator?, operator-so-far? break-if-= # state change; insert a space - add-grapheme-at-gap buffer, 0x20/space + add-code-point-utf8-at-gap buffer, 0x20/space operator-so-far? <- copy curr-operator? copy-to tokenization-needed?, 1/true } - add-grapheme-at-gap buffer, g + add-code-point-utf8-at-gap buffer, g loop } compare tokenization-needed?, 0/false @@ -406,7 +406,7 @@ fn test-infix { # helpers -# return true if x is composed entirely of operator graphemes, optionally prefixed with some '$'s +# return true if x is composed entirely of operator code-point-utf8s, optionally prefixed with some '$'s # some operator, some non-operator => pre-tokenized symbol; return false # all '$'s => return false fn operator-symbol? _x: (addr cell) -> _/eax: boolean { @@ -421,7 +421,7 @@ fn operator-symbol? _x: (addr cell) -> _/eax: boolean { var _x-data/eax: (addr stream byte) <- lookup *x-data-ah var x-data/esi: (addr stream byte) <- copy _x-data rewind-stream x-data - var g/eax: grapheme <- read-grapheme x-data + var g/eax: code-point-utf8 <- read-code-point-utf8 x-data # special case: '$' is reserved for gensyms, and can work with either # operator or non-operator symbols. { @@ -434,12 +434,12 @@ fn operator-symbol? _x: (addr cell) -> _/eax: boolean { # '$', '$$', '$$$', etc. are regular symbols return 0/false } - g <- read-grapheme x-data + g <- read-code-point-utf8 x-data loop } { { - var result/eax: boolean <- operator-grapheme? g + var result/eax: boolean <- operator-code-point-utf8? g compare result, 0/false break-if-!= return 0/false @@ -449,13 +449,13 @@ fn operator-symbol? _x: (addr cell) -> _/eax: boolean { compare done?, 0/false } break-if-!= - g <- read-grapheme x-data + g <- read-code-point-utf8 x-data loop } return 1/true } -fn operator-grapheme? g: grapheme -> _/eax: boolean { +fn operator-code-point-utf8? g: code-point-utf8 -> _/eax: boolean { # '$' is special and can be in either a symbol or operator; here we treat it as a symbol compare g, 0x25/percent { -- cgit 1.4.1-2-gfad0 profani-tty/blame/configure.ac?id=f6e0a219ffb49b77a1d022ba5b33ca5c3a763c0b'>^ 
2655d9e8 ^


cf6da8a4 ^

3134a7b6 ^

2655d9e8 ^

2519e450 ^

2655d9e8 ^


2d353751 ^



2519e450 ^

f6ca1ba1 ^

b5171cb0 ^


5235b1f1 ^

b5171cb0 ^




14edbe17 ^

c51cc954 ^











b5171cb0 ^

dcccfacb ^

82ddbf33 ^


f6ca1ba1 ^

2519e450 ^

d2035496 ^


e504c030 ^


2490c3ed ^


d2035496 ^

dcccfacb ^

3ceb9b0d ^


dcccfacb ^

2b88e2f1 ^






2519e450 ^


dcccfacb ^

2519e450 ^

dcccfacb ^

2519e450 ^

bc8a871a ^

1588106e ^

5b734156 ^

2519e450 ^

2c361644 ^


2519e450 ^


768df4d7 ^




f983438e ^

768df4d7 ^

f983438e ^

768df4d7 ^







2c976d35 ^

7fa9ecc7 ^


dcccfacb ^

0cdbfecb ^





e361ea69 ^










524b1f23 ^

2c361644 ^

b5171cb0 ^


f0ffceef ^

b5171cb0 ^












f0ffceef ^

b5171cb0 ^






dcccfacb ^


d2035496 ^


78841f2e ^

dcccfacb ^


d2035496 ^



dcccfacb ^

78841f2e ^

bd1c1394 ^

78841f2e ^

2490c3ed ^

54df2101 ^

2490c3ed ^

f8187382 ^





24ebb512 ^

54df2101 ^



2490c3ed ^


fa89e2aa ^

7e4b1b1d ^

fa89e2aa ^

1dba341d ^

50afe736 ^

fa89e2aa ^












1dba341d ^



d2035496 ^


3ceb9b0d ^









2519e450 ^


dcccfacb ^


28f5cdf7 ^

252c7c2e ^


c5a04de9 ^




2c5b4442 ^

2519e450 ^

d28930ea ^

8e0d8d2a ^

11066f20 ^

2519e450 ^

4cb302ab ^

dcccfacb ^


575b6acd ^

aa6e2284 ^

2c361644 ^

3c062438 ^




2519e450 ^

8014e902 ^

2655d9e8 ^



dcccfacb ^


b5171cb0 ^

dcccfacb ^




3cae0208 ^

3ceb9b0d ^


dcccfacb ^


1
2
3

4

5

6
7

8

9

10

11

12
13

14
15
16

17

18

19
20

21

22
23
24
25

26

27
28
29
30
31
32
33
34
35
36
37

38

39

40
41

42

43

44
45

46
47

48
49

50

51

52
53

54

55
56
57
58
59
60

61
62

63

64

65

66

67

68

69

70

71
72

73
74

75
76
77
78

79

80

81

82
83
84
85
86
87
88

89

90
91

92

93
94
95
96
97

98
99
100
101
102
103
104
105
106
107

108

109

110
111

112

113
114
115
116
117
118
119
120
121
122
123
124

125

126
127
128
129
130
131

132
133

134
135

136

137
138

139
140
141

142

143

144

145

146

147

148

149
150
151
152
153

154

155
156
157

158
159

160

161

162

163

164

165
166
167
168
169
170
171
172
173
174
175
176

177
178
179

180
181

182
183
184
185
186
187
188
189
190

191
192

193
194

195

196
197

198
199
200
201

202

203

204

205

206

207

208

209
210

211

212

213

214
215
216
217

218

219

220
221
222

223
224

225

226
227
228
229

230

231
232

233
234