diff options
| author | bptato <nincsnevem662@gmail.com> | 2025-01-18 23:14:56 +0100 |
|---|---|---|
| committer | bptato <nincsnevem662@gmail.com> | 2025-01-18 23:14:56 +0100 |
| commit | ddd639280d93da1ae51af18fe22fdd315fc2d42d (patch) | |
| tree | 2391813d5c00c0c62634fd50a31fd3bf8405cfc7 /lib | |
| parent | cb1ec272443328c80d9e4bfb17f4853ff2745b16 (diff) | |
| parent | 93f31143b25349ed071c6f564ecce45a5c2b351c (diff) | |
| download | chawan-ddd639280d93da1ae51af18fe22fdd315fc2d42d.tar.gz | |
Add 'lib/chame0/' from commit '93f31143b25349ed071c6f564ecce45a5c2b351c'
git-subtree-dir: lib/chame0
git-subtree-mainline: cb1ec272443328c80d9e4bfb17f4853ff2745b16
git-subtree-split: 93f31143b25349ed071c6f564ecce45a5c2b351c
Diffstat (limited to 'lib')
38 files changed, 10236 insertions, 0 deletions
diff --git a/lib/chame0/.gitignore b/lib/chame0/.gitignore new file mode 100644 index 00000000..6ae8e507 --- /dev/null +++ b/lib/chame0/.gitignore @@ -0,0 +1,3 @@ +/test/* +!/test/*.* +/.obj/ diff --git a/lib/chame0/.gitmodules b/lib/chame0/.gitmodules new file mode 100644 index 00000000..847256c9 --- /dev/null +++ b/lib/chame0/.gitmodules @@ -0,0 +1,6 @@ +[submodule "tests/html5lib-tests"] + path = test/html5lib-tests + url = https://github.com/html5lib/html5lib-tests.git +[submodule "test/chagashi"] + path = test/chagashi + url = https://git.sr.ht/~bptato/chagashi diff --git a/lib/chame0/Makefile b/lib/chame0/Makefile new file mode 100644 index 00000000..2b28e9ff --- /dev/null +++ b/lib/chame0/Makefile @@ -0,0 +1,17 @@ +NIM ?= nim +FLAGS += -p:. -p:test/chagashi --import:test/eprint + +.PHONY: all +all: test + +.PHONY: test +test: + $(NIM) r $(FLAGS) test/test1.nim + $(NIM) r $(FLAGS) test/tree.nim + $(NIM) r $(FLAGS) test/tokenizer.nim + $(NIM) r $(FLAGS) test/tree_charset.nim + $(NIM) r $(FLAGS) test/tree_misc.nim + +.PHONY: submodule +submodule: + git submodule update --init diff --git a/lib/chame0/NEWS b/lib/chame0/NEWS new file mode 100644 index 00000000..fbee6687 --- /dev/null +++ b/lib/chame0/NEWS @@ -0,0 +1,107 @@ +1.0.3 (2025.01.03) +* Conform to strict defs + +1.0.2 (2024.11.22) +* Minor optimizations +* Minor documentation updates + +1.0.1 (2024.07.28) +* Minor optimization +* Add test directory to skipDirs + +1.0.0 (2024.06.13) +* Drop support for parse error handling +* Switch from legacy character decoder library to Chagashi +* Return PRES_SCRIPT for the SVG script element too +First stable version. + +0.14.5 (2024.04.09) +* Fix broken parsing of HTML entities starting with a lower `z' +* Fix nimble test execution on old Nim versions +* Various refactorings: reduce duplicated code, get rid of some dead code, + move some atom mappings to compile time + +0.14.4 (2024.03.03) +* Clean up tokenizeEOF so that it's compiled correctly on old Nim + ARM + +0.14.3 (2024.02.21) +* Fix another instance of the same bug + +0.14.2 (2024.02.21) +* Fix a character reference parsing bug: e.g. in `&g<a>', the < + character was being flushed as text instead of being interpreted as + markup. + +0.14.1 (2024.02.08) +* Fix associateWithFormImpl callback regression (it was not being called for + input elements) +* Misc refactorings + +0.14.0 (2024.02.07) + +* The "bag of pointers" interface design has been dropped +* Tag and attribute names are now treated as interned strings (a user-defined + "Atom" type) +* Support processing of embedded SVG/MathML elements +* Chakasu has been made an optional dependency +* std/streams no longer used by htmlparser; now it supports chunked parsing + instead +* All tokenizer + tree builder tests passed in html5lib-tests + +Rough migration guide from the previous API: + +Users of minidom + +* nodeType is no longer supported, use the of operator to distinguish + between node types. +* minidom now only supports UTF-8; if you need support for other + charsets, use minidom_cs. +* localName is now an MAtom; to get the stringified local name, use + localNameStr. +* attrs is now a seq of Attribute tuples. Linear search this seq to find + specific attributes. +* minidom now contains several MAtom fields; to convert these to + strings, call document.atomToStr(atom). + +Users of htmlparser + +* The NodeType enum has been removed. Either copy-paste the enum + definition from a previous version, or (more efficient) use the `of` + operator to distinguish between types. +* Use of an AtomFactory is now required for consumers of htmlparser. The + easiest fix is to copy-paste the implementation found in minidom. +* Your DOM builder should be generic over a Handle and an Atom. Example: + `DOMBuilder[Node, MAtom]` +* You no longer have to copy function pointers into your DOM builder. +* It is recommended to add `include chame/htmlparseriface` to your DOM + builder module. See the htmlparseriface documentation for details. + +Switching to the new interface: + +* Add `Impl` to the name of all your procedure implementations. +* If you included chame/htmlparseriface, replace all parameters of your + procedures containing `DOMBuilder[MyHandle]` with `MyDOMBuilder`. +* setCharacterSet -> setEncodingImpl that takes a string label. +* getLocalNameImpl now must return an Atom. getTagType is no longer used. +* insertBeforeImpl must take an `Option[Handle]` +* addAttrsIfMissingImpl is now mandatory, and must take a `Table[Atom, string]` +* getNamespaceImpl is now mandatory. +* getDocumentImpl must be implemented, and must return the Handle of the + document. +* tagTypeToAtomImpl, atomToTagTypeImpl, strToAtomImpl must all be implemented. +* createHTMLElementImpl must be implemented, and must return the handle of a new + `<html>` element. +* createElement -> createElementForTokenImpl, the signature has changed + significantly. localName is the 2-in-1 replacement for both tagType and + localName. Also, you probably have to convert attributes from htmlAttrs & + xmlAttrs to your own internal representation. +* finish is no longer called at the end of parsing. Call it yourself. + +Event loop changes: + +* parseHTML is now split into three parts: initHTML5Parser, parseChunk, finish. +* If you implement scripting and/or character sets other than UTF-8, see + doc/manual.md for handling parseChunk's result. Otherwise, it is safe to + discard it. +* Do not forget to call finish after having parsed the entire document (first + for the parser, then for your own DOM builder). diff --git a/lib/chame0/README.md b/lib/chame0/README.md new file mode 100644 index 00000000..32904cf7 --- /dev/null +++ b/lib/chame0/README.md @@ -0,0 +1,98 @@ +# Chame: an HTML5 parser library in Nim + +## Usage + +Include Chame in your project using either Nimble or as a git submodule. + +``` +requires "chame" +# optional, if you want support for charsets other than UTF-8 +requires "chagashi" +``` + +Then, check the [documentation](https://chawan.net/doc/chame/) for a +description of the API. + +Note: only Nim 1.6.10+ is supported. + +## Features + +* Full compliance with the WHATWG HTML parsing standard +* Passes all tokenizer and tree builder tests in html5lib-tests (except for ones + requiring JS) +* Includes a minimal DOM implementation +* No mandatory dependencies other than the Nim standard library +* Optional character encoding support (see [minidom_cs](chame/minidom_cs.nim)) +* String interning support for tag and attribute names +* Support for chunked parsing +* document.write (no actual implementation here, but it's possible to implement + it on top of Chame) + +## Manual + +There is a manual available at [doc/manual.md](doc/manual.md). + +## To-do + +At this point the library is complete. The only remaining tasks are +optimization-related. + +Also, a small module for basic minidom utilities is planned. + +## Bugs, feedback, etc. + +Feedback, complaints, etc. are accepted at +[SourceHut](https://todo.sr.ht/~bptato/chawan) or the Nim forum +[thread](https://forum.nim-lang.org/t/10963). + +## FAQ + +### Does Chame include a DOM, JavaScript, CSS, ...? + +Chame just parses HTML and calls the callbacks supplied to it. JavaScript, +DOM manipulation, etc. are technically outside the scope of this project. + +However, Chame includes a minimal DOM interface (intuitively named minidom) +for demonstration and testing purposes. This only implements the very basics +needed for the parser to function (and for the tests to pass), and does not +have any convenience functions (like querySelector, getElementById, etc.) + +Please refer to the [Chawan](https://sr.ht/~bptato/chawan/) web browser for +an example of a complete DOM implementation that depends on Chame. + +Also see [CSS3Selectors](https://github.com/Niminem/CSS3Selectors/), a CSS +selector library for Chame's minidom which allows you to run querySelector +like you can in JS. + +If you implement a DOM library based on Chame, please notify me, so that I +can redirect users to it in this section. + +### I read the manual, but it's too complex, I don't understand anything, help + +Just call minidom.parseHTML on an std/stream.Stream object and forget about +everything else. Chances are this is enough for whatever you want to do. + +### How do I implement speculative parsing? + +No idea. Let me know if you figure something out. + +### How do you pronounce Chame? + +It is an acronym of "**Cha**wan HT**M**L (aitch-tee-e**m-e**l)." Accordingly, it is +pronounced as "cha-meh." + +## Thanks + +[SerenityOS](https://serenityos.org/)'s HTML parser has been used as a +reference when I found some parts of the specification unclear. + +Servo's HTML parser [html5ever](https://github.com/servo/html5ever) has been +the main inspiration for Chame's API. + +Finally, thanks to the standard writers for writing a very detailed +specification of the HTML5 parsing algorithm. The bulk of Chame is a direct +translation of this algorithm into Nim. + +## License + +Chame is dedicated to the public domain. See the UNLICENSE file for details. diff --git a/lib/chame0/UNLICENSE b/lib/chame0/UNLICENSE new file mode 100644 index 00000000..68a49daa --- /dev/null +++ b/lib/chame0/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to <http://unlicense.org/> diff --git a/lib/chame0/chame.nimble b/lib/chame0/chame.nimble new file mode 100644 index 00000000..190507ea --- /dev/null +++ b/lib/chame0/chame.nimble @@ -0,0 +1,13 @@ +# Package + +version = "1.0.3" +author = "bptato" +description = "HTML5 parser for Chawan" +license = "Unlicense" +skipDirs = @["test"] + +# Dependencies + +requires "nim >= 1.6.10" +when declared(taskRequires): + taskRequires "test", "chagashi >= 0.5.0" diff --git a/lib/chame0/chame/dombuilder.nim b/lib/chame0/chame/dombuilder.nim new file mode 100644 index 00000000..5497b2d0 --- /dev/null +++ b/lib/chame0/chame/dombuilder.nim @@ -0,0 +1,15 @@ +import tags + +type + SetEncodingResult* = enum + SET_ENCODING_CONTINUE, SET_ENCODING_STOP + + ParsedAttr*[Atom] = tuple + prefix: NamespacePrefix + namespace: Namespace + name: Atom + value: string + + DOMBuilderBase* = ref object of RootObj + + DOMBuilder*[Handle, Atom] = ref object of DOMBuilderBase diff --git a/lib/chame0/chame/entity_gen.nim b/lib/chame0/chame/entity_gen.nim new file mode 100644 index 00000000..2e3f6e58 --- /dev/null +++ b/lib/chame0/chame/entity_gen.nim @@ -0,0 +1,767 @@ +type Z = cstring +const entityMap*: array[2231, Z] = [ +"AElig:\xC3\x86","AElig;:\xC3\x86","AMP:&","AMP;:&","Aacute:\xC3\x81", +"Aacute;:\xC3\x81","Abreve;:\xC4\x82","Acirc:\xC3\x82","Acirc;:\xC3\x82", +"Acy;:\xD0\x90","Afr;:\xF0\x9D\x94\x84","Agrave:\xC3\x80","Agrave;:\xC3\x80", +"Alpha;:\xCE\x91","Amacr;:\xC4\x80","And;:\xE2\xA9\x93","Aogon;:\xC4\x84", +"Aopf;:\xF0\x9D\x94\xB8","ApplyFunction;:\xE2\x81\xA1","Aring:\xC3\x85", +"Aring;:\xC3\x85","Ascr;:\xF0\x9D\x92\x9C","Assign;:\xE2\x89\x94", +"Atilde:\xC3\x83","Atilde;:\xC3\x83","Auml:\xC3\x84","Auml;:\xC3\x84", +"Backslash;:\xE2\x88\x96","Barv;:\xE2\xAB\xA7","Barwed;:\xE2\x8C\x86", +"Bcy;:\xD0\x91","Because;:\xE2\x88\xB5","Bernoullis;:\xE2\x84\xAC", +"Beta;:\xCE\x92","Bfr;:\xF0\x9D\x94\x85","Bopf;:\xF0\x9D\x94\xB9", +"Breve;:\xCB\x98","Bscr;:\xE2\x84\xAC","Bumpeq;:\xE2\x89\x8E","CHcy;:\xD0\xA7", +"COPY:\xC2\xA9","COPY;:\xC2\xA9","Cacute;:\xC4\x86","Cap;:\xE2\x8B\x92", +"CapitalDifferentialD;:\xE2\x85\x85","Cayleys;:\xE2\x84\xAD","Ccaron;:\xC4\x8C", +"Ccedil:\xC3\x87","Ccedil;:\xC3\x87","Ccirc;:\xC4\x88","Cconint;:\xE2\x88\xB0", +"Cdot;:\xC4\x8A","Cedilla;:\xC2\xB8","CenterDot;:\xC2\xB7","Cfr;:\xE2\x84\xAD", +"Chi;:\xCE\xA7","CircleDot;:\xE2\x8A\x99","CircleMinus;:\xE2\x8A\x96", +"CirclePlus;:\xE2\x8A\x95","CircleTimes;:\xE2\x8A\x97", +"ClockwiseContourIntegral;:\xE2\x88\xB2","CloseCurlyDoubleQuote;:\xE2\x80\x9D", +"CloseCurlyQuote;:\xE2\x80\x99","Colon;:\xE2\x88\xB7","Colone;:\xE2\xA9\xB4", +"Congruent;:\xE2\x89\xA1","Conint;:\xE2\x88\xAF", +"ContourIntegral;:\xE2\x88\xAE","Copf;:\xE2\x84\x82","Coproduct;:\xE2\x88\x90", +"CounterClockwiseContourIntegral;:\xE2\x88\xB3","Cross;:\xE2\xA8\xAF", +"Cscr;:\xF0\x9D\x92\x9E","Cup;:\xE2\x8B\x93","CupCap;:\xE2\x89\x8D", +"DD;:\xE2\x85\x85","DDotrahd;:\xE2\xA4\x91","DJcy;:\xD0\x82","DScy;:\xD0\x85", +"DZcy;:\xD0\x8F","Dagger;:\xE2\x80\xA1","Darr;:\xE2\x86\xA1", +"Dashv;:\xE2\xAB\xA4","Dcaron;:\xC4\x8E","Dcy;:\xD0\x94","Del;:\xE2\x88\x87", +"Delta;:\xCE\x94","Dfr;:\xF0\x9D\x94\x87","DiacriticalAcute;:\xC2\xB4", +"DiacriticalDot;:\xCB\x99","DiacriticalDoubleAcute;:\xCB\x9D", +"DiacriticalGrave;:`","DiacriticalTilde;:\xCB\x9C","Diamond;:\xE2\x8B\x84", +"DifferentialD;:\xE2\x85\x86","Dopf;:\xF0\x9D\x94\xBB","Dot;:\xC2\xA8", +"DotDot;:\xE2\x83\x9C","DotEqual;:\xE2\x89\x90", +"DoubleContourIntegral;:\xE2\x88\xAF","DoubleDot;:\xC2\xA8", +"DoubleDownArrow;:\xE2\x87\x93","DoubleLeftArrow;:\xE2\x87\x90", +"DoubleLeftRightArrow;:\xE2\x87\x94","DoubleLeftTee;:\xE2\xAB\xA4", +"DoubleLongLeftArrow;:\xE2\x9F\xB8","DoubleLongLeftRightArrow;:\xE2\x9F\xBA", +"DoubleLongRightArrow;:\xE2\x9F\xB9","DoubleRightArrow;:\xE2\x87\x92", +"DoubleRightTee;:\xE2\x8A\xA8","DoubleUpArrow;:\xE2\x87\x91", +"DoubleUpDownArrow;:\xE2\x87\x95","DoubleVerticalBar;:\xE2\x88\xA5", +"DownArrow;:\xE2\x86\x93","DownArrowBar;:\xE2\xA4\x93", +"DownArrowUpArrow;:\xE2\x87\xB5","DownBreve;:\xCC\x91", +"DownLeftRightVector;:\xE2\xA5\x90","DownLeftTeeVector;:\xE2\xA5\x9E", +"DownLeftVector;:\xE2\x86\xBD","DownLeftVectorBar;:\xE2\xA5\x96", +"DownRightTeeVector;:\xE2\xA5\x9F","DownRightVector;:\xE2\x87\x81", +"DownRightVectorBar;:\xE2\xA5\x97","DownTee;:\xE2\x8A\xA4", +"DownTeeArrow;:\xE2\x86\xA7","Downarrow;:\xE2\x87\x93","Dscr;:\xF0\x9D\x92\x9F", +"Dstrok;:\xC4\x90","ENG;:\xC5\x8A","ETH:\xC3\x90","ETH;:\xC3\x90", +"Eacute:\xC3\x89","Eacute;:\xC3\x89","Ecaron;:\xC4\x9A","Ecirc:\xC3\x8A", +"Ecirc;:\xC3\x8A","Ecy;:\xD0\xAD","Edot;:\xC4\x96","Efr;:\xF0\x9D\x94\x88", +"Egrave:\xC3\x88","Egrave;:\xC3\x88","Element;:\xE2\x88\x88","Emacr;:\xC4\x92", +"EmptySmallSquare;:\xE2\x97\xBB","EmptyVerySmallSquare;:\xE2\x96\xAB", +"Eogon;:\xC4\x98","Eopf;:\xF0\x9D\x94\xBC","Epsilon;:\xCE\x95", +"Equal;:\xE2\xA9\xB5","EqualTilde;:\xE2\x89\x82","Equilibrium;:\xE2\x87\x8C", +"Escr;:\xE2\x84\xB0","Esim;:\xE2\xA9\xB3","Eta;:\xCE\x97","Euml:\xC3\x8B", +"Euml;:\xC3\x8B","Exists;:\xE2\x88\x83","ExponentialE;:\xE2\x85\x87", +"Fcy;:\xD0\xA4","Ffr;:\xF0\x9D\x94\x89","FilledSmallSquare;:\xE2\x97\xBC", +"FilledVerySmallSquare;:\xE2\x96\xAA","Fopf;:\xF0\x9D\x94\xBD", +"ForAll;:\xE2\x88\x80","Fouriertrf;:\xE2\x84\xB1","Fscr;:\xE2\x84\xB1", +"GJcy;:\xD0\x83","GT:>","GT;:>","Gamma;:\xCE\x93","Gammad;:\xCF\x9C", +"Gbreve;:\xC4\x9E","Gcedil;:\xC4\xA2","Gcirc;:\xC4\x9C","Gcy;:\xD0\x93", +"Gdot;:\xC4\xA0","Gfr;:\xF0\x9D\x94\x8A","Gg;:\xE2\x8B\x99", +"Gopf;:\xF0\x9D\x94\xBE","GreaterEqual;:\xE2\x89\xA5", +"GreaterEqualLess;:\xE2\x8B\x9B","GreaterFullEqual;:\xE2\x89\xA7", +"GreaterGreater;:\xE2\xAA\xA2","GreaterLess;:\xE2\x89\xB7", +"GreaterSlantEqual;:\xE2\xA9\xBE","GreaterTilde;:\xE2\x89\xB3", +"Gscr;:\xF0\x9D\x92\xA2","Gt;:\xE2\x89\xAB","HARDcy;:\xD0\xAA", +"Hacek;:\xCB\x87","Hat;:^","Hcirc;:\xC4\xA4","Hfr;:\xE2\x84\x8C", +"HilbertSpace;:\xE2\x84\x8B","Hopf;:\xE2\x84\x8D", +"HorizontalLine;:\xE2\x94\x80","Hscr;:\xE2\x84\x8B","Hstrok;:\xC4\xA6", +"HumpDownHump;:\xE2\x89\x8E","HumpEqual;:\xE2\x89\x8F","IEcy;:\xD0\x95", +"IJlig;:\xC4\xB2","IOcy;:\xD0\x81","Iacute:\xC3\x8D","Iacute;:\xC3\x8D", +"Icirc:\xC3\x8E","Icirc;:\xC3\x8E","Icy;:\xD0\x98","Idot;:\xC4\xB0", +"Ifr;:\xE2\x84\x91","Igrave:\xC3\x8C","Igrave;:\xC3\x8C","Im;:\xE2\x84\x91", +"Imacr;:\xC4\xAA","ImaginaryI;:\xE2\x85\x88","Implies;:\xE2\x87\x92", +"Int;:\xE2\x88\xAC","Integral;:\xE2\x88\xAB","Intersection;:\xE2\x8B\x82", +"InvisibleComma;:\xE2\x81\xA3","InvisibleTimes;:\xE2\x81\xA2","Iogon;:\xC4\xAE", +"Iopf;:\xF0\x9D\x95\x80","Iota;:\xCE\x99","Iscr;:\xE2\x84\x90", +"Itilde;:\xC4\xA8","Iukcy;:\xD0\x86","Iuml:\xC3\x8F","Iuml;:\xC3\x8F", +"Jcirc;:\xC4\xB4","Jcy;:\xD0\x99","Jfr;:\xF0\x9D\x94\x8D", +"Jopf;:\xF0\x9D\x95\x81","Jscr;:\xF0\x9D\x92\xA5","Jsercy;:\xD0\x88", +"Jukcy;:\xD0\x84","KHcy;:\xD0\xA5","KJcy;:\xD0\x8C","Kappa;:\xCE\x9A", +"Kcedil;:\xC4\xB6","Kcy;:\xD0\x9A","Kfr;:\xF0\x9D\x94\x8E", +"Kopf;:\xF0\x9D\x95\x82","Kscr;:\xF0\x9D\x92\xA6","LJcy;:\xD0\x89","LT:<", +"LT;:<","Lacute;:\xC4\xB9","Lambda;:\xCE\x9B","Lang;:\xE2\x9F\xAA", +"Laplacetrf;:\xE2\x84\x92","Larr;:\xE2\x86\x9E","Lcaron;:\xC4\xBD", +"Lcedil;:\xC4\xBB","Lcy;:\xD0\x9B","LeftAngleBracket;:\xE2\x9F\xA8", +"LeftArrow;:\xE2\x86\x90","LeftArrowBar;:\xE2\x87\xA4", +"LeftArrowRightArrow;:\xE2\x87\x86","LeftCeiling;:\xE2\x8C\x88", +"LeftDoubleBracket;:\xE2\x9F\xA6","LeftDownTeeVector;:\xE2\xA5\xA1", +"LeftDownVector;:\xE2\x87\x83","LeftDownVectorBar;:\xE2\xA5\x99", +"LeftFloor;:\xE2\x8C\x8A","LeftRightArrow;:\xE2\x86\x94", +"LeftRightVector;:\xE2\xA5\x8E","LeftTee;:\xE2\x8A\xA3", +"LeftTeeArrow;:\xE2\x86\xA4","LeftTeeVector;:\xE2\xA5\x9A", +"LeftTriangle;:\xE2\x8A\xB2","LeftTriangleBar;:\xE2\xA7\x8F", +"LeftTriangleEqual;:\xE2\x8A\xB4","LeftUpDownVector;:\xE2\xA5\x91", +"LeftUpTeeVector;:\xE2\xA5\xA0","LeftUpVector;:\xE2\x86\xBF", +"LeftUpVectorBar;:\xE2\xA5\x98","LeftVector;:\xE2\x86\xBC", +"LeftVectorBar;:\xE2\xA5\x92","Leftarrow;:\xE2\x87\x90", +"Leftrightarrow;:\xE2\x87\x94","LessEqualGreater;:\xE2\x8B\x9A", +"LessFullEqual;:\xE2\x89\xA6","LessGreater;:\xE2\x89\xB6", +"LessLess;:\xE2\xAA\xA1","LessSlantEqual;:\xE2\xA9\xBD", +"LessTilde;:\xE2\x89\xB2","Lfr;:\xF0\x9D\x94\x8F","Ll;:\xE2\x8B\x98", +"Lleftarrow;:\xE2\x87\x9A","Lmidot;:\xC4\xBF","LongLeftArrow;:\xE2\x9F\xB5", +"LongLeftRightArrow;:\xE2\x9F\xB7","LongRightArrow;:\xE2\x9F\xB6", +"Longleftarrow;:\xE2\x9F\xB8","Longleftrightarrow;:\xE2\x9F\xBA", +"Longrightarrow;:\xE2\x9F\xB9","Lopf;:\xF0\x9D\x95\x83", +"LowerLeftArrow;:\xE2\x86\x99","LowerRightArrow;:\xE2\x86\x98", +"Lscr;:\xE2\x84\x92","Lsh;:\xE2\x86\xB0","Lstrok;:\xC5\x81","Lt;:\xE2\x89\xAA", +"Map;:\xE2\xA4\x85","Mcy;:\xD0\x9C","MediumSpace;:\xE2\x81\x9F", +"Mellintrf;:\xE2\x84\xB3","Mfr;:\xF0\x9D\x94\x90","MinusPlus;:\xE2\x88\x93", +"Mopf;:\xF0\x9D\x95\x84","Mscr;:\xE2\x84\xB3","Mu;:\xCE\x9C","NJcy;:\xD0\x8A", +"Nacute;:\xC5\x83","Ncaron;:\xC5\x87","Ncedil;:\xC5\x85","Ncy;:\xD0\x9D", +"NegativeMediumSpace;:\xE2\x80\x8B","NegativeThickSpace;:\xE2\x80\x8B", +"NegativeThinSpace;:\xE2\x80\x8B","NegativeVeryThinSpace;:\xE2\x80\x8B", +"NestedGreaterGreater;:\xE2\x89\xAB","NestedLessLess;:\xE2\x89\xAA", +"NewLine;:\x0A","Nfr;:\xF0\x9D\x94\x91","NoBreak;:\xE2\x81\xA0", +"NonBreakingSpace;:\xC2\xA0","Nopf;:\xE2\x84\x95","Not;:\xE2\xAB\xAC", +"NotCongruent;:\xE2\x89\xA2","NotCupCap;:\xE2\x89\xAD", +"NotDoubleVerticalBar;:\xE2\x88\xA6","NotElement;:\xE2\x88\x89", +"NotEqual;:\xE2\x89\xA0","NotEqualTilde;:\xE2\x89\x82\xCC\xB8", +"NotExists;:\xE2\x88\x84","NotGreater;:\xE2\x89\xAF", +"NotGreaterEqual;:\xE2\x89\xB1","NotGreaterFullEqual;:\xE2\x89\xA7\xCC\xB8", +"NotGreaterGreater;:\xE2\x89\xAB\xCC\xB8","NotGreaterLess;:\xE2\x89\xB9", +"NotGreaterSlantEqual;:\xE2\xA9\xBE\xCC\xB8","NotGreaterTilde;:\xE2\x89\xB5", +"NotHumpDownHump;:\xE2\x89\x8E\xCC\xB8","NotHumpEqual;:\xE2\x89\x8F\xCC\xB8", +"NotLeftTriangle;:\xE2\x8B\xAA","NotLeftTriangleBar;:\xE2\xA7\x8F\xCC\xB8", +"NotLeftTriangleEqual;:\xE2\x8B\xAC","NotLess;:\xE2\x89\xAE", +"NotLessEqual;:\xE2\x89\xB0","NotLessGreater;:\xE2\x89\xB8", +"NotLessLess;:\xE2\x89\xAA\xCC\xB8","NotLessSlantEqual;:\xE2\xA9\xBD\xCC\xB8", +"NotLessTilde;:\xE2\x89\xB4","NotNestedGreaterGreater;:\xE2\xAA\xA2\xCC\xB8", +"NotNestedLessLess;:\xE2\xAA\xA1\xCC\xB8","NotPrecedes;:\xE2\x8A\x80", +"NotPrecedesEqual;:\xE2\xAA\xAF\xCC\xB8","NotPrecedesSlantEqual;:\xE2\x8B\xA0", +"NotReverseElement;:\xE2\x88\x8C","NotRightTriangle;:\xE2\x8B\xAB", +"NotRightTriangleBar;:\xE2\xA7\x90\xCC\xB8", +"NotRightTriangleEqual;:\xE2\x8B\xAD","NotSquareSubset;:\xE2\x8A\x8F\xCC\xB8", +"NotSquareSubsetEqual;:\xE2\x8B\xA2","NotSquareSuperset;:\xE2\x8A\x90\xCC\xB8", +"NotSquareSupersetEqual;:\xE2\x8B\xA3","NotSubset;:\xE2\x8A\x82\xE2\x83\x92", +"NotSubsetEqual;:\xE2\x8A\x88","NotSucceeds;:\xE2\x8A\x81", +"NotSucceedsEqual;:\xE2\xAA\xB0\xCC\xB8","NotSucceedsSlantEqual;:\xE2\x8B\xA1", +"NotSucceedsTilde;:\xE2\x89\xBF\xCC\xB8", +"NotSuperset;:\xE2\x8A\x83\xE2\x83\x92","NotSupersetEqual;:\xE2\x8A\x89", +"NotTilde;:\xE2\x89\x81","NotTildeEqual;:\xE2\x89\x84", +"NotTildeFullEqual;:\xE2\x89\x87","NotTildeTilde;:\xE2\x89\x89", +"NotVerticalBar;:\xE2\x88\xA4","Nscr;:\xF0\x9D\x92\xA9","Ntilde:\xC3\x91", +"Ntilde;:\xC3\x91","Nu;:\xCE\x9D","OElig;:\xC5\x92","Oacute:\xC3\x93", +"Oacute;:\xC3\x93","Ocirc:\xC3\x94","Ocirc;:\xC3\x94","Ocy;:\xD0\x9E", +"Odblac;:\xC5\x90","Ofr;:\xF0\x9D\x94\x92","Ograve:\xC3\x92","Ograve;:\xC3\x92", +"Omacr;:\xC5\x8C","Omega;:\xCE\xA9","Omicron;:\xCE\x9F", +"Oopf;:\xF0\x9D\x95\x86","OpenCurlyDoubleQuote;:\xE2\x80\x9C", +"OpenCurlyQuote;:\xE2\x80\x98","Or;:\xE2\xA9\x94","Oscr;:\xF0\x9D\x92\xAA", +"Oslash:\xC3\x98","Oslash;:\xC3\x98","Otilde:\xC3\x95","Otilde;:\xC3\x95", +"Otimes;:\xE2\xA8\xB7","Ouml:\xC3\x96","Ouml;:\xC3\x96","OverBar;:\xE2\x80\xBE", +"OverBrace;:\xE2\x8F\x9E","OverBracket;:\xE2\x8E\xB4", +"OverParenthesis;:\xE2\x8F\x9C","PartialD;:\xE2\x88\x82","Pcy;:\xD0\x9F", +"Pfr;:\xF0\x9D\x94\x93","Phi;:\xCE\xA6","Pi;:\xCE\xA0","PlusMinus;:\xC2\xB1", +"Poincareplane;:\xE2\x84\x8C","Popf;:\xE2\x84\x99","Pr;:\xE2\xAA\xBB", +"Precedes;:\xE2\x89\xBA","PrecedesEqual;:\xE2\xAA\xAF", +"PrecedesSlantEqual;:\xE2\x89\xBC","PrecedesTilde;:\xE2\x89\xBE", +"Prime;:\xE2\x80\xB3","Product;:\xE2\x88\x8F","Proportion;:\xE2\x88\xB7", +"Proportional;:\xE2\x88\x9D","Pscr;:\xF0\x9D\x92\xAB","Psi;:\xCE\xA8","QUOT:\"", +"QUOT;:\"","Qfr;:\xF0\x9D\x94\x94","Qopf;:\xE2\x84\x9A", +"Qscr;:\xF0\x9D\x92\xAC","RBarr;:\xE2\xA4\x90","REG:\xC2\xAE","REG;:\xC2\xAE", +"Racute;:\xC5\x94","Rang;:\xE2\x9F\xAB","Rarr;:\xE2\x86\xA0", +"Rarrtl;:\xE2\xA4\x96","Rcaron;:\xC5\x98","Rcedil;:\xC5\x96","Rcy;:\xD0\xA0", +"Re;:\xE2\x84\x9C","ReverseElement;:\xE2\x88\x8B", +"ReverseEquilibrium;:\xE2\x87\x8B","ReverseUpEquilibrium;:\xE2\xA5\xAF", +"Rfr;:\xE2\x84\x9C","Rho;:\xCE\xA1","RightAngleBracket;:\xE2\x9F\xA9", +"RightArrow;:\xE2\x86\x92","RightArrowBar;:\xE2\x87\xA5", +"RightArrowLeftArrow;:\xE2\x87\x84","RightCeiling;:\xE2\x8C\x89", +"RightDoubleBracket;:\xE2\x9F\xA7","RightDownTeeVector;:\xE2\xA5\x9D", +"RightDownVector;:\xE2\x87\x82","RightDownVectorBar;:\xE2\xA5\x95", +"RightFloor;:\xE2\x8C\x8B","RightTee;:\xE2\x8A\xA2", +"RightTeeArrow;:\xE2\x86\xA6","RightTeeVector;:\xE2\xA5\x9B", +"RightTriangle;:\xE2\x8A\xB3","RightTriangleBar;:\xE2\xA7\x90", +"RightTriangleEqual;:\xE2\x8A\xB5","RightUpDownVector;:\xE2\xA5\x8F", +"RightUpTeeVector;:\xE2\xA5\x9C","RightUpVector;:\xE2\x86\xBE", +"RightUpVectorBar;:\xE2\xA5\x94","RightVector;:\xE2\x87\x80", +"RightVectorBar;:\xE2\xA5\x93","Rightarrow;:\xE2\x87\x92","Ropf;:\xE2\x84\x9D", +"RoundImplies;:\xE2\xA5\xB0","Rrightarrow;:\xE2\x87\x9B","Rscr;:\xE2\x84\x9B", +"Rsh;:\xE2\x86\xB1","RuleDelayed;:\xE2\xA7\xB4","SHCHcy;:\xD0\xA9", +"SHcy;:\xD0\xA8","SOFTcy;:\xD0\xAC","Sacute;:\xC5\x9A","Sc;:\xE2\xAA\xBC", +"Scaron;:\xC5\xA0","Scedil;:\xC5\x9E","Scirc;:\xC5\x9C","Scy;:\xD0\xA1", +"Sfr;:\xF0\x9D\x94\x96","ShortDownArrow;:\xE2\x86\x93", +"ShortLeftArrow;:\xE2\x86\x90","ShortRightArrow;:\xE2\x86\x92", +"ShortUpArrow;:\xE2\x86\x91","Sigma;:\xCE\xA3","SmallCircle;:\xE2\x88\x98", +"Sopf;:\xF0\x9D\x95\x8A","Sqrt;:\xE2\x88\x9A","Square;:\xE2\x96\xA1", +"SquareIntersection;:\xE2\x8A\x93","SquareSubset;:\xE2\x8A\x8F", +"SquareSubsetEqual;:\xE2\x8A\x91","SquareSuperset;:\xE2\x8A\x90", +"SquareSupersetEqual;:\xE2\x8A\x92","SquareUnion;:\xE2\x8A\x94", +"Sscr;:\xF0\x9D\x92\xAE","Star;:\xE2\x8B\x86","Sub;:\xE2\x8B\x90", +"Subset;:\xE2\x8B\x90","SubsetEqual;:\xE2\x8A\x86","Succeeds;:\xE2\x89\xBB", +"SucceedsEqual;:\xE2\xAA\xB0","SucceedsSlantEqual;:\xE2\x89\xBD", +"SucceedsTilde;:\xE2\x89\xBF","SuchThat;:\xE2\x88\x8B","Sum;:\xE2\x88\x91", +"Sup;:\xE2\x8B\x91","Superset;:\xE2\x8A\x83","SupersetEqual;:\xE2\x8A\x87", +"Supset;:\xE2\x8B\x91","THORN:\xC3\x9E","THORN;:\xC3\x9E","TRADE;:\xE2\x84\xA2", +"TSHcy;:\xD0\x8B","TScy;:\xD0\xA6","Tab;:\x09","Tau;:\xCE\xA4", +"Tcaron;:\xC5\xA4","Tcedil;:\xC5\xA2","Tcy;:\xD0\xA2","Tfr;:\xF0\x9D\x94\x97", +"Therefore;:\xE2\x88\xB4","Theta;:\xCE\x98", +"ThickSpace;:\xE2\x81\x9F\xE2\x80\x8A","ThinSpace;:\xE2\x80\x89", +"Tilde;:\xE2\x88\xBC","TildeEqual;:\xE2\x89\x83","TildeFullEqual;:\xE2\x89\x85", +"TildeTilde;:\xE2\x89\x88","Topf;:\xF0\x9D\x95\x8B","TripleDot;:\xE2\x83\x9B", +"Tscr;:\xF0\x9D\x92\xAF","Tstrok;:\xC5\xA6","Uacute:\xC3\x9A", +"Uacute;:\xC3\x9A","Uarr;:\xE2\x86\x9F","Uarrocir;:\xE2\xA5\x89", +"Ubrcy;:\xD0\x8E","Ubreve;:\xC5\xAC","Ucirc:\xC3\x9B","Ucirc;:\xC3\x9B", +"Ucy;:\xD0\xA3","Udblac;:\xC5\xB0","Ufr;:\xF0\x9D\x94\x98","Ugrave:\xC3\x99", +"Ugrave;:\xC3\x99","Umacr;:\xC5\xAA","UnderBar;:_","UnderBrace;:\xE2\x8F\x9F", +"UnderBracket;:\xE2\x8E\xB5","UnderParenthesis;:\xE2\x8F\x9D", +"Union;:\xE2\x8B\x83","UnionPlus;:\xE2\x8A\x8E","Uogon;:\xC5\xB2", +"Uopf;:\xF0\x9D\x95\x8C","UpArrow;:\xE2\x86\x91","UpArrowBar;:\xE2\xA4\x92", +"UpArrowDownArrow;:\xE2\x87\x85","UpDownArrow;:\xE2\x86\x95", +"UpEquilibrium;:\xE2\xA5\xAE","UpTee;:\xE2\x8A\xA5","UpTeeArrow;:\xE2\x86\xA5", +"Uparrow;:\xE2\x87\x91","Updownarrow;:\xE2\x87\x95", +"UpperLeftArrow;:\xE2\x86\x96","UpperRightArrow;:\xE2\x86\x97","Upsi;:\xCF\x92", +"Upsilon;:\xCE\xA5","Uring;:\xC5\xAE","Uscr;:\xF0\x9D\x92\xB0", +"Utilde;:\xC5\xA8","Uuml:\xC3\x9C","Uuml;:\xC3\x9C","VDash;:\xE2\x8A\xAB", +"Vbar;:\xE2\xAB\xAB","Vcy;:\xD0\x92","Vdash;:\xE2\x8A\xA9", +"Vdashl;:\xE2\xAB\xA6","Vee;:\xE2\x8B\x81","Verbar;:\xE2\x80\x96", +"Vert;:\xE2\x80\x96","VerticalBar;:\xE2\x88\xA3","VerticalLine;:|", +"VerticalSeparator;:\xE2\x9D\x98","VerticalTilde;:\xE2\x89\x80", +"VeryThinSpace;:\xE2\x80\x8A","Vfr;:\xF0\x9D\x94\x99","Vopf;:\xF0\x9D\x95\x8D", +"Vscr;:\xF0\x9D\x92\xB1","Vvdash;:\xE2\x8A\xAA","Wcirc;:\xC5\xB4", +"Wedge;:\xE2\x8B\x80","Wfr;:\xF0\x9D\x94\x9A","Wopf;:\xF0\x9D\x95\x8E", +"Wscr;:\xF0\x9D\x92\xB2","Xfr;:\xF0\x9D\x94\x9B","Xi;:\xCE\x9E", +"Xopf;:\xF0\x9D\x95\x8F","Xscr;:\xF0\x9D\x92\xB3","YAcy;:\xD0\xAF", +"YIcy;:\xD0\x87","YUcy;:\xD0\xAE","Yacute:\xC3\x9D","Yacute;:\xC3\x9D", +"Ycirc;:\xC5\xB6","Ycy;:\xD0\xAB","Yfr;:\xF0\x9D\x94\x9C", +"Yopf;:\xF0\x9D\x95\x90","Yscr;:\xF0\x9D\x92\xB4","Yuml;:\xC5\xB8", +"ZHcy;:\xD0\x96","Zacute;:\xC5\xB9","Zcaron;:\xC5\xBD","Zcy;:\xD0\x97", +"Zdot;:\xC5\xBB","ZeroWidthSpace;:\xE2\x80\x8B","Zeta;:\xCE\x96", +"Zfr;:\xE2\x84\xA8","Zopf;:\xE2\x84\xA4","Zscr;:\xF0\x9D\x92\xB5", +"aacute:\xC3\xA1","aacute;:\xC3\xA1","abreve;:\xC4\x83","ac;:\xE2\x88\xBE", +"acE;:\xE2\x88\xBE\xCC\xB3","acd;:\xE2\x88\xBF","acirc:\xC3\xA2", +"acirc;:\xC3\xA2","acute:\xC2\xB4","acute;:\xC2\xB4","acy;:\xD0\xB0", +"aelig:\xC3\xA6","aelig;:\xC3\xA6","af;:\xE2\x81\xA1","afr;:\xF0\x9D\x94\x9E", +"agrave:\xC3\xA0","agrave;:\xC3\xA0","alefsym;:\xE2\x84\xB5", +"aleph;:\xE2\x84\xB5","alpha;:\xCE\xB1","amacr;:\xC4\x81","amalg;:\xE2\xA8\xBF", +"amp:&","amp;:&","and;:\xE2\x88\xA7","andand;:\xE2\xA9\x95", +"andd;:\xE2\xA9\x9C","andslope;:\xE2\xA9\x98","andv;:\xE2\xA9\x9A", +"ang;:\xE2\x88\xA0","ange;:\xE2\xA6\xA4","angle;:\xE2\x88\xA0", +"angmsd;:\xE2\x88\xA1","angmsdaa;:\xE2\xA6\xA8","angmsdab;:\xE2\xA6\xA9", +"angmsdac;:\xE2\xA6\xAA","angmsdad;:\xE2\xA6\xAB","angmsdae;:\xE2\xA6\xAC", +"angmsdaf;:\xE2\xA6\xAD","angmsdag;:\xE2\xA6\xAE","angmsdah;:\xE2\xA6\xAF", +"angrt;:\xE2\x88\x9F","angrtvb;:\xE2\x8A\xBE","angrtvbd;:\xE2\xA6\x9D", +"angsph;:\xE2\x88\xA2","angst;:\xC3\x85","angzarr;:\xE2\x8D\xBC", +"aogon;:\xC4\x85","aopf;:\xF0\x9D\x95\x92","ap;:\xE2\x89\x88", +"apE;:\xE2\xA9\xB0","apacir;:\xE2\xA9\xAF","ape;:\xE2\x89\x8A", +"apid;:\xE2\x89\x8B","apos;:\'","approx;:\xE2\x89\x88","approxeq;:\xE2\x89\x8A", +"aring:\xC3\xA5","aring;:\xC3\xA5","ascr;:\xF0\x9D\x92\xB6","ast;:*", +"asymp;:\xE2\x89\x88","asympeq;:\xE2\x89\x8D","atilde:\xC3\xA3", +"atilde;:\xC3\xA3","auml:\xC3\xA4","auml;:\xC3\xA4","awconint;:\xE2\x88\xB3", +"awint;:\xE2\xA8\x91","bNot;:\xE2\xAB\xAD","backcong;:\xE2\x89\x8C", +"backepsilon;:\xCF\xB6","backprime;:\xE2\x80\xB5","backsim;:\xE2\x88\xBD", +"backsimeq;:\xE2\x8B\x8D","barvee;:\xE2\x8A\xBD","barwed;:\xE2\x8C\x85", +"barwedge;:\xE2\x8C\x85","bbrk;:\xE2\x8E\xB5","bbrktbrk;:\xE2\x8E\xB6", +"bcong;:\xE2\x89\x8C","bcy;:\xD0\xB1","bdquo;:\xE2\x80\x9E", +"becaus;:\xE2\x88\xB5","because;:\xE2\x88\xB5","bemptyv;:\xE2\xA6\xB0", +"bepsi;:\xCF\xB6","bernou;:\xE2\x84\xAC","beta;:\xCE\xB2","beth;:\xE2\x84\xB6", +"between;:\xE2\x89\xAC","bfr;:\xF0\x9D\x94\x9F","bigcap;:\xE2\x8B\x82", +"bigcirc;:\xE2\x97\xAF","bigcup;:\xE2\x8B\x83","bigodot;:\xE2\xA8\x80", +"bigoplus;:\xE2\xA8\x81","bigotimes;:\xE2\xA8\x82","bigsqcup;:\xE2\xA8\x86", +"bigstar;:\xE2\x98\x85","bigtriangledown;:\xE2\x96\xBD", +"bigtriangleup;:\xE2\x96\xB3","biguplus;:\xE2\xA8\x84","bigvee;:\xE2\x8B\x81", +"bigwedge;:\xE2\x8B\x80","bkarow;:\xE2\xA4\x8D","blacklozenge;:\xE2\xA7\xAB", +"blacksquare;:\xE2\x96\xAA","blacktriangle;:\xE2\x96\xB4", +"blacktriangledown;:\xE2\x96\xBE","blacktriangleleft;:\xE2\x97\x82", +"blacktriangleright;:\xE2\x96\xB8","blank;:\xE2\x90\xA3","blk12;:\xE2\x96\x92", +"blk14;:\xE2\x96\x91","blk34;:\xE2\x96\x93","block;:\xE2\x96\x88", +"bne;:=\xE2\x83\xA5","bnequiv;:\xE2\x89\xA1\xE2\x83\xA5","bnot;:\xE2\x8C\x90", +"bopf;:\xF0\x9D\x95\x93","bot;:\xE2\x8A\xA5","bottom;:\xE2\x8A\xA5", +"bowtie;:\xE2\x8B\x88","boxDL;:\xE2\x95\x97","boxDR;:\xE2\x95\x94", +"boxDl;:\xE2\x95\x96","boxDr;:\xE2\x95\x93","boxH;:\xE2\x95\x90", +"boxHD;:\xE2\x95\xA6","boxHU;:\xE2\x95\xA9","boxHd;:\xE2\x95\xA4", +"boxHu;:\xE2\x95\xA7","boxUL;:\xE2\x95\x9D","boxUR;:\xE2\x95\x9A", +"boxUl;:\xE2\x95\x9C","boxUr;:\xE2\x95\x99","boxV;:\xE2\x95\x91", +"boxVH;:\xE2\x95\xAC","boxVL;:\xE2\x95\xA3","boxVR;:\xE2\x95\xA0", +"boxVh;:\xE2\x95\xAB","boxVl;:\xE2\x95\xA2","boxVr;:\xE2\x95\x9F", +"boxbox;:\xE2\xA7\x89","boxdL;:\xE2\x95\x95","boxdR;:\xE2\x95\x92", +"boxdl;:\xE2\x94\x90","boxdr;:\xE2\x94\x8C","boxh;:\xE2\x94\x80", +"boxhD;:\xE2\x95\xA5","boxhU;:\xE2\x95\xA8","boxhd;:\xE2\x94\xAC", +"boxhu;:\xE2\x94\xB4","boxminus;:\xE2\x8A\x9F","boxplus;:\xE2\x8A\x9E", +"boxtimes;:\xE2\x8A\xA0","boxuL;:\xE2\x95\x9B","boxuR;:\xE2\x95\x98", +"boxul;:\xE2\x94\x98","boxur;:\xE2\x94\x94","boxv;:\xE2\x94\x82", +"boxvH;:\xE2\x95\xAA","boxvL;:\xE2\x95\xA1","boxvR;:\xE2\x95\x9E", +"boxvh;:\xE2\x94\xBC","boxvl;:\xE2\x94\xA4","boxvr;:\xE2\x94\x9C", +"bprime;:\xE2\x80\xB5","breve;:\xCB\x98","brvbar:\xC2\xA6","brvbar;:\xC2\xA6", +"bscr;:\xF0\x9D\x92\xB7","bsemi;:\xE2\x81\x8F","bsim;:\xE2\x88\xBD", +"bsime;:\xE2\x8B\x8D","bsol;:\\","bsolb;:\xE2\xA7\x85","bsolhsub;:\xE2\x9F\x88", +"bull;:\xE2\x80\xA2","bullet;:\xE2\x80\xA2","bump;:\xE2\x89\x8E", +"bumpE;:\xE2\xAA\xAE","bumpe;:\xE2\x89\x8F","bumpeq;:\xE2\x89\x8F", +"cacute;:\xC4\x87","cap;:\xE2\x88\xA9","capand;:\xE2\xA9\x84", +"capbrcup;:\xE2\xA9\x89","capcap;:\xE2\xA9\x8B","capcup;:\xE2\xA9\x87", +"capdot;:\xE2\xA9\x80","caps;:\xE2\x88\xA9\xEF\xB8\x80","caret;:\xE2\x81\x81", +"caron;:\xCB\x87","ccaps;:\xE2\xA9\x8D","ccaron;:\xC4\x8D","ccedil:\xC3\xA7", +"ccedil;:\xC3\xA7","ccirc;:\xC4\x89","ccups;:\xE2\xA9\x8C", +"ccupssm;:\xE2\xA9\x90","cdot;:\xC4\x8B","cedil:\xC2\xB8","cedil;:\xC2\xB8", +"cemptyv;:\xE2\xA6\xB2","cent:\xC2\xA2","cent;:\xC2\xA2","centerdot;:\xC2\xB7", +"cfr;:\xF0\x9D\x94\xA0","chcy;:\xD1\x87","check;:\xE2\x9C\x93", +"checkmark;:\xE2\x9C\x93","chi;:\xCF\x87","cir;:\xE2\x97\x8B", +"cirE;:\xE2\xA7\x83","circ;:\xCB\x86","circeq;:\xE2\x89\x97", +"circlearrowleft;:\xE2\x86\xBA","circlearrowright;:\xE2\x86\xBB", +"circledR;:\xC2\xAE","circledS;:\xE2\x93\x88","circledast;:\xE2\x8A\x9B", +"circledcirc;:\xE2\x8A\x9A","circleddash;:\xE2\x8A\x9D","cire;:\xE2\x89\x97", +"cirfnint;:\xE2\xA8\x90","cirmid;:\xE2\xAB\xAF","cirscir;:\xE2\xA7\x82", +"clubs;:\xE2\x99\xA3","clubsuit;:\xE2\x99\xA3","colon;::", +"colone;:\xE2\x89\x94","coloneq;:\xE2\x89\x94","comma;:,","commat;:@", +"comp;:\xE2\x88\x81","compfn;:\xE2\x88\x98","complement;:\xE2\x88\x81", +"complexes;:\xE2\x84\x82","cong;:\xE2\x89\x85","congdot;:\xE2\xA9\xAD", +"conint;:\xE2\x88\xAE","copf;:\xF0\x9D\x95\x94","coprod;:\xE2\x88\x90", +"copy:\xC2\xA9","copy;:\xC2\xA9","copysr;:\xE2\x84\x97","crarr;:\xE2\x86\xB5", +"cross;:\xE2\x9C\x97","cscr;:\xF0\x9D\x92\xB8","csub;:\xE2\xAB\x8F", +"csube;:\xE2\xAB\x91","csup;:\xE2\xAB\x90","csupe;:\xE2\xAB\x92", +"ctdot;:\xE2\x8B\xAF","cudarrl;:\xE2\xA4\xB8","cudarrr;:\xE2\xA4\xB5", +"cuepr;:\xE2\x8B\x9E","cuesc;:\xE2\x8B\x9F","cularr;:\xE2\x86\xB6", +"cularrp;:\xE2\xA4\xBD","cup;:\xE2\x88\xAA","cupbrcap;:\xE2\xA9\x88", +"cupcap;:\xE2\xA9\x86","cupcup;:\xE2\xA9\x8A","cupdot;:\xE2\x8A\x8D", +"cupor;:\xE2\xA9\x85","cups;:\xE2\x88\xAA\xEF\xB8\x80","curarr;:\xE2\x86\xB7", +"curarrm;:\xE2\xA4\xBC","curlyeqprec;:\xE2\x8B\x9E","curlyeqsucc;:\xE2\x8B\x9F", +"curlyvee;:\xE2\x8B\x8E","curlywedge;:\xE2\x8B\x8F","curren:\xC2\xA4", +"curren;:\xC2\xA4","curvearrowleft;:\xE2\x86\xB6", +"curvearrowright;:\xE2\x86\xB7","cuvee;:\xE2\x8B\x8E","cuwed;:\xE2\x8B\x8F", +"cwconint;:\xE2\x88\xB2","cwint;:\xE2\x88\xB1","cylcty;:\xE2\x8C\xAD", +"dArr;:\xE2\x87\x93","dHar;:\xE2\xA5\xA5","dagger;:\xE2\x80\xA0", +"daleth;:\xE2\x84\xB8","darr;:\xE2\x86\x93","dash;:\xE2\x80\x90", +"dashv;:\xE2\x8A\xA3","dbkarow;:\xE2\xA4\x8F","dblac;:\xCB\x9D", +"dcaron;:\xC4\x8F","dcy;:\xD0\xB4","dd;:\xE2\x85\x86","ddagger;:\xE2\x80\xA1", +"ddarr;:\xE2\x87\x8A","ddotseq;:\xE2\xA9\xB7","deg:\xC2\xB0","deg;:\xC2\xB0", +"delta;:\xCE\xB4","demptyv;:\xE2\xA6\xB1","dfisht;:\xE2\xA5\xBF", +"dfr;:\xF0\x9D\x94\xA1","dharl;:\xE2\x87\x83","dharr;:\xE2\x87\x82", +"diam;:\xE2\x8B\x84","diamond;:\xE2\x8B\x84","diamondsuit;:\xE2\x99\xA6", +"diams;:\xE2\x99\xA6","die;:\xC2\xA8","digamma;:\xCF\x9D","disin;:\xE2\x8B\xB2", +"div;:\xC3\xB7","divide:\xC3\xB7","divide;:\xC3\xB7", +"divideontimes;:\xE2\x8B\x87","divonx;:\xE2\x8B\x87","djcy;:\xD1\x92", +"dlcorn;:\xE2\x8C\x9E","dlcrop;:\xE2\x8C\x8D","dollar;:$", +"dopf;:\xF0\x9D\x95\x95","dot;:\xCB\x99","doteq;:\xE2\x89\x90", +"doteqdot;:\xE2\x89\x91","dotminus;:\xE2\x88\xB8","dotplus;:\xE2\x88\x94", +"dotsquare;:\xE2\x8A\xA1","doublebarwedge;:\xE2\x8C\x86", +"downarrow;:\xE2\x86\x93","downdownarrows;:\xE2\x87\x8A", +"downharpoonleft;:\xE2\x87\x83","downharpoonright;:\xE2\x87\x82", +"drbkarow;:\xE2\xA4\x90","drcorn;:\xE2\x8C\x9F","drcrop;:\xE2\x8C\x8C", +"dscr;:\xF0\x9D\x92\xB9","dscy;:\xD1\x95","dsol;:\xE2\xA7\xB6", +"dstrok;:\xC4\x91","dtdot;:\xE2\x8B\xB1","dtri;:\xE2\x96\xBF", +"dtrif;:\xE2\x96\xBE","duarr;:\xE2\x87\xB5","duhar;:\xE2\xA5\xAF", +"dwangle;:\xE2\xA6\xA6","dzcy;:\xD1\x9F","dzigrarr;:\xE2\x9F\xBF", +"eDDot;:\xE2\xA9\xB7","eDot;:\xE2\x89\x91","eacute:\xC3\xA9","eacute;:\xC3\xA9", +"easter;:\xE2\xA9\xAE","ecaron;:\xC4\x9B","ecir;:\xE2\x89\x96","ecirc:\xC3\xAA", +"ecirc;:\xC3\xAA","ecolon;:\xE2\x89\x95","ecy;:\xD1\x8D","edot;:\xC4\x97", +"ee;:\xE2\x85\x87","efDot;:\xE2\x89\x92","efr;:\xF0\x9D\x94\xA2", +"eg;:\xE2\xAA\x9A","egrave:\xC3\xA8","egrave;:\xC3\xA8","egs;:\xE2\xAA\x96", +"egsdot;:\xE2\xAA\x98","el;:\xE2\xAA\x99","elinters;:\xE2\x8F\xA7", +"ell;:\xE2\x84\x93","els;:\xE2\xAA\x95","elsdot;:\xE2\xAA\x97", +"emacr;:\xC4\x93","empty;:\xE2\x88\x85","emptyset;:\xE2\x88\x85", +"emptyv;:\xE2\x88\x85","emsp13;:\xE2\x80\x84","emsp14;:\xE2\x80\x85", +"emsp;:\xE2\x80\x83","eng;:\xC5\x8B","ensp;:\xE2\x80\x82","eogon;:\xC4\x99", +"eopf;:\xF0\x9D\x95\x96","epar;:\xE2\x8B\x95","eparsl;:\xE2\xA7\xA3", +"eplus;:\xE2\xA9\xB1","epsi;:\xCE\xB5","epsilon;:\xCE\xB5","epsiv;:\xCF\xB5", +"eqcirc;:\xE2\x89\x96","eqcolon;:\xE2\x89\x95","eqsim;:\xE2\x89\x82", +"eqslantgtr;:\xE2\xAA\x96","eqslantless;:\xE2\xAA\x95","equals;:=", +"equest;:\xE2\x89\x9F","equiv;:\xE2\x89\xA1","equivDD;:\xE2\xA9\xB8", +"eqvparsl;:\xE2\xA7\xA5","erDot;:\xE2\x89\x93","erarr;:\xE2\xA5\xB1", +"escr;:\xE2\x84\xAF","esdot;:\xE2\x89\x90","esim;:\xE2\x89\x82","eta;:\xCE\xB7", +"eth:\xC3\xB0","eth;:\xC3\xB0","euml:\xC3\xAB","euml;:\xC3\xAB", +"euro;:\xE2\x82\xAC","excl;:!","exist;:\xE2\x88\x83", +"expectation;:\xE2\x84\xB0","exponentiale;:\xE2\x85\x87", +"fallingdotseq;:\xE2\x89\x92","fcy;:\xD1\x84","female;:\xE2\x99\x80", +"ffilig;:\xEF\xAC\x83","fflig;:\xEF\xAC\x80","ffllig;:\xEF\xAC\x84", +"ffr;:\xF0\x9D\x94\xA3","filig;:\xEF\xAC\x81","fjlig;:fj","flat;:\xE2\x99\xAD", +"fllig;:\xEF\xAC\x82","fltns;:\xE2\x96\xB1","fnof;:\xC6\x92", +"fopf;:\xF0\x9D\x95\x97","forall;:\xE2\x88\x80","fork;:\xE2\x8B\x94", +"forkv;:\xE2\xAB\x99","fpartint;:\xE2\xA8\x8D","frac12:\xC2\xBD", +"frac12;:\xC2\xBD","frac13;:\xE2\x85\x93","frac14:\xC2\xBC","frac14;:\xC2\xBC", +"frac15;:\xE2\x85\x95","frac16;:\xE2\x85\x99","frac18;:\xE2\x85\x9B", +"frac23;:\xE2\x85\x94","frac25;:\xE2\x85\x96","frac34:\xC2\xBE", +"frac34;:\xC2\xBE","frac35;:\xE2\x85\x97","frac38;:\xE2\x85\x9C", +"frac45;:\xE2\x85\x98","frac56;:\xE2\x85\x9A","frac58;:\xE2\x85\x9D", +"frac78;:\xE2\x85\x9E","frasl;:\xE2\x81\x84","frown;:\xE2\x8C\xA2", +"fscr;:\xF0\x9D\x92\xBB","gE;:\xE2\x89\xA7","gEl;:\xE2\xAA\x8C", +"gacute;:\xC7\xB5","gamma;:\xCE\xB3","gammad;:\xCF\x9D","gap;:\xE2\xAA\x86", +"gbreve;:\xC4\x9F","gcirc;:\xC4\x9D","gcy;:\xD0\xB3","gdot;:\xC4\xA1", +"ge;:\xE2\x89\xA5","gel;:\xE2\x8B\x9B","geq;:\xE2\x89\xA5","geqq;:\xE2\x89\xA7", +"geqslant;:\xE2\xA9\xBE","ges;:\xE2\xA9\xBE","gescc;:\xE2\xAA\xA9", +"gesdot;:\xE2\xAA\x80","gesdoto;:\xE2\xAA\x82","gesdotol;:\xE2\xAA\x84", +"gesl;:\xE2\x8B\x9B\xEF\xB8\x80","gesles;:\xE2\xAA\x94","gfr;:\xF0\x9D\x94\xA4", +"gg;:\xE2\x89\xAB","ggg;:\xE2\x8B\x99","gimel;:\xE2\x84\xB7","gjcy;:\xD1\x93", +"gl;:\xE2\x89\xB7","glE;:\xE2\xAA\x92","gla;:\xE2\xAA\xA5","glj;:\xE2\xAA\xA4", +"gnE;:\xE2\x89\xA9","gnap;:\xE2\xAA\x8A","gnapprox;:\xE2\xAA\x8A", +"gne;:\xE2\xAA\x88","gneq;:\xE2\xAA\x88","gneqq;:\xE2\x89\xA9", +"gnsim;:\xE2\x8B\xA7","gopf;:\xF0\x9D\x95\x98","grave;:`","gscr;:\xE2\x84\x8A", +"gsim;:\xE2\x89\xB3","gsime;:\xE2\xAA\x8E","gsiml;:\xE2\xAA\x90","gt:>","gt;:>", +"gtcc;:\xE2\xAA\xA7","gtcir;:\xE2\xA9\xBA","gtdot;:\xE2\x8B\x97", +"gtlPar;:\xE2\xA6\x95","gtquest;:\xE2\xA9\xBC","gtrapprox;:\xE2\xAA\x86", +"gtrarr;:\xE2\xA5\xB8","gtrdot;:\xE2\x8B\x97","gtreqless;:\xE2\x8B\x9B", +"gtreqqless;:\xE2\xAA\x8C","gtrless;:\xE2\x89\xB7","gtrsim;:\xE2\x89\xB3", +"gvertneqq;:\xE2\x89\xA9\xEF\xB8\x80","gvnE;:\xE2\x89\xA9\xEF\xB8\x80", +"hArr;:\xE2\x87\x94","hairsp;:\xE2\x80\x8A","half;:\xC2\xBD", +"hamilt;:\xE2\x84\x8B","hardcy;:\xD1\x8A","harr;:\xE2\x86\x94", +"harrcir;:\xE2\xA5\x88","harrw;:\xE2\x86\xAD","hbar;:\xE2\x84\x8F", +"hcirc;:\xC4\xA5","hearts;:\xE2\x99\xA5","heartsuit;:\xE2\x99\xA5", +"hellip;:\xE2\x80\xA6","hercon;:\xE2\x8A\xB9","hfr;:\xF0\x9D\x94\xA5", +"hksearow;:\xE2\xA4\xA5","hkswarow;:\xE2\xA4\xA6","hoarr;:\xE2\x87\xBF", +"homtht;:\xE2\x88\xBB","hookleftarrow;:\xE2\x86\xA9", +"hookrightarrow;:\xE2\x86\xAA","hopf;:\xF0\x9D\x95\x99","horbar;:\xE2\x80\x95", +"hscr;:\xF0\x9D\x92\xBD","hslash;:\xE2\x84\x8F","hstrok;:\xC4\xA7", +"hybull;:\xE2\x81\x83","hyphen;:\xE2\x80\x90","iacute:\xC3\xAD", +"iacute;:\xC3\xAD","ic;:\xE2\x81\xA3","icirc:\xC3\xAE","icirc;:\xC3\xAE", +"icy;:\xD0\xB8","iecy;:\xD0\xB5","iexcl:\xC2\xA1","iexcl;:\xC2\xA1", +"iff;:\xE2\x87\x94","ifr;:\xF0\x9D\x94\xA6","igrave:\xC3\xAC", +"igrave;:\xC3\xAC","ii;:\xE2\x85\x88","iiiint;:\xE2\xA8\x8C", +"iiint;:\xE2\x88\xAD","iinfin;:\xE2\xA7\x9C","iiota;:\xE2\x84\xA9", +"ijlig;:\xC4\xB3","imacr;:\xC4\xAB","image;:\xE2\x84\x91", +"imagline;:\xE2\x84\x90","imagpart;:\xE2\x84\x91","imath;:\xC4\xB1", +"imof;:\xE2\x8A\xB7","imped;:\xC6\xB5","in;:\xE2\x88\x88", +"incare;:\xE2\x84\x85","infin;:\xE2\x88\x9E","infintie;:\xE2\xA7\x9D", +"inodot;:\xC4\xB1","int;:\xE2\x88\xAB","intcal;:\xE2\x8A\xBA", +"integers;:\xE2\x84\xA4","intercal;:\xE2\x8A\xBA","intlarhk;:\xE2\xA8\x97", +"intprod;:\xE2\xA8\xBC","iocy;:\xD1\x91","iogon;:\xC4\xAF", +"iopf;:\xF0\x9D\x95\x9A","iota;:\xCE\xB9","iprod;:\xE2\xA8\xBC", +"iquest:\xC2\xBF","iquest;:\xC2\xBF","iscr;:\xF0\x9D\x92\xBE", +"isin;:\xE2\x88\x88","isinE;:\xE2\x8B\xB9","isindot;:\xE2\x8B\xB5", +"isins;:\xE2\x8B\xB4","isinsv;:\xE2\x8B\xB3","isinv;:\xE2\x88\x88", +"it;:\xE2\x81\xA2","itilde;:\xC4\xA9","iukcy;:\xD1\x96","iuml:\xC3\xAF", +"iuml;:\xC3\xAF","jcirc;:\xC4\xB5","jcy;:\xD0\xB9","jfr;:\xF0\x9D\x94\xA7", +"jmath;:\xC8\xB7","jopf;:\xF0\x9D\x95\x9B","jscr;:\xF0\x9D\x92\xBF", +"jsercy;:\xD1\x98","jukcy;:\xD1\x94","kappa;:\xCE\xBA","kappav;:\xCF\xB0", +"kcedil;:\xC4\xB7","kcy;:\xD0\xBA","kfr;:\xF0\x9D\x94\xA8","kgreen;:\xC4\xB8", +"khcy;:\xD1\x85","kjcy;:\xD1\x9C","kopf;:\xF0\x9D\x95\x9C", +"kscr;:\xF0\x9D\x93\x80","lAarr;:\xE2\x87\x9A","lArr;:\xE2\x87\x90", +"lAtail;:\xE2\xA4\x9B","lBarr;:\xE2\xA4\x8E","lE;:\xE2\x89\xA6", +"lEg;:\xE2\xAA\x8B","lHar;:\xE2\xA5\xA2","lacute;:\xC4\xBA", +"laemptyv;:\xE2\xA6\xB4","lagran;:\xE2\x84\x92","lambda;:\xCE\xBB", +"lang;:\xE2\x9F\xA8","langd;:\xE2\xA6\x91","langle;:\xE2\x9F\xA8", +"lap;:\xE2\xAA\x85","laquo:\xC2\xAB","laquo;:\xC2\xAB","larr;:\xE2\x86\x90", +"larrb;:\xE2\x87\xA4","larrbfs;:\xE2\xA4\x9F","larrfs;:\xE2\xA4\x9D", +"larrhk;:\xE2\x86\xA9","larrlp;:\xE2\x86\xAB","larrpl;:\xE2\xA4\xB9", +"larrsim;:\xE2\xA5\xB3","larrtl;:\xE2\x86\xA2","lat;:\xE2\xAA\xAB", +"latail;:\xE2\xA4\x99","late;:\xE2\xAA\xAD","lates;:\xE2\xAA\xAD\xEF\xB8\x80", +"lbarr;:\xE2\xA4\x8C","lbbrk;:\xE2\x9D\xB2","lbrace;:{","lbrack;:[", +"lbrke;:\xE2\xA6\x8B","lbrksld;:\xE2\xA6\x8F","lbrkslu;:\xE2\xA6\x8D", +"lcaron;:\xC4\xBE","lcedil;:\xC4\xBC","lceil;:\xE2\x8C\x88","lcub;:{", +"lcy;:\xD0\xBB","ldca;:\xE2\xA4\xB6","ldquo;:\xE2\x80\x9C", +"ldquor;:\xE2\x80\x9E","ldrdhar;:\xE2\xA5\xA7","ldrushar;:\xE2\xA5\x8B", +"ldsh;:\xE2\x86\xB2","le;:\xE2\x89\xA4","leftarrow;:\xE2\x86\x90", +"leftarrowtail;:\xE2\x86\xA2","leftharpoondown;:\xE2\x86\xBD", +"leftharpoonup;:\xE2\x86\xBC","leftleftarrows;:\xE2\x87\x87", +"leftrightarrow;:\xE2\x86\x94","leftrightarrows;:\xE2\x87\x86", +"leftrightharpoons;:\xE2\x87\x8B","leftrightsquigarrow;:\xE2\x86\xAD", +"leftthreetimes;:\xE2\x8B\x8B","leg;:\xE2\x8B\x9A","leq;:\xE2\x89\xA4", +"leqq;:\xE2\x89\xA6","leqslant;:\xE2\xA9\xBD","les;:\xE2\xA9\xBD", +"lescc;:\xE2\xAA\xA8","lesdot;:\xE2\xA9\xBF","lesdoto;:\xE2\xAA\x81", +"lesdotor;:\xE2\xAA\x83","lesg;:\xE2\x8B\x9A\xEF\xB8\x80", +"lesges;:\xE2\xAA\x93","lessapprox;:\xE2\xAA\x85","lessdot;:\xE2\x8B\x96", +"lesseqgtr;:\xE2\x8B\x9A","lesseqqgtr;:\xE2\xAA\x8B","lessgtr;:\xE2\x89\xB6", +"lesssim;:\xE2\x89\xB2","lfisht;:\xE2\xA5\xBC","lfloor;:\xE2\x8C\x8A", +"lfr;:\xF0\x9D\x94\xA9","lg;:\xE2\x89\xB6","lgE;:\xE2\xAA\x91", +"lhard;:\xE2\x86\xBD","lharu;:\xE2\x86\xBC","lharul;:\xE2\xA5\xAA", +"lhblk;:\xE2\x96\x84","ljcy;:\xD1\x99","ll;:\xE2\x89\xAA","llarr;:\xE2\x87\x87", +"llcorner;:\xE2\x8C\x9E","llhard;:\xE2\xA5\xAB","lltri;:\xE2\x97\xBA", +"lmidot;:\xC5\x80","lmoust;:\xE2\x8E\xB0","lmoustache;:\xE2\x8E\xB0", +"lnE;:\xE2\x89\xA8","lnap;:\xE2\xAA\x89","lnapprox;:\xE2\xAA\x89", +"lne;:\xE2\xAA\x87","lneq;:\xE2\xAA\x87","lneqq;:\xE2\x89\xA8", +"lnsim;:\xE2\x8B\xA6","loang;:\xE2\x9F\xAC","loarr;:\xE2\x87\xBD", +"lobrk;:\xE2\x9F\xA6","longleftarrow;:\xE2\x9F\xB5", +"longleftrightarrow;:\xE2\x9F\xB7","longmapsto;:\xE2\x9F\xBC", +"longrightarrow;:\xE2\x9F\xB6","looparrowleft;:\xE2\x86\xAB", +"looparrowright;:\xE2\x86\xAC","lopar;:\xE2\xA6\x85","lopf;:\xF0\x9D\x95\x9D", +"loplus;:\xE2\xA8\xAD","lotimes;:\xE2\xA8\xB4","lowast;:\xE2\x88\x97", +"lowbar;:_","loz;:\xE2\x97\x8A","lozenge;:\xE2\x97\x8A","lozf;:\xE2\xA7\xAB", +"lpar;:(","lparlt;:\xE2\xA6\x93","lrarr;:\xE2\x87\x86","lrcorner;:\xE2\x8C\x9F", +"lrhar;:\xE2\x87\x8B","lrhard;:\xE2\xA5\xAD","lrm;:\xE2\x80\x8E", +"lrtri;:\xE2\x8A\xBF","lsaquo;:\xE2\x80\xB9","lscr;:\xF0\x9D\x93\x81", +"lsh;:\xE2\x86\xB0","lsim;:\xE2\x89\xB2","lsime;:\xE2\xAA\x8D", +"lsimg;:\xE2\xAA\x8F","lsqb;:[","lsquo;:\xE2\x80\x98","lsquor;:\xE2\x80\x9A", +"lstrok;:\xC5\x82","lt:<","lt;:<","ltcc;:\xE2\xAA\xA6","ltcir;:\xE2\xA9\xB9", +"ltdot;:\xE2\x8B\x96","lthree;:\xE2\x8B\x8B","ltimes;:\xE2\x8B\x89", +"ltlarr;:\xE2\xA5\xB6","ltquest;:\xE2\xA9\xBB","ltrPar;:\xE2\xA6\x96", +"ltri;:\xE2\x97\x83","ltrie;:\xE2\x8A\xB4","ltrif;:\xE2\x97\x82", +"lurdshar;:\xE2\xA5\x8A","luruhar;:\xE2\xA5\xA6", +"lvertneqq;:\xE2\x89\xA8\xEF\xB8\x80","lvnE;:\xE2\x89\xA8\xEF\xB8\x80", +"mDDot;:\xE2\x88\xBA","macr:\xC2\xAF","macr;:\xC2\xAF","male;:\xE2\x99\x82", +"malt;:\xE2\x9C\xA0","maltese;:\xE2\x9C\xA0","map;:\xE2\x86\xA6", +"mapsto;:\xE2\x86\xA6","mapstodown;:\xE2\x86\xA7","mapstoleft;:\xE2\x86\xA4", +"mapstoup;:\xE2\x86\xA5","marker;:\xE2\x96\xAE","mcomma;:\xE2\xA8\xA9", +"mcy;:\xD0\xBC","mdash;:\xE2\x80\x94","measuredangle;:\xE2\x88\xA1", +"mfr;:\xF0\x9D\x94\xAA","mho;:\xE2\x84\xA7","micro:\xC2\xB5","micro;:\xC2\xB5", +"mid;:\xE2\x88\xA3","midast;:*","midcir;:\xE2\xAB\xB0","middot:\xC2\xB7", +"middot;:\xC2\xB7","minus;:\xE2\x88\x92","minusb;:\xE2\x8A\x9F", +"minusd;:\xE2\x88\xB8","minusdu;:\xE2\xA8\xAA","mlcp;:\xE2\xAB\x9B", +"mldr;:\xE2\x80\xA6","mnplus;:\xE2\x88\x93","models;:\xE2\x8A\xA7", +"mopf;:\xF0\x9D\x95\x9E","mp;:\xE2\x88\x93","mscr;:\xF0\x9D\x93\x82", +"mstpos;:\xE2\x88\xBE","mu;:\xCE\xBC","multimap;:\xE2\x8A\xB8", +"mumap;:\xE2\x8A\xB8","nGg;:\xE2\x8B\x99\xCC\xB8", +"nGt;:\xE2\x89\xAB\xE2\x83\x92","nGtv;:\xE2\x89\xAB\xCC\xB8", +"nLeftarrow;:\xE2\x87\x8D","nLeftrightarrow;:\xE2\x87\x8E", +"nLl;:\xE2\x8B\x98\xCC\xB8","nLt;:\xE2\x89\xAA\xE2\x83\x92", +"nLtv;:\xE2\x89\xAA\xCC\xB8","nRightarrow;:\xE2\x87\x8F","nVDash;:\xE2\x8A\xAF", +"nVdash;:\xE2\x8A\xAE","nabla;:\xE2\x88\x87","nacute;:\xC5\x84", +"nang;:\xE2\x88\xA0\xE2\x83\x92","nap;:\xE2\x89\x89", +"napE;:\xE2\xA9\xB0\xCC\xB8","napid;:\xE2\x89\x8B\xCC\xB8","napos;:\xC5\x89", +"napprox;:\xE2\x89\x89","natur;:\xE2\x99\xAE","natural;:\xE2\x99\xAE", +"naturals;:\xE2\x84\x95","nbsp:\xC2\xA0","nbsp;:\xC2\xA0", +"nbump;:\xE2\x89\x8E\xCC\xB8","nbumpe;:\xE2\x89\x8F\xCC\xB8", +"ncap;:\xE2\xA9\x83","ncaron;:\xC5\x88","ncedil;:\xC5\x86", +"ncong;:\xE2\x89\x87","ncongdot;:\xE2\xA9\xAD\xCC\xB8","ncup;:\xE2\xA9\x82", +"ncy;:\xD0\xBD","ndash;:\xE2\x80\x93","ne;:\xE2\x89\xA0","neArr;:\xE2\x87\x97", +"nearhk;:\xE2\xA4\xA4","nearr;:\xE2\x86\x97","nearrow;:\xE2\x86\x97", +"nedot;:\xE2\x89\x90\xCC\xB8","nequiv;:\xE2\x89\xA2","nesear;:\xE2\xA4\xA8", +"nesim;:\xE2\x89\x82\xCC\xB8","nexist;:\xE2\x88\x84","nexists;:\xE2\x88\x84", +"nfr;:\xF0\x9D\x94\xAB","ngE;:\xE2\x89\xA7\xCC\xB8","nge;:\xE2\x89\xB1", +"ngeq;:\xE2\x89\xB1","ngeqq;:\xE2\x89\xA7\xCC\xB8", +"ngeqslant;:\xE2\xA9\xBE\xCC\xB8","nges;:\xE2\xA9\xBE\xCC\xB8", +"ngsim;:\xE2\x89\xB5","ngt;:\xE2\x89\xAF","ngtr;:\xE2\x89\xAF", +"nhArr;:\xE2\x87\x8E","nharr;:\xE2\x86\xAE","nhpar;:\xE2\xAB\xB2", +"ni;:\xE2\x88\x8B","nis;:\xE2\x8B\xBC","nisd;:\xE2\x8B\xBA","niv;:\xE2\x88\x8B", +"njcy;:\xD1\x9A","nlArr;:\xE2\x87\x8D","nlE;:\xE2\x89\xA6\xCC\xB8", +"nlarr;:\xE2\x86\x9A","nldr;:\xE2\x80\xA5","nle;:\xE2\x89\xB0", +"nleftarrow;:\xE2\x86\x9A","nleftrightarrow;:\xE2\x86\xAE","nleq;:\xE2\x89\xB0", +"nleqq;:\xE2\x89\xA6\xCC\xB8","nleqslant;:\xE2\xA9\xBD\xCC\xB8", +"nles;:\xE2\xA9\xBD\xCC\xB8","nless;:\xE2\x89\xAE","nlsim;:\xE2\x89\xB4", +"nlt;:\xE2\x89\xAE","nltri;:\xE2\x8B\xAA","nltrie;:\xE2\x8B\xAC", +"nmid;:\xE2\x88\xA4","nopf;:\xF0\x9D\x95\x9F","not:\xC2\xAC","not;:\xC2\xAC", +"notin;:\xE2\x88\x89","notinE;:\xE2\x8B\xB9\xCC\xB8", +"notindot;:\xE2\x8B\xB5\xCC\xB8","notinva;:\xE2\x88\x89", +"notinvb;:\xE2\x8B\xB7","notinvc;:\xE2\x8B\xB6","notni;:\xE2\x88\x8C", +"notniva;:\xE2\x88\x8C","notnivb;:\xE2\x8B\xBE","notnivc;:\xE2\x8B\xBD", +"npar;:\xE2\x88\xA6","nparallel;:\xE2\x88\xA6", +"nparsl;:\xE2\xAB\xBD\xE2\x83\xA5","npart;:\xE2\x88\x82\xCC\xB8", +"npolint;:\xE2\xA8\x94","npr;:\xE2\x8A\x80","nprcue;:\xE2\x8B\xA0", +"npre;:\xE2\xAA\xAF\xCC\xB8","nprec;:\xE2\x8A\x80", +"npreceq;:\xE2\xAA\xAF\xCC\xB8","nrArr;:\xE2\x87\x8F","nrarr;:\xE2\x86\x9B", +"nrarrc;:\xE2\xA4\xB3\xCC\xB8","nrarrw;:\xE2\x86\x9D\xCC\xB8", +"nrightarrow;:\xE2\x86\x9B","nrtri;:\xE2\x8B\xAB","nrtrie;:\xE2\x8B\xAD", +"nsc;:\xE2\x8A\x81","nsccue;:\xE2\x8B\xA1","nsce;:\xE2\xAA\xB0\xCC\xB8", +"nscr;:\xF0\x9D\x93\x83","nshortmid;:\xE2\x88\xA4", +"nshortparallel;:\xE2\x88\xA6","nsim;:\xE2\x89\x81","nsime;:\xE2\x89\x84", +"nsimeq;:\xE2\x89\x84","nsmid;:\xE2\x88\xA4","nspar;:\xE2\x88\xA6", +"nsqsube;:\xE2\x8B\xA2","nsqsupe;:\xE2\x8B\xA3","nsub;:\xE2\x8A\x84", +"nsubE;:\xE2\xAB\x85\xCC\xB8","nsube;:\xE2\x8A\x88", +"nsubset;:\xE2\x8A\x82\xE2\x83\x92","nsubseteq;:\xE2\x8A\x88", +"nsubseteqq;:\xE2\xAB\x85\xCC\xB8","nsucc;:\xE2\x8A\x81", +"nsucceq;:\xE2\xAA\xB0\xCC\xB8","nsup;:\xE2\x8A\x85", +"nsupE;:\xE2\xAB\x86\xCC\xB8","nsupe;:\xE2\x8A\x89", +"nsupset;:\xE2\x8A\x83\xE2\x83\x92","nsupseteq;:\xE2\x8A\x89", +"nsupseteqq;:\xE2\xAB\x86\xCC\xB8","ntgl;:\xE2\x89\xB9","ntilde:\xC3\xB1", +"ntilde;:\xC3\xB1","ntlg;:\xE2\x89\xB8","ntriangleleft;:\xE2\x8B\xAA", +"ntrianglelefteq;:\xE2\x8B\xAC","ntriangleright;:\xE2\x8B\xAB", +"ntrianglerighteq;:\xE2\x8B\xAD","nu;:\xCE\xBD","num;:#","numero;:\xE2\x84\x96", +"numsp;:\xE2\x80\x87","nvDash;:\xE2\x8A\xAD","nvHarr;:\xE2\xA4\x84", +"nvap;:\xE2\x89\x8D\xE2\x83\x92","nvdash;:\xE2\x8A\xAC", +"nvge;:\xE2\x89\xA5\xE2\x83\x92","nvgt;:>\xE2\x83\x92","nvinfin;:\xE2\xA7\x9E", +"nvlArr;:\xE2\xA4\x82","nvle;:\xE2\x89\xA4\xE2\x83\x92","nvlt;:<\xE2\x83\x92", +"nvltrie;:\xE2\x8A\xB4\xE2\x83\x92","nvrArr;:\xE2\xA4\x83", +"nvrtrie;:\xE2\x8A\xB5\xE2\x83\x92","nvsim;:\xE2\x88\xBC\xE2\x83\x92", +"nwArr;:\xE2\x87\x96","nwarhk;:\xE2\xA4\xA3","nwarr;:\xE2\x86\x96", +"nwarrow;:\xE2\x86\x96","nwnear;:\xE2\xA4\xA7","oS;:\xE2\x93\x88", +"oacute:\xC3\xB3","oacute;:\xC3\xB3","oast;:\xE2\x8A\x9B","ocir;:\xE2\x8A\x9A", +"ocirc:\xC3\xB4","ocirc;:\xC3\xB4","ocy;:\xD0\xBE","odash;:\xE2\x8A\x9D", +"odblac;:\xC5\x91","odiv;:\xE2\xA8\xB8","odot;:\xE2\x8A\x99", +"odsold;:\xE2\xA6\xBC","oelig;:\xC5\x93","ofcir;:\xE2\xA6\xBF", +"ofr;:\xF0\x9D\x94\xAC","ogon;:\xCB\x9B","ograve:\xC3\xB2","ograve;:\xC3\xB2", +"ogt;:\xE2\xA7\x81","ohbar;:\xE2\xA6\xB5","ohm;:\xCE\xA9","oint;:\xE2\x88\xAE", +"olarr;:\xE2\x86\xBA","olcir;:\xE2\xA6\xBE","olcross;:\xE2\xA6\xBB", +"oline;:\xE2\x80\xBE","olt;:\xE2\xA7\x80","omacr;:\xC5\x8D","omega;:\xCF\x89", +"omicron;:\xCE\xBF","omid;:\xE2\xA6\xB6","ominus;:\xE2\x8A\x96", +"oopf;:\xF0\x9D\x95\xA0","opar;:\xE2\xA6\xB7","operp;:\xE2\xA6\xB9", +"oplus;:\xE2\x8A\x95","or;:\xE2\x88\xA8","orarr;:\xE2\x86\xBB", +"ord;:\xE2\xA9\x9D","order;:\xE2\x84\xB4","orderof;:\xE2\x84\xB4", +"ordf:\xC2\xAA","ordf;:\xC2\xAA","ordm:\xC2\xBA","ordm;:\xC2\xBA", +"origof;:\xE2\x8A\xB6","oror;:\xE2\xA9\x96","orslope;:\xE2\xA9\x97", +"orv;:\xE2\xA9\x9B","oscr;:\xE2\x84\xB4","oslash:\xC3\xB8","oslash;:\xC3\xB8", +"osol;:\xE2\x8A\x98","otilde:\xC3\xB5","otilde;:\xC3\xB5", +"otimes;:\xE2\x8A\x97","otimesas;:\xE2\xA8\xB6","ouml:\xC3\xB6", +"ouml;:\xC3\xB6","ovbar;:\xE2\x8C\xBD","par;:\xE2\x88\xA5","para:\xC2\xB6", +"para;:\xC2\xB6","parallel;:\xE2\x88\xA5","parsim;:\xE2\xAB\xB3", +"parsl;:\xE2\xAB\xBD","part;:\xE2\x88\x82","pcy;:\xD0\xBF","percnt;:%", +"period;:.","permil;:\xE2\x80\xB0","perp;:\xE2\x8A\xA5","pertenk;:\xE2\x80\xB1", +"pfr;:\xF0\x9D\x94\xAD","phi;:\xCF\x86","phiv;:\xCF\x95","phmmat;:\xE2\x84\xB3", +"phone;:\xE2\x98\x8E","pi;:\xCF\x80","pitchfork;:\xE2\x8B\x94","piv;:\xCF\x96", +"planck;:\xE2\x84\x8F","planckh;:\xE2\x84\x8E","plankv;:\xE2\x84\x8F","plus;:+", +"plusacir;:\xE2\xA8\xA3","plusb;:\xE2\x8A\x9E","pluscir;:\xE2\xA8\xA2", +"plusdo;:\xE2\x88\x94","plusdu;:\xE2\xA8\xA5","pluse;:\xE2\xA9\xB2", +"plusmn:\xC2\xB1","plusmn;:\xC2\xB1","plussim;:\xE2\xA8\xA6", +"plustwo;:\xE2\xA8\xA7","pm;:\xC2\xB1","pointint;:\xE2\xA8\x95", +"popf;:\xF0\x9D\x95\xA1","pound:\xC2\xA3","pound;:\xC2\xA3","pr;:\xE2\x89\xBA", +"prE;:\xE2\xAA\xB3","prap;:\xE2\xAA\xB7","prcue;:\xE2\x89\xBC", +"pre;:\xE2\xAA\xAF","prec;:\xE2\x89\xBA","precapprox;:\xE2\xAA\xB7", +"preccurlyeq;:\xE2\x89\xBC","preceq;:\xE2\xAA\xAF","precnapprox;:\xE2\xAA\xB9", +"precneqq;:\xE2\xAA\xB5","precnsim;:\xE2\x8B\xA8","precsim;:\xE2\x89\xBE", +"prime;:\xE2\x80\xB2","primes;:\xE2\x84\x99","prnE;:\xE2\xAA\xB5", +"prnap;:\xE2\xAA\xB9","prnsim;:\xE2\x8B\xA8","prod;:\xE2\x88\x8F", +"profalar;:\xE2\x8C\xAE","profline;:\xE2\x8C\x92","profsurf;:\xE2\x8C\x93", +"prop;:\xE2\x88\x9D","propto;:\xE2\x88\x9D","prsim;:\xE2\x89\xBE", +"prurel;:\xE2\x8A\xB0","pscr;:\xF0\x9D\x93\x85","psi;:\xCF\x88", +"puncsp;:\xE2\x80\x88","qfr;:\xF0\x9D\x94\xAE","qint;:\xE2\xA8\x8C", +"qopf;:\xF0\x9D\x95\xA2","qprime;:\xE2\x81\x97","qscr;:\xF0\x9D\x93\x86", +"quaternions;:\xE2\x84\x8D","quatint;:\xE2\xA8\x96","quest;:?", +"questeq;:\xE2\x89\x9F","quot:\"","quot;:\"","rAarr;:\xE2\x87\x9B", +"rArr;:\xE2\x87\x92","rAtail;:\xE2\xA4\x9C","rBarr;:\xE2\xA4\x8F", +"rHar;:\xE2\xA5\xA4","race;:\xE2\x88\xBD\xCC\xB1","racute;:\xC5\x95", +"radic;:\xE2\x88\x9A","raemptyv;:\xE2\xA6\xB3","rang;:\xE2\x9F\xA9", +"rangd;:\xE2\xA6\x92","range;:\xE2\xA6\xA5","rangle;:\xE2\x9F\xA9", +"raquo:\xC2\xBB","raquo;:\xC2\xBB","rarr;:\xE2\x86\x92","rarrap;:\xE2\xA5\xB5", +"rarrb;:\xE2\x87\xA5","rarrbfs;:\xE2\xA4\xA0","rarrc;:\xE2\xA4\xB3", +"rarrfs;:\xE2\xA4\x9E","rarrhk;:\xE2\x86\xAA","rarrlp;:\xE2\x86\xAC", +"rarrpl;:\xE2\xA5\x85","rarrsim;:\xE2\xA5\xB4","rarrtl;:\xE2\x86\xA3", +"rarrw;:\xE2\x86\x9D","ratail;:\xE2\xA4\x9A","ratio;:\xE2\x88\xB6", +"rationals;:\xE2\x84\x9A","rbarr;:\xE2\xA4\x8D","rbbrk;:\xE2\x9D\xB3", +"rbrace;:}","rbrack;:]","rbrke;:\xE2\xA6\x8C","rbrksld;:\xE2\xA6\x8E", +"rbrkslu;:\xE2\xA6\x90","rcaron;:\xC5\x99","rcedil;:\xC5\x97", +"rceil;:\xE2\x8C\x89","rcub;:}","rcy;:\xD1\x80","rdca;:\xE2\xA4\xB7", +"rdldhar;:\xE2\xA5\xA9","rdquo;:\xE2\x80\x9D","rdquor;:\xE2\x80\x9D", +"rdsh;:\xE2\x86\xB3","real;:\xE2\x84\x9C","realine;:\xE2\x84\x9B", +"realpart;:\xE2\x84\x9C","reals;:\xE2\x84\x9D","rect;:\xE2\x96\xAD", +"reg:\xC2\xAE","reg;:\xC2\xAE","rfisht;:\xE2\xA5\xBD","rfloor;:\xE2\x8C\x8B", +"rfr;:\xF0\x9D\x94\xAF","rhard;:\xE2\x87\x81","rharu;:\xE2\x87\x80", +"rharul;:\xE2\xA5\xAC","rho;:\xCF\x81","rhov;:\xCF\xB1", +"rightarrow;:\xE2\x86\x92","rightarrowtail;:\xE2\x86\xA3", +"rightharpoondown;:\xE2\x87\x81","rightharpoonup;:\xE2\x87\x80", +"rightleftarrows;:\xE2\x87\x84","rightleftharpoons;:\xE2\x87\x8C", +"rightrightarrows;:\xE2\x87\x89","rightsquigarrow;:\xE2\x86\x9D", +"rightthreetimes;:\xE2\x8B\x8C","ring;:\xCB\x9A","risingdotseq;:\xE2\x89\x93", +"rlarr;:\xE2\x87\x84","rlhar;:\xE2\x87\x8C","rlm;:\xE2\x80\x8F", +"rmoust;:\xE2\x8E\xB1","rmoustache;:\xE2\x8E\xB1","rnmid;:\xE2\xAB\xAE", +"roang;:\xE2\x9F\xAD","roarr;:\xE2\x87\xBE","robrk;:\xE2\x9F\xA7", +"ropar;:\xE2\xA6\x86","ropf;:\xF0\x9D\x95\xA3","roplus;:\xE2\xA8\xAE", +"rotimes;:\xE2\xA8\xB5","rpar;:)","rpargt;:\xE2\xA6\x94", +"rppolint;:\xE2\xA8\x92","rrarr;:\xE2\x87\x89","rsaquo;:\xE2\x80\xBA", +"rscr;:\xF0\x9D\x93\x87","rsh;:\xE2\x86\xB1","rsqb;:]","rsquo;:\xE2\x80\x99", +"rsquor;:\xE2\x80\x99","rthree;:\xE2\x8B\x8C","rtimes;:\xE2\x8B\x8A", +"rtri;:\xE2\x96\xB9","rtrie;:\xE2\x8A\xB5","rtrif;:\xE2\x96\xB8", +"rtriltri;:\xE2\xA7\x8E","ruluhar;:\xE2\xA5\xA8","rx;:\xE2\x84\x9E", +"sacute;:\xC5\x9B","sbquo;:\xE2\x80\x9A","sc;:\xE2\x89\xBB","scE;:\xE2\xAA\xB4", +"scap;:\xE2\xAA\xB8","scaron;:\xC5\xA1","sccue;:\xE2\x89\xBD", +"sce;:\xE2\xAA\xB0","scedil;:\xC5\x9F","scirc;:\xC5\x9D","scnE;:\xE2\xAA\xB6", +"scnap;:\xE2\xAA\xBA","scnsim;:\xE2\x8B\xA9","scpolint;:\xE2\xA8\x93", +"scsim;:\xE2\x89\xBF","scy;:\xD1\x81","sdot;:\xE2\x8B\x85", +"sdotb;:\xE2\x8A\xA1","sdote;:\xE2\xA9\xA6","seArr;:\xE2\x87\x98", +"searhk;:\xE2\xA4\xA5","searr;:\xE2\x86\x98","searrow;:\xE2\x86\x98", +"sect:\xC2\xA7","sect;:\xC2\xA7","semi;:;","seswar;:\xE2\xA4\xA9", +"setminus;:\xE2\x88\x96","setmn;:\xE2\x88\x96","sext;:\xE2\x9C\xB6", +"sfr;:\xF0\x9D\x94\xB0","sfrown;:\xE2\x8C\xA2","sharp;:\xE2\x99\xAF", +"shchcy;:\xD1\x89","shcy;:\xD1\x88","shortmid;:\xE2\x88\xA3", +"shortparallel;:\xE2\x88\xA5","shy:\xC2\xAD","shy;:\xC2\xAD","sigma;:\xCF\x83", +"sigmaf;:\xCF\x82","sigmav;:\xCF\x82","sim;:\xE2\x88\xBC", +"simdot;:\xE2\xA9\xAA","sime;:\xE2\x89\x83","simeq;:\xE2\x89\x83", +"simg;:\xE2\xAA\x9E","simgE;:\xE2\xAA\xA0","siml;:\xE2\xAA\x9D", +"simlE;:\xE2\xAA\x9F","simne;:\xE2\x89\x86","simplus;:\xE2\xA8\xA4", +"simrarr;:\xE2\xA5\xB2","slarr;:\xE2\x86\x90","smallsetminus;:\xE2\x88\x96", +"smashp;:\xE2\xA8\xB3","smeparsl;:\xE2\xA7\xA4","smid;:\xE2\x88\xA3", +"smile;:\xE2\x8C\xA3","smt;:\xE2\xAA\xAA","smte;:\xE2\xAA\xAC", +"smtes;:\xE2\xAA\xAC\xEF\xB8\x80","softcy;:\xD1\x8C","sol;:/", +"solb;:\xE2\xA7\x84","solbar;:\xE2\x8C\xBF","sopf;:\xF0\x9D\x95\xA4", +"spades;:\xE2\x99\xA0","spadesuit;:\xE2\x99\xA0","spar;:\xE2\x88\xA5", +"sqcap;:\xE2\x8A\x93","sqcaps;:\xE2\x8A\x93\xEF\xB8\x80","sqcup;:\xE2\x8A\x94", +"sqcups;:\xE2\x8A\x94\xEF\xB8\x80","sqsub;:\xE2\x8A\x8F","sqsube;:\xE2\x8A\x91", +"sqsubset;:\xE2\x8A\x8F","sqsubseteq;:\xE2\x8A\x91","sqsup;:\xE2\x8A\x90", +"sqsupe;:\xE2\x8A\x92","sqsupset;:\xE2\x8A\x90","sqsupseteq;:\xE2\x8A\x92", +"squ;:\xE2\x96\xA1","square;:\xE2\x96\xA1","squarf;:\xE2\x96\xAA", +"squf;:\xE2\x96\xAA","srarr;:\xE2\x86\x92","sscr;:\xF0\x9D\x93\x88", +"ssetmn;:\xE2\x88\x96","ssmile;:\xE2\x8C\xA3","sstarf;:\xE2\x8B\x86", +"star;:\xE2\x98\x86","starf;:\xE2\x98\x85","straightepsilon;:\xCF\xB5", +"straightphi;:\xCF\x95","strns;:\xC2\xAF","sub;:\xE2\x8A\x82", +"subE;:\xE2\xAB\x85","subdot;:\xE2\xAA\xBD","sube;:\xE2\x8A\x86", +"subedot;:\xE2\xAB\x83","submult;:\xE2\xAB\x81","subnE;:\xE2\xAB\x8B", +"subne;:\xE2\x8A\x8A","subplus;:\xE2\xAA\xBF","subrarr;:\xE2\xA5\xB9", +"subset;:\xE2\x8A\x82","subseteq;:\xE2\x8A\x86","subseteqq;:\xE2\xAB\x85", +"subsetneq;:\xE2\x8A\x8A","subsetneqq;:\xE2\xAB\x8B","subsim;:\xE2\xAB\x87", +"subsub;:\xE2\xAB\x95","subsup;:\xE2\xAB\x93","succ;:\xE2\x89\xBB", +"succapprox;:\xE2\xAA\xB8","succcurlyeq;:\xE2\x89\xBD","succeq;:\xE2\xAA\xB0", +"succnapprox;:\xE2\xAA\xBA","succneqq;:\xE2\xAA\xB6","succnsim;:\xE2\x8B\xA9", +"succsim;:\xE2\x89\xBF","sum;:\xE2\x88\x91","sung;:\xE2\x99\xAA", +"sup1:\xC2\xB9","sup1;:\xC2\xB9","sup2:\xC2\xB2","sup2;:\xC2\xB2", +"sup3:\xC2\xB3","sup3;:\xC2\xB3","sup;:\xE2\x8A\x83","supE;:\xE2\xAB\x86", +"supdot;:\xE2\xAA\xBE","supdsub;:\xE2\xAB\x98","supe;:\xE2\x8A\x87", +"supedot;:\xE2\xAB\x84","suphsol;:\xE2\x9F\x89","suphsub;:\xE2\xAB\x97", +"suplarr;:\xE2\xA5\xBB","supmult;:\xE2\xAB\x82","supnE;:\xE2\xAB\x8C", +"supne;:\xE2\x8A\x8B","supplus;:\xE2\xAB\x80","supset;:\xE2\x8A\x83", +"supseteq;:\xE2\x8A\x87","supseteqq;:\xE2\xAB\x86","supsetneq;:\xE2\x8A\x8B", +"supsetneqq;:\xE2\xAB\x8C","supsim;:\xE2\xAB\x88","supsub;:\xE2\xAB\x94", +"supsup;:\xE2\xAB\x96","swArr;:\xE2\x87\x99","swarhk;:\xE2\xA4\xA6", +"swarr;:\xE2\x86\x99","swarrow;:\xE2\x86\x99","swnwar;:\xE2\xA4\xAA", +"szlig:\xC3\x9F","szlig;:\xC3\x9F","target;:\xE2\x8C\x96","tau;:\xCF\x84", +"tbrk;:\xE2\x8E\xB4","tcaron;:\xC5\xA5","tcedil;:\xC5\xA3","tcy;:\xD1\x82", +"tdot;:\xE2\x83\x9B","telrec;:\xE2\x8C\x95","tfr;:\xF0\x9D\x94\xB1", +"there4;:\xE2\x88\xB4","therefore;:\xE2\x88\xB4","theta;:\xCE\xB8", +"thetasym;:\xCF\x91","thetav;:\xCF\x91","thickapprox;:\xE2\x89\x88", +"thicksim;:\xE2\x88\xBC","thinsp;:\xE2\x80\x89","thkap;:\xE2\x89\x88", +"thksim;:\xE2\x88\xBC","thorn:\xC3\xBE","thorn;:\xC3\xBE","tilde;:\xCB\x9C", +"times:\xC3\x97","times;:\xC3\x97","timesb;:\xE2\x8A\xA0", +"timesbar;:\xE2\xA8\xB1","timesd;:\xE2\xA8\xB0","tint;:\xE2\x88\xAD", +"toea;:\xE2\xA4\xA8","top;:\xE2\x8A\xA4","topbot;:\xE2\x8C\xB6", +"topcir;:\xE2\xAB\xB1","topf;:\xF0\x9D\x95\xA5","topfork;:\xE2\xAB\x9A", +"tosa;:\xE2\xA4\xA9","tprime;:\xE2\x80\xB4","trade;:\xE2\x84\xA2", +"triangle;:\xE2\x96\xB5","triangledown;:\xE2\x96\xBF", +"triangleleft;:\xE2\x97\x83","trianglelefteq;:\xE2\x8A\xB4", +"triangleq;:\xE2\x89\x9C","triangleright;:\xE2\x96\xB9", +"trianglerighteq;:\xE2\x8A\xB5","tridot;:\xE2\x97\xAC","trie;:\xE2\x89\x9C", +"triminus;:\xE2\xA8\xBA","triplus;:\xE2\xA8\xB9","trisb;:\xE2\xA7\x8D", +"tritime;:\xE2\xA8\xBB","trpezium;:\xE2\x8F\xA2","tscr;:\xF0\x9D\x93\x89", +"tscy;:\xD1\x86","tshcy;:\xD1\x9B","tstrok;:\xC5\xA7","twixt;:\xE2\x89\xAC", +"twoheadleftarrow;:\xE2\x86\x9E","twoheadrightarrow;:\xE2\x86\xA0", +"uArr;:\xE2\x87\x91","uHar;:\xE2\xA5\xA3","uacute:\xC3\xBA","uacute;:\xC3\xBA", +"uarr;:\xE2\x86\x91","ubrcy;:\xD1\x9E","ubreve;:\xC5\xAD","ucirc:\xC3\xBB", +"ucirc;:\xC3\xBB","ucy;:\xD1\x83","udarr;:\xE2\x87\x85","udblac;:\xC5\xB1", +"udhar;:\xE2\xA5\xAE","ufisht;:\xE2\xA5\xBE","ufr;:\xF0\x9D\x94\xB2", +"ugrave:\xC3\xB9","ugrave;:\xC3\xB9","uharl;:\xE2\x86\xBF", +"uharr;:\xE2\x86\xBE","uhblk;:\xE2\x96\x80","ulcorn;:\xE2\x8C\x9C", +"ulcorner;:\xE2\x8C\x9C","ulcrop;:\xE2\x8C\x8F","ultri;:\xE2\x97\xB8", +"umacr;:\xC5\xAB","uml:\xC2\xA8","uml;:\xC2\xA8","uogon;:\xC5\xB3", +"uopf;:\xF0\x9D\x95\xA6","uparrow;:\xE2\x86\x91","updownarrow;:\xE2\x86\x95", +"upharpoonleft;:\xE2\x86\xBF","upharpoonright;:\xE2\x86\xBE", +"uplus;:\xE2\x8A\x8E","upsi;:\xCF\x85","upsih;:\xCF\x92","upsilon;:\xCF\x85", +"upuparrows;:\xE2\x87\x88","urcorn;:\xE2\x8C\x9D","urcorner;:\xE2\x8C\x9D", +"urcrop;:\xE2\x8C\x8E","uring;:\xC5\xAF","urtri;:\xE2\x97\xB9", +"uscr;:\xF0\x9D\x93\x8A","utdot;:\xE2\x8B\xB0","utilde;:\xC5\xA9", +"utri;:\xE2\x96\xB5","utrif;:\xE2\x96\xB4","uuarr;:\xE2\x87\x88", +"uuml:\xC3\xBC","uuml;:\xC3\xBC","uwangle;:\xE2\xA6\xA7","vArr;:\xE2\x87\x95", +"vBar;:\xE2\xAB\xA8","vBarv;:\xE2\xAB\xA9","vDash;:\xE2\x8A\xA8", +"vangrt;:\xE2\xA6\x9C","varepsilon;:\xCF\xB5","varkappa;:\xCF\xB0", +"varnothing;:\xE2\x88\x85","varphi;:\xCF\x95","varpi;:\xCF\x96", +"varpropto;:\xE2\x88\x9D","varr;:\xE2\x86\x95","varrho;:\xCF\xB1", +"varsigma;:\xCF\x82","varsubsetneq;:\xE2\x8A\x8A\xEF\xB8\x80", +"varsubsetneqq;:\xE2\xAB\x8B\xEF\xB8\x80", +"varsupsetneq;:\xE2\x8A\x8B\xEF\xB8\x80", +"varsupsetneqq;:\xE2\xAB\x8C\xEF\xB8\x80","vartheta;:\xCF\x91", +"vartriangleleft;:\xE2\x8A\xB2","vartriangleright;:\xE2\x8A\xB3", +"vcy;:\xD0\xB2","vdash;:\xE2\x8A\xA2","vee;:\xE2\x88\xA8", +"veebar;:\xE2\x8A\xBB","veeeq;:\xE2\x89\x9A","vellip;:\xE2\x8B\xAE","verbar;:|", +"vert;:|","vfr;:\xF0\x9D\x94\xB3","vltri;:\xE2\x8A\xB2", +"vnsub;:\xE2\x8A\x82\xE2\x83\x92","vnsup;:\xE2\x8A\x83\xE2\x83\x92", +"vopf;:\xF0\x9D\x95\xA7","vprop;:\xE2\x88\x9D","vrtri;:\xE2\x8A\xB3", +"vscr;:\xF0\x9D\x93\x8B","vsubnE;:\xE2\xAB\x8B\xEF\xB8\x80", +"vsubne;:\xE2\x8A\x8A\xEF\xB8\x80","vsupnE;:\xE2\xAB\x8C\xEF\xB8\x80", +"vsupne;:\xE2\x8A\x8B\xEF\xB8\x80","vzigzag;:\xE2\xA6\x9A","wcirc;:\xC5\xB5", +"wedbar;:\xE2\xA9\x9F","wedge;:\xE2\x88\xA7","wedgeq;:\xE2\x89\x99", +"weierp;:\xE2\x84\x98","wfr;:\xF0\x9D\x94\xB4","wopf;:\xF0\x9D\x95\xA8", +"wp;:\xE2\x84\x98","wr;:\xE2\x89\x80","wreath;:\xE2\x89\x80", +"wscr;:\xF0\x9D\x93\x8C","xcap;:\xE2\x8B\x82","xcirc;:\xE2\x97\xAF", +"xcup;:\xE2\x8B\x83","xdtri;:\xE2\x96\xBD","xfr;:\xF0\x9D\x94\xB5", +"xhArr;:\xE2\x9F\xBA","xharr;:\xE2\x9F\xB7","xi;:\xCE\xBE", +"xlArr;:\xE2\x9F\xB8","xlarr;:\xE2\x9F\xB5","xmap;:\xE2\x9F\xBC", +"xnis;:\xE2\x8B\xBB","xodot;:\xE2\xA8\x80","xopf;:\xF0\x9D\x95\xA9", +"xoplus;:\xE2\xA8\x81","xotime;:\xE2\xA8\x82","xrArr;:\xE2\x9F\xB9", +"xrarr;:\xE2\x9F\xB6","xscr;:\xF0\x9D\x93\x8D","xsqcup;:\xE2\xA8\x86", +"xuplus;:\xE2\xA8\x84","xutri;:\xE2\x96\xB3","xvee;:\xE2\x8B\x81", +"xwedge;:\xE2\x8B\x80","yacute:\xC3\xBD","yacute;:\xC3\xBD","yacy;:\xD1\x8F", +"ycirc;:\xC5\xB7","ycy;:\xD1\x8B","yen:\xC2\xA5","yen;:\xC2\xA5", +"yfr;:\xF0\x9D\x94\xB6","yicy;:\xD1\x97","yopf;:\xF0\x9D\x95\xAA", +"yscr;:\xF0\x9D\x93\x8E","yucy;:\xD1\x8E","yuml:\xC3\xBF","yuml;:\xC3\xBF", +"zacute;:\xC5\xBA","zcaron;:\xC5\xBE","zcy;:\xD0\xB7","zdot;:\xC5\xBC", +"zeetrf;:\xE2\x84\xA8","zeta;:\xCE\xB6","zfr;:\xF0\x9D\x94\xB7", +"zhcy;:\xD0\xB6","zigrarr;:\xE2\x87\x9D","zopf;:\xF0\x9D\x95\xAB", +"zscr;:\xF0\x9D\x93\x8F","zwj;:\xE2\x80\x8D","zwnj;:\xE2\x80\x8C", +] + +const charMap*: array[char, int] = [ +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,26,38,74,128,158,166,188,200,229,236,244, +304,313,385,414,433,438,483,523,546,586,603,608,612,623,633,-1,-1,-1,-1,-1,-1, +702,818,917,983,1050,1089,1149,1177,1233,1241,1251,1405,1445,1613,1674,1743, +1754,1858,2016,2074,2126,2168,2179,2203,2217,2230,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +] diff --git a/lib/chame0/chame/htmlparser.nim b/lib/chame0/chame/htmlparser.nim new file mode 100644 index 00000000..87ab81f4 --- /dev/null +++ b/lib/chame0/chame/htmlparser.nim @@ -0,0 +1,2727 @@ +import std/macros +import std/options +import std/strutils +import std/tables + +import dombuilder +import htmltokenizer +import tags +import tokstate + +# Generics break without exporting macros. Maybe a compiler bug? +export macros + +# Export these so that htmlparseriface works seamlessly. +export dombuilder +export options +export tags + +# Export tokstate too; it is needed for fragment parsing. +export tokstate + +# Heavily inspired by html5ever's TreeSink design. +type + HTML5ParserOpts*[Handle, Atom] = object + isIframeSrcdoc*: bool + ## Is the document an iframe srcdoc? + scripting*: bool + ## Is scripting enabled for this document? + ctx*: Option[OpenElementInit[Handle, Atom]] + ## Context element for fragment parsing. When set to some Handle, + ## the fragment case is used while parsing. + ## + ## `token` must be a valid starting token for this element. + initialTokenizerState*: TokenizerState + ## The initial tokenizer state; by default, this is DATA. + openElementsInit*: seq[OpenElementInit[Handle, Atom]] + ## Initial state of the stack of open elements. By default, the stack + ## starts out empty. + ## Note: if this is initialized to a non-empty sequence, the parser will + ## start by resetting the insertion mode appropriately. + formInit*: Option[Handle] + ## Initial state of the parser's form pointer. + pushInTemplate*: bool + ## When set to true, the "in template" insertion mode is pushed to the + ## stack of template insertion modes on parser start. + + OpenElement*[Handle, Atom] = tuple + element: Handle + token: Token[Atom] ## the element's start tag token; must not be nil. + + OpenElementInit*[Handle, Atom] = tuple + element: Handle + startTagName: Atom ## the element's start tag token's name. + +type + QualifiedName[Atom] = tuple + prefix: NamespacePrefix + namespace: Namespace + localName: Atom + + HTML5Parser*[Handle, Atom] = object + quirksMode: QuirksMode + dombuilder: DOMBuilder[Handle, Atom] + opts: HTML5ParserOpts[Handle, Atom] + ctx: Option[OpenElement[Handle, Atom]] + openElements: seq[OpenElement[Handle, Atom]] + insertionMode: InsertionMode + oldInsertionMode: InsertionMode + templateModes: seq[InsertionMode] + head: Option[OpenElement[Handle, Atom]] + tokenizer: Tokenizer[Handle, Atom] + form: Option[Handle] + fosterParenting: bool + # Handle is an element. nil => marker + activeFormatting: seq[(Option[Handle], Token[Atom])] + framesetOk: bool + ignoreLF: bool + pendingTableChars: string + pendingTableCharsWhitespace: bool + caseTable: Table[Atom, Atom] + adjustedTable: Table[Atom, Atom] + foreignTable: Table[Atom, QualifiedName[Atom]] + + AdjustedInsertionLocation[Handle] = tuple[ + inside: Handle, + before: Option[Handle] + ] + +# 13.2.4.1 + InsertionMode = enum + INITIAL, BEFORE_HTML, BEFORE_HEAD, IN_HEAD, IN_HEAD_NOSCRIPT, AFTER_HEAD, + IN_BODY, TEXT, IN_TABLE, IN_TABLE_TEXT, IN_CAPTION, IN_COLUMN_GROUP, + IN_TABLE_BODY, IN_ROW, IN_CELL, IN_SELECT, IN_SELECT_IN_TABLE, IN_TEMPLATE, + AFTER_BODY, IN_FRAMESET, AFTER_FRAMESET, AFTER_AFTER_BODY, + AFTER_AFTER_FRAMESET + +type ParseResult* = enum + ## Result of parsing the passed chunk. + ## PRES_CONTINUE is returned when it is OK to continue parsing. + ## + ## PRES_STOP is returned when the parser has been stopped from + ## setEncodingImpl. + ## + ## PRES_SCRIPT is returned when a script end tag is encountered. For + ## implementations that do not support scripting, this can be treated + ## equivalently to PRES_CONTINUE. + ## + ## Implementations that *do* support scripting and implement `document.write` + ## can instead use PRES_SCRIPT to process string injected into the input + ## stream by `document.write` before continuing with parsing from the + ## network stream. In this case, script elements should be stored in e.g. the + ## DOM builder from `elementPoppedImpl`, and processed accordingly after + ## PRES_SCRIPT has been returned. + PRES_CONTINUE + PRES_STOP + PRES_SCRIPT + +# DOMBuilder interface functions +proc strToAtom[Handle, Atom](parser: HTML5Parser[Handle, Atom], s: string): + Atom = + mixin strToAtomImpl + return parser.dombuilder.strToAtomImpl(s) + +proc tagTypeToAtom[Handle, Atom](parser: HTML5Parser[Handle, Atom], + tagType: TagType): Atom = + mixin tagTypeToAtomImpl + return parser.dombuilder.tagTypeToAtomImpl(tagType) + +# Declared as extern, because Nim 1.6.10 does not recognize it in macros +# otherwise. +proc atomToTagType*[Handle, Atom](parser: HTML5Parser[Handle, Atom], + atom: Atom): TagType = + mixin atomToTagTypeImpl + return parser.dombuilder.atomToTagTypeImpl(atom) + +proc setQuirksMode[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + mode: QuirksMode) = + mixin setQuirksModeImpl + parser.quirksMode = mode + when compiles(parser.dombuilder.setQuirksModeImpl(mode)): + parser.dombuilder.setQuirksModeImpl(mode) + +proc setEncoding(parser: var HTML5Parser, cs: string): SetEncodingResult = + mixin setEncodingImpl + when compiles(parser.dombuilder.setEncodingImpl(cs)): + return parser.dombuilder.setEncodingImpl(cs) + else: + return SET_ENCODING_CONTINUE + +proc getDocument[Handle, Atom](parser: HTML5Parser[Handle, Atom]): Handle = + mixin getDocumentImpl + return parser.dombuilder.getDocumentImpl() + +proc getParentNode[Handle, Atom](parser: HTML5Parser[Handle, Atom], + handle: Handle): Option[Handle] = + mixin getParentNodeImpl + return parser.dombuilder.getParentNodeImpl(handle) + +proc getLocalName[Handle, Atom](parser: HTML5Parser[Handle, Atom], + handle: Handle): Atom = + mixin getLocalNameImpl + return parser.dombuilder.getLocalNameImpl(handle) + +proc getNamespace[Handle, Atom](parser: HTML5Parser[Handle, Atom], + handle: Handle): Namespace = + mixin getNamespaceImpl + return parser.dombuilder.getNamespaceImpl(handle) + +proc getTemplateContent[Handle, Atom](parser: HTML5Parser[Handle, Atom], + handle: Handle): Handle = + mixin getTemplateContentImpl + return parser.dombuilder.getTemplateContentImpl(handle) + +proc getTagType[Handle, Atom](parser: HTML5Parser[Handle, Atom], + handle: Handle): TagType = + if parser.getNamespace(handle) != Namespace.HTML: + return TAG_UNKNOWN + return parser.atomToTagType(parser.getLocalName(handle)) + +proc createHTMLElement[Handle, Atom](parser: HTML5Parser[Handle, Atom]): + Handle = + mixin createHTMLElementImpl + return parser.dombuilder.createHTMLElementImpl() + +proc createComment[Handle, Atom](parser: HTML5Parser[Handle, Atom], + text: string): Handle = + mixin createCommentImpl + return parser.dombuilder.createCommentImpl(text) + +proc createDocumentType[Handle, Atom](parser: HTML5Parser[Handle, Atom], + name, publicId, systemId: string): Handle = + mixin createDocumentTypeImpl + return parser.dombuilder.createDocumentTypeImpl(name, publicId, systemId) + +proc insertBefore[Handle, Atom](parser: HTML5Parser[Handle, Atom], + parent, child: Handle, before: Option[Handle]) = + mixin insertBeforeImpl + parser.dombuilder.insertBeforeImpl(parent, child, before) + +proc insertText[Handle, Atom](parser: HTML5Parser[Handle, Atom], parent: Handle, + text: string, before: Option[Handle]) = + mixin insertTextImpl + parser.dombuilder.insertTextImpl(parent, text, before) + +proc remove[Handle, Atom](parser: HTML5Parser[Handle, Atom], child: Handle) = + mixin removeImpl + parser.dombuilder.removeImpl(child) + +proc moveChildren[Handle, Atom](parser: HTML5Parser[Handle, Atom], handleFrom, + handleTo: Handle) = + mixin moveChildrenImpl + parser.dombuilder.moveChildrenImpl(handleFrom, handleTo) + +proc addAttrsIfMissing[Handle, Atom](parser: HTML5Parser[Handle, Atom], + element: Handle, attrs: Table[Atom, string]) = + mixin addAttrsIfMissingImpl + parser.dombuilder.addAttrsIfMissingImpl(element, attrs) + +proc setScriptAlreadyStarted[Handle, Atom](parser: HTML5Parser[Handle, Atom], + script: Handle) = + mixin setScriptAlreadyStartedImpl + when compiles(parser.dombuilder.setScriptAlreadyStartedImpl(script)): + parser.dombuilder.setScriptAlreadyStartedImpl(script) + +proc associateWithForm[Handle, Atom](parser: HTML5Parser[Handle, Atom], + element, form, intendedParent: Handle) = + mixin associateWithFormImpl + when compiles(parser.dombuilder.associateWithFormImpl(element, form, + intendedParent)): + parser.dombuilder.associateWithFormImpl(element, form, intendedParent) + +# Parser +func fragment(parser: HTML5Parser): bool = + return parser.ctx.isSome + +# https://html.spec.whatwg.org/multipage/parsing.html#reset-the-insertion-mode-appropriately +proc resetInsertionMode(parser: var HTML5Parser) = + template switch_insertion_mode_and_return(mode: InsertionMode) = + parser.insertionMode = mode + return + for i in countdown(parser.openElements.high, 0): + var node = parser.openElements[i] + let last = i == 0 + if parser.fragment: + node = parser.ctx.get + let tagType = parser.getTagType(node.element) + case tagType + of TAG_SELECT: + if not last: + for j in countdown(parser.openElements.high, 1): + let ancestor = parser.openElements[j].element + case parser.getTagType(ancestor) + of TAG_TEMPLATE: break + of TAG_TABLE: switch_insertion_mode_and_return IN_SELECT_IN_TABLE + else: discard + switch_insertion_mode_and_return IN_SELECT + of TAG_TD, TAG_TH: + if not last: + switch_insertion_mode_and_return IN_CELL + of TAG_TR: switch_insertion_mode_and_return IN_ROW + of TAG_TBODY, TAG_THEAD, TAG_TFOOT: + switch_insertion_mode_and_return IN_TABLE_BODY + of TAG_CAPTION: + switch_insertion_mode_and_return IN_CAPTION + of TAG_COLGROUP: switch_insertion_mode_and_return IN_COLUMN_GROUP + of TAG_TABLE: switch_insertion_mode_and_return IN_TABLE + of TAG_TEMPLATE: switch_insertion_mode_and_return parser.templateModes[^1] + of TAG_HEAD: + if not last: + switch_insertion_mode_and_return IN_HEAD + of TAG_BODY: switch_insertion_mode_and_return IN_BODY + of TAG_FRAMESET: switch_insertion_mode_and_return IN_FRAMESET + of TAG_HTML: + if parser.head.isNone: + switch_insertion_mode_and_return BEFORE_HEAD + else: + switch_insertion_mode_and_return AFTER_HEAD + else: discard + if last: + switch_insertion_mode_and_return IN_BODY + +func currentNodeToken[Handle, Atom](parser: HTML5Parser[Handle, Atom]): + OpenElement[Handle, Atom] = + return parser.openElements[^1] + +func currentNode[Handle, Atom](parser: HTML5Parser[Handle, Atom]): Handle = + return parser.currentNodeToken.element + +func currentToken[Handle, Atom](parser: HTML5Parser[Handle, Atom]): + Token[Atom] = + return parser.currentNodeToken.token + +func adjustedCurrentNodeToken[Handle, Atom](parser: HTML5Parser[Handle, Atom]): + OpenElement[Handle, Atom] = + if parser.fragment and parser.openElements.len == 1: + return parser.ctx.get + else: + return parser.currentNodeToken + +func adjustedCurrentNode[Handle, Atom](parser: HTML5Parser[Handle, Atom]): + Handle = + return parser.adjustedCurrentNodeToken.element + +proc lastElementOfTag[Handle, Atom](parser: HTML5Parser[Handle, Atom], + tagType: TagType): tuple[element: Option[Handle], pos: int] = + for i in countdown(parser.openElements.high, 0): + let element = parser.openElements[i].element + if parser.getTagType(element) == tagType: + return (some(element), i) + return (none(Handle), -1) + +func last_child_of[Handle](n: Handle): AdjustedInsertionLocation[Handle] = + (n, none(Handle)) + +func last_child_of[Handle, Atom](n: OpenElement[Handle, Atom]): + AdjustedInsertionLocation[Handle] = + last_child_of(n.element) + +# https://html.spec.whatwg.org/multipage/#appropriate-place-for-inserting-a-node +proc appropriatePlaceForInsert[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: Handle): AdjustedInsertionLocation[Handle] = + assert parser.getTagType(parser.openElements[0].element) == TAG_HTML + let targetTagType = parser.getTagType(target) + const FosterTagTypes = {TAG_TABLE, TAG_TBODY, TAG_TFOOT, TAG_THEAD, TAG_TR} + if parser.fosterParenting and targetTagType in FosterTagTypes: + let lastTemplate = parser.lastElementOfTag(TAG_TEMPLATE) + let lastTable = parser.lastElementOfTag(TAG_TABLE) + if lastTemplate.element.isSome and + (lastTable.element.isNone or lastTable.pos < lastTemplate.pos): + let content = parser.getTemplateContent(lastTemplate.element.get) + return last_child_of(content) + if lastTable.element.isNone: + return last_child_of(parser.openElements[0].element) + let parentNode = parser.getParentNode(lastTable.element.get) + if parentNode.isSome: + return (parentNode.get, lastTable.element) + let previousElement = parser.openElements[lastTable.pos - 1] + result = last_child_of(previousElement.element) + else: + result = last_child_of(target) + if parser.getTagType(result.inside) == TAG_TEMPLATE: + result = (parser.getTemplateContent(result.inside), none(Handle)) + +proc appropriatePlaceForInsert[Handle, Atom](parser: HTML5Parser[Handle, Atom]): + AdjustedInsertionLocation[Handle] = + parser.appropriatePlaceForInsert(parser.currentNode) + +proc hasElement[Handle, Atom](parser: HTML5Parser[Handle, Atom], + tags: set[TagType]): bool = + for (element, _) in parser.openElements: + if parser.getTagType(element) in tags: + return true + return false + +proc hasElement[Handle, Atom](parser: HTML5Parser[Handle, Atom], + tag: TagType): bool = + return parser.hasElement({tag}) + +const Scope = { + TAG_APPLET, TAG_CAPTION, TAG_HTML, TAG_TABLE, TAG_TD, TAG_TH, TAG_MARQUEE, + TAG_OBJECT, TAG_TEMPLATE # (+ SVG, MathML) +} + +proc hasElementInScopeWithXML[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: Handle, list: set[TagType]): bool = + for i in countdown(parser.openElements.high, 0): + let element = parser.openElements[i].element + if element == target: + return true + let localName = parser.getLocalName(element) + let tagType = parser.atomToTagType(localName) + case parser.getNamespace(element) + of Namespace.HTML: + {.linearScanEnd.} + if tagType in list: + return false + of Namespace.MATHML: + const elements = { + TAG_MI, TAG_MO, TAG_MN, TAG_MS, TAG_MTEXT, TAG_ANNOTATION_XML + } + if tagType in elements: + return false + of Namespace.SVG: + const elements = {TAG_FOREIGN_OBJECT, TAG_DESC, TAG_TITLE} + if tagType in elements: + return false + else: discard + return false + +proc hasElementInScopeWithXML[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: set[TagType], list: set[TagType]): bool = + for i in countdown(parser.openElements.high, 0): + let element = parser.openElements[i].element + let tagType = parser.atomToTagType(parser.getLocalName(element)) + case parser.getNamespace(element) + of Namespace.HTML: + if tagType in target: + return true + if tagType in list: + return false + of Namespace.MATHML: + const elements = { + TAG_MI, TAG_MO, TAG_MN, TAG_MS, TAG_MTEXT, TAG_ANNOTATION_XML + } + if tagType in elements: + return false + of Namespace.SVG: + const elements = {TAG_FOREIGN_OBJECT, TAG_DESC, TAG_TITLE} + if tagType in elements: + return false + else: discard + return false + +proc hasElementInScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: Handle): bool = + return parser.hasElementInScopeWithXML(target, Scope) + +proc hasElementInScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: set[TagType]): bool = + return parser.hasElementInScopeWithXML(target, Scope) + +proc hasElementInScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: TagType): bool = + return parser.hasElementInScope({target}) + +proc hasElementInScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + localName: Atom): bool = + let target = parser.atomToTagType(localName) + return parser.hasElementInScope(target) + +proc hasElementInListItemScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: TagType): bool = + const ListItemScope = Scope + {TAG_OL, TAG_UL} + return parser.hasElementInScopeWithXML({target}, ListItemScope) + +proc hasElementInButtonScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: TagType): bool = + const ButtonScope = Scope + {TAG_BUTTON} + return parser.hasElementInScopeWithXML({target}, ButtonScope) + +# Note: these do not include the "Scope" tags. +proc hasElementInSpecificScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: Handle, list: set[TagType]): bool = + for i in countdown(parser.openElements.high, 0): + let element = parser.openElements[i].element + if element == target: + return true + if parser.getTagType(element) in list: + return false + assert false + return false + +proc hasElementInSpecificScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: set[TagType], list: set[TagType]): bool = + for i in countdown(parser.openElements.high, 0): + let tagType = parser.getTagType(parser.openElements[i].element) + if tagType in target: + return true + if tagType in list: + return false + assert false + return false + +const TableScope = {TAG_HTML, TAG_TABLE, TAG_TEMPLATE} +proc hasElementInTableScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: TagType): bool = + return parser.hasElementInSpecificScope({target}, TableScope) + +proc hasElementInTableScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: set[TagType]): bool = + return parser.hasElementInSpecificScope(target, TableScope) + +proc hasElementInSelectScope[Handle, Atom](parser: HTML5Parser[Handle, Atom], + target: TagType): bool = + for i in countdown(parser.openElements.high, 0): + let tagType = parser.getTagType(parser.openElements[i].element) + if tagType == target: + return true + if tagType notin {TAG_OPTION, TAG_OPTGROUP}: + return false + assert false + return false + +proc createElementForToken[Handle, Atom](parser: HTML5Parser[Handle, Atom], + localName: Atom, namespace: Namespace, intendedParent: Handle, + htmlAttrs: Table[Atom, string], xmlAttrs: seq[ParsedAttr[Atom]]): Handle = + mixin createElementForTokenImpl + let element = parser.dombuilder.createElementForTokenImpl( + localName, namespace, intendedParent, htmlAttrs, xmlAttrs + ) + let tagType = parser.atomToTagType(localName) + if namespace == Namespace.HTML and tagType in FormAssociatedElements and + parser.form.isSome and not parser.hasElement(TAG_TEMPLATE) and + (tagType notin ListedElements or + parser.tagTypeToAtom(TAG_FORM) notin htmlAttrs): + parser.associateWithForm(element, parser.form.get, intendedParent) + return element + +proc createElementForToken[Handle, Atom](parser: HTML5Parser[Handle, Atom], + token: Token, namespace: Namespace, intendedParent: Handle): Handle = + # attrs not adjusted + return parser.createElementForToken(token.tagname, namespace, intendedParent, + token.attrs, @[]) + +proc createHTMLElementForToken[Handle, Atom](parser: HTML5Parser[Handle, Atom], + token: Token, intendedParent: Handle): Handle = + # attrs not adjusted + return parser.createElementForToken( + token.tagname, Namespace.HTML, intendedParent, token.attrs, @[] + ) + +proc pushElement[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + node: Handle, token: Token[Atom]) = + parser.openElements.add((node, token)) + let node = parser.adjustedCurrentNode() + parser.tokenizer.hasnonhtml = parser.getNamespace(node) != Namespace.HTML + +proc popElement[Handle, Atom](parser: var HTML5Parser[Handle, Atom]): Handle = + mixin elementPoppedImpl + result = parser.openElements.pop().element + when compiles(parser.dombuilder.elementPoppedImpl(result)): + parser.dombuilder.elementPoppedImpl(result) + if parser.openElements.len == 0: + parser.tokenizer.hasnonhtml = false + else: + let node = parser.adjustedCurrentNode() + parser.tokenizer.hasnonhtml = parser.getNamespace(node) != Namespace.HTML + +template pop_current_node = discard parser.popElement() + +proc insert[Handle, Atom](parser: HTML5Parser[Handle, Atom], + location: AdjustedInsertionLocation[Handle], node: Handle) = + parser.insertBefore(location.inside, node, location.before) + +proc append[Handle, Atom](parser: HTML5Parser[Handle, Atom], parent, node: Handle) = + parser.insertBefore(parent, node, none(Handle)) + +proc insertForeignElement[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token, localName: Atom, namespace: Namespace, stackOnly: bool, + xmlAttrs: seq[ParsedAttr[Atom]]): Handle = + let location = parser.appropriatePlaceForInsert() + let parent = location.inside + let element = parser.createElementForToken(localName, namespace, parent, + token.attrs, xmlAttrs) + if not stackOnly: + parser.insert(location, element) + parser.pushElement(element, token) + return element + +proc insertForeignElement[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token, namespace: Namespace, stackOnly: bool): Handle = + parser.insertForeignElement(token, token.tagname, namespace, stackOnly, @[]) + +proc insertHTMLElement[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token): Handle = + return parser.insertForeignElement(token, Namespace.HTML, false) + +# Note: adjustMathMLAttributes and adjustSVGAttributes both include the "adjust +# foreign attributes" step as well. +proc adjustMathMLAttributes[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + htmlAttrs: var Table[Atom, string], xmlAttrs: var seq[ParsedAttr[Atom]]) = + var deleted: seq[Atom] = @[] + for k, v in htmlAttrs.mpairs: + parser.foreignTable.withValue(k, p): + xmlAttrs.add((p[].prefix, p[].namespace, p[].localName, v)) + deleted.add(k) + var v: string = "" + if htmlAttrs.pop(parser.tagTypeToAtom(TAG_DEFINITION_URL), v): + htmlAttrs[parser.strToAtom("definitionURL")] = v + for k in deleted: + htmlAttrs.del(k) + +proc adjustSVGAttributes[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + htmlAttrs: var Table[Atom, string], xmlAttrs: var seq[ParsedAttr[Atom]]) = + var deleted: seq[Atom] = @[] + for k, v in htmlAttrs: + parser.foreignTable.withValue(k, p): + xmlAttrs.add((p[].prefix, p[].namespace, p[].localName, v)) + deleted.add(k) + for k, ak in parser.adjustedTable: + var v: string = "" + if htmlAttrs.pop(k, v): + htmlAttrs[ak] = v + for k in deleted: + htmlAttrs.del(k) + +proc insertCharacter(parser: var HTML5Parser, data: string) = + let location = parser.appropriatePlaceForInsert() + if location.inside == parser.getDocument(): + return + parser.insertText(location.inside, data, location.before) + +proc insertComment[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token, position: AdjustedInsertionLocation[Handle]) = + let comment = parser.createComment(token.data) + parser.insert(position, comment) + +proc insertComment(parser: var HTML5Parser, token: Token) = + let position = parser.appropriatePlaceForInsert() + parser.insertComment(token, position) + +const PublicIdentifierEquals = [ + "-//W3O//DTD W3 HTML Strict 3.0//EN//", + "-/W3C/DTD HTML 4.0 Transitional/EN", + "HTML" +] + +const PublicIdentifierStartsWith = [ + "+//Silmaril//dtd html Pro v0r11 19970101//", + "-//AS//DTD HTML 3.0 asWedit + extensions//", + "-//AdvaSoft Ltd//DTD HTML 3.0 asWedit + extensions//", + "-//IETF//DTD HTML 2.0 Level 1//", + "-//IETF//DTD HTML 2.0 Level 2//", + "-//IETF//DTD HTML 2.0 Strict Level 1//", + "-//IETF//DTD HTML 2.0 Strict Level 2//", + "-//IETF//DTD HTML 2.0 Strict//", + "-//IETF//DTD HTML 2.0//", + "-//IETF//DTD HTML 2.1E//", + "-//IETF//DTD HTML 3.0//", + "-//IETF//DTD HTML 3.2 Final//", + "-//IETF//DTD HTML 3.2//", + "-//IETF//DTD HTML 3//", + "-//IETF//DTD HTML Level 0//", + "-//IETF//DTD HTML Level 1//", + "-//IETF//DTD HTML Level 2//", + "-//IETF//DTD HTML Level 3//", + "-//IETF//DTD HTML Strict Level 0//", + "-//IETF//DTD HTML Strict Level 1//", + "-//IETF//DTD HTML Strict Level 2//", + "-//IETF//DTD HTML Strict Level 3//", + "-//IETF//DTD HTML Strict//", + "-//IETF//DTD HTML//", + "-//Metrius//DTD Metrius Presentational//", + "-//Microsoft//DTD Internet Explorer 2.0 HTML Strict//", + "-//Microsoft//DTD Internet Explorer 2.0 HTML//", + "-//Microsoft//DTD Internet Explorer 2.0 Tables//", + "-//Microsoft//DTD Internet Explorer 3.0 HTML Strict//", + "-//Microsoft//DTD Internet Explorer 3.0 HTML//", + "-//Microsoft//DTD Internet Explorer 3.0 Tables//", + "-//Netscape Comm. Corp.//DTD HTML//", + "-//Netscape Comm. Corp.//DTD Strict HTML//", + "-//O'Reilly and Associates//DTD HTML 2.0//", + "-//O'Reilly and Associates//DTD HTML Extended 1.0//", + "-//O'Reilly and Associates//DTD HTML Extended Relaxed 1.0//", + "-//SQ//DTD HTML 2.0 HoTMetaL + extensions//", + "-//SoftQuad Software//DTD HoTMetaL PRO 6.0::19990601::extensions to HTML 4.0//", + "-//SoftQuad//DTD HoTMetaL PRO 4.0::19971010::extensions to HTML 4.0//", + "-//Spyglass//DTD HTML 2.0 Extended//", + "-//Sun Microsystems Corp.//DTD HotJava HTML//", + "-//Sun Microsystems Corp.//DTD HotJava Strict HTML//", + "-//W3C//DTD HTML 3 1995-03-24//", + "-//W3C//DTD HTML 3.2 Draft//", + "-//W3C//DTD HTML 3.2 Final//", + "-//W3C//DTD HTML 3.2//", + "-//W3C//DTD HTML 3.2S Draft//", + "-//W3C//DTD HTML 4.0 Frameset//", + "-//W3C//DTD HTML 4.0 Transitional//", + "-//W3C//DTD HTML Experimental 19960712//", + "-//W3C//DTD HTML Experimental 970421//", + "-//W3C//DTD W3 HTML//", + "-//W3O//DTD W3 HTML 3.0//", + "-//WebTechs//DTD Mozilla HTML 2.0//", + "-//WebTechs//DTD Mozilla HTML//", +] + +const SystemIdentifierMissingAndPublicIdentifierStartsWith = [ + "-//W3C//DTD HTML 4.01 Frameset//", + "-//W3C//DTD HTML 4.01 Transitional//" +] + +const PublicIdentifierStartsWithLimited = [ + "-//W3C//DTD XHTML 1.0 Frameset//", + "-//W3C//DTD XHTML 1.0 Transitional//" +] + +const SystemIdentifierNotMissingAndPublicIdentifierStartsWith = [ + "-//W3C//DTD HTML 4.01 Frameset//", + "-//W3C//DTD HTML 4.01 Transitional//" +] + +func startsWithNoCase(str, prefix: string): bool = + if str.len < prefix.len: + return false + # prefix.len is always lower + var i = 0 + while i != prefix.len: + if str[i].toLowerAscii() != prefix[i].toLowerAscii(): + return false + inc i + true + +func equalsIgnoreCase(s1, s2: string): bool {.inline.} = + return s1.cmpIgnoreCase(s2) == 0 + +func quirksConditions(token: Token): bool = + if token.quirks: + return true + if token.name.get("") != "html": + return true + if token.sysid.get("") == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd": + return true + if token.pubid.isSome: + let pubid = token.pubid.get + for id in PublicIdentifierEquals: + if pubid.equalsIgnoreCase(id): + return true + for id in PublicIdentifierStartsWith: + if pubid.startsWithNoCase(id): + return true + if token.sysid.isNone: + for id in SystemIdentifierMissingAndPublicIdentifierStartsWith: + if pubid.startsWithNoCase(id): + return true + return false + +func limitedQuirksConditions(token: Token): bool = + if token.pubid.isNone: return false + for id in PublicIdentifierStartsWithLimited: + if token.pubid.get.startsWithNoCase(id): + return true + if token.sysid.isNone: return false + for id in SystemIdentifierNotMissingAndPublicIdentifierStartsWith: + if token.pubid.get.startsWithNoCase(id): + return true + return false + +# 13.2.6.2 +proc genericRawtextElementParsingAlgorithm(parser: var HTML5Parser, token: Token) = + discard parser.insertHTMLElement(token) + parser.tokenizer.state = RAWTEXT + parser.oldInsertionMode = parser.insertionMode + parser.insertionMode = TEXT + +proc genericRCDATAElementParsingAlgorithm(parser: var HTML5Parser, token: Token) = + discard parser.insertHTMLElement(token) + parser.tokenizer.state = RCDATA + parser.oldInsertionMode = parser.insertionMode + parser.insertionMode = TEXT + +# Pop all elements, including the specified tag. +proc popElementsIncl(parser: var HTML5Parser, tags: set[TagType]) = + while parser.getTagType(parser.popElement()) notin tags: + discard + +proc popElementsIncl(parser: var HTML5Parser, tag: TagType) = + parser.popElementsIncl({tag}) + +# Pop all elements, including the specified element. +proc popElementsIncl[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + handle: Handle) = + while parser.popElement() != handle: + discard + +# https://html.spec.whatwg.org/multipage/parsing.html#closing-elements-that-have-implied-end-tags +proc generateImpliedEndTags(parser: var HTML5Parser) = + const tags = { + TAG_DD, TAG_DT, TAG_LI, TAG_OPTGROUP, TAG_OPTION, TAG_P, TAG_RB, TAG_RP, + TAG_RT, TAG_RTC + } + while parser.getTagType(parser.currentNode) in tags: + discard parser.popElement() + +proc generateImpliedEndTags(parser: var HTML5Parser, exclude: TagType) = + let tags = { + TAG_DD, TAG_DT, TAG_LI, TAG_OPTGROUP, TAG_OPTION, TAG_P, TAG_RB, TAG_RP, + TAG_RT, TAG_RTC + } - {exclude} + while parser.getTagType(parser.currentNode) in tags: + discard parser.popElement() + +proc generateImpliedEndTagsThoroughly(parser: var HTML5Parser) = + const tags = { + TAG_CAPTION, TAG_COLGROUP, TAG_DD, TAG_DT, TAG_LI, TAG_OPTGROUP, + TAG_OPTION, TAG_P, TAG_RB, TAG_RP, TAG_RT, TAG_RTC, TAG_TBODY, TAG_TD, + TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TR + } + while parser.getTagType(parser.currentNode) in tags: + discard parser.popElement() + +# https://html.spec.whatwg.org/multipage/parsing.html#push-onto-the-list-of-active-formatting-elements +proc pushOntoActiveFormatting[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + element: Handle, token: Token) = + var count = 0 + for i in countdown(parser.activeFormatting.high, 0): + let it = parser.activeFormatting[i] + if it[0].isNone: # marker + break + if it[1].tagname != token.tagname: + continue + if parser.getNamespace(it[0].get) != parser.getNamespace(element): + continue + if it[1].attrs != token.attrs: + continue + inc count + if count == 3: + parser.activeFormatting.delete(i) + break + parser.activeFormatting.add((some(element), token)) + +proc findOpenElement[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + element: Handle): int = + for i, it in parser.openElements: + if it.element == element: + return i + return -1 + +proc reconstructActiveFormatting[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) = + type State = enum + REWIND, ADVANCE, CREATE + if parser.activeFormatting.len == 0 or + parser.activeFormatting[^1][0].isNone or + parser.findOpenElement(parser.activeFormatting[^1][0].get) != -1: + return + var i = parser.activeFormatting.high + template entry: Option[Handle] = (parser.activeFormatting[i][0]) + var state = REWIND + while true: + case state + of REWIND: + if i == 0: + state = CREATE + continue + dec i + if entry.isSome and parser.findOpenElement(entry.get) == -1: + continue + state = ADVANCE + of ADVANCE: + inc i + state = CREATE + of CREATE: + let element = parser.insertHTMLElement(parser.activeFormatting[i][1]) + parser.activeFormatting[i] = ( + some(element), parser.activeFormatting[i][1] + ) + if i != parser.activeFormatting.high: + state = ADVANCE + continue + break + +proc clearActiveFormattingTillMarker(parser: var HTML5Parser) = + while parser.activeFormatting.len > 0 and + parser.activeFormatting.pop()[0].isSome: + discard + +proc isMathMLIntegrationPoint[Handle, Atom](parser: HTML5Parser[Handle, Atom], + element: Handle): bool = + if parser.getNamespace(element) != Namespace.MATHML: + return false + let tagType = parser.atomToTagType(parser.getLocalName(element)) + return tagType in {TAG_MI, TAG_MO, TAG_MN, TAG_MS, TAG_MTEXT} + +proc isHTMLIntegrationPoint[Handle, Atom](parser: HTML5Parser[Handle, Atom], + oe: OpenElement[Handle, Atom]): bool = + let (element, token) = oe + let localName = parser.getLocalName(element) + let namespace = parser.getNamespace(element) + let tagType = parser.atomToTagType(localName) + if namespace == Namespace.MATHML: + if tagType == TAG_ANNOTATION_XML: + token.attrs.withValue(parser.tagTypeToAtom(TAG_ENCODING), p): + return p[].equalsIgnoreCase("text/html") or + p[].equalsIgnoreCase("application/xhtml+xml") + elif namespace == Namespace.SVG: + return tagType in {TAG_FOREIGN_OBJECT, TAG_DESC, TAG_TITLE} + return false + +const AsciiWhitespace = {' ', '\n', '\r', '\t', '\f'} + +func until(s: string, c1, c2: char, starti: int): string = + result = "" + for i in starti ..< s.len: + if s[i] == c1 or s[i] == c2: + break + result &= s[i] + +func extractEncFromMeta(s: string): string = + var i = 0 + while true: # Loop: + var j = 0 + while i < s.len: + template check(c: static char) = + if s[i].toLowerAscii() == c: + inc j + else: + j = 0 + case j + of 0: check 'c' + of 1: check 'h' + of 2: check 'a' + of 3: check 'r' + of 4: check 's' + of 5: check 'e' + of 6: check 't' + of 7: + inc j + break + else: discard + inc i + if j < 7: return "" + while i < s.len and s[i] in AsciiWhitespace: inc i + if i >= s.len or s[i] != '=': continue + while i < s.len and s[i] in AsciiWhitespace: inc i + break + inc i + if i >= s.len: return "" + if s[i] in {'"', '\''}: + let s2 = s.until('"', '\'', i + 1) + if s2.len == 0 or s2[^1] != s[i]: + return "" + return s2 + return s.until(';', ' ', i) + +# Find a node in the list of active formatting elements, or return -1. +func findLastActiveFormatting[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + node: Handle): int = + for i in countdown(parser.activeFormatting.high, 0): + let it = parser.activeFormatting[i][0] + if it.isSome and it.get == node: + return i + return -1 + +# > the last element in the list of active formatting elements that: +# > is between the end of the list and the last marker in the list, if any, +# > or the start of the list otherwise, and has the tag name subject. +func findLastActiveFormattingAfterMarker(parser: var HTML5Parser, + token: Token): int = + for i in countdown(parser.activeFormatting.high, 0): + let it = parser.activeFormatting[i][1] + if it == nil: + break # marker + if it.tagname == token.tagname: + return i + return -1 + +proc findLastActiveFormattingAfterMarker(parser: var HTML5Parser, + tagType: TagType): int = + for i in countdown(parser.activeFormatting.high, 0): + let it = parser.activeFormatting[i][0] + if it.isNone: + break + if parser.getTagType(it.get) == tagType: + return i + return -1 + +#https://html.spec.whatwg.org/multipage/parsing.html#the-stack-of-open-elements +const SpecialElements = { + TAG_ADDRESS, TAG_APPLET, TAG_AREA, TAG_ARTICLE, TAG_ASIDE, TAG_BASE, + TAG_BASEFONT, TAG_BGSOUND, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, + TAG_CAPTION, TAG_CENTER, TAG_COL, TAG_COLGROUP, TAG_DD, TAG_DETAILS, TAG_DIR, + TAG_DIV, TAG_DL, TAG_DT, TAG_EMBED, TAG_FIELDSET, TAG_FIGCAPTION, TAG_FIGURE, + TAG_FOOTER, TAG_FORM, TAG_FRAME, TAG_FRAMESET, TAG_H1, TAG_H2, TAG_H3, TAG_H4, + TAG_H5, TAG_H6, TAG_HEAD, TAG_HEADER, TAG_HGROUP, TAG_HR, TAG_HTML, + TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_KEYGEN, TAG_LI, TAG_LINK, TAG_LISTING, + TAG_MAIN, TAG_MARQUEE, TAG_MENU, TAG_META, TAG_NAV, TAG_NOEMBED, TAG_NOFRAMES, + TAG_NOSCRIPT, TAG_OBJECT, TAG_OL, TAG_P, TAG_PARAM, TAG_PLAINTEXT, TAG_PRE, + TAG_SCRIPT, TAG_SEARCH, TAG_SECTION, TAG_SELECT, TAG_SOURCE, TAG_STYLE, + TAG_SUMMARY, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEMPLATE, TAG_TEXTAREA, + TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TITLE, TAG_TR, TAG_TRACK, TAG_UL, TAG_WBR, + TAG_XMP +} + +proc isSpecialElement[Handle, Atom](parser: HTML5Parser[Handle, Atom], + element: Handle): bool = + let tagType = parser.atomToTagType(parser.getLocalName(element)) + case parser.getNamespace(element) + of Namespace.HTML: + {.linearScanEnd.} + return tagType in SpecialElements + of Namespace.MATHML: + const elements = { + TAG_MI, TAG_MO, TAG_MN, TAG_MS, TAG_MTEXT, TAG_ANNOTATION_XML + } + return tagType in elements + of Namespace.SVG: + return tagType in {TAG_FOREIGN_OBJECT, TAG_DESC, TAG_TITLE} + else: + return false + +# > Let furthestBlock be the topmost node in the stack of open elements that +# > is lower in the stack than formattingElement, and is an element in the +# > special category. There might not be one. +proc findFurthestBlockAfter(parser: HTML5Parser, stackIndex: int): int = + for i in stackIndex ..< parser.openElements.len: + if parser.isSpecialElement(parser.openElements[i].element): + return i + return -1 + +proc findLastActiveFormatting[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + tagTypes: set[TagType]): int = + for i in countdown(parser.activeFormatting.high, 0): + let it = parser.activeFormatting[i][0] + if it.isSome and parser.getTagType(it.get) in tagTypes: + return i + return -1 + +# If true is returned, call "any other end tag". +proc adoptionAgencyAlgorithm[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token): bool = + if parser.currentToken.tagname == token.tagname and + parser.findLastActiveFormatting(parser.currentNode) == -1: + pop_current_node + return false + for i in 0 ..< 8: # outer loop + var formattingIndex = parser.findLastActiveFormattingAfterMarker(token) + if formattingIndex == -1: + # no such element + return true + let formatting = parser.activeFormatting[formattingIndex][0].get + let stackIndex = parser.findOpenElement(formatting) + if stackIndex < 0: + parser.activeFormatting.delete(formattingIndex) + return false + if not parser.hasElementInScope(formatting): + return false + var furthestBlockIndex = parser.findFurthestBlockAfter(stackIndex) + if furthestBlockIndex == -1: + parser.popElementsIncl(formatting) + parser.activeFormatting.delete(formattingIndex) + return false + let furthestBlock = parser.openElements[furthestBlockIndex].element + let commonAncestor = parser.openElements[stackIndex - 1].element + var bookmark = formattingIndex + var node = furthestBlock + var aboveNode = parser.openElements[furthestBlockIndex - 1].element + var lastNode = furthestBlock + var j = 0 + while true: + inc j + node = aboveNode + if node == formatting: + break + let nodeStackIndex = parser.findOpenElement(node) + var nodeFormattingIndex = parser.findLastActiveFormatting(node) + if j > 3 and nodeFormattingIndex >= 0: + parser.activeFormatting.delete(nodeFormattingIndex) + if nodeFormattingIndex < bookmark: + dec bookmark # a previous node got deleted, so decrement bookmark + nodeFormattingIndex = -1 # deleted, so set to -1 + if nodeFormattingIndex < 0: + aboveNode = parser.openElements[nodeStackIndex - 1].element + parser.openElements.delete(nodeStackIndex) + if nodeStackIndex < furthestBlockIndex: + dec furthestBlockIndex + let element = parser.openElements[furthestBlockIndex].element + assert furthestBlock == element + continue + let tok = parser.activeFormatting[nodeFormattingIndex][1] + let element = parser.createHTMLElementForToken(tok, commonAncestor) + parser.activeFormatting[nodeFormattingIndex] = (some(element), tok) + parser.openElements[nodeStackIndex] = (element, tok) + aboveNode = parser.openElements[nodeStackIndex - 1].element + node = element + if lastNode == furthestBlock: + bookmark = nodeFormattingIndex + 1 + parser.remove(lastNode) + parser.append(node, lastNode) + lastNode = node + parser.remove(lastNode) + let location = parser.appropriatePlaceForInsert(commonAncestor) + parser.insert(location, lastNode) + let token = parser.activeFormatting[formattingIndex][1] + let element = parser.createHTMLElementForToken(token, furthestBlock) + parser.moveChildren(furthestBlock, element) + parser.append(furthestBlock, element) + parser.activeFormatting.insert((some(element), token), bookmark) + if formattingIndex >= bookmark: + inc formattingIndex # increment because of insert + parser.activeFormatting.delete(formattingIndex) + parser.openElements.insert((element, token), furthestBlockIndex + 1) + parser.openElements.delete(stackIndex) + return false + +proc closeP(parser: var HTML5Parser, sure = false) = + if sure or parser.hasElementInButtonScope(TAG_P): + parser.generateImpliedEndTags(TAG_P) + parser.popElementsIncl(TAG_P) + +proc newStartTagToken[Handle, Atom](parser: HTML5Parser[Handle, Atom], + t: TagType): Token[Atom] = + return Token[Atom](t: START_TAG, tagname: parser.tagTypeToAtom(t)) + +# Following is an implementation of the state (?) machine defined in +# https://html.spec.whatwg.org/multipage/parsing.html#parsing-main-inhtml +# It uses the ad-hoc pattern matching macro `match' to apply the following +# transformations: +# * First, pairs of patterns and actions are stored in tuples (and `discard' +# statements...) +# * These pairs are then assigned to token types, later mapped to legs of the +# first case statement. +# * Another case statement is constructed where needed, e.g. for switching on +# characters/tags/etc. +# * Finally, the whole thing is wrapped in a named block, to implement a +# pseudo-goto by breaking out only when the else statement needn't be +# executed. +# +# For example, the following code: +# +# match token: +# TokenType.COMMENT => (block: echo "comment") +# ("<p>", "<a>", "</div>") => (block: echo "p, a or closing div") +# ("<div>", "</p>") => (block: anything_else) +# (TokenType.START_TAG, TokenType.END_TAG) => (block: assert false, "invalid") +# other => (block: echo "anything else") +# +# (effectively) generates this: +# +# block inside_not_else: +# case token.t +# of TokenType.COMMENT: +# echo "comment" +# break inside_not_else +# of TokenType.START_TAG: +# case parser.atomToTagName(token.tagname) +# of {TAG_P, TAG_A}: +# echo "p, a or closing div" +# break inside_not_else +# of TAG_DIV: discard +# else: +# assert false +# break inside_not_else +# of TokenType.END_TAG: +# case parser.atomToTagName(token.tagname) +# of TAG_DIV: +# echo "p, a or closing div" +# break inside_not_else +# of TAG_P: discard +# else: +# assert false +# break inside_not_else +# else: discard +# echo "anything else" +# +# This duplicates any code that applies for several token types, except for the +# else branch. +type + OfBranch = (seq[NimNode], NimNode) + + OfBranchStore = object + ofBranches: seq[OfBranch] + defaultBranch: NimNode + painted: bool + + PatternType = enum + ptTokenType, ptTagType, ptOther, ptCombined + + Pattern = object + case t: PatternType + of ptTokenType: + tokt: TokenType + of ptTagType: + isend: bool + tagt: TagType + of ptCombined: + children: seq[Pattern] + of ptOther: discard + +const TagTypeMap = block: + var map: Table[string, TagType] + for tt in TagType: + map[$tt] = tt + map + +proc toPattern(s: string): Pattern = + if s == "_": + return Pattern(t: ptOther) + assert s[0] == '<' and s[^1] == '>' + let isend = s[1] == '/' + let i = if not isend: 1 else: 2 + let tagType = TagTypeMap[s[i..^2]] + assert tagType != TAG_UNKNOWN + return Pattern(t: ptTagType, isend: isend, tagt: tagType) + +proc toPattern(tt: TokenType): Pattern = + return Pattern(t: ptTokenType, tokt: tt) + +proc toPattern(s: tuple): Pattern = + result = Pattern(t: ptCombined) + for _, x in s.fieldPairs: + result.children.add(toPattern(x)) + +macro toNimNode(p: untyped): NimNode = + return p + +type MatchBuilder = object + # Stores 'of' branches + ofBranches: array[TokenType, OfBranchStore] + # Stores 'else', 'elif' branches + defaultBranch: NimNode + +func tokenBranchOn(tok: TokenType): NimNode = + assert tok in {START_TAG, END_TAG} + return quote do: + ( + let tokenTagType {.inject.} = parser.atomToTagType(token.tagname); + tokenTagType + ) + +proc addToCase(tokenCase: NimNode, branch: OfBranch) = + if branch[0].len == 1: + tokenCase.add(newNimNode(nnkOfBranch).add(branch[0][0]).add(branch[1])) + else: + var curly = newNimNode(nnkCurly) + for node in branch[0]: + curly.add(node) + tokenCase.add(newNimNode(nnkOfBranch).add(curly).add(branch[1])) + +func findAction(ofBranches: seq[OfBranch], action: NimNode): int = + for i, branch in ofBranches: + if branch[1] == action: + return i + return -1 + +proc buildBranches(builder: var MatchBuilder, patterns: openArray[Pattern], + lambdas: NimNode) = + for i in 0 ..< patterns.len: + let pattern = patterns[i] + let lambda = lambdas[i] + const noAction = {nnkSym, nnkEmpty, nnkFormalParams} + let action = lambda.findChild(it.kind notin noAction) + var patterns = @[pattern] + while patterns.len > 0: + let pattern = patterns.pop() + case pattern.t + of ptTokenType: + builder.ofBranches[pattern.tokt].defaultBranch = action + builder.ofBranches[pattern.tokt].painted = true + of ptTagType: + let tt = int(pattern.tagt) + let tokt = if not pattern.isend: START_TAG else: END_TAG + let i = builder.ofBranches[tokt].ofBranches.findAction(action) + if i != -1: + builder.ofBranches[tokt].ofBranches[i][0].add(quote do: TagType(`tt`)) + else: + let tup = (@[(quote do: TagType(`tt`))], action) + builder.ofBranches[tokt].ofBranches.add(tup) + builder.ofBranches[tokt].painted = true + of ptCombined: + for child in pattern.children: + patterns.add(child) + of ptOther: + builder.defaultBranch = action + +proc generateCaseStmts(builder: MatchBuilder, token: NimNode): NimNode = + let mainCase = newNimNode(nnkCaseStmt).add(quote do: `token`.t) + for tt in TokenType: + let ofBranch = newNimNode(nnkOfBranch).add(quote do: TokenType(`tt`)) + let tokenCase = newNimNode(nnkCaseStmt) + if builder.ofBranches[tt].defaultBranch != nil: + if builder.ofBranches[tt].ofBranches.len > 0: + tokenCase.add(tokenBranchOn(tt)) + for branch in builder.ofBranches[tt].ofBranches: + tokenCase.addToCase(branch) + tokenCase.add(newNimNode(nnkElse) + .add(builder.ofBranches[tt].defaultBranch)) + ofBranch.add(tokenCase) + mainCase.add(ofBranch) + else: + ofBranch.add(builder.ofBranches[tt].defaultBranch) + mainCase.add(ofBranch) + elif builder.ofBranches[tt].ofBranches.len > 0: + tokenCase.add(tokenBranchOn(tt)) + for branch in builder.ofBranches[tt].ofBranches: + tokenCase.addToCase(branch) + ofBranch.add(tokenCase) + tokenCase.add(newNimNode(nnkElse).add(quote do: discard)) + mainCase.add(ofBranch) + return mainCase + +macro match0(token: Token, patterns: static openArray[Pattern], + lambdas: seq[NimNode]) = + var builder = MatchBuilder() + builder.buildBranches(patterns, lambdas) + let mainCase = builder.generateCaseStmts(token) + for t in TokenType: + if not builder.ofBranches[t].painted: + mainCase.add(newNimNode(nnkElse).add(quote do: discard)) + break + var stmts = newStmtList().add(mainCase) + for stmt in builder.defaultBranch: + stmts.add(stmt) + result = newBlockStmt(ident("inside_not_else"), stmts) + +macro match(token: Token, bodyIn: untyped): untyped = + result = newStmtList() + var patterns = newTree(nnkBracket) + var lambdas: seq[NimNode] = @[] + for child in bodyIn: + if unlikely(child.kind != nnkInfix or child[0].strVal != "=>"): + error("=> expected") + let sel = child[1] + let body = child[2] + let pattern = quote do: + toPattern(`sel`) + let lambda0 = quote do: + toNimNode: + `body` + break inside_not_else + patterns.add(pattern) + lambdas.add(lambda0) + result = quote do: + match0(`token`, `patterns`, `lambdas`) + +proc processInHTMLContent[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token, insertionMode: InsertionMode): ParseResult = + template other: string = "_" + + template reprocess(tok: Token) = + return parser.processInHTMLContent(tok, parser.insertionMode) + + template reprocess(mode: InsertionMode): ParseResult = + parser.processInHTMLContent(token, mode) + + case insertionMode + of INITIAL: + match token: + TokenType.CHARACTER_WHITESPACE => (block: discard) + TokenType.COMMENT => (block: + parser.insertComment(token, last_child_of(parser.getDocument())) + ) + TokenType.DOCTYPE => (block: + let doctype = parser.createDocumentType(token.name.get(""), + token.pubid.get(""), token.sysid.get("")) + parser.append(parser.getDocument(), doctype) + if not parser.opts.isIframeSrcdoc: + if quirksConditions(token): + parser.setQuirksMode(QUIRKS) + elif limitedQuirksConditions(token): + parser.setQuirksMode(LIMITED_QUIRKS) + parser.insertionMode = BEFORE_HTML + ) + other => (block: + parser.setQuirksMode(QUIRKS) + parser.insertionMode = BEFORE_HTML + reprocess token + ) + + of BEFORE_HTML: + match token: + (TokenType.DOCTYPE, TokenType.END_TAG, + TokenType.CHARACTER_WHITESPACE) => (block: discard) + TokenType.COMMENT => (block: + parser.insertComment(token, last_child_of(parser.getDocument())) + ) + "<html>" => (block: + let intendedParent = parser.getDocument() + let element = parser.createHTMLElementForToken(token, intendedParent) + parser.append(parser.getDocument(), element) + parser.pushElement(element, token) + parser.insertionMode = BEFORE_HEAD + ) + ("</head>", "</body>", "</html>", "</br>", other) => (block: + let element = parser.createHTMLElement() + parser.append(parser.getDocument(), element) + let html = parser.newStartTagToken(TAG_HTML) + parser.pushElement(element, html) + parser.insertionMode = BEFORE_HEAD + reprocess token + ) + + of BEFORE_HEAD: + match token: + (TokenType.CHARACTER_WHITESPACE, TokenType.DOCTYPE, + TokenType.END_TAG) => (block: discard) + TokenType.COMMENT => (block: parser.insertComment(token)) + "<html>" => (block: return reprocess IN_BODY) + "<head>" => (block: + parser.head = some((parser.insertHTMLElement(token), token)) + parser.insertionMode = IN_HEAD + ) + ("</head>", "</body>", "</html>", "</br>", other) => (block: + let head = parser.newStartTagToken(TAG_HEAD) + parser.head = some((parser.insertHTMLElement(head), head)) + parser.insertionMode = IN_HEAD + reprocess token + ) + + of IN_HEAD: + match token: + TokenType.CHARACTER_WHITESPACE => (block: + parser.insertCharacter(token.s) + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + TokenType.DOCTYPE => (block: discard) + "<html>" => (block: return reprocess IN_BODY) + ("<base>", "<basefont>", "<bgsound>", "<link>") => (block: + discard parser.insertHTMLElement(token) + pop_current_node + ) + "<meta>" => (block: + discard parser.insertHTMLElement(token) + pop_current_node + token.attrs.withValue(parser.tagTypeToAtom(TAG_CHARSET), p): + case parser.setEncoding(p[]) + of SET_ENCODING_CONTINUE: + discard + of SET_ENCODING_STOP: + return PRES_STOP + do: + token.attrs.withValue(parser.tagTypeToAtom(TAG_HTTP_EQUIV), p): + if p[].equalsIgnoreCase("Content-Type"): + token.attrs.withValue(parser.tagTypeToAtom(TAG_CONTENT), p2): + let cs = extractEncFromMeta(p2[]) + if cs != "": + case parser.setEncoding(cs) + of SET_ENCODING_CONTINUE: + discard + of SET_ENCODING_STOP: + return PRES_STOP + ) + "<title>" => (block: parser.genericRCDATAElementParsingAlgorithm(token)) + "<noscript>" => (block: + if not parser.opts.scripting: + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_HEAD_NOSCRIPT + else: + parser.genericRawtextElementParsingAlgorithm(token) + ) + ("<noframes>", "<style>") => (block: + parser.genericRawtextElementParsingAlgorithm(token) + ) + "<script>" => (block: + let location = parser.appropriatePlaceForInsert() + let element = parser.createElementForToken(token, Namespace.HTML, + location.inside) + if parser.fragment: + parser.setScriptAlreadyStarted(element) + parser.insert(location, element) + parser.pushElement(element, token) + parser.tokenizer.state = SCRIPT_DATA + parser.oldInsertionMode = parser.insertionMode + parser.insertionMode = TEXT + ) + "</head>" => (block: + pop_current_node + parser.insertionMode = AFTER_HEAD + ) + "<template>" => (block: + discard parser.insertHTMLElement(token) + parser.activeFormatting.add((none(Handle), nil)) + parser.framesetOk = false + parser.insertionMode = IN_TEMPLATE + parser.templateModes.add(IN_TEMPLATE) + ) + "</template>" => (block: + if parser.hasElement(TAG_TEMPLATE): + parser.generateImpliedEndTagsThoroughly() + parser.popElementsIncl(TAG_TEMPLATE) + parser.clearActiveFormattingTillMarker() + discard parser.templateModes.pop() + parser.resetInsertionMode() + ) + ("<head>", TokenType.END_TAG) => (block: discard) + ("</body>", "</html>", "</br>", other) => (block: + pop_current_node + parser.insertionMode = AFTER_HEAD + reprocess token + ) + + of IN_HEAD_NOSCRIPT: + match token: + (TokenType.DOCTYPE, "<head>", "<noscript>", TokenType.END_TAG) => (block: + discard + ) + "<html>" => (block: return reprocess IN_BODY) + "</noscript>" => (block: + pop_current_node + parser.insertionMode = IN_HEAD + ) + (TokenType.CHARACTER_WHITESPACE, TokenType.COMMENT, + "<basefont>", "<bgsound>", "<link>", "<meta>", "<noframes>", + "<style>") => (block: + return reprocess IN_HEAD + ) + ("</br>", other) => (block: + pop_current_node + parser.insertionMode = IN_HEAD + reprocess token + ) + + of AFTER_HEAD: + match token: + TokenType.CHARACTER_WHITESPACE => (block: + parser.insertCharacter(token.s) + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + (TokenType.DOCTYPE, "<head>", TokenType.END_TAG) => (block: discard) + "<html>" => (block: return reprocess IN_BODY) + "<body>" => (block: + discard parser.insertHTMLElement(token) + parser.framesetOk = false + parser.insertionMode = IN_BODY + ) + "<frameset>" => (block: + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_FRAMESET + ) + ("<base>", "<basefont>", "<bgsound>", "<link>", "<meta>", "<noframes>", + "<script>", "<style>", "<template>", "<title>") => (block: + let (head, headTok) = parser.head.get + parser.pushElement(head, headTok) + result = reprocess IN_HEAD + if (let i = parser.findOpenElement(head); i != -1): + parser.openElements.delete(i) + ) + "</template>" => (block: return reprocess IN_HEAD) + ("</body>", "</html>", "</br>", other) => (block: + discard parser.insertHTMLElement(parser.newStartTagToken(TAG_BODY)) + parser.insertionMode = IN_BODY + reprocess token + ) + + of IN_BODY: + template any_other_start_tag() = + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + + template any_other_end_tag(tokenTagType: TagType) = + for i in countdown(parser.openElements.high, 0): + let (node, itToken) = parser.openElements[i] + if itToken.tagname == token.tagname: + parser.generateImpliedEndTags(tokenTagType) + parser.popElementsIncl(node) + break + elif parser.isSpecialElement(node): + return + + match token: + TokenType.CHARACTER_WHITESPACE => (block: + parser.reconstructActiveFormatting() + parser.insertCharacter(token.s) + ) + (TokenType.CHARACTER_NULL, TokenType.DOCTYPE) => (block: discard) + TokenType.CHARACTER => (block: + parser.reconstructActiveFormatting() + parser.insertCharacter(token.s) + parser.framesetOk = false + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + "<html>" => (block: + if not parser.hasElement(TAG_TEMPLATE): + parser.addAttrsIfMissing(parser.openElements[0].element, token.attrs) + ) + ("<base>", "<basefont>", "<bgsound>", "<link>", "<meta>", "<noframes>", + "<script>", "<style>", "<template>", "<title>", + "</template>") => (block: + return reprocess IN_HEAD + ) + "<body>" => (block: + if parser.openElements.len == 1 or + parser.getTagType(parser.openElements[1].element) != TAG_BODY or + parser.hasElement(TAG_TEMPLATE): + discard + else: + parser.framesetOk = false + parser.addAttrsIfMissing(parser.openElements[1].element, token.attrs) + ) + "<frameset>" => (block: + if parser.openElements.len == 1 or + parser.getTagType(parser.openElements[1].element) != TAG_BODY or + not parser.framesetOk: + discard + else: + parser.remove(parser.openElements[1].element) + while parser.openElements.len > 1: + pop_current_node + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_FRAMESET + ) + TokenType.EOF => (block: + if parser.templateModes.len > 0: + return reprocess IN_TEMPLATE + # stop + ) + "</body>" => (block: + if parser.hasElementInScope(TAG_BODY): + parser.insertionMode = AFTER_BODY + ) + "</html>" => (block: + if parser.hasElementInScope(TAG_BODY): + parser.insertionMode = AFTER_BODY + reprocess token + ) + ("<address>", "<article>", "<aside>", "<blockquote>", "<center>", + "<details>", "<dialog>", "<dir>", "<div>", "<dl>", "<fieldset>", + "<figcaption>", "<figure>", "<footer>", "<header>", "<hgroup>", "<main>", + "<menu>", "<nav>", "<ol>", "<p>", "<search>", "<section>", "<summary>", + "<ul>") => (block: + parser.closeP() + discard parser.insertHTMLElement(token) + ) + ("<h1>", "<h2>", "<h3>", "<h4>", "<h5>", "<h6>") => (block: + parser.closeP() + if parser.getTagType(parser.currentNode) in HTagTypes: + pop_current_node + discard parser.insertHTMLElement(token) + ) + ("<pre>", "<listing>") => (block: + parser.closeP() + discard parser.insertHTMLElement(token) + parser.ignoreLF = true + parser.framesetOk = false + ) + "<form>" => (block: + let hasTemplate = parser.hasElement(TAG_TEMPLATE) + if parser.form.isNone or hasTemplate: + parser.closeP() + let element = parser.insertHTMLElement(token) + if not hasTemplate: + parser.form = some(element) + ) + "<li>" => (block: + parser.framesetOk = false + for i in countdown(parser.openElements.high, 0): + let node = parser.openElements[i].element + let tagType = parser.getTagType(node) + case tagType + of TAG_LI: + parser.generateImpliedEndTags(TAG_LI) + parser.popElementsIncl(TAG_LI) + break + of TAG_ADDRESS, TAG_DIV, TAG_P: + discard + elif parser.isSpecialElement(node): + break + else: discard + parser.closeP() + discard parser.insertHTMLElement(token) + ) + ("<dd>", "<dt>") => (block: + parser.framesetOk = false + for i in countdown(parser.openElements.high, 0): + let node = parser.openElements[i].element + let tagType = parser.getTagType(node) + case tagType + of TAG_DD: + parser.generateImpliedEndTags(TAG_DD) + parser.popElementsIncl(TAG_DD) + break + of TAG_DT: + parser.generateImpliedEndTags(TAG_DT) + parser.popElementsIncl(TAG_DT) + break + of TAG_ADDRESS, TAG_DIV, TAG_P: + discard + elif parser.isSpecialElement(node): + break + else: discard + parser.closeP() + discard parser.insertHTMLElement(token) + ) + "<plaintext>" => (block: + parser.closeP() + discard parser.insertHTMLElement(token) + parser.tokenizer.state = PLAINTEXT + ) + "<button>" => (block: + if parser.hasElementInScope(TAG_BUTTON): + parser.generateImpliedEndTags() + parser.popElementsIncl(TAG_BUTTON) + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + parser.framesetOk = false + ) + ("</address>", "</article>", "</aside>", "</blockquote>", "</button>", + "</center>", "</details>", "</dialog>", "</dir>", "</div>", "</dl>", + "</fieldset>", "</figcaption>", "</figure>", "</footer>", "</header>", + "</hgroup>", "</listing>", "</main>", "</menu>", "</nav>", "</ol>", + "</pre>", "</search>", "</section>", "</summary>", "</ul>") => (block: + if parser.hasElementInScope(token.tagname): + parser.generateImpliedEndTags() + parser.popElementsIncl(tokenTagType) + ) + "</form>" => (block: + if not parser.hasElement(TAG_TEMPLATE): + let form = parser.form + parser.form = none(Handle) + if form.isNone or not parser.hasElementInScope(form.get): + return + let node = form.get + parser.generateImpliedEndTags() + let i = parser.findOpenElement(node) + parser.openElements.delete(i) + else: + if parser.hasElementInScope(TAG_FORM): + parser.generateImpliedEndTags() + parser.popElementsIncl(TAG_FORM) + ) + "</p>" => (block: + if not parser.hasElementInButtonScope(TAG_P): + discard parser.insertHTMLElement(parser.newStartTagToken(TAG_P)) + parser.closeP(sure = true) + ) + "</li>" => (block: + if parser.hasElementInListItemScope(TAG_LI): + parser.generateImpliedEndTags(TAG_LI) + parser.popElementsIncl(TAG_LI) + ) + ("</dd>", "</dt>") => (block: + if parser.hasElementInScope(tokenTagType): + parser.generateImpliedEndTags(tokenTagType) + parser.popElementsIncl(tokenTagType) + ) + ("</h1>", "</h2>", "</h3>", "</h4>", "</h5>", "</h6>") => (block: + if parser.hasElementInScope(HTagTypes): + parser.generateImpliedEndTags() + parser.popElementsIncl(HTagTypes) + ) + "<a>" => (block: + let i = parser.findLastActiveFormattingAfterMarker(TAG_A) + if i != -1: + let anchor = parser.activeFormatting[i][0].get + if parser.adoptionAgencyAlgorithm(token): + any_other_end_tag tokenTagType + return + let j = parser.findLastActiveFormatting(anchor) + if j != -1: + parser.activeFormatting.delete(j) + let k = parser.findOpenElement(anchor) + if k != -1: + parser.openElements.delete(k) + parser.reconstructActiveFormatting() + let element = parser.insertHTMLElement(token) + parser.pushOntoActiveFormatting(element, token) + ) + ("<b>", "<big>", "<code>", "<em>", "<font>", "<i>", "<s>", "<small>", + "<strike>", "<strong>", "<tt>", "<u>") => (block: + parser.reconstructActiveFormatting() + let element = parser.insertHTMLElement(token) + parser.pushOntoActiveFormatting(element, token) + ) + "<nobr>" => (block: + parser.reconstructActiveFormatting() + if parser.hasElementInScope(TAG_NOBR): + if parser.adoptionAgencyAlgorithm(token): + any_other_end_tag tokenTagType + return + parser.reconstructActiveFormatting() + let element = parser.insertHTMLElement(token) + parser.pushOntoActiveFormatting(element, token) + ) + ("</a>", "</b>", "</big>", "</code>", "</em>", "</font>", "</i>", + "</nobr>", "</s>", "</small>", "</strike>", "</strong>", "</tt>", + "</u>") => (block: + if parser.adoptionAgencyAlgorithm(token): + any_other_end_tag tokenTagType + return + ) + ("<applet>", "<marquee>", "<object>") => (block: + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + parser.activeFormatting.add((none(Handle), nil)) + parser.framesetOk = false + ) + ("</applet>", "</marquee>", "</object>") => (block: + if parser.hasElementInScope(tokenTagType): + parser.generateImpliedEndTags() + parser.popElementsIncl(tokenTagType) + parser.clearActiveFormattingTillMarker() + ) + "<table>" => (block: + if parser.quirksMode != QUIRKS: + parser.closeP() + discard parser.insertHTMLElement(token) + parser.framesetOk = false + parser.insertionMode = IN_TABLE + ) + "</br>" => (block: reprocess parser.newStartTagToken(TAG_BR)) + ("<area>", "<br>", "<embed>", "<img>", "<keygen>", "<wbr>") => (block: + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + pop_current_node + parser.framesetOk = false + ) + "<input>" => (block: + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + pop_current_node + token.attrs.withValue(parser.tagTypeToAtom(TAG_TYP), p): + if not p[].equalsIgnoreCase("hidden"): + parser.framesetOk = false + do: + parser.framesetOk = false + ) + ("<param>", "<source>", "<track>") => (block: + discard parser.insertHTMLElement(token) + pop_current_node + ) + "<hr>" => (block: + parser.closeP() + discard parser.insertHTMLElement(token) + pop_current_node + parser.framesetOk = false + ) + "<image>" => (block: + token.tagname = parser.tagTypeToAtom(TAG_IMG) + reprocess token + ) + "<textarea>" => (block: + discard parser.insertHTMLElement(token) + parser.ignoreLF = true + parser.tokenizer.state = RCDATA + parser.oldInsertionMode = parser.insertionMode + parser.framesetOk = false + parser.insertionMode = TEXT + ) + "<xmp>" => (block: + parser.closeP() + parser.reconstructActiveFormatting() + parser.framesetOk = false + parser.genericRawtextElementParsingAlgorithm(token) + ) + "<iframe>" => (block: + parser.framesetOk = false + parser.genericRawtextElementParsingAlgorithm(token) + ) + "<noembed>" => (block: + parser.genericRawtextElementParsingAlgorithm(token) + ) + "<noscript>" => (block: + if parser.opts.scripting: + parser.genericRawtextElementParsingAlgorithm(token) + else: + any_other_start_tag + ) + "<select>" => (block: + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + parser.framesetOk = false + const TableInsertionModes = { + IN_TABLE, IN_CAPTION, IN_TABLE_BODY, IN_ROW, IN_CELL + } + if parser.insertionMode in TableInsertionModes: + parser.insertionMode = IN_SELECT_IN_TABLE + else: + parser.insertionMode = IN_SELECT + ) + ("<optgroup>", "<option>") => (block: + if parser.getTagType(parser.currentNode) == TAG_OPTION: + pop_current_node + parser.reconstructActiveFormatting() + discard parser.insertHTMLElement(token) + ) + ("<rb>", "<rtc>") => (block: + if parser.hasElementInScope(TAG_RUBY): + parser.generateImpliedEndTags() + discard parser.insertHTMLElement(token) + ) + ("<rp>", "<rt>") => (block: + if parser.hasElementInScope(TAG_RUBY): + parser.generateImpliedEndTags(TAG_RTC) + discard parser.insertHTMLElement(token) + ) + "<math>" => (block: + parser.reconstructActiveFormatting() + var xmlAttrs: seq[ParsedAttr[Atom]] = @[] + parser.adjustMathMLAttributes(token.attrs, xmlAttrs) + discard parser.insertForeignElement(token, token.tagname, + Namespace.MATHML, false, xmlAttrs) + if token.selfclosing: + pop_current_node + ) + "<svg>" => (block: + parser.reconstructActiveFormatting() + var xmlAttrs: seq[ParsedAttr[Atom]] = @[] + parser.adjustSVGAttributes(token.attrs, xmlAttrs) + discard parser.insertForeignElement(token, token.tagname, Namespace.SVG, + false, xmlAttrs) + if token.selfclosing: + pop_current_node + ) + ("<caption>", "<col>", "<colgroup>", "<frame>", "<head>", "<tbody>", + "<td>", "<tfoot>", "<th>", "<thead>", "<tr>") => (block: + discard + ) + TokenType.START_TAG => (block: any_other_start_tag) + TokenType.END_TAG => (block: + let tokenTagType = parser.atomToTagType(token.tagname) + any_other_end_tag tokenTagType + ) + + of TEXT: + match token: + TokenType.CHARACTER_NULL => (block: + # "This can never be a U+0000 NULL character; the tokenizer converts + # those to U+FFFD REPLACEMENT CHARACTER characters." + assert false + ) + (TokenType.CHARACTER, TokenType.CHARACTER_WHITESPACE) => (block: + parser.insertCharacter(token.s) + ) + TokenType.EOF => (block: + if parser.getTagType(parser.currentNode) == TAG_SCRIPT: + parser.setScriptAlreadyStarted(parser.currentNode) + pop_current_node + parser.insertionMode = parser.oldInsertionMode + reprocess token + ) + TokenType.END_TAG => (block: + pop_current_node + parser.insertionMode = parser.oldInsertionMode + if parser.opts.scripting: + if parser.atomToTagType(token.tagname) == TAG_SCRIPT: + return PRES_SCRIPT + ) + + of IN_TABLE: + template clear_the_stack_back_to_a_table_context() = + const tags = {TAG_TABLE, TAG_TEMPLATE, TAG_HTML} + while parser.getTagType(parser.currentNode) notin tags: + pop_current_node + + match token: + (TokenType.CHARACTER, TokenType.CHARACTER_WHITESPACE, + TokenType.CHARACTER_NULL) => (block: + const CanHaveText = { + TAG_TABLE, TAG_TBODY, TAG_TFOOT, TAG_THEAD, TAG_TR + } + if parser.getTagType(parser.currentNode) in CanHaveText: + parser.pendingTableChars = "" + parser.pendingTableCharsWhitespace = true + parser.oldInsertionMode = parser.insertionMode + parser.insertionMode = IN_TABLE_TEXT + reprocess token + else: # anything else + parser.fosterParenting = true + result = reprocess IN_BODY + parser.fosterParenting = false + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + TokenType.DOCTYPE => (block: discard) + "<caption>" => (block: + clear_the_stack_back_to_a_table_context + parser.activeFormatting.add((none(Handle), nil)) + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_CAPTION + ) + "<colgroup>" => (block: + clear_the_stack_back_to_a_table_context + let colgroupTok = parser.newStartTagToken(TAG_COLGROUP) + discard parser.insertHTMLElement(colgroupTok) + parser.insertionMode = IN_COLUMN_GROUP + ) + "<col>" => (block: + clear_the_stack_back_to_a_table_context + let colgroupTok = parser.newStartTagToken(TAG_COLGROUP) + discard parser.insertHTMLElement(colgroupTok) + parser.insertionMode = IN_COLUMN_GROUP + reprocess token + ) + ("<tbody>", "<tfoot>", "<thead>") => (block: + clear_the_stack_back_to_a_table_context + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_TABLE_BODY + ) + ("<td>", "<th>", "<tr>") => (block: + clear_the_stack_back_to_a_table_context + discard parser.insertHTMLElement(parser.newStartTagToken(TAG_TBODY)) + parser.insertionMode = IN_TABLE_BODY + reprocess token + ) + "<table>" => (block: + if parser.hasElementInTableScope(TAG_TABLE): + parser.popElementsIncl(TAG_TABLE) + parser.resetInsertionMode() + reprocess token + ) + "</table>" => (block: + if parser.hasElementInTableScope(TAG_TABLE): + parser.popElementsIncl(TAG_TABLE) + parser.resetInsertionMode() + ) + ("</body>", "</caption>", "</col>", "</colgroup>", "</html>", "</tbody>", + "</td>", "</tfoot>", "</th>", "</thead>", "</tr>") => (block: discard) + ("<style>", "<script>", "<template>", "</template>") => (block: + return reprocess IN_HEAD + ) + "<input>" => (block: + token.attrs.withValue(parser.tagTypeToAtom(TAG_TYP), p): + if not p[].equalsIgnoreCase("hidden"): + # anything else + parser.fosterParenting = true + result = reprocess IN_BODY + parser.fosterParenting = false + else: + discard parser.insertHTMLElement(token) + pop_current_node + do: + # anything else + parser.fosterParenting = true + result = reprocess IN_BODY + parser.fosterParenting = false + ) + "<form>" => (block: + if parser.form.isNone and not parser.hasElement(TAG_TEMPLATE): + parser.form = some(parser.insertHTMLElement(token)) + pop_current_node + ) + TokenType.EOF => (block: return reprocess IN_BODY) + other => (block: + parser.fosterParenting = true + result = reprocess IN_BODY + parser.fosterParenting = false + ) + + of IN_TABLE_TEXT: + match token: + TokenType.CHARACTER_NULL => (block: discard) + TokenType.CHARACTER_WHITESPACE => (block: + parser.pendingTableChars &= token.s + ) + TokenType.CHARACTER => (block: + parser.pendingTableCharsWhitespace = false + parser.pendingTableChars &= token.s + ) + other => (block: + if not parser.pendingTableCharsWhitespace: + # I *think* this is effectively the same thing the specification + # wants... + parser.fosterParenting = true + parser.reconstructActiveFormatting() + parser.insertCharacter(parser.pendingTableChars) + parser.framesetOk = false + parser.fosterParenting = false + else: + parser.insertCharacter(parser.pendingTableChars) + parser.insertionMode = parser.oldInsertionMode + reprocess token + ) + + of IN_CAPTION: + match token: + "</caption>" => (block: + if parser.hasElementInTableScope(TAG_CAPTION): + parser.generateImpliedEndTags() + parser.popElementsIncl(TAG_CAPTION) + parser.clearActiveFormattingTillMarker() + parser.insertionMode = IN_TABLE + ) + ("<caption>", "<col>", "<colgroup>", "<tbody>", "<td>", "<tfoot>", + "<th>", "<thead>", "<tr>", "</table>") => (block: + if parser.hasElementInTableScope(TAG_CAPTION): + parser.generateImpliedEndTags() + parser.popElementsIncl(TAG_CAPTION) + parser.clearActiveFormattingTillMarker() + parser.insertionMode = IN_TABLE + reprocess token + ) + ("</body>", "</col>", "</colgroup>", "</html>", "</tbody>", "</td>", + "</tfoot>", "</th>", "</thead>", "</tr>") => (block: discard) + other => (block: return reprocess IN_BODY) + + of IN_COLUMN_GROUP: + match token: + TokenType.CHARACTER_WHITESPACE => (block: + parser.insertCharacter(token.s) + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + (TokenType.DOCTYPE, "</col>") => (block: discard) + "<html>" => (block: return reprocess IN_BODY) + "<col>" => (block: + discard parser.insertHTMLElement(token) + pop_current_node + ) + "</colgroup>" => (block: + if parser.getTagType(parser.currentNode) == TAG_COLGROUP: + pop_current_node + parser.insertionMode = IN_TABLE + ) + ("<template>", "</template>") => (block: return reprocess IN_HEAD) + TokenType.EOF => (block: return reprocess IN_BODY) + other => (block: + if parser.getTagType(parser.currentNode) == TAG_COLGROUP: + pop_current_node + parser.insertionMode = IN_TABLE + reprocess token + ) + + of IN_TABLE_BODY: + template clear_the_stack_back_to_a_table_body_context() = + const tags = {TAG_TBODY, TAG_TFOOT, TAG_THEAD, TAG_TEMPLATE, TAG_HTML} + while parser.getTagType(parser.currentNode) notin tags: + pop_current_node + + match token: + "<tr>" => (block: + clear_the_stack_back_to_a_table_body_context + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_ROW + ) + ("<th>", "<td>") => (block: + clear_the_stack_back_to_a_table_body_context + discard parser.insertHTMLElement(parser.newStartTagToken(TAG_TR)) + parser.insertionMode = IN_ROW + reprocess token + ) + ("</tbody>", "</tfoot>", "</thead>") => (block: + if parser.hasElementInTableScope(tokenTagType): + clear_the_stack_back_to_a_table_body_context + pop_current_node + parser.insertionMode = IN_TABLE + ) + ("<caption>", "<col>", "<colgroup>", "<tbody>", "<tfoot>", "<thead>", + "</table>") => (block: + if parser.hasElementInTableScope({TAG_TBODY, TAG_THEAD, TAG_TFOOT}): + clear_the_stack_back_to_a_table_body_context + pop_current_node + parser.insertionMode = IN_TABLE + reprocess token + ) + ("</body>", "</caption>", "</col>", "</colgroup>", "</html>", "</td>", + "</th>", "</tr>") => (block: discard) + other => (block: return reprocess IN_TABLE) + + of IN_ROW: + template clear_the_stack_back_to_a_table_row_context() = + while parser.getTagType(parser.currentNode) notin {TAG_TR, TAG_TEMPLATE, TAG_HTML}: + pop_current_node + + match token: + ("<th>", "<td>") => (block: + clear_the_stack_back_to_a_table_row_context + discard parser.insertHTMLElement(token) + parser.insertionMode = IN_CELL + parser.activeFormatting.add((none(Handle), nil)) + ) + "</tr>" => (block: + if parser.hasElementInTableScope(TAG_TR): + clear_the_stack_back_to_a_table_row_context + pop_current_node + parser.insertionMode = IN_TABLE_BODY + ) + ("<caption>", "<col>", "<colgroup>", "<tbody>", "<tfoot>", "<thead>", + "<tr>", "</table>") => (block: + if parser.hasElementInTableScope(TAG_TR): + clear_the_stack_back_to_a_table_row_context + pop_current_node + parser.insertionMode = IN_TABLE_BODY + reprocess token + ) + ("</tbody>", "</tfoot>", "</thead>") => (block: + if parser.hasElementInTableScope({tokenTagType, TAG_TR}): + clear_the_stack_back_to_a_table_row_context + pop_current_node + parser.insertionMode = IN_TABLE_BODY + reprocess token + ) + ("</body>", "</caption>", "</col>", "</colgroup>", "</html>", "</td>", + "</th>") => (block: discard) + other => (block: return reprocess IN_TABLE) + + of IN_CELL: + template close_cell() = + parser.generateImpliedEndTags() + parser.popElementsIncl({TAG_TD, TAG_TH}) + parser.clearActiveFormattingTillMarker() + parser.insertionMode = IN_ROW + + match token: + ("</td>", "</th>") => (block: + if parser.hasElementInTableScope(tokenTagType): + parser.generateImpliedEndTags() + parser.popElementsIncl(tokenTagType) + parser.clearActiveFormattingTillMarker() + parser.insertionMode = IN_ROW + ) + ("<caption>", "<col>", "<colgroup>", "<tbody>", "<td>", "<tfoot>", + "<th>", "<thead>", "<tr>") => (block: + if parser.hasElementInTableScope({TAG_TD, TAG_TH}): + close_cell + reprocess token + ) + ("</body>", "</caption>", "</col>", "</colgroup>", "</html>") => (block: + discard + ) + ("</table>", "</tbody>", "</tfoot>", "</thead>", "</tr>") => (block: + if parser.hasElementInTableScope(tokenTagType): + close_cell + reprocess token + ) + other => (block: return reprocess IN_BODY) + + of IN_SELECT: + match token: + (TokenType.CHARACTER, TokenType.CHARACTER_WHITESPACE) => (block: + parser.insertCharacter(token.s) + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + "<html>" => (block: return reprocess IN_BODY) + "<option>" => (block: + if parser.getTagType(parser.currentNode) == TAG_OPTION: + pop_current_node + discard parser.insertHTMLElement(token) + ) + "<optgroup>" => (block: + if parser.getTagType(parser.currentNode) == TAG_OPTION: + pop_current_node + if parser.getTagType(parser.currentNode) == TAG_OPTGROUP: + pop_current_node + discard parser.insertHTMLElement(token) + ) + "<hr>" => (block: + if parser.getTagType(parser.currentNode) == TAG_OPTION: + pop_current_node + if parser.getTagType(parser.currentNode) == TAG_OPTGROUP: + pop_current_node + discard parser.insertHTMLElement(token) + pop_current_node + ) + "</optgroup>" => (block: + if parser.getTagType(parser.currentNode) == TAG_OPTION: + if parser.openElements.len > 1: + let tagType = parser.getTagType(parser.openElements[^2].element) + if tagType == TAG_OPTGROUP: + pop_current_node + if parser.getTagType(parser.currentNode) == TAG_OPTGROUP: + pop_current_node + ) + "</option>" => (block: + if parser.getTagType(parser.currentNode) == TAG_OPTION: + pop_current_node + ) + "</select>" => (block: + if parser.hasElementInSelectScope(TAG_SELECT): + parser.popElementsIncl(TAG_SELECT) + parser.resetInsertionMode() + ) + "<select>" => (block: + if parser.hasElementInSelectScope(TAG_SELECT): + parser.popElementsIncl(TAG_SELECT) + parser.resetInsertionMode() + ) + ("<input>", "<keygen>", "<textarea>") => (block: + if parser.hasElementInSelectScope(TAG_SELECT): + parser.popElementsIncl(TAG_SELECT) + parser.resetInsertionMode() + reprocess token + ) + ("<script>", "<template>", "</template>") => (block: + return reprocess IN_HEAD + ) + TokenType.EOF => (block: return reprocess IN_BODY) + other => (block: discard) + + of IN_SELECT_IN_TABLE: + match token: + ("<caption>", "<table>", "<tbody>", "<tfoot>", "<thead>", "<tr>", "<td>", + "<th>") => (block: + parser.popElementsIncl(TAG_SELECT) + parser.resetInsertionMode() + reprocess token + ) + ("</caption>", "</table>", "</tbody>", "</tfoot>", "</thead>", "</tr>", + "</td>", "</th>") => (block: + if parser.hasElementInTableScope(tokenTagType): + parser.popElementsIncl(TAG_SELECT) + parser.resetInsertionMode() + reprocess token + ) + other => (block: return reprocess IN_SELECT) + + of IN_TEMPLATE: + match token: + (TokenType.CHARACTER, TokenType.CHARACTER_WHITESPACE, + TokenType.CHARACTER_NULL, TokenType.DOCTYPE) => (block: + return reprocess IN_BODY + ) + ("<base>", "<basefont>", "<bgsound>", "<link>", "<meta>", "<noframes>", + "<script>", "<style>", "<template>", "<title>", "</template>") => (block: + return reprocess IN_HEAD + ) + ("<caption>", "<colgroup>", "<tbody>", "<tfoot>", "<thead>") => (block: + discard parser.templateModes.pop() + parser.templateModes.add(IN_TABLE) + parser.insertionMode = IN_TABLE + reprocess token + ) + "<col>" => (block: + discard parser.templateModes.pop() + parser.templateModes.add(IN_COLUMN_GROUP) + parser.insertionMode = IN_COLUMN_GROUP + reprocess token + ) + "<tr>" => (block: + discard parser.templateModes.pop() + parser.templateModes.add(IN_TABLE_BODY) + parser.insertionMode = IN_TABLE_BODY + reprocess token + ) + ("<td>", "<th>") => (block: + discard parser.templateModes.pop() + parser.templateModes.add(IN_ROW) + parser.insertionMode = IN_ROW + reprocess token + ) + TokenType.START_TAG => (block: + discard parser.templateModes.pop() + parser.templateModes.add(IN_BODY) + parser.insertionMode = IN_BODY + reprocess token + ) + TokenType.END_TAG => (block: discard) + TokenType.EOF => (block: + if not parser.hasElement(TAG_TEMPLATE): + discard # stop + else: + parser.popElementsIncl(TAG_TEMPLATE) + parser.clearActiveFormattingTillMarker() + discard parser.templateModes.pop() + parser.resetInsertionMode() + reprocess token + ) + + of AFTER_BODY: + match token: + TokenType.CHARACTER_WHITESPACE => (block: + return reprocess IN_BODY + ) + TokenType.COMMENT => (block: + parser.insertComment(token, last_child_of(parser.openElements[0])) + ) + TokenType.DOCTYPE => (block: discard) + "<html>" => (block: return reprocess IN_BODY) + "</html>" => (block: + if not parser.fragment: + parser.insertionMode = AFTER_AFTER_BODY + ) + TokenType.EOF => (block: discard) # stop + other => (block: + parser.insertionMode = IN_BODY + reprocess token + ) + + of IN_FRAMESET: + match token: + TokenType.CHARACTER_WHITESPACE => (block: + parser.insertCharacter(token.s) + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + TokenType.DOCTYPE => (block: discard) + "<html>" => (block: return reprocess IN_BODY) + "<frameset>" => (block: discard parser.insertHTMLElement(token)) + "</frameset>" => (block: + if parser.getTagType(parser.currentNode) != TAG_HTML: + pop_current_node + if not parser.fragment and + parser.getTagType(parser.currentNode) != TAG_FRAMESET: + parser.insertionMode = AFTER_FRAMESET + ) + "<frame>" => (block: + discard parser.insertHTMLElement(token) + pop_current_node + ) + "<noframes>" => (block: return reprocess IN_HEAD) + TokenType.EOF => (block: + # stop + ) + other => (block: discard) + + of AFTER_FRAMESET: + match token: + TokenType.CHARACTER_WHITESPACE => (block: + parser.insertCharacter(token.s) + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + "<html>" => (block: return reprocess IN_BODY) + "</html>" => (block: parser.insertionMode = AFTER_AFTER_FRAMESET) + "<noframes>" => (block: return reprocess IN_HEAD) + other => (block: discard) + + of AFTER_AFTER_BODY: + match token: + TokenType.COMMENT => (block: + parser.insertComment(token, last_child_of(parser.getDocument())) + ) + (TokenType.DOCTYPE, TokenType.CHARACTER_WHITESPACE, "<html>") => (block: + return reprocess IN_BODY + ) + TokenType.EOF => (block: discard) # stop + other => (block: + parser.insertionMode = IN_BODY + reprocess token + ) + + of AFTER_AFTER_FRAMESET: + match token: + TokenType.COMMENT => (block: + parser.insertComment(token, last_child_of(parser.getDocument())) + ) + (TokenType.DOCTYPE, TokenType.CHARACTER_WHITESPACE, "<html>") => (block: + return reprocess IN_BODY + ) + "<noframes>" => (block: return reprocess IN_HEAD) + other => (block: discard) + return PRES_CONTINUE + +proc processInForeignContent[Handle, Atom]( + parser: var HTML5Parser[Handle, Atom], token: Token): ParseResult = + template script_end_tag() = + pop_current_node + return PRES_SCRIPT + + template any_other_start_tag() = + let namespace = parser.getNamespace(parser.adjustedCurrentNode) + var tagname = token.tagname + var xmlAttrs: seq[ParsedAttr[Atom]] = @[] + if namespace == Namespace.SVG: + parser.caseTable.withValue(tagname, p): + tagname = p[] + parser.adjustSVGAttributes(token.attrs, xmlAttrs) + elif namespace == Namespace.MATHML: + parser.adjustMathMLAttributes(token.attrs, xmlAttrs) + discard parser.insertForeignElement(token, tagname, namespace, false, + xmlAttrs) + if token.selfclosing: + if namespace == Namespace.SVG and + parser.atomToTagType(token.tagname) == TAG_SCRIPT: + script_end_tag + else: + pop_current_node + + template any_other_end_tag() = + for i in countdown(parser.openElements.high, 0): # loop + if i == 0: # fragment case + assert parser.fragment + break + let (node, nodeToken) = parser.openElements[i] + if i != parser.openElements.high and + parser.getNamespace(node) == Namespace.HTML: + return parser.processInHTMLContent(token, parser.insertionMode) + if nodeToken.tagname == token.tagname: + # Compare the start tag token, since it is guaranteed to be lower case. + # (The local name might have been adjusted to a non-lower-case string.) + parser.popElementsIncl(node) + break + + match token: + TokenType.CHARACTER_NULL => (block: parser.insertCharacter("\uFFFD")) + TokenType.CHARACTER_WHITESPACE => (block: parser.insertCharacter(token.s)) + TokenType.CHARACTER => (block: + parser.insertCharacter(token.s) + parser.framesetOk = false + ) + TokenType.COMMENT => (block: parser.insertComment(token)) + TokenType.DOCTYPE => (block: discard) + ("<b>", "<big>", "<blockquote>", "<body>", "<br>", "<center>", "<code>", + "<dd>", "<div>", "<dl>", "<dt>", "<em>", "<embed>", "<h1>", "<h2>", + "<h3>", "<h4>", "<h5>", "<h6>", "<head>", "<hr>", "<i>", "<img>", "<li>", + "<listing>", "<menu>", "<meta>", "<nobr>", "<ol>", "<p>", "<pre>", + "<ruby>", "<s>", "<small>", "<span>", "<strong>", "<strike>", "<sub>", + "<sup>", "<table>", "<tt>", "<u>", "<ul>", "<var>", + "<font>", # only if has "color", "face", or "size" + "</br>", "</p>") => (block: + if parser.atomToTagType(token.tagname) == TAG_FONT: + let atColor = parser.tagTypeToAtom(TAG_COLOR) + let atFace = parser.tagTypeToAtom(TAG_FACE) + let atSize = parser.tagTypeToAtom(TAG_SIZE) + if atColor notin token.attrs and atFace notin token.attrs and + atSize notin token.attrs: + any_other_start_tag + return + while not parser.isMathMLIntegrationPoint(parser.currentNode) and + not parser.isHTMLIntegrationPoint(parser.currentNodeToken) and + parser.getNamespace(parser.currentNode) != Namespace.HTML: + pop_current_node + return parser.processInHTMLContent(token, parser.insertionMode) + ) + TokenType.START_TAG => (block: + any_other_start_tag + ) + "</script>" => (block: + let namespace = parser.getNamespace(parser.currentNode) + let localName = parser.currentToken.tagname + # Any atom corresponding to the string "script" must have the same + # value as TAG_SCRIPT, so this is correct. + if namespace == Namespace.SVG and + parser.atomToTagType(localName) == TAG_SCRIPT: + script_end_tag + else: + any_other_end_tag + ) + TokenType.END_TAG => (block: any_other_end_tag) + return PRES_CONTINUE + +proc processToken[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + token: Token[Atom]): ParseResult = + if parser.ignoreLF: + parser.ignoreLF = false + if token.t == CHARACTER_WHITESPACE: + if token.s[0] == '\n': + if token.s.len == 1: + return PRES_CONTINUE + else: + token.s.delete(0..0) + if parser.openElements.len == 0 or + parser.getNamespace(parser.adjustedCurrentNode) == Namespace.HTML: + return parser.processInHTMLContent(token, parser.insertionMode) + let oe = parser.adjustedCurrentNodeToken + let oeTagType = parser.atomToTagType(oe.token.tagname) + let namespace = parser.getNamespace(oe.element) + const CharacterToken = {CHARACTER, CHARACTER_WHITESPACE, CHARACTER_NULL} + let mmlnoatoms = {TAG_MGLYPH, TAG_MALIGNMARK} + let ismmlip = parser.isMathMLIntegrationPoint(oe.element) + let ishtmlip = parser.isHTMLIntegrationPoint(oe) + if token.t == START_TAG and ( + let tagType = parser.atomToTagType(token.tagname) + ismmlip and tagType notin mmlnoatoms or ishtmlip or + namespace == Namespace.MATHML and oeTagType == TAG_ANNOTATION_XML and + tagType == TAG_SVG + ) or token.t in CharacterToken and (ismmlip or ishtmlip): + return parser.processInHTMLContent(token, parser.insertionMode) + return parser.processInForeignContent(token) + +const CaseTable = { + "altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "fedropshadow": "feDropShadow", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath", +} + +proc createCaseTable[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) = + for (k, v) in CaseTable: + let ka = parser.strToAtom(k) + let va = parser.strToAtom(v) + parser.caseTable[ka] = va + +const AdjustedTable = { + "attributename": "attributeName", + "attributetype": "attributeType", + "basefrequency": "baseFrequency", + "baseprofile": "baseProfile", + "calcmode": "calcMode", + "clippathunits": "clipPathUnits", + "diffuseconstant": "diffuseConstant", + "edgemode": "edgeMode", + "filterunits": "filterUnits", + "glyphref": "glyphRef", + "gradienttransform": "gradientTransform", + "gradientunits": "gradientUnits", + "kernelmatrix": "kernelMatrix", + "kernelunitlength": "kernelUnitLength", + "keypoints": "keyPoints", + "keysplines": "keySplines", + "keytimes": "keyTimes", + "lengthadjust": "lengthAdjust", + "limitingconeangle": "limitingConeAngle", + "markerheight": "markerHeight", + "markerunits": "markerUnits", + "markerwidth": "markerWidth", + "maskcontentunits": "maskContentUnits", + "maskunits": "maskUnits", + "numoctaves": "numOctaves", + "pathlength": "pathLength", + "patterncontentunits": "patternContentUnits", + "patterntransform": "patternTransform", + "patternunits": "patternUnits", + "pointsatx": "pointsAtX", + "pointsaty": "pointsAtY", + "pointsatz": "pointsAtZ", + "preservealpha": "preserveAlpha", + "preserveaspectratio": "preserveAspectRatio", + "primitiveunits": "primitiveUnits", + "refx": "refX", + "refy": "refY", + "repeatcount": "repeatCount", + "repeatdur": "repeatDur", + "requiredextensions": "requiredExtensions", + "requiredfeatures": "requiredFeatures", + "specularconstant": "specularConstant", + "specularexponent": "specularExponent", + "spreadmethod": "spreadMethod", + "startoffset": "startOffset", + "stddeviation": "stdDeviation", + "stitchtiles": "stitchTiles", + "surfacescale": "surfaceScale", + "systemlanguage": "systemLanguage", + "tablevalues": "tableValues", + "targetx": "targetX", + "targety": "targetY", + "textlength": "textLength", + "viewbox": "viewBox", + "viewtarget": "viewTarget", + "xchannelselector": "xChannelSelector", + "ychannelselector": "yChannelSelector", + "zoomandpan": "zoomAndPan", +} + +proc createAdjustedTable[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) = + for (k, v) in AdjustedTable: + let ka = parser.strToAtom(k) + let va = parser.strToAtom(v) + parser.adjustedTable[ka] = va + +const ForeignTable = { + "xlink:actuate": (PREFIX_XLINK, "actuate", Namespace.XLINK), + "xlink:arcrole": (PREFIX_XLINK, "arcrole", Namespace.XLINK), + "xlink:href": (PREFIX_XLINK, "href", Namespace.XLINK), + "xlink:role": (PREFIX_XLINK, "role", Namespace.XLINK), + "xlink:show": (PREFIX_XLINK, "show", Namespace.XLINK), + "xlink:title": (PREFIX_XLINK, "title", Namespace.XLINK), + "xlink:type": (PREFIX_XLINK, "type", Namespace.XLINK), + "xml:lang": (PREFIX_XML, "lang", Namespace.XML), + "xml:space": (PREFIX_XML, "space", Namespace.XML), + "xmlns": (NO_PREFIX, "xmlns", Namespace.XMLNS), + "xmlns:xlink": (PREFIX_XMLNS, "xlink", Namespace.XMLNS), +} + +proc createForeignTable[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) = + for (oldName, qualName) in ForeignTable: + let (prefix, newName, ns) = qualName + let oldNameAtom = parser.strToAtom(oldName) + let newNameAtom = parser.strToAtom(newName) + parser.foreignTable[oldNameAtom] = (prefix, ns, newNameAtom) + +proc initHTML5Parser*[Handle, Atom](dombuilder: DOMBuilder[Handle, Atom], + opts: HTML5ParserOpts[Handle, Atom]): HTML5Parser[Handle, Atom] = + ## Create and initialize a new HTML5Parser object from dombuilder `dombuilder` + ## and parser options `opts`. + ## + ## The generic `Handle` must be the node handle type of the DOM builder. The + ## generic `Atom` must be the interned string type of the DOM builder. + var parser = HTML5Parser[Handle, Atom]( + dombuilder: dombuilder, + opts: opts, + form: opts.formInit, + framesetOk: true + ) + if opts.ctx.isSome: + let ctxInit = opts.ctx.get + let ctx: OpenElement[Handle, Atom] = (ctxInit.element, Token[Atom]( + t: START_TAG, + tagname: ctxInit.startTagName + )) + parser.ctx = some(ctx) + for (element, tagName) in opts.openElementsInit: + let it = (element, Token[Atom](t: START_TAG, tagname: tagName)) + parser.openElements.add(it) + parser.createCaseTable() + parser.createAdjustedTable() + parser.createForeignTable() + if opts.pushInTemplate: + parser.templateModes.add(IN_TEMPLATE) + if opts.openElementsInit.len > 0: + parser.resetInsertionMode() + let tokstate = opts.initialTokenizerState + parser.tokenizer = newTokenizer[Handle, Atom](dombuilder, tokstate) + return parser + +proc parseChunk*[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + inputBuf: openArray[char]): ParseResult = + ## Parse a chunk of characters stored in `inputBuf` with `parser`. + var running = true + parser.tokenizer.inputBufIdx = 0 + while running: + running = parser.tokenizer.tokenize(inputBuf) != trDone + for i, token in parser.tokenizer.tokqueue: + let pres = parser.processToken(token) + if pres != PRES_CONTINUE: + assert pres != PRES_SCRIPT or i == parser.tokenizer.tokqueue.high + return pres + return PRES_CONTINUE + +func getInsertionPoint*(parser: HTML5Parser): int = + return parser.tokenizer.inputBufIdx + +proc finish*[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) = + ## Finish parsing the document associated with `parser`. + ## This will process an EOF token, and pop all elements from the stack of + ## open elements one by one. + var running = true + while running: + running = parser.tokenizer.finish() != trDone + for token in parser.tokenizer.tokqueue: + let pres = parser.processToken(token) + assert pres == PRES_CONTINUE + # pres == PRES_SCRIPT: this is unreachable. + # * Tokenizer's tokenizeEOF() can not emit end tag tokens, ergo no + # </script> will be processed here. + # * In some cases, tokenize() is called before tokenizeEOF(), to flush + # characters stuck in the internal peekBuf. This can happen if: + # 1. eatStr returns esrRetry in a previous pass. Here, peekBuf can + # contain any prefix of strings passed to eatStr/NoCase(), which + # crucially is never a potential </script> tag (or indeed, + # nothing that can start with </). + # 2. The "named character reference" state is interrupted. In this + # case, peekBuf is a prefix of at least one named character reference; + # obviously these can not be </script> tags either, since they + # all match the regex `&[a-zA-Z]+'. + # pres == PRES_STOP: unreachable for reasons almost identical to those + # outlined in PRES_SCRIPT. PRES_STOP can only be returned after a <meta> + # tag is processed; just like with end tags, the tokenizer cannot emit + # start tags in finish(). + discard parser.processInHTMLContent(Token[Atom](t: EOF), parser.insertionMode) + while parser.openElements.len > 0: + pop_current_node diff --git a/lib/chame0/chame/htmlparseriface.nim b/lib/chame0/chame/htmlparseriface.nim new file mode 100644 index 00000000..5be4c536 --- /dev/null +++ b/lib/chame0/chame/htmlparseriface.nim @@ -0,0 +1,343 @@ +when defined(nimdocdummy): + ## Interface definitions for htmlparser. + ## + ## This exists to make implementing the DOMBuilder interface less painful. Two + ## categories of hooks exist: + ## 1. Mandatory hooks: these must be implemented by all users, or Chame will + ## not compile. + ## 2. Optional hooks: these may be omitted if your DOM does not need + ## them. (You do not have to do anything special for this, just don't + ## implement them.) + ## + ## Usage: + ## 1. Put a type clause with your generic types in your DOM builder interface: + ## ```nim + ## type + ## DOMBuilderImpl = MyDOMBuilder + ## AtomImpl = MyAtom + ## HandleImpl = MyHandle + ## ``` + ## 2. **Include** (**not** import) this file: + ## ```nim + ## include chame/htmlparseriface + ## ``` + ## 3. Implement all `*Impl` functions until the compiler no longer complains. + ## + ## Then you can call `parseHTML` with your custom DOMBuilder. + ## + ## You can also just implement the required functions without including + ## this interface, but then you will have to do the casting from + ## `DOMBuilder[HandleImpl, AtomImpl]` -> `DOMBuilderImpl` manually and will + ## get uglier error messages when a function is missing. + ## + ## Note that when using this interface you can't use procs with different side + ## effects than declared, so e.g. `func getDocumentImpl(...` **will not work**. + ## You must use `proc getDocumentImpl(...` instead. + ## + ## Also, make sure that parameter names match the ones defined here, + ## otherwise you are likely to get strange compilation errors. + ## + ## ## Optional hooks + ## Following procedures are optional hooks; implementations of this interface + ## can choose to leave them out without getting compilation errors. + ## + ## + ## ```nim + ## proc setQuirksModeImpl(builder: DOMBuilderBase, quirksMode: QuirksMode) + ## ``` + ## + ## Set quirks mode to either `QUIRKS` or `LIMITED_QUIRKS`. `NO_QUIRKS` is the + ## default and is therefore never passed here. + ## + ## + ## ```nim + ## proc setEncodingImpl(builder: DOMBuilderBase, encoding: string): + ## SetEncodingResult + ## ``` + ## + ## Called whenever a <meta charset=... or a <meta http-equiv=... tag + ## containing a non-empty character set is encountered. A SetEncodingResult + ## return value is expected, which is either `SET_ENCODING_STOP`, stopping + ## the parser, or `SET_ENCODING_CONTINUE`, allowing the parser to continue. + ## + ## Note that htmlparser no longer contains any encoding-related logic, not + ## even UTF-8 validation. Implementing this is left to the caller. (For an + ## example, see minidom_cs which implements decoding of all character sets + ## in the WHATWG recommendation.) + ## + ## + ## ```nim + ## proc elementPoppedImpl(builder: DOMBuilderBase, handle: HandleImpl) + ## ``` + ## + ## Called when an element is popped from the stack of open elements + ## (i.e. when it has been closed.) + ## + ## + ## ```nim + ## proc setScriptAlreadyStartedImpl(builder: DOMBuilderBase, handle: HandleImpl) + ## ``` + ## + ## Set the "already started" flag for the script element. + ## + ## Note: this flag is not togglable, so implementations of this callback + ## should just set the flag to true. + ## + ## + ## ```nim + ## proc associateWithFormImpl(builder: DOMBuilderBase, element, form, + ## intendedParent: HandleImpl) + ## ``` + ## + ## Called after createElement. Attempts to set form for form-associated + ## elements. + ## + ## Note: the DOM builder is responsible for checking whether the intended + ## parent and the form element are in the same tree. + # Dummy definitions + import std/tables + import htmlparser + type + HandleImpl = int + AtomImpl = int + DOMBuilderImpl = DOMBuilder[HandleImpl, AtomImpl] + +# DOMBuilder +static: + # DOMBuilderImpl must be an instance of DOMBuilder with the handle type + # HandleImpl and atom type AtomImpl. + doAssert DOMBuilderImpl is DOMBuilder[HandleImpl, AtomImpl] + +converter toDOMBuilderImpl(dombuilder: DOMBuilder[HandleImpl, AtomImpl]): + DOMBuilderImpl = + return DOMBuilderImpl(dombuilder) + +when defined(nimdocdummy): + import std/macros + + macro doc(f: untyped) = + f[0] = newNimNode(nnkPostfix).add(ident("*")).add(f[0]) + f +else: + macro doc(f: untyped) = f + +proc strToAtomImpl(builder: DOMBuilderImpl, s: string): AtomImpl {.doc.} + ## Turn a string `s` into an Atom. + ## + ## This must always convert *every* string with the same value into the + ## same Atom. We recommend using a hash table-based approach for this + ## (see e.g. MAtomFactory). + +proc tagTypeToAtomImpl(builder: DOMBuilderImpl, tagType: TagType): AtomImpl + {.doc.} + ## Turn a TagType `tagType` into an Atom. + ## + ## Every TagType `tagType` must always be converted into the same atom as + ## that represented by its stringifier. An implementation could be: + ## ```nim + ## proc tagTypeToAtomImpl(builder: DOMBuilderImpl, tt: TagType): AtomImpl = + ## assert tt != TAG_UNKNOWN # parser never calls this with TAG_UNKNOWN. + ## return builder.strToAtomImpl($tt) + ## ``` + +proc atomToTagTypeImpl(builder: DOMBuilderImpl, atom: AtomImpl): TagType {.doc.} + ## Turn an Atom `atom` into a TagType. This is the inverse function of + ## `tagTypeToAtomImpl`. + +proc getDocumentImpl(builder: DOMBuilderImpl): HandleImpl {.doc.} + ## Get the root document node's handle. + ## This must not return nil, not even in the fragment parsing case. + +proc getParentNodeImpl(builder: DOMBuilderImpl, handle: HandleImpl): + Option[HandleImpl] {.doc.} + ## Retrieve a handle to the parent node. + ## May return none(Handle) if no parent node exists. + +proc createHTMLElementImpl(builder: DOMBuilderImpl): HandleImpl {.doc.} + ## Create a new <html> element node. The tag type of the created element must + ## be TAG_HTML, and its namespace must be Namespace.HTML. + ## + ## This should not be confused with the "create an element for a token" + ## step, which is not executed here. + +proc createElementForTokenImpl(builder: DOMBuilderImpl, localName: AtomImpl, + namespace: Namespace, intendedParent: HandleImpl, + htmlAttrs: Table[AtomImpl, string], xmlAttrs: seq[ParsedAttr[AtomImpl]]): + HandleImpl {.doc.} + ## Create a new element node. + ## + ## `localName` is an Atom representing the tag name of the start token. + ## + ## Note that the parser determines the TagType of an element by its namespace + ## and localName; for non-HTML elements it is always considered TAG_UNKNOWN. + ## + ## (However, tokens have no namespace, so TAG_SVG and TAG_MATHML can still + ## used on them.) + ## + ## `namespace` is the namespace of the new element. For HTML elements, + ## it's HTML; for embedded SVG/MathML elements, it is Namespace.SVG or + ## Namespace.MATHML. No other namespace is used currently. + ## + ## `htmlAttrs` is a seq of the new elements attributes. It only contains + ## attributes with prefix NO_PREFIX and namespace NO_NAMESPACE; adjusted + ## foreign of embedded SVG/MathML elements that *do* have namespaces are + ## *not* included, these can be found in `xmlAttrs`. All attribute names in + ## `htmlAttrs` are guaranteed to be unique, but the parser makes no guarantees + ## about the order of the attributes. (TODO maybe attrs should be a hash + ## table after all?) + ## + ## `xmlAttrs` is a list of (XML) adjusted attributes. They are only set + ## for elements in the MathML or SVG namespace, for which there are + ## pre-defined attributes in the standard with names whose casing, namespace, + ## and namespace prefixes must be adjusted by the parser. + ## + ## `intendedParent` is the intended parent of the element, as passed to the + ## "create an element for a token" step. This may be used for looking up + ## custom element definitions. + ## + ## Implementers of this function are encouraged to consult the + ## [create an element for a token](https://html.spec.whatwg.org/multipage/parsing.html#create-an-element-for-the-token) + ## section of the standard. In particular, steps 3 (Let document be intended + ## parent's node document.) to 13 (If element is a resettable element...) + ## should be implemented. + ## + ## Note that step 14. (If element is a form-associated element...) should + ## *not* be implemented here. In fact, it is impossible to do so without + ## access to the parser internals, so for this step, the parser will call + ## associateWithFormImpl if all conditions (except "the intended parent is + ## in the same tree as the element pointed to by the form element pointer") + ## are fulfilled. + +proc getLocalNameImpl(builder: DOMBuilderImpl, handle: HandleImpl): AtomImpl + {.doc.} + ## Retrieve the local name (also known as the tag name) of the element + ## represented by `handle`. + +proc getNamespaceImpl(builder: DOMBuilderImpl, handle: HandleImpl): Namespace + {.doc.} + ## Retrieve the namespace of element. For HTML elements, this should be + ## `Namespace.HTML`. For embedded SVG or MathML elements, it should be + ## `Namespace.SVG` or `Namespace.MathML`, respectively. + ## + ## (In general, you should just return the namespace that was passed to + ## `createElement`.) + +proc getTemplateContentImpl(builder: DOMBuilderImpl, handle: HandleImpl): + HandleImpl {.doc.} + ## Retrieve a handle to the template element's contents. Every element + ## where `builder.atomToTagTypeImpl(element.localName)` equals TAG_TEMPLATE + ## and `builder.getNamespaceImpl(element)` equals `Namespace.HTML` must have + ## an associated "template contents" node which this function must return. + +proc addAttrsIfMissingImpl(builder: DOMBuilderImpl, handle: HandleImpl, + attrs: Table[AtomImpl, string]) {.doc.} + ## Add the attributes in `attrs` to the element node `element`. + ## This is only called with the HTML and BODY tags, when more than one + ## exists in a document. + ## + ## Pseudocode implementation: + ## ```nim + ## for attr in attrs: + ## if attr.name notin element.attrs: + ## element.attrs.add(attr) + ## ``` + +proc createCommentImpl(builder: DOMBuilderImpl, text: string): HandleImpl + {.doc.} + ## Create a new comment node. `text` is a string representing the new comment + ## node's character data. + +proc createDocumentTypeImpl(builder: DOMBuilderImpl, name, publicId, + systemId: string): HandleImpl {.doc.} + ## Create a new document type node. + +proc insertBeforeImpl(builder: DOMBuilderImpl, parent, child: HandleImpl, + before: Option[HandleImpl]) {.doc.} + ## Insert node `child` before the node called `before`. + ## + ## If `before` is `none(Handle)`, `child` is expected to be appended to + ## `parent`'s node list. + ## + ## If `child` is a text, and its previous sibling after insertion is a + ## text as well, then they should be merged. `before` is never a + ## text node (and thus never has to be merged). + ## + ## Note: parent may be either an Element or a Document node. + +proc insertTextImpl(builder: DOMBuilderImpl, parent: HandleImpl, text: string, + before: Option[HandleImpl]) {.doc.} + ## Insert a text node at the specified location with contents `text`. If + ## the specified location has a previous sibling that is a text node, no new + ## text node should be created, but instead `text` should be appended to the + ## previous sibling's character data (or if `before` is `none(Handle)`, + ## to the last element). + +proc removeImpl(builder: DOMBuilderImpl, child: HandleImpl) {.doc.} + ## If `child` does not have a parent node, do nothing. Otherwise, remove + ## `child` from its parent node. + +proc moveChildrenImpl(builder: DOMBuilderImpl, fromNode, toNode: HandleImpl) + {.doc.} + ## Remove all children from the node `fromHandle`, then append them to + ## `toHandle`. + +when defined(nimdocdummy): + # Dummy definitions + proc strToAtomImpl(builder: DOMBuilderImpl, s: string): AtomImpl = + discard + + proc tagTypeToAtomImpl(builder: DOMBuilderImpl, tagType: TagType): AtomImpl = + discard + + proc atomToTagTypeImpl(builder: DOMBuilderImpl, atom: AtomImpl): TagType = + discard + + proc getDocumentImpl(builder: DOMBuilderImpl): HandleImpl = + discard + + proc getParentNodeImpl(builder: DOMBuilderImpl, handle: HandleImpl): + Option[HandleImpl] = + discard + + proc createHTMLElementImpl(builder: DOMBuilderImpl): HandleImpl = discard + + proc createElementForTokenImpl(builder: DOMBuilderImpl, localName: AtomImpl, + namespace: Namespace, intendedParent: HandleImpl, + htmlAttrs: Table[AtomImpl, string], xmlAttrs: seq[ParsedAttr[AtomImpl]]): + HandleImpl = discard + + proc getLocalNameImpl(builder: DOMBuilderImpl, handle: HandleImpl): AtomImpl = + discard + + proc getNamespaceImpl(builder: DOMBuilderImpl, handle: HandleImpl): + Namespace = + discard + + proc getTemplateContentImpl(builder: DOMBuilderImpl, handle: HandleImpl): + HandleImpl = + discard + + proc addAttrsIfMissingImpl(builder: DOMBuilderImpl, handle: HandleImpl, + attrs: Table[AtomImpl, string]) = + discard + + proc createCommentImpl(builder: DOMBuilderImpl, text: string): HandleImpl = + discard + + proc createDocumentTypeImpl(builder: DOMBuilderImpl, name, publicId, + systemId: string): HandleImpl = + discard + + proc insertBeforeImpl(builder: DOMBuilderImpl, parent, child: HandleImpl, + before: Option[HandleImpl]) = + discard + + proc insertTextImpl(builder: DOMBuilderImpl, parent: HandleImpl, text: string, + before: Option[HandleImpl]) = + discard + + proc removeImpl(builder: DOMBuilderImpl, child: HandleImpl) = + discard + + proc moveChildrenImpl(builder: DOMBuilderImpl, fromNode, toNode: HandleImpl) = + discard diff --git a/lib/chame0/chame/htmltokenizer.nim b/lib/chame0/chame/htmltokenizer.nim new file mode 100644 index 00000000..e9180a4d --- /dev/null +++ b/lib/chame0/chame/htmltokenizer.nim @@ -0,0 +1,1422 @@ +{.experimental: "overloadableEnums".} + +import std/options +import std/strformat +import std/strutils +import std/tables + +import dombuilder +import entity_gen +import tokstate + +export tokstate + +type + Tokenizer*[Handle, Atom] = object + dombuilder: DOMBuilder[Handle, Atom] + state*: TokenizerState + rstate: TokenizerState # return state + # temporary buffer (mentioned by the standard, but also used for attribute + # names) + tmp: string + code: uint32 # codepoint of current numeric character reference + tok: Token[Atom] # current token to be emitted + laststart*: Token[Atom] # last start tag token + attrna: Atom # atom representing attrn after the attribute name is closed + attrv: string # buffer for attribute values + attr: bool # is there already an attr in the previous values? + hasnonhtml*: bool # does the stack of open elements have a non-HTML node? + tokqueue*: seq[Token[Atom]] # queue of tokens to be emitted in this iteration + charbuf: string # buffer for character tokens + tagNameBuf: string # buffer for storing the tag name + peekBuf: array[64, char] # a stack with the last element at peekBufLen - 1 + peekBufLen: int + inputBufIdx*: int # last character consumed in input buf + ignoreLF: bool # ignore the next consumed line feed (for CRLF normalization) + isend: bool # if consume returns -1 and isend, we are at EOF + isws: bool # is the current character token whitespace-only? + + TokenType* = enum + DOCTYPE, START_TAG, END_TAG, COMMENT, CHARACTER, CHARACTER_WHITESPACE, + CHARACTER_NULL, EOF + + Token*[Atom] = ref object + case t*: TokenType + of DOCTYPE: + quirks*: bool + name*: Option[string] + pubid*: Option[string] + sysid*: Option[string] + of START_TAG, END_TAG: + selfclosing*: bool + tagname*: Atom + attrs*: Table[Atom, string] + of CHARACTER, CHARACTER_WHITESPACE: + s*: string + of COMMENT: + data*: string + of EOF, CHARACTER_NULL: discard + +const AsciiUpperAlpha = {'A'..'Z'} +const AsciiLowerAlpha = {'a'..'z'} +const AsciiAlpha = (AsciiUpperAlpha + AsciiLowerAlpha) +const AsciiDigit = {'0'..'9'} +const AsciiAlphaNumeric = AsciiAlpha + AsciiDigit +const AsciiWhitespace = {' ', '\n', '\r', '\t', '\f'} + +func `$`*(tok: Token): string = + case tok.t + of DOCTYPE: fmt"{tok.t} {tok.name} {tok.pubid} {tok.sysid} {tok.quirks}" + of START_TAG, END_TAG: fmt"{tok.t} {tok.tagname} {tok.selfclosing} {tok.attrs}" + of CHARACTER, CHARACTER_WHITESPACE: $tok.t & " " & tok.s + of CHARACTER_NULL: $tok.t + of COMMENT: fmt"{tok.t} {tok.data}" + of EOF: fmt"{tok.t}" + +proc strToAtom[Handle, Atom](tokenizer: Tokenizer[Handle, Atom], + s: string): Atom = + mixin strToAtomImpl + return tokenizer.dombuilder.strToAtomImpl(s) + +proc newTokenizer*[Handle, Atom](dombuilder: DOMBuilder[Handle, Atom], + initialState = DATA): Tokenizer[Handle, Atom] = + return Tokenizer[Handle, Atom](state: initialState, dombuilder: dombuilder) + +proc reconsume(tokenizer: var Tokenizer, s: openArray[char]) = + for i in countdown(s.high, 0): + tokenizer.peekBuf[tokenizer.peekBufLen] = s[i] + inc tokenizer.peekBufLen + +proc reconsume(tokenizer: var Tokenizer, c: char) = + tokenizer.peekBuf[tokenizer.peekBufLen] = c + inc tokenizer.peekBufLen + +proc consume(tokenizer: var Tokenizer, ibuf: openArray[char]): int = + if tokenizer.peekBufLen > 0: + dec tokenizer.peekBufLen + return int(tokenizer.peekBuf[tokenizer.peekBufLen]) + if tokenizer.inputBufIdx >= ibuf.len: + return -1 + var c = ibuf[tokenizer.inputBufIdx] + inc tokenizer.inputBufIdx + if tokenizer.ignoreLF: + if c == '\n': + if tokenizer.inputBufIdx >= ibuf.len: + return -1 + c = ibuf[tokenizer.inputBufIdx] + inc tokenizer.inputBufIdx + tokenizer.ignoreLF = false + if c == '\r': + tokenizer.ignoreLF = true + c = '\n' + return int(c) + +proc flushChars[Handle, Atom](tokenizer: var Tokenizer[Handle, Atom]) = + if tokenizer.charbuf.len > 0: + if tokenizer.isws: + tokenizer.tokqueue.add(Token[Atom]( + t: CHARACTER_WHITESPACE, + s: tokenizer.charbuf + )) + else: + tokenizer.tokqueue.add(Token[Atom](t: CHARACTER, s: tokenizer.charbuf)) + tokenizer.isws = false + tokenizer.charbuf.setLen(0) + +const AttributeStates = { + ATTRIBUTE_VALUE_DOUBLE_QUOTED, ATTRIBUTE_VALUE_SINGLE_QUOTED, + ATTRIBUTE_VALUE_UNQUOTED +} + +func consumedAsAttribute(tokenizer: Tokenizer): bool = + return tokenizer.rstate in AttributeStates + +proc appendToAttrValue(tokenizer: var Tokenizer, s: openArray[char]) = + if tokenizer.attr: + for c in s: + tokenizer.attrv &= c + +proc emit(tokenizer: var Tokenizer, c: char) = + let isws = c in AsciiWhitespace + if tokenizer.isws != isws: + # Emit whitespace & non-whitespace separately. + tokenizer.flushChars() + tokenizer.isws = isws + tokenizer.charbuf &= c + +type CharRefResult = tuple[i, ci: int, entry: cstring] + +proc findCharRef(tokenizer: var Tokenizer, c: char, ibuf: openArray[char]): + CharRefResult = + var i = charMap[c] + if i == -1: + return (0, 0, nil) + tokenizer.tmp &= c + var entry = entityMap[i] + var ci = 1 + let oc = c + while entry != nil and entry[ci] != ':': + let n = tokenizer.consume(ibuf) + if n == -1 and not tokenizer.isend: + # We must retry at the next iteration :/ + #TODO it would be nice to save state in this case + return (-1, 0, nil) + if n != int(entry[ci]): + entry = nil + # i is not the right entry. + while entry == nil: + if i == 0: + # See below; we avoid flushing the last character consumed by + # decrementing `ci'. + dec ci + break + dec i + entry = entityMap[i] + if oc != entry[0]: + # Out of entries that start with the character `oc'; give up. + # We must not flush the last character consumed as a character + # reference, since it is not a prefix of any entry (and can indeed be + # markup; e.g. in the case of `&g<'), so decrement `ci' here. + dec ci + entry = nil + break + var j = 1 + while j < tokenizer.tmp.len - 1: + if entry[j] == ':': + # Full match: entry is a prefix of the previous entry we inspected. + break + if tokenizer.tmp[j + 1] != entry[j]: + # Characters consumed until now are not a prefix of entry. + # Try the next one instead. + entry = nil + break + inc j + if entry != nil: + if entry[j] == ':': + # Full match: make sure the outer loop exits. + ci = j - 1 + elif int(entry[j]) == n: + # Partial match, *including c*. (This is never reached with n == -1) + ci = j + else: + # Continue with the loop. + # If entry is set to non-nil after this iteration, then ci will + # also be set appropriately. + # Otherwise, if entry remains nil, ci will point to the first + # non-matching character in tmp; this will be reconsumed. + entry = nil + if n != -1: + tokenizer.tmp &= cast[char](n) + inc ci + return (i, ci, entry) + +proc appendAttrOrEmit(tokenizer: var Tokenizer, s: openArray[char]) = + if tokenizer.consumedAsAttribute(): + tokenizer.appendToAttrValue(s) + else: + for c in s: + tokenizer.emit(c) + +proc numericCharacterReferenceEndState(tokenizer: var Tokenizer) = + const ControlMap = [ + 0x20ACu16, 0, 0x201A, 0x192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x2C6, 0x2030, 0x160, 0x2039, 0x152, 0, 0x17D, 0, + 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x2DC, 0x2122, 0x161, 0x203A, 0x153, 0, 0x17E, 0x178 + ] + var u = tokenizer.code + let cc = u - 0x80 + if cc < ControlMap.len and ControlMap[cc] != 0: + u = ControlMap[cc] + elif u == 0x00 or u > 0x10FFFF or u in 0xD800u32..0xDFFFu32: + u = 0xFFFD + var s = "" + if u < 0x80: + s = $char(u) + elif u < 0x800: + s = char(u shr 6 or 0xC0) & + char(u and 0x3F or 0x80) + elif u < 0x10000: + s = char(u shr 12 or 0xE0) & + char(u shr 6 and 0x3F or 0x80) & + char(u and 0x3F or 0x80) + else: + s = char(u shr 18 or 0xF0) & + char(u shr 12 and 0x3F or 0x80) & + char(u shr 6 and 0x3F or 0x80) & + char(u and 0x3F or 0x80) + tokenizer.appendAttrOrEmit(s) + +proc flushAttr(tokenizer: var Tokenizer) = + tokenizer.tok.attrs[tokenizer.attrna] = tokenizer.attrv + +type EatStrResult = enum + esrFail, esrSuccess, esrRetry + +proc eatStr(tokenizer: var Tokenizer, c: char, s, ibuf: openArray[char]): + EatStrResult = + var cs = $c + for c in s: + let n = tokenizer.consume(ibuf) + if n != -1: + cs &= cast[char](n) + if n != int(c): + tokenizer.reconsume(cs) + if n == -1 and not tokenizer.isend: + return esrRetry + return esrFail + return esrSuccess + +proc eatStrNoCase0(tokenizer: var Tokenizer, c: char, s, ibuf: openArray[char]): + EatStrResult = + var cs = $c + for c in s: + let n = tokenizer.consume(ibuf) + if n != -1: + cs &= cast[char](n) + if n == -1 or cast[char](n).toLowerAscii() != c: + tokenizer.reconsume(cs) + if n == -1 and not tokenizer.isend: + return esrRetry + return esrFail + return esrSuccess + +# convenience template for eatStrNoCase0 (to make sure it's called correctly) +template eatStrNoCase(tokenizer: var Tokenizer, c: char, s: static string, + ibuf: openArray[char]): EatStrResult = + const s0 = s.toLowerAscii() + tokenizer.eatStrNoCase0(c, s0, ibuf) + +proc flushTagName(tokenizer: var Tokenizer) = + tokenizer.tok.tagname = tokenizer.strToAtom(tokenizer.tagNameBuf) + +proc emitTmp(tokenizer: var Tokenizer) = + if tokenizer.isws: + tokenizer.flushChars() + tokenizer.charbuf &= "</" + tokenizer.charbuf &= tokenizer.tmp + +# if true, redo +proc tokenizeEOF[Handle, Atom](tokenizer: var Tokenizer[Handle, Atom]): bool = + tokenizer.tokqueue.setLen(0) + if tokenizer.isws: + tokenizer.flushChars() + case tokenizer.state + of TAG_OPEN, RCDATA_LESS_THAN_SIGN, RAWTEXT_LESS_THAN_SIGN, + SCRIPT_DATA_LESS_THAN_SIGN, SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN: + tokenizer.charbuf &= '<' + of END_TAG_OPEN, RCDATA_END_TAG_OPEN, RAWTEXT_END_TAG_OPEN, + SCRIPT_DATA_END_TAG_OPEN, SCRIPT_DATA_ESCAPED_END_TAG_OPEN: + tokenizer.charbuf &= "</" + of RCDATA_END_TAG_NAME, RAWTEXT_END_TAG_NAME, SCRIPT_DATA_END_TAG_NAME, + SCRIPT_DATA_ESCAPED_END_TAG_NAME: + tokenizer.emitTmp() + of BOGUS_COMMENT, BOGUS_DOCTYPE, COMMENT_END_DASH, + COMMENT_END, COMMENT_END_BANG, COMMENT_LESS_THAN_SIGN_BANG_DASH, + COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH, COMMENT_START_DASH, COMMENT, + COMMENT_START, COMMENT_LESS_THAN_SIGN, COMMENT_LESS_THAN_SIGN_BANG: + tokenizer.flushChars() + tokenizer.tokqueue.add(tokenizer.tok) + of MARKUP_DECLARATION_OPEN: + tokenizer.flushChars() + tokenizer.tokqueue.add(Token[Atom](t: COMMENT)) + of DOCTYPE, BEFORE_DOCTYPE_NAME: + tokenizer.flushChars() + tokenizer.tokqueue.add(Token[Atom](t: DOCTYPE, quirks: true)) + of DOCTYPE_NAME, AFTER_DOCTYPE_NAME, AFTER_DOCTYPE_PUBLIC_KEYWORD, + BEFORE_DOCTYPE_PUBLIC_IDENTIFIER, + DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED, + DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED, + AFTER_DOCTYPE_PUBLIC_IDENTIFIER, + BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS, + AFTER_DOCTYPE_SYSTEM_KEYWORD, BEFORE_DOCTYPE_SYSTEM_IDENTIFIER, + DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED, + DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED, + AFTER_DOCTYPE_SYSTEM_IDENTIFIER: + tokenizer.tok.quirks = true + tokenizer.flushChars() + tokenizer.tokqueue.add(tokenizer.tok) + of CDATA_SECTION_BRACKET: + tokenizer.charbuf &= ']' + of CDATA_SECTION_END: + tokenizer.charbuf &= "]]" + of CHARACTER_REFERENCE: + tokenizer.appendAttrOrEmit("&") + tokenizer.state = tokenizer.rstate + return true + of AMBIGUOUS_AMPERSAND_STATE: + tokenizer.state = tokenizer.rstate + return true + of NAMED_CHARACTER_REFERENCE, HEXADECIMAL_CHARACTER_REFERENCE_START, + NUMERIC_CHARACTER_REFERENCE: + tokenizer.appendAttrOrEmit(tokenizer.tmp) + tokenizer.state = tokenizer.rstate + return true + of HEXADECIMAL_CHARACTER_REFERENCE, DECIMAL_CHARACTER_REFERENCE: + tokenizer.numericCharacterReferenceEndState() + # we unnecessarily consumed once so reconsume + tokenizer.state = tokenizer.rstate + return true + else: discard + tokenizer.flushChars() + false + +type TokenizeResult* = enum + trDone, trEmit + +proc tokenize*[Handle, Atom](tokenizer: var Tokenizer[Handle, Atom], + ibuf: openArray[char]): TokenizeResult = + template emit(s: static string) = + static: + doAssert AsciiWhitespace notin s + if tokenizer.isws: + tokenizer.flushChars() + tokenizer.charbuf &= s + template emit(ch: char) = + tokenizer.emit(ch) + template emit_null = + tokenizer.flushChars() + tokenizer.tokqueue.add(Token[Atom](t: CHARACTER_NULL)) + template prepare_attrs_if_start = + if tokenizer.tok.t == START_TAG and tokenizer.attr: + tokenizer.flushAttr() + template emit_tok = + tokenizer.flushChars() + tokenizer.tokqueue.add(tokenizer.tok) + template emit_replacement = emit "\uFFFD" + template switch_state(s: TokenizerState) = + tokenizer.state = s + template switch_state_return(s: TokenizerState) = + tokenizer.rstate = tokenizer.state + tokenizer.state = s + template is_appropriate_end_tag_token(): bool = + tokenizer.laststart != nil and + tokenizer.laststart.tagname == tokenizer.tok.tagname + template start_new_attribute = + if tokenizer.tok.t == START_TAG and tokenizer.attr: + # This can also be called with tok.t == END_TAG, in that case we do + # not want to flush attributes. + tokenizer.flushAttr() + tokenizer.tmp = "" + tokenizer.attrv = "" + tokenizer.attr = true + template leave_attribute_name_state = + tokenizer.attrna = tokenizer.strToAtom(tokenizer.tmp) + if tokenizer.attrna in tokenizer.tok.attrs: + tokenizer.attr = false + template new_token(t: Token) = + tokenizer.attr = false + tokenizer.tok = t + + tokenizer.tokqueue.setLen(0) + + while true: + if tokenizer.tokqueue.len > 0: + return trEmit + let n = tokenizer.consume(ibuf) + if n == -1: + break # trDone + let c = cast[char](n) + template reconsume_in(s: TokenizerState) = + tokenizer.reconsume(c) + switch_state s + + case tokenizer.state + of DATA: + case c + of '&': switch_state_return CHARACTER_REFERENCE + of '<': switch_state TAG_OPEN + of '\0': emit_null + else: emit c + + of RCDATA: + case c + of '&': switch_state_return CHARACTER_REFERENCE + of '<': switch_state RCDATA_LESS_THAN_SIGN + of '\0': emit_replacement + else: emit c + + of RAWTEXT: + case c + of '<': switch_state RAWTEXT_LESS_THAN_SIGN + of '\0': emit_replacement + else: emit c + + of SCRIPT_DATA: + case c + of '<': switch_state SCRIPT_DATA_LESS_THAN_SIGN + of '\0': emit_replacement + else: emit c + + of PLAINTEXT: + case c + of '\0': emit_replacement + else: emit c + + of TAG_OPEN: + case c + of '!': switch_state MARKUP_DECLARATION_OPEN + of '/': switch_state END_TAG_OPEN + of AsciiAlpha: + new_token Token[Atom](t: START_TAG) + tokenizer.laststart = tokenizer.tok + tokenizer.tagNameBuf = $c.toLowerAscii() + # note: was reconsume + switch_state TAG_NAME + of '?': + new_token Token[Atom](t: COMMENT, data: "?") + # note: was reconsume + switch_state BOGUS_COMMENT + else: + emit '<' + reconsume_in DATA + + of END_TAG_OPEN: + case c + of AsciiAlpha: + new_token Token[Atom](t: END_TAG) + tokenizer.tagNameBuf = $c.toLowerAscii() + # note: was reconsume + switch_state TAG_NAME + of '>': switch_state DATA + else: + new_token Token[Atom](t: COMMENT) + reconsume_in BOGUS_COMMENT + + of TAG_NAME: + case c + of AsciiWhitespace: + tokenizer.flushTagName() + switch_state BEFORE_ATTRIBUTE_NAME + of '/': + tokenizer.flushTagName() + switch_state SELF_CLOSING_START_TAG + of '>': + switch_state DATA + tokenizer.flushTagName() + emit_tok + of '\0': tokenizer.tagNameBuf &= "\uFFFD" + else: tokenizer.tagNameBuf &= c.toLowerAscii() + + of RCDATA_LESS_THAN_SIGN: + case c + of '/': + tokenizer.tmp = "" + switch_state RCDATA_END_TAG_OPEN + else: + emit '<' + reconsume_in RCDATA + + of RCDATA_END_TAG_OPEN: + case c + of AsciiAlpha: + new_token Token[Atom](t: END_TAG) + tokenizer.tagNameBuf = $c.toLowerAscii() + tokenizer.tmp &= c + # note: was reconsume + switch_state RCDATA_END_TAG_NAME + else: + emit "</" + reconsume_in RCDATA + + of RCDATA_END_TAG_NAME: + template anything_else = + tokenizer.emitTmp() + reconsume_in RCDATA + case c + of AsciiWhitespace: + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state BEFORE_ATTRIBUTE_NAME + else: + anything_else + of '/': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state SELF_CLOSING_START_TAG + else: + anything_else + of '>': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state DATA + emit_tok + else: + anything_else + of AsciiAlpha: # note: merged upper & lower + tokenizer.tagNameBuf &= c.toLowerAscii() + tokenizer.tmp &= c + else: + anything_else + + of RAWTEXT_LESS_THAN_SIGN: + case c + of '/': + tokenizer.tmp = "" + switch_state RAWTEXT_END_TAG_OPEN + else: + emit '<' + reconsume_in RAWTEXT + + of RAWTEXT_END_TAG_OPEN: + case c + of AsciiAlpha: + new_token Token[Atom](t: END_TAG) + tokenizer.tagNameBuf = $c.toLowerAscii() + tokenizer.tmp &= c + # note: was reconsume + switch_state RAWTEXT_END_TAG_NAME + else: + emit "</" + reconsume_in RAWTEXT + + of RAWTEXT_END_TAG_NAME: + template anything_else = + tokenizer.emitTmp() + reconsume_in RAWTEXT + case c + of AsciiWhitespace: + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state BEFORE_ATTRIBUTE_NAME + else: + anything_else + of '/': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state SELF_CLOSING_START_TAG + else: + anything_else + of '>': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state DATA + emit_tok + else: + anything_else + of AsciiAlpha: # note: merged upper & lower + tokenizer.tagNameBuf &= c.toLowerAscii() + tokenizer.tmp &= c + else: + anything_else + + of SCRIPT_DATA_LESS_THAN_SIGN: + case c + of '/': + tokenizer.tmp = "" + switch_state SCRIPT_DATA_END_TAG_OPEN + of '!': + switch_state SCRIPT_DATA_ESCAPE_START + emit "<!" + else: + emit '<' + reconsume_in SCRIPT_DATA + + of SCRIPT_DATA_END_TAG_OPEN: + case c + of AsciiAlpha: + new_token Token[Atom](t: END_TAG) + tokenizer.tagNameBuf = $c.toLowerAscii() + tokenizer.tmp &= c + # note: was reconsume + switch_state SCRIPT_DATA_END_TAG_NAME + else: + emit "</" + reconsume_in SCRIPT_DATA + + of SCRIPT_DATA_END_TAG_NAME: + template anything_else = + tokenizer.emitTmp() + reconsume_in SCRIPT_DATA + case c + of AsciiWhitespace: + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state BEFORE_ATTRIBUTE_NAME + else: + anything_else + of '/': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state SELF_CLOSING_START_TAG + else: + anything_else + of '>': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state DATA + emit_tok + else: + anything_else + of AsciiAlpha: # note: merged upper & lower + tokenizer.tagNameBuf &= c.toLowerAscii() + tokenizer.tmp &= c + else: + anything_else + + of SCRIPT_DATA_ESCAPE_START: + case c + of '-': + switch_state SCRIPT_DATA_ESCAPE_START_DASH + emit '-' + else: + reconsume_in SCRIPT_DATA + + of SCRIPT_DATA_ESCAPE_START_DASH: + case c + of '-': + switch_state SCRIPT_DATA_ESCAPED_DASH_DASH + emit '-' + else: + reconsume_in SCRIPT_DATA + + of SCRIPT_DATA_ESCAPED: + case c + of '-': + switch_state SCRIPT_DATA_ESCAPED_DASH + emit '-' + of '<': switch_state SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN + of '\0': emit_replacement + else: emit c + + of SCRIPT_DATA_ESCAPED_DASH: + case c + of '-': + switch_state SCRIPT_DATA_ESCAPED_DASH_DASH + emit '-' + of '<': + switch_state SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN + of '\0': + switch_state SCRIPT_DATA_ESCAPED + emit_replacement + else: + switch_state SCRIPT_DATA_ESCAPED + emit c + + of SCRIPT_DATA_ESCAPED_DASH_DASH: + case c + of '-': + emit '-' + of '<': + switch_state SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN + of '>': + switch_state SCRIPT_DATA + emit '>' + of '\0': + switch_state SCRIPT_DATA_ESCAPED + emit_replacement + else: + switch_state SCRIPT_DATA_ESCAPED + emit c + + of SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN: + case c + of '/': + tokenizer.tmp = "" + switch_state SCRIPT_DATA_ESCAPED_END_TAG_OPEN + of AsciiAlpha: + tokenizer.tmp = $c.toLowerAscii() + emit '<' + emit c + # note: was reconsume + switch_state SCRIPT_DATA_DOUBLE_ESCAPE_START + else: + emit '<' + reconsume_in SCRIPT_DATA_ESCAPED + + of SCRIPT_DATA_ESCAPED_END_TAG_OPEN: + if c in AsciiAlpha: + new_token Token[Atom](t: END_TAG) + tokenizer.tagNameBuf = $c.toLowerAscii() + tokenizer.tmp &= c + # note: was reconsume + switch_state SCRIPT_DATA_ESCAPED_END_TAG_NAME + else: + emit "</" + reconsume_in SCRIPT_DATA_ESCAPED + + of SCRIPT_DATA_ESCAPED_END_TAG_NAME: + template anything_else = + tokenizer.emitTmp() + reconsume_in SCRIPT_DATA_ESCAPED + case c + of AsciiWhitespace: + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state BEFORE_ATTRIBUTE_NAME + else: + anything_else + of '/': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state SELF_CLOSING_START_TAG + else: + anything_else + of '>': + tokenizer.flushTagName() + if is_appropriate_end_tag_token: + switch_state DATA + emit_tok + else: + anything_else + of AsciiAlpha: + tokenizer.tagNameBuf &= c.toLowerAscii() + tokenizer.tmp &= c + else: + anything_else + + of SCRIPT_DATA_DOUBLE_ESCAPE_START: + case c + of AsciiWhitespace, '/', '>': + if tokenizer.tmp == "script": + switch_state SCRIPT_DATA_DOUBLE_ESCAPED + else: + switch_state SCRIPT_DATA_ESCAPED + emit c + of AsciiAlpha: # note: merged upper & lower + tokenizer.tmp &= c.toLowerAscii() + emit c + else: reconsume_in SCRIPT_DATA_ESCAPED + + of SCRIPT_DATA_DOUBLE_ESCAPED: + case c + of '-': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED_DASH + emit '-' + of '<': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN + emit '<' + of '\0': emit_replacement + else: emit c + + of SCRIPT_DATA_DOUBLE_ESCAPED_DASH: + case c + of '-': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH + emit '-' + of '<': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN + emit '<' + of '\0': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED + emit_replacement + else: + switch_state SCRIPT_DATA_DOUBLE_ESCAPED + emit c + + of SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH: + case c + of '-': emit '-' + of '<': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN + emit '<' + of '>': + switch_state SCRIPT_DATA + emit '>' + of '\0': + switch_state SCRIPT_DATA_DOUBLE_ESCAPED + emit_replacement + else: + switch_state SCRIPT_DATA_DOUBLE_ESCAPED + emit c + + of SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN: + case c + of '/': + tokenizer.tmp = "" + switch_state SCRIPT_DATA_DOUBLE_ESCAPE_END + emit '/' + else: reconsume_in SCRIPT_DATA_DOUBLE_ESCAPED + + of SCRIPT_DATA_DOUBLE_ESCAPE_END: + case c + of AsciiWhitespace, '/', '>': + if tokenizer.tmp == "script": + switch_state SCRIPT_DATA_ESCAPED + else: + switch_state SCRIPT_DATA_DOUBLE_ESCAPED + emit c + of AsciiAlpha: # note: merged upper & lower + tokenizer.tmp &= c.toLowerAscii() + emit c + else: + reconsume_in SCRIPT_DATA_DOUBLE_ESCAPED + + of BEFORE_ATTRIBUTE_NAME: + case c + of AsciiWhitespace: discard + of '/', '>': reconsume_in AFTER_ATTRIBUTE_NAME + of '=': + start_new_attribute + tokenizer.tmp &= c + switch_state ATTRIBUTE_NAME + else: + start_new_attribute + reconsume_in ATTRIBUTE_NAME + + of ATTRIBUTE_NAME: + case c + of AsciiWhitespace, '/', '>': + leave_attribute_name_state + reconsume_in AFTER_ATTRIBUTE_NAME + of '=': + leave_attribute_name_state + switch_state BEFORE_ATTRIBUTE_VALUE + of '\0': + tokenizer.tmp &= "\uFFFD" + else: + tokenizer.tmp &= c.toLowerAscii() + + of AFTER_ATTRIBUTE_NAME: + case c + of AsciiWhitespace: discard + of '/': switch_state SELF_CLOSING_START_TAG + of '=': switch_state BEFORE_ATTRIBUTE_VALUE + of '>': + switch_state DATA + prepare_attrs_if_start + emit_tok + else: + start_new_attribute + reconsume_in ATTRIBUTE_NAME + + of BEFORE_ATTRIBUTE_VALUE: + case c + of AsciiWhitespace: discard + of '"': switch_state ATTRIBUTE_VALUE_DOUBLE_QUOTED + of '\'': switch_state ATTRIBUTE_VALUE_SINGLE_QUOTED + of '>': + switch_state DATA + prepare_attrs_if_start + emit_tok + else: reconsume_in ATTRIBUTE_VALUE_UNQUOTED + + of ATTRIBUTE_VALUE_DOUBLE_QUOTED: + case c + of '"': switch_state AFTER_ATTRIBUTE_VALUE_QUOTED + of '&': switch_state_return CHARACTER_REFERENCE + of '\0': tokenizer.appendToAttrValue("\uFFFD") + else: tokenizer.appendToAttrValue([c]) + + of ATTRIBUTE_VALUE_SINGLE_QUOTED: + case c + of '\'': switch_state AFTER_ATTRIBUTE_VALUE_QUOTED + of '&': switch_state_return CHARACTER_REFERENCE + of '\0': tokenizer.appendToAttrValue("\uFFFD") + else: tokenizer.appendToAttrValue([c]) + + of ATTRIBUTE_VALUE_UNQUOTED: + case c + of AsciiWhitespace: switch_state BEFORE_ATTRIBUTE_NAME + of '&': switch_state_return CHARACTER_REFERENCE + of '>': + switch_state DATA + prepare_attrs_if_start + emit_tok + of '\0': tokenizer.appendToAttrValue("\uFFFD") + else: tokenizer.appendToAttrValue([c]) + + of AFTER_ATTRIBUTE_VALUE_QUOTED: + case c + of AsciiWhitespace: + switch_state BEFORE_ATTRIBUTE_NAME + of '/': + switch_state SELF_CLOSING_START_TAG + of '>': + switch_state DATA + prepare_attrs_if_start + emit_tok + else: reconsume_in BEFORE_ATTRIBUTE_NAME + + of SELF_CLOSING_START_TAG: + case c + of '>': + tokenizer.tok.selfclosing = true + switch_state DATA + prepare_attrs_if_start + emit_tok + else: reconsume_in BEFORE_ATTRIBUTE_NAME + + of BOGUS_COMMENT: + assert tokenizer.tok.t == COMMENT + case c + of '>': + switch_state DATA + emit_tok + of '\0': tokenizer.tok.data &= "\uFFFD" + else: tokenizer.tok.data &= c + + of MARKUP_DECLARATION_OPEN: # note: rewritten to fit case model as we consume a char anyway + template anything_else = + new_token Token[Atom](t: COMMENT) + switch_state BOGUS_COMMENT + case c + of '-': + case tokenizer.eatStr(c, "-", ibuf) + of esrSuccess: + new_token Token[Atom](t: COMMENT) + tokenizer.state = COMMENT_START + of esrRetry: break + of esrFail: anything_else + of 'D', 'd': + case tokenizer.eatStrNoCase(c, "octype", ibuf) + of esrSuccess: switch_state DOCTYPE + of esrRetry: break + of esrFail: anything_else + of '[': + case tokenizer.eatStr(c, "CDATA[", ibuf) + of esrSuccess: + if tokenizer.hasnonhtml: + switch_state CDATA_SECTION + else: + new_token Token[Atom](t: COMMENT, data: "[CDATA[") + switch_state BOGUS_COMMENT + of esrRetry: break + of esrFail: anything_else + else: + # eat didn't reconsume, do it ourselves + tokenizer.reconsume(c) + anything_else + + of COMMENT_START: + case c + of '-': switch_state COMMENT_START_DASH + of '>': + switch_state DATA + emit_tok + else: reconsume_in COMMENT + + of COMMENT_START_DASH: + case c + of '-': switch_state COMMENT_END + of '>': + switch_state DATA + emit_tok + else: + tokenizer.tok.data &= '-' + reconsume_in COMMENT + + of COMMENT: + case c + of '<': + tokenizer.tok.data &= c + switch_state COMMENT_LESS_THAN_SIGN + of '-': switch_state COMMENT_END_DASH + of '\0': tokenizer.tok.data &= "\uFFFD" + else: tokenizer.tok.data &= c + + of COMMENT_LESS_THAN_SIGN: + case c + of '!': + tokenizer.tok.data &= c + switch_state COMMENT_LESS_THAN_SIGN_BANG + of '<': tokenizer.tok.data &= c + else: reconsume_in COMMENT + + of COMMENT_LESS_THAN_SIGN_BANG: + case c + of '-': switch_state COMMENT_LESS_THAN_SIGN_BANG_DASH + else: reconsume_in COMMENT + + of COMMENT_LESS_THAN_SIGN_BANG_DASH: + case c + of '-': switch_state COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH + else: reconsume_in COMMENT_END_DASH + + of COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH: + case c + of '>': + # note: was reconsume (comment end) + switch_state DATA + emit_tok + else: reconsume_in COMMENT_END + + of COMMENT_END_DASH: + case c + of '-': switch_state COMMENT_END + else: + tokenizer.tok.data &= '-' + reconsume_in COMMENT + + of COMMENT_END: + case c + of '>': + switch_state DATA + emit_tok + of '!': switch_state COMMENT_END_BANG + of '-': tokenizer.tok.data &= '-' + else: + tokenizer.tok.data &= "--" + reconsume_in COMMENT + + of COMMENT_END_BANG: + case c + of '-': + tokenizer.tok.data &= "--!" + switch_state COMMENT_END_DASH + of '>': + switch_state DATA + emit_tok + else: + tokenizer.tok.data &= "--!" + reconsume_in COMMENT + + of DOCTYPE: + if c notin AsciiWhitespace: + tokenizer.reconsume(c) + switch_state BEFORE_DOCTYPE_NAME + + of BEFORE_DOCTYPE_NAME: + case c + of AsciiWhitespace: discard + of '\0': + new_token Token[Atom](t: DOCTYPE, name: some("\uFFFD")) + switch_state DOCTYPE_NAME + of '>': + new_token Token[Atom](t: DOCTYPE, quirks: true) + switch_state DATA + emit_tok + else: + new_token Token[Atom](t: DOCTYPE, name: some($c.toLowerAscii())) + switch_state DOCTYPE_NAME + + of DOCTYPE_NAME: + case c + of AsciiWhitespace: switch_state AFTER_DOCTYPE_NAME + of '>': + switch_state DATA + emit_tok + of '\0': tokenizer.tok.name.get &= "\uFFFD" + else: tokenizer.tok.name.get &= c.toLowerAscii() + + of AFTER_DOCTYPE_NAME: # note: rewritten to fit case model as we consume a char anyway + template anything_else = + tokenizer.tok.quirks = true + switch_state BOGUS_DOCTYPE + case c + of AsciiWhitespace: discard + of '>': + switch_state DATA + emit_tok + of 'p', 'P': + case tokenizer.eatStrNoCase(c, "ublic", ibuf) + of esrSuccess: switch_state AFTER_DOCTYPE_PUBLIC_KEYWORD + of esrRetry: break + of esrFail: anything_else + of 's', 'S': + case tokenizer.eatStrNoCase(c, "ystem", ibuf) + of esrSuccess: switch_state AFTER_DOCTYPE_SYSTEM_KEYWORD + of esrRetry: break + of esrFail: anything_else + else: + # eat didn't reconsume, do it ourselves + tokenizer.reconsume(c) + anything_else + + of AFTER_DOCTYPE_PUBLIC_KEYWORD: + case c + of AsciiWhitespace: switch_state BEFORE_DOCTYPE_PUBLIC_IDENTIFIER + of '"': + tokenizer.tok.pubid = some("") + switch_state DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED + of '\'': + tokenizer.tok.pubid = some("") + switch_state DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: + tokenizer.tok.quirks = true + reconsume_in BOGUS_DOCTYPE + + of BEFORE_DOCTYPE_PUBLIC_IDENTIFIER: + case c + of AsciiWhitespace: discard + of '"': + tokenizer.tok.pubid = some("") + switch_state DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED + of '\'': + tokenizer.tok.pubid = some("") + switch_state DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: + tokenizer.tok.quirks = true + reconsume_in BOGUS_DOCTYPE + + of DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED: + case c + of '"': switch_state AFTER_DOCTYPE_PUBLIC_IDENTIFIER + of '\0': tokenizer.tok.pubid.get &= "\uFFFD" + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: tokenizer.tok.pubid.get &= c + + of DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED: + case c + of '\'': switch_state AFTER_DOCTYPE_PUBLIC_IDENTIFIER + of '\0': tokenizer.tok.pubid.get &= "\uFFFD" + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: tokenizer.tok.pubid.get &= c + + of AFTER_DOCTYPE_PUBLIC_IDENTIFIER: + case c + of AsciiWhitespace: + switch_state BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS + of '>': + switch_state DATA + emit_tok + of '"': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED + of '\'': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED + else: + tokenizer.tok.quirks = true + reconsume_in BOGUS_DOCTYPE + + of BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS: + case c + of AsciiWhitespace: discard + of '>': + switch_state DATA + emit_tok + of '"': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED + of '\'': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED + else: + tokenizer.tok.quirks = true + reconsume_in BOGUS_DOCTYPE + + of AFTER_DOCTYPE_SYSTEM_KEYWORD: + case c + of AsciiWhitespace: switch_state BEFORE_DOCTYPE_SYSTEM_IDENTIFIER + of '"': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED + of '\'': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: + tokenizer.tok.quirks = true + reconsume_in BOGUS_DOCTYPE + + of BEFORE_DOCTYPE_SYSTEM_IDENTIFIER: + case c + of AsciiWhitespace: discard + of '"': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED + of '\'': + tokenizer.tok.sysid = some("") + switch_state DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: + tokenizer.tok.quirks = true + reconsume_in BOGUS_DOCTYPE + + of DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED: + case c + of '"': switch_state AFTER_DOCTYPE_SYSTEM_IDENTIFIER + of '\0': tokenizer.tok.sysid.get &= "\uFFFD" + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: tokenizer.tok.sysid.get &= c + + of DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED: + case c + of '\'': switch_state AFTER_DOCTYPE_SYSTEM_IDENTIFIER + of '\0': tokenizer.tok.sysid.get &= "\uFFFD" + of '>': + tokenizer.tok.quirks = true + switch_state DATA + emit_tok + else: + tokenizer.tok.sysid.get &= c + + of AFTER_DOCTYPE_SYSTEM_IDENTIFIER: + case c + of AsciiWhitespace: discard + of '>': + switch_state DATA + emit_tok + else: + switch_state BOGUS_DOCTYPE + + of BOGUS_DOCTYPE: + if c == '>': + emit_tok + switch_state DATA + + of CDATA_SECTION: + case c + of ']': switch_state CDATA_SECTION_BRACKET + of '\0': + # "U+0000 NULL characters are handled in the tree construction stage, + # as part of the in foreign content insertion mode, which is the only + # place where CDATA sections can appear." + emit_null + else: + emit c + + of CDATA_SECTION_BRACKET: + if c == ']': + switch_state CDATA_SECTION_END + else: + emit ']' + reconsume_in CDATA_SECTION + + of CDATA_SECTION_END: + case c + of ']': emit ']' + of '>': switch_state DATA + else: + emit "]]" + reconsume_in CDATA_SECTION + + of CHARACTER_REFERENCE: + case c + of AsciiAlpha: + tokenizer.tmp = "&" + reconsume_in NAMED_CHARACTER_REFERENCE + of '#': + tokenizer.tmp = "&#" + switch_state NUMERIC_CHARACTER_REFERENCE + else: + tokenizer.appendAttrOrEmit("&") + reconsume_in tokenizer.rstate + + of NAMED_CHARACTER_REFERENCE: + let (i, ci, entry) = tokenizer.findCharRef(c, ibuf) + if entry == nil and i == -1: + # -1 encountered without isend + tokenizer.reconsume(tokenizer.tmp.toOpenArray(1, tokenizer.tmp.high)) + tokenizer.tmp = "&" + break + # Move back the pointer & shorten the buffer to the last match. + # (Add 1, because we do not store the starting & char in entityMap, + # but tmp starts with an &.) + tokenizer.reconsume(tokenizer.tmp.toOpenArray(ci + 1, tokenizer.tmp.high)) + tokenizer.tmp.setLen(ci + 1) + if entry != nil and entry[ci] == ':': + let n = tokenizer.consume(ibuf) + let sc = tokenizer.consumedAsAttribute() and tokenizer.tmp[^1] != ';' + if sc and n != -1 and cast[char](n) in {'='} + AsciiAlphaNumeric: + tokenizer.reconsume(cast[char](n)) + tokenizer.appendAttrOrEmit(tokenizer.tmp) + switch_state tokenizer.rstate + elif sc and n == -1 and not tokenizer.isend: + # We have to redo the above check. + tokenizer.reconsume(tokenizer.tmp.toOpenArray(1, tokenizer.tmp.high)) + tokenizer.tmp = "&" + break + else: + if n != -1: + tokenizer.reconsume(cast[char](n)) + tokenizer.tmp = "" + var ci = ci + 1 + while (let c = entry[ci]; c != '\0'): + tokenizer.tmp &= c + inc ci + tokenizer.appendAttrOrEmit(tokenizer.tmp) + switch_state tokenizer.rstate + else: + tokenizer.appendAttrOrEmit(tokenizer.tmp) + switch_state AMBIGUOUS_AMPERSAND_STATE + + of AMBIGUOUS_AMPERSAND_STATE: + if c in AsciiAlpha: + tokenizer.appendAttrOrEmit([c]) + else: + reconsume_in tokenizer.rstate + + of NUMERIC_CHARACTER_REFERENCE: + tokenizer.code = 0 + case c + of 'x', 'X': + tokenizer.tmp &= c + switch_state HEXADECIMAL_CHARACTER_REFERENCE_START + of AsciiDigit: + tokenizer.code = uint32(c) - uint32('0') + # note: was reconsume + switch_state DECIMAL_CHARACTER_REFERENCE + else: + tokenizer.appendAttrOrEmit(tokenizer.tmp) + reconsume_in tokenizer.rstate + + of HEXADECIMAL_CHARACTER_REFERENCE_START: + let c2 = c.toLowerAscii() + case c2 + of AsciiDigit: + tokenizer.code = uint32(c2) - uint32('0') + # note: was reconsume + switch_state HEXADECIMAL_CHARACTER_REFERENCE + of 'a'..'f': + tokenizer.code = uint32(c2) - uint32('a') + 10 + # note: was reconsume + switch_state HEXADECIMAL_CHARACTER_REFERENCE + else: + tokenizer.appendAttrOrEmit(tokenizer.tmp) + reconsume_in tokenizer.rstate + + of HEXADECIMAL_CHARACTER_REFERENCE: + let c2 = c.toLowerAscii() + case c2 + of AsciiDigit: + if tokenizer.code < 0x10FFFF: + tokenizer.code *= 0x10 + tokenizer.code += uint32(c2) - uint32('0') + of 'a'..'f': + if tokenizer.code < 0x10FFFF: + tokenizer.code *= 0x10 + tokenizer.code += uint32(c2) - uint32('a') + 10 + else: + if c != ';': + tokenizer.reconsume(c) + tokenizer.numericCharacterReferenceEndState() + switch_state tokenizer.rstate + + of DECIMAL_CHARACTER_REFERENCE: + if c in AsciiDigit: + if tokenizer.code < 0x10FFFF: + tokenizer.code *= 10 + tokenizer.code += uint32(c) - uint32('0') + else: + if c != ';': + tokenizer.reconsume(c) + tokenizer.numericCharacterReferenceEndState() + switch_state tokenizer.rstate + + return trDone + +proc finish*[Handle, Atom](tokenizer: var Tokenizer[Handle, Atom]): + TokenizeResult = + if tokenizer.peekBufLen > 0: + tokenizer.isend = true + if tokenizer.tokenize([]) != trDone: + return trEmit + if tokenizer.tokenizeEOF(): + let r = tokenizer.tokenizeEOF() + assert not r + return trDone diff --git a/lib/chame0/chame/minidom.nim b/lib/chame0/chame/minidom.nim new file mode 100644 index 00000000..0fe10e46 --- /dev/null +++ b/lib/chame0/chame/minidom.nim @@ -0,0 +1,497 @@ +## Minimal DOMBuilder example. Implements the absolute minimum required +## for htmlparser to work correctly. +## +## For an example of a complete implementation (with JS support and +## document.write), see Chawan's chadombuilder. +## +## Note: this only works with UTF-8 inputs. For a variant that can +## switch encodings when meta tags are encountered etc. see +## [chame/minidom_cs](minidom_cs.html). + +import std/algorithm +import std/hashes +import std/options +import std/sets +import std/streams +import std/tables + +import htmlparser +import tags + +export tags + +# Atom implementation +#TODO maybe we should use a better hash map. +const MAtomFactoryStrMapLength = 1024 # must be a power of 2 +static: + doAssert (MAtomFactoryStrMapLength and (MAtomFactoryStrMapLength - 1)) == 0 + +type + MAtom* = distinct int + + MAtomFactory* = ref object of RootObj + strMap: array[MAtomFactoryStrMapLength, seq[MAtom]] + atomMap: seq[string] + +# Mandatory Atom functions +func `==`*(a, b: MAtom): bool {.borrow.} +func hash*(atom: MAtom): Hash {.borrow.} + +func strToAtom*(factory: MAtomFactory, s: string): MAtom + +proc newMAtomFactory*(): MAtomFactory = + const minCap = int(TagType.high) + 1 + let factory = MAtomFactory( + atomMap: newSeqOfCap[string](minCap), + ) + factory.atomMap.add("") # skip TAG_UNKNOWN + for tagType in TagType(int(TAG_UNKNOWN) + 1) .. TagType.high: + discard factory.strToAtom($tagType) + return factory + +func strToAtom*(factory: MAtomFactory, s: string): MAtom = + let h = s.hash() + let i = h and (factory.strMap.len - 1) + for atom in factory.strMap[i]: + if factory.atomMap[int(atom)] == s: + # Found + return atom + # Not found + let atom = MAtom(factory.atomMap.len) + factory.atomMap.add(s) + factory.strMap[i].add(atom) + return atom + +func tagTypeToAtom*(factory: MAtomFactory, tagType: TagType): MAtom = + assert tagType != TAG_UNKNOWN + return MAtom(tagType) + +func atomToStr*(factory: MAtomFactory, atom: MAtom): string = + return factory.atomMap[int(atom)] + +# Node types +type + Attribute* = ParsedAttr[MAtom] + + Node* = ref object of RootObj + childList*: seq[Node] + parentNode* {.cursor.}: Node + + CharacterData* = ref object of Node + data*: string + + Comment* = ref object of CharacterData + + Document* = ref object of Node + factory*: MAtomFactory + + Text* = ref object of CharacterData + + DocumentType* = ref object of Node + name*: string + publicId*: string + systemId*: string + + Element* = ref object of Node + localName*: MAtom + namespace*: Namespace + attrs*: seq[Attribute] + document*: Document + + DocumentFragment* = ref object of Node + + HTMLTemplateElement* = ref object of Element + content*: DocumentFragment + +type + MiniDOMBuilder* = ref object of DOMBuilder[Node, MAtom] + document*: Document + factory*: MAtomFactory + +type + DOMBuilderImpl = MiniDOMBuilder + AtomImpl = MAtom + HandleImpl = Node + +include htmlparseriface + +func toTagType*(atom: MAtom): TagType {.inline.} = + if int(atom) <= int(high(TagType)): + return TagType(atom) + return TAG_UNKNOWN + +func tagType*(element: Element): TagType = + return element.localName.toTagType() + +func cmp*(a, b: MAtom): int {.inline.} = + return cmp(int(a), int(b)) + +# We use this to validate input strings, since htmltokenizer/htmlparser does no +# input validation. +proc toValidUTF8(s: string): string = + result = "" + var i = 0 + while i < s.len: + if int(s[i]) < 0x80: + result &= s[i] + inc i + elif int(s[i]) shr 5 == 0x6: + if i + 1 < s.len and int(s[i + 1]) shr 6 == 2: + result &= s[i] + result &= s[i + 1] + else: + result &= "\uFFFD" + i += 2 + elif int(s[i]) shr 4 == 0xE: + if i + 2 < s.len and int(s[i + 1]) shr 6 == 2 and + int(s[i + 2]) shr 6 == 2: + result &= s[i] + result &= s[i + 1] + result &= s[i + 2] + else: + result &= "\uFFFD" + i += 3 + elif int(s[i]) shr 3 == 0x1E: + if i + 3 < s.len and int(s[i + 1]) shr 6 == 2 and + int(s[i + 2]) shr 6 == 2 and int(s[i + 3]) shr 6 == 2: + result &= s[i] + result &= s[i + 1] + result &= s[i + 2] + result &= s[i + 3] + else: + result &= "\uFFFD" + i += 4 + else: + result &= "\uFFFD" + inc i + +proc localNameStr*(element: Element): string = + return element.document.factory.atomToStr(element.localName) + +iterator attrsStr*(element: Element): tuple[name, value: string] = + let factory = element.document.factory + for attr in element.attrs: + var name = "" + if attr.prefix != NO_PREFIX: + name &= $attr.prefix & ':' + name &= factory.atomToStr(attr.name) + yield (name, attr.value) + +# htmlparseriface implementation +proc strToAtomImpl(builder: MiniDOMBuilder, s: string): MAtom = + return builder.factory.strToAtom(s) + +proc tagTypeToAtomImpl(builder: MiniDOMBuilder, tagType: TagType): MAtom = + return builder.factory.tagTypeToAtom(tagType) + +proc atomToTagTypeImpl(builder: MiniDOMBuilder, atom: MAtom): TagType = + return atom.toTagType() + +proc getDocumentImpl(builder: MiniDOMBuilder): Node = + return builder.document + +proc getParentNodeImpl(builder: MiniDOMBuilder, handle: Node): Option[Node] = + return option(handle.parentNode) + +proc createElement(document: Document, localName: MAtom, namespace: Namespace): + Element = + let element = if localName.toTagType() == TAG_TEMPLATE and + namespace == Namespace.HTML: + HTMLTemplateElement( + content: DocumentFragment() + ) + else: + Element() + element.localName = localName + element.namespace = namespace + element.document = document + return element + +proc createHTMLElementImpl(builder: MiniDOMBuilder): Node = + let localName = builder.factory.tagTypeToAtom(TAG_HTML) + return builder.document.createElement(localName, Namespace.HTML) + +proc createElementForTokenImpl(builder: MiniDOMBuilder, localName: MAtom, + namespace: Namespace, intendedParent: Node, htmlAttrs: Table[MAtom, string], + xmlAttrs: seq[Attribute]): Node = + let element = builder.document.createElement(localName, namespace) + element.attrs = xmlAttrs + for k, v in htmlAttrs: + element.attrs.add((NO_PREFIX, NO_NAMESPACE, k, v.toValidUTF8())) + element.attrs.sort(func(a, b: Attribute): int = cmp(a.name, b.name)) + return element + +proc getLocalNameImpl(builder: MiniDOMBuilder, handle: Node): MAtom = + return Element(handle).localName + +proc getNamespaceImpl(builder: MiniDOMBuilder, handle: Node): Namespace = + return Element(handle).namespace + +proc getTemplateContentImpl(builder: MiniDOMBuilder, handle: Node): Node = + return HTMLTemplateElement(handle).content + +proc createCommentImpl(builder: MiniDOMBuilder, text: string): Node = + return Comment(data: text.toValidUTF8()) + +proc createDocumentTypeImpl(builder: MiniDOMBuilder, name, publicId, + systemId: string): Node = + return DocumentType( + name: name.toValidUTF8(), + publicId: publicId.toValidUTF8(), + systemId: systemId.toValidUTF8() + ) + +func countElementChildren(node: Node): int = + result = 0 + for child in node.childList: + if child of Element: + inc result + +func hasTextChild(node: Node): bool = + for child in node.childList: + if child of Text: + return true + return false + +func hasElementChild(node: Node): bool = + for child in node.childList: + if child of Element: + return true + return false + +func hasDocumentTypeChild(node: Node): bool = + for child in node.childList: + if child of DocumentType: + return true + return false + +func isHostIncludingInclusiveAncestor(a, b: Node): bool = + var b = b + while b != nil: + if b == a: + return true + b = b.parentNode + return false + +func hasPreviousElementSibling(node: Node): bool = + for n in node.parentNode.childList: + if n == node: + break + if n of Element: + return true + return false + +func hasNextDocumentTypeSibling(node: Node): bool = + for i in countdown(node.parentNode.childList.len, 0): + let n = node.parentNode.childList[i] + if n == node: + break + if n of DocumentType: + return true + return false + +func isValidParent(node: Node): bool = + return node of Element or node of Document or node of DocumentFragment + +func isValidChild(node: Node): bool = + return node.isValidParent or node of DocumentType or node of CharacterData + +# WARNING the ordering of the arguments in the standard is whack so this +# doesn't match that +func preInsertionValidity*(parent, node: Node, before: Node): bool = + if not parent.isValidParent: + return false + if node.isHostIncludingInclusiveAncestor(parent): + return false + if before != nil and before.parentNode != parent: + return false + if not node.isValidChild: + return false + if node of Text and parent of Document: + return false + if node of DocumentType and not (parent of Document): + return false + if parent of Document: + if node of DocumentFragment: + let elems = node.countElementChildren() + if elems > 1 or node.hasTextChild(): + return false + elif elems == 1 and (parent.hasElementChild() or + before != nil and + (before of DocumentType or before.hasNextDocumentTypeSibling())): + return false + elif node of Element: + if parent.hasElementChild(): + return false + elif before != nil and (before of DocumentType or + before.hasNextDocumentTypeSibling()): + return false + elif node of DocumentType: + if parent.hasDocumentTypeChild() or + before != nil and before.hasPreviousElementSibling() or + before == nil and parent.hasElementChild(): + return false + return true # no exception reached + +proc insertBefore(parent, child: Node, before: Option[Node]) = + let before = before.get(nil) + if parent.preInsertionValidity(child, before): + assert child.parentNode == nil + if before == nil: + parent.childList.add(child) + else: + let i = parent.childList.find(before) + parent.childList.insert(child, i) + child.parentNode = parent + +proc insertBeforeImpl(builder: MiniDOMBuilder, parent, child: Node, + before: Option[Node]) = + parent.insertBefore(child, before) + +proc insertTextImpl(builder: MiniDOMBuilder, parent: Node, text: string, + before: Option[Node]) = + let text = text.toValidUTF8() + let before = before.get(nil) + let prevSibling = if before != nil: + let i = parent.childList.find(before) + if i == 0: + nil + else: + parent.childList[i - 1] + elif parent.childList.len > 0: + parent.childList[^1] + else: + nil + if prevSibling != nil and prevSibling of Text: + Text(prevSibling).data &= text + else: + let text = Text(data: text) + parent.insertBefore(text, option(before)) + +proc removeImpl(builder: MiniDOMBuilder, child: Node) = + if child.parentNode != nil: + let i = child.parentNode.childList.find(child) + child.parentNode.childList.delete(i) + child.parentNode = nil + +proc moveChildrenImpl(builder: MiniDOMBuilder, fromNode, toNode: Node) = + let tomove = @(fromNode.childList) + fromNode.childList.setLen(0) + for child in tomove: + child.parentNode = nil + toNode.insertBefore(child, none(Node)) + +proc addAttrsIfMissingImpl(builder: MiniDOMBuilder, handle: Node, + attrs: Table[MAtom, string]) = + let element = Element(handle) + var oldNames = initHashSet[MAtom]() + for attr in element.attrs: + oldNames.incl(attr.name) + for name, value in attrs: + if name notin oldNames: + element.attrs.add((NO_PREFIX, NO_NAMESPACE, name, value.toValidUTF8())) + element.attrs.sort(func(a, b: Attribute): int = cmp(a.name, b.name)) + +method setEncodingImpl(builder: MiniDOMBuilder, encoding: string): + SetEncodingResult {.base.} = + # Provided as a method for minidom_cs to override. + return SET_ENCODING_CONTINUE + +proc newMiniDOMBuilder*(factory: MAtomFactory): MiniDOMBuilder = + let document = Document(factory: factory) + let builder = MiniDOMBuilder( + document: document, + factory: factory + ) + return builder + +proc parseFromStream(parser: var HTML5Parser[Node, MAtom], + inputStream: Stream) = + var buffer {.noinit.}: array[4096, char] + while true: + let n = inputStream.readData(addr buffer[0], buffer.len) + if n == 0: break + # res can be PRES_CONTINUE or PRES_SCRIPTING. PRES_STOP is only returned + # on charset switching, and minidom does not support that. + var res = parser.parseChunk(buffer.toOpenArray(0, n - 1)) + # Important: we must repeat parseChunk with the same contents for the script + # end tag result, with reprocess = true. + # + # (This is only relevant for calls where scripting = true; with scripting = + # false, PRES_SCRIPT would never be returned.) + var ip = 0 + while res == PRES_SCRIPT and (ip += parser.getInsertionPoint(); ip != n): + res = parser.parseChunk(buffer.toOpenArray(ip, n - 1)) + parser.finish() + +proc parseHTML*(inputStream: Stream, opts = HTML5ParserOpts[Node, MAtom](), + factory = newMAtomFactory()): Document = + ## Read, parse and return an HTML document from `inputStream`, using + ## parser options `opts` and MAtom factory `factory`. + ## + ## `inputStream` is not required to be seekable. + ## + ## For a description of `HTML5ParserOpts`, see the `htmlparser` module's + ## documentation. + let builder = newMiniDOMBuilder(factory) + var parser = initHTML5Parser(builder, opts) + parser.parseFromStream(inputStream) + return builder.document + +proc parseHTMLFragment*(inputStream: Stream, element: Element, + opts: HTML5ParserOpts[Node, MAtom], factory = newMAtomFactory()): + seq[Node] = + ## Read, parse and return the children of an HTML fragment from `inputStream`, + ## using context element `element` and parser options `opts`. + ## + ## For information on `opts` (an `HTML5ParserOpts` object), please consult + ## the documentation of chame/htmlparser.nim. + ## + ## For details on the HTML fragment parsing algorithm, see + ## https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments + ## + ## Note: the members `ctx`, `initialTokenizerState`, `openElementsInit` and + ## `pushInTemplate` of `opts` are overridden (in accordance with the standard). + let builder = newMiniDOMBuilder(factory) + let document = builder.document + let state = if element.namespace != Namespace.HTML: + DATA + else: + case element.tagType + of TAG_TITLE, TAG_TEXTAREA: RCDATA + of TAG_STYLE, TAG_XMP, TAG_IFRAME, TAG_NOEMBED, TAG_NOFRAMES: RAWTEXT + of TAG_SCRIPT: SCRIPT_DATA + of TAG_NOSCRIPT: DATA # no scripting + of TAG_PLAINTEXT: PLAINTEXT + else: DATA + let htmlAtom = builder.factory.tagTypeToAtom(TAG_HTML) + let root = Element( + localName: htmlAtom, + namespace: HTML, + document: document + ) + document.childList = @[Node(root)] + var opts = opts + opts.ctx = some((Node(element), element.localName)) + opts.initialTokenizerState = state + opts.openElementsInit = @[(Node(root), htmlAtom)] + opts.pushInTemplate = element.tagType == TAG_TEMPLATE + var parser = initHTML5Parser(builder, opts) + parser.parseFromStream(inputStream) + return root.childList + +proc parseHTMLFragment*(s: string, element: Element): seq[Node] = + ## Convenience wrapper around parseHTMLFragment with opts. + ## + ## Read, parse and return the children of an HTML fragment from the string `s`, + ## using context element `element`. + ## + ## For details on the HTML fragment parsing algorithm, see + ## https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments + let inputStream = newStringStream(s) + let opts = HTML5ParserOpts[Node, MAtom]( + isIframeSrcdoc: false, + scripting: false, + pushInTemplate: element.tagType == TAG_TEMPLATE + ) + return parseHTMLFragment(inputStream, element, opts) diff --git a/lib/chame0/chame/minidom_cs.nim b/lib/chame0/chame/minidom_cs.nim new file mode 100644 index 00000000..6b183785 --- /dev/null +++ b/lib/chame0/chame/minidom_cs.nim @@ -0,0 +1,165 @@ +## A demonstration of using the Chagashi encoding library in combination +## with the Chame HTML parser. +## +## For the most part, this is the same as minidom, except it supports +## decoding documents with arbitrary character sets. +## +## Note: this is not implemented for the fragment parsing algorithm, +## because that is only defined for UTF-8 in the standard. +## +## For a version without the encoding library dependency, see +## [minidom](minidom.html). + +import std/streams + +import minidom +import htmlparser +import tags + +import chagashi/charset +import chagashi/decoder + +export minidom +export tags + +type CharsetConfidence = enum + ccTentative, ccCertain + +type CharsetMiniDOMBuilder = ref object of MiniDOMBuilder + charset: Charset + confidence: CharsetConfidence + +method setEncodingImpl(builder: CharsetMiniDOMBuilder, encoding: string): + SetEncodingResult = + let charset = getCharset(encoding) + if charset == CHARSET_UNKNOWN: + return SET_ENCODING_CONTINUE + if builder.charset in {CHARSET_UTF_16_LE, CHARSET_UTF_16_BE}: + builder.confidence = ccCertain + return SET_ENCODING_CONTINUE + builder.confidence = ccCertain + if charset == builder.charset: + return SET_ENCODING_CONTINUE + if charset == CHARSET_X_USER_DEFINED: + builder.charset = CHARSET_WINDOWS_1252 + else: + builder.charset = charset + return SET_ENCODING_STOP + +proc newCharsetMiniDOMBuilder(factory: MAtomFactory): CharsetMiniDOMBuilder = + let document = Document(factory: factory) + let builder = CharsetMiniDOMBuilder(document: document, factory: factory) + return builder + +#TODO this should be handled by decoderstream +proc bomSniff(inputStream: Stream): Charset = + let bom = inputStream.readStr(2) + if bom == "\xFE\xFF": + return CHARSET_UTF_16_BE + if bom == "\xFF\xFE": + return CHARSET_UTF_16_LE + if bom == "\xEF\xBB": + if inputStream.readChar() == '\xBF': + return CHARSET_UTF_8 + inputStream.setPosition(0) + return CHARSET_UNKNOWN + +proc parseHTML*(inputStream: Stream, opts: HTML5ParserOpts[Node, MAtom], + charsets: seq[Charset], seekable = true, + factory = newMAtomFactory()): Document = + ## Read, parse and return an HTML document from `inputStream`. + ## + ## `charsets` is a list of input character sets to try. If empty, it will be + ## initialized to `@[CHARSET_UTF_8]`. + ## + ## The list of fallback charsets is used as follows: + ## + ## * A charset stack is initialized to `charsets`, reversed. This + ## means that the first charset specified in `charsets` is on top of + ## the stack. (e.g. say `charsets = @[CHARSET_UTF_16_LE, CHARSET_UTF_8]`, + ## then utf-16-le is tried before utf-8.) + ## * BOM sniffing is attempted. If successful, confidence is set to + ## certain and the resulting charset is used (i.e. other character + ## sets will not be tried for decoding this document.) + ## * If the charset stack is empty, UTF-8 is pushed on top. + ## * Attempt to parse the document with the first charset on top of + ## the stack. + ## * If BOM sniffing was unsuccessful, and a <meta charset=...> tag + ## is encountered, parsing is restarted with the specified charset. + ## No further attempts are made to detect the encoding, and decoder + ## errors are signaled by U+FFFD replacement characters. + ## * Otherwise, each charset on the charset stack is tried until either no + ## decoding errors are encountered, or only one charset is left. For + ## the last charset, decoder errors are signaled by U+FFFD replacement + ## characters. + ## + ## `seekable` must be true only if `inputStream` is seekable; if set to true, + ## `inputStream.setPosition(0)` must work. + ## + ## Note that `seekable = false` disables automatic character set detection; + ## even `<meta charset=...` tags will be disregarded. + ## (TODO: this should be improved in the future; theoretically we could still + ## switch between ASCII-compatible charsets before non-ASCII is encountered.) + let builder = newCharsetMiniDOMBuilder(factory) + var charsetStack: seq[Charset] = @[] + for i in countdown(charsets.high, 0): + charsetStack.add(charsets[i]) + var seekable = seekable + var inputStream = inputStream + if seekable: + let scs = inputStream.bomSniff() + if scs != CHARSET_UNKNOWN: + charsetStack.add(scs) + builder.confidence = ccCertain + seekable = false + if charsetStack.len == 0: + charsetStack.add(DefaultCharset) # UTF-8 + while true: + builder.charset = charsetStack.pop() + if seekable and charsetStack.len > 0: + builder.confidence = ccTentative # used in the next iteration + else: + builder.confidence = ccCertain + var parser = initHTML5Parser(builder, opts) + var iq {.noinit.}: array[4096, char] + let decoder = newTextDecoder(builder.charset) + let errorMode = [ + ccTentative: demFatal, + ccCertain: demReplacement + ][builder.confidence] + var ctx = initTextDecoderContext(decoder, errorMode) + while true: + let n = inputStream.readData(addr iq[0], iq.len) + var finish = n < iq.len + for chunk in ctx.decode(iq.toOpenArrayByte(0, n - 1), finish = finish): + # res can be PRES_SCRIPT, PRES_STOP or PRES_CONTINUE. + var res = parser.parseChunk(chunk.toOpenArray()) + # For PRES_SCRIPT, we must re-feed the same chunk as in minidom, but + # starting from the current insertion point. + var ip = 0 + while res == PRES_SCRIPT and + (ip += parser.getInsertionPoint(); ip != chunk.len): + res = parser.parseChunk(chunk.toOpenArray(ip, chunk.high)) + # PRES_STOP is returned when we return SET_ENCODING_STOP from + # setEncodingImpl. We immediately stop parsing in this case. + if res == PRES_STOP: + finish = true + break + if finish: + break + parser.finish() + if builder.confidence == ccCertain and seekable: + # A meta tag describing the charset has been found; force use of this + # charset. + inputStream.setPosition(0) + builder.document = Document(factory: factory) + charsetStack.add(builder.charset) + seekable = false + continue + if ctx.failed and seekable: + # Retry with another charset. + inputStream.setPosition(0) + builder.document = Document(factory: factory) + continue + break + return builder.document diff --git a/lib/chame0/chame/res/entity.json b/lib/chame0/chame/res/entity.json new file mode 100644 index 00000000..557170b4 --- /dev/null +++ b/lib/chame0/chame/res/entity.json @@ -0,0 +1,2233 @@ +{ + "Æ": { "codepoints": [198], "characters": "\u00C6" }, + "Æ": { "codepoints": [198], "characters": "\u00C6" }, + "&": { "codepoints": [38], "characters": "\u0026" }, + "&": { "codepoints": [38], "characters": "\u0026" }, + "Á": { "codepoints": [193], "characters": "\u00C1" }, + "Á": { "codepoints": [193], "characters": "\u00C1" }, + "Ă": { "codepoints": [258], "characters": "\u0102" }, + "Â": { "codepoints": [194], "characters": "\u00C2" }, + "Â": { "codepoints": [194], "characters": "\u00C2" }, + "А": { "codepoints": [1040], "characters": "\u0410" }, + "𝔄": { "codepoints": [120068], "characters": "\uD835\uDD04" }, + "À": { "codepoints": [192], "characters": "\u00C0" }, + "À": { "codepoints": [192], "characters": "\u00C0" }, + "Α": { "codepoints": [913], "characters": "\u0391" }, + "Ā": { "codepoints": [256], "characters": "\u0100" }, + "⩓": { "codepoints": [10835], "characters": "\u2A53" }, + "Ą": { "codepoints": [260], "characters": "\u0104" }, + "𝔸": { "codepoints": [120120], "characters": "\uD835\uDD38" }, + "⁡": { "codepoints": [8289], "characters": "\u2061" }, + "Å": { "codepoints": [197], "characters": "\u00C5" }, + "Å": { "codepoints": [197], "characters": "\u00C5" }, + "𝒜": { "codepoints": [119964], "characters": "\uD835\uDC9C" }, + "≔": { "codepoints": [8788], "characters": "\u2254" }, + "Ã": { "codepoints": [195], "characters": "\u00C3" }, + "Ã": { "codepoints": [195], "characters": "\u00C3" }, + "Ä": { "codepoints": [196], "characters": "\u00C4" }, + "Ä": { "codepoints": [196], "characters": "\u00C4" }, + "∖": { "codepoints": [8726], "characters": "\u2216" }, + "⫧": { "codepoints": [10983], "characters": "\u2AE7" }, + "⌆": { "codepoints": [8966], "characters": "\u2306" }, + "Б": { "codepoints": [1041], "characters": "\u0411" }, + "∵": { "codepoints": [8757], "characters": "\u2235" }, + "ℬ": { "codepoints": [8492], "characters": "\u212C" }, + "Β": { "codepoints": [914], "characters": "\u0392" }, + "𝔅": { "codepoints": [120069], "characters": "\uD835\uDD05" }, + "𝔹": { "codepoints": [120121], "characters": "\uD835\uDD39" }, + "˘": { "codepoints": [728], "characters": "\u02D8" }, + "ℬ": { "codepoints": [8492], "characters": "\u212C" }, + "≎": { "codepoints": [8782], "characters": "\u224E" }, + "Ч": { "codepoints": [1063], "characters": "\u0427" }, + "©": { "codepoints": [169], "characters": "\u00A9" }, + "©": { "codepoints": [169], "characters": "\u00A9" }, + "Ć": { "codepoints": [262], "characters": "\u0106" }, + "⋒": { "codepoints": [8914], "characters": "\u22D2" }, + "ⅅ": { "codepoints": [8517], "characters": "\u2145" }, + "ℭ": { "codepoints": [8493], "characters": "\u212D" }, + "Č": { "codepoints": [268], "characters": "\u010C" }, + "Ç": { "codepoints": [199], "characters": "\u00C7" }, + "Ç": { "codepoints": [199], "characters": "\u00C7" }, + "Ĉ": { "codepoints": [264], "characters": "\u0108" }, + "∰": { "codepoints": [8752], "characters": "\u2230" }, + "Ċ": { "codepoints": [266], "characters": "\u010A" }, + "¸": { "codepoints": [184], "characters": "\u00B8" }, + "·": { "codepoints": [183], "characters": "\u00B7" }, + "ℭ": { "codepoints": [8493], "characters": "\u212D" }, + "Χ": { "codepoints": [935], "characters": "\u03A7" }, + "⊙": { "codepoints": [8857], "characters": "\u2299" }, + "⊖": { "codepoints": [8854], "characters": "\u2296" }, + "⊕": { "codepoints": [8853], "characters": "\u2295" }, + "⊗": { "codepoints": [8855], "characters": "\u2297" }, + "∲": { "codepoints": [8754], "characters": "\u2232" }, + "”": { "codepoints": [8221], "characters": "\u201D" }, + "’": { "codepoints": [8217], "characters": "\u2019" }, + "∷": { "codepoints": [8759], "characters": "\u2237" }, + "⩴": { "codepoints": [10868], "characters": "\u2A74" }, + "≡": { "codepoints": [8801], "characters": "\u2261" }, + "∯": { "codepoints": [8751], "characters": "\u222F" }, + "∮": { "codepoints": [8750], "characters": "\u222E" }, + "ℂ": { "codepoints": [8450], "characters": "\u2102" }, + "∐": { "codepoints": [8720], "characters": "\u2210" }, + "∳": { "codepoints": [8755], "characters": "\u2233" }, + "⨯": { "codepoints": [10799], "characters": "\u2A2F" }, + "𝒞": { "codepoints": [119966], "characters": "\uD835\uDC9E" }, + "⋓": { "codepoints": [8915], "characters": "\u22D3" }, + "≍": { "codepoints": [8781], "characters": "\u224D" }, + "ⅅ": { "codepoints": [8517], "characters": "\u2145" }, + "⤑": { "codepoints": [10513], "characters": "\u2911" }, + "Ђ": { "codepoints": [1026], "characters": "\u0402" }, + "Ѕ": { "codepoints": [1029], "characters": "\u0405" }, + "Џ": { "codepoints": [1039], "characters": "\u040F" }, + "‡": { "codepoints": [8225], "characters": "\u2021" }, + "↡": { "codepoints": [8609], "characters": "\u21A1" }, + "⫤": { "codepoints": [10980], "characters": "\u2AE4" }, + "Ď": { "codepoints": [270], "characters": "\u010E" }, + "Д": { "codepoints": [1044], "characters": "\u0414" }, + "∇": { "codepoints": [8711], "characters": "\u2207" }, + "Δ": { "codepoints": [916], "characters": "\u0394" }, + "𝔇": { "codepoints": [120071], "characters": "\uD835\uDD07" }, + "´": { "codepoints": [180], "characters": "\u00B4" }, + "˙": { "codepoints": [729], "characters": "\u02D9" }, + "˝": { "codepoints": [733], "characters": "\u02DD" }, + "`": { "codepoints": [96], "characters": "\u0060" }, + "˜": { "codepoints": [732], "characters": "\u02DC" }, + "⋄": { "codepoints": [8900], "characters": "\u22C4" }, + "ⅆ": { "codepoints": [8518], "characters": "\u2146" }, + "𝔻": { "codepoints": [120123], "characters": "\uD835\uDD3B" }, + "¨": { "codepoints": [168], "characters": "\u00A8" }, + "⃜": { "codepoints": [8412], "characters": "\u20DC" }, + "≐": { "codepoints": [8784], "characters": "\u2250" }, + "∯": { "codepoints": [8751], "characters": "\u222F" }, + "¨": { "codepoints": [168], "characters": "\u00A8" }, + "⇓": { "codepoints": [8659], "characters": "\u21D3" }, + "⇐": { "codepoints": [8656], "characters": "\u21D0" }, + "⇔": { "codepoints": [8660], "characters": "\u21D4" }, + "⫤": { "codepoints": [10980], "characters": "\u2AE4" }, + "⟸": { "codepoints": [10232], "characters": "\u27F8" }, + "⟺": { "codepoints": [10234], "characters": "\u27FA" }, + "⟹": { "codepoints": [10233], "characters": "\u27F9" }, + "⇒": { "codepoints": [8658], "characters": "\u21D2" }, + "⊨": { "codepoints": [8872], "characters": "\u22A8" }, + "⇑": { "codepoints": [8657], "characters": "\u21D1" }, + "⇕": { "codepoints": [8661], "characters": "\u21D5" }, + "∥": { "codepoints": [8741], "characters": "\u2225" }, + "↓": { "codepoints": [8595], "characters": "\u2193" }, + "⤓": { "codepoints": [10515], "characters": "\u2913" }, + "⇵": { "codepoints": [8693], "characters": "\u21F5" }, + "̑": { "codepoints": [785], "characters": "\u0311" }, + "⥐": { "codepoints": [10576], "characters": "\u2950" }, + "⥞": { "codepoints": [10590], "characters": "\u295E" }, + "↽": { "codepoints": [8637], "characters": "\u21BD" }, + "⥖": { "codepoints": [10582], "characters": "\u2956" }, + "⥟": { "codepoints": [10591], "characters": "\u295F" }, + "⇁": { "codepoints": [8641], "characters": "\u21C1" }, + "⥗": { "codepoints": [10583], "characters": "\u2957" }, + "⊤": { "codepoints": [8868], "characters": "\u22A4" }, + "↧": { "codepoints": [8615], "characters": "\u21A7" }, + "⇓": { "codepoints": [8659], "characters": "\u21D3" }, + "𝒟": { "codepoints": [119967], "characters": "\uD835\uDC9F" }, + "Đ": { "codepoints": [272], "characters": "\u0110" }, + "Ŋ": { "codepoints": [330], "characters": "\u014A" }, + "Ð": { "codepoints": [208], "characters": "\u00D0" }, + "Ð": { "codepoints": [208], "characters": "\u00D0" }, + "É": { "codepoints": [201], "characters": "\u00C9" }, + "É": { "codepoints": [201], "characters": "\u00C9" }, + "Ě": { "codepoints": [282], "characters": "\u011A" }, + "Ê": { "codepoints": [202], "characters": "\u00CA" }, + "Ê": { "codepoints": [202], "characters": "\u00CA" }, + "Э": { "codepoints": [1069], "characters": "\u042D" }, + "Ė": { "codepoints": [278], "characters": "\u0116" }, + "𝔈": { "codepoints": [120072], "characters": "\uD835\uDD08" }, + "È": { "codepoints": [200], "characters": "\u00C8" }, + "È": { "codepoints": [200], "characters": "\u00C8" }, + "∈": { "codepoints": [8712], "characters": "\u2208" }, + "Ē": { "codepoints": [274], "characters": "\u0112" }, + "◻": { "codepoints": [9723], "characters": "\u25FB" }, + "▫": { "codepoints": [9643], "characters": "\u25AB" }, + "Ę": { "codepoints": [280], "characters": "\u0118" }, + "𝔼": { "codepoints": [120124], "characters": "\uD835\uDD3C" }, + "Ε": { "codepoints": [917], "characters": "\u0395" }, + "⩵": { "codepoints": [10869], "characters": "\u2A75" }, + "≂": { "codepoints": [8770], "characters": "\u2242" }, + "⇌": { "codepoints": [8652], "characters": "\u21CC" }, + "ℰ": { "codepoints": [8496], "characters": "\u2130" }, + "⩳": { "codepoints": [10867], "characters": "\u2A73" }, + "Η": { "codepoints": [919], "characters": "\u0397" }, + "Ë": { "codepoints": [203], "characters": "\u00CB" }, + "Ë": { "codepoints": [203], "characters": "\u00CB" }, + "∃": { "codepoints": [8707], "characters": "\u2203" }, + "ⅇ": { "codepoints": [8519], "characters": "\u2147" }, + "Ф": { "codepoints": [1060], "characters": "\u0424" }, + "𝔉": { "codepoints": [120073], "characters": "\uD835\uDD09" }, + "◼": { "codepoints": [9724], "characters": "\u25FC" }, + "▪": { "codepoints": [9642], "characters": "\u25AA" }, + "𝔽": { "codepoints": [120125], "characters": "\uD835\uDD3D" }, + "∀": { "codepoints": [8704], "characters": "\u2200" }, + "ℱ": { "codepoints": [8497], "characters": "\u2131" }, + "ℱ": { "codepoints": [8497], "characters": "\u2131" }, + "Ѓ": { "codepoints": [1027], "characters": "\u0403" }, + ">": { "codepoints": [62], "characters": "\u003E" }, + ">": { "codepoints": [62], "characters": "\u003E" }, + "Γ": { "codepoints": [915], "characters": "\u0393" }, + "Ϝ": { "codepoints": [988], "characters": "\u03DC" }, + "Ğ": { "codepoints": [286], "characters": "\u011E" }, + "Ģ": { "codepoints": [290], "characters": "\u0122" }, + "Ĝ": { "codepoints": [284], "characters": "\u011C" }, + "Г": { "codepoints": [1043], "characters": "\u0413" }, + "Ġ": { "codepoints": [288], "characters": "\u0120" }, + "𝔊": { "codepoints": [120074], "characters": "\uD835\uDD0A" }, + "⋙": { "codepoints": [8921], "characters": "\u22D9" }, + "𝔾": { "codepoints": [120126], "characters": "\uD835\uDD3E" }, + "≥": { "codepoints": [8805], "characters": "\u2265" }, + "⋛": { "codepoints": [8923], "characters": "\u22DB" }, + "≧": { "codepoints": [8807], "characters": "\u2267" }, + "⪢": { "codepoints": [10914], "characters": "\u2AA2" }, + "≷": { "codepoints": [8823], "characters": "\u2277" }, + "⩾": { "codepoints": [10878], "characters": "\u2A7E" }, + "≳": { "codepoints": [8819], "characters": "\u2273" }, + "𝒢": { "codepoints": [119970], "characters": "\uD835\uDCA2" }, + "≫": { "codepoints": [8811], "characters": "\u226B" }, + "Ъ": { "codepoints": [1066], "characters": "\u042A" }, + "ˇ": { "codepoints": [711], "characters": "\u02C7" }, + "^": { "codepoints": [94], "characters": "\u005E" }, + "Ĥ": { "codepoints": [292], "characters": "\u0124" }, + "ℌ": { "codepoints": [8460], "characters": "\u210C" }, + "ℋ": { "codepoints": [8459], "characters": "\u210B" }, + "ℍ": { "codepoints": [8461], "characters": "\u210D" }, + "─": { "codepoints": [9472], "characters": "\u2500" }, + "ℋ": { "codepoints": [8459], "characters": "\u210B" }, + "Ħ": { "codepoints": [294], "characters": "\u0126" }, + "≎": { "codepoints": [8782], "characters": "\u224E" }, + "≏": { "codepoints": [8783], "characters": "\u224F" }, + "Е": { "codepoints": [1045], "characters": "\u0415" }, + "IJ": { "codepoints": [306], "characters": "\u0132" }, + "Ё": { "codepoints": [1025], "characters": "\u0401" }, + "Í": { "codepoints": [205], "characters": "\u00CD" }, + "Í": { "codepoints": [205], "characters": "\u00CD" }, + "Î": { "codepoints": [206], "characters": "\u00CE" }, + "Î": { "codepoints": [206], "characters": "\u00CE" }, + "И": { "codepoints": [1048], "characters": "\u0418" }, + "İ": { "codepoints": [304], "characters": "\u0130" }, + "ℑ": { "codepoints": [8465], "characters": "\u2111" }, + "Ì": { "codepoints": [204], "characters": "\u00CC" }, + "Ì": { "codepoints": [204], "characters": "\u00CC" }, + "ℑ": { "codepoints": [8465], "characters": "\u2111" }, + "Ī": { "codepoints": [298], "characters": "\u012A" }, + "ⅈ": { "codepoints": [8520], "characters": "\u2148" }, + "⇒": { "codepoints": [8658], "characters": "\u21D2" }, + "∬": { "codepoints": [8748], "characters": "\u222C" }, + "∫": { "codepoints": [8747], "characters": "\u222B" }, + "⋂": { "codepoints": [8898], "characters": "\u22C2" }, + "⁣": { "codepoints": [8291], "characters": "\u2063" }, + "⁢": { "codepoints": [8290], "characters": "\u2062" }, + "Į": { "codepoints": [302], "characters": "\u012E" }, + "𝕀": { "codepoints": [120128], "characters": "\uD835\uDD40" }, + "Ι": { "codepoints": [921], "characters": "\u0399" }, + "ℐ": { "codepoints": [8464], "characters": "\u2110" }, + "Ĩ": { "codepoints": [296], "characters": "\u0128" }, + "І": { "codepoints": [1030], "characters": "\u0406" }, + "Ï": { "codepoints": [207], "characters": "\u00CF" }, + "Ï": { "codepoints": [207], "characters": "\u00CF" }, + "Ĵ": { "codepoints": [308], "characters": "\u0134" }, + "Й": { "codepoints": [1049], "characters": "\u0419" }, + "𝔍": { "codepoints": [120077], "characters": "\uD835\uDD0D" }, + "𝕁": { "codepoints": [120129], "characters": "\uD835\uDD41" }, + "𝒥": { "codepoints": [119973], "characters": "\uD835\uDCA5" }, + "Ј": { "codepoints": [1032], "characters": "\u0408" }, + "Є": { "codepoints": [1028], "characters": "\u0404" }, + "Х": { "codepoints": [1061], "characters": "\u0425" }, + "Ќ": { "codepoints": [1036], "characters": "\u040C" }, + "Κ": { "codepoints": [922], "characters": "\u039A" }, + "Ķ": { "codepoints": [310], "characters": "\u0136" }, + "К": { "codepoints": [1050], "characters": "\u041A" }, + "𝔎": { "codepoints": [120078], "characters": "\uD835\uDD0E" }, + "𝕂": { "codepoints": [120130], "characters": "\uD835\uDD42" }, + "𝒦": { "codepoints": [119974], "characters": "\uD835\uDCA6" }, + "Љ": { "codepoints": [1033], "characters": "\u0409" }, + "<": { "codepoints": [60], "characters": "\u003C" }, + "<": { "codepoints": [60], "characters": "\u003C" }, + "Ĺ": { "codepoints": [313], "characters": "\u0139" }, + "Λ": { "codepoints": [923], "characters": "\u039B" }, + "⟪": { "codepoints": [10218], "characters": "\u27EA" }, + "ℒ": { "codepoints": [8466], "characters": "\u2112" }, + "↞": { "codepoints": [8606], "characters": "\u219E" }, + "Ľ": { "codepoints": [317], "characters": "\u013D" }, + "Ļ": { "codepoints": [315], "characters": "\u013B" }, + "Л": { "codepoints": [1051], "characters": "\u041B" }, + "⟨": { "codepoints": [10216], "characters": "\u27E8" }, + "←": { "codepoints": [8592], "characters": "\u2190" }, + "⇤": { "codepoints": [8676], "characters": "\u21E4" }, + "⇆": { "codepoints": [8646], "characters": "\u21C6" }, + "⌈": { "codepoints": [8968], "characters": "\u2308" }, + "⟦": { "codepoints": [10214], "characters": "\u27E6" }, + "⥡": { "codepoints": [10593], "characters": "\u2961" }, + "⇃": { "codepoints": [8643], "characters": "\u21C3" }, + "⥙": { "codepoints": [10585], "characters": "\u2959" }, + "⌊": { "codepoints": [8970], "characters": "\u230A" }, + "↔": { "codepoints": [8596], "characters": "\u2194" }, + "⥎": { "codepoints": [10574], "characters": "\u294E" }, + "⊣": { "codepoints": [8867], "characters": "\u22A3" }, + "↤": { "codepoints": [8612], "characters": "\u21A4" }, + "⥚": { "codepoints": [10586], "characters": "\u295A" }, + "⊲": { "codepoints": [8882], "characters": "\u22B2" }, + "⧏": { "codepoints": [10703], "characters": "\u29CF" }, + "⊴": { "codepoints": [8884], "characters": "\u22B4" }, + "⥑": { "codepoints": [10577], "characters": "\u2951" }, + "⥠": { "codepoints": [10592], "characters": "\u2960" }, + "↿": { "codepoints": [8639], "characters": "\u21BF" }, + "⥘": { "codepoints": [10584], "characters": "\u2958" }, + "↼": { "codepoints": [8636], "characters": "\u21BC" }, + "⥒": { "codepoints": [10578], "characters": "\u2952" }, + "⇐": { "codepoints": [8656], "characters": "\u21D0" }, + "⇔": { "codepoints": [8660], "characters": "\u21D4" }, + "⋚": { "codepoints": [8922], "characters": "\u22DA" }, + "≦": { "codepoints": [8806], "characters": "\u2266" }, + "≶": { "codepoints": [8822], "characters": "\u2276" }, + "⪡": { "codepoints": [10913], "characters": "\u2AA1" }, + "⩽": { "codepoints": [10877], "characters": "\u2A7D" }, + "≲": { "codepoints": [8818], "characters": "\u2272" }, + "𝔏": { "codepoints": [120079], "characters": "\uD835\uDD0F" }, + "⋘": { "codepoints": [8920], "characters": "\u22D8" }, + "⇚": { "codepoints": [8666], "characters": "\u21DA" }, + "Ŀ": { "codepoints": [319], "characters": "\u013F" }, + "⟵": { "codepoints": [10229], "characters": "\u27F5" }, + "⟷": { "codepoints": [10231], "characters": "\u27F7" }, + "⟶": { "codepoints": [10230], "characters": "\u27F6" }, + "⟸": { "codepoints": [10232], "characters": "\u27F8" }, + "⟺": { "codepoints": [10234], "characters": "\u27FA" }, + "⟹": { "codepoints": [10233], "characters": "\u27F9" }, + "𝕃": { "codepoints": [120131], "characters": "\uD835\uDD43" }, + "↙": { "codepoints": [8601], "characters": "\u2199" }, + "↘": { "codepoints": [8600], "characters": "\u2198" }, + "ℒ": { "codepoints": [8466], "characters": "\u2112" }, + "↰": { "codepoints": [8624], "characters": "\u21B0" }, + "Ł": { "codepoints": [321], "characters": "\u0141" }, + "≪": { "codepoints": [8810], "characters": "\u226A" }, + "⤅": { "codepoints": [10501], "characters": "\u2905" }, + "М": { "codepoints": [1052], "characters": "\u041C" }, + " ": { "codepoints": [8287], "characters": "\u205F" }, + "ℳ": { "codepoints": [8499], "characters": "\u2133" }, + "𝔐": { "codepoints": [120080], "characters": "\uD835\uDD10" }, + "∓": { "codepoints": [8723], "characters": "\u2213" }, + "𝕄": { "codepoints": [120132], "characters": "\uD835\uDD44" }, + "ℳ": { "codepoints": [8499], "characters": "\u2133" }, + "Μ": { "codepoints": [924], "characters": "\u039C" }, + "Њ": { "codepoints": [1034], "characters": "\u040A" }, + "Ń": { "codepoints": [323], "characters": "\u0143" }, + "Ň": { "codepoints": [327], "characters": "\u0147" }, + "Ņ": { "codepoints": [325], "characters": "\u0145" }, + "Н": { "codepoints": [1053], "characters": "\u041D" }, + "​": { "codepoints": [8203], "characters": "\u200B" }, + "​": { "codepoints": [8203], "characters": "\u200B" }, + "​": { "codepoints": [8203], "characters": "\u200B" }, + "​": { "codepoints": [8203], "characters": "\u200B" }, + "≫": { "codepoints": [8811], "characters": "\u226B" }, + "≪": { "codepoints": [8810], "characters": "\u226A" }, + "
": { "codepoints": [10], "characters": "\u000A" }, + "𝔑": { "codepoints": [120081], "characters": "\uD835\uDD11" }, + "⁠": { "codepoints": [8288], "characters": "\u2060" }, + " ": { "codepoints": [160], "characters": "\u00A0" }, + "ℕ": { "codepoints": [8469], "characters": "\u2115" }, + "⫬": { "codepoints": [10988], "characters": "\u2AEC" }, + "≢": { "codepoints": [8802], "characters": "\u2262" }, + "≭": { "codepoints": [8813], "characters": "\u226D" }, + "∦": { "codepoints": [8742], "characters": "\u2226" }, + "∉": { "codepoints": [8713], "characters": "\u2209" }, + "≠": { "codepoints": [8800], "characters": "\u2260" }, + "≂̸": { "codepoints": [8770, 824], "characters": "\u2242\u0338" }, + "∄": { "codepoints": [8708], "characters": "\u2204" }, + "≯": { "codepoints": [8815], "characters": "\u226F" }, + "≱": { "codepoints": [8817], "characters": "\u2271" }, + "≧̸": { "codepoints": [8807, 824], "characters": "\u2267\u0338" }, + "≫̸": { "codepoints": [8811, 824], "characters": "\u226B\u0338" }, + "≹": { "codepoints": [8825], "characters": "\u2279" }, + "⩾̸": { "codepoints": [10878, 824], "characters": "\u2A7E\u0338" }, + "≵": { "codepoints": [8821], "characters": "\u2275" }, + "≎̸": { "codepoints": [8782, 824], "characters": "\u224E\u0338" }, + "≏̸": { "codepoints": [8783, 824], "characters": "\u224F\u0338" }, + "⋪": { "codepoints": [8938], "characters": "\u22EA" }, + "⧏̸": { "codepoints": [10703, 824], "characters": "\u29CF\u0338" }, + "⋬": { "codepoints": [8940], "characters": "\u22EC" }, + "≮": { "codepoints": [8814], "characters": "\u226E" }, + "≰": { "codepoints": [8816], "characters": "\u2270" }, + "≸": { "codepoints": [8824], "characters": "\u2278" }, + "≪̸": { "codepoints": [8810, 824], "characters": "\u226A\u0338" }, + "⩽̸": { "codepoints": [10877, 824], "characters": "\u2A7D\u0338" }, + "≴": { "codepoints": [8820], "characters": "\u2274" }, + "⪢̸": { "codepoints": [10914, 824], "characters": "\u2AA2\u0338" }, + "⪡̸": { "codepoints": [10913, 824], "characters": "\u2AA1\u0338" }, + "⊀": { "codepoints": [8832], "characters": "\u2280" }, + "⪯̸": { "codepoints": [10927, 824], "characters": "\u2AAF\u0338" }, + "⋠": { "codepoints": [8928], "characters": "\u22E0" }, + "∌": { "codepoints": [8716], "characters": "\u220C" }, + "⋫": { "codepoints": [8939], "characters": "\u22EB" }, + "⧐̸": { "codepoints": [10704, 824], "characters": "\u29D0\u0338" }, + "⋭": { "codepoints": [8941], "characters": "\u22ED" }, + "⊏̸": { "codepoints": [8847, 824], "characters": "\u228F\u0338" }, + "⋢": { "codepoints": [8930], "characters": "\u22E2" }, + "⊐̸": { "codepoints": [8848, 824], "characters": "\u2290\u0338" }, + "⋣": { "codepoints": [8931], "characters": "\u22E3" }, + "⊂⃒": { "codepoints": [8834, 8402], "characters": "\u2282\u20D2" }, + "⊈": { "codepoints": [8840], "characters": "\u2288" }, + "⊁": { "codepoints": [8833], "characters": "\u2281" }, + "⪰̸": { "codepoints": [10928, 824], "characters": "\u2AB0\u0338" }, + "⋡": { "codepoints": [8929], "characters": "\u22E1" }, + "≿̸": { "codepoints": [8831, 824], "characters": "\u227F\u0338" }, + "⊃⃒": { "codepoints": [8835, 8402], "characters": "\u2283\u20D2" }, + "⊉": { "codepoints": [8841], "characters": "\u2289" }, + "≁": { "codepoints": [8769], "characters": "\u2241" }, + "≄": { "codepoints": [8772], "characters": "\u2244" }, + "≇": { "codepoints": [8775], "characters": "\u2247" }, + "≉": { "codepoints": [8777], "characters": "\u2249" }, + "∤": { "codepoints": [8740], "characters": "\u2224" }, + "𝒩": { "codepoints": [119977], "characters": "\uD835\uDCA9" }, + "Ñ": { "codepoints": [209], "characters": "\u00D1" }, + "Ñ": { "codepoints": [209], "characters": "\u00D1" }, + "Ν": { "codepoints": [925], "characters": "\u039D" }, + "Œ": { "codepoints": [338], "characters": "\u0152" }, + "Ó": { "codepoints": [211], "characters": "\u00D3" }, + "Ó": { "codepoints": [211], "characters": "\u00D3" }, + "Ô": { "codepoints": [212], "characters": "\u00D4" }, + "Ô": { "codepoints": [212], "characters": "\u00D4" }, + "О": { "codepoints": [1054], "characters": "\u041E" }, + "Ő": { "codepoints": [336], "characters": "\u0150" }, + "𝔒": { "codepoints": [120082], "characters": "\uD835\uDD12" }, + "Ò": { "codepoints": [210], "characters": "\u00D2" }, + "Ò": { "codepoints": [210], "characters": "\u00D2" }, + "Ō": { "codepoints": [332], "characters": "\u014C" }, + "Ω": { "codepoints": [937], "characters": "\u03A9" }, + "Ο": { "codepoints": [927], "characters": "\u039F" }, + "𝕆": { "codepoints": [120134], "characters": "\uD835\uDD46" }, + "“": { "codepoints": [8220], "characters": "\u201C" }, + "‘": { "codepoints": [8216], "characters": "\u2018" }, + "⩔": { "codepoints": [10836], "characters": "\u2A54" }, + "𝒪": { "codepoints": [119978], "characters": "\uD835\uDCAA" }, + "Ø": { "codepoints": [216], "characters": "\u00D8" }, + "Ø": { "codepoints": [216], "characters": "\u00D8" }, + "Õ": { "codepoints": [213], "characters": "\u00D5" }, + "Õ": { "codepoints": [213], "characters": "\u00D5" }, + "⨷": { "codepoints": [10807], "characters": "\u2A37" }, + "Ö": { "codepoints": [214], "characters": "\u00D6" }, + "Ö": { "codepoints": [214], "characters": "\u00D6" }, + "‾": { "codepoints": [8254], "characters": "\u203E" }, + "⏞": { "codepoints": [9182], "characters": "\u23DE" }, + "⎴": { "codepoints": [9140], "characters": "\u23B4" }, + "⏜": { "codepoints": [9180], "characters": "\u23DC" }, + "∂": { "codepoints": [8706], "characters": "\u2202" }, + "П": { "codepoints": [1055], "characters": "\u041F" }, + "𝔓": { "codepoints": [120083], "characters": "\uD835\uDD13" }, + "Φ": { "codepoints": [934], "characters": "\u03A6" }, + "Π": { "codepoints": [928], "characters": "\u03A0" }, + "±": { "codepoints": [177], "characters": "\u00B1" }, + "ℌ": { "codepoints": [8460], "characters": "\u210C" }, + "ℙ": { "codepoints": [8473], "characters": "\u2119" }, + "⪻": { "codepoints": [10939], "characters": "\u2ABB" }, + "≺": { "codepoints": [8826], "characters": "\u227A" }, + "⪯": { "codepoints": [10927], "characters": "\u2AAF" }, + "≼": { "codepoints": [8828], "characters": "\u227C" }, + "≾": { "codepoints": [8830], "characters": "\u227E" }, + "″": { "codepoints": [8243], "characters": "\u2033" }, + "∏": { "codepoints": [8719], "characters": "\u220F" }, + "∷": { "codepoints": [8759], "characters": "\u2237" }, + "∝": { "codepoints": [8733], "characters": "\u221D" }, + "𝒫": { "codepoints": [119979], "characters": "\uD835\uDCAB" }, + "Ψ": { "codepoints": [936], "characters": "\u03A8" }, + """: { "codepoints": [34], "characters": "\u0022" }, + """: { "codepoints": [34], "characters": "\u0022" }, + "𝔔": { "codepoints": [120084], "characters": "\uD835\uDD14" }, + "ℚ": { "codepoints": [8474], "characters": "\u211A" }, + "𝒬": { "codepoints": [119980], "characters": "\uD835\uDCAC" }, + "⤐": { "codepoints": [10512], "characters": "\u2910" }, + "®": { "codepoints": [174], "characters": "\u00AE" }, + "®": { "codepoints": [174], "characters": "\u00AE" }, + "Ŕ": { "codepoints": [340], "characters": "\u0154" }, + "⟫": { "codepoints": [10219], "characters": "\u27EB" }, + "↠": { "codepoints": [8608], "characters": "\u21A0" }, + "⤖": { "codepoints": [10518], "characters": "\u2916" }, + "Ř": { "codepoints": [344], "characters": "\u0158" }, + "Ŗ": { "codepoints": [342], "characters": "\u0156" }, + "Р": { "codepoints": [1056], "characters": "\u0420" }, + "ℜ": { "codepoints": [8476], "characters": "\u211C" }, + "∋": { "codepoints": [8715], "characters": "\u220B" }, + "⇋": { "codepoints": [8651], "characters": "\u21CB" }, + "⥯": { "codepoints": [10607], "characters": "\u296F" }, + "ℜ": { "codepoints": [8476], "characters": "\u211C" }, + "Ρ": { "codepoints": [929], "characters": "\u03A1" }, + "⟩": { "codepoints": [10217], "characters": "\u27E9" }, + "→": { "codepoints": [8594], "characters": "\u2192" }, + "⇥": { "codepoints": [8677], "characters": "\u21E5" }, + "⇄": { "codepoints": [8644], "characters": "\u21C4" }, + "⌉": { "codepoints": [8969], "characters": "\u2309" }, + "⟧": { "codepoints": [10215], "characters": "\u27E7" }, + "⥝": { "codepoints": [10589], "characters": "\u295D" }, + "⇂": { "codepoints": [8642], "characters": "\u21C2" }, + "⥕": { "codepoints": [10581], "characters": "\u2955" }, + "⌋": { "codepoints": [8971], "characters": "\u230B" }, + "⊢": { "codepoints": [8866], "characters": "\u22A2" }, + "↦": { "codepoints": [8614], "characters": "\u21A6" }, + "⥛": { "codepoints": [10587], "characters": "\u295B" }, + "⊳": { "codepoints": [8883], "characters": "\u22B3" }, + "⧐": { "codepoints": [10704], "characters": "\u29D0" }, + "⊵": { "codepoints": [8885], "characters": "\u22B5" }, + "⥏": { "codepoints": [10575], "characters": "\u294F" }, + "⥜": { "codepoints": [10588], "characters": "\u295C" }, + "↾": { "codepoints": [8638], "characters": "\u21BE" }, + "⥔": { "codepoints": [10580], "characters": "\u2954" }, + "⇀": { "codepoints": [8640], "characters": "\u21C0" }, + "⥓": { "codepoints": [10579], "characters": "\u2953" }, + "⇒": { "codepoints": [8658], "characters": "\u21D2" }, + "ℝ": { "codepoints": [8477], "characters": "\u211D" }, + "⥰": { "codepoints": [10608], "characters": "\u2970" }, + "⇛": { "codepoints": [8667], "characters": "\u21DB" }, + "ℛ": { "codepoints": [8475], "characters": "\u211B" }, + "↱": { "codepoints": [8625], "characters": "\u21B1" }, + "⧴": { "codepoints": [10740], "characters": "\u29F4" }, + "Щ": { "codepoints": [1065], "characters": "\u0429" }, + "Ш": { "codepoints": [1064], "characters": "\u0428" }, + "Ь": { "codepoints": [1068], "characters": "\u042C" }, + "Ś": { "codepoints": [346], "characters": "\u015A" }, + "⪼": { "codepoints": [10940], "characters": "\u2ABC" }, + "Š": { "codepoints": [352], "characters": "\u0160" }, + "Ş": { "codepoints": [350], "characters": "\u015E" }, + "Ŝ": { "codepoints": [348], "characters": "\u015C" }, + "С": { "codepoints": [1057], "characters": "\u0421" }, + "𝔖": { "codepoints": [120086], "characters": "\uD835\uDD16" }, + "↓": { "codepoints": [8595], "characters": "\u2193" }, + "←": { "codepoints": [8592], "characters": "\u2190" }, + "→": { "codepoints": [8594], "characters": "\u2192" }, + "↑": { "codepoints": [8593], "characters": "\u2191" }, + "Σ": { "codepoints": [931], "characters": "\u03A3" }, + "∘": { "codepoints": [8728], "characters": "\u2218" }, + "𝕊": { "codepoints": [120138], "characters": "\uD835\uDD4A" }, + "√": { "codepoints": [8730], "characters": "\u221A" }, + "□": { "codepoints": [9633], "characters": "\u25A1" }, + "⊓": { "codepoints": [8851], "characters": "\u2293" }, + "⊏": { "codepoints": [8847], "characters": "\u228F" }, + "⊑": { "codepoints": [8849], "characters": "\u2291" }, + "⊐": { "codepoints": [8848], "characters": "\u2290" }, + "⊒": { "codepoints": [8850], "characters": "\u2292" }, + "⊔": { "codepoints": [8852], "characters": "\u2294" }, + "𝒮": { "codepoints": [119982], "characters": "\uD835\uDCAE" }, + "⋆": { "codepoints": [8902], "characters": "\u22C6" }, + "⋐": { "codepoints": [8912], "characters": "\u22D0" }, + "⋐": { "codepoints": [8912], "characters": "\u22D0" }, + "⊆": { "codepoints": [8838], "characters": "\u2286" }, + "≻": { "codepoints": [8827], "characters": "\u227B" }, + "⪰": { "codepoints": [10928], "characters": "\u2AB0" }, + "≽": { "codepoints": [8829], "characters": "\u227D" }, + "≿": { "codepoints": [8831], "characters": "\u227F" }, + "∋": { "codepoints": [8715], "characters": "\u220B" }, + "∑": { "codepoints": [8721], "characters": "\u2211" }, + "⋑": { "codepoints": [8913], "characters": "\u22D1" }, + "⊃": { "codepoints": [8835], "characters": "\u2283" }, + "⊇": { "codepoints": [8839], "characters": "\u2287" }, + "⋑": { "codepoints": [8913], "characters": "\u22D1" }, + "Þ": { "codepoints": [222], "characters": "\u00DE" }, + "Þ": { "codepoints": [222], "characters": "\u00DE" }, + "™": { "codepoints": [8482], "characters": "\u2122" }, + "Ћ": { "codepoints": [1035], "characters": "\u040B" }, + "Ц": { "codepoints": [1062], "characters": "\u0426" }, + "	": { "codepoints": [9], "characters": "\u0009" }, + "Τ": { "codepoints": [932], "characters": "\u03A4" }, + "Ť": { "codepoints": [356], "characters": "\u0164" }, + "Ţ": { "codepoints": [354], "characters": "\u0162" }, + "Т": { "codepoints": [1058], "characters": "\u0422" }, + "𝔗": { "codepoints": [120087], "characters": "\uD835\uDD17" }, + "∴": { "codepoints": [8756], "characters": "\u2234" }, + "Θ": { "codepoints": [920], "characters": "\u0398" }, + "  ": { "codepoints": [8287, 8202], "characters": "\u205F\u200A" }, + " ": { "codepoints": [8201], "characters": "\u2009" }, + "∼": { "codepoints": [8764], "characters": "\u223C" }, + "≃": { "codepoints": [8771], "characters": "\u2243" }, + "≅": { "codepoints": [8773], "characters": "\u2245" }, + "≈": { "codepoints": [8776], "characters": "\u2248" }, + "𝕋": { "codepoints": [120139], "characters": "\uD835\uDD4B" }, + "⃛": { "codepoints": [8411], "characters": "\u20DB" }, + "𝒯": { "codepoints": [119983], "characters": "\uD835\uDCAF" }, + "Ŧ": { "codepoints": [358], "characters": "\u0166" }, + "Ú": { "codepoints": [218], "characters": "\u00DA" }, + "Ú": { "codepoints": [218], "characters": "\u00DA" }, + "↟": { "codepoints": [8607], "characters": "\u219F" }, + "⥉": { "codepoints": [10569], "characters": "\u2949" }, + "Ў": { "codepoints": [1038], "characters": "\u040E" }, + "Ŭ": { "codepoints": [364], "characters": "\u016C" }, + "Û": { "codepoints": [219], "characters": "\u00DB" }, + "Û": { "codepoints": [219], "characters": "\u00DB" }, + "У": { "codepoints": [1059], "characters": "\u0423" }, + "Ű": { "codepoints": [368], "characters": "\u0170" }, + "𝔘": { "codepoints": [120088], "characters": "\uD835\uDD18" }, + "Ù": { "codepoints": [217], "characters": "\u00D9" }, + "Ù": { "codepoints": [217], "characters": "\u00D9" }, + "Ū": { "codepoints": [362], "characters": "\u016A" }, + "_": { "codepoints": [95], "characters": "\u005F" }, + "⏟": { "codepoints": [9183], "characters": "\u23DF" }, + "⎵": { "codepoints": [9141], "characters": "\u23B5" }, + "⏝": { "codepoints": [9181], "characters": "\u23DD" }, + "⋃": { "codepoints": [8899], "characters": "\u22C3" }, + "⊎": { "codepoints": [8846], "characters": "\u228E" }, + "Ų": { "codepoints": [370], "characters": "\u0172" }, + "𝕌": { "codepoints": [120140], "characters": "\uD835\uDD4C" }, + "↑": { "codepoints": [8593], "characters": "\u2191" }, + "⤒": { "codepoints": [10514], "characters": "\u2912" }, + "⇅": { "codepoints": [8645], "characters": "\u21C5" }, + "↕": { "codepoints": [8597], "characters": "\u2195" }, + "⥮": { "codepoints": [10606], "characters": "\u296E" }, + "⊥": { "codepoints": [8869], "characters": "\u22A5" }, + "↥": { "codepoints": [8613], "characters": "\u21A5" }, + "⇑": { "codepoints": [8657], "characters": "\u21D1" }, + "⇕": { "codepoints": [8661], "characters": "\u21D5" }, + "↖": { "codepoints": [8598], "characters": "\u2196" }, + "↗": { "codepoints": [8599], "characters": "\u2197" }, + "ϒ": { "codepoints": [978], "characters": "\u03D2" }, + "Υ": { "codepoints": [933], "characters": "\u03A5" }, + "Ů": { "codepoints": [366], "characters": "\u016E" }, + "𝒰": { "codepoints": [119984], "characters": "\uD835\uDCB0" }, + "Ũ": { "codepoints": [360], "characters": "\u0168" }, + "Ü": { "codepoints": [220], "characters": "\u00DC" }, + "Ü": { "codepoints": [220], "characters": "\u00DC" }, + "⊫": { "codepoints": [8875], "characters": "\u22AB" }, + "⫫": { "codepoints": [10987], "characters": "\u2AEB" }, + "В": { "codepoints": [1042], "characters": "\u0412" }, + "⊩": { "codepoints": [8873], "characters": "\u22A9" }, + "⫦": { "codepoints": [10982], "characters": "\u2AE6" }, + "⋁": { "codepoints": [8897], "characters": "\u22C1" }, + "‖": { "codepoints": [8214], "characters": "\u2016" }, + "‖": { "codepoints": [8214], "characters": "\u2016" }, + "∣": { "codepoints": [8739], "characters": "\u2223" }, + "|": { "codepoints": [124], "characters": "\u007C" }, + "❘": { "codepoints": [10072], "characters": "\u2758" }, + "≀": { "codepoints": [8768], "characters": "\u2240" }, + " ": { "codepoints": [8202], "characters": "\u200A" }, + "𝔙": { "codepoints": [120089], "characters": "\uD835\uDD19" }, + "𝕍": { "codepoints": [120141], "characters": "\uD835\uDD4D" }, + "𝒱": { "codepoints": [119985], "characters": "\uD835\uDCB1" }, + "⊪": { "codepoints": [8874], "characters": "\u22AA" }, + "Ŵ": { "codepoints": [372], "characters": "\u0174" }, + "⋀": { "codepoints": [8896], "characters": "\u22C0" }, + "𝔚": { "codepoints": [120090], "characters": "\uD835\uDD1A" }, + "𝕎": { "codepoints": [120142], "characters": "\uD835\uDD4E" }, + "𝒲": { "codepoints": [119986], "characters": "\uD835\uDCB2" }, + "𝔛": { "codepoints": [120091], "characters": "\uD835\uDD1B" }, + "Ξ": { "codepoints": [926], "characters": "\u039E" }, + "𝕏": { "codepoints": [120143], "characters": "\uD835\uDD4F" }, + "𝒳": { "codepoints": [119987], "characters": "\uD835\uDCB3" }, + "Я": { "codepoints": [1071], "characters": "\u042F" }, + "Ї": { "codepoints": [1031], "characters": "\u0407" }, + "Ю": { "codepoints": [1070], "characters": "\u042E" }, + "Ý": { "codepoints": [221], "characters": "\u00DD" }, + "Ý": { "codepoints": [221], "characters": "\u00DD" }, + "Ŷ": { "codepoints": [374], "characters": "\u0176" }, + "Ы": { "codepoints": [1067], "characters": "\u042B" }, + "𝔜": { "codepoints": [120092], "characters": "\uD835\uDD1C" }, + "𝕐": { "codepoints": [120144], "characters": "\uD835\uDD50" }, + "𝒴": { "codepoints": [119988], "characters": "\uD835\uDCB4" }, + "Ÿ": { "codepoints": [376], "characters": "\u0178" }, + "Ж": { "codepoints": [1046], "characters": "\u0416" }, + "Ź": { "codepoints": [377], "characters": "\u0179" }, + "Ž": { "codepoints": [381], "characters": "\u017D" }, + "З": { "codepoints": [1047], "characters": "\u0417" }, + "Ż": { "codepoints": [379], "characters": "\u017B" }, + "​": { "codepoints": [8203], "characters": "\u200B" }, + "Ζ": { "codepoints": [918], "characters": "\u0396" }, + "ℨ": { "codepoints": [8488], "characters": "\u2128" }, + "ℤ": { "codepoints": [8484], "characters": "\u2124" }, + "𝒵": { "codepoints": [119989], "characters": "\uD835\uDCB5" }, + "á": { "codepoints": [225], "characters": "\u00E1" }, + "á": { "codepoints": [225], "characters": "\u00E1" }, + "ă": { "codepoints": [259], "characters": "\u0103" }, + "∾": { "codepoints": [8766], "characters": "\u223E" }, + "∾̳": { "codepoints": [8766, 819], "characters": "\u223E\u0333" }, + "∿": { "codepoints": [8767], "characters": "\u223F" }, + "â": { "codepoints": [226], "characters": "\u00E2" }, + "â": { "codepoints": [226], "characters": "\u00E2" }, + "´": { "codepoints": [180], "characters": "\u00B4" }, + "´": { "codepoints": [180], "characters": "\u00B4" }, + "а": { "codepoints": [1072], "characters": "\u0430" }, + "æ": { "codepoints": [230], "characters": "\u00E6" }, + "æ": { "codepoints": [230], "characters": "\u00E6" }, + "⁡": { "codepoints": [8289], "characters": "\u2061" }, + "𝔞": { "codepoints": [120094], "characters": "\uD835\uDD1E" }, + "à": { "codepoints": [224], "characters": "\u00E0" }, + "à": { "codepoints": [224], "characters": "\u00E0" }, + "ℵ": { "codepoints": [8501], "characters": "\u2135" }, + "ℵ": { "codepoints": [8501], "characters": "\u2135" }, + "α": { "codepoints": [945], "characters": "\u03B1" }, + "ā": { "codepoints": [257], "characters": "\u0101" }, + "⨿": { "codepoints": [10815], "characters": "\u2A3F" }, + "&": { "codepoints": [38], "characters": "\u0026" }, + "&": { "codepoints": [38], "characters": "\u0026" }, + "∧": { "codepoints": [8743], "characters": "\u2227" }, + "⩕": { "codepoints": [10837], "characters": "\u2A55" }, + "⩜": { "codepoints": [10844], "characters": "\u2A5C" }, + "⩘": { "codepoints": [10840], "characters": "\u2A58" }, + "⩚": { "codepoints": [10842], "characters": "\u2A5A" }, + "∠": { "codepoints": [8736], "characters": "\u2220" }, + "⦤": { "codepoints": [10660], "characters": "\u29A4" }, + "∠": { "codepoints": [8736], "characters": "\u2220" }, + "∡": { "codepoints": [8737], "characters": "\u2221" }, + "⦨": { "codepoints": [10664], "characters": "\u29A8" }, + "⦩": { "codepoints": [10665], "characters": "\u29A9" }, + "⦪": { "codepoints": [10666], "characters": "\u29AA" }, + "⦫": { "codepoints": [10667], "characters": "\u29AB" }, + "⦬": { "codepoints": [10668], "characters": "\u29AC" }, + "⦭": { "codepoints": [10669], "characters": "\u29AD" }, + "⦮": { "codepoints": [10670], "characters": "\u29AE" }, + "⦯": { "codepoints": [10671], "characters": "\u29AF" }, + "∟": { "codepoints": [8735], "characters": "\u221F" }, + "⊾": { "codepoints": [8894], "characters": "\u22BE" }, + "⦝": { "codepoints": [10653], "characters": "\u299D" }, + "∢": { "codepoints": [8738], "characters": "\u2222" }, + "Å": { "codepoints": [197], "characters": "\u00C5" }, + "⍼": { "codepoints": [9084], "characters": "\u237C" }, + "ą": { "codepoints": [261], "characters": "\u0105" }, + "𝕒": { "codepoints": [120146], "characters": "\uD835\uDD52" }, + "≈": { "codepoints": [8776], "characters": "\u2248" }, + "⩰": { "codepoints": [10864], "characters": "\u2A70" }, + "⩯": { "codepoints": [10863], "characters": "\u2A6F" }, + "≊": { "codepoints": [8778], "characters": "\u224A" }, + "≋": { "codepoints": [8779], "characters": "\u224B" }, + "'": { "codepoints": [39], "characters": "\u0027" }, + "≈": { "codepoints": [8776], "characters": "\u2248" }, + "≊": { "codepoints": [8778], "characters": "\u224A" }, + "å": { "codepoints": [229], "characters": "\u00E5" }, + "å": { "codepoints": [229], "characters": "\u00E5" }, + "𝒶": { "codepoints": [119990], "characters": "\uD835\uDCB6" }, + "*": { "codepoints": [42], "characters": "\u002A" }, + "≈": { "codepoints": [8776], "characters": "\u2248" }, + "≍": { "codepoints": [8781], "characters": "\u224D" }, + "ã": { "codepoints": [227], "characters": "\u00E3" }, + "ã": { "codepoints": [227], "characters": "\u00E3" }, + "ä": { "codepoints": [228], "characters": "\u00E4" }, + "ä": { "codepoints": [228], "characters": "\u00E4" }, + "∳": { "codepoints": [8755], "characters": "\u2233" }, + "⨑": { "codepoints": [10769], "characters": "\u2A11" }, + "⫭": { "codepoints": [10989], "characters": "\u2AED" }, + "≌": { "codepoints": [8780], "characters": "\u224C" }, + "϶": { "codepoints": [1014], "characters": "\u03F6" }, + "‵": { "codepoints": [8245], "characters": "\u2035" }, + "∽": { "codepoints": [8765], "characters": "\u223D" }, + "⋍": { "codepoints": [8909], "characters": "\u22CD" }, + "⊽": { "codepoints": [8893], "characters": "\u22BD" }, + "⌅": { "codepoints": [8965], "characters": "\u2305" }, + "⌅": { "codepoints": [8965], "characters": "\u2305" }, + "⎵": { "codepoints": [9141], "characters": "\u23B5" }, + "⎶": { "codepoints": [9142], "characters": "\u23B6" }, + "≌": { "codepoints": [8780], "characters": "\u224C" }, + "б": { "codepoints": [1073], "characters": "\u0431" }, + "„": { "codepoints": [8222], "characters": "\u201E" }, + "∵": { "codepoints": [8757], "characters": "\u2235" }, + "∵": { "codepoints": [8757], "characters": "\u2235" }, + "⦰": { "codepoints": [10672], "characters": "\u29B0" }, + "϶": { "codepoints": [1014], "characters": "\u03F6" }, + "ℬ": { "codepoints": [8492], "characters": "\u212C" }, + "β": { "codepoints": [946], "characters": "\u03B2" }, + "ℶ": { "codepoints": [8502], "characters": "\u2136" }, + "≬": { "codepoints": [8812], "characters": "\u226C" }, + "𝔟": { "codepoints": [120095], "characters": "\uD835\uDD1F" }, + "⋂": { "codepoints": [8898], "characters": "\u22C2" }, + "◯": { "codepoints": [9711], "characters": "\u25EF" }, + "⋃": { "codepoints": [8899], "characters": "\u22C3" }, + "⨀": { "codepoints": [10752], "characters": "\u2A00" }, + "⨁": { "codepoints": [10753], "characters": "\u2A01" }, + "⨂": { "codepoints": [10754], "characters": "\u2A02" }, + "⨆": { "codepoints": [10758], "characters": "\u2A06" }, + "★": { "codepoints": [9733], "characters": "\u2605" }, + "▽": { "codepoints": [9661], "characters": "\u25BD" }, + "△": { "codepoints": [9651], "characters": "\u25B3" }, + "⨄": { "codepoints": [10756], "characters": "\u2A04" }, + "⋁": { "codepoints": [8897], "characters": "\u22C1" }, + "⋀": { "codepoints": [8896], "characters": "\u22C0" }, + "⤍": { "codepoints": [10509], "characters": "\u290D" }, + "⧫": { "codepoints": [10731], "characters": "\u29EB" }, + "▪": { "codepoints": [9642], "characters": "\u25AA" }, + "▴": { "codepoints": [9652], "characters": "\u25B4" }, + "▾": { "codepoints": [9662], "characters": "\u25BE" }, + "◂": { "codepoints": [9666], "characters": "\u25C2" }, + "▸": { "codepoints": [9656], "characters": "\u25B8" }, + "␣": { "codepoints": [9251], "characters": "\u2423" }, + "▒": { "codepoints": [9618], "characters": "\u2592" }, + "░": { "codepoints": [9617], "characters": "\u2591" }, + "▓": { "codepoints": [9619], "characters": "\u2593" }, + "█": { "codepoints": [9608], "characters": "\u2588" }, + "=⃥": { "codepoints": [61, 8421], "characters": "\u003D\u20E5" }, + "≡⃥": { "codepoints": [8801, 8421], "characters": "\u2261\u20E5" }, + "⌐": { "codepoints": [8976], "characters": "\u2310" }, + "𝕓": { "codepoints": [120147], "characters": "\uD835\uDD53" }, + "⊥": { "codepoints": [8869], "characters": "\u22A5" }, + "⊥": { "codepoints": [8869], "characters": "\u22A5" }, + "⋈": { "codepoints": [8904], "characters": "\u22C8" }, + "╗": { "codepoints": [9559], "characters": "\u2557" }, + "╔": { "codepoints": [9556], "characters": "\u2554" }, + "╖": { "codepoints": [9558], "characters": "\u2556" }, + "╓": { "codepoints": [9555], "characters": "\u2553" }, + "═": { "codepoints": [9552], "characters": "\u2550" }, + "╦": { "codepoints": [9574], "characters": "\u2566" }, + "╩": { "codepoints": [9577], "characters": "\u2569" }, + "╤": { "codepoints": [9572], "characters": "\u2564" }, + "╧": { "codepoints": [9575], "characters": "\u2567" }, + "╝": { "codepoints": [9565], "characters": "\u255D" }, + "╚": { "codepoints": [9562], "characters": "\u255A" }, + "╜": { "codepoints": [9564], "characters": "\u255C" }, + "╙": { "codepoints": [9561], "characters": "\u2559" }, + "║": { "codepoints": [9553], "characters": "\u2551" }, + "╬": { "codepoints": [9580], "characters": "\u256C" }, + "╣": { "codepoints": [9571], "characters": "\u2563" }, + "╠": { "codepoints": [9568], "characters": "\u2560" }, + "╫": { "codepoints": [9579], "characters": "\u256B" }, + "╢": { "codepoints": [9570], "characters": "\u2562" }, + "╟": { "codepoints": [9567], "characters": "\u255F" }, + "⧉": { "codepoints": [10697], "characters": "\u29C9" }, + "╕": { "codepoints": [9557], "characters": "\u2555" }, + "╒": { "codepoints": [9554], "characters": "\u2552" }, + "┐": { "codepoints": [9488], "characters": "\u2510" }, + "┌": { "codepoints": [9484], "characters": "\u250C" }, + "─": { "codepoints": [9472], "characters": "\u2500" }, + "╥": { "codepoints": [9573], "characters": "\u2565" }, + "╨": { "codepoints": [9576], "characters": "\u2568" }, + "┬": { "codepoints": [9516], "characters": "\u252C" }, + "┴": { "codepoints": [9524], "characters": "\u2534" }, + "⊟": { "codepoints": [8863], "characters": "\u229F" }, + "⊞": { "codepoints": [8862], "characters": "\u229E" }, + "⊠": { "codepoints": [8864], "characters": "\u22A0" }, + "╛": { "codepoints": [9563], "characters": "\u255B" }, + "╘": { "codepoints": [9560], "characters": "\u2558" }, + "┘": { "codepoints": [9496], "characters": "\u2518" }, + "└": { "codepoints": [9492], "characters": "\u2514" }, + "│": { "codepoints": [9474], "characters": "\u2502" }, + "╪": { "codepoints": [9578], "characters": "\u256A" }, + "╡": { "codepoints": [9569], "characters": "\u2561" }, + "╞": { "codepoints": [9566], "characters": "\u255E" }, + "┼": { "codepoints": [9532], "characters": "\u253C" }, + "┤": { "codepoints": [9508], "characters": "\u2524" }, + "├": { "codepoints": [9500], "characters": "\u251C" }, + "‵": { "codepoints": [8245], "characters": "\u2035" }, + "˘": { "codepoints": [728], "characters": "\u02D8" }, + "¦": { "codepoints": [166], "characters": "\u00A6" }, + "¦": { "codepoints": [166], "characters": "\u00A6" }, + "𝒷": { "codepoints": [119991], "characters": "\uD835\uDCB7" }, + "⁏": { "codepoints": [8271], "characters": "\u204F" }, + "∽": { "codepoints": [8765], "characters": "\u223D" }, + "⋍": { "codepoints": [8909], "characters": "\u22CD" }, + "\": { "codepoints": [92], "characters": "\u005C" }, + "⧅": { "codepoints": [10693], "characters": "\u29C5" }, + "⟈": { "codepoints": [10184], "characters": "\u27C8" }, + "•": { "codepoints": [8226], "characters": "\u2022" }, + "•": { "codepoints": [8226], "characters": "\u2022" }, + "≎": { "codepoints": [8782], "characters": "\u224E" }, + "⪮": { "codepoints": [10926], "characters": "\u2AAE" }, + "≏": { "codepoints": [8783], "characters": "\u224F" }, + "≏": { "codepoints": [8783], "characters": "\u224F" }, + "ć": { "codepoints": [263], "characters": "\u0107" }, + "∩": { "codepoints": [8745], "characters": "\u2229" }, + "⩄": { "codepoints": [10820], "characters": "\u2A44" }, + "⩉": { "codepoints": [10825], "characters": "\u2A49" }, + "⩋": { "codepoints": [10827], "characters": "\u2A4B" }, + "⩇": { "codepoints": [10823], "characters": "\u2A47" }, + "⩀": { "codepoints": [10816], "characters": "\u2A40" }, + "∩︀": { "codepoints": [8745, 65024], "characters": "\u2229\uFE00" }, + "⁁": { "codepoints": [8257], "characters": "\u2041" }, + "ˇ": { "codepoints": [711], "characters": "\u02C7" }, + "⩍": { "codepoints": [10829], "characters": "\u2A4D" }, + "č": { "codepoints": [269], "characters": "\u010D" }, + "ç": { "codepoints": [231], "characters": "\u00E7" }, + "ç": { "codepoints": [231], "characters": "\u00E7" }, + "ĉ": { "codepoints": [265], "characters": "\u0109" }, + "⩌": { "codepoints": [10828], "characters": "\u2A4C" }, + "⩐": { "codepoints": [10832], "characters": "\u2A50" }, + "ċ": { "codepoints": [267], "characters": "\u010B" }, + "¸": { "codepoints": [184], "characters": "\u00B8" }, + "¸": { "codepoints": [184], "characters": "\u00B8" }, + "⦲": { "codepoints": [10674], "characters": "\u29B2" }, + "¢": { "codepoints": [162], "characters": "\u00A2" }, + "¢": { "codepoints": [162], "characters": "\u00A2" }, + "·": { "codepoints": [183], "characters": "\u00B7" }, + "𝔠": { "codepoints": [120096], "characters": "\uD835\uDD20" }, + "ч": { "codepoints": [1095], "characters": "\u0447" }, + "✓": { "codepoints": [10003], "characters": "\u2713" }, + "✓": { "codepoints": [10003], "characters": "\u2713" }, + "χ": { "codepoints": [967], "characters": "\u03C7" }, + "○": { "codepoints": [9675], "characters": "\u25CB" }, + "⧃": { "codepoints": [10691], "characters": "\u29C3" }, + "ˆ": { "codepoints": [710], "characters": "\u02C6" }, + "≗": { "codepoints": [8791], "characters": "\u2257" }, + "↺": { "codepoints": [8634], "characters": "\u21BA" }, + "↻": { "codepoints": [8635], "characters": "\u21BB" }, + "®": { "codepoints": [174], "characters": "\u00AE" }, + "Ⓢ": { "codepoints": [9416], "characters": "\u24C8" }, + "⊛": { "codepoints": [8859], "characters": "\u229B" }, + "⊚": { "codepoints": [8858], "characters": "\u229A" }, + "⊝": { "codepoints": [8861], "characters": "\u229D" }, + "≗": { "codepoints": [8791], "characters": "\u2257" }, + "⨐": { "codepoints": [10768], "characters": "\u2A10" }, + "⫯": { "codepoints": [10991], "characters": "\u2AEF" }, + "⧂": { "codepoints": [10690], "characters": "\u29C2" }, + "♣": { "codepoints": [9827], "characters": "\u2663" }, + "♣": { "codepoints": [9827], "characters": "\u2663" }, + ":": { "codepoints": [58], "characters": "\u003A" }, + "≔": { "codepoints": [8788], "characters": "\u2254" }, + "≔": { "codepoints": [8788], "characters": "\u2254" }, + ",": { "codepoints": [44], "characters": "\u002C" }, + "@": { "codepoints": [64], "characters": "\u0040" }, + "∁": { "codepoints": [8705], "characters": "\u2201" }, + "∘": { "codepoints": [8728], "characters": "\u2218" }, + "∁": { "codepoints": [8705], "characters": "\u2201" }, + "ℂ": { "codepoints": [8450], "characters": "\u2102" }, + "≅": { "codepoints": [8773], "characters": "\u2245" }, + "⩭": { "codepoints": [10861], "characters": "\u2A6D" }, + "∮": { "codepoints": [8750], "characters": "\u222E" }, + "𝕔": { "codepoints": [120148], "characters": "\uD835\uDD54" }, + "∐": { "codepoints": [8720], "characters": "\u2210" }, + "©": { "codepoints": [169], "characters": "\u00A9" }, + "©": { "codepoints": [169], "characters": "\u00A9" }, + "℗": { "codepoints": [8471], "characters": "\u2117" }, + "↵": { "codepoints": [8629], "characters": "\u21B5" }, + "✗": { "codepoints": [10007], "characters": "\u2717" }, + "𝒸": { "codepoints": [119992], "characters": "\uD835\uDCB8" }, + "⫏": { "codepoints": [10959], "characters": "\u2ACF" }, + "⫑": { "codepoints": [10961], "characters": "\u2AD1" }, + "⫐": { "codepoints": [10960], "characters": "\u2AD0" }, + "⫒": { "codepoints": [10962], "characters": "\u2AD2" }, + "⋯": { "codepoints": [8943], "characters": "\u22EF" }, + "⤸": { "codepoints": [10552], "characters": "\u2938" }, + "⤵": { "codepoints": [10549], "characters": "\u2935" }, + "⋞": { "codepoints": [8926], "characters": "\u22DE" }, + "⋟": { "codepoints": [8927], "characters": "\u22DF" }, + "↶": { "codepoints": [8630], "characters": "\u21B6" }, + "⤽": { "codepoints": [10557], "characters": "\u293D" }, + "∪": { "codepoints": [8746], "characters": "\u222A" }, + "⩈": { "codepoints": [10824], "characters": "\u2A48" }, + "⩆": { "codepoints": [10822], "characters": "\u2A46" }, + "⩊": { "codepoints": [10826], "characters": "\u2A4A" }, + "⊍": { "codepoints": [8845], "characters": "\u228D" }, + "⩅": { "codepoints": [10821], "characters": "\u2A45" }, + "∪︀": { "codepoints": [8746, 65024], "characters": "\u222A\uFE00" }, + "↷": { "codepoints": [8631], "characters": "\u21B7" }, + "⤼": { "codepoints": [10556], "characters": "\u293C" }, + "⋞": { "codepoints": [8926], "characters": "\u22DE" }, + "⋟": { "codepoints": [8927], "characters": "\u22DF" }, + "⋎": { "codepoints": [8910], "characters": "\u22CE" }, + "⋏": { "codepoints": [8911], "characters": "\u22CF" }, + "¤": { "codepoints": [164], "characters": "\u00A4" }, + "¤": { "codepoints": [164], "characters": "\u00A4" }, + "↶": { "codepoints": [8630], "characters": "\u21B6" }, + "↷": { "codepoints": [8631], "characters": "\u21B7" }, + "⋎": { "codepoints": [8910], "characters": "\u22CE" }, + "⋏": { "codepoints": [8911], "characters": "\u22CF" }, + "∲": { "codepoints": [8754], "characters": "\u2232" }, + "∱": { "codepoints": [8753], "characters": "\u2231" }, + "⌭": { "codepoints": [9005], "characters": "\u232D" }, + "⇓": { "codepoints": [8659], "characters": "\u21D3" }, + "⥥": { "codepoints": [10597], "characters": "\u2965" }, + "†": { "codepoints": [8224], "characters": "\u2020" }, + "ℸ": { "codepoints": [8504], "characters": "\u2138" }, + "↓": { "codepoints": [8595], "characters": "\u2193" }, + "‐": { "codepoints": [8208], "characters": "\u2010" }, + "⊣": { "codepoints": [8867], "characters": "\u22A3" }, + "⤏": { "codepoints": [10511], "characters": "\u290F" }, + "˝": { "codepoints": [733], "characters": "\u02DD" }, + "ď": { "codepoints": [271], "characters": "\u010F" }, + "д": { "codepoints": [1076], "characters": "\u0434" }, + "ⅆ": { "codepoints": [8518], "characters": "\u2146" }, + "‡": { "codepoints": [8225], "characters": "\u2021" }, + "⇊": { "codepoints": [8650], "characters": "\u21CA" }, + "⩷": { "codepoints": [10871], "characters": "\u2A77" }, + "°": { "codepoints": [176], "characters": "\u00B0" }, + "°": { "codepoints": [176], "characters": "\u00B0" }, + "δ": { "codepoints": [948], "characters": "\u03B4" }, + "⦱": { "codepoints": [10673], "characters": "\u29B1" }, + "⥿": { "codepoints": [10623], "characters": "\u297F" }, + "𝔡": { "codepoints": [120097], "characters": "\uD835\uDD21" }, + "⇃": { "codepoints": [8643], "characters": "\u21C3" }, + "⇂": { "codepoints": [8642], "characters": "\u21C2" }, + "⋄": { "codepoints": [8900], "characters": "\u22C4" }, + "⋄": { "codepoints": [8900], "characters": "\u22C4" }, + "♦": { "codepoints": [9830], "characters": "\u2666" }, + "♦": { "codepoints": [9830], "characters": "\u2666" }, + "¨": { "codepoints": [168], "characters": "\u00A8" }, + "ϝ": { "codepoints": [989], "characters": "\u03DD" }, + "⋲": { "codepoints": [8946], "characters": "\u22F2" }, + "÷": { "codepoints": [247], "characters": "\u00F7" }, + "÷": { "codepoints": [247], "characters": "\u00F7" }, + "÷": { "codepoints": [247], "characters": "\u00F7" }, + "⋇": { "codepoints": [8903], "characters": "\u22C7" }, + "⋇": { "codepoints": [8903], "characters": "\u22C7" }, + "ђ": { "codepoints": [1106], "characters": "\u0452" }, + "⌞": { "codepoints": [8990], "characters": "\u231E" }, + "⌍": { "codepoints": [8973], "characters": "\u230D" }, + "$": { "codepoints": [36], "characters": "\u0024" }, + "𝕕": { "codepoints": [120149], "characters": "\uD835\uDD55" }, + "˙": { "codepoints": [729], "characters": "\u02D9" }, + "≐": { "codepoints": [8784], "characters": "\u2250" }, + "≑": { "codepoints": [8785], "characters": "\u2251" }, + "∸": { "codepoints": [8760], "characters": "\u2238" }, + "∔": { "codepoints": [8724], "characters": "\u2214" }, + "⊡": { "codepoints": [8865], "characters": "\u22A1" }, + "⌆": { "codepoints": [8966], "characters": "\u2306" }, + "↓": { "codepoints": [8595], "characters": "\u2193" }, + "⇊": { "codepoints": [8650], "characters": "\u21CA" }, + "⇃": { "codepoints": [8643], "characters": "\u21C3" }, + "⇂": { "codepoints": [8642], "characters": "\u21C2" }, + "⤐": { "codepoints": [10512], "characters": "\u2910" }, + "⌟": { "codepoints": [8991], "characters": "\u231F" }, + "⌌": { "codepoints": [8972], "characters": "\u230C" }, + "𝒹": { "codepoints": [119993], "characters": "\uD835\uDCB9" }, + "ѕ": { "codepoints": [1109], "characters": "\u0455" }, + "⧶": { "codepoints": [10742], "characters": "\u29F6" }, + "đ": { "codepoints": [273], "characters": "\u0111" }, + "⋱": { "codepoints": [8945], "characters": "\u22F1" }, + "▿": { "codepoints": [9663], "characters": "\u25BF" }, + "▾": { "codepoints": [9662], "characters": "\u25BE" }, + "⇵": { "codepoints": [8693], "characters": "\u21F5" }, + "⥯": { "codepoints": [10607], "characters": "\u296F" }, + "⦦": { "codepoints": [10662], "characters": "\u29A6" }, + "џ": { "codepoints": [1119], "characters": "\u045F" }, + "⟿": { "codepoints": [10239], "characters": "\u27FF" }, + "⩷": { "codepoints": [10871], "characters": "\u2A77" }, + "≑": { "codepoints": [8785], "characters": "\u2251" }, + "é": { "codepoints": [233], "characters": "\u00E9" }, + "é": { "codepoints": [233], "characters": "\u00E9" }, + "⩮": { "codepoints": [10862], "characters": "\u2A6E" }, + "ě": { "codepoints": [283], "characters": "\u011B" }, + "≖": { "codepoints": [8790], "characters": "\u2256" }, + "ê": { "codepoints": [234], "characters": "\u00EA" }, + "ê": { "codepoints": [234], "characters": "\u00EA" }, + "≕": { "codepoints": [8789], "characters": "\u2255" }, + "э": { "codepoints": [1101], "characters": "\u044D" }, + "ė": { "codepoints": [279], "characters": "\u0117" }, + "ⅇ": { "codepoints": [8519], "characters": "\u2147" }, + "≒": { "codepoints": [8786], "characters": "\u2252" }, + "𝔢": { "codepoints": [120098], "characters": "\uD835\uDD22" }, + "⪚": { "codepoints": [10906], "characters": "\u2A9A" }, + "è": { "codepoints": [232], "characters": "\u00E8" }, + "è": { "codepoints": [232], "characters": "\u00E8" }, + "⪖": { "codepoints": [10902], "characters": "\u2A96" }, + "⪘": { "codepoints": [10904], "characters": "\u2A98" }, + "⪙": { "codepoints": [10905], "characters": "\u2A99" }, + "⏧": { "codepoints": [9191], "characters": "\u23E7" }, + "ℓ": { "codepoints": [8467], "characters": "\u2113" }, + "⪕": { "codepoints": [10901], "characters": "\u2A95" }, + "⪗": { "codepoints": [10903], "characters": "\u2A97" }, + "ē": { "codepoints": [275], "characters": "\u0113" }, + "∅": { "codepoints": [8709], "characters": "\u2205" }, + "∅": { "codepoints": [8709], "characters": "\u2205" }, + "∅": { "codepoints": [8709], "characters": "\u2205" }, + " ": { "codepoints": [8196], "characters": "\u2004" }, + " ": { "codepoints": [8197], "characters": "\u2005" }, + " ": { "codepoints": [8195], "characters": "\u2003" }, + "ŋ": { "codepoints": [331], "characters": "\u014B" }, + " ": { "codepoints": [8194], "characters": "\u2002" }, + "ę": { "codepoints": [281], "characters": "\u0119" }, + "𝕖": { "codepoints": [120150], "characters": "\uD835\uDD56" }, + "⋕": { "codepoints": [8917], "characters": "\u22D5" }, + "⧣": { "codepoints": [10723], "characters": "\u29E3" }, + "⩱": { "codepoints": [10865], "characters": "\u2A71" }, + "ε": { "codepoints": [949], "characters": "\u03B5" }, + "ε": { "codepoints": [949], "characters": "\u03B5" }, + "ϵ": { "codepoints": [1013], "characters": "\u03F5" }, + "≖": { "codepoints": [8790], "characters": "\u2256" }, + "≕": { "codepoints": [8789], "characters": "\u2255" }, + "≂": { "codepoints": [8770], "characters": "\u2242" }, + "⪖": { "codepoints": [10902], "characters": "\u2A96" }, + "⪕": { "codepoints": [10901], "characters": "\u2A95" }, + "=": { "codepoints": [61], "characters": "\u003D" }, + "≟": { "codepoints": [8799], "characters": "\u225F" }, + "≡": { "codepoints": [8801], "characters": "\u2261" }, + "⩸": { "codepoints": [10872], "characters": "\u2A78" }, + "⧥": { "codepoints": [10725], "characters": "\u29E5" }, + "≓": { "codepoints": [8787], "characters": "\u2253" }, + "⥱": { "codepoints": [10609], "characters": "\u2971" }, + "ℯ": { "codepoints": [8495], "characters": "\u212F" }, + "≐": { "codepoints": [8784], "characters": "\u2250" }, + "≂": { "codepoints": [8770], "characters": "\u2242" }, + "η": { "codepoints": [951], "characters": "\u03B7" }, + "ð": { "codepoints": [240], "characters": "\u00F0" }, + "ð": { "codepoints": [240], "characters": "\u00F0" }, + "ë": { "codepoints": [235], "characters": "\u00EB" }, + "ë": { "codepoints": [235], "characters": "\u00EB" }, + "€": { "codepoints": [8364], "characters": "\u20AC" }, + "!": { "codepoints": [33], "characters": "\u0021" }, + "∃": { "codepoints": [8707], "characters": "\u2203" }, + "ℰ": { "codepoints": [8496], "characters": "\u2130" }, + "ⅇ": { "codepoints": [8519], "characters": "\u2147" }, + "≒": { "codepoints": [8786], "characters": "\u2252" }, + "ф": { "codepoints": [1092], "characters": "\u0444" }, + "♀": { "codepoints": [9792], "characters": "\u2640" }, + "ffi": { "codepoints": [64259], "characters": "\uFB03" }, + "ff": { "codepoints": [64256], "characters": "\uFB00" }, + "ffl": { "codepoints": [64260], "characters": "\uFB04" }, + "𝔣": { "codepoints": [120099], "characters": "\uD835\uDD23" }, + "fi": { "codepoints": [64257], "characters": "\uFB01" }, + "fj": { "codepoints": [102, 106], "characters": "\u0066\u006A" }, + "♭": { "codepoints": [9837], "characters": "\u266D" }, + "fl": { "codepoints": [64258], "characters": "\uFB02" }, + "▱": { "codepoints": [9649], "characters": "\u25B1" }, + "ƒ": { "codepoints": [402], "characters": "\u0192" }, + "𝕗": { "codepoints": [120151], "characters": "\uD835\uDD57" }, + "∀": { "codepoints": [8704], "characters": "\u2200" }, + "⋔": { "codepoints": [8916], "characters": "\u22D4" }, + "⫙": { "codepoints": [10969], "characters": "\u2AD9" }, + "⨍": { "codepoints": [10765], "characters": "\u2A0D" }, + "½": { "codepoints": [189], "characters": "\u00BD" }, + "½": { "codepoints": [189], "characters": "\u00BD" }, + "⅓": { "codepoints": [8531], "characters": "\u2153" }, + "¼": { "codepoints": [188], "characters": "\u00BC" }, + "¼": { "codepoints": [188], "characters": "\u00BC" }, + "⅕": { "codepoints": [8533], "characters": "\u2155" }, + "⅙": { "codepoints": [8537], "characters": "\u2159" }, + "⅛": { "codepoints": [8539], "characters": "\u215B" }, + "⅔": { "codepoints": [8532], "characters": "\u2154" }, + "⅖": { "codepoints": [8534], "characters": "\u2156" }, + "¾": { "codepoints": [190], "characters": "\u00BE" }, + "¾": { "codepoints": [190], "characters": "\u00BE" }, + "⅗": { "codepoints": [8535], "characters": "\u2157" }, + "⅜": { "codepoints": [8540], "characters": "\u215C" }, + "⅘": { "codepoints": [8536], "characters": "\u2158" }, + "⅚": { "codepoints": [8538], "characters": "\u215A" }, + "⅝": { "codepoints": [8541], "characters": "\u215D" }, + "⅞": { "codepoints": [8542], "characters": "\u215E" }, + "⁄": { "codepoints": [8260], "characters": "\u2044" }, + "⌢": { "codepoints": [8994], "characters": "\u2322" }, + "𝒻": { "codepoints": [119995], "characters": "\uD835\uDCBB" }, + "≧": { "codepoints": [8807], "characters": "\u2267" }, + "⪌": { "codepoints": [10892], "characters": "\u2A8C" }, + "ǵ": { "codepoints": [501], "characters": "\u01F5" }, + "γ": { "codepoints": [947], "characters": "\u03B3" }, + "ϝ": { "codepoints": [989], "characters": "\u03DD" }, + "⪆": { "codepoints": [10886], "characters": "\u2A86" }, + "ğ": { "codepoints": [287], "characters": "\u011F" }, + "ĝ": { "codepoints": [285], "characters": "\u011D" }, + "г": { "codepoints": [1075], "characters": "\u0433" }, + "ġ": { "codepoints": [289], "characters": "\u0121" }, + "≥": { "codepoints": [8805], "characters": "\u2265" }, + "⋛": { "codepoints": [8923], "characters": "\u22DB" }, + "≥": { "codepoints": [8805], "characters": "\u2265" }, + "≧": { "codepoints": [8807], "characters": "\u2267" }, + "⩾": { "codepoints": [10878], "characters": "\u2A7E" }, + "⩾": { "codepoints": [10878], "characters": "\u2A7E" }, + "⪩": { "codepoints": [10921], "characters": "\u2AA9" }, + "⪀": { "codepoints": [10880], "characters": "\u2A80" }, + "⪂": { "codepoints": [10882], "characters": "\u2A82" }, + "⪄": { "codepoints": [10884], "characters": "\u2A84" }, + "⋛︀": { "codepoints": [8923, 65024], "characters": "\u22DB\uFE00" }, + "⪔": { "codepoints": [10900], "characters": "\u2A94" }, + "𝔤": { "codepoints": [120100], "characters": "\uD835\uDD24" }, + "≫": { "codepoints": [8811], "characters": "\u226B" }, + "⋙": { "codepoints": [8921], "characters": "\u22D9" }, + "ℷ": { "codepoints": [8503], "characters": "\u2137" }, + "ѓ": { "codepoints": [1107], "characters": "\u0453" }, + "≷": { "codepoints": [8823], "characters": "\u2277" }, + "⪒": { "codepoints": [10898], "characters": "\u2A92" }, + "⪥": { "codepoints": [10917], "characters": "\u2AA5" }, + "⪤": { "codepoints": [10916], "characters": "\u2AA4" }, + "≩": { "codepoints": [8809], "characters": "\u2269" }, + "⪊": { "codepoints": [10890], "characters": "\u2A8A" }, + "⪊": { "codepoints": [10890], "characters": "\u2A8A" }, + "⪈": { "codepoints": [10888], "characters": "\u2A88" }, + "⪈": { "codepoints": [10888], "characters": "\u2A88" }, + "≩": { "codepoints": [8809], "characters": "\u2269" }, + "⋧": { "codepoints": [8935], "characters": "\u22E7" }, + "𝕘": { "codepoints": [120152], "characters": "\uD835\uDD58" }, + "`": { "codepoints": [96], "characters": "\u0060" }, + "ℊ": { "codepoints": [8458], "characters": "\u210A" }, + "≳": { "codepoints": [8819], "characters": "\u2273" }, + "⪎": { "codepoints": [10894], "characters": "\u2A8E" }, + "⪐": { "codepoints": [10896], "characters": "\u2A90" }, + ">": { "codepoints": [62], "characters": "\u003E" }, + ">": { "codepoints": [62], "characters": "\u003E" }, + "⪧": { "codepoints": [10919], "characters": "\u2AA7" }, + "⩺": { "codepoints": [10874], "characters": "\u2A7A" }, + "⋗": { "codepoints": [8919], "characters": "\u22D7" }, + "⦕": { "codepoints": [10645], "characters": "\u2995" }, + "⩼": { "codepoints": [10876], "characters": "\u2A7C" }, + "⪆": { "codepoints": [10886], "characters": "\u2A86" }, + "⥸": { "codepoints": [10616], "characters": "\u2978" }, + "⋗": { "codepoints": [8919], "characters": "\u22D7" }, + "⋛": { "codepoints": [8923], "characters": "\u22DB" }, + "⪌": { "codepoints": [10892], "characters": "\u2A8C" }, + "≷": { "codepoints": [8823], "characters": "\u2277" }, + "≳": { "codepoints": [8819], "characters": "\u2273" }, + "≩︀": { "codepoints": [8809, 65024], "characters": "\u2269\uFE00" }, + "≩︀": { "codepoints": [8809, 65024], "characters": "\u2269\uFE00" }, + "⇔": { "codepoints": [8660], "characters": "\u21D4" }, + " ": { "codepoints": [8202], "characters": "\u200A" }, + "½": { "codepoints": [189], "characters": "\u00BD" }, + "ℋ": { "codepoints": [8459], "characters": "\u210B" }, + "ъ": { "codepoints": [1098], "characters": "\u044A" }, + "↔": { "codepoints": [8596], "characters": "\u2194" }, + "⥈": { "codepoints": [10568], "characters": "\u2948" }, + "↭": { "codepoints": [8621], "characters": "\u21AD" }, + "ℏ": { "codepoints": [8463], "characters": "\u210F" }, + "ĥ": { "codepoints": [293], "characters": "\u0125" }, + "♥": { "codepoints": [9829], "characters": "\u2665" }, + "♥": { "codepoints": [9829], "characters": "\u2665" }, + "…": { "codepoints": [8230], "characters": "\u2026" }, + "⊹": { "codepoints": [8889], "characters": "\u22B9" }, + "𝔥": { "codepoints": [120101], "characters": "\uD835\uDD25" }, + "⤥": { "codepoints": [10533], "characters": "\u2925" }, + "⤦": { "codepoints": [10534], "characters": "\u2926" }, + "⇿": { "codepoints": [8703], "characters": "\u21FF" }, + "∻": { "codepoints": [8763], "characters": "\u223B" }, + "↩": { "codepoints": [8617], "characters": "\u21A9" }, + "↪": { "codepoints": [8618], "characters": "\u21AA" }, + "𝕙": { "codepoints": [120153], "characters": "\uD835\uDD59" }, + "―": { "codepoints": [8213], "characters": "\u2015" }, + "𝒽": { "codepoints": [119997], "characters": "\uD835\uDCBD" }, + "ℏ": { "codepoints": [8463], "characters": "\u210F" }, + "ħ": { "codepoints": [295], "characters": "\u0127" }, + "⁃": { "codepoints": [8259], "characters": "\u2043" }, + "‐": { "codepoints": [8208], "characters": "\u2010" }, + "í": { "codepoints": [237], "characters": "\u00ED" }, + "í": { "codepoints": [237], "characters": "\u00ED" }, + "⁣": { "codepoints": [8291], "characters": "\u2063" }, + "î": { "codepoints": [238], "characters": "\u00EE" }, + "î": { "codepoints": [238], "characters": "\u00EE" }, + "и": { "codepoints": [1080], "characters": "\u0438" }, + "е": { "codepoints": [1077], "characters": "\u0435" }, + "¡": { "codepoints": [161], "characters": "\u00A1" }, + "¡": { "codepoints": [161], "characters": "\u00A1" }, + "⇔": { "codepoints": [8660], "characters": "\u21D4" }, + "𝔦": { "codepoints": [120102], "characters": "\uD835\uDD26" }, + "ì": { "codepoints": [236], "characters": "\u00EC" }, + "ì": { "codepoints": [236], "characters": "\u00EC" }, + "ⅈ": { "codepoints": [8520], "characters": "\u2148" }, + "⨌": { "codepoints": [10764], "characters": "\u2A0C" }, + "∭": { "codepoints": [8749], "characters": "\u222D" }, + "⧜": { "codepoints": [10716], "characters": "\u29DC" }, + "℩": { "codepoints": [8489], "characters": "\u2129" }, + "ij": { "codepoints": [307], "characters": "\u0133" }, + "ī": { "codepoints": [299], "characters": "\u012B" }, + "ℑ": { "codepoints": [8465], "characters": "\u2111" }, + "ℐ": { "codepoints": [8464], "characters": "\u2110" }, + "ℑ": { "codepoints": [8465], "characters": "\u2111" }, + "ı": { "codepoints": [305], "characters": "\u0131" }, + "⊷": { "codepoints": [8887], "characters": "\u22B7" }, + "Ƶ": { "codepoints": [437], "characters": "\u01B5" }, + "∈": { "codepoints": [8712], "characters": "\u2208" }, + "℅": { "codepoints": [8453], "characters": "\u2105" }, + "∞": { "codepoints": [8734], "characters": "\u221E" }, + "⧝": { "codepoints": [10717], "characters": "\u29DD" }, + "ı": { "codepoints": [305], "characters": "\u0131" }, + "∫": { "codepoints": [8747], "characters": "\u222B" }, + "⊺": { "codepoints": [8890], "characters": "\u22BA" }, + "ℤ": { "codepoints": [8484], "characters": "\u2124" }, + "⊺": { "codepoints": [8890], "characters": "\u22BA" }, + "⨗": { "codepoints": [10775], "characters": "\u2A17" }, + "⨼": { "codepoints": [10812], "characters": "\u2A3C" }, + "ё": { "codepoints": [1105], "characters": "\u0451" }, + "į": { "codepoints": [303], "characters": "\u012F" }, + "𝕚": { "codepoints": [120154], "characters": "\uD835\uDD5A" }, + "ι": { "codepoints": [953], "characters": "\u03B9" }, + "⨼": { "codepoints": [10812], "characters": "\u2A3C" }, + "¿": { "codepoints": [191], "characters": "\u00BF" }, + "¿": { "codepoints": [191], "characters": "\u00BF" }, + "𝒾": { "codepoints": [119998], "characters": "\uD835\uDCBE" }, + "∈": { "codepoints": [8712], "characters": "\u2208" }, + "⋹": { "codepoints": [8953], "characters": "\u22F9" }, + "⋵": { "codepoints": [8949], "characters": "\u22F5" }, + "⋴": { "codepoints": [8948], "characters": "\u22F4" }, + "⋳": { "codepoints": [8947], "characters": "\u22F3" }, + "∈": { "codepoints": [8712], "characters": "\u2208" }, + "⁢": { "codepoints": [8290], "characters": "\u2062" }, + "ĩ": { "codepoints": [297], "characters": "\u0129" }, + "і": { "codepoints": [1110], "characters": "\u0456" }, + "ï": { "codepoints": [239], "characters": "\u00EF" }, + "ï": { "codepoints": [239], "characters": "\u00EF" }, + "ĵ": { "codepoints": [309], "characters": "\u0135" }, + "й": { "codepoints": [1081], "characters": "\u0439" }, + "𝔧": { "codepoints": [120103], "characters": "\uD835\uDD27" }, + "ȷ": { "codepoints": [567], "characters": "\u0237" }, + "𝕛": { "codepoints": [120155], "characters": "\uD835\uDD5B" }, + "𝒿": { "codepoints": [119999], "characters": "\uD835\uDCBF" }, + "ј": { "codepoints": [1112], "characters": "\u0458" }, + "є": { "codepoints": [1108], "characters": "\u0454" }, + "κ": { "codepoints": [954], "characters": "\u03BA" }, + "ϰ": { "codepoints": [1008], "characters": "\u03F0" }, + "ķ": { "codepoints": [311], "characters": "\u0137" }, + "к": { "codepoints": [1082], "characters": "\u043A" }, + "𝔨": { "codepoints": [120104], "characters": "\uD835\uDD28" }, + "ĸ": { "codepoints": [312], "characters": "\u0138" }, + "х": { "codepoints": [1093], "characters": "\u0445" }, + "ќ": { "codepoints": [1116], "characters": "\u045C" }, + "𝕜": { "codepoints": [120156], "characters": "\uD835\uDD5C" }, + "𝓀": { "codepoints": [120000], "characters": "\uD835\uDCC0" }, + "⇚": { "codepoints": [8666], "characters": "\u21DA" }, + "⇐": { "codepoints": [8656], "characters": "\u21D0" }, + "⤛": { "codepoints": [10523], "characters": "\u291B" }, + "⤎": { "codepoints": [10510], "characters": "\u290E" }, + "≦": { "codepoints": [8806], "characters": "\u2266" }, + "⪋": { "codepoints": [10891], "characters": "\u2A8B" }, + "⥢": { "codepoints": [10594], "characters": "\u2962" }, + "ĺ": { "codepoints": [314], "characters": "\u013A" }, + "⦴": { "codepoints": [10676], "characters": "\u29B4" }, + "ℒ": { "codepoints": [8466], "characters": "\u2112" }, + "λ": { "codepoints": [955], "characters": "\u03BB" }, + "⟨": { "codepoints": [10216], "characters": "\u27E8" }, + "⦑": { "codepoints": [10641], "characters": "\u2991" }, + "⟨": { "codepoints": [10216], "characters": "\u27E8" }, + "⪅": { "codepoints": [10885], "characters": "\u2A85" }, + "«": { "codepoints": [171], "characters": "\u00AB" }, + "«": { "codepoints": [171], "characters": "\u00AB" }, + "←": { "codepoints": [8592], "characters": "\u2190" }, + "⇤": { "codepoints": [8676], "characters": "\u21E4" }, + "⤟": { "codepoints": [10527], "characters": "\u291F" }, + "⤝": { "codepoints": [10525], "characters": "\u291D" }, + "↩": { "codepoints": [8617], "characters": "\u21A9" }, + "↫": { "codepoints": [8619], "characters": "\u21AB" }, + "⤹": { "codepoints": [10553], "characters": "\u2939" }, + "⥳": { "codepoints": [10611], "characters": "\u2973" }, + "↢": { "codepoints": [8610], "characters": "\u21A2" }, + "⪫": { "codepoints": [10923], "characters": "\u2AAB" }, + "⤙": { "codepoints": [10521], "characters": "\u2919" }, + "⪭": { "codepoints": [10925], "characters": "\u2AAD" }, + "⪭︀": { "codepoints": [10925, 65024], "characters": "\u2AAD\uFE00" }, + "⤌": { "codepoints": [10508], "characters": "\u290C" }, + "❲": { "codepoints": [10098], "characters": "\u2772" }, + "{": { "codepoints": [123], "characters": "\u007B" }, + "[": { "codepoints": [91], "characters": "\u005B" }, + "⦋": { "codepoints": [10635], "characters": "\u298B" }, + "⦏": { "codepoints": [10639], "characters": "\u298F" }, + "⦍": { "codepoints": [10637], "characters": "\u298D" }, + "ľ": { "codepoints": [318], "characters": "\u013E" }, + "ļ": { "codepoints": [316], "characters": "\u013C" }, + "⌈": { "codepoints": [8968], "characters": "\u2308" }, + "{": { "codepoints": [123], "characters": "\u007B" }, + "л": { "codepoints": [1083], "characters": "\u043B" }, + "⤶": { "codepoints": [10550], "characters": "\u2936" }, + "“": { "codepoints": [8220], "characters": "\u201C" }, + "„": { "codepoints": [8222], "characters": "\u201E" }, + "⥧": { "codepoints": [10599], "characters": "\u2967" }, + "⥋": { "codepoints": [10571], "characters": "\u294B" }, + "↲": { "codepoints": [8626], "characters": "\u21B2" }, + "≤": { "codepoints": [8804], "characters": "\u2264" }, + "←": { "codepoints": [8592], "characters": "\u2190" }, + "↢": { "codepoints": [8610], "characters": "\u21A2" }, + "↽": { "codepoints": [8637], "characters": "\u21BD" }, + "↼": { "codepoints": [8636], "characters": "\u21BC" }, + "⇇": { "codepoints": [8647], "characters": "\u21C7" }, + "↔": { "codepoints": [8596], "characters": "\u2194" }, + "⇆": { "codepoints": [8646], "characters": "\u21C6" }, + "⇋": { "codepoints": [8651], "characters": "\u21CB" }, + "↭": { "codepoints": [8621], "characters": "\u21AD" }, + "⋋": { "codepoints": [8907], "characters": "\u22CB" }, + "⋚": { "codepoints": [8922], "characters": "\u22DA" }, + "≤": { "codepoints": [8804], "characters": "\u2264" }, + "≦": { "codepoints": [8806], "characters": "\u2266" }, + "⩽": { "codepoints": [10877], "characters": "\u2A7D" }, + "⩽": { "codepoints": [10877], "characters": "\u2A7D" }, + "⪨": { "codepoints": [10920], "characters": "\u2AA8" }, + "⩿": { "codepoints": [10879], "characters": "\u2A7F" }, + "⪁": { "codepoints": [10881], "characters": "\u2A81" }, + "⪃": { "codepoints": [10883], "characters": "\u2A83" }, + "⋚︀": { "codepoints": [8922, 65024], "characters": "\u22DA\uFE00" }, + "⪓": { "codepoints": [10899], "characters": "\u2A93" }, + "⪅": { "codepoints": [10885], "characters": "\u2A85" }, + "⋖": { "codepoints": [8918], "characters": "\u22D6" }, + "⋚": { "codepoints": [8922], "characters": "\u22DA" }, + "⪋": { "codepoints": [10891], "characters": "\u2A8B" }, + "≶": { "codepoints": [8822], "characters": "\u2276" }, + "≲": { "codepoints": [8818], "characters": "\u2272" }, + "⥼": { "codepoints": [10620], "characters": "\u297C" }, + "⌊": { "codepoints": [8970], "characters": "\u230A" }, + "𝔩": { "codepoints": [120105], "characters": "\uD835\uDD29" }, + "≶": { "codepoints": [8822], "characters": "\u2276" }, + "⪑": { "codepoints": [10897], "characters": "\u2A91" }, + "↽": { "codepoints": [8637], "characters": "\u21BD" }, + "↼": { "codepoints": [8636], "characters": "\u21BC" }, + "⥪": { "codepoints": [10602], "characters": "\u296A" }, + "▄": { "codepoints": [9604], "characters": "\u2584" }, + "љ": { "codepoints": [1113], "characters": "\u0459" }, + "≪": { "codepoints": [8810], "characters": "\u226A" }, + "⇇": { "codepoints": [8647], "characters": "\u21C7" }, + "⌞": { "codepoints": [8990], "characters": "\u231E" }, + "⥫": { "codepoints": [10603], "characters": "\u296B" }, + "◺": { "codepoints": [9722], "characters": "\u25FA" }, + "ŀ": { "codepoints": [320], "characters": "\u0140" }, + "⎰": { "codepoints": [9136], "characters": "\u23B0" }, + "⎰": { "codepoints": [9136], "characters": "\u23B0" }, + "≨": { "codepoints": [8808], "characters": "\u2268" }, + "⪉": { "codepoints": [10889], "characters": "\u2A89" }, + "⪉": { "codepoints": [10889], "characters": "\u2A89" }, + "⪇": { "codepoints": [10887], "characters": "\u2A87" }, + "⪇": { "codepoints": [10887], "characters": "\u2A87" }, + "≨": { "codepoints": [8808], "characters": "\u2268" }, + "⋦": { "codepoints": [8934], "characters": "\u22E6" }, + "⟬": { "codepoints": [10220], "characters": "\u27EC" }, + "⇽": { "codepoints": [8701], "characters": "\u21FD" }, + "⟦": { "codepoints": [10214], "characters": "\u27E6" }, + "⟵": { "codepoints": [10229], "characters": "\u27F5" }, + "⟷": { "codepoints": [10231], "characters": "\u27F7" }, + "⟼": { "codepoints": [10236], "characters": "\u27FC" }, + "⟶": { "codepoints": [10230], "characters": "\u27F6" }, + "↫": { "codepoints": [8619], "characters": "\u21AB" }, + "↬": { "codepoints": [8620], "characters": "\u21AC" }, + "⦅": { "codepoints": [10629], "characters": "\u2985" }, + "𝕝": { "codepoints": [120157], "characters": "\uD835\uDD5D" }, + "⨭": { "codepoints": [10797], "characters": "\u2A2D" }, + "⨴": { "codepoints": [10804], "characters": "\u2A34" }, + "∗": { "codepoints": [8727], "characters": "\u2217" }, + "_": { "codepoints": [95], "characters": "\u005F" }, + "◊": { "codepoints": [9674], "characters": "\u25CA" }, + "◊": { "codepoints": [9674], "characters": "\u25CA" }, + "⧫": { "codepoints": [10731], "characters": "\u29EB" }, + "(": { "codepoints": [40], "characters": "\u0028" }, + "⦓": { "codepoints": [10643], "characters": "\u2993" }, + "⇆": { "codepoints": [8646], "characters": "\u21C6" }, + "⌟": { "codepoints": [8991], "characters": "\u231F" }, + "⇋": { "codepoints": [8651], "characters": "\u21CB" }, + "⥭": { "codepoints": [10605], "characters": "\u296D" }, + "‎": { "codepoints": [8206], "characters": "\u200E" }, + "⊿": { "codepoints": [8895], "characters": "\u22BF" }, + "‹": { "codepoints": [8249], "characters": "\u2039" }, + "𝓁": { "codepoints": [120001], "characters": "\uD835\uDCC1" }, + "↰": { "codepoints": [8624], "characters": "\u21B0" }, + "≲": { "codepoints": [8818], "characters": "\u2272" }, + "⪍": { "codepoints": [10893], "characters": "\u2A8D" }, + "⪏": { "codepoints": [10895], "characters": "\u2A8F" }, + "[": { "codepoints": [91], "characters": "\u005B" }, + "‘": { "codepoints": [8216], "characters": "\u2018" }, + "‚": { "codepoints": [8218], "characters": "\u201A" }, + "ł": { "codepoints": [322], "characters": "\u0142" }, + "<": { "codepoints": [60], "characters": "\u003C" }, + "<": { "codepoints": [60], "characters": "\u003C" }, + "⪦": { "codepoints": [10918], "characters": "\u2AA6" }, + "⩹": { "codepoints": [10873], "characters": "\u2A79" }, + "⋖": { "codepoints": [8918], "characters": "\u22D6" }, + "⋋": { "codepoints": [8907], "characters": "\u22CB" }, + "⋉": { "codepoints": [8905], "characters": "\u22C9" }, + "⥶": { "codepoints": [10614], "characters": "\u2976" }, + "⩻": { "codepoints": [10875], "characters": "\u2A7B" }, + "⦖": { "codepoints": [10646], "characters": "\u2996" }, + "◃": { "codepoints": [9667], "characters": "\u25C3" }, + "⊴": { "codepoints": [8884], "characters": "\u22B4" }, + "◂": { "codepoints": [9666], "characters": "\u25C2" }, + "⥊": { "codepoints": [10570], "characters": "\u294A" }, + "⥦": { "codepoints": [10598], "characters": "\u2966" }, + "≨︀": { "codepoints": [8808, 65024], "characters": "\u2268\uFE00" }, + "≨︀": { "codepoints": [8808, 65024], "characters": "\u2268\uFE00" }, + "∺": { "codepoints": [8762], "characters": "\u223A" }, + "¯": { "codepoints": [175], "characters": "\u00AF" }, + "¯": { "codepoints": [175], "characters": "\u00AF" }, + "♂": { "codepoints": [9794], "characters": "\u2642" }, + "✠": { "codepoints": [10016], "characters": "\u2720" }, + "✠": { "codepoints": [10016], "characters": "\u2720" }, + "↦": { "codepoints": [8614], "characters": "\u21A6" }, + "↦": { "codepoints": [8614], "characters": "\u21A6" }, + "↧": { "codepoints": [8615], "characters": "\u21A7" }, + "↤": { "codepoints": [8612], "characters": "\u21A4" }, + "↥": { "codepoints": [8613], "characters": "\u21A5" }, + "▮": { "codepoints": [9646], "characters": "\u25AE" }, + "⨩": { "codepoints": [10793], "characters": "\u2A29" }, + "м": { "codepoints": [1084], "characters": "\u043C" }, + "—": { "codepoints": [8212], "characters": "\u2014" }, + "∡": { "codepoints": [8737], "characters": "\u2221" }, + "𝔪": { "codepoints": [120106], "characters": "\uD835\uDD2A" }, + "℧": { "codepoints": [8487], "characters": "\u2127" }, + "µ": { "codepoints": [181], "characters": "\u00B5" }, + "µ": { "codepoints": [181], "characters": "\u00B5" }, + "∣": { "codepoints": [8739], "characters": "\u2223" }, + "*": { "codepoints": [42], "characters": "\u002A" }, + "⫰": { "codepoints": [10992], "characters": "\u2AF0" }, + "·": { "codepoints": [183], "characters": "\u00B7" }, + "·": { "codepoints": [183], "characters": "\u00B7" }, + "−": { "codepoints": [8722], "characters": "\u2212" }, + "⊟": { "codepoints": [8863], "characters": "\u229F" }, + "∸": { "codepoints": [8760], "characters": "\u2238" }, + "⨪": { "codepoints": [10794], "characters": "\u2A2A" }, + "⫛": { "codepoints": [10971], "characters": "\u2ADB" }, + "…": { "codepoints": [8230], "characters": "\u2026" }, + "∓": { "codepoints": [8723], "characters": "\u2213" }, + "⊧": { "codepoints": [8871], "characters": "\u22A7" }, + "𝕞": { "codepoints": [120158], "characters": "\uD835\uDD5E" }, + "∓": { "codepoints": [8723], "characters": "\u2213" }, + "𝓂": { "codepoints": [120002], "characters": "\uD835\uDCC2" }, + "∾": { "codepoints": [8766], "characters": "\u223E" }, + "μ": { "codepoints": [956], "characters": "\u03BC" }, + "⊸": { "codepoints": [8888], "characters": "\u22B8" }, + "⊸": { "codepoints": [8888], "characters": "\u22B8" }, + "⋙̸": { "codepoints": [8921, 824], "characters": "\u22D9\u0338" }, + "≫⃒": { "codepoints": [8811, 8402], "characters": "\u226B\u20D2" }, + "≫̸": { "codepoints": [8811, 824], "characters": "\u226B\u0338" }, + "⇍": { "codepoints": [8653], "characters": "\u21CD" }, + "⇎": { "codepoints": [8654], "characters": "\u21CE" }, + "⋘̸": { "codepoints": [8920, 824], "characters": "\u22D8\u0338" }, + "≪⃒": { "codepoints": [8810, 8402], "characters": "\u226A\u20D2" }, + "≪̸": { "codepoints": [8810, 824], "characters": "\u226A\u0338" }, + "⇏": { "codepoints": [8655], "characters": "\u21CF" }, + "⊯": { "codepoints": [8879], "characters": "\u22AF" }, + "⊮": { "codepoints": [8878], "characters": "\u22AE" }, + "∇": { "codepoints": [8711], "characters": "\u2207" }, + "ń": { "codepoints": [324], "characters": "\u0144" }, + "∠⃒": { "codepoints": [8736, 8402], "characters": "\u2220\u20D2" }, + "≉": { "codepoints": [8777], "characters": "\u2249" }, + "⩰̸": { "codepoints": [10864, 824], "characters": "\u2A70\u0338" }, + "≋̸": { "codepoints": [8779, 824], "characters": "\u224B\u0338" }, + "ʼn": { "codepoints": [329], "characters": "\u0149" }, + "≉": { "codepoints": [8777], "characters": "\u2249" }, + "♮": { "codepoints": [9838], "characters": "\u266E" }, + "♮": { "codepoints": [9838], "characters": "\u266E" }, + "ℕ": { "codepoints": [8469], "characters": "\u2115" }, + " ": { "codepoints": [160], "characters": "\u00A0" }, + " ": { "codepoints": [160], "characters": "\u00A0" }, + "≎̸": { "codepoints": [8782, 824], "characters": "\u224E\u0338" }, + "≏̸": { "codepoints": [8783, 824], "characters": "\u224F\u0338" }, + "⩃": { "codepoints": [10819], "characters": "\u2A43" }, + "ň": { "codepoints": [328], "characters": "\u0148" }, + "ņ": { "codepoints": [326], "characters": "\u0146" }, + "≇": { "codepoints": [8775], "characters": "\u2247" }, + "⩭̸": { "codepoints": [10861, 824], "characters": "\u2A6D\u0338" }, + "⩂": { "codepoints": [10818], "characters": "\u2A42" }, + "н": { "codepoints": [1085], "characters": "\u043D" }, + "–": { "codepoints": [8211], "characters": "\u2013" }, + "≠": { "codepoints": [8800], "characters": "\u2260" }, + "⇗": { "codepoints": [8663], "characters": "\u21D7" }, + "⤤": { "codepoints": [10532], "characters": "\u2924" }, + "↗": { "codepoints": [8599], "characters": "\u2197" }, + "↗": { "codepoints": [8599], "characters": "\u2197" }, + "≐̸": { "codepoints": [8784, 824], "characters": "\u2250\u0338" }, + "≢": { "codepoints": [8802], "characters": "\u2262" }, + "⤨": { "codepoints": [10536], "characters": "\u2928" }, + "≂̸": { "codepoints": [8770, 824], "characters": "\u2242\u0338" }, + "∄": { "codepoints": [8708], "characters": "\u2204" }, + "∄": { "codepoints": [8708], "characters": "\u2204" }, + "𝔫": { "codepoints": [120107], "characters": "\uD835\uDD2B" }, + "≧̸": { "codepoints": [8807, 824], "characters": "\u2267\u0338" }, + "≱": { "codepoints": [8817], "characters": "\u2271" }, + "≱": { "codepoints": [8817], "characters": "\u2271" }, + "≧̸": { "codepoints": [8807, 824], "characters": "\u2267\u0338" }, + "⩾̸": { "codepoints": [10878, 824], "characters": "\u2A7E\u0338" }, + "⩾̸": { "codepoints": [10878, 824], "characters": "\u2A7E\u0338" }, + "≵": { "codepoints": [8821], "characters": "\u2275" }, + "≯": { "codepoints": [8815], "characters": "\u226F" }, + "≯": { "codepoints": [8815], "characters": "\u226F" }, + "⇎": { "codepoints": [8654], "characters": "\u21CE" }, + "↮": { "codepoints": [8622], "characters": "\u21AE" }, + "⫲": { "codepoints": [10994], "characters": "\u2AF2" }, + "∋": { "codepoints": [8715], "characters": "\u220B" }, + "⋼": { "codepoints": [8956], "characters": "\u22FC" }, + "⋺": { "codepoints": [8954], "characters": "\u22FA" }, + "∋": { "codepoints": [8715], "characters": "\u220B" }, + "њ": { "codepoints": [1114], "characters": "\u045A" }, + "⇍": { "codepoints": [8653], "characters": "\u21CD" }, + "≦̸": { "codepoints": [8806, 824], "characters": "\u2266\u0338" }, + "↚": { "codepoints": [8602], "characters": "\u219A" }, + "‥": { "codepoints": [8229], "characters": "\u2025" }, + "≰": { "codepoints": [8816], "characters": "\u2270" }, + "↚": { "codepoints": [8602], "characters": "\u219A" }, + "↮": { "codepoints": [8622], "characters": "\u21AE" }, + "≰": { "codepoints": [8816], "characters": "\u2270" }, + "≦̸": { "codepoints": [8806, 824], "characters": "\u2266\u0338" }, + "⩽̸": { "codepoints": [10877, 824], "characters": "\u2A7D\u0338" }, + "⩽̸": { "codepoints": [10877, 824], "characters": "\u2A7D\u0338" }, + "≮": { "codepoints": [8814], "characters": "\u226E" }, + "≴": { "codepoints": [8820], "characters": "\u2274" }, + "≮": { "codepoints": [8814], "characters": "\u226E" }, + "⋪": { "codepoints": [8938], "characters": "\u22EA" }, + "⋬": { "codepoints": [8940], "characters": "\u22EC" }, + "∤": { "codepoints": [8740], "characters": "\u2224" }, + "𝕟": { "codepoints": [120159], "characters": "\uD835\uDD5F" }, + "¬": { "codepoints": [172], "characters": "\u00AC" }, + "¬": { "codepoints": [172], "characters": "\u00AC" }, + "∉": { "codepoints": [8713], "characters": "\u2209" }, + "⋹̸": { "codepoints": [8953, 824], "characters": "\u22F9\u0338" }, + "⋵̸": { "codepoints": [8949, 824], "characters": "\u22F5\u0338" }, + "∉": { "codepoints": [8713], "characters": "\u2209" }, + "⋷": { "codepoints": [8951], "characters": "\u22F7" }, + "⋶": { "codepoints": [8950], "characters": "\u22F6" }, + "∌": { "codepoints": [8716], "characters": "\u220C" }, + "∌": { "codepoints": [8716], "characters": "\u220C" }, + "⋾": { "codepoints": [8958], "characters": "\u22FE" }, + "⋽": { "codepoints": [8957], "characters": "\u22FD" }, + "∦": { "codepoints": [8742], "characters": "\u2226" }, + "∦": { "codepoints": [8742], "characters": "\u2226" }, + "⫽⃥": { "codepoints": [11005, 8421], "characters": "\u2AFD\u20E5" }, + "∂̸": { "codepoints": [8706, 824], "characters": "\u2202\u0338" }, + "⨔": { "codepoints": [10772], "characters": "\u2A14" }, + "⊀": { "codepoints": [8832], "characters": "\u2280" }, + "⋠": { "codepoints": [8928], "characters": "\u22E0" }, + "⪯̸": { "codepoints": [10927, 824], "characters": "\u2AAF\u0338" }, + "⊀": { "codepoints": [8832], "characters": "\u2280" }, + "⪯̸": { "codepoints": [10927, 824], "characters": "\u2AAF\u0338" }, + "⇏": { "codepoints": [8655], "characters": "\u21CF" }, + "↛": { "codepoints": [8603], "characters": "\u219B" }, + "⤳̸": { "codepoints": [10547, 824], "characters": "\u2933\u0338" }, + "↝̸": { "codepoints": [8605, 824], "characters": "\u219D\u0338" }, + "↛": { "codepoints": [8603], "characters": "\u219B" }, + "⋫": { "codepoints": [8939], "characters": "\u22EB" }, + "⋭": { "codepoints": [8941], "characters": "\u22ED" }, + "⊁": { "codepoints": [8833], "characters": "\u2281" }, + "⋡": { "codepoints": [8929], "characters": "\u22E1" }, + "⪰̸": { "codepoints": [10928, 824], "characters": "\u2AB0\u0338" }, + "𝓃": { "codepoints": [120003], "characters": "\uD835\uDCC3" }, + "∤": { "codepoints": [8740], "characters": "\u2224" }, + "∦": { "codepoints": [8742], "characters": "\u2226" }, + "≁": { "codepoints": [8769], "characters": "\u2241" }, + "≄": { "codepoints": [8772], "characters": "\u2244" }, + "≄": { "codepoints": [8772], "characters": "\u2244" }, + "∤": { "codepoints": [8740], "characters": "\u2224" }, + "∦": { "codepoints": [8742], "characters": "\u2226" }, + "⋢": { "codepoints": [8930], "characters": "\u22E2" }, + "⋣": { "codepoints": [8931], "characters": "\u22E3" }, + "⊄": { "codepoints": [8836], "characters": "\u2284" }, + "⫅̸": { "codepoints": [10949, 824], "characters": "\u2AC5\u0338" }, + "⊈": { "codepoints": [8840], "characters": "\u2288" }, + "⊂⃒": { "codepoints": [8834, 8402], "characters": "\u2282\u20D2" }, + "⊈": { "codepoints": [8840], "characters": "\u2288" }, + "⫅̸": { "codepoints": [10949, 824], "characters": "\u2AC5\u0338" }, + "⊁": { "codepoints": [8833], "characters": "\u2281" }, + "⪰̸": { "codepoints": [10928, 824], "characters": "\u2AB0\u0338" }, + "⊅": { "codepoints": [8837], "characters": "\u2285" }, + "⫆̸": { "codepoints": [10950, 824], "characters": "\u2AC6\u0338" }, + "⊉": { "codepoints": [8841], "characters": "\u2289" }, + "⊃⃒": { "codepoints": [8835, 8402], "characters": "\u2283\u20D2" }, + "⊉": { "codepoints": [8841], "characters": "\u2289" }, + "⫆̸": { "codepoints": [10950, 824], "characters": "\u2AC6\u0338" }, + "≹": { "codepoints": [8825], "characters": "\u2279" }, + "ñ": { "codepoints": [241], "characters": "\u00F1" }, + "ñ": { "codepoints": [241], "characters": "\u00F1" }, + "≸": { "codepoints": [8824], "characters": "\u2278" }, + "⋪": { "codepoints": [8938], "characters": "\u22EA" }, + "⋬": { "codepoints": [8940], "characters": "\u22EC" }, + "⋫": { "codepoints": [8939], "characters": "\u22EB" }, + "⋭": { "codepoints": [8941], "characters": "\u22ED" }, + "ν": { "codepoints": [957], "characters": "\u03BD" }, + "#": { "codepoints": [35], "characters": "\u0023" }, + "№": { "codepoints": [8470], "characters": "\u2116" }, + " ": { "codepoints": [8199], "characters": "\u2007" }, + "⊭": { "codepoints": [8877], "characters": "\u22AD" }, + "⤄": { "codepoints": [10500], "characters": "\u2904" }, + "≍⃒": { "codepoints": [8781, 8402], "characters": "\u224D\u20D2" }, + "⊬": { "codepoints": [8876], "characters": "\u22AC" }, + "≥⃒": { "codepoints": [8805, 8402], "characters": "\u2265\u20D2" }, + ">⃒": { "codepoints": [62, 8402], "characters": "\u003E\u20D2" }, + "⧞": { "codepoints": [10718], "characters": "\u29DE" }, + "⤂": { "codepoints": [10498], "characters": "\u2902" }, + "≤⃒": { "codepoints": [8804, 8402], "characters": "\u2264\u20D2" }, + "<⃒": { "codepoints": [60, 8402], "characters": "\u003C\u20D2" }, + "⊴⃒": { "codepoints": [8884, 8402], "characters": "\u22B4\u20D2" }, + "⤃": { "codepoints": [10499], "characters": "\u2903" }, + "⊵⃒": { "codepoints": [8885, 8402], "characters": "\u22B5\u20D2" }, + "∼⃒": { "codepoints": [8764, 8402], "characters": "\u223C\u20D2" }, + "⇖": { "codepoints": [8662], "characters": "\u21D6" }, + "⤣": { "codepoints": [10531], "characters": "\u2923" }, + "↖": { "codepoints": [8598], "characters": "\u2196" }, + "↖": { "codepoints": [8598], "characters": "\u2196" }, + "⤧": { "codepoints": [10535], "characters": "\u2927" }, + "Ⓢ": { "codepoints": [9416], "characters": "\u24C8" }, + "ó": { "codepoints": [243], "characters": "\u00F3" }, + "ó": { "codepoints": [243], "characters": "\u00F3" }, + "⊛": { "codepoints": [8859], "characters": "\u229B" }, + "⊚": { "codepoints": [8858], "characters": "\u229A" }, + "ô": { "codepoints": [244], "characters": "\u00F4" }, + "ô": { "codepoints": [244], "characters": "\u00F4" }, + "о": { "codepoints": [1086], "characters": "\u043E" }, + "⊝": { "codepoints": [8861], "characters": "\u229D" }, + "ő": { "codepoints": [337], "characters": "\u0151" }, + "⨸": { "codepoints": [10808], "characters": "\u2A38" }, + "⊙": { "codepoints": [8857], "characters": "\u2299" }, + "⦼": { "codepoints": [10684], "characters": "\u29BC" }, + "œ": { "codepoints": [339], "characters": "\u0153" }, + "⦿": { "codepoints": [10687], "characters": "\u29BF" }, + "𝔬": { "codepoints": [120108], "characters": "\uD835\uDD2C" }, + "˛": { "codepoints": [731], "characters": "\u02DB" }, + "ò": { "codepoints": [242], "characters": "\u00F2" }, + "ò": { "codepoints": [242], "characters": "\u00F2" }, + "⧁": { "codepoints": [10689], "characters": "\u29C1" }, + "⦵": { "codepoints": [10677], "characters": "\u29B5" }, + "Ω": { "codepoints": [937], "characters": "\u03A9" }, + "∮": { "codepoints": [8750], "characters": "\u222E" }, + "↺": { "codepoints": [8634], "characters": "\u21BA" }, + "⦾": { "codepoints": [10686], "characters": "\u29BE" }, + "⦻": { "codepoints": [10683], "characters": "\u29BB" }, + "‾": { "codepoints": [8254], "characters": "\u203E" }, + "⧀": { "codepoints": [10688], "characters": "\u29C0" }, + "ō": { "codepoints": [333], "characters": "\u014D" }, + "ω": { "codepoints": [969], "characters": "\u03C9" }, + "ο": { "codepoints": [959], "characters": "\u03BF" }, + "⦶": { "codepoints": [10678], "characters": "\u29B6" }, + "⊖": { "codepoints": [8854], "characters": "\u2296" }, + "𝕠": { "codepoints": [120160], "characters": "\uD835\uDD60" }, + "⦷": { "codepoints": [10679], "characters": "\u29B7" }, + "⦹": { "codepoints": [10681], "characters": "\u29B9" }, + "⊕": { "codepoints": [8853], "characters": "\u2295" }, + "∨": { "codepoints": [8744], "characters": "\u2228" }, + "↻": { "codepoints": [8635], "characters": "\u21BB" }, + "⩝": { "codepoints": [10845], "characters": "\u2A5D" }, + "ℴ": { "codepoints": [8500], "characters": "\u2134" }, + "ℴ": { "codepoints": [8500], "characters": "\u2134" }, + "ª": { "codepoints": [170], "characters": "\u00AA" }, + "ª": { "codepoints": [170], "characters": "\u00AA" }, + "º": { "codepoints": [186], "characters": "\u00BA" }, + "º": { "codepoints": [186], "characters": "\u00BA" }, + "⊶": { "codepoints": [8886], "characters": "\u22B6" }, + "⩖": { "codepoints": [10838], "characters": "\u2A56" }, + "⩗": { "codepoints": [10839], "characters": "\u2A57" }, + "⩛": { "codepoints": [10843], "characters": "\u2A5B" }, + "ℴ": { "codepoints": [8500], "characters": "\u2134" }, + "ø": { "codepoints": [248], "characters": "\u00F8" }, + "ø": { "codepoints": [248], "characters": "\u00F8" }, + "⊘": { "codepoints": [8856], "characters": "\u2298" }, + "õ": { "codepoints": [245], "characters": "\u00F5" }, + "õ": { "codepoints": [245], "characters": "\u00F5" }, + "⊗": { "codepoints": [8855], "characters": "\u2297" }, + "⨶": { "codepoints": [10806], "characters": "\u2A36" }, + "ö": { "codepoints": [246], "characters": "\u00F6" }, + "ö": { "codepoints": [246], "characters": "\u00F6" }, + "⌽": { "codepoints": [9021], "characters": "\u233D" }, + "∥": { "codepoints": [8741], "characters": "\u2225" }, + "¶": { "codepoints": [182], "characters": "\u00B6" }, + "¶": { "codepoints": [182], "characters": "\u00B6" }, + "∥": { "codepoints": [8741], "characters": "\u2225" }, + "⫳": { "codepoints": [10995], "characters": "\u2AF3" }, + "⫽": { "codepoints": [11005], "characters": "\u2AFD" }, + "∂": { "codepoints": [8706], "characters": "\u2202" }, + "п": { "codepoints": [1087], "characters": "\u043F" }, + "%": { "codepoints": [37], "characters": "\u0025" }, + ".": { "codepoints": [46], "characters": "\u002E" }, + "‰": { "codepoints": [8240], "characters": "\u2030" }, + "⊥": { "codepoints": [8869], "characters": "\u22A5" }, + "‱": { "codepoints": [8241], "characters": "\u2031" }, + "𝔭": { "codepoints": [120109], "characters": "\uD835\uDD2D" }, + "φ": { "codepoints": [966], "characters": "\u03C6" }, + "ϕ": { "codepoints": [981], "characters": "\u03D5" }, + "ℳ": { "codepoints": [8499], "characters": "\u2133" }, + "☎": { "codepoints": [9742], "characters": "\u260E" }, + "π": { "codepoints": [960], "characters": "\u03C0" }, + "⋔": { "codepoints": [8916], "characters": "\u22D4" }, + "ϖ": { "codepoints": [982], "characters": "\u03D6" }, + "ℏ": { "codepoints": [8463], "characters": "\u210F" }, + "ℎ": { "codepoints": [8462], "characters": "\u210E" }, + "ℏ": { "codepoints": [8463], "characters": "\u210F" }, + "+": { "codepoints": [43], "characters": "\u002B" }, + "⨣": { "codepoints": [10787], "characters": "\u2A23" }, + "⊞": { "codepoints": [8862], "characters": "\u229E" }, + "⨢": { "codepoints": [10786], "characters": "\u2A22" }, + "∔": { "codepoints": [8724], "characters": "\u2214" }, + "⨥": { "codepoints": [10789], "characters": "\u2A25" }, + "⩲": { "codepoints": [10866], "characters": "\u2A72" }, + "±": { "codepoints": [177], "characters": "\u00B1" }, + "±": { "codepoints": [177], "characters": "\u00B1" }, + "⨦": { "codepoints": [10790], "characters": "\u2A26" }, + "⨧": { "codepoints": [10791], "characters": "\u2A27" }, + "±": { "codepoints": [177], "characters": "\u00B1" }, + "⨕": { "codepoints": [10773], "characters": "\u2A15" }, + "𝕡": { "codepoints": [120161], "characters": "\uD835\uDD61" }, + "£": { "codepoints": [163], "characters": "\u00A3" }, + "£": { "codepoints": [163], "characters": "\u00A3" }, + "≺": { "codepoints": [8826], "characters": "\u227A" }, + "⪳": { "codepoints": [10931], "characters": "\u2AB3" }, + "⪷": { "codepoints": [10935], "characters": "\u2AB7" }, + "≼": { "codepoints": [8828], "characters": "\u227C" }, + "⪯": { "codepoints": [10927], "characters": "\u2AAF" }, + "≺": { "codepoints": [8826], "characters": "\u227A" }, + "⪷": { "codepoints": [10935], "characters": "\u2AB7" }, + "≼": { "codepoints": [8828], "characters": "\u227C" }, + "⪯": { "codepoints": [10927], "characters": "\u2AAF" }, + "⪹": { "codepoints": [10937], "characters": "\u2AB9" }, + "⪵": { "codepoints": [10933], "characters": "\u2AB5" }, + "⋨": { "codepoints": [8936], "characters": "\u22E8" }, + "≾": { "codepoints": [8830], "characters": "\u227E" }, + "′": { "codepoints": [8242], "characters": "\u2032" }, + "ℙ": { "codepoints": [8473], "characters": "\u2119" }, + "⪵": { "codepoints": [10933], "characters": "\u2AB5" }, + "⪹": { "codepoints": [10937], "characters": "\u2AB9" }, + "⋨": { "codepoints": [8936], "characters": "\u22E8" }, + "∏": { "codepoints": [8719], "characters": "\u220F" }, + "⌮": { "codepoints": [9006], "characters": "\u232E" }, + "⌒": { "codepoints": [8978], "characters": "\u2312" }, + "⌓": { "codepoints": [8979], "characters": "\u2313" }, + "∝": { "codepoints": [8733], "characters": "\u221D" }, + "∝": { "codepoints": [8733], "characters": "\u221D" }, + "≾": { "codepoints": [8830], "characters": "\u227E" }, + "⊰": { "codepoints": [8880], "characters": "\u22B0" }, + "𝓅": { "codepoints": [120005], "characters": "\uD835\uDCC5" }, + "ψ": { "codepoints": [968], "characters": "\u03C8" }, + " ": { "codepoints": [8200], "characters": "\u2008" }, + "𝔮": { "codepoints": [120110], "characters": "\uD835\uDD2E" }, + "⨌": { "codepoints": [10764], "characters": "\u2A0C" }, + "𝕢": { "codepoints": [120162], "characters": "\uD835\uDD62" }, + "⁗": { "codepoints": [8279], "characters": "\u2057" }, + "𝓆": { "codepoints": [120006], "characters": "\uD835\uDCC6" }, + "ℍ": { "codepoints": [8461], "characters": "\u210D" }, + "⨖": { "codepoints": [10774], "characters": "\u2A16" }, + "?": { "codepoints": [63], "characters": "\u003F" }, + "≟": { "codepoints": [8799], "characters": "\u225F" }, + """: { "codepoints": [34], "characters": "\u0022" }, + """: { "codepoints": [34], "characters": "\u0022" }, + "⇛": { "codepoints": [8667], "characters": "\u21DB" }, + "⇒": { "codepoints": [8658], "characters": "\u21D2" }, + "⤜": { "codepoints": [10524], "characters": "\u291C" }, + "⤏": { "codepoints": [10511], "characters": "\u290F" }, + "⥤": { "codepoints": [10596], "characters": "\u2964" }, + "∽̱": { "codepoints": [8765, 817], "characters": "\u223D\u0331" }, + "ŕ": { "codepoints": [341], "characters": "\u0155" }, + "√": { "codepoints": [8730], "characters": "\u221A" }, + "⦳": { "codepoints": [10675], "characters": "\u29B3" }, + "⟩": { "codepoints": [10217], "characters": "\u27E9" }, + "⦒": { "codepoints": [10642], "characters": "\u2992" }, + "⦥": { "codepoints": [10661], "characters": "\u29A5" }, + "⟩": { "codepoints": [10217], "characters": "\u27E9" }, + "»": { "codepoints": [187], "characters": "\u00BB" }, + "»": { "codepoints": [187], "characters": "\u00BB" }, + "→": { "codepoints": [8594], "characters": "\u2192" }, + "⥵": { "codepoints": [10613], "characters": "\u2975" }, + "⇥": { "codepoints": [8677], "characters": "\u21E5" }, + "⤠": { "codepoints": [10528], "characters": "\u2920" }, + "⤳": { "codepoints": [10547], "characters": "\u2933" }, + "⤞": { "codepoints": [10526], "characters": "\u291E" }, + "↪": { "codepoints": [8618], "characters": "\u21AA" }, + "↬": { "codepoints": [8620], "characters": "\u21AC" }, + "⥅": { "codepoints": [10565], "characters": "\u2945" }, + "⥴": { "codepoints": [10612], "characters": "\u2974" }, + "↣": { "codepoints": [8611], "characters": "\u21A3" }, + "↝": { "codepoints": [8605], "characters": "\u219D" }, + "⤚": { "codepoints": [10522], "characters": "\u291A" }, + "∶": { "codepoints": [8758], "characters": "\u2236" }, + "ℚ": { "codepoints": [8474], "characters": "\u211A" }, + "⤍": { "codepoints": [10509], "characters": "\u290D" }, + "❳": { "codepoints": [10099], "characters": "\u2773" }, + "}": { "codepoints": [125], "characters": "\u007D" }, + "]": { "codepoints": [93], "characters": "\u005D" }, + "⦌": { "codepoints": [10636], "characters": "\u298C" }, + "⦎": { "codepoints": [10638], "characters": "\u298E" }, + "⦐": { "codepoints": [10640], "characters": "\u2990" }, + "ř": { "codepoints": [345], "characters": "\u0159" }, + "ŗ": { "codepoints": [343], "characters": "\u0157" }, + "⌉": { "codepoints": [8969], "characters": "\u2309" }, + "}": { "codepoints": [125], "characters": "\u007D" }, + "р": { "codepoints": [1088], "characters": "\u0440" }, + "⤷": { "codepoints": [10551], "characters": "\u2937" }, + "⥩": { "codepoints": [10601], "characters": "\u2969" }, + "”": { "codepoints": [8221], "characters": "\u201D" }, + "”": { "codepoints": [8221], "characters": "\u201D" }, + "↳": { "codepoints": [8627], "characters": "\u21B3" }, + "ℜ": { "codepoints": [8476], "characters": "\u211C" }, + "ℛ": { "codepoints": [8475], "characters": "\u211B" }, + "ℜ": { "codepoints": [8476], "characters": "\u211C" }, + "ℝ": { "codepoints": [8477], "characters": "\u211D" }, + "▭": { "codepoints": [9645], "characters": "\u25AD" }, + "®": { "codepoints": [174], "characters": "\u00AE" }, + "®": { "codepoints": [174], "characters": "\u00AE" }, + "⥽": { "codepoints": [10621], "characters": "\u297D" }, + "⌋": { "codepoints": [8971], "characters": "\u230B" }, + "𝔯": { "codepoints": [120111], "characters": "\uD835\uDD2F" }, + "⇁": { "codepoints": [8641], "characters": "\u21C1" }, + "⇀": { "codepoints": [8640], "characters": "\u21C0" }, + "⥬": { "codepoints": [10604], "characters": "\u296C" }, + "ρ": { "codepoints": [961], "characters": "\u03C1" }, + "ϱ": { "codepoints": [1009], "characters": "\u03F1" }, + "→": { "codepoints": [8594], "characters": "\u2192" }, + "↣": { "codepoints": [8611], "characters": "\u21A3" }, + "⇁": { "codepoints": [8641], "characters": "\u21C1" }, + "⇀": { "codepoints": [8640], "characters": "\u21C0" }, + "⇄": { "codepoints": [8644], "characters": "\u21C4" }, + "⇌": { "codepoints": [8652], "characters": "\u21CC" }, + "⇉": { "codepoints": [8649], "characters": "\u21C9" }, + "↝": { "codepoints": [8605], "characters": "\u219D" }, + "⋌": { "codepoints": [8908], "characters": "\u22CC" }, + "˚": { "codepoints": [730], "characters": "\u02DA" }, + "≓": { "codepoints": [8787], "characters": "\u2253" }, + "⇄": { "codepoints": [8644], "characters": "\u21C4" }, + "⇌": { "codepoints": [8652], "characters": "\u21CC" }, + "‏": { "codepoints": [8207], "characters": "\u200F" }, + "⎱": { "codepoints": [9137], "characters": "\u23B1" }, + "⎱": { "codepoints": [9137], "characters": "\u23B1" }, + "⫮": { "codepoints": [10990], "characters": "\u2AEE" }, + "⟭": { "codepoints": [10221], "characters": "\u27ED" }, + "⇾": { "codepoints": [8702], "characters": "\u21FE" }, + "⟧": { "codepoints": [10215], "characters": "\u27E7" }, + "⦆": { "codepoints": [10630], "characters": "\u2986" }, + "𝕣": { "codepoints": [120163], "characters": "\uD835\uDD63" }, + "⨮": { "codepoints": [10798], "characters": "\u2A2E" }, + "⨵": { "codepoints": [10805], "characters": "\u2A35" }, + ")": { "codepoints": [41], "characters": "\u0029" }, + "⦔": { "codepoints": [10644], "characters": "\u2994" }, + "⨒": { "codepoints": [10770], "characters": "\u2A12" }, + "⇉": { "codepoints": [8649], "characters": "\u21C9" }, + "›": { "codepoints": [8250], "characters": "\u203A" }, + "𝓇": { "codepoints": [120007], "characters": "\uD835\uDCC7" }, + "↱": { "codepoints": [8625], "characters": "\u21B1" }, + "]": { "codepoints": [93], "characters": "\u005D" }, + "’": { "codepoints": [8217], "characters": "\u2019" }, + "’": { "codepoints": [8217], "characters": "\u2019" }, + "⋌": { "codepoints": [8908], "characters": "\u22CC" }, + "⋊": { "codepoints": [8906], "characters": "\u22CA" }, + "▹": { "codepoints": [9657], "characters": "\u25B9" }, + "⊵": { "codepoints": [8885], "characters": "\u22B5" }, + "▸": { "codepoints": [9656], "characters": "\u25B8" }, + "⧎": { "codepoints": [10702], "characters": "\u29CE" }, + "⥨": { "codepoints": [10600], "characters": "\u2968" }, + "℞": { "codepoints": [8478], "characters": "\u211E" }, + "ś": { "codepoints": [347], "characters": "\u015B" }, + "‚": { "codepoints": [8218], "characters": "\u201A" }, + "≻": { "codepoints": [8827], "characters": "\u227B" }, + "⪴": { "codepoints": [10932], "characters": "\u2AB4" }, + "⪸": { "codepoints": [10936], "characters": "\u2AB8" }, + "š": { "codepoints": [353], "characters": "\u0161" }, + "≽": { "codepoints": [8829], "characters": "\u227D" }, + "⪰": { "codepoints": [10928], "characters": "\u2AB0" }, + "ş": { "codepoints": [351], "characters": "\u015F" }, + "ŝ": { "codepoints": [349], "characters": "\u015D" }, + "⪶": { "codepoints": [10934], "characters": "\u2AB6" }, + "⪺": { "codepoints": [10938], "characters": "\u2ABA" }, + "⋩": { "codepoints": [8937], "characters": "\u22E9" }, + "⨓": { "codepoints": [10771], "characters": "\u2A13" }, + "≿": { "codepoints": [8831], "characters": "\u227F" }, + "с": { "codepoints": [1089], "characters": "\u0441" }, + "⋅": { "codepoints": [8901], "characters": "\u22C5" }, + "⊡": { "codepoints": [8865], "characters": "\u22A1" }, + "⩦": { "codepoints": [10854], "characters": "\u2A66" }, + "⇘": { "codepoints": [8664], "characters": "\u21D8" }, + "⤥": { "codepoints": [10533], "characters": "\u2925" }, + "↘": { "codepoints": [8600], "characters": "\u2198" }, + "↘": { "codepoints": [8600], "characters": "\u2198" }, + "§": { "codepoints": [167], "characters": "\u00A7" }, + "§": { "codepoints": [167], "characters": "\u00A7" }, + ";": { "codepoints": [59], "characters": "\u003B" }, + "⤩": { "codepoints": [10537], "characters": "\u2929" }, + "∖": { "codepoints": [8726], "characters": "\u2216" }, + "∖": { "codepoints": [8726], "characters": "\u2216" }, + "✶": { "codepoints": [10038], "characters": "\u2736" }, + "𝔰": { "codepoints": [120112], "characters": "\uD835\uDD30" }, + "⌢": { "codepoints": [8994], "characters": "\u2322" }, + "♯": { "codepoints": [9839], "characters": "\u266F" }, + "щ": { "codepoints": [1097], "characters": "\u0449" }, + "ш": { "codepoints": [1096], "characters": "\u0448" }, + "∣": { "codepoints": [8739], "characters": "\u2223" }, + "∥": { "codepoints": [8741], "characters": "\u2225" }, + "­": { "codepoints": [173], "characters": "\u00AD" }, + "­": { "codepoints": [173], "characters": "\u00AD" }, + "σ": { "codepoints": [963], "characters": "\u03C3" }, + "ς": { "codepoints": [962], "characters": "\u03C2" }, + "ς": { "codepoints": [962], "characters": "\u03C2" }, + "∼": { "codepoints": [8764], "characters": "\u223C" }, + "⩪": { "codepoints": [10858], "characters": "\u2A6A" }, + "≃": { "codepoints": [8771], "characters": "\u2243" }, + "≃": { "codepoints": [8771], "characters": "\u2243" }, + "⪞": { "codepoints": [10910], "characters": "\u2A9E" }, + "⪠": { "codepoints": [10912], "characters": "\u2AA0" }, + "⪝": { "codepoints": [10909], "characters": "\u2A9D" }, + "⪟": { "codepoints": [10911], "characters": "\u2A9F" }, + "≆": { "codepoints": [8774], "characters": "\u2246" }, + "⨤": { "codepoints": [10788], "characters": "\u2A24" }, + "⥲": { "codepoints": [10610], "characters": "\u2972" }, + "←": { "codepoints": [8592], "characters": "\u2190" }, + "∖": { "codepoints": [8726], "characters": "\u2216" }, + "⨳": { "codepoints": [10803], "characters": "\u2A33" }, + "⧤": { "codepoints": [10724], "characters": "\u29E4" }, + "∣": { "codepoints": [8739], "characters": "\u2223" }, + "⌣": { "codepoints": [8995], "characters": "\u2323" }, + "⪪": { "codepoints": [10922], "characters": "\u2AAA" }, + "⪬": { "codepoints": [10924], "characters": "\u2AAC" }, + "⪬︀": { "codepoints": [10924, 65024], "characters": "\u2AAC\uFE00" }, + "ь": { "codepoints": [1100], "characters": "\u044C" }, + "/": { "codepoints": [47], "characters": "\u002F" }, + "⧄": { "codepoints": [10692], "characters": "\u29C4" }, + "⌿": { "codepoints": [9023], "characters": "\u233F" }, + "𝕤": { "codepoints": [120164], "characters": "\uD835\uDD64" }, + "♠": { "codepoints": [9824], "characters": "\u2660" }, + "♠": { "codepoints": [9824], "characters": "\u2660" }, + "∥": { "codepoints": [8741], "characters": "\u2225" }, + "⊓": { "codepoints": [8851], "characters": "\u2293" }, + "⊓︀": { "codepoints": [8851, 65024], "characters": "\u2293\uFE00" }, + "⊔": { "codepoints": [8852], "characters": "\u2294" }, + "⊔︀": { "codepoints": [8852, 65024], "characters": "\u2294\uFE00" }, + "⊏": { "codepoints": [8847], "characters": "\u228F" }, + "⊑": { "codepoints": [8849], "characters": "\u2291" }, + "⊏": { "codepoints": [8847], "characters": "\u228F" }, + "⊑": { "codepoints": [8849], "characters": "\u2291" }, + "⊐": { "codepoints": [8848], "characters": "\u2290" }, + "⊒": { "codepoints": [8850], "characters": "\u2292" }, + "⊐": { "codepoints": [8848], "characters": "\u2290" }, + "⊒": { "codepoints": [8850], "characters": "\u2292" }, + "□": { "codepoints": [9633], "characters": "\u25A1" }, + "□": { "codepoints": [9633], "characters": "\u25A1" }, + "▪": { "codepoints": [9642], "characters": "\u25AA" }, + "▪": { "codepoints": [9642], "characters": "\u25AA" }, + "→": { "codepoints": [8594], "characters": "\u2192" }, + "𝓈": { "codepoints": [120008], "characters": "\uD835\uDCC8" }, + "∖": { "codepoints": [8726], "characters": "\u2216" }, + "⌣": { "codepoints": [8995], "characters": "\u2323" }, + "⋆": { "codepoints": [8902], "characters": "\u22C6" }, + "☆": { "codepoints": [9734], "characters": "\u2606" }, + "★": { "codepoints": [9733], "characters": "\u2605" }, + "ϵ": { "codepoints": [1013], "characters": "\u03F5" }, + "ϕ": { "codepoints": [981], "characters": "\u03D5" }, + "¯": { "codepoints": [175], "characters": "\u00AF" }, + "⊂": { "codepoints": [8834], "characters": "\u2282" }, + "⫅": { "codepoints": [10949], "characters": "\u2AC5" }, + "⪽": { "codepoints": [10941], "characters": "\u2ABD" }, + "⊆": { "codepoints": [8838], "characters": "\u2286" }, + "⫃": { "codepoints": [10947], "characters": "\u2AC3" }, + "⫁": { "codepoints": [10945], "characters": "\u2AC1" }, + "⫋": { "codepoints": [10955], "characters": "\u2ACB" }, + "⊊": { "codepoints": [8842], "characters": "\u228A" }, + "⪿": { "codepoints": [10943], "characters": "\u2ABF" }, + "⥹": { "codepoints": [10617], "characters": "\u2979" }, + "⊂": { "codepoints": [8834], "characters": "\u2282" }, + "⊆": { "codepoints": [8838], "characters": "\u2286" }, + "⫅": { "codepoints": [10949], "characters": "\u2AC5" }, + "⊊": { "codepoints": [8842], "characters": "\u228A" }, + "⫋": { "codepoints": [10955], "characters": "\u2ACB" }, + "⫇": { "codepoints": [10951], "characters": "\u2AC7" }, + "⫕": { "codepoints": [10965], "characters": "\u2AD5" }, + "⫓": { "codepoints": [10963], "characters": "\u2AD3" }, + "≻": { "codepoints": [8827], "characters": "\u227B" }, + "⪸": { "codepoints": [10936], "characters": "\u2AB8" }, + "≽": { "codepoints": [8829], "characters": "\u227D" }, + "⪰": { "codepoints": [10928], "characters": "\u2AB0" }, + "⪺": { "codepoints": [10938], "characters": "\u2ABA" }, + "⪶": { "codepoints": [10934], "characters": "\u2AB6" }, + "⋩": { "codepoints": [8937], "characters": "\u22E9" }, + "≿": { "codepoints": [8831], "characters": "\u227F" }, + "∑": { "codepoints": [8721], "characters": "\u2211" }, + "♪": { "codepoints": [9834], "characters": "\u266A" }, + "¹": { "codepoints": [185], "characters": "\u00B9" }, + "¹": { "codepoints": [185], "characters": "\u00B9" }, + "²": { "codepoints": [178], "characters": "\u00B2" }, + "²": { "codepoints": [178], "characters": "\u00B2" }, + "³": { "codepoints": [179], "characters": "\u00B3" }, + "³": { "codepoints": [179], "characters": "\u00B3" }, + "⊃": { "codepoints": [8835], "characters": "\u2283" }, + "⫆": { "codepoints": [10950], "characters": "\u2AC6" }, + "⪾": { "codepoints": [10942], "characters": "\u2ABE" }, + "⫘": { "codepoints": [10968], "characters": "\u2AD8" }, + "⊇": { "codepoints": [8839], "characters": "\u2287" }, + "⫄": { "codepoints": [10948], "characters": "\u2AC4" }, + "⟉": { "codepoints": [10185], "characters": "\u27C9" }, + "⫗": { "codepoints": [10967], "characters": "\u2AD7" }, + "⥻": { "codepoints": [10619], "characters": "\u297B" }, + "⫂": { "codepoints": [10946], "characters": "\u2AC2" }, + "⫌": { "codepoints": [10956], "characters": "\u2ACC" }, + "⊋": { "codepoints": [8843], "characters": "\u228B" }, + "⫀": { "codepoints": [10944], "characters": "\u2AC0" }, + "⊃": { "codepoints": [8835], "characters": "\u2283" }, + "⊇": { "codepoints": [8839], "characters": "\u2287" }, + "⫆": { "codepoints": [10950], "characters": "\u2AC6" }, + "⊋": { "codepoints": [8843], "characters": "\u228B" }, + "⫌": { "codepoints": [10956], "characters": "\u2ACC" }, + "⫈": { "codepoints": [10952], "characters": "\u2AC8" }, + "⫔": { "codepoints": [10964], "characters": "\u2AD4" }, + "⫖": { "codepoints": [10966], "characters": "\u2AD6" }, + "⇙": { "codepoints": [8665], "characters": "\u21D9" }, + "⤦": { "codepoints": [10534], "characters": "\u2926" }, + "↙": { "codepoints": [8601], "characters": "\u2199" }, + "↙": { "codepoints": [8601], "characters": "\u2199" }, + "⤪": { "codepoints": [10538], "characters": "\u292A" }, + "ß": { "codepoints": [223], "characters": "\u00DF" }, + "ß": { "codepoints": [223], "characters": "\u00DF" }, + "⌖": { "codepoints": [8982], "characters": "\u2316" }, + "τ": { "codepoints": [964], "characters": "\u03C4" }, + "⎴": { "codepoints": [9140], "characters": "\u23B4" }, + "ť": { "codepoints": [357], "characters": "\u0165" }, + "ţ": { "codepoints": [355], "characters": "\u0163" }, + "т": { "codepoints": [1090], "characters": "\u0442" }, + "⃛": { "codepoints": [8411], "characters": "\u20DB" }, + "⌕": { "codepoints": [8981], "characters": "\u2315" }, + "𝔱": { "codepoints": [120113], "characters": "\uD835\uDD31" }, + "∴": { "codepoints": [8756], "characters": "\u2234" }, + "∴": { "codepoints": [8756], "characters": "\u2234" }, + "θ": { "codepoints": [952], "characters": "\u03B8" }, + "ϑ": { "codepoints": [977], "characters": "\u03D1" }, + "ϑ": { "codepoints": [977], "characters": "\u03D1" }, + "≈": { "codepoints": [8776], "characters": "\u2248" }, + "∼": { "codepoints": [8764], "characters": "\u223C" }, + " ": { "codepoints": [8201], "characters": "\u2009" }, + "≈": { "codepoints": [8776], "characters": "\u2248" }, + "∼": { "codepoints": [8764], "characters": "\u223C" }, + "þ": { "codepoints": [254], "characters": "\u00FE" }, + "þ": { "codepoints": [254], "characters": "\u00FE" }, + "˜": { "codepoints": [732], "characters": "\u02DC" }, + "×": { "codepoints": [215], "characters": "\u00D7" }, + "×": { "codepoints": [215], "characters": "\u00D7" }, + "⊠": { "codepoints": [8864], "characters": "\u22A0" }, + "⨱": { "codepoints": [10801], "characters": "\u2A31" }, + "⨰": { "codepoints": [10800], "characters": "\u2A30" }, + "∭": { "codepoints": [8749], "characters": "\u222D" }, + "⤨": { "codepoints": [10536], "characters": "\u2928" }, + "⊤": { "codepoints": [8868], "characters": "\u22A4" }, + "⌶": { "codepoints": [9014], "characters": "\u2336" }, + "⫱": { "codepoints": [10993], "characters": "\u2AF1" }, + "𝕥": { "codepoints": [120165], "characters": "\uD835\uDD65" }, + "⫚": { "codepoints": [10970], "characters": "\u2ADA" }, + "⤩": { "codepoints": [10537], "characters": "\u2929" }, + "‴": { "codepoints": [8244], "characters": "\u2034" }, + "™": { "codepoints": [8482], "characters": "\u2122" }, + "▵": { "codepoints": [9653], "characters": "\u25B5" }, + "▿": { "codepoints": [9663], "characters": "\u25BF" }, + "◃": { "codepoints": [9667], "characters": "\u25C3" }, + "⊴": { "codepoints": [8884], "characters": "\u22B4" }, + "≜": { "codepoints": [8796], "characters": "\u225C" }, + "▹": { "codepoints": [9657], "characters": "\u25B9" }, + "⊵": { "codepoints": [8885], "characters": "\u22B5" }, + "◬": { "codepoints": [9708], "characters": "\u25EC" }, + "≜": { "codepoints": [8796], "characters": "\u225C" }, + "⨺": { "codepoints": [10810], "characters": "\u2A3A" }, + "⨹": { "codepoints": [10809], "characters": "\u2A39" }, + "⧍": { "codepoints": [10701], "characters": "\u29CD" }, + "⨻": { "codepoints": [10811], "characters": "\u2A3B" }, + "⏢": { "codepoints": [9186], "characters": "\u23E2" }, + "𝓉": { "codepoints": [120009], "characters": "\uD835\uDCC9" }, + "ц": { "codepoints": [1094], "characters": "\u0446" }, + "ћ": { "codepoints": [1115], "characters": "\u045B" }, + "ŧ": { "codepoints": [359], "characters": "\u0167" }, + "≬": { "codepoints": [8812], "characters": "\u226C" }, + "↞": { "codepoints": [8606], "characters": "\u219E" }, + "↠": { "codepoints": [8608], "characters": "\u21A0" }, + "⇑": { "codepoints": [8657], "characters": "\u21D1" }, + "⥣": { "codepoints": [10595], "characters": "\u2963" }, + "ú": { "codepoints": [250], "characters": "\u00FA" }, + "ú": { "codepoints": [250], "characters": "\u00FA" }, + "↑": { "codepoints": [8593], "characters": "\u2191" }, + "ў": { "codepoints": [1118], "characters": "\u045E" }, + "ŭ": { "codepoints": [365], "characters": "\u016D" }, + "û": { "codepoints": [251], "characters": "\u00FB" }, + "û": { "codepoints": [251], "characters": "\u00FB" }, + "у": { "codepoints": [1091], "characters": "\u0443" }, + "⇅": { "codepoints": [8645], "characters": "\u21C5" }, + "ű": { "codepoints": [369], "characters": "\u0171" }, + "⥮": { "codepoints": [10606], "characters": "\u296E" }, + "⥾": { "codepoints": [10622], "characters": "\u297E" }, + "𝔲": { "codepoints": [120114], "characters": "\uD835\uDD32" }, + "ù": { "codepoints": [249], "characters": "\u00F9" }, + "ù": { "codepoints": [249], "characters": "\u00F9" }, + "↿": { "codepoints": [8639], "characters": "\u21BF" }, + "↾": { "codepoints": [8638], "characters": "\u21BE" }, + "▀": { "codepoints": [9600], "characters": "\u2580" }, + "⌜": { "codepoints": [8988], "characters": "\u231C" }, + "⌜": { "codepoints": [8988], "characters": "\u231C" }, + "⌏": { "codepoints": [8975], "characters": "\u230F" }, + "◸": { "codepoints": [9720], "characters": "\u25F8" }, + "ū": { "codepoints": [363], "characters": "\u016B" }, + "¨": { "codepoints": [168], "characters": "\u00A8" }, + "¨": { "codepoints": [168], "characters": "\u00A8" }, + "ų": { "codepoints": [371], "characters": "\u0173" }, + "𝕦": { "codepoints": [120166], "characters": "\uD835\uDD66" }, + "↑": { "codepoints": [8593], "characters": "\u2191" }, + "↕": { "codepoints": [8597], "characters": "\u2195" }, + "↿": { "codepoints": [8639], "characters": "\u21BF" }, + "↾": { "codepoints": [8638], "characters": "\u21BE" }, + "⊎": { "codepoints": [8846], "characters": "\u228E" }, + "υ": { "codepoints": [965], "characters": "\u03C5" }, + "ϒ": { "codepoints": [978], "characters": "\u03D2" }, + "υ": { "codepoints": [965], "characters": "\u03C5" }, + "⇈": { "codepoints": [8648], "characters": "\u21C8" }, + "⌝": { "codepoints": [8989], "characters": "\u231D" }, + "⌝": { "codepoints": [8989], "characters": "\u231D" }, + "⌎": { "codepoints": [8974], "characters": "\u230E" }, + "ů": { "codepoints": [367], "characters": "\u016F" }, + "◹": { "codepoints": [9721], "characters": "\u25F9" }, + "𝓊": { "codepoints": [120010], "characters": "\uD835\uDCCA" }, + "⋰": { "codepoints": [8944], "characters": "\u22F0" }, + "ũ": { "codepoints": [361], "characters": "\u0169" }, + "▵": { "codepoints": [9653], "characters": "\u25B5" }, + "▴": { "codepoints": [9652], "characters": "\u25B4" }, + "⇈": { "codepoints": [8648], "characters": "\u21C8" }, + "ü": { "codepoints": [252], "characters": "\u00FC" }, + "ü": { "codepoints": [252], "characters": "\u00FC" }, + "⦧": { "codepoints": [10663], "characters": "\u29A7" }, + "⇕": { "codepoints": [8661], "characters": "\u21D5" }, + "⫨": { "codepoints": [10984], "characters": "\u2AE8" }, + "⫩": { "codepoints": [10985], "characters": "\u2AE9" }, + "⊨": { "codepoints": [8872], "characters": "\u22A8" }, + "⦜": { "codepoints": [10652], "characters": "\u299C" }, + "ϵ": { "codepoints": [1013], "characters": "\u03F5" }, + "ϰ": { "codepoints": [1008], "characters": "\u03F0" }, + "∅": { "codepoints": [8709], "characters": "\u2205" }, + "ϕ": { "codepoints": [981], "characters": "\u03D5" }, + "ϖ": { "codepoints": [982], "characters": "\u03D6" }, + "∝": { "codepoints": [8733], "characters": "\u221D" }, + "↕": { "codepoints": [8597], "characters": "\u2195" }, + "ϱ": { "codepoints": [1009], "characters": "\u03F1" }, + "ς": { "codepoints": [962], "characters": "\u03C2" }, + "⊊︀": { "codepoints": [8842, 65024], "characters": "\u228A\uFE00" }, + "⫋︀": { "codepoints": [10955, 65024], "characters": "\u2ACB\uFE00" }, + "⊋︀": { "codepoints": [8843, 65024], "characters": "\u228B\uFE00" }, + "⫌︀": { "codepoints": [10956, 65024], "characters": "\u2ACC\uFE00" }, + "ϑ": { "codepoints": [977], "characters": "\u03D1" }, + "⊲": { "codepoints": [8882], "characters": "\u22B2" }, + "⊳": { "codepoints": [8883], "characters": "\u22B3" }, + "в": { "codepoints": [1074], "characters": "\u0432" }, + "⊢": { "codepoints": [8866], "characters": "\u22A2" }, + "∨": { "codepoints": [8744], "characters": "\u2228" }, + "⊻": { "codepoints": [8891], "characters": "\u22BB" }, + "≚": { "codepoints": [8794], "characters": "\u225A" }, + "⋮": { "codepoints": [8942], "characters": "\u22EE" }, + "|": { "codepoints": [124], "characters": "\u007C" }, + "|": { "codepoints": [124], "characters": "\u007C" }, + "𝔳": { "codepoints": [120115], "characters": "\uD835\uDD33" }, + "⊲": { "codepoints": [8882], "characters": "\u22B2" }, + "⊂⃒": { "codepoints": [8834, 8402], "characters": "\u2282\u20D2" }, + "⊃⃒": { "codepoints": [8835, 8402], "characters": "\u2283\u20D2" }, + "𝕧": { "codepoints": [120167], "characters": "\uD835\uDD67" }, + "∝": { "codepoints": [8733], "characters": "\u221D" }, + "⊳": { "codepoints": [8883], "characters": "\u22B3" }, + "𝓋": { "codepoints": [120011], "characters": "\uD835\uDCCB" }, + "⫋︀": { "codepoints": [10955, 65024], "characters": "\u2ACB\uFE00" }, + "⊊︀": { "codepoints": [8842, 65024], "characters": "\u228A\uFE00" }, + "⫌︀": { "codepoints": [10956, 65024], "characters": "\u2ACC\uFE00" }, + "⊋︀": { "codepoints": [8843, 65024], "characters": "\u228B\uFE00" }, + "⦚": { "codepoints": [10650], "characters": "\u299A" }, + "ŵ": { "codepoints": [373], "characters": "\u0175" }, + "⩟": { "codepoints": [10847], "characters": "\u2A5F" }, + "∧": { "codepoints": [8743], "characters": "\u2227" }, + "≙": { "codepoints": [8793], "characters": "\u2259" }, + "℘": { "codepoints": [8472], "characters": "\u2118" }, + "𝔴": { "codepoints": [120116], "characters": "\uD835\uDD34" }, + "𝕨": { "codepoints": [120168], "characters": "\uD835\uDD68" }, + "℘": { "codepoints": [8472], "characters": "\u2118" }, + "≀": { "codepoints": [8768], "characters": "\u2240" }, + "≀": { "codepoints": [8768], "characters": "\u2240" }, + "𝓌": { "codepoints": [120012], "characters": "\uD835\uDCCC" }, + "⋂": { "codepoints": [8898], "characters": "\u22C2" }, + "◯": { "codepoints": [9711], "characters": "\u25EF" }, + "⋃": { "codepoints": [8899], "characters": "\u22C3" }, + "▽": { "codepoints": [9661], "characters": "\u25BD" }, + "𝔵": { "codepoints": [120117], "characters": "\uD835\uDD35" }, + "⟺": { "codepoints": [10234], "characters": "\u27FA" }, + "⟷": { "codepoints": [10231], "characters": "\u27F7" }, + "ξ": { "codepoints": [958], "characters": "\u03BE" }, + "⟸": { "codepoints": [10232], "characters": "\u27F8" }, + "⟵": { "codepoints": [10229], "characters": "\u27F5" }, + "⟼": { "codepoints": [10236], "characters": "\u27FC" }, + "⋻": { "codepoints": [8955], "characters": "\u22FB" }, + "⨀": { "codepoints": [10752], "characters": "\u2A00" }, + "𝕩": { "codepoints": [120169], "characters": "\uD835\uDD69" }, + "⨁": { "codepoints": [10753], "characters": "\u2A01" }, + "⨂": { "codepoints": [10754], "characters": "\u2A02" }, + "⟹": { "codepoints": [10233], "characters": "\u27F9" }, + "⟶": { "codepoints": [10230], "characters": "\u27F6" }, + "𝓍": { "codepoints": [120013], "characters": "\uD835\uDCCD" }, + "⨆": { "codepoints": [10758], "characters": "\u2A06" }, + "⨄": { "codepoints": [10756], "characters": "\u2A04" }, + "△": { "codepoints": [9651], "characters": "\u25B3" }, + "⋁": { "codepoints": [8897], "characters": "\u22C1" }, + "⋀": { "codepoints": [8896], "characters": "\u22C0" }, + "ý": { "codepoints": [253], "characters": "\u00FD" }, + "ý": { "codepoints": [253], "characters": "\u00FD" }, + "я": { "codepoints": [1103], "characters": "\u044F" }, + "ŷ": { "codepoints": [375], "characters": "\u0177" }, + "ы": { "codepoints": [1099], "characters": "\u044B" }, + "¥": { "codepoints": [165], "characters": "\u00A5" }, + "¥": { "codepoints": [165], "characters": "\u00A5" }, + "𝔶": { "codepoints": [120118], "characters": "\uD835\uDD36" }, + "ї": { "codepoints": [1111], "characters": "\u0457" }, + "𝕪": { "codepoints": [120170], "characters": "\uD835\uDD6A" }, + "𝓎": { "codepoints": [120014], "characters": "\uD835\uDCCE" }, + "ю": { "codepoints": [1102], "characters": "\u044E" }, + "ÿ": { "codepoints": [255], "characters": "\u00FF" }, + "ÿ": { "codepoints": [255], "characters": "\u00FF" }, + "ź": { "codepoints": [378], "characters": "\u017A" }, + "ž": { "codepoints": [382], "characters": "\u017E" }, + "з": { "codepoints": [1079], "characters": "\u0437" }, + "ż": { "codepoints": [380], "characters": "\u017C" }, + "ℨ": { "codepoints": [8488], "characters": "\u2128" }, + "ζ": { "codepoints": [950], "characters": "\u03B6" }, + "𝔷": { "codepoints": [120119], "characters": "\uD835\uDD37" }, + "ж": { "codepoints": [1078], "characters": "\u0436" }, + "⇝": { "codepoints": [8669], "characters": "\u21DD" }, + "𝕫": { "codepoints": [120171], "characters": "\uD835\uDD6B" }, + "𝓏": { "codepoints": [120015], "characters": "\uD835\uDCCF" }, + "‍": { "codepoints": [8205], "characters": "\u200D" }, + "‌": { "codepoints": [8204], "characters": "\u200C" } +} diff --git a/lib/chame0/chame/res/genentity.nim b/lib/chame0/chame/res/genentity.nim new file mode 100644 index 00000000..b25a8993 --- /dev/null +++ b/lib/chame0/chame/res/genentity.nim @@ -0,0 +1,49 @@ +import std/json +import std/streams +import std/strutils + +type LineWriter = object + s: Stream + line: string + +proc write(writer: var LineWriter, s: string) = + if s.len + writer.line.len > 80: + writer.s.writeLine(writer.line) + writer.line = "" + writer.line &= s + +proc flush(writer: var LineWriter) = + writer.s.writeLine(writer.line) + writer.line = "" + +proc main() = + let entityJson = parseJson(readFile("entity.json")) + echo "type Z = cstring" + var writer = LineWriter(s: newFileStream(stdout)) + var cc: char + var charMap: array[char, int] + for i in charMap.mitems: + i = -1 + var entityMap: seq[tuple[name, value: string]] + for k, v in entityJson: + entityMap.add((k.substr(1), v{"characters"}.getStr())) + let n = entityMap.len + echo "const entityMap*: array[" & $n & ", Z] = [" + var i = 0 + for (k, v) in entityMap: + if k[0] != cc: + charMap[cc] = i - 1 + cc = k[0] + writer.write((k & ":" & v).escape() & ",") + inc i + assert cc == 'z' + charMap[cc] = i - 1 + writer.flush() + echo "]" + echo "" + echo "const charMap*: array[char, int] = [" + for c in char.low..char.high: + writer.write($charMap[c] & ",") + writer.flush() + echo "]" +main() diff --git a/lib/chame0/chame/tags.nim b/lib/chame0/chame/tags.nim new file mode 100644 index 00000000..09df69f0 --- /dev/null +++ b/lib/chame0/chame/tags.nim @@ -0,0 +1,197 @@ +type + TagType* = enum + TAG_UNKNOWN = "" + TAG_APPLET = "applet" + TAG_BIG = "big" + TAG_HTML = "html" + TAG_BASE = "base" + TAG_BASEFONT = "basefont" + TAG_BGSOUND = "bgsound" + TAG_HEAD = "head" + TAG_LINK = "link" + TAG_LISTING = "listing" + TAG_META = "meta" + TAG_STYLE = "style" + TAG_TITLE = "title" + TAG_BODY = "body" + TAG_ADDRESS = "address" + TAG_ARTICLE = "article" + TAG_ASIDE = "aside" + TAG_FOOTER = "footer" + TAG_HEADER = "header" + TAG_H1 = "h1" + TAG_H2 = "h2" + TAG_H3 = "h3" + TAG_H4 = "h4" + TAG_H5 = "h5" + TAG_H6 = "h6" + TAG_HGROUP = "hgroup" + TAG_MAIN = "main" + TAG_NAV = "nav" + TAG_SEARCH = "search" + TAG_SECTION = "section" + TAG_BLOCKQUOTE = "blockquote" + TAG_DD = "dd" + TAG_DIV = "div" + TAG_DL = "dl" + TAG_DT = "dt" + TAG_FIGCAPTION = "figcaption" + TAG_FIGURE = "figure" + TAG_HR = "hr" + TAG_LI = "li" + TAG_OL = "ol" + TAG_P = "p" + TAG_PRE = "pre" + TAG_UL = "ul" + TAG_A = "a" + TAG_ABBR = "abbr" + TAG_B = "b" + TAG_BDI = "bdi" + TAG_BDO = "bdo" + TAG_BR = "br" + TAG_NOBR = "nobr" + TAG_CITE = "cite" + TAG_CODE = "code" + TAG_DATA = "data" + TAG_DFN = "dfn" + TAG_EM = "em" + TAG_EMBED = "embed" + TAG_I = "i" + TAG_KBD = "kbd" + TAG_MARK = "mark" + TAG_MARQUEE = "marquee" + TAG_Q = "q" + TAG_RB = "rb" + TAG_RP = "rp" + TAG_RT = "rt" + TAG_RTC = "rtc" + TAG_RUBY = "ruby" + TAG_S = "s" + TAG_SAMP = "samp" + TAG_SMALL = "small" + TAG_SPAN = "span" + TAG_STRONG = "strong" + TAG_SUB = "sub" + TAG_SUP = "sup" + TAG_TIME = "time" + TAG_U = "u" + TAG_VAR = "var" + TAG_WBR = "wbr" + TAG_AREA = "area" + TAG_AUDIO = "audio" + TAG_IMG = "img" + TAG_IMAGE = "image" + TAG_MAP = "map" + TAG_TRACK = "track" + TAG_VIDEO = "video" + TAG_IFRAME = "iframe" + TAG_OBJECT = "object" + TAG_PARAM = "param" + TAG_PICTURE = "picture" + TAG_PORTAL = "portal" + TAG_SOURCE = "source" + TAG_CANVAS = "canvas" + TAG_NOSCRIPT = "noscript" + TAG_NOEMBED = "noembed" + TAG_PLAINTEXT = "plaintext" + TAG_XMP = "xmp" + TAG_SCRIPT = "script" + TAG_DEL = "del" + TAG_INS = "ins" + TAG_CAPTION = "caption" + TAG_COL = "col" + TAG_COLGROUP = "colgroup" + TAG_TABLE = "table" + TAG_TBODY = "tbody" + TAG_TD = "td" + TAG_TFOOT = "tfoot" + TAG_TH = "th" + TAG_THEAD = "thead" + TAG_TR = "tr" + TAG_BUTTON = "button" + TAG_DATALIST = "datalist" + TAG_FIELDSET = "fieldset" + TAG_FORM = "form" + TAG_INPUT = "input" + TAG_KEYGEN = "keygen" + TAG_LABEL = "label" + TAG_LEGEND = "legend" + TAG_METER = "meter" + TAG_OPTGROUP = "optgroup" + TAG_OPTION = "option" + TAG_OUTPUT = "output" + TAG_PROGRESS = "progress" + TAG_SELECT = "select" + TAG_TEXTAREA = "textarea" + TAG_DETAILS = "details" + TAG_DIALOG = "dialog" + TAG_MENU = "menu" + TAG_SUMMARY = "summary" + TAG_BLINK = "blink" + TAG_CENTER = "center" + TAG_CONTENT = "content" + TAG_DIR = "dir" + TAG_FONT = "font" + TAG_FRAME = "frame" + TAG_NOFRAMES = "noframes" + TAG_FRAMESET = "frameset" + TAG_STRIKE = "strike" + TAG_TT = "tt" + TAG_TEMPLATE = "template" + TAG_SARCASM = "sarcasm" + TAG_MATH = "math" + TAG_SVG = "svg" + TAG_CHARSET = "charset" + TAG_HTTP_EQUIV = "http-equiv" + TAG_TYP = "type" + TAG_DEFINITION_URL = "definitionurl" + TAG_ANNOTATION_XML = "annotation-xml" + TAG_FOREIGN_OBJECT = "foreignObject" + TAG_DESC = "desc" + TAG_ENCODING = "encoding" + TAG_MI = "mi" + TAG_MO = "mo" + TAG_MN = "mn" + TAG_MS = "ms" + TAG_MTEXT = "mtext" + TAG_MGLYPH = "mglyph" + TAG_MALIGNMARK = "malignmark" + TAG_COLOR = "color" + TAG_FACE = "face" + TAG_SIZE = "size" + + QuirksMode* = enum + NO_QUIRKS, QUIRKS, LIMITED_QUIRKS + + Namespace* = enum + NO_NAMESPACE = "", + HTML = "http://www.w3.org/1999/xhtml", + MATHML = "http://www.w3.org/1998/Math/MathML", + SVG = "http://www.w3.org/2000/svg", + XLINK = "http://www.w3.org/1999/xlink", + XML = "http://www.w3.org/XML/1998/namespace", + XMLNS = "http://www.w3.org/2000/xmlns/" + NAMESPACE_UNKNOWN = "" + + NamespacePrefix* = enum + NO_PREFIX = "" + PREFIX_XLINK = "xlink" + PREFIX_XML = "xml" + PREFIX_XMLNS = "xmlns" + PREFIX_UNKNOWN = "" + +const AllTagTypes* = {TagType.low..TagType.high} + +const HTagTypes* = { + TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6 +} + +# 4.10.2 Categories +const FormAssociatedElements* = { + TAG_BUTTON, TAG_FIELDSET, TAG_INPUT, TAG_OBJECT, TAG_OUTPUT, TAG_SELECT, + TAG_TEXTAREA, TAG_IMG +} + +const ListedElements* = { + TAG_FIELDSET, TAG_INPUT, TAG_OBJECT, TAG_OUTPUT, TAG_SELECT, TAG_TEXTAREA +} diff --git a/lib/chame0/chame/tokstate.nim b/lib/chame0/chame/tokstate.nim new file mode 100644 index 00000000..0d794ea0 --- /dev/null +++ b/lib/chame0/chame/tokstate.nim @@ -0,0 +1,32 @@ +type TokenizerState* = enum + DATA, CHARACTER_REFERENCE, TAG_OPEN, RCDATA, RCDATA_LESS_THAN_SIGN, + RAWTEXT, RAWTEXT_LESS_THAN_SIGN, SCRIPT_DATA, SCRIPT_DATA_LESS_THAN_SIGN, + PLAINTEXT, MARKUP_DECLARATION_OPEN, END_TAG_OPEN, BOGUS_COMMENT, TAG_NAME, + BEFORE_ATTRIBUTE_NAME, RCDATA_END_TAG_OPEN, RCDATA_END_TAG_NAME, + RAWTEXT_END_TAG_OPEN, RAWTEXT_END_TAG_NAME, SELF_CLOSING_START_TAG, + SCRIPT_DATA_END_TAG_OPEN, SCRIPT_DATA_ESCAPE_START, + SCRIPT_DATA_END_TAG_NAME, SCRIPT_DATA_ESCAPE_START_DASH, + SCRIPT_DATA_ESCAPED_DASH_DASH, SCRIPT_DATA_ESCAPED, + SCRIPT_DATA_ESCAPED_DASH, SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN, + SCRIPT_DATA_ESCAPED_END_TAG_OPEN, SCRIPT_DATA_DOUBLE_ESCAPE_START, + SCRIPT_DATA_ESCAPED_END_TAG_NAME, SCRIPT_DATA_DOUBLE_ESCAPED, + SCRIPT_DATA_DOUBLE_ESCAPED_DASH, SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN, + SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH, SCRIPT_DATA_DOUBLE_ESCAPE_END, + AFTER_ATTRIBUTE_NAME, ATTRIBUTE_NAME, BEFORE_ATTRIBUTE_VALUE, + ATTRIBUTE_VALUE_DOUBLE_QUOTED, ATTRIBUTE_VALUE_SINGLE_QUOTED, + ATTRIBUTE_VALUE_UNQUOTED, AFTER_ATTRIBUTE_VALUE_QUOTED, COMMENT_START, + CDATA_SECTION, COMMENT_START_DASH, COMMENT, COMMENT_END, + COMMENT_LESS_THAN_SIGN, COMMENT_END_DASH, COMMENT_LESS_THAN_SIGN_BANG, + COMMENT_LESS_THAN_SIGN_BANG_DASH, COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH, + COMMENT_END_BANG, DOCTYPE, BEFORE_DOCTYPE_NAME, DOCTYPE_NAME, + AFTER_DOCTYPE_NAME, AFTER_DOCTYPE_PUBLIC_KEYWORD, + AFTER_DOCTYPE_SYSTEM_KEYWORD, BOGUS_DOCTYPE, + BEFORE_DOCTYPE_PUBLIC_IDENTIFIER, DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED, + DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED, AFTER_DOCTYPE_PUBLIC_IDENTIFIER, + BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS, + DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED, + DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED, BEFORE_DOCTYPE_SYSTEM_IDENTIFIER, + AFTER_DOCTYPE_SYSTEM_IDENTIFIER, CDATA_SECTION_BRACKET, CDATA_SECTION_END, + NAMED_CHARACTER_REFERENCE, NUMERIC_CHARACTER_REFERENCE, + AMBIGUOUS_AMPERSAND_STATE, HEXADECIMAL_CHARACTER_REFERENCE_START, + HEXADECIMAL_CHARACTER_REFERENCE, DECIMAL_CHARACTER_REFERENCE diff --git a/lib/chame0/chame/version.nim b/lib/chame0/chame/version.nim new file mode 100644 index 00000000..6c29dff8 --- /dev/null +++ b/lib/chame0/chame/version.nim @@ -0,0 +1,3 @@ +const Major* = 1 +const Minor* = 0 +const Patch* = 3 diff --git a/lib/chame0/doc/.index.md b/lib/chame0/doc/.index.md new file mode 100644 index 00000000..a38863e6 --- /dev/null +++ b/lib/chame0/doc/.index.md @@ -0,0 +1,18 @@ +# Documentation of Chame, an HTML5 parsing library written in Nim. + +To get started, please read the [readme](https://git.sr.ht/~bptato/chame). + +Index of the (stable) public API: + +* [minidom](minidom.html): minimal DOM module and a high-level interface to the + HTML parser. +* [minidom_cs](minidom_cs.html): minidom with support for non-UTF-8 character + sets. +* [htmlparser](htmlparser.html): low-level interface to the HTML parser. +* [htmlparseriface](htmlparseriface.html): forward declarations for the HTML + parser +* [tags](tags.html): enum definitions for Chame (tags, namespaces, etc.) +* [tokstate](tokstate.html): tokenizer states for the fragment parsing + algorithm + +Also of interest: the Chame [manual](manual.html). diff --git a/lib/chame0/doc/manual.md b/lib/chame0/doc/manual.md new file mode 100644 index 00000000..afa50b56 --- /dev/null +++ b/lib/chame0/doc/manual.md @@ -0,0 +1,273 @@ +# Using Chame + +Chame is divided into two parts: a low-level API ([htmlparser](htmlparser.html)) +and a high-level API ([minidom](minidom.html), [minidom_cs](minidom_cs.html)). +The high-level APIs build on top of htmlparser, and are easier to use. However, +they give consumers less control than htmlparser. + +Here we describe both APIs. + +## Table of Contents + +* [Basic concepts](#basic-concepts) + - [Standards](#standards) + - [String interning](#string-interning) + - [String validation](#string-validation) +* [High-level API (minidom, minidom_cs)](#high-level-api-minidom-minidom_cs) +* [Low-level API (htmlparser)](#low-level-api-htmlparser) + - [Functions and procedures](#functions-and-procedures) + * [initHTML5Parser](#inithtml5parser) + * [parseChunk](#parsechunk) + * [finish](#finish) + - [Generic parameters](#generic-parameters) +* [Example](#example) + +## Basic concepts + +### Standards + +Chame implements HTML5 parsing as described in the +[Parsing HTML documents](https://html.spec.whatwg.org/multipage/parsing.html) +section of WHATWG's living standard. Note that this document may change at +any time, and newer additions might take some time to implement in Chame. + +Users of the low-level API are encouraged to consult the appropriate sections +of the standard while implementing hooks provided by htmlparser. + +### String interning + +To achieve O(1) comparisons of tag and attribute names and a lower +memory footprint, Chame uses +[string interning](https://en.wikipedia.org/wiki/String_interning). +While minidom users can simply call the appropriate conversion functions +on Document.factory, consumers of htmlparser must implement string +interning themselves, be that through MAtomFactory or a custom solution. + +### String validation + +Note that (as per standard) the tokenization stage strips out all NUL +characters, so strings from the parser can be safely converted to cstrings. + +htmlparser itself does no UTF-8 validation; it is up to the DOM builder to +validate the input. Non-ASCII characters are treated as opaque characters, +so parsing of ASCII-compatible character sets should just work with the caveat +that the strings from htmlparser will not necessarily be valid UTF-8. This is +not a problem in minidom, since it abstracts over this difficulty. + +## High-level API (minidom, minidom_cs) + +minidom has two main entry points: `parseHTML` and `parseHTMLFragment`. For +parsing documents, `parseHTML` should be used; `parseHTMLFragment` is for parsing +incomplete document fragments. + +e.g. in a browser, the `innerHTML` setter would use `parseHTMLFragment`, while +`DOMParser.parseFromString` would use `parseHTML`. + +The input stream must be passed as a `Stream` object from `std/streams`. Both +`parseHTML` and `parseHTMLFragment` return only when the input stream has been +completely consumed from the stream. For chunked parsing, you must use the +low-level htmlparser API instead. + +minidom (and minidom_cs) implements string interning using `MAtomFactory`, and +interned strings in minidom are represented using `MAtom`s. Every `MAtom` is +guaranteed to point to a valid UTF-8 string. To convert a Nim string into an +`MAtom`, use the `MAtomFactory.strToAtom` function. To convert an `MAtom` into a +Nim string, use the `MAtomFactory.atomToStr` function. + +Note: it is always more efficient to convert strings to atoms (i.e. to use +`strToAtom`) than to do it the other way. `MAtom`s are just integers, so +storing, copying, and comparing them is a lot cheaper than the same operations +with strings. + +The output is a DOM tree, with the root node being a `Document`. The root +Document node also contains a `MAtomFactory` instance, which can be used to +convert `MAtom`s back to strings (through `atomToStr`). + +Strings returned from minidom are guaranteed to be valid UTF-8. Note however +that minidom only understands UTF-8 documents. For parsing documents with +character sets other than UTF-8, minidom_cs must be used. The `parseHTML` +function of minidom_cs is also able to BOM sniff, interpret meta charset +tags, and optionally retry parsing with a predefined list of character +sets (using the companion character decoding library Chagashi). + +## Low-level API (htmlparser) + +### Functions and procedures + +htmlparser has three defined procedures: `initHTML5Parser`, `parseChunk`, and +`finish`. A `getInsertionPoint` function is available as well. + +#### initHTML5Parser + +```nim +# Signature +proc initHTML5Parser[Handle, Atom](dombuilder: DOMBuilder[Handle, Atom], + opts: HTML5ParserOpts[Handle, Atom]): HTML5Parser[Handle, Atom] +``` + +The `initHTML5Parser` procedure requires a user-defined DOMBuilder object +derived from the `DOMBuilder[Handle, Atom]` generic object reference. + +To implement all interfaces necessary for htmlparser, please include +[htmlparseriface](htmlparseriface.html) in your DOM builder module; it contains +forward-declarations for all procedures that `HTML5Parser` depends on. Feel +free to study/copy [minidom](minidom.html)'s implementations. + +The return value is an `HTML5Parser[Handle, Atom]` object. Note that this is +a rather large object that is passed by value; if possible, avoid copying it +at all. + +#### parseChunk + +```nim +# Signature +proc parseChunk[Handle, Atom](parser: var HTML5Parser[Handle, Atom], + inputBuf: openArray[char], reprocess = false): ParseResult +``` + +`parseChunk` consumes all data passed in `inputBuf`. During this, the +appropriate functions (`createElementImpl`, etc.) will be called by the parser. + +`parseChunk` returns a `ParseResult`, which is one of the following values: + +* `PRES_CONTINUE`: the caller should continue with parsing the next chunk of + data when it is available. (It's also fine to do delay processing the next + call by processing something different first.) +* `PRES_STOP`: parsing was stopped by your setEncodingImpl implementation. The + caller is expected to restart parsing from the beginning using a **new** + `HTML5Parser` object. WARNING: do *not* re-use the current HTML5Parser for + this. +* `PRES_SCRIPT`: a `</script>` end tag has been encountered, which immediately + suspended parsing. In the next `parseChunk` call, the caller is expected to + pass the **same** buffer (`inputBuf`) as in the current one. For details, + see below. + +Special care is required when implementing programs with scripting support. The +HTML5 standard requires the parser to be re-entrant for supporting the +`document.write` JavaScript function; therefore the parser suspends itself upon +encountering a `</script>` end tag, returning a `PRES_SCRIPT` `ParseResult`. + +At this point, implementations have two options. + +##### Option 1: Continue parsing the current buffer + +If either: + +* your implementation does not support `document.write`, or +* no `document.write` call has been issued by the script, or +* parsing of all buffers passed by `document.write` calls has finished, + +then you can simply resume parsing the current buffer by calling `parseChunk` +again with an openArray that uses the same backing buffer, except starting +from `parser.getInsertionPoint()`. `minidom`, which pretends to support +scripting in test cases, but does not actually execute scripts, has an example +of this: + +```nim +var buffer: array[4096, char] +while true: + let n = inputStream.readData(addr buffer[0], buffer.len) + if n == 0: break + # res can be PRES_CONTINUE or PRES_SCRIPTING. PRES_STOP is only returned + # on charset switching, and minidom does not support that. + var res = parser.parseChunk(buffer.toOpenArray(0, n - 1)) + # Important: we must repeat parseChunk with the same contents for the script + # end tag result, with reprocess = true. + # + # (This is only relevant for calls where scripting = true; with scripting = + # false, PRES_SCRIPT would never be returned.) + var ip = 0 + while res == PRES_SCRIPT and (ip += parser.getInsertionPoint(); ip != n): + res = parser.parseChunk(buffer.toOpenArray(ip, n - 1)) +parser.finish() +``` + +Note the while loop; `parseChunk` will return `PRES_SCRIPT` multiple times +for a single chunk if it contains several scripts. + +Also note that `minidom` does not handle `PRES_STOP`, since it does not support +legacy encodings. For an implementation that *does* handle `PRES_STOP`, see +`minidom_cs`. + +##### Option 2: Parse buffers passed by `document.write` + +Per standard, it is possible to insert buffers into the stream from scripts +using the `document.write` function. + +It is possible to implement this, but it is somewhat too involved to give a +detailed explanation of it here. Please refer to Chawan's implementation in +html/chadombuilder and html/dom. (Good luck.) + +#### finish + +After having parsed all chunks of your document with `parseChunk`, you **must** +call the `finish` function. This is necessary because the parser may still have +some non-flushed characters in an internal buffer. e.g. when the parser receives +the string `>`, it is not clear whether the character reference refers to a +"greater than" sign, or a longer character reference like `≳`; `finish` +confirms that the reference is indeed a `>` sign. Also, the parser has to +execute certain actions on encountering the `EOF` token, which only `finish` can +produce. + +`finish` must never be called twice, and any `parseChunk` call after `finish` +is invalid. + +### Generic parameters + +`initHTML5Parser` takes two generic parameters: `Handle` and `Atom`. + +`Handle` is conceptually a unique pointer to a node in the document. A naive +single-threaded implementation (like minidom) may simply implement this as +a Nim `ref` to an object. However, this is not mandatory; since `Handle` is +a generic parameter, any type is accepted. For example, multi-processing +implementations that use message passing might instead prefer to use an +integer ID that refers to an object owned by a different thread. + +Similarly, `Atom` is a unique pointer to a string. This means that +`DOMBuilder.strToAtom` must always return the same Atom for every string whose +contents are equivalent. Additionally, `atomToTagType` and `tagTypeToAtom` must +operate as if `TagType` values were equivalent to the contents of its +stringifier. (i.e. `tagTypeToAtom(tagType) == strToAtom($tagType)` for all tag +types except `TAG_UNKNOWN`, which is never passed to `tagTypeToAtom`.) + +Note that htmlparser does not *require* an `atomToStr` procedure, so it is not +even necessary to store interned strings in a format compatible with the Nim +string type. (Obviously, some way to stringify atoms is required for most use +cases, but it need not be exposed to Chame.) + +## Example + +A simple example with minidom: dumps all text on a page. + +```Nim +# Compile with nim c -d:ssl +# List text found between HTML tags on the target website. +import std/httpclient +import std/os +import std/strutils +import chame/minidom + +if paramCount() != 1: + echo "Usage: " & paramStr(0) & " [URL]" + quit(1) +let client = newHttpClient() +let res = client.get(paramStr(1)) +let document = parseHTML(res.bodyStream) +var stack = @[Node(document)] +while stack.len > 0: + let node = stack.pop() + if node of Text: + let s = Text(node).data.strip() + if s != "": + echo s + for i in countdown(node.childList.high, 0): + stack.add(node.childList[i]) +``` + +For more advanced usage of minidom, please study tests/tree.nim and +tests/shared/tree_common.nim which together constitute a test runner of +html5lib-tests. + +For an example implementation of [htmlparseriface](htmlparseriface.html), please +check the source code of [minidom](minidom.html) (and if you need legacy charset +support, [minidom_cs](minidom_cs.html)). diff --git a/lib/chame0/gendoc.sh b/lib/chame0/gendoc.sh new file mode 100755 index 00000000..4e7f5578 --- /dev/null +++ b/lib/chame0/gendoc.sh @@ -0,0 +1,25 @@ +#!/bin/sh +mkdir -p .obj/doc +for f in chame/*.nim +do if test "$f" = "chame/htmlparseriface.nim" + then nim doc -d:nimdocdummy --outdir:.obj/doc "$f" + else nim doc -p:test/chagashi/ --outdir:.obj/doc "$f" + fi + sed -i \ + -e '/<\!-- Google fonts -->/,+2d' \ + -e 's/theindex.html/index.html/g' \ + ".obj/doc/$(basename "$f" .nim).html" +done +makehtml() { + printf '<!DOCTYPE html> +<head> +<meta name=viewport content="width=device-width, initial-scale=1"> +<title>%s</title> +</head> +<body> +' "$2" + cat "$1" | pandoc + printf '</body>\n' +} +makehtml doc/manual.md "Chame manual" > .obj/doc/manual.html +makehtml doc/.index.md "Chame documentation" > .obj/doc/index.html diff --git a/lib/chame0/test/chagashi b/lib/chame0/test/chagashi new file mode 160000 +Subproject f77bd4183aad90aca00f387844de2ef8f75ce84 diff --git a/lib/chame0/test/config.nims b/lib/chame0/test/config.nims new file mode 100644 index 00000000..80091ff6 --- /dev/null +++ b/lib/chame0/test/config.nims @@ -0,0 +1 @@ +switch("path", "$projectDir/../src") diff --git a/lib/chame0/test/custom.dat b/lib/chame0/test/custom.dat new file mode 100644 index 00000000..f3463935 --- /dev/null +++ b/lib/chame0/test/custom.dat @@ -0,0 +1,23 @@ +#data +<!DOCTYPE html> +<plaintext> +いろはにほへと +ちりぬるをわか +よたれそつねな +らむうゐのおく +やまけふこえて +あさくゆめみし +ゑひもせす +#document +| <html> +| <head> +| <body> +| <plaintext> +| " +縺�繧阪�ッ縺ォ縺サ縺ク縺ィ +縺。繧翫〓繧九r繧上° +繧医◆繧後◎縺、縺ュ縺ェ +繧峨��縺�繧舌�ョ縺翫¥ +繧�縺セ縺代�オ縺薙∴縺ヲ +縺ゅ&縺阪f繧√∩縺� +繧代�イ繧ゅ○縺�" diff --git a/lib/chame0/test/eprint.nim b/lib/chame0/test/eprint.nim new file mode 100644 index 00000000..ef86818d --- /dev/null +++ b/lib/chame0/test/eprint.nim @@ -0,0 +1,22 @@ +{.used.} + +template eprint0(s: varargs[string]) = + {.cast(noSideEffect), cast(tags: []), cast(raises: []).}: + var o = "" + for i in 0 ..< s.len: + if i != 0: + o &= ' ' + o &= s[i] + when not declared(stderr): + echo o + else: + o &= '\n' + stderr.write(o) + +when defined(release): + func eprint*(s: varargs[string, `$`]) + {.deprecated: "eprint is for debugging only".} = + eprint0(s) +else: + func eprint*(s: varargs[string, `$`]) = + eprint0(s) diff --git a/lib/chame0/test/gtrsim.dat b/lib/chame0/test/gtrsim.dat new file mode 100644 index 00000000..1049dbd2 --- /dev/null +++ b/lib/chame0/test/gtrsim.dat @@ -0,0 +1,46 @@ +#data +>rsim +#errors +#document +| <html> +| <head> +| <body> +| ">rsim" + +#data +> +#errors +#document +| <html> +| <head> +| <body> +| ">" + +#data +<h2>&g</h2> +#errors +#document +| <html> +| <head> +| <body> +| <h2> +| "&g" + +#data +<H2>Q&A</H2> +#errors +#document +| <html> +| <head> +| <body> +| <h2> +| "Q&A" + +#data +ź +#errors +#document +| <html> +| <head> +| <body> +| "ナコ" diff --git a/lib/chame0/test/html5lib-tests b/lib/chame0/test/html5lib-tests new file mode 160000 +Subproject a9f44960a9fedf265093d22b2aa3c7ca123727b diff --git a/lib/chame0/test/latin1.dat b/lib/chame0/test/latin1.dat new file mode 100644 index 00000000..bbca37da --- /dev/null +++ b/lib/chame0/test/latin1.dat @@ -0,0 +1,13 @@ +#data +<meta charset=latin1> +゚ +#errors +#document +| <html> +| <head> +| <meta> +| charset="latin1" +| " +" +| <body> +| "テ�" diff --git a/lib/chame0/test/shared/tree_common.nim b/lib/chame0/test/shared/tree_common.nim new file mode 100644 index 00000000..cac15100 --- /dev/null +++ b/lib/chame0/test/shared/tree_common.nim @@ -0,0 +1,334 @@ +import std/algorithm +import std/options +import std/strutils +import std/tables +import std/unittest + +import test1 +import chame/htmlparser +import chame/minidom + +type + TCError = object + s: string + + FragmentType = enum + FT_HTML, FT_SVG, FT_MATHML + + TCFragment = object + fragmentType: FragmentType + ctx: Element + + ScriptMode = enum + SCRIPT_BOTH, SCRIPT_OFF, SCRIPT_ON + + TCTest = object + data: string + errors: seq[TCError] + newerrors: seq[TCError] + fragment: Option[TCFragment] + script: ScriptMode + document: Document + + TCTestParser = object + s: string + i: int + pi: int + factory: MAtomFactory + linei: int + +func has(ctx: TCTestParser): bool = + return ctx.i < ctx.s.len + +proc reconsumeLine(ctx: var TCTestParser) = + ctx.i = ctx.pi + dec ctx.linei + +proc consumeLine(ctx: var TCTestParser): string = + result = "" + ctx.pi = ctx.i + inc ctx.linei + while ctx.has: + if ctx.s[ctx.i] == '\n': + inc ctx.i + break + result &= ctx.s[ctx.i] + inc ctx.i + +proc parseTestData(ctx: var TCTestParser): string = + result = "" + while ctx.has: + let line = ctx.consumeLine() + if line == "#errors": + return + if result.len > 0: + result &= '\n' + result &= line + doAssert false, "errors expected" + +proc parseTestNewErrors(ctx: var TCTestParser): seq[TCError] = + result = @[] + while ctx.has: + let line = ctx.consumeLine() + case line + of "#document-fragment", "#script-off", "#script-on", "#document": + ctx.reconsumeLine() + return + result.add(TCError(s: line)) + +proc parseTestErrors(ctx: var TCTestParser): seq[TCError] = + result = @[] + while ctx.has: + let line = ctx.consumeLine() + case line + of "#new-errors": + ctx.reconsumeLine() + result.add(ctx.parseTestNewErrors()) + return + of "#document-fragment", "#script-off", "#script-on", "#document": + ctx.reconsumeLine() + return + result.add(TCError(s: line)) + +proc parseTestFragment(ctx: var TCTestParser): TCFragment = + var line = ctx.consumeLine() + var fragmentType = FT_HTML + if line.startsWith("svg "): + fragmentType = FT_SVG + line = line.substr("svg ".len) + elif line.startsWith("math "): + fragmentType = FT_MATHML + line = line.substr("math ".len) + let namespace = case fragmentType + of FT_SVG: Namespace.SVG + of FT_MATHML: Namespace.MATHML + of FT_HTML: Namespace.HTML + let element = Element( + namespace: namespace, + localName: ctx.factory.strToAtom(line) + ) + return TCFragment( + fragmentType: fragmentType, + ctx: element + ) + +proc parseDoctype(ctx: TCTestParser, s: string): DocumentType = + let doctype = DocumentType() + var i = "<!DOCTYPE ".len + while i < s.len and s[i] != ' ' and s[i] != '>': + doctype.name &= s[i] + inc i + while s[i] == ' ': + inc i + if s[i] == '>': + return doctype + assert s[i] == '"' + inc i + while i < s.len and s[i] != '"': + doctype.publicId &= s[i] + inc i + assert s[i] == '"' + inc i + assert s[i] == ' ' + inc i + assert s[i] == '"' + inc i + while i < s.len and s[i] != '"': + doctype.systemId &= s[i] + inc i + while i + 1 < s.len and s[i + 1] == '"': + doctype.systemId &= s[i] + inc i + assert s[i] == '"' + inc i + assert s[i] == '>' + return doctype + +func until(s: string, c: set[char]): string = + result = "" + var i = 0 + while i < s.len: + if s[i] in c: + break + result.add(s[i]) + inc i + +func until(s: string, c: char): string = s.until({c}) + +func after(s: string, c: set[char]): string = + var i = 0 + while i < s.len: + if s[i] in c: + return s.substr(i + 1) + inc i + return "" + +func after(s: string, c: char): string = s.after({c}) + +proc parseTestDocument(ctx: var TCTestParser): Document = + result = Document(factory: ctx.factory) + var stack = @[Node(result)] + template top: auto = stack[^1] + var thistext: Text = nil + var thiscomment: Comment = nil + var indent = 1 + template pop_node = + let node = stack.pop() + if node of Element: + Element(node).attrs.sort(proc(a, b: Attribute): int = cmp(a.name, b.name)) + indent -= 2 + while ctx.has: + let line = ctx.consumeLine() + if thistext != nil: + if line.endsWith("\""): + thistext.data &= line.substr(0, line.high - 1) + thistext = nil + else: + thistext.data &= line & "\n" + continue + if thiscomment != nil: + if line.endsWith(" -->"): + thiscomment.data &= line.substr(0, line.high - " -->".len) + thiscomment = nil + else: + thiscomment.data &= line & "\n" + continue + if line == "": + break + assert line[0] == '|' and line[1] == ' ' + while indent >= line.len or not line.startsWith('|' & ' '.repeat(indent)): + let node = stack.pop() + if node of Element: + Element(node).attrs.sort(proc(a, b: Attribute): int = cmp(a.name, b.name)) + indent -= 2 + let str = line.substr(indent + 1) + if str.startsWith("<!DOCTYPE "): + let doctype = ctx.parseDoctype(str) + top.childList.add(doctype) + elif str.startsWith("<!-- "): + let comment = minidom.Comment() + top.childList.add(comment) + if not str.endsWith(" -->"): + comment.data = str.substr("<!-- ".len) & "\n" + thiscomment = comment + else: + comment.data = str.substr("<!-- ".len, str.high - " -->".len) + elif str.startsWith("<?"): + assert false, "todo" + elif str.startsWith("<") and str.endsWith(">"): + var nameStr = str.substr(1, str.high - 1) + var namespace = Namespace.HTML + if nameStr.startsWith("svg "): + nameStr = nameStr.substr("svg ".len) + namespace = Namespace.SVG + elif nameStr.startsWith("math "): + nameStr = nameStr.substr("math ".len) + namespace = Namespace.MATHML + let element = if nameStr == "template": + HTMLTemplateElement() + else: + Element() + element.localName = ctx.factory.strToAtom(nameStr) + element.namespace = namespace + element.document = result + top.childList.add(element) + stack.add(element) + indent += 2 + elif str == "content": + let fragment = DocumentFragment() + HTMLTemplateElement(top).content = fragment + stack.add(fragment) + indent += 2 + elif str[0] == '"': + let text = Text() + top.childList.add(text) + if str[^1] != '"' or str.len == 1: + text.data = str.substr(1) & "\n" + thistext = text + else: + text.data = str.substr(1, str.high - 1) + else: + assert '=' in str + var name = str.until('=') + var prefix = NO_PREFIX + var ns = NO_NAMESPACE + if name.startsWith("xml "): + ns = Namespace.XML + prefix = PREFIX_XML + name = name.substr("xml ".len) + elif name.startsWith("xmlns "): + ns = Namespace.XMLNS + prefix = PREFIX_XMLNS + name = name.substr("xmlns ".len) + elif name.startsWith("xlink "): + ns = Namespace.XLINK + prefix = PREFIX_XLINK + name = name.substr("xlink ".len) + let na = ctx.factory.strToAtom(name) + let value = str.after('=')[1..^2] + Element(top).attrs.add((prefix, ns, na, value)) + while indent > 1: + pop_node + +proc parseTest(ctx: var TCTestParser): TCTest = + doAssert ctx.consumeLine() == "#data" + var t = TCTest() + t.data = ctx.parseTestData() + t.errors = ctx.parseTestErrors() + while ctx.has: + let line = ctx.consumeLine() + case line + of "#document-fragment": + t.fragment = some(ctx.parseTestFragment()) + of "#script-off": + t.script = SCRIPT_OFF + of "#script-on": + t.script = SCRIPT_ON + of "#document": + assert t.document == nil + t.document = ctx.parseTestDocument() + break + of "": + break + return t + +proc parseTests*(s: string, factory: MAtomFactory): seq[TCTest] = + result = @[] + var parser = TCTestParser(s: s, factory: factory) + while parser.i < s.len: + let test = parser.parseTest() + result.add(test) + var s = "" + for x in test.document.childList: + s &= $x & '\n' + +proc checkTest(nodein, nodep: Node) = + check nodein.childList.len == nodep.childList.len + if nodein.childList.len != nodep.childList.len: + echo nodein + echo nodep + if nodein of Element: + check nodep of Element + let nodein = Element(nodein) + let nodep = Element(nodep) + check nodein.localName == nodep.localName + check nodein.namespace == nodep.namespace + if nodein.attrs != nodep.attrs: + echo "NODEIN", $nodein + echo "NODEP", $nodep + check nodein.attrs == nodep.attrs + elif nodein of DocumentFragment: + assert false + elif nodein of CharacterData: + check nodep of CharacterData + check CharacterData(nodein).data == CharacterData(nodep).data + elif nodein of DocumentType: + check nodep of DocumentType + let nodein = DocumentType(nodein) + let nodep = DocumentType(nodep) + check nodein.name == nodep.name + check nodein.publicId == nodep.publicId + check nodein.systemId == nodep.systemId + for i in 0 ..< nodein.childList.len: + checkTest(nodein.childList[i], nodep.childList[i]) + diff --git a/lib/chame0/test/sjis.dat b/lib/chame0/test/sjis.dat new file mode 100644 index 00000000..6c2ac6bc --- /dev/null +++ b/lib/chame0/test/sjis.dat @@ -0,0 +1,23 @@ +#data +<plaintext> +いろはにほへと +ちりぬるをわか +よたれそつねな +らむうゐのおく +やまけふこえて +あさくゆめみし +ゑひもせす +#errors +#document +| <html> +| <head> +| <body> +| <plaintext> +| " +縺�繧阪�ッ縺ォ縺サ縺ク縺ィ +縺。繧翫〓繧九r繧上° +繧医◆繧後◎縺、縺ュ縺ェ +繧峨��縺�繧舌�ョ縺翫¥ +繧�縺セ縺代�オ縺薙∴縺ヲ +縺ゅ&縺上f繧√∩縺� +繧代�イ繧ゅ○縺�" diff --git a/lib/chame0/test/test1.nim b/lib/chame0/test/test1.nim new file mode 100644 index 00000000..b00cb4ac --- /dev/null +++ b/lib/chame0/test/test1.nim @@ -0,0 +1,92 @@ +# "simple" test to check if it compiles +# (actually, I just copy-pasted a throwaway test project here) + +import std/unittest +import std/streams + +import chame/tags +import chame/minidom + +func escapeText(s: string, attribute_mode = false): string = + result = "" + var nbsp_mode = false + var nbsp_prev = '\0' + for c in s: + if nbsp_mode: + if c == char(0xA0): + result &= " " + else: + result &= nbsp_prev & c + nbsp_mode = false + elif c == '&': + result &= "&" + elif c == char(0xC2): + nbsp_mode = true + nbsp_prev = c + elif attribute_mode and c == '"': + result &= """ + elif not attribute_mode and c == '<': + result &= "<" + elif not attribute_mode and c == '>': + result &= ">" + else: + result &= c + +func `$`*(node: Node): string = + result = "" + if node of Element: + let element = Element(node) + var x = "" + if element.namespace == Namespace.SVG: + x = "svg " + elif element.namespace == Namespace.MATHML: + x = "math " + result = "<" & x & element.localNameStr + for k, v in element.attrsStr: + result &= ' ' & k & "=\"" & v.escapeText(true) & "\"" + result &= ">" + for node in element.childList: + result &= $node + result &= "</" & x & element.localNameStr & ">" + elif node of Text: + let text = Text(node) + result = text.data.escapeText() + elif node of Comment: + result = "<!-- " & Comment(node).data & "-->" + elif node of DocumentType: + result = "<!DOCTYPE" & ' ' & DocumentType(node).name & ">" + elif node of Document: + result = "Node of Document" + else: + assert false + +# This is, in fact, standards-compliant behavior. +# Don't ask. +test "simple html serialization test": + const inhtml = """ +<!DOCTYPE html> +<html> +<head> +</head> +<body> +<main> +Hello, world! +</main> +</body> +</html>""" + const outhtml = """ +<!DOCTYPE html> +<html><head> +</head> +<body> +<main> +Hello, world! +</main> + +</body></html> +""" + let document = parseHTML(newStringStream(inhtml)) + var s = "" + for x in document.childList: + s &= $x & '\n' + check s == outhtml diff --git a/lib/chame0/test/tokenizer.nim b/lib/chame0/test/tokenizer.nim new file mode 100644 index 00000000..1d6ae41e --- /dev/null +++ b/lib/chame0/test/tokenizer.nim @@ -0,0 +1,275 @@ +import std/json +import std/options +import std/tables +import std/unicode +import std/unittest + +import chame/htmltokenizer +import chame/minidom + +const hexCharMap = (func(): array[char, int] = + for i in 0..255: + case chr(i) + of '0'..'9': result[char(i)] = i - ord('0') + of 'a'..'f': result[char(i)] = i - ord('a') + 10 + of 'A'..'F': result[char(i)] = i - ord('A') + 10 + else: result[char(i)] = -1 +)() + +func hexValue(c: char): int = + return hexCharMap[c] + +func doubleEscape(input: string): string = + var s = "" + var esc = 0 + var u: uint32 = 0 + for c in input: + if esc == 0: + if c == '\\': + inc esc + else: + s &= c + elif esc == 1: + if c == 'u': + inc esc + else: + s &= '\\' + dec esc + s &= c + elif esc < 6: # 2 + 4 + inc esc + if esc == 3: + u = 0x00 + let n = hexValue(c) + doAssert n != -1 + u *= 0x10 + u += uint32(n) + if esc == 6: + s &= $cast[Rune](u) + esc = 0 + return s + +proc getAttrs(factory: MAtomFactory, o: JsonNode, esc: bool): + Table[MAtom, string] = + result = Table[MAtom, string]() + for k, v in o: + let k = factory.strToAtom(k) + if esc: + result[k] = v.getStr().doubleEscape() + else: + result[k] = v.getStr() + +proc getToken(factory: MAtomFactory, a: seq[JsonNode], esc: bool): + Token[MAtom] = + case a[0].getStr() + of "StartTag": + return Token[MAtom]( + t: START_TAG, + tagname: factory.strToAtom(a[1].getStr()), + attrs: getAttrs(factory, a[2], esc), + selfclosing: a.len > 3 and a[3].getBool() + ) + of "EndTag": + return Token[MAtom]( + t: END_TAG, + tagname: factory.strToAtom(a[1].getStr()) + ) + of "Character": + let s = if esc: + doubleEscape(a[1].getStr()) + else: + a[1].getStr() + return Token[MAtom]( + t: CHARACTER, + s: s + ) + of "DOCTYPE": + return Token[MAtom]( + t: TokenType.DOCTYPE, + quirks: not a[4].getBool(), # yes, this is reversed. don't ask + name: if a[1].kind == JNull: none(string) else: some(a[1].getStr()), + pubid: if a[2].kind == JNull: none(string) else: some(a[2].getStr()), + sysid: if a[3].kind == JNull: none(string) else: some(a[3].getStr()) + ) + of "Comment": + let s = if esc: + doubleEscape(a[1].getStr()) + else: + a[1].getStr() + return Token[MAtom]( + t: TokenType.COMMENT, + data: s + ) + else: discard + +proc checkEquals(factory: MAtomFactory, tok, otok: Token, desc: string) = + doAssert otok.t == tok.t, desc & " (tok t: " & $tok.t & " otok t: " & + $otok.t & ")" + case tok.t + of TokenType.DOCTYPE: + doAssert tok.name == otok.name, desc & " (" & "tok name: " & $tok.name & + " otok name: " & $otok.name & ")" + doAssert tok.pubid == otok.pubid, desc & " (" & "tok pubid: " & + $tok.pubid & " otok pubid: " & $otok.pubid & ")" + doAssert tok.sysid == otok.sysid, desc + doAssert tok.quirks == otok.quirks, desc + of TokenType.START_TAG, TokenType.END_TAG: + doAssert tok.tagname == otok.tagname, desc & " (tok tagname: " & + factory.atomToStr(tok.tagname) & " otok tagname " & + factory.atomToStr(otok.tagname) & ")" + if tok.t == TokenType.START_TAG: + #TODO not sure if this is the best solution. but end tags can't really + # be self-closing... + # Maybe use a separate "self-closing tag" token type? + doAssert tok.selfclosing == otok.selfclosing, desc + var attrs = "" + var i = 0 + for name, value in tok.attrs: + if i > 0: + attrs &= " " + attrs &= factory.atomToStr(name) + attrs &= "=" + attrs &= "'" & value & "'" + inc i + var oattrs = "" + i = 0 + for name, value in otok.attrs: + if i > 0: + oattrs &= " " + oattrs &= factory.atomToStr(name) + oattrs &= "=" + oattrs &= "'" & value & "'" + inc i + doAssert tok.attrs == otok.attrs, desc & " (tok attrs: " & attrs & + " otok attrs (" & oattrs & ")" + of TokenType.CHARACTER, TokenType.CHARACTER_WHITESPACE: + doAssert tok.s == otok.s, desc & " (tok s: " & tok.s & " otok s: " & + otok.s & ")" + of TokenType.COMMENT: + doAssert tok.data == otok.data, desc & " (tok data: " & tok.data & + " otok data: " & otok.data & ")" + of EOF, CHARACTER_NULL: discard + +proc runTest(builder: MiniDOMBuilder, desc: string, + output: seq[JsonNode], laststart: MAtom, esc: bool, + input: string, state = TokenizerState.DATA) = + let factory = builder.factory + var tokenizer = newTokenizer(builder, state) + tokenizer.laststart = Token[MAtom](t: START_TAG, tagname: laststart) + var i = 0 + var chartok: Token[MAtom] = nil + var toks = newSeq[Token[MAtom]]() + while true: + let res = tokenizer.tokenize(input.toOpenArray(0, input.high)) + toks.add(tokenizer.tokqueue) + if res == trDone: + break + while true: + let res = tokenizer.finish() + toks.add(tokenizer.tokqueue) + if res == trDone: + break + for tok in toks: + check tok != nil + if chartok != nil and tok.t notin {CHARACTER, CHARACTER_WHITESPACE, + CHARACTER_NULL}: + let otok = getToken(factory, output[i].getElems(), esc) + checkEquals(factory, chartok, otok, desc) + inc i + chartok = nil + if tok.t in {CHARACTER, CHARACTER_WHITESPACE}: + if chartok == nil: + chartok = Token[MAtom](t: CHARACTER) + chartok.s &= tok.s + elif tok.t == CHARACTER_NULL: + if chartok == nil: + chartok = Token[MAtom](t: CHARACTER) + chartok.s &= char(0) + else: + let otok = getToken(factory, output[i].getElems(), esc) + checkEquals(factory, tok, otok, desc) + inc i + +func getState(s: string): TokenizerState = + case s + of "Data state": + return DATA + of "PLAINTEXT state": + return PLAINTEXT + of "RCDATA state": + return RCDATA + of "RAWTEXT state": + return RAWTEXT + of "Script data state": + return SCRIPT_DATA + of "CDATA section state": + return CDATA_SECTION + else: + doAssert false, "Unknown state: " & s + +const rootpath = "test/html5lib-tests/tokenizer/" + +proc runTests(filename: string) = + let tests = parseFile(rootpath & filename){"tests"} + for t in tests: + let desc = t{"description"}.getStr() + var input = t{"input"}.getStr() + let esc = "doubleEscaped" in t and t{"doubleEscaped"}.getBool() + if esc: + input = doubleEscape(input) + let output = t{"output"}.getElems() + let laststart0 = if "lastStartTag" in t: + t{"lastStartTag"}.getStr() + else: + "" + let factory = newMAtomFactory() + let builder = newMiniDOMBuilder(factory) + let laststart = builder.factory.strToAtom(laststart0) + if "initialStates" notin t: + runTest(builder, desc, output, laststart, esc, input) + else: + for state in t{"initialStates"}: + let state = getState(state.getStr()) + runTest(builder, desc, output, laststart, esc, input, state) + +test "contentModelFlags": + runTests("contentModelFlags.test") + +test "domjs": + runTests("domjs.test") + +test "entities": + runTests("entities.test") + +test "escapeFlag": + runTests("escapeFlag.test") + +test "namedEntities": + runTests("namedEntities.test") + +test "numericEntities": + runTests("numericEntities.test") + +#test "pendingSpecChanges": +# runTests("pendingSpecChanges.test") + +test "test1": + runTests("test1.test") + +test "test2": + runTests("test2.test") + +test "test3": + runTests("test3.test") + +test "test4": + runTests("test4.test") + +test "unicodeChars": + runTests("unicodeChars.test") + +test "unicodeCharsProblematic": + runTests("unicodeCharsProblematic.test") + +#test "xmlViolation": +# runTests("xmlViolation.test") diff --git a/lib/chame0/test/tree.nim b/lib/chame0/test/tree.nim new file mode 100644 index 00000000..87731f62 --- /dev/null +++ b/lib/chame0/test/tree.nim @@ -0,0 +1,215 @@ +include shared/tree_common + +import std/streams + +proc runTest(test: TCTest, factory: MAtomFactory, scripting, print: bool) = + let ss = newStringStream(test.data) + let opts = HTML5ParserOpts[Node, MAtom]( + scripting: scripting + ) + let pdoc = if test.fragment.isNone: + parseHTML(ss, opts, factory) + else: + let ctx = Element() + ctx[] = test.fragment.get.ctx[] + let childList = parseHTMLFragment(ss, ctx, opts, factory) + for child in childList: + if ctx.preInsertionValidity(child, nil): + ctx.childList.add(child) + Document(childList: ctx.childList) + if print: + var ins = "" + for x in test.document.childList: + ins &= $x & '\n' + var ps = "" + for x in pdoc.childList: + ps &= $x & '\n' + echo "data ", test.data + echo "indoc ", $ins + echo "psdoc ", $ps + checkTest(test.document, pdoc) + +const rootpath = "test/html5lib-tests/tree-construction/" + +proc runTests(filename: string, print = false) = + let factory = newMAtomFactory() + let tests = parseTests(readFile(rootpath & filename), factory) + for test in tests: + case test.script + of SCRIPT_OFF: + test.runTest(factory, scripting = false, print = print) + of SCRIPT_ON: + test.runTest(factory, scripting = true, print = print) + of SCRIPT_BOTH: + test.runTest(factory, scripting = false, print = print) + test.runTest(factory, scripting = true, print = print) + +test "tests1.dat": + runTests("tests1.dat") + +test "tests2.dat": + runTests("tests2.dat") + +test "tests3.dat": + runTests("tests3.dat") + +test "tests4.dat": + runTests("tests4.dat") + +test "tests5.dat": + runTests("tests5.dat") + +test "tests6.dat": + runTests("tests6.dat") + +test "tests7.dat": + runTests("tests7.dat") + +test "tests8.dat": + runTests("tests8.dat") + +test "tests9.dat": + runTests("tests9.dat") + +test "tests10.dat": + runTests("tests10.dat") + +test "tests11.dat": + runTests("tests11.dat") + +test "tests12.dat": + runTests("tests12.dat") + +# no tests13 in html5lib-tests :) + +test "tests14.dat": + runTests("tests14.dat") + +test "tests15.dat": + runTests("tests15.dat") + +test "tests16.dat": + runTests("tests16.dat") + +test "tests17.dat": + runTests("tests17.dat") + +test "tests18.dat": + runTests("tests18.dat") + +test "tests19.dat": + runTests("tests19.dat") + +test "tests20.dat": + runTests("tests20.dat") + +test "tests21.dat": + runTests("tests21.dat") + +test "tests22.dat": + runTests("tests22.dat") + +test "tests23.dat": + runTests("tests23.dat") + +test "tests24.dat": + runTests("tests24.dat") + +test "tests25.dat": + runTests("tests25.dat") + +test "tests26.dat": + runTests("tests26.dat") + +test "adoption01.dat": + runTests("adoption01.dat") + +test "adoption02.dat": + runTests("adoption02.dat") + +test "blocks.dat": + runTests("blocks.dat") + +test "comments01.dat": + runTests("comments01.dat") + +test "doctype01.dat": + runTests("doctype01.dat") + +test "domjs-unsafe.dat": + runTests("domjs-unsafe.dat") + +test "entities01.dat": + runTests("entities01.dat") + +test "entities02.dat": + runTests("entities02.dat") + +test "foreign-fragment.dat": + runTests("foreign-fragment.dat") + +test "html5test-com.dat": + runTests("html5test-com.dat") + +test "inbody01.dat": + runTests("inbody01.dat") + +test "isindex.dat": + runTests("isindex.dat") + +test "main-element.dat": + runTests("main-element.dat") + +test "math.dat": + runTests("math.dat") + +test "menuitem-element.dat": + runTests("menuitem-element.dat") + +test "namespace-sensitivity.dat": + runTests("namespace-sensitivity.dat") + +test "noscript01.dat": + runTests("noscript01.dat") + +test "pending-spec-changes.dat": + runTests("pending-spec-changes.dat") + +test "pending-spec-changes-plain-text-unsafe.dat": + runTests("pending-spec-changes-plain-text-unsafe.dat") + +test "plain-text-unsafe.dat": + runTests("plain-text-unsafe.dat") + +test "quirks01.dat": + runTests("quirks01.dat") + +test "ruby.dat": + runTests("ruby.dat") + +test "scriptdata01.dat": + runTests("scriptdata01.dat") + +test "search-element.dat": + runTests("search-element.dat") + +test "svg.dat": + runTests("svg.dat") + +test "tables01.dat": + runTests("tables01.dat") + +test "template.dat": + runTests("template.dat") + +test "tests_innerHTML_1.dat": + runTests("tests_innerHTML_1.dat") + +test "tricky01.dat": + runTests("tricky01.dat") + +test "webkit01.dat": + runTests("webkit01.dat") + +test "webkit02.dat": + runTests("webkit02.dat") diff --git a/lib/chame0/test/tree_charset.nim b/lib/chame0/test/tree_charset.nim new file mode 100644 index 00000000..d9069c94 --- /dev/null +++ b/lib/chame0/test/tree_charset.nim @@ -0,0 +1,50 @@ +include shared/tree_common + +import std/streams +import chame/minidom_cs +import chagashi/charset + +proc runTest(test: TCTest, factory: MAtomFactory, scripting: bool, + labels: openArray[string]) = + let ss = newStringStream(test.data) + let opts = HTML5ParserOpts[Node, MAtom](scripting: scripting) + assert test.fragment.isNone + var charsets: seq[Charset] = @[] + for s in labels: + let cs = getCharset(s) + assert cs != CHARSET_UNKNOWN + charsets.add(cs) + let pdoc = parseHTML(ss, opts, charsets, factory = factory) + #[ + var ins = "" + for x in test.document.childList: + ins &= $x & '\n' + var ps = "" + for x in pdoc.childList: + ps &= $x & '\n' + echo "data ", test.data + echo "indoc ", $ins + echo "psdoc ", $ps + ]# + checkTest(test.document, pdoc) + +const rootpath = "test/" + +proc runTests(filename: string, labels: openArray[string]) = + let factory = newMAtomFactory() + let tests = parseTests(readFile(rootpath & filename), factory) + for test in tests: + case test.script + of SCRIPT_OFF: + test.runTest(factory, scripting = false, labels) + of SCRIPT_ON: + test.runTest(factory, scripting = true, labels) + of SCRIPT_BOTH: + test.runTest(factory, scripting = false, labels) + test.runTest(factory, scripting = true, labels) + +test "sjis.dat": + runTests("sjis.dat", ["utf8", "sjis", "latin1"]) + +test "latin1.dat": + runTests("latin1.dat", ["utf8"]) diff --git a/lib/chame0/test/tree_misc.nim b/lib/chame0/test/tree_misc.nim new file mode 100644 index 00000000..b26270cd --- /dev/null +++ b/lib/chame0/test/tree_misc.nim @@ -0,0 +1,39 @@ +include shared/tree_common + +import std/streams + +proc runTest(test: TCTest, factory: MAtomFactory, scripting: bool) = + let ss = newStringStream(test.data) + let opts = HTML5ParserOpts[Node, MAtom](scripting: scripting) + assert test.fragment.isNone + let pdoc = parseHTML(ss, opts, factory = factory) + #[ + var ins = "" + for x in test.document.childList: + ins &= $x & '\n' + var ps = "" + for x in pdoc.childList: + ps &= $x & '\n' + echo "data ", test.data + echo "indoc ", $ins + echo "psdoc ", $ps + ]# + checkTest(test.document, pdoc) + +const rootpath = "test/" + +proc runTests(filename: string) = + let factory = newMAtomFactory() + let tests = parseTests(readFile(rootpath & filename), factory) + for test in tests: + case test.script + of SCRIPT_OFF: + test.runTest(factory, scripting = false) + of SCRIPT_ON: + test.runTest(factory, scripting = true) + of SCRIPT_BOTH: + test.runTest(factory, scripting = false) + test.runTest(factory, scripting = true) + +test "gtrsim.dat": + runTests("gtrsim.dat") diff --git a/lib/chame0/todo b/lib/chame0/todo new file mode 100644 index 00000000..80ad5420 --- /dev/null +++ b/lib/chame0/todo @@ -0,0 +1,4 @@ +interface: +* reduce use of getNamespace (maybe store ns on the stack of open elements?) +etc: +* add getElementById and some more basic stuff to minidom diff --git a/lib/chame0/updver b/lib/chame0/updver new file mode 100755 index 00000000..eb02af93 --- /dev/null +++ b/lib/chame0/updver @@ -0,0 +1,59 @@ +#!/bin/sh + +die() +{ + echo "$*" 2>&1 + exit 1 +} + +test "$1" || die "Usage: updver [version]" + +vv=$(printf '%s\n' "$1" | sed 's/v//') + +major=$(printf '%s\n' "$vv" | sed 's/\..*//') +minor=$(printf '%s\n' "$vv" | sed 's/[^.]*\.\([^.]*\)\..*/\1/') +patch=$(printf '%s\n' "$vv" | sed 's/.*\.//') + +vs="$major.$minor.$patch" + +oldhdr=$(head -1 NEWS) + +msgfile=$(mktemp) + +printf '%s (%s)\n\n' "$vs" "$(date +'%Y.%m.%d')" > "$msgfile" +cat NEWS >> "$msgfile" + +test -n "$EDITOR" || die 'missing $EDITOR env var' +$EDITOR "$msgfile" + +printf 'Ok? (y/n) ' +read -r ok +case "$ok" in +y|Y) ;; +*) die "Aborted. (File is $msgfile.)" ;; +esac + +cp "$msgfile" NEWS +echo "/version/c +version = \"$vs\" +. +p +wq" | ed -s chame.nimble + +echo "/Major/s/[0-9]*$/$major/ +/Minor/s/[0-9]*$/$minor/ +/Patch/s/[0-9]*$/$patch/ +g/.*/p +wq" | ed -s chame/version.nim + +git add . +git commit -m "Version $vs" + +tmp2=$(mktemp) + +while read line +do if test "$line" = "$oldhdr"; then break; fi + printf '%s\n' "$line" +done <NEWS >"$tmp2" + +git tag -faeF "$tmp2" "v$vs" |