blob: 9d8843f064058414352b17879500fb7627299a6e (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
#
#
# Nim's Runtime Library
# (c) Copyright 2012 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## This module is based on Python's Unidecode module by Tomaz Solc,
## which in turn is based on the ``Text::Unidecode`` Perl module by
## Sean M. Burke
## (http://search.cpan.org/~sburke/Text-Unidecode-0.04/lib/Text/Unidecode.pm ).
##
## It provides a single proc that does Unicode to ASCII transliterations:
## It finds the sequence of ASCII characters that is the closest approximation
## to the Unicode string.
##
## For example, the closest to string "Äußerst" in ASCII is "Ausserst". Some
## information is lost in this transformation, of course, since several Unicode
## strings can be transformed in the same ASCII representation. So this is a
## strictly one-way transformation. However a human reader will probably
## still be able to guess what original string was meant from the context.
##
## This module needs the data file "unidecode.dat" to work: You can either
## ship this file with your application and initialize this module with the
## `loadUnidecodeTable` proc or you can define the ``embedUnidecodeTable``
## symbol to embed the file as a resource into your application.
import unicode
when defined(embedUnidecodeTable):
import strutils
const translationTable = splitLines(slurp"unidecode/unidecode.dat")
else:
# shared is fine for threading:
var translationTable: seq[string]
proc loadUnidecodeTable*(datafile = "unidecode.dat") =
## loads the datafile that `unidecode` to work. Unless this module is
## compiled with the ``embedUnidecodeTable`` symbol defined, this needs
## to be called by the main thread before any thread can make a call
## to `unidecode`.
when not defined(embedUnidecodeTable):
newSeq(translationTable, 0xffff)
var i = 0
for line in lines(datafile):
translationTable[i] = line.string
inc(i)
proc unidecode*(s: string): string =
## Finds the sequence of ASCII characters that is the closest approximation
## to the UTF-8 string `s`.
##
## Example:
##
## ..code-block:: nim
##
## unidecode("\x53\x17\x4E\xB0")
##
## Results in: "Bei Jing"
##
assert(not isNil(translationTable))
result = ""
for r in runes(s):
var c = int(r)
if c <=% 127: add(result, chr(c))
elif c <% translationTable.len: add(result, translationTable[c-128])
when isMainModule:
loadUnidecodeTable("lib/pure/unidecode/unidecode.dat")
assert unidecode("Äußerst") == "Ausserst"
|