forked from latex3/latex2e
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathload-unicode-data.tex
244 lines (244 loc) · 8.61 KB
/
load-unicode-data.tex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
% File load-unicode-data.tex
%
% Copyright 2015,2016 The LaTeX3 Project
%
% It may be distributed and/or modified under the conditions of
% the LaTeX Project Public License (LPPL), either version 1.3c of
% this license or (at your option) any later version. The latest
% version of this license is in the file
% http://www.latex-project.org/lppl.txt.
%
% Issues with this file should be reported at
% https://github.com/latex3/unicode-data
%
% This file parses a number of data files provided by the Unicode Consortium
% and when used with used Unicode-capable engine sets up a range of TeX-related
% parameters based on the extracted information.
%
% From the file UnicodeData.txt the following properties are set:
% - \catcode 11 for all letters (Unicode class "L")
% - \catcode 11 for all combining marks (Unicode class "M")
% - \sfcode 999 for all code points of class "Lu" (upper case letters)
% - \lccode for all of class "Ll" (lower case letters) to the code point
% itself, and \uccode to the upper case mapping (or if not given
% to the code point itself)
% - \uccode for all of class "Lu" (upper case letters) to the code point
% itself, and \lccode to the lower case mapping (or if not given
% to the code point itself)
% - \lccode and \uccode for all of class "Lt" (title case letters) to the
% lower and upper case mappings (or if not given to the code point itself)
% - \lccode and \uccode for all other letter code points are set to
% the code point itself
% - \lccode and/or \uccode for non-letter code points for which an upper
% or lower case mapping is given
% - \sfcode 0 (ignored) for code points of Unicode classes "Pe" (closing
% punctuation marks) and "Pf" (final quotation marks)
% - \Umathcode for all letters in the base plane class 7 (var) and for
% the supplementary plane class 0 (regular)
%
% =============================================================================
%
% The data can only be loaded by Unicode engines. Currently this is limited to
% XeTeX and LuaTeX, both of which define \Umathcode.
\ifx\Umathcode\undefined
\expandafter\endinput
\fi
% Just in case, check for the e-TeX extensions.
\ifx\eTeXversion\undefined
\expandafter\endinput
\fi
% This file can be loaded in IniTeX mode so the category codes of |{|, |}| and
% |#| may not be correct. Everything is done in a group so that only the
% settings we want to propagate are made available generally.
\begingroup
\catcode`\{=1 %
\catcode`\}=2 %
\catcode`\#=6 %
% Write some basic information to the log.
\catcode`\^=7 %
\newlinechar=`\^^J %
\message{^^J}%
\message{load-unicode-data.tex v1.2 (2016-02-02)^^J}%
\message{Reading Unicode data^^J}%
% The first stage of parsing is dealing with the fact that there are lots of
% data items separated by |;|. Of those, only a few are needed so they are
% picked out and everything else is dropped. There is one complication: there
% are a few cases in the data file of ranges which are marked by the descriptor
% |First| and a matching |Last|. A separate routine is used to handle these
% cases.
\def\parseunicodedataI#1;#2;#3;#4;#5;#6;#7;#8;#9;{%
\parseunicodedataII#1;#3;#2 First>\relax
}%
\def\parseunicodedataII#1;#2;#3 First>#4\relax{%
\ifx\relax#4\relax
\expandafter\parseunicodedataIII
\else
\expandafter\parseunicodedataVII
\fi
#1;#2;%
}%
\def\parseunicodedataIII#1;#2;#3;#4;#5;#6;#7;#8\relax{%
\parseunicodedataIV{#1}{#2}{#6}{#7}%
}%
% At this stage we have a `normal' data line with four pieces of information:
% the code point, the Unicode class and the (possibly empty) upper and lower
% case mappings. A few utility macros are defined, then we branch based on the
% Unicode class. Notice that for all letter-like code points we first set the
% |\lccode| and |\uccode| values to the code point itself then test for the
% classes where a different setting might be appropriate. For non-letters
% there is a check to see if any mappings are available, and also for trailing
% punctuation to set the appropriate |\sfcode|.
\def\Ll{Ll}%
\def\Lt{Lt}%
\def\Lu{Lu}%
\def\Pe{Pe}%
\def\Pf{Pf}%
\def\firsttoken#1#2\relax{#1}%
\def\parseunicodedataIV#1#2#3#4{%
\ifnum 0%
\if L\firsttoken#2?\relax 1\fi
\if M\firsttoken#2?\relax 1\fi
>0 %
\parseunicodedataV{"#1}%
\def\temp{#2}%
\ifx\Ll\temp
\parseunicodedataVI\uccode{#1}{#3}%
\fi
\ifx\Lt\temp
\parseunicodedataVI\uccode{#1}{#3}%
\parseunicodedataVI\lccode{#1}{#4}%
\fi
\ifx\Lu\temp
\parseunicodedataVI\lccode{#1}{#4}%
\global\sfcode"#1=999 %
\fi
% Letters in base plane are class~$7$, those in the supplementary plane are
% class~$0$.
\ifnum"#1<10000 %
\global\Umathcode"#1="7"01"#1 %
\else
\global\Umathcode"#1="0"01"#1 %
\fi
\else
\def\temp{#2}%
\ifnum 0\ifx\temp\Pe 1\fi\ifx\temp\Pf 1\fi>0 %
\global\sfcode"#1=0 %
\fi
\ifx\relax#3\relax
\else
\global\uccode"#1="#3 %
\fi
\ifx\relax#4\relax
\else
\global\lccode"#1="#4 %
\fi
\fi
}%
% A simple auxiliary for all letter-like code points: the |\lccode| and
% |\uccode| may get reset for cased letters but this means the initial
% setting can't be forgotten.
\def\parseunicodedataV#1{%
\global\catcode#1=11 %
\global\lccode#1=#1 %
\global\uccode#1=#1 %
}%
% An auxiliary to deal with the fact that some cased letters don't actually
% have a case mapping available.
\def\parseunicodedataVI#1#2#3{%
\ifx\relax#3\relax
\else
\global#1"#2="#3 %
\fi
}%
% For lines that were the |First>| of a range, read the data source again for
% last line. Lines for letters then trigger a loop over the entire range. These
% are always non-cased letters.
\def\parseunicodedataVII#1;#2;#3\relax{%
\read0 to \unicodedataline
\expandafter\parseunicodedataXII\unicodedataline\relax#1;#2\relax
}%
\def\parseunicodedataXII#1;#2\relax#3;#4\relax{%
\if L\firsttoken#4?\relax
\begingroup
\count0="#3 %
\loop
\unless\ifnum\count0>"#1 %
\parseunicodedataV{\count0 }%
\advance\count0 by 1 %
\repeat
\endgroup
\fi
}%
% From plain: may not be defined (yet).
\def\loop#1\repeat{\def\body{#1}\iterate}%
\def\iterate{%
\body
\let\next\iterate
\else
\let\next\relax
\fi
\next
}%
\let\repeat\fi
% There is no version data in |UnicodeData.txt|: log that it is being used with
% a hard-coded date (when the download took place). This obviously needs to be
% updated when a new download takes place!
\message{\string# UnicodeData-8.0.0.txt^^J}%
\message{\string# Downloaded 2015-12-01 09:00:00 GMT [JAW]^^J}%
% Actually loading the file requires an input stream, done directly.
% There is a blank line at the end of the data source so there is a check
% here for a |\par|.
\def\storedpar{\par}%
\openin0=UnicodeData.txt %
\loop\unless\ifeof0 %
\read0 to \unicodedataline
\unless\ifx\unicodedataline\storedpar
\expandafter\parseunicodedataI\unicodedataline\relax
\fi
\repeat
\closein0 %
% All of the other data files have some common aspects to their format. We
% therefore begin with some shared code. First a check for a comment line:
% these can be skipped. (Currently only |MathClass.txt| is used by this code
% is also usable with the other Unicode data files.)
\edef\hash{\string#}%
\def\parseunicodedataI#1\relax{%
\unless\if\hash\firsttoken#1?\relax
\parseunicodedataII#1\relax
\fi
}%
% The first entry in all of the files is a code point or range of code points:
% set up to find a range. The definition of |\parseunicodedataIV| will depend on
% the data being processed and may need to split the remainder of the line
% further.
\def\parseunicodedataII#1;#2\relax{%
\parseunicodedataIII#1....\relax{#2}%
}%
\def\parseunicodedataIII#1..#2..#3\relax#4{%
\ifx\relax#2\relax
\parseunicodedataIV{#1}{#1}#4\relax
\else
\parseunicodedataIV{#1}{#2}#4\relax
\fi
}%
% A shared routine for reading the data files: only one part of the parser
% has to be altered.
\def\readandparse#1{%
\catcode`\#=12 %
\openin0=#1.txt %
% Read two lines from the source file to extract the version information: it is
% always the first two lines of the file.
\read0 to \unicodedataline
\message{\unicodedataline ^^J}%
\read0 to \unicodedataline
\message{\unicodedataline ^^J}%
\loop\unless\ifeof0 %
\read0 to \unicodedataline
\unless\ifx\unicodedataline\storedpar
\expandafter\parseunicodedataI\unicodedataline\relax
\fi
\repeat
\catcode`\#=6 %
\closein0 %
}%
\endgroup