| 1 |
(* File to be processed by wlex, not ocamllex ! *)
|
| 2 |
|
| 3 |
classes
|
| 4 |
encoding_error
|
| 5 |
xml_char
|
| 6 |
blank
|
| 7 |
lowercase uppercase ascii_digit
|
| 8 |
"#_<>=.,:;+-*/@&{}[]()|?`\"\\\'!"
|
| 9 |
|
| 10 |
unicode_base_char
|
| 11 |
unicode_ideographic
|
| 12 |
unicode_combining_char
|
| 13 |
unicode_digit
|
| 14 |
unicode_extender
|
| 15 |
|
| 16 |
|
| 17 |
{
|
| 18 |
let keywords = Hashtbl.create 17
|
| 19 |
|
| 20 |
let in_comment = ref false
|
| 21 |
|
| 22 |
let error = Location.raise_loc
|
| 23 |
exception Illegal_character of char
|
| 24 |
exception Unterminated_comment
|
| 25 |
exception Unterminated_string
|
| 26 |
exception Unterminated_string_in_comment
|
| 27 |
|
| 28 |
|
| 29 |
(* Buffer for string literals (always encoded in UTF8).
|
| 30 |
Each character is encoded in two consecutives code point;
|
| 31 |
the first one gives the number of bytes in the input document;
|
| 32 |
the second one gives the Unicode representation *)
|
| 33 |
|
| 34 |
let string_buff = Buffer.create 1024
|
| 35 |
|
| 36 |
let store_len ?(add=0) lexbuf =
|
| 37 |
let l = add + (Lexing.lexeme_end lexbuf) - (Lexing.lexeme_start lexbuf) in
|
| 38 |
Encodings.Utf8.store string_buff l
|
| 39 |
|
| 40 |
let store_ascii = Buffer.add_char string_buff
|
| 41 |
let store_char = Buffer.add_string string_buff
|
| 42 |
let store_code = Encodings.Utf8.store string_buff
|
| 43 |
let get_stored_string () =
|
| 44 |
let s = Buffer.contents string_buff in
|
| 45 |
Buffer.clear string_buff;
|
| 46 |
s
|
| 47 |
let store_special = function
|
| 48 |
| 'n' -> store_ascii '\n'
|
| 49 |
| 'r' -> store_ascii '\r'
|
| 50 |
| 't' -> store_ascii '\t'
|
| 51 |
| c -> raise (Illegal_character '\\')
|
| 52 |
|
| 53 |
let string_start_pos = ref 0;;
|
| 54 |
let comment_start_pos : int list ref = ref [];;
|
| 55 |
|
| 56 |
let numeric_char s =
|
| 57 |
int_of_string (String.sub s 1 (String.length s - 2))
|
| 58 |
|
| 59 |
|
| 60 |
let hexa_digit = function
|
| 61 |
| '0'..'9' as c -> (Char.code c) - (Char.code '0')
|
| 62 |
| 'a'..'f' as c -> (Char.code c) - (Char.code 'a') + 10
|
| 63 |
| _ -> failwith "Invalid hexadecimal digit" (* TODO: error loc *)
|
| 64 |
|
| 65 |
|
| 66 |
let hexa_char s =
|
| 67 |
let rec aux i accu =
|
| 68 |
if i = String.length s - 1 then accu
|
| 69 |
else aux (succ i) (accu * 16 + hexa_digit s.[i])
|
| 70 |
in
|
| 71 |
aux 0 0
|
| 72 |
|
| 73 |
let rec tag_of_tag s i =
|
| 74 |
match s.[i] with
|
| 75 |
| '\008' | '\009' | '\010' | '\013' | '\032' -> tag_of_tag s (i+1)
|
| 76 |
| _ -> String.sub s i (String.length s - i)
|
| 77 |
|
| 78 |
}
|
| 79 |
|
| 80 |
let letter = lowercase | uppercase | unicode_base_char | unicode_ideographic
|
| 81 |
let digit = ascii_digit | unicode_digit
|
| 82 |
let character = [ ^ encoding_error ]
|
| 83 |
|
| 84 |
let ncname_char =
|
| 85 |
letter | digit | [ ".-_" ] | unicode_combining_char | unicode_extender
|
| 86 |
let ncname = (letter | '_' ) ncname_char*
|
| 87 |
let qname = (ncname ':')? ncname
|
| 88 |
|
| 89 |
rule token = parse
|
| 90 |
blank+ { token engine lexbuf }
|
| 91 |
| qname
|
| 92 |
{
|
| 93 |
let s = Lexing.lexeme lexbuf in
|
| 94 |
if (s.[0] >= 'A') && (s.[0] <= 'Z')
|
| 95 |
then "UIDENT",s
|
| 96 |
else if Hashtbl.mem keywords s then "",s else "LIDENT",s
|
| 97 |
}
|
| 98 |
| '-'? ascii_digit+
|
| 99 |
{ "INT",Lexing.lexeme lexbuf }
|
| 100 |
| [ "<>=.,:;+-*/@&{}[]()|?`!" ]
|
| 101 |
| "->" | "::" | ";;" | "--" | ":=" | "\\" | "++"
|
| 102 |
| "{|" | "|}" | "<=" | ">=" | "<<" | ">>"
|
| 103 |
| ["?+*"] "?" | "#"
|
| 104 |
{ "",Lexing.lexeme lexbuf }
|
| 105 |
| "#" lowercase+ { "DIRECTIVE",Lexing.lexeme lexbuf }
|
| 106 |
| '"' | "'"
|
| 107 |
{ let string_start = Lexing.lexeme_start lexbuf in
|
| 108 |
string_start_pos := string_start;
|
| 109 |
let double_quote = Lexing.lexeme_char lexbuf 0 = '"' in
|
| 110 |
string (Lexing.lexeme lexbuf) engine lexbuf;
|
| 111 |
lexbuf.Lexing.lex_start_pos <-
|
| 112 |
string_start - lexbuf.Lexing.lex_abs_pos;
|
| 113 |
(if double_quote then "STRING2" else "STRING1"),
|
| 114 |
(get_stored_string()) }
|
| 115 |
|
| 116 |
| "(*"
|
| 117 |
{ comment_start_pos := [Lexing.lexeme_start lexbuf];
|
| 118 |
in_comment := true;
|
| 119 |
comment engine lexbuf;
|
| 120 |
in_comment := false;
|
| 121 |
token engine lexbuf }
|
| 122 |
|
| 123 |
| eof
|
| 124 |
{ "EOI","" }
|
| 125 |
| _
|
| 126 |
{ error
|
| 127 |
(Lexing.lexeme_start lexbuf) (Lexing.lexeme_end lexbuf)
|
| 128 |
(Illegal_character ((Lexing.lexeme lexbuf).[0])) }
|
| 129 |
|
| 130 |
and comment = parse
|
| 131 |
"(*"
|
| 132 |
{ comment_start_pos := Lexing.lexeme_start lexbuf :: !comment_start_pos;
|
| 133 |
comment engine lexbuf;
|
| 134 |
}
|
| 135 |
| "*)"
|
| 136 |
{ comment_start_pos := List.tl !comment_start_pos;
|
| 137 |
if !comment_start_pos <> [] then comment engine lexbuf;
|
| 138 |
}
|
| 139 |
| '"' | "'"
|
| 140 |
{ string_start_pos := Lexing.lexeme_start lexbuf;
|
| 141 |
Buffer.clear string_buff;
|
| 142 |
let ender = Lexing.lexeme lexbuf in
|
| 143 |
(try string ender engine lexbuf
|
| 144 |
with Location.Location (_,_,Unterminated_string) ->
|
| 145 |
let st = List.hd !comment_start_pos in
|
| 146 |
error st (st+2) Unterminated_string_in_comment);
|
| 147 |
Buffer.clear string_buff;
|
| 148 |
comment engine lexbuf }
|
| 149 |
| eof
|
| 150 |
{ let st = List.hd !comment_start_pos in
|
| 151 |
error st (st+2) Unterminated_comment
|
| 152 |
}
|
| 153 |
| _
|
| 154 |
{ comment engine lexbuf }
|
| 155 |
|
| 156 |
and string ender = parse
|
| 157 |
| '"' | "'"
|
| 158 |
{ let c = Lexing.lexeme lexbuf in
|
| 159 |
if c = ender then ()
|
| 160 |
else (store_len lexbuf;
|
| 161 |
store_char (Lexing.lexeme lexbuf);
|
| 162 |
string ender engine lexbuf) }
|
| 163 |
| '\\' ['\\' '"' '\'']
|
| 164 |
{ store_len lexbuf;
|
| 165 |
store_ascii (Lexing.lexeme_char lexbuf 1);
|
| 166 |
string ender engine lexbuf }
|
| 167 |
| '\\' lowercase
|
| 168 |
{ let c = Lexing.lexeme_char lexbuf 1 in
|
| 169 |
if c = 'x'
|
| 170 |
then parse_hexa_char engine lexbuf
|
| 171 |
else (store_len lexbuf; store_special c);
|
| 172 |
string ender engine lexbuf }
|
| 173 |
| '\\' ascii_digit+ ';'
|
| 174 |
{ store_len lexbuf;
|
| 175 |
store_code (numeric_char (Lexing.lexeme lexbuf));
|
| 176 |
string ender engine lexbuf }
|
| 177 |
| '\\'
|
| 178 |
{ error
|
| 179 |
(Lexing.lexeme_start lexbuf) (Lexing.lexeme_end lexbuf)
|
| 180 |
(Illegal_character '\\') }
|
| 181 |
| eof
|
| 182 |
{ error !string_start_pos (!string_start_pos+1) Unterminated_string }
|
| 183 |
| _
|
| 184 |
{ store_len lexbuf;
|
| 185 |
store_code (Char.code (Lexing.lexeme_char lexbuf 0));
|
| 186 |
(* Adapt when source is UTF8 *)
|
| 187 |
string ender engine lexbuf }
|
| 188 |
|
| 189 |
and parse_hexa_char = parse
|
| 190 |
| (ascii_digit|lowercase)+ ';'
|
| 191 |
{ store_len ~add:2 lexbuf;
|
| 192 |
store_code (hexa_char (Lexing.lexeme lexbuf)) }
|
| 193 |
| _
|
| 194 |
{ error
|
| 195 |
(Lexing.lexeme_start lexbuf) (Lexing.lexeme_end lexbuf)
|
| 196 |
(Illegal_character '\\') }
|
| 197 |
|
| 198 |
|
| 199 |
|
| 200 |
{
|
| 201 |
|
| 202 |
let delta_loc = ref 0
|
| 203 |
let set_delta_loc dl = delta_loc := dl
|
| 204 |
|
| 205 |
(* For synchronization on errors in the toplevel ... *)
|
| 206 |
(* Issue: file inclusion *)
|
| 207 |
let lexbuf = ref None
|
| 208 |
let last_tok = ref ("","")
|
| 209 |
|
| 210 |
let lexer_func_of_wlex lexfun lexengine cs =
|
| 211 |
let dl = !delta_loc in
|
| 212 |
delta_loc := 0;
|
| 213 |
let lb =
|
| 214 |
Lexing.from_function
|
| 215 |
(fun s n ->
|
| 216 |
try s.[0] <- Stream.next cs; 1
|
| 217 |
with Stream.Failure -> 0)
|
| 218 |
in
|
| 219 |
lexbuf := Some lb;
|
| 220 |
let next () =
|
| 221 |
let tok = lexfun lexengine lb in
|
| 222 |
let loc = (Lexing.lexeme_start lb + dl,
|
| 223 |
Lexing.lexeme_end lb + dl) in
|
| 224 |
last_tok := tok;
|
| 225 |
(tok, loc)
|
| 226 |
in
|
| 227 |
Token.make_stream_and_location next
|
| 228 |
|
| 229 |
let register_kw (s1,s2) =
|
| 230 |
if s1 = "" then
|
| 231 |
match s2.[0] with
|
| 232 |
| 'a' .. 'z' when not (Hashtbl.mem keywords s2) ->
|
| 233 |
Hashtbl.add keywords s2 ()
|
| 234 |
| _ -> ()
|
| 235 |
|
| 236 |
|
| 237 |
let lexer lexfun lexengine =
|
| 238 |
{
|
| 239 |
Token.tok_func = lexer_func_of_wlex lexfun lexengine;
|
| 240 |
Token.tok_using = register_kw;
|
| 241 |
Token.tok_removing = (fun _ -> ());
|
| 242 |
Token.tok_match = Token.default_match;
|
| 243 |
Token.tok_text = Token.lexer_text
|
| 244 |
}
|
| 245 |
|
| 246 |
let classes =
|
| 247 |
let c i = (i,i) in
|
| 248 |
let i ch1 ch2 = (Char.code ch1, Char.code ch2) in
|
| 249 |
[ unicode_base_char,
|
| 250 |
[ 0x00C0,0x00D6; 0x00D8,0x00F6;
|
| 251 |
0x00F8,0x00FF; 0x0100,0x0131; 0x0134,0x013E; 0x0141,0x0148;
|
| 252 |
0x014A,0x017E; 0x0180,0x01C3; 0x01CD,0x01F0; 0x01F4,0x01F5;
|
| 253 |
0x01FA,0x0217; 0x0250,0x02A8; 0x02BB,0x02C1; 0x0386,0x0386;
|
| 254 |
0x0388,0x038A; 0x038C,0x038C; 0x038E,0x03A1; 0x03A3,0x03CE;
|
| 255 |
0x03D0,0x03D6; 0x03DA,0x03DA; 0x03DC,0x03DC; 0x03DE,0x03DE;
|
| 256 |
0x03E0,0x03E0; 0x03E2,0x03F3;
|
| 257 |
0x0401,0x040C; 0x040E,0x044F; 0x0451,0x045C; 0x045E,0x0481;
|
| 258 |
0x0490,0x04C4; 0x04C7,0x04C8; 0x04CB,0x04CC; 0x04D0,0x04EB;
|
| 259 |
0x04EE,0x04F5; 0x04F8,0x04F9; 0x0531,0x0556; 0x0559,0x0559;
|
| 260 |
0x0561,0x0586; 0x05D0,0x05EA; 0x05F0,0x05F2; 0x0621,0x063A;
|
| 261 |
0x0641,0x064A; 0x0671,0x06B7; 0x06BA,0x06BE; 0x06C0,0x06CE;
|
| 262 |
0x06D0,0x06D3; 0x06D5,0x06D5; 0x06E5,0x06E6; 0x0905,0x0939;
|
| 263 |
0x093D,0x093D;
|
| 264 |
0x0958,0x0961; 0x0985,0x098C; 0x098F,0x0990; 0x0993,0x09A8;
|
| 265 |
0x09AA,0x09B0; 0x09B2,0x09B2; 0x09B6,0x09B9; 0x09DC,0x09DD;
|
| 266 |
0x09DF,0x09E1; 0x09F0,0x09F1; 0x0A05,0x0A0A; 0x0A0F,0x0A10;
|
| 267 |
0x0A13,0x0A28; 0x0A2A,0x0A30; 0x0A32,0x0A33; 0x0A35,0x0A36;
|
| 268 |
0x0A38,0x0A39; 0x0A59,0x0A5C; 0x0A5E,0x0A5E; 0x0A72,0x0A74;
|
| 269 |
0x0A85,0x0A8B; 0x0A8D,0x0A8D; 0x0A8F,0x0A91; 0x0A93,0x0AA8;
|
| 270 |
0x0AAA,0x0AB0; 0x0AB2,0x0AB3; 0x0AB5,0x0AB9; 0x0ABD,0x0ABD;
|
| 271 |
0x0AE0,0x0AE0;
|
| 272 |
0x0B05,0x0B0C; 0x0B0F,0x0B10; 0x0B13,0x0B28; 0x0B2A,0x0B30;
|
| 273 |
0x0B32,0x0B33; 0x0B36,0x0B39; 0x0B3D,0x0B3D; 0x0B5C,0x0B5D;
|
| 274 |
0x0B5F,0x0B61; 0x0B85,0x0B8A; 0x0B8E,0x0B90; 0x0B92,0x0B95;
|
| 275 |
0x0B99,0x0B9A; 0x0B9C,0x0B9C; 0x0B9E,0x0B9F; 0x0BA3,0x0BA4;
|
| 276 |
0x0BA8,0x0BAA; 0x0BAE,0x0BB5; 0x0BB7,0x0BB9; 0x0C05,0x0C0C;
|
| 277 |
0x0C0E,0x0C10; 0x0C12,0x0C28; 0x0C2A,0x0C33; 0x0C35,0x0C39;
|
| 278 |
0x0C60,0x0C61; 0x0C85,0x0C8C; 0x0C8E,0x0C90; 0x0C92,0x0CA8;
|
| 279 |
0x0CAA,0x0CB3; 0x0CB5,0x0CB9; 0x0CDE,0x0CDE; 0x0CE0,0x0CE1;
|
| 280 |
0x0D05,0x0D0C; 0x0D0E,0x0D10; 0x0D12,0x0D28; 0x0D2A,0x0D39;
|
| 281 |
0x0D60,0x0D61; 0x0E01,0x0E2E; 0x0E30,0x0E30; 0x0E32,0x0E33;
|
| 282 |
0x0E40,0x0E45; 0x0E81,0x0E82; 0x0E84,0x0E84; 0x0E87,0x0E88;
|
| 283 |
0x0E8A,0x0E8A;
|
| 284 |
0x0E8D,0x0E8D; 0x0E94,0x0E97; 0x0E99,0x0E9F; 0x0EA1,0x0EA3;
|
| 285 |
0x0EA5,0x0EA5;
|
| 286 |
0x0EA7,0x0EA7; 0x0EAA,0x0EAB; 0x0EAD,0x0EAE; 0x0EB0,0x0EB0;
|
| 287 |
0x0EB2,0x0EB3;
|
| 288 |
0x0EBD,0x0EBD; 0x0EC0,0x0EC4; 0x0F40,0x0F47; 0x0F49,0x0F69;
|
| 289 |
0x10A0,0x10C5; 0x10D0,0x10F6; 0x1100,0x1100; 0x1102,0x1103;
|
| 290 |
0x1105,0x1107; 0x1109,0x1109; 0x110B,0x110C; 0x110E,0x1112;
|
| 291 |
0x113C,0x113C;
|
| 292 |
0x113E,0x113E; 0x1140,0x1140; 0x114C,0x114C; 0x114E,0x114E;
|
| 293 |
0x1150,0x1150; 0x1154,0x1155; 0x1159,0x1159;
|
| 294 |
0x115F,0x1161; 0x1163,0x1163; 0x1165,0x1165; 0x1167,0x1167;
|
| 295 |
0x1169,0x1169; 0x116D,0x116E;
|
| 296 |
0x1172,0x1173; 0x1175,0x1175; 0x119E,0x119E; 0x11A8,0x11A8;
|
| 297 |
0x11AB,0x11AB; 0x11AE,0x11AF;
|
| 298 |
0x11B7,0x11B8; 0x11BA,0x11BA; 0x11BC,0x11C2; 0x11EB,0x11EB;
|
| 299 |
0x11F0,0x11F0; 0x11F9,0x11F9;
|
| 300 |
0x1E00,0x1E9B; 0x1EA0,0x1EF9; 0x1F00,0x1F15; 0x1F18,0x1F1D;
|
| 301 |
0x1F20,0x1F45; 0x1F48,0x1F4D; 0x1F50,0x1F57; 0x1F59,0x1F59;
|
| 302 |
0x1F5B,0x1F5B;
|
| 303 |
0x1F5D,0x1F5D; 0x1F5F,0x1F7D; 0x1F80,0x1FB4; 0x1FB6,0x1FBC;
|
| 304 |
0x1FBE,0x1FBE;
|
| 305 |
0x1FC2,0x1FC4; 0x1FC6,0x1FCC; 0x1FD0,0x1FD3; 0x1FD6,0x1FDB;
|
| 306 |
0x1FE0,0x1FEC; 0x1FF2,0x1FF4; 0x1FF6,0x1FFC; 0x2126,0x2126;
|
| 307 |
0x212A,0x212B; 0x212E,0x212E; 0x2180,0x2182; 0x3041,0x3094;
|
| 308 |
0x30A1,0x30FA; 0x3105,0x312C; (* 0xAC00,0xD7A3 *) ];
|
| 309 |
|
| 310 |
unicode_ideographic,
|
| 311 |
[ 0x3007,0x3007; 0x3021,0x3029 (* 0x4E00-0x9FA5 *) ];
|
| 312 |
|
| 313 |
unicode_combining_char,
|
| 314 |
[ 0x0300,0x0345; 0x0360,0x0361; 0x0483,0x0486; 0x0591,0x05A1;
|
| 315 |
0x05A3,0x05B9; 0x05BB,0x05BD; 0x05BF,0x05BF; 0x05C1,0x05C2;
|
| 316 |
0x05C4,0x05C4; 0x064B,0x0652; 0x0670,0x0670; 0x06D6,0x06DC;
|
| 317 |
0x06DD,0x06DF; 0x06E0,0x06E4; 0x06E7,0x06E8; 0x06EA,0x06ED;
|
| 318 |
0x0901,0x0903; 0x093C,0x093C; 0x093E,0x094C; 0x094D,0x094D;
|
| 319 |
0x0951,0x0954; 0x0962,0x0963; 0x0981,0x0983; 0x09BC,0x09BC;
|
| 320 |
0x09BE,0x09BE; 0x09BF,0x09BF; 0x09C0,0x09C4; 0x09C7,0x09C8;
|
| 321 |
0x09CB,0x09CD; 0x09D7,0x09D7; 0x09E2,0x09E3; 0x0A02,0x0A02;
|
| 322 |
0x0A3C,0x0A3C; 0x0A3E,0x0A3E; 0x0A3F,0x0A3F; 0x0A40,0x0A42;
|
| 323 |
0x0A47,0x0A48; 0x0A4B,0x0A4D; 0x0A70,0x0A71; 0x0A81,0x0A83;
|
| 324 |
0x0ABC,0x0ABC; 0x0ABE,0x0AC5; 0x0AC7,0x0AC9; 0x0ACB,0x0ACD;
|
| 325 |
0x0B01,0x0B03; 0x0B3C,0x0B3C; 0x0B3E,0x0B43; 0x0B47,0x0B48;
|
| 326 |
0x0B4B,0x0B4D; 0x0B56,0x0B57; 0x0B82,0x0B83; 0x0BBE,0x0BC2;
|
| 327 |
0x0BC6,0x0BC8; 0x0BCA,0x0BCD; 0x0BD7,0x0BD7; 0x0C01,0x0C03;
|
| 328 |
0x0C3E,0x0C44; 0x0C46,0x0C48; 0x0C4A,0x0C4D; 0x0C55,0x0C56;
|
| 329 |
0x0C82,0x0C83; 0x0CBE,0x0CC4; 0x0CC6,0x0CC8; 0x0CCA,0x0CCD;
|
| 330 |
0x0CD5,0x0CD6; 0x0D02,0x0D03; 0x0D3E,0x0D43; 0x0D46,0x0D48;
|
| 331 |
0x0D4A,0x0D4D; 0x0D57,0x0D57; 0x0E31,0x0E31; 0x0E34,0x0E3A;
|
| 332 |
0x0E47,0x0E4E; 0x0EB1,0x0EB1; 0x0EB4,0x0EB9; 0x0EBB,0x0EBC;
|
| 333 |
0x0EC8,0x0ECD; 0x0F18,0x0F19; 0x0F35,0x0F35; 0x0F37,0x0F37;
|
| 334 |
0x0F39,0x0F39; 0x0F3E,0x0F3E; 0x0F3F,0x0F3F; 0x0F71,0x0F84;
|
| 335 |
0x0F86,0x0F8B; 0x0F90,0x0F95; 0x0F97,0x0F97; 0x0F99,0x0FAD;
|
| 336 |
0x0FB1,0x0FB7; 0x0FB9,0x0FB9; 0x20D0,0x20DC; 0x20E1,0x20E1;
|
| 337 |
0x302A,0x302F; 0x3099,0x3099; 0x309A,0x309A ];
|
| 338 |
|
| 339 |
unicode_digit,
|
| 340 |
[ 0x0660,0x0669; 0x06F0,0x06F9; 0x0966,0x096F; 0x09E6,0x09EF;
|
| 341 |
0x0A66,0x0A6F; 0x0AE6,0x0AEF; 0x0B66,0x0B6F; 0x0BE7,0x0BEF;
|
| 342 |
0x0C66,0x0C6F; 0x0CE6,0x0CEF; 0x0D66,0x0D6F; 0x0E50,0x0E59;
|
| 343 |
0x0ED0,0x0ED9; 0x0F20,0x0F29 ];
|
| 344 |
|
| 345 |
|
| 346 |
unicode_extender,
|
| 347 |
[ 0x00B7,0x00B7; 0x02D0,0x02D1; 0x0387,0x0387; 0x0640,0x0640;
|
| 348 |
0x0E46,0x0E46; 0x0EC6,0x0EC6; 0x3005,0x3005; 0x3031,0x3035;
|
| 349 |
0x309D,0x309E; 0x30FC,0x30FE ];
|
| 350 |
|
| 351 |
ascii_digit,
|
| 352 |
[ i '0' '9'];
|
| 353 |
|
| 354 |
lowercase,
|
| 355 |
[i 'a' 'z'];
|
| 356 |
|
| 357 |
uppercase,
|
| 358 |
[i 'A' 'Z'];
|
| 359 |
|
| 360 |
blank,
|
| 361 |
[c 8; c 9; c 10; c 13; c 32]
|
| 362 |
]
|
| 363 |
|
| 364 |
let table =
|
| 365 |
assert(nb_classes <= 256);
|
| 366 |
let v = String.make 0x312d (Char.chr encoding_error) in
|
| 367 |
let fill_int c (i, j) = String.fill v i (j-i+1) c in
|
| 368 |
let fill_class (c, l) = List.iter (fill_int (Char.chr c)) l in
|
| 369 |
let fill_char (ch, cl) = v.[ch] <- Char.chr cl in
|
| 370 |
List.iter fill_class classes;
|
| 371 |
List.iter fill_char one_char_classes;
|
| 372 |
v
|
| 373 |
|
| 374 |
let utf8_engine =
|
| 375 |
Lex_engines.engine_tiny_utf8 table
|
| 376 |
(fun code ->
|
| 377 |
if code >= 0x4E00 && code <= 0x9FA5 then
|
| 378 |
unicode_ideographic
|
| 379 |
else if code >= 0xAC00 && code <= 0xD7A3 then
|
| 380 |
unicode_base_char
|
| 381 |
else if code <= 0xD7FF || (code >= 0xE000 && code <= 0xFFFD) ||
|
| 382 |
(code >= 0x10000 && code <= 0x10FFFF) then
|
| 383 |
xml_char
|
| 384 |
else encoding_error)
|
| 385 |
|
| 386 |
let latin1_engine = Lex_engines.engine_tiny_8bit table
|
| 387 |
}
|
Parent Directory
| 
Revision Log