2 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4 * Copyright 1998-2005 The OpenLDAP Foundation.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted only as authorized by the OpenLDAP
11 * A copy of this license is available in file LICENSE in the
12 * top-level directory of the distribution or, alternatively, at
13 * <http://www.OpenLDAP.org/license.html>.
15 /* Copyright 2001 Computing Research Labs, New Mexico State University
17 * Permission is hereby granted, free of charge, to any person obtaining a
18 * copy of this software and associated documentation files (the "Software"),
19 * to deal in the Software without restriction, including without limitation
20 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
21 * and/or sell copies of the Software, and to permit persons to whom the
22 * Software is furnished to do so, subject to the following conditions:
24 * The above copyright notice and this permission notice shall be included in
25 * all copies or substantial portions of the Software.
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
30 * THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY
31 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
32 * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
33 * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
35 /* $Id: ucgendat.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
38 #include "ldap_config.h"
42 #include <ac/stdlib.h>
43 #include <ac/string.h>
44 #include <ac/unistd.h>
51 #define HARDCODE_DATA 1
55 #define ishdigit(cc) (((cc) >= '0' && (cc) <= '9') ||\
56 ((cc) >= 'A' && (cc) <= 'F') ||\
57 ((cc) >= 'a' && (cc) <= 'f'))
60 * A header written to the output file with the byte-order-mark and the number
63 static ac_uint2 hdr[2] = {0xfeff, 0};
66 #define NEEDPROPS (NUMPROPS + (4 - (NUMPROPS & 3)))
74 * List of properties expected to be found in the Unicode Character Database
75 * including some implementation specific properties.
77 * The implementation specific properties are:
78 * Cm = Composed (can be decomposed)
80 * Sy = Symmetric (has left and right forms)
85 * Cp = Defined character
87 static _prop_t props[NUMPROPS] = {
88 {"Mn", 2}, {"Mc", 2}, {"Me", 2}, {"Nd", 2}, {"Nl", 2}, {"No", 2},
89 {"Zs", 2}, {"Zl", 2}, {"Zp", 2}, {"Cc", 2}, {"Cf", 2}, {"Cs", 2},
90 {"Co", 2}, {"Cn", 2}, {"Lu", 2}, {"Ll", 2}, {"Lt", 2}, {"Lm", 2},
91 {"Lo", 2}, {"Pc", 2}, {"Pd", 2}, {"Ps", 2}, {"Pe", 2}, {"Po", 2},
92 {"Sm", 2}, {"Sc", 2}, {"Sk", 2}, {"So", 2}, {"L", 1}, {"R", 1},
93 {"EN", 2}, {"ES", 2}, {"ET", 2}, {"AN", 2}, {"CS", 2}, {"B", 1},
94 {"S", 1}, {"WS", 2}, {"ON", 2},
95 {"Cm", 2}, {"Nb", 2}, {"Sy", 2}, {"Hd", 2}, {"Qm", 2}, {"Mr", 2},
96 {"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}, {"AL", 2}
105 static _ranges_t proptbl[NUMPROPS];
108 * Make sure this array is sized to be on a 4-byte boundary at compile time.
110 static ac_uint2 propcnt[NEEDPROPS];
113 * Array used to collect a decomposition before adding it to the decomposition
116 static ac_uint4 dectmp[64];
117 static ac_uint4 dectmp_size;
127 * List of decomposition. Created and expanded in order as the characters are
128 * encountered. First list contains canonical mappings, second also includes
129 * compatibility mappings.
131 static _decomp_t *decomps;
132 static ac_uint4 decomps_used;
133 static ac_uint4 decomps_size;
135 static _decomp_t *kdecomps;
136 static ac_uint4 kdecomps_used;
137 static ac_uint4 kdecomps_size;
140 * Composition exclusion table stuff.
142 #define COMPEX_SET(c) (compexs[(c) >> 5] |= (1 << ((c) & 31)))
143 #define COMPEX_TEST(c) (compexs[(c) >> 5] & (1 << ((c) & 31)))
144 static ac_uint4 compexs[8192];
147 * Struct for holding a composition pair, and array of composition pairs
156 static _comp_t *comps;
157 static ac_uint4 comps_used;
160 * Types and lists for handling lists of case mappings.
168 static _case_t *upper;
169 static _case_t *lower;
170 static _case_t *title;
171 static ac_uint4 upper_used;
172 static ac_uint4 upper_size;
173 static ac_uint4 lower_used;
174 static ac_uint4 lower_size;
175 static ac_uint4 title_used;
176 static ac_uint4 title_size;
179 * Array used to collect case mappings before adding them to a list.
181 static ac_uint4 cases[3];
184 * An array to hold ranges for combining classes.
186 static ac_uint4 *ccl;
187 static ac_uint4 ccl_used;
188 static ac_uint4 ccl_size;
191 * Structures for handling numbers.
204 * Arrays to hold the mapping of codes to numbers.
206 static _codeidx_t *ncodes;
207 static ac_uint4 ncodes_used;
208 static ac_uint4 ncodes_size;
211 static ac_uint4 nums_used;
212 static ac_uint4 nums_size;
215 * Array for holding numbers.
218 static ac_uint4 nums_used;
219 static ac_uint4 nums_size;
222 add_range(ac_uint4 start, ac_uint4 end, char *p1, char *p2)
228 for (k = 0; k < 2; k++) {
240 for (i = 0; i < NUMPROPS; i++) {
241 if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
251 * Resize the range list if necessary.
253 if (rlp->used == rlp->size) {
255 rlp->ranges = (ac_uint4 *)
256 malloc(sizeof(ac_uint4) << 3);
258 rlp->ranges = (ac_uint4 *)
259 realloc((char *) rlp->ranges,
260 sizeof(ac_uint4) * (rlp->size + 8));
265 * If this is the first code for this property list, just add it
268 if (rlp->used == 0) {
269 rlp->ranges[0] = start;
270 rlp->ranges[1] = end;
276 * Optimize the case of adding the range to the end.
279 if (start > rlp->ranges[j]) {
281 rlp->ranges[j++] = start;
282 rlp->ranges[j++] = end;
288 * Need to locate the insertion point.
291 i < rlp->used && start > rlp->ranges[i + 1] + 1; i += 2) ;
294 * If the start value lies in the current range, then simply set the
295 * new end point of the range to the end value passed as a parameter.
297 if (rlp->ranges[i] <= start && start <= rlp->ranges[i + 1] + 1) {
298 rlp->ranges[i + 1] = end;
303 * Shift following values up by two.
305 for (j = rlp->used; j > i; j -= 2) {
306 rlp->ranges[j] = rlp->ranges[j - 2];
307 rlp->ranges[j + 1] = rlp->ranges[j - 1];
311 * Add the new range at the insertion point.
313 rlp->ranges[i] = start;
314 rlp->ranges[i + 1] = end;
320 ordered_range_insert(ac_uint4 c, char *name, int len)
330 * Deal with directionality codes introduced in Unicode 3.0.
332 if ((len == 2 && memcmp(name, "BN", 2) == 0) ||
334 (memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
335 memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
336 memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0))) {
338 * Mark all of these as Other Neutral to preserve compatibility with
345 for (i = 0; i < NUMPROPS; i++) {
346 if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
354 * Have a match, so insert the code in order.
359 * Resize the range list if necessary.
361 if (rlp->used == rlp->size) {
363 rlp->ranges = (ac_uint4 *)
364 malloc(sizeof(ac_uint4) << 3);
366 rlp->ranges = (ac_uint4 *)
367 realloc((char *) rlp->ranges,
368 sizeof(ac_uint4) * (rlp->size + 8));
373 * If this is the first code for this property list, just add it
376 if (rlp->used == 0) {
377 rlp->ranges[0] = rlp->ranges[1] = c;
383 * Optimize the cases of extending the last range and adding new ranges to
388 s = rlp->ranges[j - 1];
392 * Extend the last range.
400 * Start another range on the end.
403 rlp->ranges[j] = rlp->ranges[j + 1] = c;
410 * The code is a duplicate of a code in the last range, so just return.
415 * The code should be inserted somewhere before the last range in the
416 * list. Locate the insertion point.
419 i < rlp->used && c > rlp->ranges[i + 1] + 1; i += 2) ;
422 e = rlp->ranges[i + 1];
426 * Simply extend the current range.
428 rlp->ranges[i + 1] = c;
431 * Add a new entry before the current location. Shift all entries
432 * before the current one up by one to make room.
434 for (j = rlp->used; j > i; j -= 2) {
435 rlp->ranges[j] = rlp->ranges[j - 2];
436 rlp->ranges[j + 1] = rlp->ranges[j - 1];
438 rlp->ranges[i] = rlp->ranges[i + 1] = c;
445 add_decomp(ac_uint4 code, short compat)
448 _decomp_t **pdecomps;
449 ac_uint4 *pdecomps_used;
450 ac_uint4 *pdecomps_size;
453 pdecomps = &kdecomps;
454 pdecomps_used = &kdecomps_used;
455 pdecomps_size = &kdecomps_size;
458 pdecomps_used = &decomps_used;
459 pdecomps_size = &decomps_size;
463 * Add the code to the composite property.
466 ordered_range_insert(code, "Cm", 2);
470 * Locate the insertion point for the code.
472 for (i = 0; i < *pdecomps_used && code > (*pdecomps)[i].code; i++) ;
475 * Allocate space for a new decomposition.
477 if (*pdecomps_used == *pdecomps_size) {
478 if (*pdecomps_size == 0)
479 *pdecomps = (_decomp_t *) malloc(sizeof(_decomp_t) << 3);
481 *pdecomps = (_decomp_t *)
482 realloc((char *) *pdecomps,
483 sizeof(_decomp_t) * (*pdecomps_size + 8));
484 (void) memset((char *) (*pdecomps + *pdecomps_size), '\0',
485 sizeof(_decomp_t) << 3);
489 if (i < *pdecomps_used && code != (*pdecomps)[i].code) {
491 * Shift the decomps up by one if the codes don't match.
493 for (j = *pdecomps_used; j > i; j--)
494 (void) AC_MEMCPY((char *) &(*pdecomps)[j], (char *) &(*pdecomps)[j - 1],
499 * Insert or replace a decomposition.
501 size = dectmp_size + (4 - (dectmp_size & 3));
502 if ((*pdecomps)[i].size < size) {
503 if ((*pdecomps)[i].size == 0)
504 (*pdecomps)[i].decomp = (ac_uint4 *)
505 malloc(sizeof(ac_uint4) * size);
507 (*pdecomps)[i].decomp = (ac_uint4 *)
508 realloc((char *) (*pdecomps)[i].decomp,
509 sizeof(ac_uint4) * size);
510 (*pdecomps)[i].size = size;
513 if ((*pdecomps)[i].code != code)
516 (*pdecomps)[i].code = code;
517 (*pdecomps)[i].used = dectmp_size;
518 (void) AC_MEMCPY((char *) (*pdecomps)[i].decomp, (char *) dectmp,
519 sizeof(ac_uint4) * dectmp_size);
522 * NOTICE: This needs changing later so it is more general than simply
523 * pairs. This calculation is done here to simplify allocation elsewhere.
525 if (!compat && dectmp_size == 2)
530 add_title(ac_uint4 code)
535 * Always map the code to itself.
539 if (title_used == title_size) {
541 title = (_case_t *) malloc(sizeof(_case_t) << 3);
543 title = (_case_t *) realloc((char *) title,
544 sizeof(_case_t) * (title_size + 8));
549 * Locate the insertion point.
551 for (i = 0; i < title_used && code > title[i].key; i++) ;
553 if (i < title_used) {
555 * Shift the array up by one.
557 for (j = title_used; j > i; j--)
558 (void) AC_MEMCPY((char *) &title[j], (char *) &title[j - 1],
562 title[i].key = cases[2]; /* Title */
563 title[i].other1 = cases[0]; /* Upper */
564 title[i].other2 = cases[1]; /* Lower */
570 add_upper(ac_uint4 code)
575 * Always map the code to itself.
580 * If the title case character is not present, then make it the same as
586 if (upper_used == upper_size) {
588 upper = (_case_t *) malloc(sizeof(_case_t) << 3);
590 upper = (_case_t *) realloc((char *) upper,
591 sizeof(_case_t) * (upper_size + 8));
596 * Locate the insertion point.
598 for (i = 0; i < upper_used && code > upper[i].key; i++) ;
600 if (i < upper_used) {
602 * Shift the array up by one.
604 for (j = upper_used; j > i; j--)
605 (void) AC_MEMCPY((char *) &upper[j], (char *) &upper[j - 1],
609 upper[i].key = cases[0]; /* Upper */
610 upper[i].other1 = cases[1]; /* Lower */
611 upper[i].other2 = cases[2]; /* Title */
617 add_lower(ac_uint4 code)
622 * Always map the code to itself.
627 * If the title case character is empty, then make it the same as the
633 if (lower_used == lower_size) {
635 lower = (_case_t *) malloc(sizeof(_case_t) << 3);
637 lower = (_case_t *) realloc((char *) lower,
638 sizeof(_case_t) * (lower_size + 8));
643 * Locate the insertion point.
645 for (i = 0; i < lower_used && code > lower[i].key; i++) ;
647 if (i < lower_used) {
649 * Shift the array up by one.
651 for (j = lower_used; j > i; j--)
652 (void) AC_MEMCPY((char *) &lower[j], (char *) &lower[j - 1],
656 lower[i].key = cases[1]; /* Lower */
657 lower[i].other1 = cases[0]; /* Upper */
658 lower[i].other2 = cases[2]; /* Title */
664 ordered_ccl_insert(ac_uint4 c, ac_uint4 ccl_code)
668 if (ccl_used == ccl_size) {
670 ccl = (ac_uint4 *) malloc(sizeof(ac_uint4) * 24);
673 realloc((char *) ccl, sizeof(ac_uint4) * (ccl_size + 24));
678 * Optimize adding the first item.
688 * Handle the special case of extending the range on the end. This
689 * requires that the combining class codes are the same.
691 if (ccl_code == ccl[ccl_used - 1] && c == ccl[ccl_used - 2] + 1) {
692 ccl[ccl_used - 2] = c;
697 * Handle the special case of adding another range on the end.
699 if (c > ccl[ccl_used - 2] + 1 ||
700 (c == ccl[ccl_used - 2] + 1 && ccl_code != ccl[ccl_used - 1])) {
703 ccl[ccl_used++] = ccl_code;
708 * Locate either the insertion point or range for the code.
710 for (i = 0; i < ccl_used && c > ccl[i + 1] + 1; i += 3) ;
712 if (ccl_code == ccl[i + 2] && c == ccl[i + 1] + 1) {
714 * Extend an existing range.
718 } else if (c < ccl[i]) {
720 * Start a new range before the current location.
722 for (j = ccl_used; j > i; j -= 3) {
724 ccl[j - 1] = ccl[j - 4];
725 ccl[j - 2] = ccl[j - 5];
727 ccl[i] = ccl[i + 1] = c;
728 ccl[i + 2] = ccl_code;
733 * Adds a number if it does not already exist and returns an index value
737 make_number(short num, short denom)
742 * Determine if the number already exists.
744 for (n = 0; n < nums_used; n++) {
745 if (nums[n].numerator == num && nums[n].denominator == denom)
749 if (nums_used == nums_size) {
751 nums = (_num_t *) malloc(sizeof(_num_t) << 3);
753 nums = (_num_t *) realloc((char *) nums,
754 sizeof(_num_t) * (nums_size + 8));
759 nums[n].numerator = num;
760 nums[n].denominator = denom;
766 add_number(ac_uint4 code, short num, short denom)
771 * Insert the code in order.
773 for (i = 0; i < ncodes_used && code > ncodes[i].code; i++) ;
776 * Handle the case of the codes matching and simply replace the number
777 * that was there before.
779 if (i < ncodes_used && code == ncodes[i].code) {
780 ncodes[i].idx = make_number(num, denom);
785 * Resize the array if necessary.
787 if (ncodes_used == ncodes_size) {
788 if (ncodes_size == 0)
789 ncodes = (_codeidx_t *) malloc(sizeof(_codeidx_t) << 3);
791 ncodes = (_codeidx_t *)
792 realloc((char *) ncodes, sizeof(_codeidx_t) * (ncodes_size + 8));
798 * Shift things around to insert the code if necessary.
800 if (i < ncodes_used) {
801 for (j = ncodes_used; j > i; j--) {
802 ncodes[j].code = ncodes[j - 1].code;
803 ncodes[j].idx = ncodes[j - 1].idx;
806 ncodes[i].code = code;
807 ncodes[i].idx = make_number(num, denom);
813 * This routine assumes that the line is a valid Unicode Character Database
819 ac_uint4 i, lineno, skip, code, ccl_code;
820 short wnum, neg, number[2], compat;
821 char line[512], *s, *e;
824 while (fgets(line, sizeof(line), in)) {
825 if( (s=strchr(line, '\n')) ) *s = '\0';
829 * Skip blank lines and lines that start with a '#'.
831 if (line[0] == 0 || line[0] == '#')
835 * If lines need to be skipped, do it here.
843 * Collect the code. The code can be up to 6 hex digits in length to
844 * allow surrogates to be specified.
846 for (s = line, i = code = 0; *s != ';' && i < 6; i++, s++) {
848 if (*s >= '0' && *s <= '9')
850 else if (*s >= 'A' && *s <= 'F')
851 code += (*s - 'A') + 10;
852 else if (*s >= 'a' && *s <= 'f')
853 code += (*s - 'a') + 10;
857 * Handle the following special cases:
858 * 1. 4E00-9FA5 CJK Ideographs.
859 * 2. AC00-D7A3 Hangul Syllables.
860 * 3. D800-DFFF Surrogates.
861 * 4. E000-F8FF Private Use Area.
862 * 5. F900-FA2D Han compatibility.
863 * ...Plus additional ranges in newer Unicode versions...
867 /* CJK Ideograph Extension A */
868 add_range(0x3400, 0x4db5, "Lo", "L");
870 add_range(0x3400, 0x4db5, "Cp", 0);
876 * The Han ideographs.
878 add_range(0x4e00, 0x9fff, "Lo", "L");
881 * Add the characters to the defined category.
883 add_range(0x4e00, 0x9fa5, "Cp", 0);
889 * The Hangul syllables.
891 add_range(0xac00, 0xd7a3, "Lo", "L");
894 * Add the characters to the defined category.
896 add_range(0xac00, 0xd7a3, "Cp", 0);
902 * Make a range of all surrogates and assume some default
905 add_range(0x010000, 0x10ffff, "Cs", "L");
910 * The Private Use area. Add with a default set of properties.
912 add_range(0xe000, 0xf8ff, "Co", "L");
917 * The CJK compatibility area.
919 add_range(0xf900, 0xfaff, "Lo", "L");
922 * Add the characters to the defined category.
924 add_range(0xf900, 0xfaff, "Cp", 0);
929 /* CJK Ideograph Extension B */
930 add_range(0x20000, 0x2a6d6, "Lo", "L");
932 add_range(0x20000, 0x2a6d6, "Cp", 0);
937 /* Plane 15 private use */
938 add_range(0xf0000, 0xffffd, "Co", "L");
943 /* Plane 16 private use */
944 add_range(0x100000, 0x10fffd, "Co", "L");
953 * Add the code to the defined category.
955 ordered_range_insert(code, "Cp", 2);
958 * Locate the first character property field.
960 for (i = 0; *s != 0 && i < 2; s++) {
964 for (e = s; *e && *e != ';'; e++) ;
966 ordered_range_insert(code, s, e - s);
969 * Locate the combining class code.
971 for (s = e; *s != 0 && i < 3; s++) {
977 * Convert the combining class code from decimal.
979 for (ccl_code = 0, e = s; *e && *e != ';'; e++)
980 ccl_code = (ccl_code * 10) + (*e - '0');
983 * Add the code if it not 0.
986 ordered_ccl_insert(code, ccl_code);
989 * Locate the second character property field.
991 for (s = e; *s != 0 && i < 4; s++) {
995 for (e = s; *e && *e != ';'; e++) ;
997 ordered_range_insert(code, s, e - s);
1000 * Check for a decomposition.
1007 * Skip compatibility formatting tag.
1009 while (*s++ != '>');
1012 * Collect the codes of the decomposition.
1014 for (dectmp_size = 0; *s != ';'; ) {
1016 * Skip all leading non-hex digits.
1018 while (!ishdigit(*s))
1021 for (dectmp[dectmp_size] = 0; ishdigit(*s); s++) {
1022 dectmp[dectmp_size] <<= 4;
1023 if (*s >= '0' && *s <= '9')
1024 dectmp[dectmp_size] += *s - '0';
1025 else if (*s >= 'A' && *s <= 'F')
1026 dectmp[dectmp_size] += (*s - 'A') + 10;
1027 else if (*s >= 'a' && *s <= 'f')
1028 dectmp[dectmp_size] += (*s - 'a') + 10;
1034 * If there are any codes in the temporary decomposition array,
1035 * then add the character with its decomposition.
1037 if (dectmp_size > 0) {
1039 add_decomp(code, 0);
1041 add_decomp(code, 1);
1046 * Skip to the number field.
1048 for (i = 0; i < 3 && *s; s++) {
1054 * Scan the number in.
1056 number[0] = number[1] = 0;
1057 for (e = s, neg = wnum = 0; *e && *e != ';'; e++) {
1065 * Move the the denominator of the fraction.
1073 number[wnum] = (number[wnum] * 10) + (*e - '0');
1078 * Adjust the denominator in case of integers and add the number.
1083 add_number(code, number[0], number[1]);
1087 * Skip to the start of the possible case mappings.
1089 for (s = e, i = 0; i < 4 && *s; s++) {
1095 * Collect the case mappings.
1097 cases[0] = cases[1] = cases[2] = 0;
1098 for (i = 0; i < 3; i++) {
1099 while (ishdigit(*s)) {
1101 if (*s >= '0' && *s <= '9')
1102 cases[i] += *s - '0';
1103 else if (*s >= 'A' && *s <= 'F')
1104 cases[i] += (*s - 'A') + 10;
1105 else if (*s >= 'a' && *s <= 'f')
1106 cases[i] += (*s - 'a') + 10;
1112 if (cases[0] && cases[1])
1114 * Add the upper and lower mappings for a title case character.
1119 * Add the lower and title case mappings for the upper case
1125 * Add the upper and title case mappings for the lower case
1133 find_decomp(ac_uint4 code, short compat)
1139 r = (compat ? kdecomps_used : decomps_used) - 1;
1140 decs = compat ? kdecomps : decomps;
1143 if (code > decs[m].code)
1145 else if (code < decs[m].code)
1154 decomp_it(_decomp_t *d, short compat)
1159 for (i = 0; i < d->used; i++) {
1160 if ((dp = find_decomp(d->decomp[i], compat)) != 0)
1161 decomp_it(dp, compat);
1163 dectmp[dectmp_size++] = d->decomp[i];
1168 * Expand all decompositions by recursively decomposing each character
1169 * in the decomposition.
1176 for (i = 0; i < decomps_used; i++) {
1178 decomp_it(&decomps[i], 0);
1179 if (dectmp_size > 0)
1180 add_decomp(decomps[i].code, 0);
1183 for (i = 0; i < kdecomps_used; i++) {
1185 decomp_it(&kdecomps[i], 1);
1186 if (dectmp_size > 0)
1187 add_decomp(kdecomps[i].code, 1);
1192 cmpcomps(const void *v_comp1, const void *v_comp2)
1194 const _comp_t *comp1 = v_comp1, *comp2 = v_comp2;
1195 long diff = comp1->code1 - comp2->code1;
1198 diff = comp1->code2 - comp2->code2;
1203 * Load composition exclusion data
1206 read_compexdata(FILE *in)
1212 (void) memset((char *) compexs, 0, sizeof(compexs));
1214 while (fgets(line, sizeof(line), in)) {
1215 if( (s=strchr(line, '\n')) ) *s = '\0';
1217 * Skip blank lines and lines that start with a '#'.
1219 if (line[0] == 0 || line[0] == '#')
1223 * Collect the code. Assume max 6 digits
1226 for (s = line, i = code = 0; *s != '#' && i < 6; i++, s++) {
1227 if (isspace((unsigned char)*s)) break;
1229 if (*s >= '0' && *s <= '9')
1231 else if (*s >= 'A' && *s <= 'F')
1232 code += (*s - 'A') + 10;
1233 else if (*s >= 'a' && *s <= 'f')
1234 code += (*s - 'a') + 10;
1241 * Creates array of compositions from decomposition array
1248 comps = (_comp_t *) malloc(comps_used * sizeof(_comp_t));
1250 for (i = cu = 0; i < decomps_used; i++) {
1251 if (decomps[i].used != 2 || COMPEX_TEST(decomps[i].code))
1253 comps[cu].comp = decomps[i].code;
1254 comps[cu].count = 2;
1255 comps[cu].code1 = decomps[i].decomp[0];
1256 comps[cu].code2 = decomps[i].decomp[1];
1260 qsort(comps, comps_used, sizeof(_comp_t), cmpcomps);
1265 write_case(FILE *out, _case_t *tab, int num, int first)
1269 for (i=0; i<num; i++) {
1270 if (first) first = 0;
1271 else fprintf(out, ",");
1272 fprintf(out, "\n\t0x%08lx, 0x%08lx, 0x%08lx",
1273 (unsigned long) tab[i].key, (unsigned long) tab[i].other1,
1274 (unsigned long) tab[i].other2);
1278 #define PREF "static const "
1283 write_cdata(char *opath)
1287 ac_uint4 i, idx, nprops;
1288 #if !(HARDCODE_DATA)
1289 ac_uint2 casecnt[2];
1295 /*****************************************************************
1297 * Generate the ctype data.
1299 *****************************************************************/
1302 * Open the output file.
1304 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "uctable.h", opath);
1305 if ((out = fopen(path, "w")) == 0)
1309 * Open the ctype.dat file.
1311 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "ctype.dat", opath);
1312 if ((out = fopen(path, "wb")) == 0)
1317 * Collect the offsets for the properties. The offsets array is
1318 * on a 4-byte boundary to keep things efficient for architectures
1319 * that need such a thing.
1321 for (i = idx = 0; i < NUMPROPS; i++) {
1322 propcnt[i] = (proptbl[i].used != 0) ? idx : 0xffff;
1323 idx += proptbl[i].used;
1327 * Add the sentinel index which is used by the binary search as the upper
1328 * bound for a search.
1333 * Record the actual number of property lists. This may be different than
1334 * the number of offsets actually written because of aligning on a 4-byte
1340 * Calculate the byte count needed and pad the property counts array to a
1343 if ((bytes = sizeof(ac_uint2) * (NUMPROPS + 1)) & 3)
1344 bytes += 4 - (bytes & 3);
1345 nprops = bytes / sizeof(ac_uint2);
1346 bytes += sizeof(ac_uint4) * idx;
1349 fprintf(out, PREF "ac_uint4 _ucprop_size = %d;\n\n", NUMPROPS);
1351 fprintf(out, PREF "ac_uint2 _ucprop_offsets[] = {");
1353 for (i = 0; i<nprops; i++) {
1354 if (i) fprintf(out, ",");
1355 if (!(i&7)) fprintf(out, "\n\t");
1356 else fprintf(out, " ");
1357 fprintf(out, "0x%04x", propcnt[i]);
1359 fprintf(out, "\n};\n\n");
1361 fprintf(out, PREF "ac_uint4 _ucprop_ranges[] = {");
1364 for (i = 0; i < NUMPROPS; i++) {
1365 if (proptbl[i].used > 0) {
1366 for (j=0; j<proptbl[i].used; j++) {
1367 if (k) fprintf(out, ",");
1368 if (!(k&3)) fprintf(out,"\n\t");
1369 else fprintf(out, " ");
1371 fprintf(out, "0x%08lx", (unsigned long) proptbl[i].ranges[j]);
1375 fprintf(out, "\n};\n\n");
1380 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1383 * Write the byte count.
1385 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1388 * Write the property list counts.
1390 fwrite((char *) propcnt, sizeof(ac_uint2), nprops, out);
1393 * Write the property lists.
1395 for (i = 0; i < NUMPROPS; i++) {
1396 if (proptbl[i].used > 0)
1397 fwrite((char *) proptbl[i].ranges, sizeof(ac_uint4),
1398 proptbl[i].used, out);
1404 /*****************************************************************
1406 * Generate the case mapping data.
1408 *****************************************************************/
1411 fprintf(out, PREF "ac_uint4 _uccase_size = %ld;\n\n",
1412 (long) (upper_used + lower_used + title_used));
1414 fprintf(out, PREF "ac_uint2 _uccase_len[2] = {%ld, %ld};\n\n",
1415 (long) upper_used, (long) lower_used);
1416 fprintf(out, PREF "ac_uint4 _uccase_map[] = {");
1420 * Write the upper case table.
1422 write_case(out, upper, upper_used, 1);
1426 * Write the lower case table.
1428 write_case(out, lower, lower_used, !upper_used);
1432 * Write the title case table.
1434 write_case(out, title, title_used, !(upper_used||lower_used));
1436 if (!(upper_used || lower_used || title_used))
1437 fprintf(out, "\t0");
1439 fprintf(out, "\n};\n\n");
1442 * Open the case.dat file.
1444 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "case.dat", opath);
1445 if ((out = fopen(path, "wb")) == 0)
1449 * Write the case mapping tables.
1451 hdr[1] = upper_used + lower_used + title_used;
1452 casecnt[0] = upper_used;
1453 casecnt[1] = lower_used;
1458 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1461 * Write the upper and lower case table sizes.
1463 fwrite((char *) casecnt, sizeof(ac_uint2), 2, out);
1467 * Write the upper case table.
1469 fwrite((char *) upper, sizeof(_case_t), upper_used, out);
1473 * Write the lower case table.
1475 fwrite((char *) lower, sizeof(_case_t), lower_used, out);
1479 * Write the title case table.
1481 fwrite((char *) title, sizeof(_case_t), title_used, out);
1486 /*****************************************************************
1488 * Generate the composition data.
1490 *****************************************************************/
1493 * Create compositions from decomposition data
1498 fprintf(out, PREF "ac_uint4 _uccomp_size = %ld;\n\n",
1501 fprintf(out, PREF "ac_uint4 _uccomp_data[] = {");
1504 * Now, if comps exist, write them out.
1506 if (comps_used > 0) {
1507 for (i=0; i<comps_used; i++) {
1508 if (i) fprintf(out, ",");
1509 fprintf(out, "\n\t0x%08lx, 0x%08lx, 0x%08lx, 0x%08lx",
1510 (unsigned long) comps[i].comp, (unsigned long) comps[i].count,
1511 (unsigned long) comps[i].code1, (unsigned long) comps[i].code2);
1514 fprintf(out, "\t0");
1516 fprintf(out, "\n};\n\n");
1519 * Open the comp.dat file.
1521 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "comp.dat", opath);
1522 if ((out = fopen(path, "wb")) == 0)
1528 hdr[1] = (ac_uint2) comps_used * 4;
1529 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1532 * Write out the byte count to maintain header size.
1534 bytes = comps_used * sizeof(_comp_t);
1535 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1538 * Now, if comps exist, write them out.
1541 fwrite((char *) comps, sizeof(_comp_t), comps_used, out);
1546 /*****************************************************************
1548 * Generate the decomposition data.
1550 *****************************************************************/
1553 * Fully expand all decompositions before generating the output file.
1558 fprintf(out, PREF "ac_uint4 _ucdcmp_size = %ld;\n\n",
1561 fprintf(out, PREF "ac_uint4 _ucdcmp_nodes[] = {");
1565 * Write the list of decomp nodes.
1567 for (i = idx = 0; i < decomps_used; i++) {
1568 fprintf(out, "\n\t0x%08lx, 0x%08lx,",
1569 (unsigned long) decomps[i].code, (unsigned long) idx);
1570 idx += decomps[i].used;
1574 * Write the sentinel index as the last decomp node.
1576 fprintf(out, "\n\t0x%08lx\n};\n\n", (unsigned long) idx);
1578 fprintf(out, PREF "ac_uint4 _ucdcmp_decomp[] = {");
1580 * Write the decompositions themselves.
1583 for (i = 0; i < decomps_used; i++)
1584 for (j=0; j<decomps[i].used; j++) {
1585 if (k) fprintf(out, ",");
1586 if (!(k&3)) fprintf(out,"\n\t");
1587 else fprintf(out, " ");
1589 fprintf(out, "0x%08lx", (unsigned long) decomps[i].decomp[j]);
1591 fprintf(out, "\n};\n\n");
1595 * Open the decomp.dat file.
1597 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "decomp.dat", opath);
1598 if ((out = fopen(path, "wb")) == 0)
1601 hdr[1] = decomps_used;
1606 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1609 * Write a temporary byte count which will be calculated as the
1610 * decompositions are written out.
1613 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1617 * Write the list of decomp nodes.
1619 for (i = idx = 0; i < decomps_used; i++) {
1620 fwrite((char *) &decomps[i].code, sizeof(ac_uint4), 1, out);
1621 fwrite((char *) &idx, sizeof(ac_uint4), 1, out);
1622 idx += decomps[i].used;
1626 * Write the sentinel index as the last decomp node.
1628 fwrite((char *) &idx, sizeof(ac_uint4), 1, out);
1631 * Write the decompositions themselves.
1633 for (i = 0; i < decomps_used; i++)
1634 fwrite((char *) decomps[i].decomp, sizeof(ac_uint4),
1635 decomps[i].used, out);
1638 * Seek back to the beginning and write the byte count.
1640 bytes = (sizeof(ac_uint4) * idx) +
1641 (sizeof(ac_uint4) * ((hdr[1] << 1) + 1));
1642 fseek(out, sizeof(ac_uint2) << 1, 0L);
1643 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1649 #ifdef HARDCODE_DATA
1650 fprintf(out, PREF "ac_uint4 _uckdcmp_size = %ld;\n\n",
1651 kdecomps_used * 2L);
1653 fprintf(out, PREF "ac_uint4 _uckdcmp_nodes[] = {");
1655 if (kdecomps_used) {
1657 * Write the list of kdecomp nodes.
1659 for (i = idx = 0; i < kdecomps_used; i++) {
1660 fprintf(out, "\n\t0x%08lx, 0x%08lx,",
1661 (unsigned long) kdecomps[i].code, (unsigned long) idx);
1662 idx += kdecomps[i].used;
1666 * Write the sentinel index as the last decomp node.
1668 fprintf(out, "\n\t0x%08lx\n};\n\n", (unsigned long) idx);
1670 fprintf(out, PREF "ac_uint4 _uckdcmp_decomp[] = {");
1673 * Write the decompositions themselves.
1676 for (i = 0; i < kdecomps_used; i++)
1677 for (j=0; j<kdecomps[i].used; j++) {
1678 if (k) fprintf(out, ",");
1679 if (!(k&3)) fprintf(out,"\n\t");
1680 else fprintf(out, " ");
1682 fprintf(out, "0x%08lx", (unsigned long) kdecomps[i].decomp[j]);
1684 fprintf(out, "\n};\n\n");
1688 * Open the kdecomp.dat file.
1690 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "kdecomp.dat", opath);
1691 if ((out = fopen(path, "wb")) == 0)
1694 hdr[1] = kdecomps_used;
1699 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1702 * Write a temporary byte count which will be calculated as the
1703 * decompositions are written out.
1706 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1708 if (kdecomps_used) {
1710 * Write the list of kdecomp nodes.
1712 for (i = idx = 0; i < kdecomps_used; i++) {
1713 fwrite((char *) &kdecomps[i].code, sizeof(ac_uint4), 1, out);
1714 fwrite((char *) &idx, sizeof(ac_uint4), 1, out);
1715 idx += kdecomps[i].used;
1719 * Write the sentinel index as the last decomp node.
1721 fwrite((char *) &idx, sizeof(ac_uint4), 1, out);
1724 * Write the decompositions themselves.
1726 for (i = 0; i < kdecomps_used; i++)
1727 fwrite((char *) kdecomps[i].decomp, sizeof(ac_uint4),
1728 kdecomps[i].used, out);
1731 * Seek back to the beginning and write the byte count.
1733 bytes = (sizeof(ac_uint4) * idx) +
1734 (sizeof(ac_uint4) * ((hdr[1] << 1) + 1));
1735 fseek(out, sizeof(ac_uint2) << 1, 0L);
1736 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1742 /*****************************************************************
1744 * Generate the combining class data.
1746 *****************************************************************/
1747 #ifdef HARDCODE_DATA
1748 fprintf(out, PREF "ac_uint4 _uccmcl_size = %ld;\n\n", (long) ccl_used);
1750 fprintf(out, PREF "ac_uint4 _uccmcl_nodes[] = {");
1754 * Write the combining class ranges out.
1756 for (i = 0; i<ccl_used; i++) {
1757 if (i) fprintf(out, ",");
1758 if (!(i&3)) fprintf(out, "\n\t");
1759 else fprintf(out, " ");
1760 fprintf(out, "0x%08lx", (unsigned long) ccl[i]);
1763 fprintf(out, "\t0");
1765 fprintf(out, "\n};\n\n");
1768 * Open the cmbcl.dat file.
1770 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "cmbcl.dat", opath);
1771 if ((out = fopen(path, "wb")) == 0)
1775 * Set the number of ranges used. Each range has a combining class which
1776 * means each entry is a 3-tuple.
1778 hdr[1] = ccl_used / 3;
1783 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1786 * Write out the byte count to maintain header size.
1788 bytes = ccl_used * sizeof(ac_uint4);
1789 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1793 * Write the combining class ranges out.
1795 fwrite((char *) ccl, sizeof(ac_uint4), ccl_used, out);
1800 /*****************************************************************
1802 * Generate the number data.
1804 *****************************************************************/
1807 fprintf(out, PREF "ac_uint4 _ucnum_size = %lu;\n\n",
1808 (unsigned long)ncodes_used<<1);
1810 fprintf(out, PREF "ac_uint4 _ucnum_nodes[] = {");
1813 * Now, if number mappings exist, write them out.
1815 if (ncodes_used > 0) {
1816 for (i = 0; i<ncodes_used; i++) {
1817 if (i) fprintf(out, ",");
1818 if (!(i&1)) fprintf(out, "\n\t");
1819 else fprintf(out, " ");
1820 fprintf(out, "0x%08lx, 0x%08lx",
1821 (unsigned long) ncodes[i].code, (unsigned long) ncodes[i].idx);
1823 fprintf(out, "\n};\n\n");
1825 fprintf(out, PREF "short _ucnum_vals[] = {");
1826 for (i = 0; i<nums_used; i++) {
1827 if (i) fprintf(out, ",");
1828 if (!(i&3)) fprintf(out, "\n\t");
1829 else fprintf(out, " ");
1830 if (nums[i].numerator < 0) {
1831 fprintf(out, "%6d, 0x%04x",
1832 nums[i].numerator, nums[i].denominator);
1834 fprintf(out, "0x%04x, 0x%04x",
1835 nums[i].numerator, nums[i].denominator);
1838 fprintf(out, "\n};\n\n");
1842 * Open the num.dat file.
1844 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "num.dat", opath);
1845 if ((out = fopen(path, "wb")) == 0)
1849 * The count part of the header will be the total number of codes that
1852 hdr[1] = (ac_uint2) (ncodes_used << 1);
1853 bytes = (ncodes_used * sizeof(_codeidx_t)) + (nums_used * sizeof(_num_t));
1858 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1861 * Write out the byte count to maintain header size.
1863 fwrite((char *) &bytes, sizeof(ac_uint4), 1, out);
1866 * Now, if number mappings exist, write them out.
1868 if (ncodes_used > 0) {
1869 fwrite((char *) ncodes, sizeof(_codeidx_t), ncodes_used, out);
1870 fwrite((char *) nums, sizeof(_num_t), nums_used, out);
1881 "Usage: %s [-o output-directory|-x composition-exclusions]", prog);
1882 fprintf(stderr, " datafile1 datafile2 ...\n\n");
1884 "-o output-directory\n\t\tWrite the output files to a different");
1885 fprintf(stderr, " directory (default: .).\n");
1887 "-x composition-exclusion\n\t\tFile of composition codes");
1888 fprintf(stderr, " that should be excluded.\n");
1893 main(int argc, char *argv[])
1898 prog = lutil_progname( "ucgendat", argc, argv );
1907 if (argv[0][0] == '-') {
1908 switch (argv[0][1]) {
1917 if ((in = fopen(argv[0], "r")) == 0)
1919 "%s: unable to open composition exclusion file %s\n",
1922 read_compexdata(in);
1931 if (in != stdin && in != NULL)
1933 if ((in = fopen(argv[0], "r")) == 0)
1934 fprintf(stderr, "%s: unable to open ctype file %s\n",