2 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4 * Copyright 1998-2003 The OpenLDAP Foundation.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted only as authorized by the OpenLDAP
11 * A copy of this license is available in file LICENSE in the
12 * top-level directory of the distribution or, alternatively, at
13 * <http://www.OpenLDAP.org/license.html>.
15 /* Copyright 2001 Computing Research Labs, New Mexico State University
17 * Permission is hereby granted, free of charge, to any person obtaining a
18 * copy of this software and associated documentation files (the "Software"),
19 * to deal in the Software without restriction, including without limitation
20 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
21 * and/or sell copies of the Software, and to permit persons to whom the
22 * Software is furnished to do so, subject to the following conditions:
24 * The above copyright notice and this permission notice shall be included in
25 * all copies or substantial portions of the Software.
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
30 * THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY
31 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
32 * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
33 * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
35 /* $Id: ucgendat.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
38 #include "ldap_config.h"
41 #include <ac/stdlib.h>
42 #include <ac/string.h>
43 #include <ac/unistd.h>
50 #define ishdigit(cc) (((cc) >= '0' && (cc) <= '9') ||\
51 ((cc) >= 'A' && (cc) <= 'F') ||\
52 ((cc) >= 'a' && (cc) <= 'f'))
55 * A header written to the output file with the byte-order-mark and the number
58 static unsigned short hdr[2] = {0xfeff, 0};
61 #define NEEDPROPS (NUMPROPS + (4 - (NUMPROPS & 3)))
69 * List of properties expected to be found in the Unicode Character Database
70 * including some implementation specific properties.
72 * The implementation specific properties are:
73 * Cm = Composed (can be decomposed)
75 * Sy = Symmetric (has left and right forms)
80 * Cp = Defined character
82 static _prop_t props[NUMPROPS] = {
83 {"Mn", 2}, {"Mc", 2}, {"Me", 2}, {"Nd", 2}, {"Nl", 2}, {"No", 2},
84 {"Zs", 2}, {"Zl", 2}, {"Zp", 2}, {"Cc", 2}, {"Cf", 2}, {"Cs", 2},
85 {"Co", 2}, {"Cn", 2}, {"Lu", 2}, {"Ll", 2}, {"Lt", 2}, {"Lm", 2},
86 {"Lo", 2}, {"Pc", 2}, {"Pd", 2}, {"Ps", 2}, {"Pe", 2}, {"Po", 2},
87 {"Sm", 2}, {"Sc", 2}, {"Sk", 2}, {"So", 2}, {"L", 1}, {"R", 1},
88 {"EN", 2}, {"ES", 2}, {"ET", 2}, {"AN", 2}, {"CS", 2}, {"B", 1},
89 {"S", 1}, {"WS", 2}, {"ON", 2},
90 {"Cm", 2}, {"Nb", 2}, {"Sy", 2}, {"Hd", 2}, {"Qm", 2}, {"Mr", 2},
91 {"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}, {"AL", 2}
95 unsigned long *ranges;
100 static _ranges_t proptbl[NUMPROPS];
103 * Make sure this array is sized to be on a 4-byte boundary at compile time.
105 static unsigned short propcnt[NEEDPROPS];
108 * Array used to collect a decomposition before adding it to the decomposition
111 static unsigned long dectmp[64];
112 static unsigned long dectmp_size;
118 unsigned long *decomp;
122 * List of decomposition. Created and expanded in order as the characters are
123 * encountered. First list contains canonical mappings, second also includes
124 * compatibility mappings.
126 static _decomp_t *decomps;
127 static unsigned long decomps_used;
128 static unsigned long decomps_size;
130 static _decomp_t *kdecomps;
131 static unsigned long kdecomps_used;
132 static unsigned long kdecomps_size;
135 * Composition exclusion table stuff.
137 #define COMPEX_SET(c) (compexs[(c) >> 5] |= (1 << ((c) & 31)))
138 #define COMPEX_TEST(c) (compexs[(c) >> 5] & (1 << ((c) & 31)))
139 static unsigned long compexs[2048];
142 * Struct for holding a composition pair, and array of composition pairs
151 static _comp_t *comps;
152 static unsigned long comps_used;
155 * Types and lists for handling lists of case mappings.
159 unsigned long other1;
160 unsigned long other2;
163 static _case_t *upper;
164 static _case_t *lower;
165 static _case_t *title;
166 static unsigned long upper_used;
167 static unsigned long upper_size;
168 static unsigned long lower_used;
169 static unsigned long lower_size;
170 static unsigned long title_used;
171 static unsigned long title_size;
174 * Array used to collect case mappings before adding them to a list.
176 static unsigned long cases[3];
179 * An array to hold ranges for combining classes.
181 static unsigned long *ccl;
182 static unsigned long ccl_used;
183 static unsigned long ccl_size;
186 * Structures for handling numbers.
199 * Arrays to hold the mapping of codes to numbers.
201 static _codeidx_t *ncodes;
202 static unsigned long ncodes_used;
203 static unsigned long ncodes_size;
206 static unsigned long nums_used;
207 static unsigned long nums_size;
210 * Array for holding numbers.
213 static unsigned long nums_used;
214 static unsigned long nums_size;
217 add_range(unsigned long start, unsigned long end, char *p1, char *p2)
223 for (k = 0; k < 2; k++) {
235 for (i = 0; i < NUMPROPS; i++) {
236 if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
246 * Resize the range list if necessary.
248 if (rlp->used == rlp->size) {
250 rlp->ranges = (unsigned long *)
251 malloc(sizeof(unsigned long) << 3);
253 rlp->ranges = (unsigned long *)
254 realloc((char *) rlp->ranges,
255 sizeof(unsigned long) * (rlp->size + 8));
260 * If this is the first code for this property list, just add it
263 if (rlp->used == 0) {
264 rlp->ranges[0] = start;
265 rlp->ranges[1] = end;
271 * Optimize the case of adding the range to the end.
274 if (start > rlp->ranges[j]) {
276 rlp->ranges[j++] = start;
277 rlp->ranges[j++] = end;
283 * Need to locate the insertion point.
286 i < rlp->used && start > rlp->ranges[i + 1] + 1; i += 2) ;
289 * If the start value lies in the current range, then simply set the
290 * new end point of the range to the end value passed as a parameter.
292 if (rlp->ranges[i] <= start && start <= rlp->ranges[i + 1] + 1) {
293 rlp->ranges[i + 1] = end;
298 * Shift following values up by two.
300 for (j = rlp->used; j > i; j -= 2) {
301 rlp->ranges[j] = rlp->ranges[j - 2];
302 rlp->ranges[j + 1] = rlp->ranges[j - 1];
306 * Add the new range at the insertion point.
308 rlp->ranges[i] = start;
309 rlp->ranges[i + 1] = end;
315 ordered_range_insert(unsigned long c, char *name, int len)
325 * Deal with directionality codes introduced in Unicode 3.0.
327 if ((len == 2 && memcmp(name, "BN", 2) == 0) ||
329 (memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
330 memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
331 memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0))) {
333 * Mark all of these as Other Neutral to preserve compatibility with
340 for (i = 0; i < NUMPROPS; i++) {
341 if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
349 * Have a match, so insert the code in order.
354 * Resize the range list if necessary.
356 if (rlp->used == rlp->size) {
358 rlp->ranges = (unsigned long *)
359 malloc(sizeof(unsigned long) << 3);
361 rlp->ranges = (unsigned long *)
362 realloc((char *) rlp->ranges,
363 sizeof(unsigned long) * (rlp->size + 8));
368 * If this is the first code for this property list, just add it
371 if (rlp->used == 0) {
372 rlp->ranges[0] = rlp->ranges[1] = c;
378 * Optimize the cases of extending the last range and adding new ranges to
383 s = rlp->ranges[j - 1];
387 * Extend the last range.
395 * Start another range on the end.
398 rlp->ranges[j] = rlp->ranges[j + 1] = c;
405 * The code is a duplicate of a code in the last range, so just return.
410 * The code should be inserted somewhere before the last range in the
411 * list. Locate the insertion point.
414 i < rlp->used && c > rlp->ranges[i + 1] + 1; i += 2) ;
417 e = rlp->ranges[i + 1];
421 * Simply extend the current range.
423 rlp->ranges[i + 1] = c;
426 * Add a new entry before the current location. Shift all entries
427 * before the current one up by one to make room.
429 for (j = rlp->used; j > i; j -= 2) {
430 rlp->ranges[j] = rlp->ranges[j - 2];
431 rlp->ranges[j + 1] = rlp->ranges[j - 1];
433 rlp->ranges[i] = rlp->ranges[i + 1] = c;
440 add_decomp(unsigned long code, short compat)
442 unsigned long i, j, size;
443 _decomp_t **pdecomps;
444 unsigned long *pdecomps_used;
445 unsigned long *pdecomps_size;
448 pdecomps = &kdecomps;
449 pdecomps_used = &kdecomps_used;
450 pdecomps_size = &kdecomps_size;
453 pdecomps_used = &decomps_used;
454 pdecomps_size = &decomps_size;
458 * Add the code to the composite property.
461 ordered_range_insert(code, "Cm", 2);
465 * Locate the insertion point for the code.
467 for (i = 0; i < *pdecomps_used && code > (*pdecomps)[i].code; i++) ;
470 * Allocate space for a new decomposition.
472 if (*pdecomps_used == *pdecomps_size) {
473 if (*pdecomps_size == 0)
474 *pdecomps = (_decomp_t *) malloc(sizeof(_decomp_t) << 3);
476 *pdecomps = (_decomp_t *)
477 realloc((char *) *pdecomps,
478 sizeof(_decomp_t) * (*pdecomps_size + 8));
479 (void) memset((char *) (*pdecomps + *pdecomps_size), '\0',
480 sizeof(_decomp_t) << 3);
484 if (i < *pdecomps_used && code != (*pdecomps)[i].code) {
486 * Shift the decomps up by one if the codes don't match.
488 for (j = *pdecomps_used; j > i; j--)
489 (void) AC_MEMCPY((char *) &(*pdecomps)[j], (char *) &(*pdecomps)[j - 1],
494 * Insert or replace a decomposition.
496 size = dectmp_size + (4 - (dectmp_size & 3));
497 if ((*pdecomps)[i].size < size) {
498 if ((*pdecomps)[i].size == 0)
499 (*pdecomps)[i].decomp = (unsigned long *)
500 malloc(sizeof(unsigned long) * size);
502 (*pdecomps)[i].decomp = (unsigned long *)
503 realloc((char *) (*pdecomps)[i].decomp,
504 sizeof(unsigned long) * size);
505 (*pdecomps)[i].size = size;
508 if ((*pdecomps)[i].code != code)
511 (*pdecomps)[i].code = code;
512 (*pdecomps)[i].used = dectmp_size;
513 (void) AC_MEMCPY((char *) (*pdecomps)[i].decomp, (char *) dectmp,
514 sizeof(unsigned long) * dectmp_size);
517 * NOTICE: This needs changing later so it is more general than simply
518 * pairs. This calculation is done here to simplify allocation elsewhere.
520 if (!compat && dectmp_size == 2)
525 add_title(unsigned long code)
530 * Always map the code to itself.
534 if (title_used == title_size) {
536 title = (_case_t *) malloc(sizeof(_case_t) << 3);
538 title = (_case_t *) realloc((char *) title,
539 sizeof(_case_t) * (title_size + 8));
544 * Locate the insertion point.
546 for (i = 0; i < title_used && code > title[i].key; i++) ;
548 if (i < title_used) {
550 * Shift the array up by one.
552 for (j = title_used; j > i; j--)
553 (void) AC_MEMCPY((char *) &title[j], (char *) &title[j - 1],
557 title[i].key = cases[2]; /* Title */
558 title[i].other1 = cases[0]; /* Upper */
559 title[i].other2 = cases[1]; /* Lower */
565 add_upper(unsigned long code)
570 * Always map the code to itself.
575 * If the title case character is not present, then make it the same as
581 if (upper_used == upper_size) {
583 upper = (_case_t *) malloc(sizeof(_case_t) << 3);
585 upper = (_case_t *) realloc((char *) upper,
586 sizeof(_case_t) * (upper_size + 8));
591 * Locate the insertion point.
593 for (i = 0; i < upper_used && code > upper[i].key; i++) ;
595 if (i < upper_used) {
597 * Shift the array up by one.
599 for (j = upper_used; j > i; j--)
600 (void) AC_MEMCPY((char *) &upper[j], (char *) &upper[j - 1],
604 upper[i].key = cases[0]; /* Upper */
605 upper[i].other1 = cases[1]; /* Lower */
606 upper[i].other2 = cases[2]; /* Title */
612 add_lower(unsigned long code)
617 * Always map the code to itself.
622 * If the title case character is empty, then make it the same as the
628 if (lower_used == lower_size) {
630 lower = (_case_t *) malloc(sizeof(_case_t) << 3);
632 lower = (_case_t *) realloc((char *) lower,
633 sizeof(_case_t) * (lower_size + 8));
638 * Locate the insertion point.
640 for (i = 0; i < lower_used && code > lower[i].key; i++) ;
642 if (i < lower_used) {
644 * Shift the array up by one.
646 for (j = lower_used; j > i; j--)
647 (void) AC_MEMCPY((char *) &lower[j], (char *) &lower[j - 1],
651 lower[i].key = cases[1]; /* Lower */
652 lower[i].other1 = cases[0]; /* Upper */
653 lower[i].other2 = cases[2]; /* Title */
659 ordered_ccl_insert(unsigned long c, unsigned long ccl_code)
663 if (ccl_used == ccl_size) {
665 ccl = (unsigned long *) malloc(sizeof(unsigned long) * 24);
667 ccl = (unsigned long *)
668 realloc((char *) ccl, sizeof(unsigned long) * (ccl_size + 24));
673 * Optimize adding the first item.
683 * Handle the special case of extending the range on the end. This
684 * requires that the combining class codes are the same.
686 if (ccl_code == ccl[ccl_used - 1] && c == ccl[ccl_used - 2] + 1) {
687 ccl[ccl_used - 2] = c;
692 * Handle the special case of adding another range on the end.
694 if (c > ccl[ccl_used - 2] + 1 ||
695 (c == ccl[ccl_used - 2] + 1 && ccl_code != ccl[ccl_used - 1])) {
698 ccl[ccl_used++] = ccl_code;
703 * Locate either the insertion point or range for the code.
705 for (i = 0; i < ccl_used && c > ccl[i + 1] + 1; i += 3) ;
707 if (ccl_code == ccl[i + 2] && c == ccl[i + 1] + 1) {
709 * Extend an existing range.
713 } else if (c < ccl[i]) {
715 * Start a new range before the current location.
717 for (j = ccl_used; j > i; j -= 3) {
719 ccl[j - 1] = ccl[j - 4];
720 ccl[j - 2] = ccl[j - 5];
722 ccl[i] = ccl[i + 1] = c;
723 ccl[i + 2] = ccl_code;
728 * Adds a number if it does not already exist and returns an index value
732 make_number(short num, short denom)
737 * Determine if the number already exists.
739 for (n = 0; n < nums_used; n++) {
740 if (nums[n].numerator == num && nums[n].denominator == denom)
744 if (nums_used == nums_size) {
746 nums = (_num_t *) malloc(sizeof(_num_t) << 3);
748 nums = (_num_t *) realloc((char *) nums,
749 sizeof(_num_t) * (nums_size + 8));
754 nums[n].numerator = num;
755 nums[n].denominator = denom;
761 add_number(unsigned long code, short num, short denom)
766 * Insert the code in order.
768 for (i = 0; i < ncodes_used && code > ncodes[i].code; i++) ;
771 * Handle the case of the codes matching and simply replace the number
772 * that was there before.
774 if (i < ncodes_used && code == ncodes[i].code) {
775 ncodes[i].idx = make_number(num, denom);
780 * Resize the array if necessary.
782 if (ncodes_used == ncodes_size) {
783 if (ncodes_size == 0)
784 ncodes = (_codeidx_t *) malloc(sizeof(_codeidx_t) << 3);
786 ncodes = (_codeidx_t *)
787 realloc((char *) ncodes, sizeof(_codeidx_t) * (ncodes_size + 8));
793 * Shift things around to insert the code if necessary.
795 if (i < ncodes_used) {
796 for (j = ncodes_used; j > i; j--) {
797 ncodes[j].code = ncodes[j - 1].code;
798 ncodes[j].idx = ncodes[j - 1].idx;
801 ncodes[i].code = code;
802 ncodes[i].idx = make_number(num, denom);
808 * This routine assumes that the line is a valid Unicode Character Database
814 unsigned long i, lineno, skip, code, ccl_code;
815 short wnum, neg, number[2], compat;
816 char line[512], *s, *e;
819 while (fgets(line, sizeof(line), in)) {
820 if( (s=strchr(line, '\n')) ) *s = '\0';
824 * Skip blank lines and lines that start with a '#'.
826 if (line[0] == 0 || line[0] == '#')
830 * If lines need to be skipped, do it here.
838 * Collect the code. The code can be up to 6 hex digits in length to
839 * allow surrogates to be specified.
841 for (s = line, i = code = 0; *s != ';' && i < 6; i++, s++) {
843 if (*s >= '0' && *s <= '9')
845 else if (*s >= 'A' && *s <= 'F')
846 code += (*s - 'A') + 10;
847 else if (*s >= 'a' && *s <= 'f')
848 code += (*s - 'a') + 10;
852 * Handle the following special cases:
853 * 1. 4E00-9FA5 CJK Ideographs.
854 * 2. AC00-D7A3 Hangul Syllables.
855 * 3. D800-DFFF Surrogates.
856 * 4. E000-F8FF Private Use Area.
857 * 5. F900-FA2D Han compatibility.
862 * The Han ideographs.
864 add_range(0x4e00, 0x9fff, "Lo", "L");
867 * Add the characters to the defined category.
869 add_range(0x4e00, 0x9fa5, "Cp", 0);
875 * The Hangul syllables.
877 add_range(0xac00, 0xd7a3, "Lo", "L");
880 * Add the characters to the defined category.
882 add_range(0xac00, 0xd7a3, "Cp", 0);
888 * Make a range of all surrogates and assume some default
891 add_range(0x010000, 0x10ffff, "Cs", "L");
896 * The Private Use area. Add with a default set of properties.
898 add_range(0xe000, 0xf8ff, "Co", "L");
903 * The CJK compatibility area.
905 add_range(0xf900, 0xfaff, "Lo", "L");
908 * Add the characters to the defined category.
910 add_range(0xf900, 0xfaff, "Cp", 0);
919 * Add the code to the defined category.
921 ordered_range_insert(code, "Cp", 2);
924 * Locate the first character property field.
926 for (i = 0; *s != 0 && i < 2; s++) {
930 for (e = s; *e && *e != ';'; e++) ;
932 ordered_range_insert(code, s, e - s);
935 * Locate the combining class code.
937 for (s = e; *s != 0 && i < 3; s++) {
943 * Convert the combining class code from decimal.
945 for (ccl_code = 0, e = s; *e && *e != ';'; e++)
946 ccl_code = (ccl_code * 10) + (*e - '0');
949 * Add the code if it not 0.
952 ordered_ccl_insert(code, ccl_code);
955 * Locate the second character property field.
957 for (s = e; *s != 0 && i < 4; s++) {
961 for (e = s; *e && *e != ';'; e++) ;
963 ordered_range_insert(code, s, e - s);
966 * Check for a decomposition.
973 * Skip compatibility formatting tag.
978 * Collect the codes of the decomposition.
980 for (dectmp_size = 0; *s != ';'; ) {
982 * Skip all leading non-hex digits.
984 while (!ishdigit(*s))
987 for (dectmp[dectmp_size] = 0; ishdigit(*s); s++) {
988 dectmp[dectmp_size] <<= 4;
989 if (*s >= '0' && *s <= '9')
990 dectmp[dectmp_size] += *s - '0';
991 else if (*s >= 'A' && *s <= 'F')
992 dectmp[dectmp_size] += (*s - 'A') + 10;
993 else if (*s >= 'a' && *s <= 'f')
994 dectmp[dectmp_size] += (*s - 'a') + 10;
1000 * If there are any codes in the temporary decomposition array,
1001 * then add the character with its decomposition.
1003 if (dectmp_size > 0) {
1005 add_decomp(code, 0);
1007 add_decomp(code, 1);
1012 * Skip to the number field.
1014 for (i = 0; i < 3 && *s; s++) {
1020 * Scan the number in.
1022 number[0] = number[1] = 0;
1023 for (e = s, neg = wnum = 0; *e && *e != ';'; e++) {
1031 * Move the the denominator of the fraction.
1039 number[wnum] = (number[wnum] * 10) + (*e - '0');
1044 * Adjust the denominator in case of integers and add the number.
1047 number[1] = number[0];
1049 add_number(code, number[0], number[1]);
1053 * Skip to the start of the possible case mappings.
1055 for (s = e, i = 0; i < 4 && *s; s++) {
1061 * Collect the case mappings.
1063 cases[0] = cases[1] = cases[2] = 0;
1064 for (i = 0; i < 3; i++) {
1065 while (ishdigit(*s)) {
1067 if (*s >= '0' && *s <= '9')
1068 cases[i] += *s - '0';
1069 else if (*s >= 'A' && *s <= 'F')
1070 cases[i] += (*s - 'A') + 10;
1071 else if (*s >= 'a' && *s <= 'f')
1072 cases[i] += (*s - 'a') + 10;
1078 if (cases[0] && cases[1])
1080 * Add the upper and lower mappings for a title case character.
1085 * Add the lower and title case mappings for the upper case
1091 * Add the upper and title case mappings for the lower case
1099 find_decomp(unsigned long code, short compat)
1105 r = (compat ? kdecomps_used : decomps_used) - 1;
1106 decs = compat ? kdecomps : decomps;
1109 if (code > decs[m].code)
1111 else if (code < decs[m].code)
1120 decomp_it(_decomp_t *d, short compat)
1125 for (i = 0; i < d->used; i++) {
1126 if ((dp = find_decomp(d->decomp[i], compat)) != 0)
1127 decomp_it(dp, compat);
1129 dectmp[dectmp_size++] = d->decomp[i];
1134 * Expand all decompositions by recursively decomposing each character
1135 * in the decomposition.
1142 for (i = 0; i < decomps_used; i++) {
1144 decomp_it(&decomps[i], 0);
1145 if (dectmp_size > 0)
1146 add_decomp(decomps[i].code, 0);
1149 for (i = 0; i < kdecomps_used; i++) {
1151 decomp_it(&kdecomps[i], 1);
1152 if (dectmp_size > 0)
1153 add_decomp(kdecomps[i].code, 1);
1158 cmpcomps(const void *v_comp1, const void *v_comp2)
1160 const _comp_t *comp1 = v_comp1, *comp2 = v_comp2;
1161 long diff = comp1->code1 - comp2->code1;
1164 diff = comp1->code2 - comp2->code2;
1169 * Load composition exclusion data
1172 read_compexdata(FILE *in)
1174 unsigned short i, code;
1177 (void) memset((char *) compexs, 0, sizeof(unsigned long) << 11);
1179 while (fgets(line, sizeof(line), in)) {
1180 if( (s=strchr(line, '\n')) ) *s = '\0';
1182 * Skip blank lines and lines that start with a '#'.
1184 if (line[0] == 0 || line[0] == '#')
1188 * Collect the code. Assume max 4 digits
1191 for (s = line, i = code = 0; *s != '#' && i < 4; i++, s++) {
1193 if (*s >= '0' && *s <= '9')
1195 else if (*s >= 'A' && *s <= 'F')
1196 code += (*s - 'A') + 10;
1197 else if (*s >= 'a' && *s <= 'f')
1198 code += (*s - 'a') + 10;
1205 * Creates array of compositions from decomposition array
1210 unsigned long i, cu;
1212 comps = (_comp_t *) malloc(comps_used * sizeof(_comp_t));
1214 for (i = cu = 0; i < decomps_used; i++) {
1215 if (decomps[i].used != 2 || COMPEX_TEST(decomps[i].code))
1217 comps[cu].comp = decomps[i].code;
1218 comps[cu].count = 2;
1219 comps[cu].code1 = decomps[i].decomp[0];
1220 comps[cu].code2 = decomps[i].decomp[1];
1224 qsort(comps, comps_used, sizeof(_comp_t), cmpcomps);
1228 write_cdata(char *opath)
1232 unsigned long i, idx, nprops;
1233 unsigned short casecnt[2];
1236 /*****************************************************************
1238 * Generate the ctype data.
1240 *****************************************************************/
1243 * Open the ctype.dat file.
1245 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "ctype.dat", opath);
1246 if ((out = fopen(path, "wb")) == 0)
1250 * Collect the offsets for the properties. The offsets array is
1251 * on a 4-byte boundary to keep things efficient for architectures
1252 * that need such a thing.
1254 for (i = idx = 0; i < NUMPROPS; i++) {
1255 propcnt[i] = (proptbl[i].used != 0) ? idx : 0xffff;
1256 idx += proptbl[i].used;
1260 * Add the sentinel index which is used by the binary search as the upper
1261 * bound for a search.
1266 * Record the actual number of property lists. This may be different than
1267 * the number of offsets actually written because of aligning on a 4-byte
1273 * Calculate the byte count needed and pad the property counts array to a
1276 if ((bytes = sizeof(unsigned short) * (NUMPROPS + 1)) & 3)
1277 bytes += 4 - (bytes & 3);
1278 nprops = bytes / sizeof(unsigned short);
1279 bytes += sizeof(unsigned long) * idx;
1284 fwrite((char *) hdr, sizeof(ac_uint2), 2, out);
1287 * Write the byte count.
1289 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1292 * Write the property list counts.
1294 fwrite((char *) propcnt, sizeof(unsigned short), nprops, out);
1297 * Write the property lists.
1299 for (i = 0; i < NUMPROPS; i++) {
1300 if (proptbl[i].used > 0)
1301 fwrite((char *) proptbl[i].ranges, sizeof(unsigned long),
1302 proptbl[i].used, out);
1307 /*****************************************************************
1309 * Generate the case mapping data.
1311 *****************************************************************/
1314 * Open the case.dat file.
1316 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "case.dat", opath);
1317 if ((out = fopen(path, "wb")) == 0)
1321 * Write the case mapping tables.
1323 hdr[1] = upper_used + lower_used + title_used;
1324 casecnt[0] = upper_used;
1325 casecnt[1] = lower_used;
1330 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1333 * Write the upper and lower case table sizes.
1335 fwrite((char *) casecnt, sizeof(unsigned short), 2, out);
1339 * Write the upper case table.
1341 fwrite((char *) upper, sizeof(_case_t), upper_used, out);
1345 * Write the lower case table.
1347 fwrite((char *) lower, sizeof(_case_t), lower_used, out);
1351 * Write the title case table.
1353 fwrite((char *) title, sizeof(_case_t), title_used, out);
1357 /*****************************************************************
1359 * Generate the composition data.
1361 *****************************************************************/
1364 * Create compositions from decomposition data
1369 * Open the comp.dat file.
1371 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "comp.dat", opath);
1372 if ((out = fopen(path, "wb")) == 0)
1378 hdr[1] = (unsigned short) comps_used * 4;
1379 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1382 * Write out the byte count to maintain header size.
1384 bytes = comps_used * sizeof(_comp_t);
1385 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1388 * Now, if comps exist, write them out.
1391 fwrite((char *) comps, sizeof(_comp_t), comps_used, out);
1395 /*****************************************************************
1397 * Generate the decomposition data.
1399 *****************************************************************/
1402 * Fully expand all decompositions before generating the output file.
1407 * Open the decomp.dat file.
1409 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "decomp.dat", opath);
1410 if ((out = fopen(path, "wb")) == 0)
1413 hdr[1] = decomps_used;
1418 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1421 * Write a temporary byte count which will be calculated as the
1422 * decompositions are written out.
1425 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1429 * Write the list of decomp nodes.
1431 for (i = idx = 0; i < decomps_used; i++) {
1432 fwrite((char *) &decomps[i].code, sizeof(unsigned long), 1, out);
1433 fwrite((char *) &idx, sizeof(unsigned long), 1, out);
1434 idx += decomps[i].used;
1438 * Write the sentinel index as the last decomp node.
1440 fwrite((char *) &idx, sizeof(unsigned long), 1, out);
1443 * Write the decompositions themselves.
1445 for (i = 0; i < decomps_used; i++)
1446 fwrite((char *) decomps[i].decomp, sizeof(unsigned long),
1447 decomps[i].used, out);
1450 * Seek back to the beginning and write the byte count.
1452 bytes = (sizeof(unsigned long) * idx) +
1453 (sizeof(unsigned long) * ((hdr[1] << 1) + 1));
1454 fseek(out, sizeof(unsigned short) << 1, 0L);
1455 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1461 * Open the kdecomp.dat file.
1463 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "kdecomp.dat", opath);
1464 if ((out = fopen(path, "wb")) == 0)
1467 hdr[1] = kdecomps_used;
1472 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1475 * Write a temporary byte count which will be calculated as the
1476 * decompositions are written out.
1479 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1481 if (kdecomps_used) {
1483 * Write the list of kdecomp nodes.
1485 for (i = idx = 0; i < kdecomps_used; i++) {
1486 fwrite((char *) &kdecomps[i].code, sizeof(unsigned long), 1, out);
1487 fwrite((char *) &idx, sizeof(unsigned long), 1, out);
1488 idx += kdecomps[i].used;
1492 * Write the sentinel index as the last decomp node.
1494 fwrite((char *) &idx, sizeof(unsigned long), 1, out);
1497 * Write the decompositions themselves.
1499 for (i = 0; i < kdecomps_used; i++)
1500 fwrite((char *) kdecomps[i].decomp, sizeof(unsigned long),
1501 kdecomps[i].used, out);
1504 * Seek back to the beginning and write the byte count.
1506 bytes = (sizeof(unsigned long) * idx) +
1507 (sizeof(unsigned long) * ((hdr[1] << 1) + 1));
1508 fseek(out, sizeof(unsigned short) << 1, 0L);
1509 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1514 /*****************************************************************
1516 * Generate the combining class data.
1518 *****************************************************************/
1521 * Open the cmbcl.dat file.
1523 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "cmbcl.dat", opath);
1524 if ((out = fopen(path, "wb")) == 0)
1528 * Set the number of ranges used. Each range has a combining class which
1529 * means each entry is a 3-tuple.
1531 hdr[1] = ccl_used / 3;
1536 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1539 * Write out the byte count to maintain header size.
1541 bytes = ccl_used * sizeof(unsigned long);
1542 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1546 * Write the combining class ranges out.
1548 fwrite((char *) ccl, sizeof(unsigned long), ccl_used, out);
1552 /*****************************************************************
1554 * Generate the number data.
1556 *****************************************************************/
1559 * Open the num.dat file.
1561 snprintf(path, sizeof path, "%s" LDAP_DIRSEP "num.dat", opath);
1562 if ((out = fopen(path, "wb")) == 0)
1566 * The count part of the header will be the total number of codes that
1569 hdr[1] = (unsigned short) (ncodes_used << 1);
1570 bytes = (ncodes_used * sizeof(_codeidx_t)) + (nums_used * sizeof(_num_t));
1575 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1578 * Write out the byte count to maintain header size.
1580 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1583 * Now, if number mappings exist, write them out.
1585 if (ncodes_used > 0) {
1586 fwrite((char *) ncodes, sizeof(_codeidx_t), ncodes_used, out);
1587 fwrite((char *) nums, sizeof(_num_t), nums_used, out);
1597 "Usage: %s [-o output-directory|-x composition-exclusions]", prog);
1598 fprintf(stderr, " datafile1 datafile2 ...\n\n");
1600 "-o output-directory\n\t\tWrite the output files to a different");
1601 fprintf(stderr, " directory (default: .).\n");
1603 "-x composition-exclusion\n\t\tFile of composition codes");
1604 fprintf(stderr, " that should be excluded.\n");
1609 main(int argc, char *argv[])
1614 prog = lutil_progname( "ucgendat", argc, argv );
1623 if (argv[0][0] == '-') {
1624 switch (argv[0][1]) {
1633 if ((in = fopen(argv[0], "r")) == 0)
1635 "%s: unable to open composition exclusion file %s\n",
1638 read_compexdata(in);
1647 if (in != stdin && in != NULL)
1649 if ((in = fopen(argv[0], "r")) == 0)
1650 fprintf(stderr, "%s: unable to open ctype file %s\n",