3 * Copyright 2001 Computing Research Labs, New Mexico State University
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
20 * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
21 * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 /* $Id: ucgendat.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
26 #include "ldap_config.h"
29 #include <ac/stdlib.h>
30 #include <ac/string.h>
31 #include <ac/unistd.h>
33 #define ishdigit(cc) (((cc) >= '0' && (cc) <= '9') ||\
34 ((cc) >= 'A' && (cc) <= 'F') ||\
35 ((cc) >= 'a' && (cc) <= 'f'))
38 * A header written to the output file with the byte-order-mark and the number
41 static unsigned short hdr[2] = {0xfeff, 0};
44 #define NEEDPROPS (NUMPROPS + (4 - (NUMPROPS & 3)))
52 * List of properties expected to be found in the Unicode Character Database
53 * including some implementation specific properties.
55 * The implementation specific properties are:
56 * Cm = Composed (can be decomposed)
58 * Sy = Symmetric (has left and right forms)
63 * Cp = Defined character
65 static _prop_t props[NUMPROPS] = {
66 {"Mn", 2}, {"Mc", 2}, {"Me", 2}, {"Nd", 2}, {"Nl", 2}, {"No", 2},
67 {"Zs", 2}, {"Zl", 2}, {"Zp", 2}, {"Cc", 2}, {"Cf", 2}, {"Cs", 2},
68 {"Co", 2}, {"Cn", 2}, {"Lu", 2}, {"Ll", 2}, {"Lt", 2}, {"Lm", 2},
69 {"Lo", 2}, {"Pc", 2}, {"Pd", 2}, {"Ps", 2}, {"Pe", 2}, {"Po", 2},
70 {"Sm", 2}, {"Sc", 2}, {"Sk", 2}, {"So", 2}, {"L", 1}, {"R", 1},
71 {"EN", 2}, {"ES", 2}, {"ET", 2}, {"AN", 2}, {"CS", 2}, {"B", 1},
72 {"S", 1}, {"WS", 2}, {"ON", 2},
73 {"Cm", 2}, {"Nb", 2}, {"Sy", 2}, {"Hd", 2}, {"Qm", 2}, {"Mr", 2},
74 {"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}, {"AL", 2}
78 unsigned long *ranges;
83 static _ranges_t proptbl[NUMPROPS];
86 * Make sure this array is sized to be on a 4-byte boundary at compile time.
88 static unsigned short propcnt[NEEDPROPS];
91 * Array used to collect a decomposition before adding it to the decomposition
94 static unsigned long dectmp[64];
95 static unsigned long dectmp_size;
101 unsigned long *decomp;
105 * List of decomposition. Created and expanded in order as the characters are
108 static _decomp_t *decomps;
109 static unsigned long decomps_used;
110 static unsigned long decomps_size;
113 * Composition exclusion table stuff.
115 #define COMPEX_SET(c) (compexs[(c) >> 5] |= (1 << ((c) & 31)))
116 #define COMPEX_TEST(c) (compexs[(c) >> 5] & (1 << ((c) & 31)))
117 static unsigned long compexs[2048];
120 * Struct for holding a composition pair, and array of composition pairs
129 static _comp_t *comps;
130 static unsigned long comps_used;
133 * Types and lists for handling lists of case mappings.
137 unsigned long other1;
138 unsigned long other2;
141 static _case_t *upper;
142 static _case_t *lower;
143 static _case_t *title;
144 static unsigned long upper_used;
145 static unsigned long upper_size;
146 static unsigned long lower_used;
147 static unsigned long lower_size;
148 static unsigned long title_used;
149 static unsigned long title_size;
152 * Array used to collect case mappings before adding them to a list.
154 static unsigned long cases[3];
157 * An array to hold ranges for combining classes.
159 static unsigned long *ccl;
160 static unsigned long ccl_used;
161 static unsigned long ccl_size;
164 * Structures for handling numbers.
177 * Arrays to hold the mapping of codes to numbers.
179 static _codeidx_t *ncodes;
180 static unsigned long ncodes_used;
181 static unsigned long ncodes_size;
184 static unsigned long nums_used;
185 static unsigned long nums_size;
188 * Array for holding numbers.
191 static unsigned long nums_used;
192 static unsigned long nums_size;
195 add_range(unsigned long start, unsigned long end, char *p1, char *p2)
201 for (k = 0; k < 2; k++) {
213 for (i = 0; i < NUMPROPS; i++) {
214 if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
224 * Resize the range list if necessary.
226 if (rlp->used == rlp->size) {
228 rlp->ranges = (unsigned long *)
229 malloc(sizeof(unsigned long) << 3);
231 rlp->ranges = (unsigned long *)
232 realloc((char *) rlp->ranges,
233 sizeof(unsigned long) * (rlp->size + 8));
238 * If this is the first code for this property list, just add it
241 if (rlp->used == 0) {
242 rlp->ranges[0] = start;
243 rlp->ranges[1] = end;
249 * Optimize the case of adding the range to the end.
252 if (start > rlp->ranges[j]) {
254 rlp->ranges[j++] = start;
255 rlp->ranges[j++] = end;
261 * Need to locate the insertion point.
264 i < rlp->used && start > rlp->ranges[i + 1] + 1; i += 2) ;
267 * If the start value lies in the current range, then simply set the
268 * new end point of the range to the end value passed as a parameter.
270 if (rlp->ranges[i] <= start && start <= rlp->ranges[i + 1] + 1) {
271 rlp->ranges[i + 1] = end;
276 * Shift following values up by two.
278 for (j = rlp->used; j > i; j -= 2) {
279 rlp->ranges[j] = rlp->ranges[j - 2];
280 rlp->ranges[j + 1] = rlp->ranges[j - 1];
284 * Add the new range at the insertion point.
286 rlp->ranges[i] = start;
287 rlp->ranges[i + 1] = end;
293 ordered_range_insert(unsigned long c, char *name, int len)
303 * Deal with directionality codes introduced in Unicode 3.0.
305 if ((len == 2 && memcmp(name, "BN", 2) == 0) ||
307 (memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
308 memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
309 memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0))) {
311 * Mark all of these as Other Neutral to preserve compatibility with
318 for (i = 0; i < NUMPROPS; i++) {
319 if (props[i].len == len && memcmp(props[i].name, name, len) == 0)
327 * Have a match, so insert the code in order.
332 * Resize the range list if necessary.
334 if (rlp->used == rlp->size) {
336 rlp->ranges = (unsigned long *)
337 malloc(sizeof(unsigned long) << 3);
339 rlp->ranges = (unsigned long *)
340 realloc((char *) rlp->ranges,
341 sizeof(unsigned long) * (rlp->size + 8));
346 * If this is the first code for this property list, just add it
349 if (rlp->used == 0) {
350 rlp->ranges[0] = rlp->ranges[1] = c;
356 * Optimize the cases of extending the last range and adding new ranges to
361 s = rlp->ranges[j - 1];
365 * Extend the last range.
373 * Start another range on the end.
376 rlp->ranges[j] = rlp->ranges[j + 1] = c;
383 * The code is a duplicate of a code in the last range, so just return.
388 * The code should be inserted somewhere before the last range in the
389 * list. Locate the insertion point.
392 i < rlp->used && c > rlp->ranges[i + 1] + 1; i += 2) ;
395 e = rlp->ranges[i + 1];
399 * Simply extend the current range.
401 rlp->ranges[i + 1] = c;
404 * Add a new entry before the current location. Shift all entries
405 * before the current one up by one to make room.
407 for (j = rlp->used; j > i; j -= 2) {
408 rlp->ranges[j] = rlp->ranges[j - 2];
409 rlp->ranges[j + 1] = rlp->ranges[j - 1];
411 rlp->ranges[i] = rlp->ranges[i + 1] = c;
418 add_decomp(unsigned long code)
420 unsigned long i, j, size;
423 * Add the code to the composite property.
425 ordered_range_insert(code, "Cm", 2);
428 * Locate the insertion point for the code.
430 for (i = 0; i < decomps_used && code > decomps[i].code; i++) ;
433 * Allocate space for a new decomposition.
435 if (decomps_used == decomps_size) {
436 if (decomps_size == 0)
437 decomps = (_decomp_t *) malloc(sizeof(_decomp_t) << 3);
439 decomps = (_decomp_t *)
440 realloc((char *) decomps,
441 sizeof(_decomp_t) * (decomps_size + 8));
442 (void) memset((char *) (decomps + decomps_size), '\0',
443 sizeof(_decomp_t) << 3);
447 if (i < decomps_used && code != decomps[i].code) {
449 * Shift the decomps up by one if the codes don't match.
451 for (j = decomps_used; j > i; j--)
452 (void) AC_MEMCPY((char *) &decomps[j], (char *) &decomps[j - 1],
457 * Insert or replace a decomposition.
459 size = dectmp_size + (4 - (dectmp_size & 3));
460 if (decomps[i].size < size) {
461 if (decomps[i].size == 0)
462 decomps[i].decomp = (unsigned long *)
463 malloc(sizeof(unsigned long) * size);
465 decomps[i].decomp = (unsigned long *)
466 realloc((char *) decomps[i].decomp,
467 sizeof(unsigned long) * size);
468 decomps[i].size = size;
471 if (decomps[i].code != code)
474 decomps[i].code = code;
475 decomps[i].used = dectmp_size;
476 (void) AC_MEMCPY((char *) decomps[i].decomp, (char *) dectmp,
477 sizeof(unsigned long) * dectmp_size);
480 * NOTICE: This needs changing later so it is more general than simply
481 * pairs. This calculation is done here to simplify allocation elsewhere.
483 if (dectmp_size == 2)
488 add_title(unsigned long code)
493 * Always map the code to itself.
497 if (title_used == title_size) {
499 title = (_case_t *) malloc(sizeof(_case_t) << 3);
501 title = (_case_t *) realloc((char *) title,
502 sizeof(_case_t) * (title_size + 8));
507 * Locate the insertion point.
509 for (i = 0; i < title_used && code > title[i].key; i++) ;
511 if (i < title_used) {
513 * Shift the array up by one.
515 for (j = title_used; j > i; j--)
516 (void) AC_MEMCPY((char *) &title[j], (char *) &title[j - 1],
520 title[i].key = cases[2]; /* Title */
521 title[i].other1 = cases[0]; /* Upper */
522 title[i].other2 = cases[1]; /* Lower */
528 add_upper(unsigned long code)
533 * Always map the code to itself.
538 * If the title case character is not present, then make it the same as
544 if (upper_used == upper_size) {
546 upper = (_case_t *) malloc(sizeof(_case_t) << 3);
548 upper = (_case_t *) realloc((char *) upper,
549 sizeof(_case_t) * (upper_size + 8));
554 * Locate the insertion point.
556 for (i = 0; i < upper_used && code > upper[i].key; i++) ;
558 if (i < upper_used) {
560 * Shift the array up by one.
562 for (j = upper_used; j > i; j--)
563 (void) AC_MEMCPY((char *) &upper[j], (char *) &upper[j - 1],
567 upper[i].key = cases[0]; /* Upper */
568 upper[i].other1 = cases[1]; /* Lower */
569 upper[i].other2 = cases[2]; /* Title */
575 add_lower(unsigned long code)
580 * Always map the code to itself.
585 * If the title case character is empty, then make it the same as the
591 if (lower_used == lower_size) {
593 lower = (_case_t *) malloc(sizeof(_case_t) << 3);
595 lower = (_case_t *) realloc((char *) lower,
596 sizeof(_case_t) * (lower_size + 8));
601 * Locate the insertion point.
603 for (i = 0; i < lower_used && code > lower[i].key; i++) ;
605 if (i < lower_used) {
607 * Shift the array up by one.
609 for (j = lower_used; j > i; j--)
610 (void) AC_MEMCPY((char *) &lower[j], (char *) &lower[j - 1],
614 lower[i].key = cases[1]; /* Lower */
615 lower[i].other1 = cases[0]; /* Upper */
616 lower[i].other2 = cases[2]; /* Title */
622 ordered_ccl_insert(unsigned long c, unsigned long ccl_code)
626 if (ccl_used == ccl_size) {
628 ccl = (unsigned long *) malloc(sizeof(unsigned long) * 24);
630 ccl = (unsigned long *)
631 realloc((char *) ccl, sizeof(unsigned long) * (ccl_size + 24));
636 * Optimize adding the first item.
646 * Handle the special case of extending the range on the end. This
647 * requires that the combining class codes are the same.
649 if (ccl_code == ccl[ccl_used - 1] && c == ccl[ccl_used - 2] + 1) {
650 ccl[ccl_used - 2] = c;
655 * Handle the special case of adding another range on the end.
657 if (c > ccl[ccl_used - 2] + 1 ||
658 (c == ccl[ccl_used - 2] + 1 && ccl_code != ccl[ccl_used - 1])) {
661 ccl[ccl_used++] = ccl_code;
666 * Locate either the insertion point or range for the code.
668 for (i = 0; i < ccl_used && c > ccl[i + 1] + 1; i += 3) ;
670 if (ccl_code == ccl[i + 2] && c == ccl[i + 1] + 1) {
672 * Extend an existing range.
676 } else if (c < ccl[i]) {
678 * Start a new range before the current location.
680 for (j = ccl_used; j > i; j -= 3) {
682 ccl[j - 1] = ccl[j - 4];
683 ccl[j - 2] = ccl[j - 5];
685 ccl[i] = ccl[i + 1] = c;
686 ccl[i + 2] = ccl_code;
691 * Adds a number if it does not already exist and returns an index value
695 make_number(short num, short denom)
700 * Determine if the number already exists.
702 for (n = 0; n < nums_used; n++) {
703 if (nums[n].numerator == num && nums[n].denominator == denom)
707 if (nums_used == nums_size) {
709 nums = (_num_t *) malloc(sizeof(_num_t) << 3);
711 nums = (_num_t *) realloc((char *) nums,
712 sizeof(_num_t) * (nums_size + 8));
717 nums[n].numerator = num;
718 nums[n].denominator = denom;
724 add_number(unsigned long code, short num, short denom)
729 * Insert the code in order.
731 for (i = 0; i < ncodes_used && code > ncodes[i].code; i++) ;
734 * Handle the case of the codes matching and simply replace the number
735 * that was there before.
737 if (ncodes_used > 0 && code == ncodes[i].code) {
738 ncodes[i].idx = make_number(num, denom);
743 * Resize the array if necessary.
745 if (ncodes_used == ncodes_size) {
746 if (ncodes_size == 0)
747 ncodes = (_codeidx_t *) malloc(sizeof(_codeidx_t) << 3);
749 ncodes = (_codeidx_t *)
750 realloc((char *) ncodes, sizeof(_codeidx_t) * (ncodes_size + 8));
756 * Shift things around to insert the code if necessary.
758 if (i < ncodes_used) {
759 for (j = ncodes_used; j > i; j--) {
760 ncodes[j].code = ncodes[j - 1].code;
761 ncodes[j].idx = ncodes[j - 1].idx;
764 ncodes[i].code = code;
765 ncodes[i].idx = make_number(num, denom);
771 * This routine assumes that the line is a valid Unicode Character Database
777 unsigned long i, lineno, skip, code, ccl_code;
778 short wnum, neg, number[2];
779 char line[512], *s, *e;
782 while (fscanf(in, "%[^\n]\n", line) != EOF) {
786 * Skip blank lines and lines that start with a '#'.
788 if (line[0] == 0 || line[0] == '#')
792 * If lines need to be skipped, do it here.
800 * Collect the code. The code can be up to 6 hex digits in length to
801 * allow surrogates to be specified.
803 for (s = line, i = code = 0; *s != ';' && i < 6; i++, s++) {
805 if (*s >= '0' && *s <= '9')
807 else if (*s >= 'A' && *s <= 'F')
808 code += (*s - 'A') + 10;
809 else if (*s >= 'a' && *s <= 'f')
810 code += (*s - 'a') + 10;
814 * Handle the following special cases:
815 * 1. 4E00-9FA5 CJK Ideographs.
816 * 2. AC00-D7A3 Hangul Syllables.
817 * 3. D800-DFFF Surrogates.
818 * 4. E000-F8FF Private Use Area.
819 * 5. F900-FA2D Han compatibility.
824 * The Han ideographs.
826 add_range(0x4e00, 0x9fff, "Lo", "L");
829 * Add the characters to the defined category.
831 add_range(0x4e00, 0x9fa5, "Cp", 0);
837 * The Hangul syllables.
839 add_range(0xac00, 0xd7a3, "Lo", "L");
842 * Add the characters to the defined category.
844 add_range(0xac00, 0xd7a3, "Cp", 0);
850 * Make a range of all surrogates and assume some default
853 add_range(0x010000, 0x10ffff, "Cs", "L");
858 * The Private Use area. Add with a default set of properties.
860 add_range(0xe000, 0xf8ff, "Co", "L");
865 * The CJK compatibility area.
867 add_range(0xf900, 0xfaff, "Lo", "L");
870 * Add the characters to the defined category.
872 add_range(0xf900, 0xfaff, "Cp", 0);
881 * Add the code to the defined category.
883 ordered_range_insert(code, "Cp", 2);
886 * Locate the first character property field.
888 for (i = 0; *s != 0 && i < 2; s++) {
892 for (e = s; *e && *e != ';'; e++) ;
894 ordered_range_insert(code, s, e - s);
897 * Locate the combining class code.
899 for (s = e; *s != 0 && i < 3; s++) {
905 * Convert the combining class code from decimal.
907 for (ccl_code = 0, e = s; *e && *e != ';'; e++)
908 ccl_code = (ccl_code * 10) + (*e - '0');
911 * Add the code if it not 0.
914 ordered_ccl_insert(code, ccl_code);
917 * Locate the second character property field.
919 for (s = e; *s != 0 && i < 4; s++) {
923 for (e = s; *e && *e != ';'; e++) ;
925 ordered_range_insert(code, s, e - s);
928 * Check for a decomposition.
931 if (*s != ';' && *s != '<') {
933 * Collect the codes of the decomposition.
935 for (dectmp_size = 0; *s != ';'; ) {
937 * Skip all leading non-hex digits.
939 while (!ishdigit(*s))
942 for (dectmp[dectmp_size] = 0; ishdigit(*s); s++) {
943 dectmp[dectmp_size] <<= 4;
944 if (*s >= '0' && *s <= '9')
945 dectmp[dectmp_size] += *s - '0';
946 else if (*s >= 'A' && *s <= 'F')
947 dectmp[dectmp_size] += (*s - 'A') + 10;
948 else if (*s >= 'a' && *s <= 'f')
949 dectmp[dectmp_size] += (*s - 'a') + 10;
955 * If there are any codes in the temporary decomposition array,
956 * then add the character with its decomposition.
963 * Skip to the number field.
965 for (i = 0; i < 3 && *s; s++) {
971 * Scan the number in.
973 number[0] = number[1] = 0;
974 for (e = s, neg = wnum = 0; *e && *e != ';'; e++) {
982 * Move the the denominator of the fraction.
990 number[wnum] = (number[wnum] * 10) + (*e - '0');
995 * Adjust the denominator in case of integers and add the number.
998 number[1] = number[0];
1000 add_number(code, number[0], number[1]);
1004 * Skip to the start of the possible case mappings.
1006 for (s = e, i = 0; i < 4 && *s; s++) {
1012 * Collect the case mappings.
1014 cases[0] = cases[1] = cases[2] = 0;
1015 for (i = 0; i < 3; i++) {
1016 while (ishdigit(*s)) {
1018 if (*s >= '0' && *s <= '9')
1019 cases[i] += *s - '0';
1020 else if (*s >= 'A' && *s <= 'F')
1021 cases[i] += (*s - 'A') + 10;
1022 else if (*s >= 'a' && *s <= 'f')
1023 cases[i] += (*s - 'a') + 10;
1029 if (cases[0] && cases[1])
1031 * Add the upper and lower mappings for a title case character.
1036 * Add the lower and title case mappings for the upper case
1042 * Add the upper and title case mappings for the lower case
1050 find_decomp(unsigned long code)
1055 r = decomps_used - 1;
1058 if (code > decomps[m].code)
1060 else if (code < decomps[m].code)
1069 decomp_it(_decomp_t *d)
1074 for (i = 0; i < d->used; i++) {
1075 if ((dp = find_decomp(d->decomp[i])) != 0)
1078 dectmp[dectmp_size++] = d->decomp[i];
1083 * Expand all decompositions by recursively decomposing each character
1084 * in the decomposition.
1091 for (i = 0; i < decomps_used; i++) {
1093 decomp_it(&decomps[i]);
1094 if (dectmp_size > 0)
1095 add_decomp(decomps[i].code);
1100 cmpcomps(_comp_t *comp1, _comp_t *comp2)
1102 long diff = comp1->code1 - comp2->code1;
1105 diff = comp1->code2 - comp2->code2;
1110 * Load composition exclusion data
1113 read_compexdata(FILE *in)
1115 unsigned short i, code;
1118 (void) memset((char *) compexs, 0, sizeof(unsigned long) << 11);
1120 while (fscanf(in, "%[^\n]\n", line) != EOF) {
1122 * Skip blank lines and lines that start with a '#'.
1124 if (line[0] == 0 || line[0] == '#')
1128 * Collect the code. Assume max 4 digits
1131 for (s = line, i = code = 0; *s != '#' && i < 4; i++, s++) {
1133 if (*s >= '0' && *s <= '9')
1135 else if (*s >= 'A' && *s <= 'F')
1136 code += (*s - 'A') + 10;
1137 else if (*s >= 'a' && *s <= 'f')
1138 code += (*s - 'a') + 10;
1145 * Creates array of compositions from decomposition array
1150 unsigned long i, cu;
1152 comps = (_comp_t *) malloc(comps_used * sizeof(_comp_t));
1154 for (i = cu = 0; i < decomps_used; i++) {
1155 if (decomps[i].used != 2 || COMPEX_TEST(decomps[i].code))
1157 comps[cu].comp = decomps[i].code;
1158 comps[cu].count = 2;
1159 comps[cu].code1 = decomps[i].decomp[0];
1160 comps[cu].code2 = decomps[i].decomp[1];
1164 qsort(comps, comps_used, sizeof(_comp_t),
1165 (int (*)(const void *, const void *)) cmpcomps);
1169 write_cdata(char *opath)
1172 unsigned long i, idx, bytes, nprops;
1173 unsigned short casecnt[2];
1176 /*****************************************************************
1178 * Generate the ctype data.
1180 *****************************************************************/
1183 * Open the ctype.dat file.
1185 sprintf(path, "%s%sctype.dat", opath, LDAP_DIRSEP);
1186 if ((out = fopen(path, "wb")) == 0)
1190 * Collect the offsets for the properties. The offsets array is
1191 * on a 4-byte boundary to keep things efficient for architectures
1192 * that need such a thing.
1194 for (i = idx = 0; i < NUMPROPS; i++) {
1195 propcnt[i] = (proptbl[i].used != 0) ? idx : 0xffff;
1196 idx += proptbl[i].used;
1200 * Add the sentinel index which is used by the binary search as the upper
1201 * bound for a search.
1206 * Record the actual number of property lists. This may be different than
1207 * the number of offsets actually written because of aligning on a 4-byte
1213 * Calculate the byte count needed and pad the property counts array to a
1216 if ((bytes = sizeof(unsigned short) * (NUMPROPS + 1)) & 3)
1217 bytes += 4 - (bytes & 3);
1218 nprops = bytes / sizeof(unsigned short);
1219 bytes += sizeof(unsigned long) * idx;
1224 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1227 * Write the byte count.
1229 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1232 * Write the property list counts.
1234 fwrite((char *) propcnt, sizeof(unsigned short), nprops, out);
1237 * Write the property lists.
1239 for (i = 0; i < NUMPROPS; i++) {
1240 if (proptbl[i].used > 0)
1241 fwrite((char *) proptbl[i].ranges, sizeof(unsigned long),
1242 proptbl[i].used, out);
1247 /*****************************************************************
1249 * Generate the case mapping data.
1251 *****************************************************************/
1254 * Open the case.dat file.
1256 sprintf(path, "%s%scase.dat", opath, LDAP_DIRSEP);
1257 if ((out = fopen(path, "wb")) == 0)
1261 * Write the case mapping tables.
1263 hdr[1] = upper_used + lower_used + title_used;
1264 casecnt[0] = upper_used;
1265 casecnt[1] = lower_used;
1270 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1273 * Write the upper and lower case table sizes.
1275 fwrite((char *) casecnt, sizeof(unsigned short), 2, out);
1279 * Write the upper case table.
1281 fwrite((char *) upper, sizeof(_case_t), upper_used, out);
1285 * Write the lower case table.
1287 fwrite((char *) lower, sizeof(_case_t), lower_used, out);
1291 * Write the title case table.
1293 fwrite((char *) title, sizeof(_case_t), title_used, out);
1297 /*****************************************************************
1299 * Generate the composition data.
1301 *****************************************************************/
1304 * Create compositions from decomposition data
1309 * Open the comp.dat file.
1311 sprintf(path, "%s%scomp.dat", opath, LDAP_DIRSEP);
1312 if ((out = fopen(path, "wb")) == 0)
1318 hdr[1] = (unsigned short) comps_used * 4;
1319 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1322 * Write out the byte count to maintain header size.
1324 bytes = comps_used * sizeof(_comp_t);
1325 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1328 * Now, if comps exist, write them out.
1331 fwrite((char *) comps, sizeof(_comp_t), comps_used, out);
1335 /*****************************************************************
1337 * Generate the decomposition data.
1339 *****************************************************************/
1342 * Fully expand all decompositions before generating the output file.
1347 * Open the decomp.dat file.
1349 sprintf(path, "%s%sdecomp.dat", opath, LDAP_DIRSEP);
1350 if ((out = fopen(path, "wb")) == 0)
1353 hdr[1] = decomps_used;
1358 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1361 * Write a temporary byte count which will be calculated as the
1362 * decompositions are written out.
1365 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1369 * Write the list of decomp nodes.
1371 for (i = idx = 0; i < decomps_used; i++) {
1372 fwrite((char *) &decomps[i].code, sizeof(unsigned long), 1, out);
1373 fwrite((char *) &idx, sizeof(unsigned long), 1, out);
1374 idx += decomps[i].used;
1378 * Write the sentinel index as the last decomp node.
1380 fwrite((char *) &idx, sizeof(unsigned long), 1, out);
1383 * Write the decompositions themselves.
1385 for (i = 0; i < decomps_used; i++)
1386 fwrite((char *) decomps[i].decomp, sizeof(unsigned long),
1387 decomps[i].used, out);
1390 * Seek back to the beginning and write the byte count.
1392 bytes = (sizeof(unsigned long) * idx) +
1393 (sizeof(unsigned long) * ((hdr[1] << 1) + 1));
1394 fseek(out, sizeof(unsigned short) << 1, 0L);
1395 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1400 /*****************************************************************
1402 * Generate the combining class data.
1404 *****************************************************************/
1407 * Open the cmbcl.dat file.
1409 sprintf(path, "%s%scmbcl.dat", opath, LDAP_DIRSEP);
1410 if ((out = fopen(path, "wb")) == 0)
1414 * Set the number of ranges used. Each range has a combining class which
1415 * means each entry is a 3-tuple.
1417 hdr[1] = ccl_used / 3;
1422 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1425 * Write out the byte count to maintain header size.
1427 bytes = ccl_used * sizeof(unsigned long);
1428 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1432 * Write the combining class ranges out.
1434 fwrite((char *) ccl, sizeof(unsigned long), ccl_used, out);
1438 /*****************************************************************
1440 * Generate the number data.
1442 *****************************************************************/
1445 * Open the num.dat file.
1447 sprintf(path, "%s%snum.dat", opath, LDAP_DIRSEP);
1448 if ((out = fopen(path, "wb")) == 0)
1452 * The count part of the header will be the total number of codes that
1455 hdr[1] = (unsigned short) (ncodes_used << 1);
1456 bytes = (ncodes_used * sizeof(_codeidx_t)) + (nums_used * sizeof(_num_t));
1461 fwrite((char *) hdr, sizeof(unsigned short), 2, out);
1464 * Write out the byte count to maintain header size.
1466 fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
1469 * Now, if number mappings exist, write them out.
1471 if (ncodes_used > 0) {
1472 fwrite((char *) ncodes, sizeof(_codeidx_t), ncodes_used, out);
1473 fwrite((char *) nums, sizeof(_num_t), nums_used, out);
1483 "Usage: %s [-o output-directory|-x composition-exclusions]", prog);
1484 fprintf(stderr, " datafile1 datafile2 ...\n\n");
1486 "-o output-directory\n\t\tWrite the output files to a different");
1487 fprintf(stderr, " directory (default: .).\n");
1489 "-x composition-exclusion\n\t\tFile of composition codes");
1490 fprintf(stderr, " that should be excluded.\n");
1495 main(int argc, char *argv[])
1500 if ((prog = strrchr(argv[0], *LDAP_DIRSEP)) != 0)
1512 if (argv[0][0] == '-') {
1513 switch (argv[0][1]) {
1522 if ((in = fopen(argv[0], "rb")) == 0)
1524 "%s: unable to open composition exclusion file %s\n",
1527 read_compexdata(in);
1536 if (in != stdin && in != NULL)
1538 if ((in = fopen(argv[0], "rb")) == 0)
1539 fprintf(stderr, "%s: unable to open ctype file %s\n",