3 * Copyright 2000 The OpenLDAP Foundation, All Rights Reserved.
4 * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
7 * Copyright 1999 Computing Research Labs, New Mexico State University
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
16 * The above copyright notice and this permission notice shall be included in
17 * all copies or substantial portions of the Software.
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY
23 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
24 * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
25 * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 static char rcsid[] __attribute__ ((unused)) = "$Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $";
31 static char rcsid[] = "$Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $";
39 #include <ac/string.h>
40 #include <ac/unistd.h>
45 /**************************************************************************
47 * Miscellaneous types, data, and support functions.
49 **************************************************************************/
56 unsigned short len[2];
61 * A simple array of 32-bit masks for lookup.
63 static unsigned long masks32[32] = {
64 0x00000001, 0x00000002, 0x00000004, 0x00000008, 0x00000010, 0x00000020,
65 0x00000040, 0x00000080, 0x00000100, 0x00000200, 0x00000400, 0x00000800,
66 0x00001000, 0x00002000, 0x00004000, 0x00008000, 0x00010000, 0x00020000,
67 0x00040000, 0x00080000, 0x00100000, 0x00200000, 0x00400000, 0x00800000,
68 0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000,
69 0x40000000, 0x80000000
72 #define endian_short(cc) (((cc) >> 8) | (((cc) & 0xff) << 8))
73 #define endian_long(cc) ((((cc) & 0xff) << 24)|((((cc) >> 8) & 0xff) << 16)|\
74 ((((cc) >> 16) & 0xff) << 8)|((cc) >> 24))
77 _ucopenfile(char *paths, char *filename, char *mode)
80 char *fp, *dp, *pp, path[BUFSIZ];
82 if (filename == 0 || *filename == 0)
88 while (*dp && *dp != ':')
97 if ((f = fopen(path, mode)) != 0)
107 /**************************************************************************
109 * Support for the character properties.
111 **************************************************************************/
113 static unsigned long _ucprop_size;
114 static unsigned short *_ucprop_offsets;
115 static unsigned long *_ucprop_ranges;
118 _ucprop_load(char *paths, int reload)
121 unsigned long size, i;
124 if (_ucprop_size > 0) {
127 * The character properties have already been loaded.
132 * Unload the current character property data in preparation for
133 * loading a new copy. Only the first array has to be deallocated
134 * because all the memory for the arrays is allocated as a single
137 free((char *) _ucprop_offsets);
141 if ((in = _ucopenfile(paths, "ctype.dat", "rb")) == 0)
147 fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
149 if (hdr.bom == 0xfffe) {
150 hdr.cnt = endian_short(hdr.cnt);
151 hdr.size.bytes = endian_long(hdr.size.bytes);
154 if ((_ucprop_size = hdr.cnt) == 0) {
160 * Allocate all the storage needed for the lookup table.
162 _ucprop_offsets = (unsigned short *) malloc(hdr.size.bytes);
165 * Calculate the offset into the storage for the ranges. The offsets
166 * array is on a 4-byte boundary and one larger than the value provided in
167 * the header count field. This means the offset to the ranges must be
168 * calculated after aligning the count to a 4-byte boundary.
170 if ((size = ((hdr.cnt + 1) * sizeof(unsigned short))) & 3)
171 size += 4 - (size & 3);
173 _ucprop_ranges = (unsigned long *) (_ucprop_offsets + size);
176 * Load the offset array.
178 fread((char *) _ucprop_offsets, sizeof(unsigned short), size, in);
181 * Do an endian swap if necessary. Don't forget there is an extra node on
182 * the end with the final index.
184 if (hdr.bom == 0xfffe) {
185 for (i = 0; i <= _ucprop_size; i++)
186 _ucprop_offsets[i] = endian_short(_ucprop_offsets[i]);
190 * Load the ranges. The number of elements is in the last array position
193 fread((char *) _ucprop_ranges, sizeof(unsigned long),
194 _ucprop_offsets[_ucprop_size], in);
199 * Do an endian swap if necessary.
201 if (hdr.bom == 0xfffe) {
202 for (i = 0; i < _ucprop_offsets[_ucprop_size]; i++)
203 _ucprop_ranges[i] = endian_long(_ucprop_ranges[i]);
210 if (_ucprop_size == 0)
214 * Only need to free the offsets because the memory is allocated as a
217 free((char *) _ucprop_offsets);
222 _ucprop_lookup(unsigned long code, unsigned long n)
227 * There is an extra node on the end of the offsets to allow this routine
228 * to work right. If the index is 0xffff, then there are no nodes for the
231 if ((l = _ucprop_offsets[n]) == 0xffff)
235 * Locate the next offset that is not 0xffff. The sentinel at the end of
236 * the array is the max index value.
239 n + m < _ucprop_size && _ucprop_offsets[n + m] == 0xffff; m++) ;
241 r = _ucprop_offsets[n + m] - 1;
245 * Determine a "mid" point and adjust to make sure the mid point is at
246 * the beginning of a range pair.
250 if (code > _ucprop_ranges[m + 1])
252 else if (code < _ucprop_ranges[m])
254 else if (code >= _ucprop_ranges[m] && code <= _ucprop_ranges[m + 1])
261 ucisprop(unsigned long code, unsigned long mask1, unsigned long mask2)
265 if (mask1 == 0 && mask2 == 0)
268 for (i = 0; mask1 && i < 32; i++) {
269 if ((mask1 & masks32[i]) && _ucprop_lookup(code, i))
273 for (i = 32; mask2 && i < _ucprop_size; i++) {
274 if ((mask2 & masks32[i & 31]) && _ucprop_lookup(code, i))
281 /**************************************************************************
283 * Support for case mapping.
285 **************************************************************************/
287 static unsigned long _uccase_size;
288 static unsigned short _uccase_len[2];
289 static unsigned long *_uccase_map;
292 _uccase_load(char *paths, int reload)
298 if (_uccase_size > 0) {
301 * The case mappings have already been loaded.
305 free((char *) _uccase_map);
309 if ((in = _ucopenfile(paths, "case.dat", "rb")) == 0)
315 fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
317 if (hdr.bom == 0xfffe) {
318 hdr.cnt = endian_short(hdr.cnt);
319 hdr.size.len[0] = endian_short(hdr.size.len[0]);
320 hdr.size.len[1] = endian_short(hdr.size.len[1]);
324 * Set the node count and lengths of the upper and lower case mapping
327 _uccase_size = hdr.cnt * 3;
328 _uccase_len[0] = hdr.size.len[0] * 3;
329 _uccase_len[1] = hdr.size.len[1] * 3;
331 _uccase_map = (unsigned long *)
332 malloc(_uccase_size * sizeof(unsigned long));
335 * Load the case mapping table.
337 fread((char *) _uccase_map, sizeof(unsigned long), _uccase_size, in);
340 * Do an endian swap if necessary.
342 if (hdr.bom == 0xfffe) {
343 for (i = 0; i < _uccase_size; i++)
344 _uccase_map[i] = endian_long(_uccase_map[i]);
351 if (_uccase_size == 0)
354 free((char *) _uccase_map);
359 _uccase_lookup(unsigned long code, long l, long r, int field)
364 * Do the binary search.
368 * Determine a "mid" point and adjust to make sure the mid point is at
369 * the beginning of a case mapping triple.
373 if (code > _uccase_map[m])
375 else if (code < _uccase_map[m])
377 else if (code == _uccase_map[m])
378 return _uccase_map[m + field];
385 uctoupper(unsigned long code)
393 if (ucislower(code)) {
395 * The character is lower case.
399 r = (l + _uccase_len[1]) - 3;
402 * The character is title case.
405 l = _uccase_len[0] + _uccase_len[1];
406 r = _uccase_size - 3;
408 return _uccase_lookup(code, l, r, field);
412 uctolower(unsigned long code)
420 if (ucisupper(code)) {
422 * The character is upper case.
426 r = _uccase_len[0] - 3;
429 * The character is title case.
432 l = _uccase_len[0] + _uccase_len[1];
433 r = _uccase_size - 3;
435 return _uccase_lookup(code, l, r, field);
439 uctotitle(unsigned long code)
448 * The offset will always be the same for converting to title case.
452 if (ucisupper(code)) {
454 * The character is upper case.
457 r = _uccase_len[0] - 3;
460 * The character is lower case.
463 r = (l + _uccase_len[1]) - 3;
465 return _uccase_lookup(code, l, r, field);
468 /**************************************************************************
470 * Support for decompositions.
472 **************************************************************************/
474 static unsigned long _ucdcmp_size;
475 static unsigned long *_ucdcmp_nodes;
476 static unsigned long *_ucdcmp_decomp;
479 _ucdcmp_load(char *paths, int reload)
482 unsigned long size, i;
485 if (_ucdcmp_size > 0) {
488 * The decompositions have already been loaded.
492 free((char *) _ucdcmp_nodes);
496 if ((in = _ucopenfile(paths, "decomp.dat", "rb")) == 0)
502 fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
504 if (hdr.bom == 0xfffe) {
505 hdr.cnt = endian_short(hdr.cnt);
506 hdr.size.bytes = endian_long(hdr.size.bytes);
509 _ucdcmp_size = hdr.cnt << 1;
510 _ucdcmp_nodes = (unsigned long *) malloc(hdr.size.bytes);
511 _ucdcmp_decomp = _ucdcmp_nodes + (_ucdcmp_size + 1);
514 * Read the decomposition data in.
516 size = hdr.size.bytes / sizeof(unsigned long);
517 fread((char *) _ucdcmp_nodes, sizeof(unsigned long), size, in);
520 * Do an endian swap if necessary.
522 if (hdr.bom == 0xfffe) {
523 for (i = 0; i < size; i++)
524 _ucdcmp_nodes[i] = endian_long(_ucdcmp_nodes[i]);
531 if (_ucdcmp_size == 0)
535 * Only need to free the offsets because the memory is allocated as a
538 free((char *) _ucdcmp_nodes);
543 ucdecomp(unsigned long code, unsigned long *num, unsigned long **decomp)
548 r = _ucdcmp_nodes[_ucdcmp_size] - 1;
552 * Determine a "mid" point and adjust to make sure the mid point is at
553 * the beginning of a code+offset pair.
557 if (code > _ucdcmp_nodes[m])
559 else if (code < _ucdcmp_nodes[m])
561 else if (code == _ucdcmp_nodes[m]) {
562 *num = _ucdcmp_nodes[m + 3] - _ucdcmp_nodes[m + 1];
563 *decomp = &_ucdcmp_decomp[_ucdcmp_nodes[m + 1]];
571 ucdecomp_hangul(unsigned long code, unsigned long *num, unsigned long decomp[])
573 if (!ucishangul(code))
577 decomp[0] = 0x1100 + (unsigned long) (code / 588);
578 decomp[1] = 0x1161 + (unsigned long) ((code % 588) / 28);
579 decomp[2] = 0x11a7 + (unsigned long) (code % 28);
580 *num = (decomp[2] != 0x11a7) ? 3 : 2;
585 /**************************************************************************
587 * Support for combining classes.
589 **************************************************************************/
591 static unsigned long _uccmcl_size;
592 static unsigned long *_uccmcl_nodes;
595 _uccmcl_load(char *paths, int reload)
601 if (_uccmcl_size > 0) {
604 * The combining classes have already been loaded.
608 free((char *) _uccmcl_nodes);
612 if ((in = _ucopenfile(paths, "cmbcl.dat", "rb")) == 0)
618 fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
620 if (hdr.bom == 0xfffe) {
621 hdr.cnt = endian_short(hdr.cnt);
622 hdr.size.bytes = endian_long(hdr.size.bytes);
625 _uccmcl_size = hdr.cnt * 3;
626 _uccmcl_nodes = (unsigned long *) malloc(hdr.size.bytes);
629 * Read the combining classes in.
631 fread((char *) _uccmcl_nodes, sizeof(unsigned long), _uccmcl_size, in);
634 * Do an endian swap if necessary.
636 if (hdr.bom == 0xfffe) {
637 for (i = 0; i < _uccmcl_size; i++)
638 _uccmcl_nodes[i] = endian_long(_uccmcl_nodes[i]);
645 if (_uccmcl_size == 0)
648 free((char *) _uccmcl_nodes);
653 uccombining_class(unsigned long code)
658 r = _uccmcl_size - 1;
663 if (code > _uccmcl_nodes[m + 1])
665 else if (code < _uccmcl_nodes[m])
667 else if (code >= _uccmcl_nodes[m] && code <= _uccmcl_nodes[m + 1])
668 return _uccmcl_nodes[m + 2];
673 /**************************************************************************
675 * Support for numeric values.
677 **************************************************************************/
679 static unsigned long *_ucnum_nodes;
680 static unsigned long _ucnum_size;
681 static short *_ucnum_vals;
684 _ucnumb_load(char *paths, int reload)
687 unsigned long size, i;
690 if (_ucnum_size > 0) {
693 * The numbers have already been loaded.
697 free((char *) _ucnum_nodes);
701 if ((in = _ucopenfile(paths, "num.dat", "rb")) == 0)
707 fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
709 if (hdr.bom == 0xfffe) {
710 hdr.cnt = endian_short(hdr.cnt);
711 hdr.size.bytes = endian_long(hdr.size.bytes);
714 _ucnum_size = hdr.cnt;
715 _ucnum_nodes = (unsigned long *) malloc(hdr.size.bytes);
716 _ucnum_vals = (short *) (_ucnum_nodes + _ucnum_size);
719 * Read the combining classes in.
721 fread((char *) _ucnum_nodes, sizeof(unsigned char), hdr.size.bytes, in);
724 * Do an endian swap if necessary.
726 if (hdr.bom == 0xfffe) {
727 for (i = 0; i < _ucnum_size; i++)
728 _ucnum_nodes[i] = endian_long(_ucnum_nodes[i]);
731 * Determine the number of values that have to be adjusted.
733 size = (hdr.size.bytes -
734 (_ucnum_size * (sizeof(unsigned long) << 1))) /
737 for (i = 0; i < size; i++)
738 _ucnum_vals[i] = endian_short(_ucnum_vals[i]);
745 if (_ucnum_size == 0)
748 free((char *) _ucnum_nodes);
753 ucnumber_lookup(unsigned long code, struct ucnumber *num)
762 * Determine a "mid" point and adjust to make sure the mid point is at
763 * the beginning of a code+offset pair.
767 if (code > _ucnum_nodes[m])
769 else if (code < _ucnum_nodes[m])
772 vp = _ucnum_vals + _ucnum_nodes[m + 1];
773 num->numerator = (int) *vp++;
774 num->denominator = (int) *vp;
782 ucdigit_lookup(unsigned long code, int *digit)
791 * Determine a "mid" point and adjust to make sure the mid point is at
792 * the beginning of a code+offset pair.
796 if (code > _ucnum_nodes[m])
798 else if (code < _ucnum_nodes[m])
801 vp = _ucnum_vals + _ucnum_nodes[m + 1];
802 if (*vp == *(vp + 1)) {
813 ucgetnumber(unsigned long code)
818 * Initialize with some arbitrary value, because the caller simply cannot
819 * tell for sure if the code is a number without calling the ucisnumber()
820 * macro before calling this function.
822 num.numerator = num.denominator = -111;
824 (void) ucnumber_lookup(code, &num);
830 ucgetdigit(unsigned long code)
835 * Initialize with some arbitrary value, because the caller simply cannot
836 * tell for sure if the code is a number without calling the ucisdigit()
837 * macro before calling this function.
841 (void) ucdigit_lookup(code, &dig);
846 /**************************************************************************
848 * Setup and cleanup routines.
850 **************************************************************************/
853 ucdata_load(char *paths, int masks)
855 if (masks & UCDATA_CTYPE)
856 _ucprop_load(paths, 0);
857 if (masks & UCDATA_CASE)
858 _uccase_load(paths, 0);
859 if (masks & UCDATA_DECOMP)
860 _ucdcmp_load(paths, 0);
861 if (masks & UCDATA_CMBCL)
862 _uccmcl_load(paths, 0);
863 if (masks & UCDATA_NUM)
864 _ucnumb_load(paths, 0);
868 ucdata_unload(int masks)
870 if (masks & UCDATA_CTYPE)
872 if (masks & UCDATA_CASE)
874 if (masks & UCDATA_DECOMP)
876 if (masks & UCDATA_CMBCL)
878 if (masks & UCDATA_NUM)
883 ucdata_reload(char *paths, int masks)
885 if (masks & UCDATA_CTYPE)
886 _ucprop_load(paths, 1);
887 if (masks & UCDATA_CASE)
888 _uccase_load(paths, 1);
889 if (masks & UCDATA_DECOMP)
890 _ucdcmp_load(paths, 1);
891 if (masks & UCDATA_CMBCL)
892 _uccmcl_load(paths, 1);
893 if (masks & UCDATA_NUM)
894 _ucnumb_load(paths, 1);
903 unsigned long i, lo, *dec;
911 printf("NOT WEAK\n");
913 printf("LOWER 0x%04lX\n", uctolower(0xff3a));
914 printf("UPPER 0x%04lX\n", uctoupper(0xff5a));
916 if (ucisalpha(0x1d5))
919 printf("NOT ALPHA\n");
921 if (ucisupper(0x1d5)) {
923 lo = uctolower(0x1d5);
924 printf("0x%04lx\n", lo);
925 lo = uctotitle(0x1d5);
926 printf("0x%04lx\n", lo);
928 printf("NOT UPPER\n");
930 if (ucistitle(0x1d5))
933 printf("NOT TITLE\n");
935 if (uciscomposite(0x1d5))
936 printf("COMPOSITE\n");
938 printf("NOT COMPOSITE\n");
940 if (ucdecomp(0x1d5, &lo, &dec)) {
941 for (i = 0; i < lo; i++)
942 printf("0x%04lx ", dec[i]);
946 if ((lo = uccombining_class(0x41)) != 0)
947 printf("0x41 CCL %ld\n", lo);
949 if (ucisxdigit(0xfeff))
950 printf("0xFEFF HEX DIGIT\n");
952 printf("0xFEFF NOT HEX DIGIT\n");
954 if (ucisdefined(0x10000))
955 printf("0x10000 DEFINED\n");
957 printf("0x10000 NOT DEFINED\n");
959 if (ucnumber_lookup(0x30, &num)) {
960 if (num.numerator != num.denominator)
961 printf("UCNUMBER: 0x30 = %d/%d\n", num.numerator, num.denominator);
963 printf("UCNUMBER: 0x30 = %d\n", num.numerator);
965 printf("UCNUMBER: 0x30 NOT A NUMBER\n");
967 if (ucnumber_lookup(0xbc, &num)) {
968 if (num.numerator != num.denominator)
969 printf("UCNUMBER: 0xbc = %d/%d\n", num.numerator, num.denominator);
971 printf("UCNUMBER: 0xbc = %d\n", num.numerator);
973 printf("UCNUMBER: 0xbc NOT A NUMBER\n");
976 if (ucnumber_lookup(0xff19, &num)) {
977 if (num.numerator != num.denominator)
978 printf("UCNUMBER: 0xff19 = %d/%d\n", num.numerator, num.denominator);
980 printf("UCNUMBER: 0xff19 = %d\n", num.numerator);
982 printf("UCNUMBER: 0xff19 NOT A NUMBER\n");
984 if (ucnumber_lookup(0x4e00, &num)) {
985 if (num.numerator != num.denominator)
986 printf("UCNUMBER: 0x4e00 = %d/%d\n", num.numerator, num.denominator);
988 printf("UCNUMBER: 0x4e00 = %d\n", num.numerator);
990 printf("UCNUMBER: 0x4e00 NOT A NUMBER\n");
992 if (ucdigit_lookup(0x06f9, &dig))
993 printf("UCDIGIT: 0x6f9 = %d\n", dig);
995 printf("UCDIGIT: 0x6f9 NOT A NUMBER\n");
997 dig = ucgetdigit(0x0969);
998 printf("UCGETDIGIT: 0x969 = %d\n", dig);
1000 num = ucgetnumber(0x30);
1001 if (num.numerator != num.denominator)
1002 printf("UCGETNUMBER: 0x30 = %d/%d\n", num.numerator, num.denominator);
1004 printf("UCGETNUMBER: 0x30 = %d\n", num.numerator);
1006 num = ucgetnumber(0xbc);
1007 if (num.numerator != num.denominator)
1008 printf("UCGETNUMBER: 0xbc = %d/%d\n", num.numerator, num.denominator);
1010 printf("UCGETNUMBER: 0xbc = %d\n", num.numerator);
1012 num = ucgetnumber(0xff19);
1013 if (num.numerator != num.denominator)
1014 printf("UCGETNUMBER: 0xff19 = %d/%d\n", num.numerator, num.denominator);
1016 printf("UCGETNUMBER: 0xff19 = %d\n", num.numerator);