* COPYING RESTRICTIONS APPLY, see COPYRIGHT file
*/
/*
- * Copyright 1999 Computing Research Labs, New Mexico State University
+ * Copyright 2001 Computing Research Labs, New Mexico State University
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
-/* $Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $" */
+/* $Id: ucdata.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
#include "portable.h"
return _uccase_lookup(code, l, r, field);
}
+/**************************************************************************
+ *
+ * Support for compositions.
+ *
+ **************************************************************************/
+
+static unsigned long _uccomp_size;
+static unsigned long *_uccomp_data;
+
+/*
+ * Return -1 on error, 0 if okay
+ */
+static int
+_uccomp_load(char *paths, int reload)
+{
+ FILE *in;
+ unsigned long size, i;
+ _ucheader_t hdr;
+
+ if (_uccomp_size > 0) {
+ if (!reload)
+ /*
+ * The compositions have already been loaded.
+ */
+ return 0;
+
+ free((char *) _uccomp_data);
+ _uccomp_size = 0;
+ }
+
+ if ((in = _ucopenfile(paths, "comp.dat", "rb")) == 0)
+ return -1;
+
+ /*
+ * Load the header.
+ */
+ fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
+
+ if (hdr.bom == 0xfffe) {
+ hdr.cnt = endian_short(hdr.cnt);
+ hdr.size.bytes = endian_long(hdr.size.bytes);
+ }
+
+ _uccomp_size = hdr.cnt;
+ _uccomp_data = (unsigned long *) malloc(hdr.size.bytes);
+
+ /*
+ * Read the composition data in.
+ */
+ size = hdr.size.bytes / sizeof(unsigned long);
+ fread((char *) _uccomp_data, sizeof(unsigned long), size, in);
+
+ /*
+ * Do an endian swap if necessary.
+ */
+ if (hdr.bom == 0xfffe) {
+ for (i = 0; i < size; i++)
+ _uccomp_data[i] = endian_long(_uccomp_data[i]);
+ }
+
+ /*
+ * Assume that the data is ordered on count, so that all compositions
+ * of length 2 come first. Only handling length 2 for now.
+ */
+ for (i = 1; i < size; i += 4)
+ if (_uccomp_data[i] != 2)
+ break;
+ _uccomp_size = i - 1;
+
+ return 0;
+}
+
+static void
+_uccomp_unload(void)
+{
+ if (_uccomp_size == 0)
+ return;
+
+ free((char *) _uccomp_data);
+ _uccomp_size = 0;
+}
+
+int
+uccomp(unsigned long node1, unsigned long node2, unsigned long *comp)
+{
+ int l, r, m;
+
+ l = 0;
+ r = _uccomp_size - 1;
+
+ while (l <= r) {
+ m = ((r + l) >> 1);
+ m -= m & 3;
+ if (node1 > _uccomp_data[m+2])
+ l = m + 4;
+ else if (node1 < _uccomp_data[m+2])
+ r = m - 4;
+ else if (node2 > _uccomp_data[m+3])
+ l = m + 4;
+ else if (node2 < _uccomp_data[m+3])
+ r = m - 4;
+ else {
+ *comp = _uccomp_data[m];
+ return 1;
+ }
+ }
+ return 0;
+}
+
+int
+uccomp_hangul(unsigned long *str, int len)
+{
+ const int SBase = 0xAC00, LBase = 0x1100,
+ VBase = 0x1161, TBase = 0x11A7,
+ LCount = 19, VCount = 21, TCount = 28,
+ NCount = VCount * TCount, /* 588 */
+ SCount = LCount * NCount; /* 11172 */
+
+ int i, rlen;
+ unsigned long ch, last, lindex, sindex;
+
+ last = str[0];
+ rlen = 1;
+ for ( i = 1; i < len; i++ ) {
+ ch = str[i];
+
+ /* check if two current characters are L and V */
+ lindex = last - LBase;
+ if (0 <= lindex && lindex < LCount) {
+ unsigned long vindex = ch - VBase;
+ if (0 <= vindex && vindex < VCount) {
+ /* make syllable of form LV */
+ last = SBase + (lindex * VCount + vindex) * TCount;
+ str[rlen-1] = last; /* reset last */
+ continue;
+ }
+ }
+
+ /* check if two current characters are LV and T */
+ sindex = last - SBase;
+ if (0 <= sindex && sindex < SCount && (sindex % TCount) == 0) {
+ unsigned long tindex = ch - TBase;
+ if (0 <= tindex && tindex <= TCount) {
+ /* make syllable of form LVT */
+ last += tindex;
+ str[rlen-1] = last; /* reset last */
+ continue;
+ }
+ }
+
+ /* if neither case was true, just add the character */
+ last = ch;
+ str[rlen] = ch;
+ rlen++;
+ }
+ return rlen;
+}
+
+int
+uccanoncomp(unsigned long *str, int len)
+{
+ int i, stpos, copos;
+ unsigned long cl, prevcl, st, ch, co;
+
+ st = str[0];
+ stpos = 0;
+ copos = 1;
+ prevcl = uccombining_class(st) == 0 ? 0 : 256;
+
+ for (i = 1; i < len; i++) {
+ ch = str[i];
+ cl = uccombining_class(ch);
+ if (uccomp(st, ch, &co) && (prevcl < cl || prevcl == 0))
+ st = str[stpos] = co;
+ else {
+ if (cl == 0) {
+ stpos = copos;
+ st = ch;
+ }
+ prevcl = cl;
+ str[copos++] = ch;
+ }
+ }
+
+ return uccomp_hangul(str, copos);
+}
+
/**************************************************************************
*
* Support for decompositions.
if (_ucdcmp_size > 0) {
if (!reload)
- /*
- * The decompositions have already been loaded.
- */
+ /*
+ * The decompositions have already been loaded.
+ */
return 0;
free((char *) _ucdcmp_nodes);
}
if ((in = _ucopenfile(paths, "decomp.dat", "rb")) == 0)
- return -1;
+ return -1;
/*
* Load the header.
*/
if (hdr.bom == 0xfffe) {
for (i = 0; i < size; i++)
- _ucdcmp_nodes[i] = endian_long(_ucdcmp_nodes[i]);
+ _ucdcmp_nodes[i] = endian_long(_ucdcmp_nodes[i]);
}
return 0;
}
return 1;
}
+int
+uccanondecomp(const unsigned long *in, int inlen,
+ unsigned long **out, int *outlen)
+{
+ int i, j, k, l, size;
+ unsigned long num, class, *decomp, hangdecomp[3];
+
+ size = inlen;
+ *out = (unsigned long *) malloc(size * sizeof(**out));
+ if (*out == NULL)
+ return *outlen = -1;
+
+ i = 0;
+ for (j = 0; j < inlen; j++) {
+ if (ucdecomp(in[j], &num, &decomp)) {
+ if (size - i < num) {
+ size = inlen + i - j + num - 1;
+ *out = (unsigned long *) realloc(*out, size * sizeof(**out));
+ if (*out == NULL)
+ return *outlen = -1;
+ }
+ for (k = 0; k < num; k++) {
+ class = uccombining_class(decomp[k]);
+ if (class == 0) {
+ (*out)[i] = decomp[k];
+ } else {
+ for (l = i; l > 0; l--)
+ if (class >= uccombining_class((*out)[l-1]))
+ break;
+ memmove(*out + l + 1, *out + l, (i - l) * sizeof(**out));
+ (*out)[l] = decomp[k];
+ }
+ i++;
+ }
+ } else if (ucdecomp_hangul(in[j], &num, hangdecomp)) {
+ if (size - i < num) {
+ size = inlen + i - j + num - 1;
+ *out = (unsigned long *) realloc(*out, size * sizeof(**out));
+ if (*out == NULL)
+ return *outlen = -1;
+ }
+ for (k = 0; k < num; k++) {
+ (*out)[i] = hangdecomp[k];
+ i++;
+ }
+ } else {
+ if (size - i < 1) {
+ size = inlen + i - j;
+ *out = (unsigned long *) realloc(*out, size * sizeof(**out));
+ if (*out == NULL)
+ return *outlen = -1;
+ }
+ class = uccombining_class(in[j]);
+ if (class == 0) {
+ (*out)[i] = in[j];
+ } else {
+ for (l = i; l > 0; l--)
+ if (class >= uccombining_class((*out)[l-1]))
+ break;
+ memmove(*out + l + 1, *out + l, (i - l) * sizeof(**out));
+ (*out)[l] = in[j];
+ }
+ i++;
+ }
+ }
+ return *outlen = i;
+}
+
/**************************************************************************
*
* Support for combining classes.
if (_uccmcl_size > 0) {
if (!reload)
- /*
- * The combining classes have already been loaded.
- */
- return 0;
+ /*
+ * The combining classes have already been loaded.
+ */
+ return 0;
free((char *) _uccmcl_nodes);
_uccmcl_size = 0;
}
if ((in = _ucopenfile(paths, "cmbcl.dat", "rb")) == 0)
- return -1;
+ return -1;
/*
* Load the header.
*/
if (hdr.bom == 0xfffe) {
for (i = 0; i < _uccmcl_size; i++)
- _uccmcl_nodes[i] = endian_long(_uccmcl_nodes[i]);
+ _uccmcl_nodes[i] = endian_long(_uccmcl_nodes[i]);
}
return 0;
}
ucdata_load(char *paths, int masks)
{
int error = 0;
+
if (masks & UCDATA_CTYPE)
- error |= _ucprop_load(paths, 0) < 0 ? UCDATA_CTYPE : 0;
+ error |= _ucprop_load(paths, 0) < 0 ? UCDATA_CTYPE : 0;
if (masks & UCDATA_CASE)
- error |= _uccase_load(paths, 0) < 0 ? UCDATA_CASE : 0;
+ error |= _uccase_load(paths, 0) < 0 ? UCDATA_CASE : 0;
if (masks & UCDATA_DECOMP)
- error |= _ucdcmp_load(paths, 0) < 0 ? UCDATA_DECOMP : 0;
+ error |= _ucdcmp_load(paths, 0) < 0 ? UCDATA_DECOMP : 0;
if (masks & UCDATA_CMBCL)
- error |= _uccmcl_load(paths, 0) < 0 ? UCDATA_CMBCL : 0;
+ error |= _uccmcl_load(paths, 0) < 0 ? UCDATA_CMBCL : 0;
if (masks & UCDATA_NUM)
- error |= _ucnumb_load(paths, 0) < 0 ? UCDATA_NUM : 0;
+ error |= _ucnumb_load(paths, 0) < 0 ? UCDATA_NUM : 0;
+ if (masks & UCDATA_COMP)
+ error |= _uccomp_load(paths, 0) < 0 ? UCDATA_COMP : 0;
+
return -error;
}
_uccmcl_unload();
if (masks & UCDATA_NUM)
_ucnumb_unload();
+ if (masks & UCDATA_COMP)
+ _uccomp_unload();
}
/*
ucdata_reload(char *paths, int masks)
{
int error = 0;
+
if (masks & UCDATA_CTYPE)
- error |= _ucprop_load(paths, 1) < 0 ? UCDATA_CTYPE : 0;
+ error |= _ucprop_load(paths, 1) < 0 ? UCDATA_CTYPE : 0;
if (masks & UCDATA_CASE)
- error |= _uccase_load(paths, 1) < 0 ? UCDATA_CASE : 0;
+ error |= _uccase_load(paths, 1) < 0 ? UCDATA_CASE : 0;
if (masks & UCDATA_DECOMP)
- error |= _ucdcmp_load(paths, 1) < 0 ? UCDATA_DECOMP : 0;
+ error |= _ucdcmp_load(paths, 1) < 0 ? UCDATA_DECOMP : 0;
if (masks & UCDATA_CMBCL)
- error |= _uccmcl_load(paths, 1) < 0 ? UCDATA_CMBCL : 0;
+ error |= _uccmcl_load(paths, 1) < 0 ? UCDATA_CMBCL : 0;
if (masks & UCDATA_NUM)
- error |= _ucnumb_load(paths, 1) < 0 ? UCDATA_NUM : 0;
+ error |= _ucnumb_load(paths, 1) < 0 ? UCDATA_NUM : 0;
+ if (masks & UCDATA_COMP)
+ error |= _uccomp_load(paths, 1) < 0 ? UCDATA_COMP : 0;
+
return -error;
}
* COPYING RESTRICTIONS APPLY, see COPYRIGHT file
*/
/*
- * Copyright 1999 Computing Research Labs, New Mexico State University
+ * Copyright 2001 Computing Research Labs, New Mexico State University
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
#define _h_ucdata
/*
- * $Id: ucdata.h,v 1.5 1999/11/19 15:24:29 mleisher Exp $
+ * $Id: ucdata.h,v 1.6 2001/01/02 18:46:20 mleisher Exp $
*/
LDAP_BEGIN_DECL
#undef __
#define __(x) x
-#define UCDATA_VERSION "2.3"
+#define UCDATA_VERSION "2.4"
/**************************************************************************
*
extern unsigned long uctolower __((unsigned long code));
extern unsigned long uctotitle __((unsigned long code));
+/**************************************************************************
+ *
+ * Functions for getting compositions.
+ *
+ **************************************************************************/
+
+/*
+ * This routine determines if there exists a composition of node1 and node2.
+ * If it returns 0, there is no composition. Any other value indicates a
+ * composition was returned in comp.
+ */
+extern int uccomp __((unsigned long node1, unsigned long node2,
+ unsigned long *comp));
+
+/*
+ * Does Hangul composition on the string str with length len, and returns
+ * the length of the composed string.
+ */
+extern int uccomp_hangul __((unsigned long *str, int len));
+
+/*
+ * Does canonical composition on the string str with length len, and returns
+ * the length of the composed string.
+ */
+extern int uccanoncomp __((unsigned long *str, int len));
+
/**************************************************************************
*
* Functions for getting decompositions.
* returned.
*/
extern int ucdecomp __((unsigned long code, unsigned long *num,
-
unsigned long **decomp));
/*
extern int ucdecomp_hangul __((unsigned long code, unsigned long *num,
unsigned long decomp[]));
+/*
+ * This routine does canonical decomposition of the string in of length
+ * inlen, and returns the decomposed string in out with length outlen.
+ * The memory for out is allocated by this routine. It returns the length
+ * of the decomposed string if okay, and -1 on error.
+ */
+extern int uccanondecomp __((const unsigned long *in, int inlen,
+ unsigned long **out, int *outlen));
+
/**************************************************************************
*
* Functions for getting combining classes.
#define UCDATA_DECOMP 0x04
#define UCDATA_CMBCL 0x08
#define UCDATA_NUM 0x10
+#define UCDATA_COMP 0x20
#define UCDATA_ALL (UCDATA_CASE|UCDATA_CTYPE|UCDATA_DECOMP|\
- UCDATA_CMBCL|UCDATA_NUM)
+ UCDATA_CMBCL|UCDATA_NUM|UCDATA_COMP)
/*
* Functions to load, unload, and reload specific data files.
/* $OpenLDAP$ */
/*
- * Copyright 1999 Computing Research Labs, New Mexico State University
+ * Copyright 2001 Computing Research Labs, New Mexico State University
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
-/* $Id: ucgendat.c,v 1.3 1999/10/07 20:49:56 mleisher Exp $" */
+/* $Id: ucgendat.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
#include "portable.h"
*/
static unsigned short hdr[2] = {0xfeff, 0};
-#define NUMPROPS 49
+#define NUMPROPS 50
#define NEEDPROPS (NUMPROPS + (4 - (NUMPROPS & 3)))
typedef struct {
{"EN", 2}, {"ES", 2}, {"ET", 2}, {"AN", 2}, {"CS", 2}, {"B", 1},
{"S", 1}, {"WS", 2}, {"ON", 2},
{"Cm", 2}, {"Nb", 2}, {"Sy", 2}, {"Hd", 2}, {"Qm", 2}, {"Mr", 2},
- {"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}
+ {"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}, {"AL", 2}
};
typedef struct {
static unsigned long decomps_used;
static unsigned long decomps_size;
+/*
+ * Composition exclusion table stuff.
+ */
+#define COMPEX_SET(c) (compexs[(c) >> 15] |= (1 << ((c) & 31)))
+#define COMPEX_TEST(c) (compexs[(c) >> 15] & (1 << ((c) & 31)))
+static unsigned long compexs[2048];
+
+/*
+ * Struct for holding a composition pair, and array of composition pairs
+ */
+typedef struct {
+ unsigned long comp;
+ unsigned long count;
+ unsigned long code1;
+ unsigned long code2;
+} _comp_t;
+
+static _comp_t *comps;
+static unsigned long comps_used;
+
/*
* Types and lists for handling lists of case mappings.
*/
/*
* Deal with directionality codes introduced in Unicode 3.0.
*/
- if (len == 2) {
- if (memcmp(name, "AL", 2) == 0) {
- /*
- * Mark the Arabic letters as having RTL directionality.
- */
- len = 1;
- name = "R";
- } else if (memcmp(name, "BN", 2) == 0) {
- /*
- * Mark the control characters as being Other Neutrals.
- */
- len = 2;
- name = "ON";
- }
- } else if (len == 3 &&
- (memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
- memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
- memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0)) {
+ if ((len == 2 && memcmp(name, "BN", 2) == 0) ||
+ (len == 3 &&
+ (memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
+ memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
+ memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0))) {
/*
* Mark all of these as Other Neutral to preserve compatibility with
* older versions.
(void) AC_MEMCPY((char *) decomps[i].decomp, (char *) dectmp,
sizeof(unsigned long) * dectmp_size);
+ /*
+ * NOTICE: This needs changing later so it is more general than simply
+ * pairs. This calculation is done here to simplify allocation elsewhere.
+ */
+ if (dectmp_size == 2)
+ comps_used++;
}
static void
}
/*
- * If there is more than one code in the temporary decomposition
- * array, then add the character with its decomposition.
+ * If there are any codes in the temporary decomposition array,
+ * then add the character with its decomposition.
*/
- if (dectmp_size > 1)
+ if (dectmp_size > 0)
add_decomp(code);
}
}
}
+static int
+cmpcomps(_comp_t *comp1, _comp_t *comp2)
+{
+ long diff = comp1->code1 - comp2->code1;
+
+ if (!diff)
+ diff = comp1->code2 - comp2->code2;
+ return (int) diff;
+}
+
+/*
+ * Load composition exclusion data
+ */
+static void
+read_compexdata(FILE *in)
+{
+ unsigned short i, code;
+ char line[512], *s;
+
+ (void) memset((char *) compexs, 0, sizeof(unsigned long) << 11);
+
+ while (fscanf(in, "%[^\n]\n", line) != EOF) {
+ /*
+ * Skip blank lines and lines that start with a '#'.
+ */
+ if (line[0] == 0 || line[0] == '#')
+ continue;
+
+ /*
+ * Collect the code. Assume max 4 digits
+ */
+
+ for (s = line, i = code = 0; *s != '#' && i < 4; i++, s++) {
+ code <<= 4;
+ if (*s >= '0' && *s <= '9')
+ code += *s - '0';
+ else if (*s >= 'A' && *s <= 'F')
+ code += (*s - 'A') + 10;
+ else if (*s >= 'a' && *s <= 'f')
+ code += (*s - 'a') + 10;
+ }
+ COMPEX_SET(code);
+ }
+}
+
+/*
+ * Creates array of compositions from decomposition array
+ */
+static void
+create_comps(void)
+{
+ unsigned long i, cu;
+
+ comps = (_comp_t *) malloc(comps_used * sizeof(_comp_t));
+
+ for (i = cu = 0; i < decomps_used; i++) {
+ if (decomps[i].used != 2 || COMPEX_TEST(decomps[i].code))
+ continue;
+ comps[cu].comp = decomps[i].code;
+ comps[cu].count = 2;
+ comps[cu].code1 = decomps[i].decomp[0];
+ comps[cu].code2 = decomps[i].decomp[1];
+ cu++;
+ }
+ qsort(comps, comps_used, sizeof(_comp_t),
+ (int (*)(const void *, const void *)) cmpcomps);
+}
+
static void
write_cdata(char *opath)
{
fclose(out);
+ /*****************************************************************
+ *
+ * Generate the composition data.
+ *
+ *****************************************************************/
+
+ /*
+ * Create compositions from decomposition data
+ */
+ create_comps();
+
+ /*
+ * Open the comp.dat file.
+ */
+ sprintf(path, "%s/comp.dat", opath);
+ if ((out = fopen(path, "wb")) == 0)
+ return;
+
+ /*
+ * Write the header.
+ */
+ hdr[1] = (unsigned short) comps_used * 4;
+ fwrite((char *) hdr, sizeof(unsigned short), 2, out);
+
+ /*
+ * Write out the byte count to maintain header size.
+ */
+ bytes = comps_used * sizeof(_comp_t);
+ fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
+
+ /*
+ * Now, if comps exist, write them out.
+ */
+ if (comps_used > 0)
+ fwrite((char *) comps, sizeof(_comp_t), comps_used, out);
+
+ fclose(out);
+
/*****************************************************************
*
* Generate the decomposition data.
fclose(out);
}
+static void
+usage(char *prog)
+{
+ fprintf(stderr,
+ "Usage: %s [-o output-directory|-x composition-exclusions]", prog);
+ fprintf(stderr, " datafile1 datafile2 ...\n\n");
+ fprintf(stderr,
+ "-o output-directory\n\t\tWrite the output files to a different");
+ fprintf(stderr, " directory (default: .).\n");
+ fprintf(stderr,
+ "-x composition-exclusion\n\t\tFile of composition codes");
+ fprintf(stderr, " that should be excluded.\n");
+ exit(1);
+}
+
int
main(int argc, char *argv[])
{
argv++;
while (argc > 0) {
- if (argv[0][0] == '-' && argv[0][1] == 'o') {
- argc--;
- argv++;
- opath = argv[0];
+ if (argv[0][0] == '-') {
+ switch (argv[0][1]) {
+ case 'o':
+ argc--;
+ argv++;
+ opath = argv[0];
+ break;
+ case 'x':
+ argc--;
+ argv++;
+ if ((in = fopen(argv[0], "rb")) == 0)
+ fprintf(stderr,
+ "%s: unable to open composition exclusion file %s\n",
+ prog, argv[0]);
+ else {
+ read_compexdata(in);
+ fclose(in);
+ in = 0;
+ }
+ break;
+ default:
+ usage(prog);
+ }
} else {
if (in != stdin && in != NULL)
fclose(in);
read_cdata(in);
fclose(in);
in = 0;
- }
+ }
}
argc--;
argv++;
projects / openldap / commitdiff