X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=libraries%2Flibldap%2Futf-8.c;h=ab42cfbf5e15f8ffb6425f32283a212979e97028;hb=3b9f4a82ee478c943a66696adf9133dc9f503e16;hp=2069b591f3088cedac60474bed65c5f52a837042;hpb=aa934d0666cac55d7a07ae458511a9eb82cea3b4;p=openldap diff --git a/libraries/libldap/utf-8.c b/libraries/libldap/utf-8.c index 2069b591f3..ab42cfbf5e 100644 --- a/libraries/libldap/utf-8.c +++ b/libraries/libldap/utf-8.c @@ -1,12 +1,17 @@ /* $OpenLDAP$ */ /* - * Copyright 1998-1999 The OpenLDAP Foundation, All Rights Reserved. + * Copyright 1998-2002 The OpenLDAP Foundation, All Rights Reserved. * COPYING RESTRICTIONS APPLY, see COPYRIGHT file */ /* * Basic UTF-8 routines * + * These routines are "dumb". Though they understand UTF-8, + * they don't grok Unicode. That is, they can push bits, + * but don't have a clue what the bits represent. That's + * good enough for use with the LDAP Client SDK. + * * These routines are not optimized. */ @@ -20,24 +25,25 @@ #include #include +#include "ldap_utf8.h" + #include "ldap-int.h" #include "ldap_defaults.h" -#define UTF8_ISASCII(u) ( !((u) & ~0x7f) ) -#define UCS4_INVALID 0x80000000U +/* + * Basic UTF-8 routines + */ /* * return the number of bytes required to hold the - * NULL-terminated UTF-8 string INCLUDING the + * NULL-terminated UTF-8 string NOT INCLUDING the * termination. */ ber_len_t ldap_utf8_bytes( const char * p ) { - ber_len_t bytes = 0; + ber_len_t bytes; - if( p == NULL ) return bytes; - - while( p[bytes++] ) { + for( bytes=0; p[bytes]; bytes++ ) { /* EMPTY */ ; } @@ -47,233 +53,355 @@ ber_len_t ldap_utf8_bytes( const char * p ) ber_len_t ldap_utf8_chars( const char * p ) { /* could be optimized and could check for invalid sequences */ - ber_len_t chars; - - for( chars=0; *p ; chars++ ) { - int charlen = ldap_utf8_charlen( p ); + ber_len_t chars=0; - if( !charlen ) return chars; - - p = &p[charlen]; - }; + for( ; *p ; LDAP_UTF8_INCR(p) ) { + chars++; + } return chars; } -int ldap_utf8_charlen( const char * p ) +/* return offset to next character */ +int ldap_utf8_offset( const char * p ) { - unsigned c = * (const unsigned char *) p; + return LDAP_UTF8_NEXT(p) - p; +} - if ((c & 0xfe ) == 0xfc) { - return 6; - } +/* + * Returns length indicated by first byte. + */ +const char ldap_utf8_lentab[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 }; - if ((c & 0xfc ) == 0xf8) { - return 5; - } +int ldap_utf8_charlen( const char * p ) +{ + if (!(*p & 0x80)) + return 1; - if ((c & 0xf8 ) == 0xf0) { - return 4; - } + return ldap_utf8_lentab[*(const unsigned char *)p ^ 0x80]; +} - if ((c & 0xf0 ) == 0xe0) { - return 3; - } +/* + * Make sure the UTF-8 char used the shortest possible encoding + * returns charlen if valid, 0 if not. + * + * Here are the valid UTF-8 encodings, taken from RFC 2279 page 4. + * The table is slightly modified from that of the RFC. + * + * UCS-4 range (hex) UTF-8 sequence (binary) + * 0000 0000-0000 007F 0....... + * 0000 0080-0000 07FF 110++++. 10...... + * 0000 0800-0000 FFFF 1110++++ 10+..... 10...... + * 0001 0000-001F FFFF 11110+++ 10++.... 10...... 10...... + * 0020 0000-03FF FFFF 111110++ 10+++... 10...... 10...... 10...... + * 0400 0000-7FFF FFFF 1111110+ 10++++.. 10...... 10...... 10...... 10...... + * + * The '.' bits are "don't cares". When validating a UTF-8 sequence, + * at least one of the '+' bits must be set, otherwise the character + * should have been encoded in fewer octets. Note that in the two-octet + * case, only the first octet needs to be validated, and this is done + * in the ldap_utf8_lentab[] above. + */ - if ((c & 0xe0 ) == 0xc0) { - return 2; - } +/* mask of required bits in second octet */ +#undef c +#define c const char +c ldap_utf8_mintab[] = { + (c)0x20, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, + (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, + (c)0x30, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, + (c)0x38, (c)0x80, (c)0x80, (c)0x80, (c)0x3c, (c)0x80, (c)0x00, (c)0x00 }; +#undef c + +int ldap_utf8_charlen2( const char * p ) +{ + int i = LDAP_UTF8_CHARLEN( p ); - if ((c & 0x80 ) == 0x80) { - /* INVALID */ - return 0; + if ( i > 2 ) { + if ( !( ldap_utf8_mintab[*p & 0x1f] & p[1] ) ) + i = 0; } - - return 1; + return i; } -ber_int_t ldap_utf8_to_ucs4( const char * p ) +/* conv UTF-8 to UCS-4, useful for comparisons */ +ldap_ucs4_t ldap_x_utf8_to_ucs4( const char * p ) { + const unsigned char *c = p; + ldap_ucs4_t ch; int len, i; - ber_int_t c = * (const unsigned char *) p; - - if ((c & 0xFE ) == 0xFC) { - len = 6; - c &= 0x01; - - } else if ((c & 0xFC ) == 0xF8) { - len = 5; - c &= 0x03; + static unsigned char mask[] = { + 0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 }; - } else if ((c & 0xF8 ) == 0xF0) { - len = 4; - c &= 0x07; + len = LDAP_UTF8_CHARLEN2(p, len); - } else if ((c & 0xF0 ) == 0xE0) { - len = 3; - c &= 0x0F; + if( len == 0 ) return LDAP_UCS4_INVALID; - } else if ((c & 0xE0 ) == 0xC0) { - len = 2; - c &= 0x1F; - - } else if ((c & 0x80 ) == 0x80) { - return UCS4_INVALID; - - } else { - return c; - } + ch = c[0] & mask[len]; for(i=1; i < len; i++) { - ber_int_t ch = ((const unsigned char *) p)[i]; - - if ((ch & 0xc0) != 0x80) { - return UCS4_INVALID; + if ((c[i] & 0xc0) != 0x80) { + return LDAP_UCS4_INVALID; } - c <<= 6; - c |= ch & 0x3f; + ch <<= 6; + ch |= c[i] & 0x3f; } - return c; + return ch; } -int ldap_ucs4_to_utf8( ber_int_t c, char *buf ) +/* conv UCS-4 to UTF-8, not used */ +int ldap_x_ucs4_to_utf8( ldap_ucs4_t c, char *buf ) { int len=0; unsigned char* p = buf; - if(buf == NULL) return 0; - if ( c < 0 ) { - /* not a valid Unicode character */ + /* not a valid Unicode character */ + if ( c < 0 ) return 0; + + /* Just return length, don't convert */ + if(buf == NULL) { + if( c < 0x80 ) return 1; + else if( c < 0x800 ) return 2; + else if( c < 0x10000 ) return 3; + else if( c < 0x200000 ) return 4; + else if( c < 0x4000000 ) return 5; + else return 6; + } - } else if( c < 0x80 ) { + if( c < 0x80 ) { p[len++] = c; } else if( c < 0x800 ) { p[len++] = 0xc0 | ( c >> 6 ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( c & 0x3f ); } else if( c < 0x10000 ) { p[len++] = 0xe0 | ( c >> 12 ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); } else if( c < 0x200000 ) { p[len++] = 0xf0 | ( c >> 18 ); - p[len++] = 0x80 | ( (c >> 12) & 0x3F ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 12) & 0x3f ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); - } else if( c < 0x400000 ) { + } else if( c < 0x4000000 ) { p[len++] = 0xf8 | ( c >> 24 ); - p[len++] = 0x80 | ( (c >> 18) & 0x3F ); - p[len++] = 0x80 | ( (c >> 12) & 0x3F ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 18) & 0x3f ); + p[len++] = 0x80 | ( (c >> 12) & 0x3f ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); } else /* if( c < 0x80000000 ) */ { p[len++] = 0xfc | ( c >> 30 ); - p[len++] = 0x80 | ( (c >> 24) & 0x3F ); - p[len++] = 0x80 | ( (c >> 18) & 0x3F ); - p[len++] = 0x80 | ( (c >> 12) & 0x3F ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 24) & 0x3f ); + p[len++] = 0x80 | ( (c >> 18) & 0x3f ); + p[len++] = 0x80 | ( (c >> 12) & 0x3f ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); } - buf[len] = '\0'; return len; } -char* ldap_utf8_next( const char * p ) +#define LDAP_UCS_UTF8LEN(c) \ + c < 0 ? 0 : (c < 0x80 ? 1 : (c < 0x800 ? 2 : (c < 0x10000 ? 3 : \ + (c < 0x200000 ? 4 : (c < 0x4000000 ? 5 : 6))))) + +/* Convert a string to UTF-8 format. The input string is expected to + * have characters of 1, 2, or 4 octets (in network byte order) + * corresponding to the ASN.1 T61STRING, BMPSTRING, and UNIVERSALSTRING + * types respectively. (Here T61STRING just means that there is one + * octet per character and characters may use the high bit of the octet. + * The characters are assumed to use ISO mappings, no provision is made + * for converting from T.61 coding rules to Unicode.) + */ + +int +ldap_ucs_to_utf8s( struct berval *ucs, int csize, struct berval *utf8s ) { - int len = ldap_utf8_charlen( p ); + unsigned char *in, *end; + char *ptr; + ldap_ucs4_t u; + int i, l = 0; + + utf8s->bv_val = NULL; + utf8s->bv_len = 0; + + in = (unsigned char *)ucs->bv_val; + + /* Make sure we stop at an even multiple of csize */ + end = in + ( ucs->bv_len & ~(csize-1) ); + + for (; in < end; ) { + u = *in++; + if (csize > 1) { + u <<= 8; + u |= *in++; + } + if (csize > 2) { + u <<= 8; + u |= *in++; + u <<= 8; + u |= *in++; + } + i = LDAP_UCS_UTF8LEN(u); + if (i == 0) + return LDAP_INVALID_SYNTAX; + l += i; + } - return len ? (char *) &p[len] : NULL; + utf8s->bv_val = LDAP_MALLOC( l+1 ); + if (utf8s->bv_val == NULL) + return LDAP_NO_MEMORY; + utf8s->bv_len = l; + + ptr = utf8s->bv_val; + for (in = (unsigned char *)ucs->bv_val; in < end; ) { + u = *in++; + if (csize > 1) { + u <<= 8; + u |= *in++; + } + if (csize > 2) { + u <<= 8; + u |= *in++; + u <<= 8; + u |= *in++; + } + ptr += ldap_x_ucs4_to_utf8(u, ptr); + } + *ptr = '\0'; + return LDAP_SUCCESS; } -char* ldap_utf8_prev( const char * p ) +/* + * Advance to the next UTF-8 character + * + * Ignores length of multibyte character, instead rely on + * continuation markers to find start of next character. + * This allows for "resyncing" of when invalid characters + * are provided provided the start of the next character + * is appears within the 6 bytes examined. + */ +char* ldap_utf8_next( const char * p ) { int i; const unsigned char *u = p; - for( i = -1; i >= -6 ; i-- ) { - if ( u[i] & 0xC0 != 0x80 ) return (char *) &p[i]; + if( LDAP_UTF8_ISASCII(u) ) { + return (char *) &p[1]; } - return NULL; -} + for( i=1; i<6; i++ ) { + if ( ( u[i] & 0xc0 ) != 0x80 ) { + return (char *) &p[i]; + } + } -int ldap_utf8_isascii( const char * p ) -{ - unsigned c = * (const unsigned char *) p; - return UTF8_ISASCII(c); + return (char *) &p[i]; } -int ldap_utf8_isdigit( const char * p ) +/* + * Advance to the previous UTF-8 character + * + * Ignores length of multibyte character, instead rely on + * continuation markers to find start of next character. + * This allows for "resyncing" of when invalid characters + * are provided provided the start of the next character + * is appears within the 6 bytes examined. + */ +char* ldap_utf8_prev( const char * p ) { - unsigned c = * (const unsigned char *) p; + int i; + const unsigned char *u = p; - if(!UTF8_ISASCII(c)) return 0; + for( i=-1; i>-6 ; i-- ) { + if ( ( u[i] & 0xc0 ) != 0x80 ) { + return (char *) &p[i]; + } + } - return c >= '0' && c <= '9'; + return (char *) &p[i]; } -int ldap_utf8_isxdigit( const char * p ) +/* + * Copy one UTF-8 character from src to dst returning + * number of bytes copied. + * + * Ignores length of multibyte character, instead rely on + * continuation markers to find start of next character. + * This allows for "resyncing" of when invalid characters + * are provided provided the start of the next character + * is appears within the 6 bytes examined. + */ +int ldap_utf8_copy( char* dst, const char *src ) { - unsigned c = * (const unsigned char *) p; - - if(!UTF8_ISASCII(c)) return 0; + int i; + const unsigned char *u = src; - return ( c >= '0' && c <= '9' ) - || ( c >= 'A' && c <= 'F' ) - || ( c >= 'a' && c <= 'f' ); -} + dst[0] = src[0]; -int ldap_utf8_isalpha( const char * p ) -{ - unsigned c = * (const unsigned char *) p; + if( LDAP_UTF8_ISASCII(u) ) { + return 1; + } - if(!UTF8_ISASCII(c)) return 0; + for( i=1; i<6; i++ ) { + if ( ( u[i] & 0xc0 ) != 0x80 ) { + return i; + } + dst[i] = src[i]; + } - return ( c >= 'A' && c <= 'Z' ) - || ( c >= 'a' && c <= 'z' ); + return i; } -int ldap_utf8_isalnum( const char * p ) +#ifndef UTF8_ALPHA_CTYPE +/* + * UTF-8 ctype routines + * Only deals with characters < 0x80 (ie: US-ASCII) + */ + +int ldap_utf8_isascii( const char * p ) { unsigned c = * (const unsigned char *) p; - - if(!UTF8_ISASCII(c)) return 0; - - return ( c >= '0' && c <= '9' ) - || ( c >= 'A' && c <= 'Z' ) - || ( c >= 'a' && c <= 'z' ); + return LDAP_ASCII(c); } -int ldap_utf8_islower( const char * p ) +int ldap_utf8_isdigit( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_ASCII(c)) return 0; - return ( c >= 'a' && c <= 'z' ); + return LDAP_DIGIT( c ); } -int ldap_utf8_isupper( const char * p ) +int ldap_utf8_isxdigit( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_ASCII(c)) return 0; - return ( c >= 'A' && c <= 'Z' ); + return LDAP_HEX(c); } int ldap_utf8_isspace( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_ASCII(c)) return 0; switch(c) { case ' ': @@ -288,122 +416,119 @@ int ldap_utf8_isspace( const char * p ) return 0; } -char* ldap_utf8_fgetc( FILE *s, char *buf ) +/* + * These are not needed by the C SDK and are + * not "good enough" for general use. + */ +int ldap_utf8_isalpha( const char * p ) { - int i; - unsigned char *p; - unsigned int c; - int len; + unsigned c = * (const unsigned char *) p; - if( s == NULL ) return NULL; + if(!LDAP_ASCII(c)) return 0; - p = buf; + return LDAP_ALPHA(c); +} - c = fgetc( s ); - if( c == EOF ) { - p[0] = -1; - return NULL; - } +int ldap_utf8_isalnum( const char * p ) +{ + unsigned c = * (const unsigned char *) p; + + if(!LDAP_ASCII(c)) return 0; - p[0] = c; + return LDAP_ALNUM(c); +} - len = ldap_utf8_charlen( buf ); +int ldap_utf8_islower( const char * p ) +{ + unsigned c = * (const unsigned char *) p; - if( len < 1 ) return NULL; + if(!LDAP_ASCII(c)) return 0; - for( i = 1; i < len; i++ ) { - unsigned int c = fgetc( s ); - if( c == EOF ) { - p[i] = -1; - return NULL; - } - if( c & 0xC0 != 0x80 ) { - ungetc( c, s ); - p[i] = -1; - return NULL; - } - p[i] = c; + return LDAP_LOWER(c); +} + +int ldap_utf8_isupper( const char * p ) +{ + unsigned c = * (const unsigned char *) p; + + if(!LDAP_ASCII(c)) return 0; + + return LDAP_UPPER(c); +} +#endif + + +/* + * UTF-8 string routines + */ + +/* like strchr() */ +char * (ldap_utf8_strchr)( const char *str, const char *chr ) +{ + for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) { + if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( chr ) ) { + return (char *) str; + } } - return buf; + return NULL; } +/* like strcspn() but returns number of bytes, not characters */ ber_len_t (ldap_utf8_strcspn)( const char *str, const char *set ) { - int len; const char *cstr; + const char *cset; - for( cstr = str; *cstr != '\0'; cstr += len ) { - const char *cset; - - for( cset = set; ; cset += len ) { - if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) { + for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) { + for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) { + if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) { return cstr - str; } - - len = ldap_utf8_charlen(cset); - if( !len ) break; } - - len = ldap_utf8_charlen(cstr); - if( !len ) break; } return cstr - str; } +/* like strspn() but returns number of bytes, not characters */ ber_len_t (ldap_utf8_strspn)( const char *str, const char *set ) { - int len; const char *cstr; + const char *cset; - for( cstr = str; *cstr != '\0'; cstr += len ) { - const char *cset; - - for( cset = set; ; cset += len ) { + for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) { + for( cset = set; ; LDAP_UTF8_INCR(cset) ) { if( *cset == '\0' ) { return cstr - str; } - if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) { + if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) { break; } - - len = ldap_utf8_charlen(cset); - if( !len ) break; } - - len = ldap_utf8_charlen(cstr); - if( !len ) break; } return cstr - str; } +/* like strpbrk(), replaces strchr() as well */ char *(ldap_utf8_strpbrk)( const char *str, const char *set ) { - int len; - const char *cstr; - - for( cstr = str; *cstr != '\0'; cstr += len ) { + for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) { const char *cset; - for( cset = set; ; cset += len ) { - if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) { - return cstr; + for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) { + if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( cset ) ) { + return (char *) str; } - - len = ldap_utf8_charlen(cset); - if( !len ) break; } - - len = ldap_utf8_charlen(cstr); - if( !len ) break; } return NULL; } +/* like strtok_r(), not strtok() */ char *(ldap_utf8_strtok)(char *str, const char *sep, char **last) { char *begin; @@ -420,17 +545,12 @@ char *(ldap_utf8_strtok)(char *str, const char *sep, char **last) return NULL; } - end = &begin[ ldap_utf8_strcpn( begin, sep ) ]; + end = &begin[ ldap_utf8_strcspn( begin, sep ) ]; if( *end != '\0' ) { - int len = ldap_utf8_charlen( end ); + char *next = LDAP_UTF8_NEXT( end ); *end = '\0'; - - if( len ) { - end += len; - } else { - end = NULL; - } + end = next; } *last = end;