X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;ds=sidebyside;f=libraries%2Flibldap%2Futf-8.c;h=6b2ac969e1046d0f8be70daa30daa521fde189eb;hb=2b0819c4a9209784f762ec154ce4721038522a8a;hp=2069b591f3088cedac60474bed65c5f52a837042;hpb=aa934d0666cac55d7a07ae458511a9eb82cea3b4;p=openldap diff --git a/libraries/libldap/utf-8.c b/libraries/libldap/utf-8.c index 2069b591f3..6b2ac969e1 100644 --- a/libraries/libldap/utf-8.c +++ b/libraries/libldap/utf-8.c @@ -1,12 +1,17 @@ /* $OpenLDAP$ */ /* - * Copyright 1998-1999 The OpenLDAP Foundation, All Rights Reserved. + * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved. * COPYING RESTRICTIONS APPLY, see COPYRIGHT file */ /* * Basic UTF-8 routines * + * These routines are "dumb". Though they understand UTF-8, + * they don't grok Unicode. That is, they can push bits, + * but don't have a clue what the bits represent. That's + * good enough for use with the LDAP Client SDK. + * * These routines are not optimized. */ @@ -20,24 +25,28 @@ #include #include +#include "ldap_utf8.h" + #include "ldap-int.h" #include "ldap_defaults.h" -#define UTF8_ISASCII(u) ( !((u) & ~0x7f) ) -#define UCS4_INVALID 0x80000000U +#undef LDAP_IS_ASCII +#define LDAP_IS_ASCII(uc) ((uc) < 0x80) + +/* + * Basic UTF-8 routines + */ /* * return the number of bytes required to hold the - * NULL-terminated UTF-8 string INCLUDING the + * NULL-terminated UTF-8 string NOT INCLUDING the * termination. */ ber_len_t ldap_utf8_bytes( const char * p ) { - ber_len_t bytes = 0; + ber_len_t bytes; - if( p == NULL ) return bytes; - - while( p[bytes++] ) { + for( bytes=0; p[bytes]; bytes++ ) { /* EMPTY */ ; } @@ -47,19 +56,26 @@ ber_len_t ldap_utf8_bytes( const char * p ) ber_len_t ldap_utf8_chars( const char * p ) { /* could be optimized and could check for invalid sequences */ - ber_len_t chars; - - for( chars=0; *p ; chars++ ) { - int charlen = ldap_utf8_charlen( p ); + ber_len_t chars=0; - if( !charlen ) return chars; - - p = &p[charlen]; + for( ; *p ; LDAP_UTF8_INCR(p) ) { + chars++; }; return chars; } +/* return offset to next character */ +int ldap_utf8_offset( const char * p ) +{ + return LDAP_UTF8_NEXT(p) - p; +} + +/* + * Returns length indicated by first byte. + * + * This function should use a table lookup. + */ int ldap_utf8_charlen( const char * p ) { unsigned c = * (const unsigned char *) p; @@ -92,53 +108,35 @@ int ldap_utf8_charlen( const char * p ) return 1; } -ber_int_t ldap_utf8_to_ucs4( const char * p ) +/* conv UTF-8 to UCS-4, useful for comparisons */ +ldap_ucs4_t ldap_x_utf8_to_ucs4( const char * p ) { + const unsigned char *c = p; + ldap_ucs4_t ch; int len, i; - ber_int_t c = * (const unsigned char *) p; + static unsigned char mask[] = { + 0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 }; - if ((c & 0xFE ) == 0xFC) { - len = 6; - c &= 0x01; + len = LDAP_UTF8_CHARLEN(p); - } else if ((c & 0xFC ) == 0xF8) { - len = 5; - c &= 0x03; + if( len == 0 ) return LDAP_UCS4_INVALID; - } else if ((c & 0xF8 ) == 0xF0) { - len = 4; - c &= 0x07; - - } else if ((c & 0xF0 ) == 0xE0) { - len = 3; - c &= 0x0F; - - } else if ((c & 0xE0 ) == 0xC0) { - len = 2; - c &= 0x1F; - - } else if ((c & 0x80 ) == 0x80) { - return UCS4_INVALID; - - } else { - return c; - } + ch = c[0] & mask[len]; for(i=1; i < len; i++) { - ber_int_t ch = ((const unsigned char *) p)[i]; - - if ((ch & 0xc0) != 0x80) { - return UCS4_INVALID; + if ((c[i] & 0xc0) != 0x80) { + return LDAP_UCS4_INVALID; } - c <<= 6; - c |= ch & 0x3f; + ch <<= 6; + ch |= c[i] & 0x3f; } - return c; + return ch; } -int ldap_ucs4_to_utf8( ber_int_t c, char *buf ) +/* conv UCS-4 to UTF-8, not used */ +int ldap_x_ucs4_to_utf8( ldap_ucs4_t c, char *buf ) { int len=0; unsigned char* p = buf; @@ -152,69 +150,136 @@ int ldap_ucs4_to_utf8( ber_int_t c, char *buf ) } else if( c < 0x800 ) { p[len++] = 0xc0 | ( c >> 6 ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( c & 0x3f ); } else if( c < 0x10000 ) { p[len++] = 0xe0 | ( c >> 12 ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); } else if( c < 0x200000 ) { p[len++] = 0xf0 | ( c >> 18 ); - p[len++] = 0x80 | ( (c >> 12) & 0x3F ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 12) & 0x3f ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); - } else if( c < 0x400000 ) { + } else if( c < 0x4000000 ) { p[len++] = 0xf8 | ( c >> 24 ); - p[len++] = 0x80 | ( (c >> 18) & 0x3F ); - p[len++] = 0x80 | ( (c >> 12) & 0x3F ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 18) & 0x3f ); + p[len++] = 0x80 | ( (c >> 12) & 0x3f ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); } else /* if( c < 0x80000000 ) */ { p[len++] = 0xfc | ( c >> 30 ); - p[len++] = 0x80 | ( (c >> 24) & 0x3F ); - p[len++] = 0x80 | ( (c >> 18) & 0x3F ); - p[len++] = 0x80 | ( (c >> 12) & 0x3F ); - p[len++] = 0x80 | ( (c >> 6) & 0x3F ); - p[len++] = 0x80 | ( c & 0x3F ); + p[len++] = 0x80 | ( (c >> 24) & 0x3f ); + p[len++] = 0x80 | ( (c >> 18) & 0x3f ); + p[len++] = 0x80 | ( (c >> 12) & 0x3f ); + p[len++] = 0x80 | ( (c >> 6) & 0x3f ); + p[len++] = 0x80 | ( c & 0x3f ); } buf[len] = '\0'; return len; } +/* + * Advance to the next UTF-8 character + * + * Ignores length of multibyte character, instead rely on + * continuation markers to find start of next character. + * This allows for "resyncing" of when invalid characters + * are provided provided the start of the next character + * is appears within the 6 bytes examined. + */ char* ldap_utf8_next( const char * p ) { - int len = ldap_utf8_charlen( p ); + int i; + const unsigned char *u = p; + + if( LDAP_UTF8_ISASCII(u) ) { + return (char *) &p[1]; + } + + for( i=1; i<6; i++ ) { + if ( ( u[i] & 0xc0 ) != 0x80 ) { + return (char *) &p[i]; + } + } - return len ? (char *) &p[len] : NULL; + return (char *) &p[i]; } +/* + * Advance to the previous UTF-8 character + * + * Ignores length of multibyte character, instead rely on + * continuation markers to find start of next character. + * This allows for "resyncing" of when invalid characters + * are provided provided the start of the next character + * is appears within the 6 bytes examined. + */ char* ldap_utf8_prev( const char * p ) { int i; const unsigned char *u = p; - for( i = -1; i >= -6 ; i-- ) { - if ( u[i] & 0xC0 != 0x80 ) return (char *) &p[i]; + for( i=-1; i>-6 ; i-- ) { + if ( ( u[i] & 0xc0 ) != 0x80 ) { + return (char *) &p[i]; + } } - return NULL; + return (char *) &p[i]; +} + +/* + * Copy one UTF-8 character from src to dst returning + * number of bytes copied. + * + * Ignores length of multibyte character, instead rely on + * continuation markers to find start of next character. + * This allows for "resyncing" of when invalid characters + * are provided provided the start of the next character + * is appears within the 6 bytes examined. + */ +int ldap_utf8_copy( char* dst, const char *src ) +{ + int i; + const unsigned char *u = src; + + dst[0] = src[0]; + + if( LDAP_UTF8_ISASCII(u) ) { + return 1; + } + + for( i=1; i<6; i++ ) { + if ( ( u[i] & 0xc0 ) != 0x80 ) { + return i; + } + dst[i] = src[i]; + } + + return i; } +/* + * UTF-8 ctype routines + * Only deals with characters < 0x80 (ie: US-ASCII) + */ + int ldap_utf8_isascii( const char * p ) { unsigned c = * (const unsigned char *) p; - return UTF8_ISASCII(c); + return LDAP_IS_ASCII(c); } int ldap_utf8_isdigit( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_IS_ASCII(c)) return 0; return c >= '0' && c <= '9'; } @@ -223,18 +288,42 @@ int ldap_utf8_isxdigit( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_IS_ASCII(c)) return 0; return ( c >= '0' && c <= '9' ) || ( c >= 'A' && c <= 'F' ) || ( c >= 'a' && c <= 'f' ); } +int ldap_utf8_isspace( const char * p ) +{ + unsigned c = * (const unsigned char *) p; + + if(!LDAP_IS_ASCII(c)) return 0; + + switch(c) { + case ' ': + case '\t': + case '\n': + case '\r': + case '\v': + case '\f': + return 1; + } + + return 0; +} + +#ifndef UTF8_ALPHA_CTYPE +/* + * These are not needed by the C SDK and are + * not "good enough" for general use. + */ int ldap_utf8_isalpha( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_IS_ASCII(c)) return 0; return ( c >= 'A' && c <= 'Z' ) || ( c >= 'a' && c <= 'z' ); @@ -244,7 +333,7 @@ int ldap_utf8_isalnum( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_IS_ASCII(c)) return 0; return ( c >= '0' && c <= '9' ) || ( c >= 'A' && c <= 'Z' ) @@ -255,7 +344,7 @@ int ldap_utf8_islower( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_IS_ASCII(c)) return 0; return ( c >= 'a' && c <= 'z' ); } @@ -264,146 +353,84 @@ int ldap_utf8_isupper( const char * p ) { unsigned c = * (const unsigned char *) p; - if(!UTF8_ISASCII(c)) return 0; + if(!LDAP_IS_ASCII(c)) return 0; return ( c >= 'A' && c <= 'Z' ); } +#endif -int ldap_utf8_isspace( const char * p ) -{ - unsigned c = * (const unsigned char *) p; - - if(!UTF8_ISASCII(c)) return 0; - switch(c) { - case ' ': - case '\t': - case '\n': - case '\r': - case '\v': - case '\f': - return 1; - } - - return 0; -} +/* + * UTF-8 string routines + */ -char* ldap_utf8_fgetc( FILE *s, char *buf ) +/* like strchr() */ +char * (ldap_utf8_strchr)( const char *str, const char *chr ) { - int i; - unsigned char *p; - unsigned int c; - int len; - - if( s == NULL ) return NULL; - - p = buf; - - c = fgetc( s ); - if( c == EOF ) { - p[0] = -1; - return NULL; + for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) { + if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( chr ) ) { + return (char *) str; + } } - p[0] = c; - - len = ldap_utf8_charlen( buf ); - - if( len < 1 ) return NULL; - - for( i = 1; i < len; i++ ) { - unsigned int c = fgetc( s ); - if( c == EOF ) { - p[i] = -1; - return NULL; - } - if( c & 0xC0 != 0x80 ) { - ungetc( c, s ); - p[i] = -1; - return NULL; - } - p[i] = c; - } - - return buf; + return NULL; } +/* like strcspn() but returns number of bytes, not characters */ ber_len_t (ldap_utf8_strcspn)( const char *str, const char *set ) { - int len; const char *cstr; + const char *cset; - for( cstr = str; *cstr != '\0'; cstr += len ) { - const char *cset; - - for( cset = set; ; cset += len ) { - if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) { + for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) { + for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) { + if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) { return cstr - str; } - - len = ldap_utf8_charlen(cset); - if( !len ) break; } - - len = ldap_utf8_charlen(cstr); - if( !len ) break; } return cstr - str; } +/* like strspn() but returns number of bytes, not characters */ ber_len_t (ldap_utf8_strspn)( const char *str, const char *set ) { - int len; const char *cstr; + const char *cset; - for( cstr = str; *cstr != '\0'; cstr += len ) { - const char *cset; - - for( cset = set; ; cset += len ) { + for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) { + for( cset = set; ; LDAP_UTF8_INCR(cset) ) { if( *cset == '\0' ) { return cstr - str; } - if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) { + if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) { break; } - - len = ldap_utf8_charlen(cset); - if( !len ) break; } - - len = ldap_utf8_charlen(cstr); - if( !len ) break; } return cstr - str; } +/* like strpbrk(), replaces strchr() as well */ char *(ldap_utf8_strpbrk)( const char *str, const char *set ) { - int len; - const char *cstr; - - for( cstr = str; *cstr != '\0'; cstr += len ) { + for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) { const char *cset; - for( cset = set; ; cset += len ) { - if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) { - return cstr; + for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) { + if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( cset ) ) { + return (char *) str; } - - len = ldap_utf8_charlen(cset); - if( !len ) break; } - - len = ldap_utf8_charlen(cstr); - if( !len ) break; } return NULL; } +/* like strtok_r(), not strtok() */ char *(ldap_utf8_strtok)(char *str, const char *sep, char **last) { char *begin; @@ -420,17 +447,12 @@ char *(ldap_utf8_strtok)(char *str, const char *sep, char **last) return NULL; } - end = &begin[ ldap_utf8_strcpn( begin, sep ) ]; + end = &begin[ ldap_utf8_strcspn( begin, sep ) ]; if( *end != '\0' ) { - int len = ldap_utf8_charlen( end ); + char *next = LDAP_UTF8_NEXT( end ); *end = '\0'; - - if( len ) { - end += len; - } else { - end = NULL; - } + end = next; } *last = end;