X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=libraries%2Flibldap%2Futf-8.c;h=8520bdb6c1b224c6559c425654d576efe7ab5ca0;hb=62295c0928c071685dbbfb98644cadaeb37333b8;hp=1523b7b4a47baf17982a5131d002ed5bc56cec8f;hpb=cee040a3218637b541aa6c17fb074dc840752fab;p=openldap
diff --git a/libraries/libldap/utf-8.c b/libraries/libldap/utf-8.c
index 1523b7b4a4..8520bdb6c1 100644
--- a/libraries/libldap/utf-8.c
+++ b/libraries/libldap/utf-8.c
@@ -1,11 +1,19 @@
+/* utf-8.c -- Basic UTF-8 routines */
/* $OpenLDAP$ */
-/*
- * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved.
- * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
+/* This work is part of OpenLDAP Software .
+ *
+ * Copyright 1998-2009 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * .
*/
-
-/*
- * Basic UTF-8 routines
+/* Basic UTF-8 routines
*
* These routines are "dumb". Though they understand UTF-8,
* they don't grok Unicode. That is, they can push bits,
@@ -25,16 +33,11 @@
#include
#include
+#include "ldap_utf8.h"
+
#include "ldap-int.h"
#include "ldap_defaults.h"
-#undef ISASCII
-#define ISASCII(uc) ((uc) < 0x100)
-
-/*
- * Basic UTF-8 routines
- */
-
/*
* return the number of bytes required to hold the
* NULL-terminated UTF-8 string NOT INCLUDING the
@@ -58,7 +61,7 @@ ber_len_t ldap_utf8_chars( const char * p )
for( ; *p ; LDAP_UTF8_INCR(p) ) {
chars++;
- };
+ }
return chars;
}
@@ -71,51 +74,78 @@ int ldap_utf8_offset( const char * p )
/*
* Returns length indicated by first byte.
- *
- * This function should use a table lookup.
*/
+const char ldap_utf8_lentab[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 };
+
int ldap_utf8_charlen( const char * p )
{
- unsigned c = * (const unsigned char *) p;
-
- if ((c & 0xfe ) == 0xfc) {
- return 6;
- }
-
- if ((c & 0xfc ) == 0xf8) {
- return 5;
- }
+ if (!(*p & 0x80))
+ return 1;
- if ((c & 0xf8 ) == 0xf0) {
- return 4;
- }
+ return ldap_utf8_lentab[*(const unsigned char *)p ^ 0x80];
+}
- if ((c & 0xf0 ) == 0xe0) {
- return 3;
- }
+/*
+ * Make sure the UTF-8 char used the shortest possible encoding
+ * returns charlen if valid, 0 if not.
+ *
+ * Here are the valid UTF-8 encodings, taken from RFC 2279 page 4.
+ * The table is slightly modified from that of the RFC.
+ *
+ * UCS-4 range (hex) UTF-8 sequence (binary)
+ * 0000 0000-0000 007F 0.......
+ * 0000 0080-0000 07FF 110++++. 10......
+ * 0000 0800-0000 FFFF 1110++++ 10+..... 10......
+ * 0001 0000-001F FFFF 11110+++ 10++.... 10...... 10......
+ * 0020 0000-03FF FFFF 111110++ 10+++... 10...... 10...... 10......
+ * 0400 0000-7FFF FFFF 1111110+ 10++++.. 10...... 10...... 10...... 10......
+ *
+ * The '.' bits are "don't cares". When validating a UTF-8 sequence,
+ * at least one of the '+' bits must be set, otherwise the character
+ * should have been encoded in fewer octets. Note that in the two-octet
+ * case, only the first octet needs to be validated, and this is done
+ * in the ldap_utf8_lentab[] above.
+ */
- if ((c & 0xe0 ) == 0xc0) {
- return 2;
- }
+/* mask of required bits in second octet */
+#undef c
+#define c const char
+c ldap_utf8_mintab[] = {
+ (c)0x20, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80,
+ (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80,
+ (c)0x30, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80,
+ (c)0x38, (c)0x80, (c)0x80, (c)0x80, (c)0x3c, (c)0x80, (c)0x00, (c)0x00 };
+#undef c
+
+int ldap_utf8_charlen2( const char * p )
+{
+ int i = LDAP_UTF8_CHARLEN( p );
- if ((c & 0x80 ) == 0x80) {
- /* INVALID */
- return 0;
+ if ( i > 2 ) {
+ if ( !( ldap_utf8_mintab[*p & 0x1f] & p[1] ) )
+ i = 0;
}
-
- return 1;
+ return i;
}
/* conv UTF-8 to UCS-4, useful for comparisons */
-ldap_ucs4_t ldap_utf8_to_ucs4( const char * p )
+ldap_ucs4_t ldap_x_utf8_to_ucs4( const char * p )
{
- const unsigned char *c = p;
+ const unsigned char *c = (const unsigned char *) p;
ldap_ucs4_t ch;
int len, i;
static unsigned char mask[] = {
0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 };
- len = LDAP_UTF8_CHARLEN(p);
+ len = LDAP_UTF8_CHARLEN2(p, len);
if( len == 0 ) return LDAP_UCS4_INVALID;
@@ -134,16 +164,25 @@ ldap_ucs4_t ldap_utf8_to_ucs4( const char * p )
}
/* conv UCS-4 to UTF-8, not used */
-int ldap_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
+int ldap_x_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
{
int len=0;
- unsigned char* p = buf;
- if(buf == NULL) return 0;
-
- if ( c < 0 ) {
- /* not a valid Unicode character */
+ unsigned char* p = (unsigned char *) buf;
+
+ /* not a valid Unicode character */
+ if ( c < 0 ) return 0;
+
+ /* Just return length, don't convert */
+ if(buf == NULL) {
+ if( c < 0x80 ) return 1;
+ else if( c < 0x800 ) return 2;
+ else if( c < 0x10000 ) return 3;
+ else if( c < 0x200000 ) return 4;
+ else if( c < 0x4000000 ) return 5;
+ else return 6;
+ }
- } else if( c < 0x80 ) {
+ if( c < 0x80 ) {
p[len++] = c;
} else if( c < 0x800 ) {
@@ -161,7 +200,7 @@ int ldap_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
p[len++] = 0x80 | ( (c >> 6) & 0x3f );
p[len++] = 0x80 | ( c & 0x3f );
- } else if( c < 0x400000 ) {
+ } else if( c < 0x4000000 ) {
p[len++] = 0xf8 | ( c >> 24 );
p[len++] = 0x80 | ( (c >> 18) & 0x3f );
p[len++] = 0x80 | ( (c >> 12) & 0x3f );
@@ -177,10 +216,80 @@ int ldap_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
p[len++] = 0x80 | ( c & 0x3f );
}
- buf[len] = '\0';
return len;
}
+#define LDAP_UCS_UTF8LEN(c) \
+ c < 0 ? 0 : (c < 0x80 ? 1 : (c < 0x800 ? 2 : (c < 0x10000 ? 3 : \
+ (c < 0x200000 ? 4 : (c < 0x4000000 ? 5 : 6)))))
+
+/* Convert a string to UTF-8 format. The input string is expected to
+ * have characters of 1, 2, or 4 octets (in network byte order)
+ * corresponding to the ASN.1 T61STRING, BMPSTRING, and UNIVERSALSTRING
+ * types respectively. (Here T61STRING just means that there is one
+ * octet per character and characters may use the high bit of the octet.
+ * The characters are assumed to use ISO mappings, no provision is made
+ * for converting from T.61 coding rules to Unicode.)
+ */
+
+int
+ldap_ucs_to_utf8s( struct berval *ucs, int csize, struct berval *utf8s )
+{
+ unsigned char *in, *end;
+ char *ptr;
+ ldap_ucs4_t u;
+ int i, l = 0;
+
+ utf8s->bv_val = NULL;
+ utf8s->bv_len = 0;
+
+ in = (unsigned char *)ucs->bv_val;
+
+ /* Make sure we stop at an even multiple of csize */
+ end = in + ( ucs->bv_len & ~(csize-1) );
+
+ for (; in < end; ) {
+ u = *in++;
+ if (csize > 1) {
+ u <<= 8;
+ u |= *in++;
+ }
+ if (csize > 2) {
+ u <<= 8;
+ u |= *in++;
+ u <<= 8;
+ u |= *in++;
+ }
+ i = LDAP_UCS_UTF8LEN(u);
+ if (i == 0)
+ return LDAP_INVALID_SYNTAX;
+ l += i;
+ }
+
+ utf8s->bv_val = LDAP_MALLOC( l+1 );
+ if (utf8s->bv_val == NULL)
+ return LDAP_NO_MEMORY;
+ utf8s->bv_len = l;
+
+ ptr = utf8s->bv_val;
+ for (in = (unsigned char *)ucs->bv_val; in < end; ) {
+ u = *in++;
+ if (csize > 1) {
+ u <<= 8;
+ u |= *in++;
+ }
+ if (csize > 2) {
+ u <<= 8;
+ u |= *in++;
+ u <<= 8;
+ u |= *in++;
+ }
+ ptr += ldap_x_ucs4_to_utf8(u, ptr);
+ }
+ *ptr = '\0';
+ return LDAP_SUCCESS;
+}
+
/*
* Advance to the next UTF-8 character
*
@@ -193,7 +302,7 @@ int ldap_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
char* ldap_utf8_next( const char * p )
{
int i;
- const unsigned char *u = p;
+ const unsigned char *u = (const unsigned char *) p;
if( LDAP_UTF8_ISASCII(u) ) {
return (char *) &p[1];
@@ -220,7 +329,7 @@ char* ldap_utf8_next( const char * p )
char* ldap_utf8_prev( const char * p )
{
int i;
- const unsigned char *u = p;
+ const unsigned char *u = (const unsigned char *) p;
for( i=-1; i>-6 ; i-- ) {
if ( ( u[i] & 0xc0 ) != 0x80 ) {
@@ -244,7 +353,7 @@ char* ldap_utf8_prev( const char * p )
int ldap_utf8_copy( char* dst, const char *src )
{
int i;
- const unsigned char *u = src;
+ const unsigned char *u = (const unsigned char *) src;
dst[0] = src[0];
@@ -262,42 +371,41 @@ int ldap_utf8_copy( char* dst, const char *src )
return i;
}
+#ifndef UTF8_ALPHA_CTYPE
/*
* UTF-8 ctype routines
- * Only deals with characters < 0x100 (ie: US-ASCII)
+ * Only deals with characters < 0x80 (ie: US-ASCII)
*/
int ldap_utf8_isascii( const char * p )
{
unsigned c = * (const unsigned char *) p;
- return ISASCII(c);
+ return LDAP_ASCII(c);
}
int ldap_utf8_isdigit( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
- return c >= '0' && c <= '9';
+ return LDAP_DIGIT( c );
}
int ldap_utf8_isxdigit( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
- return ( c >= '0' && c <= '9' )
- || ( c >= 'A' && c <= 'F' )
- || ( c >= 'a' && c <= 'f' );
+ return LDAP_HEX(c);
}
int ldap_utf8_isspace( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
switch(c) {
case ' ':
@@ -312,7 +420,6 @@ int ldap_utf8_isspace( const char * p )
return 0;
}
-#ifndef UTF8_ALPHA_CTYPE
/*
* These are not needed by the C SDK and are
* not "good enough" for general use.
@@ -321,39 +428,36 @@ int ldap_utf8_isalpha( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
- return ( c >= 'A' && c <= 'Z' )
- || ( c >= 'a' && c <= 'z' );
+ return LDAP_ALPHA(c);
}
int ldap_utf8_isalnum( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
- return ( c >= '0' && c <= '9' )
- || ( c >= 'A' && c <= 'Z' )
- || ( c >= 'a' && c <= 'z' );
+ return LDAP_ALNUM(c);
}
int ldap_utf8_islower( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
- return ( c >= 'a' && c <= 'z' );
+ return LDAP_LOWER(c);
}
int ldap_utf8_isupper( const char * p )
{
unsigned c = * (const unsigned char *) p;
- if(!ISASCII(c)) return 0;
+ if(!LDAP_ASCII(c)) return 0;
- return ( c >= 'A' && c <= 'Z' );
+ return LDAP_UPPER(c);
}
#endif
@@ -366,7 +470,7 @@ int ldap_utf8_isupper( const char * p )
char * (ldap_utf8_strchr)( const char *str, const char *chr )
{
for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
- if( ldap_utf8_to_ucs4( str ) == ldap_utf8_to_ucs4( chr ) ) {
+ if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( chr ) ) {
return (char *) str;
}
}
@@ -382,7 +486,7 @@ ber_len_t (ldap_utf8_strcspn)( const char *str, const char *set )
for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
- if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) {
+ if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) {
return cstr - str;
}
}
@@ -403,7 +507,7 @@ ber_len_t (ldap_utf8_strspn)( const char *str, const char *set )
return cstr - str;
}
- if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) {
+ if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) {
break;
}
}
@@ -419,7 +523,7 @@ char *(ldap_utf8_strpbrk)( const char *str, const char *set )
const char *cset;
for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
- if( ldap_utf8_to_ucs4( str ) == ldap_utf8_to_ucs4( cset ) ) {
+ if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( cset ) ) {
return (char *) str;
}
}