X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=libraries%2Flibldap%2Futf-8.c;h=0d12f010a3bd45612226044c765323d47c5332d5;hb=55dba4395f81f75ad8f247d3c0fad7119fe1ea9d;hp=f2ed924bc5677403f2f6435b4844e547e488856d;hpb=14210f5e34cf1784d9b102d52047f5aa93ce75e6;p=openldap

diff --git a/libraries/libldap/utf-8.c b/libraries/libldap/utf-8.c
index f2ed924bc5..0d12f010a3 100644
--- a/libraries/libldap/utf-8.c
+++ b/libraries/libldap/utf-8.c
@@ -1,12 +1,17 @@
 /* $OpenLDAP$ */
 /*
- * Copyright 1998-1999 The OpenLDAP Foundation, All Rights Reserved.
+ * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved.
  * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
  */
 
 /*
  * Basic UTF-8 routines
  *
+ * These routines are "dumb".  Though they understand UTF-8,
+ * they don't grok Unicode.  That is, they can push bits,
+ * but don't have a clue what the bits represent.  That's
+ * good enough for use with the LDAP Client SDK.
+ *
  * These routines are not optimized.
  */
 
@@ -23,13 +28,25 @@
 #include "ldap-int.h"
 #include "ldap_defaults.h"
 
-#define UTF8_ISASCII(u)	( !((u) & ~0x7f) )
+#undef ISASCII
+#define ISASCII(uc)	((uc) < 0x100)
+#undef UCS4_INVALID
+#define UCS4_INVALID	0x80000000U
+
+/*
+ * Basic UTF-8 routines
+ */
 
+/*
+ * return the number of bytes required to hold the
+ * NULL-terminated UTF-8 string NOT INCLUDING the
+ * termination.
+ */
 ber_len_t ldap_utf8_bytes( const char * p )
 {
 	ber_len_t bytes;
 
-	for( bytes=0; p[bytes] ; bytes++ ) {
+	for( bytes=0; p[bytes]; bytes++ ) {
 		/* EMPTY */ ;
 	}
 
@@ -39,38 +56,50 @@ ber_len_t ldap_utf8_bytes( const char * p )
 ber_len_t ldap_utf8_chars( const char * p )
 {
 	/* could be optimized and could check for invalid sequences */
-	ber_len_t chars;
-
-	for( chars=0; *p ; chars++ ) {
-		int charlen = ldap_utf8_charlen( p );
-
-		if( !charlen ) return chars;
+	ber_len_t chars=0;
 
-		p = &p[charlen];
+	for( ; *p ; LDAP_UTF8_INCR(p) ) {
+		chars++;
 	};
 
 	return chars;
 }
 
+/* return offset to next character */
+int ldap_utf8_offset( const char * p )
+{
+	return LDAP_UTF8_NEXT(p) - p;
+}
+
+/*
+ * Returns length indicated by first byte.
+ *
+ * This function should use a table lookup.
+ */
 int ldap_utf8_charlen( const char * p )
 {
-	unsigned c;
+	unsigned c = * (const unsigned char *) p;
 
-	if ((c & 0xFE ) == 0xFC) {
+	if ((c & 0xfe ) == 0xfc) {
 		return 6;
 	}
-	if ((c & 0xFC ) == 0xF8) {
+
+	if ((c & 0xfc ) == 0xf8) {
 		return 5;
 	}
-	if ((c & 0xF8 ) == 0xF0) {
+
+	if ((c & 0xf8 ) == 0xf0) {
 		return 4;
 	}
-	if ((c & 0xF0 ) == 0xE0) {
+
+	if ((c & 0xf0 ) == 0xe0) {
 		return 3;
 	}
-	if ((c & 0xE0 ) == 0xC0) {
+
+	if ((c & 0xe0 ) == 0xc0) {
 		return 2;
 	}
+
 	if ((c & 0x80 ) == 0x80) {
 		/* INVALID */
 		return 0;
@@ -79,36 +108,178 @@ int ldap_utf8_charlen( const char * p )
 	return 1;
 }
 
-char* ldap_utf8_next( char * p )
+/* conv UTF-8 to UCS-4, useful for comparisons */
+ldap_ucs4_t ldap_utf8_to_ucs4( const char * p )
 {
-	int len = ldap_utf8_charlen( p );
+    const unsigned char *c = p;
+    ldap_ucs4_t ch;
+	int len, i;
+	static unsigned char mask[] = {
+		0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 };
+
+	len = LDAP_UTF8_CHARLEN(p);
+
+	if( len == 0 ) return UCS4_INVALID;
+
+	ch = c[0] & mask[len];
+
+	for(i=1; i < len; i++) {
+		if ((c[i] & 0xc0) != 0x80) {
+			return UCS4_INVALID;
+		}
+
+		ch <<= 6;
+		ch |= c[i] & 0x3f;
+	}
 
-	return len ? &p[len] : NULL;
+	return ch;
 }
 
-char* ldap_utf8_prev( char * p )
+/* conv UCS-4 to UTF-8, not used */
+int ldap_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
+{
+	int len=0;
+	unsigned char* p = buf;
+	if(buf == NULL) return 0;
+
+	if ( c < 0 ) {
+		/* not a valid Unicode character */
+
+	} else if( c < 0x80 ) {
+		p[len++] = c;
+
+	} else if( c < 0x800 ) {
+		p[len++] = 0xc0 | ( c >> 6 );
+		p[len++] = 0x80 | ( c & 0x3f );
+
+	} else if( c < 0x10000 ) {
+		p[len++] = 0xe0 | ( c >> 12 );
+		p[len++] = 0x80 | ( (c >> 6) & 0x3f );
+		p[len++] = 0x80 | ( c & 0x3f );
+
+	} else if( c < 0x200000 ) {
+		p[len++] = 0xf0 | ( c >> 18 );
+		p[len++] = 0x80 | ( (c >> 12) & 0x3f );
+		p[len++] = 0x80 | ( (c >> 6) & 0x3f );
+		p[len++] = 0x80 | ( c & 0x3f );
+
+	} else if( c < 0x400000 ) {
+		p[len++] = 0xf8 | ( c >> 24 );
+		p[len++] = 0x80 | ( (c >> 18) & 0x3f );
+		p[len++] = 0x80 | ( (c >> 12) & 0x3f );
+		p[len++] = 0x80 | ( (c >> 6) & 0x3f );
+		p[len++] = 0x80 | ( c & 0x3f );
+
+	} else /* if( c < 0x80000000 ) */ {
+		p[len++] = 0xfc | ( c >> 30 );
+		p[len++] = 0x80 | ( (c >> 24) & 0x3f );
+		p[len++] = 0x80 | ( (c >> 18) & 0x3f );
+		p[len++] = 0x80 | ( (c >> 12) & 0x3f );
+		p[len++] = 0x80 | ( (c >> 6) & 0x3f );
+		p[len++] = 0x80 | ( c & 0x3f );
+	}
+
+	buf[len] = '\0';
+	return len;
+}
+
+/*
+ * Advance to the next UTF-8 character
+ *
+ * Ignores length of multibyte character, instead rely on
+ * continuation markers to find start of next character.
+ * This allows for "resyncing" of when invalid characters
+ * are provided provided the start of the next character
+ * is appears within the 6 bytes examined.
+ */
+char* ldap_utf8_next( const char * p )
 {
 	int i;
-	unsigned char *u = p;
+	const unsigned char *u = p;
 
-	for( i = -1; i >= -6 ; i-- ) {
-		if ( u[i] & 0xC0 != 0x80 ) return &p[i];
+	if( LDAP_UTF8_ISASCII(u) ) {
+		return (char *) &p[1];
 	}
 
-	return NULL;
+	for( i=1; i<6; i++ ) {
+		if ( ( u[i] & 0xc0 ) != 0x80 ) {
+			return (char *) &p[i];
+		}
+	}
+
+	return (char *) &p[i];
+}
+
+/*
+ * Advance to the previous UTF-8 character
+ *
+ * Ignores length of multibyte character, instead rely on
+ * continuation markers to find start of next character.
+ * This allows for "resyncing" of when invalid characters
+ * are provided provided the start of the next character
+ * is appears within the 6 bytes examined.
+ */
+char* ldap_utf8_prev( const char * p )
+{
+	int i;
+	const unsigned char *u = p;
+
+	for( i=-1; i>-6 ; i-- ) {
+		if ( ( u[i] & 0xc0 ) != 0x80 ) {
+			return (char *) &p[i];
+		}
+	}
+
+	return (char *) &p[i];
+}
+
+/*
+ * Copy one UTF-8 character from src to dst returning
+ * number of bytes copied.
+ *
+ * Ignores length of multibyte character, instead rely on
+ * continuation markers to find start of next character.
+ * This allows for "resyncing" of when invalid characters
+ * are provided provided the start of the next character
+ * is appears within the 6 bytes examined.
+ */
+int ldap_utf8_copy( char* dst, const char *src )
+{
+	int i;
+	const unsigned char *u = src;
+
+	dst[0] = src[0];
+
+	if( LDAP_UTF8_ISASCII(u) ) {
+		return 1;
+	}
+
+	for( i=1; i<6; i++ ) {
+		if ( ( u[i] & 0xc0 ) != 0x80 ) {
+			return i; 
+		}
+		dst[i] = src[i];
+	}
+
+	return i;
 }
 
+/*
+ * UTF-8 ctype routines
+ * Only deals with characters < 0x100 (ie: US-ASCII)
+ */
+
 int ldap_utf8_isascii( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
-	return UTF8_ISASCII(c);
+	return ISASCII(c);
 }
 
 int ldap_utf8_isdigit( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
 
-	if(!UTF8_ISASCII(c)) return 0;
+	if(!ISASCII(c)) return 0;
 
 	return c >= '0' && c <= '9';
 }
@@ -117,18 +288,42 @@ int ldap_utf8_isxdigit( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
 
-	if(!UTF8_ISASCII(c)) return 0;
+	if(!ISASCII(c)) return 0;
 
 	return ( c >= '0' && c <= '9' )
 		|| ( c >= 'A' && c <= 'F' )
 		|| ( c >= 'a' && c <= 'f' );
 }
 
+int ldap_utf8_isspace( const char * p )
+{
+	unsigned c = * (const unsigned char *) p;
+
+	if(!ISASCII(c)) return 0;
+
+	switch(c) {
+	case ' ':
+	case '\t':
+	case '\n':
+	case '\r':
+	case '\v':
+	case '\f':
+		return 1;
+	}
+
+	return 0;
+}
+
+#ifndef UTF8_ALPHA_CTYPE
+/*
+ * These are not needed by the C SDK and are
+ * not "good enough" for general use.
+ */
 int ldap_utf8_isalpha( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
 
-	if(!UTF8_ISASCII(c)) return 0;
+	if(!ISASCII(c)) return 0;
 
 	return ( c >= 'A' && c <= 'Z' )
 		|| ( c >= 'a' && c <= 'z' );
@@ -138,7 +333,7 @@ int ldap_utf8_isalnum( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
 
-	if(!UTF8_ISASCII(c)) return 0;
+	if(!ISASCII(c)) return 0;
 
 	return ( c >= '0' && c <= '9' )
 		|| ( c >= 'A' && c <= 'Z' )
@@ -149,7 +344,7 @@ int ldap_utf8_islower( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
 
-	if(!UTF8_ISASCII(c)) return 0;
+	if(!ISASCII(c)) return 0;
 
 	return ( c >= 'a' && c <= 'z' );
 }
@@ -158,66 +353,109 @@ int ldap_utf8_isupper( const char * p )
 {
 	unsigned c = * (const unsigned char *) p;
 
-	if(!UTF8_ISASCII(c)) return 0;
+	if(!ISASCII(c)) return 0;
 
 	return ( c >= 'A' && c <= 'Z' );
 }
+#endif
 
-int ldap_utf8_isspace( const char * p )
+
+/*
+ * UTF-8 string routines
+ */
+
+/* like strchr() */
+char * (ldap_utf8_strchr)( const char *str, const char *chr )
 {
-	unsigned c = * (const unsigned char *) p;
+	for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
+		if( ldap_utf8_to_ucs4( str ) == ldap_utf8_to_ucs4( chr ) ) {
+			return (char *) str;
+		} 
+	}
 
-	if(!UTF8_ISASCII(c)) return 0;
+	return NULL;
+}
 
-	switch(c) {
-	case ' ':
-	case '\t':
-	case '\n':
-	case '\r':
-	case '\v':
-	case '\f':
-		return 1;
+/* like strcspn() but returns number of bytes, not characters */
+ber_len_t (ldap_utf8_strcspn)( const char *str, const char *set )
+{
+	const char *cstr;
+	const char *cset;
+
+	for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
+		for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
+			if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) {
+				return cstr - str;
+			} 
+		}
 	}
 
-	return 0;
+	return cstr - str;
 }
 
-char* ldap_utf8_fgetc( FILE *s, char *buf )
+/* like strspn() but returns number of bytes, not characters */
+ber_len_t (ldap_utf8_strspn)( const char *str, const char *set )
 {
-	int i;
-	unsigned char *p;
-	unsigned int c;
-	int len;
+	const char *cstr;
+	const char *cset;
 
-	if( s == NULL ) return NULL;
+	for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
 
-	p = buf;
+		for( cset = set; ; LDAP_UTF8_INCR(cset) ) {
+			if( *cset == '\0' ) {
+				return cstr - str;
+			}
 
-	c = fgetc( s );
-	if( c == EOF ) {
-		p[0] = -1;
-		return NULL;
+			if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) {
+				break;
+			} 
+		}
 	}
 
-	p[0] = c;
-
-	len = ldap_utf8_charlen( buf );
+	return cstr - str;
+}
 
-	if( len < 1 ) return NULL;
+/* like strpbrk(), replaces strchr() as well */
+char *(ldap_utf8_strpbrk)( const char *str, const char *set )
+{
+	for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
+		const char *cset;
 
-	for( i = 1; i < len; i++ ) {
-		unsigned int c = fgetc( s );
-		if( c == EOF ) {
-			p[i] = -1;
-			return NULL;
-		}
-		if( c & 0xC0 != 0x80 ) {
-			ungetc( c, s );
-			p[i] = -1;
-			return NULL;
+		for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
+			if( ldap_utf8_to_ucs4( str ) == ldap_utf8_to_ucs4( cset ) ) {
+				return (char *) str;
+			} 
 		}
-		p[i] = c;
 	}
 
-	return buf;
+	return NULL;
+}
+
+/* like strtok_r(), not strtok() */
+char *(ldap_utf8_strtok)(char *str, const char *sep, char **last)
+{
+	char *begin;
+	char *end;
+
+	if( last == NULL ) return NULL;
+
+	begin = str ? str : *last;
+
+	begin += ldap_utf8_strspn( begin, sep );
+
+	if( *begin == '\0' ) {
+		*last = NULL;
+		return NULL;
+	}
+
+	end = &begin[ ldap_utf8_strcspn( begin, sep ) ];
+
+	if( *end != '\0' ) {
+		char *next = LDAP_UTF8_NEXT( end );
+		*end = '\0';
+		end = next;
+	}
+
+	*last = end;
+	return begin;
 }