3 * Copyright 1998-2002 The OpenLDAP Foundation, All Rights Reserved.
4 * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
10 * These routines are "dumb". Though they understand UTF-8,
11 * they don't grok Unicode. That is, they can push bits,
12 * but don't have a clue what the bits represent. That's
13 * good enough for use with the LDAP Client SDK.
15 * These routines are not optimized.
22 #include <ac/stdlib.h>
24 #include <ac/socket.h>
25 #include <ac/string.h>
28 #include "ldap_utf8.h"
31 #include "ldap_defaults.h"
34 * Basic UTF-8 routines
38 * return the number of bytes required to hold the
39 * NULL-terminated UTF-8 string NOT INCLUDING the
42 ber_len_t ldap_utf8_bytes( const char * p )
46 for( bytes=0; p[bytes]; bytes++ ) {
53 ber_len_t ldap_utf8_chars( const char * p )
55 /* could be optimized and could check for invalid sequences */
58 for( ; *p ; LDAP_UTF8_INCR(p) ) {
65 /* return offset to next character */
66 int ldap_utf8_offset( const char * p )
68 return LDAP_UTF8_NEXT(p) - p;
72 * Returns length indicated by first byte.
74 * This function should use a table lookup.
76 const char ldap_utf8_lentab[] = {
77 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
78 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
79 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
80 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
82 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
83 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
84 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 };
86 int ldap_utf8_charlen( const char * p )
91 return ldap_utf8_lentab[*(unsigned char *)p ^ 0x80];
94 /* conv UTF-8 to UCS-4, useful for comparisons */
95 ldap_ucs4_t ldap_x_utf8_to_ucs4( const char * p )
97 const unsigned char *c = p;
100 static unsigned char mask[] = {
101 0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 };
103 len = LDAP_UTF8_CHARLEN(p);
105 if( len == 0 ) return LDAP_UCS4_INVALID;
107 ch = c[0] & mask[len];
109 for(i=1; i < len; i++) {
110 if ((c[i] & 0xc0) != 0x80) {
111 return LDAP_UCS4_INVALID;
121 /* conv UCS-4 to UTF-8, not used */
122 int ldap_x_ucs4_to_utf8( ldap_ucs4_t c, char *buf )
125 unsigned char* p = buf;
126 if(buf == NULL) return 0;
129 /* not a valid Unicode character */
131 } else if( c < 0x80 ) {
134 } else if( c < 0x800 ) {
135 p[len++] = 0xc0 | ( c >> 6 );
136 p[len++] = 0x80 | ( c & 0x3f );
138 } else if( c < 0x10000 ) {
139 p[len++] = 0xe0 | ( c >> 12 );
140 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
141 p[len++] = 0x80 | ( c & 0x3f );
143 } else if( c < 0x200000 ) {
144 p[len++] = 0xf0 | ( c >> 18 );
145 p[len++] = 0x80 | ( (c >> 12) & 0x3f );
146 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
147 p[len++] = 0x80 | ( c & 0x3f );
149 } else if( c < 0x4000000 ) {
150 p[len++] = 0xf8 | ( c >> 24 );
151 p[len++] = 0x80 | ( (c >> 18) & 0x3f );
152 p[len++] = 0x80 | ( (c >> 12) & 0x3f );
153 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
154 p[len++] = 0x80 | ( c & 0x3f );
156 } else /* if( c < 0x80000000 ) */ {
157 p[len++] = 0xfc | ( c >> 30 );
158 p[len++] = 0x80 | ( (c >> 24) & 0x3f );
159 p[len++] = 0x80 | ( (c >> 18) & 0x3f );
160 p[len++] = 0x80 | ( (c >> 12) & 0x3f );
161 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
162 p[len++] = 0x80 | ( c & 0x3f );
170 * Advance to the next UTF-8 character
172 * Ignores length of multibyte character, instead rely on
173 * continuation markers to find start of next character.
174 * This allows for "resyncing" of when invalid characters
175 * are provided provided the start of the next character
176 * is appears within the 6 bytes examined.
178 char* ldap_utf8_next( const char * p )
181 const unsigned char *u = p;
183 if( LDAP_UTF8_ISASCII(u) ) {
184 return (char *) &p[1];
187 for( i=1; i<6; i++ ) {
188 if ( ( u[i] & 0xc0 ) != 0x80 ) {
189 return (char *) &p[i];
193 return (char *) &p[i];
197 * Advance to the previous UTF-8 character
199 * Ignores length of multibyte character, instead rely on
200 * continuation markers to find start of next character.
201 * This allows for "resyncing" of when invalid characters
202 * are provided provided the start of the next character
203 * is appears within the 6 bytes examined.
205 char* ldap_utf8_prev( const char * p )
208 const unsigned char *u = p;
210 for( i=-1; i>-6 ; i-- ) {
211 if ( ( u[i] & 0xc0 ) != 0x80 ) {
212 return (char *) &p[i];
216 return (char *) &p[i];
220 * Copy one UTF-8 character from src to dst returning
221 * number of bytes copied.
223 * Ignores length of multibyte character, instead rely on
224 * continuation markers to find start of next character.
225 * This allows for "resyncing" of when invalid characters
226 * are provided provided the start of the next character
227 * is appears within the 6 bytes examined.
229 int ldap_utf8_copy( char* dst, const char *src )
232 const unsigned char *u = src;
236 if( LDAP_UTF8_ISASCII(u) ) {
240 for( i=1; i<6; i++ ) {
241 if ( ( u[i] & 0xc0 ) != 0x80 ) {
250 #ifndef UTF8_ALPHA_CTYPE
252 * UTF-8 ctype routines
253 * Only deals with characters < 0x80 (ie: US-ASCII)
256 int ldap_utf8_isascii( const char * p )
258 unsigned c = * (const unsigned char *) p;
259 return LDAP_ASCII(c);
262 int ldap_utf8_isdigit( const char * p )
264 unsigned c = * (const unsigned char *) p;
266 if(!LDAP_ASCII(c)) return 0;
268 return LDAP_DIGIT( c );
271 int ldap_utf8_isxdigit( const char * p )
273 unsigned c = * (const unsigned char *) p;
275 if(!LDAP_ASCII(c)) return 0;
280 int ldap_utf8_isspace( const char * p )
282 unsigned c = * (const unsigned char *) p;
284 if(!LDAP_ASCII(c)) return 0;
300 * These are not needed by the C SDK and are
301 * not "good enough" for general use.
303 int ldap_utf8_isalpha( const char * p )
305 unsigned c = * (const unsigned char *) p;
307 if(!LDAP_ASCII(c)) return 0;
309 return LDAP_ALPHA(c);
312 int ldap_utf8_isalnum( const char * p )
314 unsigned c = * (const unsigned char *) p;
316 if(!LDAP_ASCII(c)) return 0;
318 return LDAP_ALNUM(c);
321 int ldap_utf8_islower( const char * p )
323 unsigned c = * (const unsigned char *) p;
325 if(!LDAP_ASCII(c)) return 0;
327 return LDAP_LOWER(c);
330 int ldap_utf8_isupper( const char * p )
332 unsigned c = * (const unsigned char *) p;
334 if(!LDAP_ASCII(c)) return 0;
336 return LDAP_UPPER(c);
342 * UTF-8 string routines
346 char * (ldap_utf8_strchr)( const char *str, const char *chr )
348 for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
349 if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( chr ) ) {
357 /* like strcspn() but returns number of bytes, not characters */
358 ber_len_t (ldap_utf8_strcspn)( const char *str, const char *set )
363 for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
364 for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
365 if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) {
374 /* like strspn() but returns number of bytes, not characters */
375 ber_len_t (ldap_utf8_strspn)( const char *str, const char *set )
380 for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
381 for( cset = set; ; LDAP_UTF8_INCR(cset) ) {
382 if( *cset == '\0' ) {
386 if( ldap_x_utf8_to_ucs4( cstr ) == ldap_x_utf8_to_ucs4( cset ) ) {
395 /* like strpbrk(), replaces strchr() as well */
396 char *(ldap_utf8_strpbrk)( const char *str, const char *set )
398 for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
401 for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
402 if( ldap_x_utf8_to_ucs4( str ) == ldap_x_utf8_to_ucs4( cset ) ) {
411 /* like strtok_r(), not strtok() */
412 char *(ldap_utf8_strtok)(char *str, const char *sep, char **last)
417 if( last == NULL ) return NULL;
419 begin = str ? str : *last;
421 begin += ldap_utf8_strspn( begin, sep );
423 if( *begin == '\0' ) {
428 end = &begin[ ldap_utf8_strcspn( begin, sep ) ];
431 char *next = LDAP_UTF8_NEXT( end );