3 * Copyright 1998-2000 The OpenLDAP Foundation, All Rights Reserved.
4 * COPYING RESTRICTIONS APPLY, see COPYRIGHT file
10 * These routines are "dumb". Though they understand UTF-8,
11 * they don't grok Unicode. That is, they can push bits,
12 * but don't have a clue what the bits represent. That's
13 * good enough for use with the LDAP Client SDK.
15 * These routines are not optimized.
22 #include <ac/stdlib.h>
24 #include <ac/socket.h>
25 #include <ac/string.h>
29 #include "ldap_defaults.h"
32 #define ISASCII(uc) ((uc) < 0x100)
34 #define UCS4_INVALID 0x80000000U
37 * Basic UTF-8 routines
41 * return the number of bytes required to hold the
42 * NULL-terminated UTF-8 string NOT INCLUDING the
45 ber_len_t ldap_utf8_bytes( const char * p )
49 for( bytes=0; p[bytes]; bytes++ ) {
56 ber_len_t ldap_utf8_chars( const char * p )
58 /* could be optimized and could check for invalid sequences */
61 for( ; *p ; LDAP_UTF8_INCR(p) ) {
68 /* return offset to next character */
69 int ldap_utf8_offset( const char * p )
71 return LDAP_UTF8_NEXT(p) - p;
75 * Returns length indicated by first byte.
77 * This function should use a table lookup.
79 int ldap_utf8_charlen( const char * p )
81 unsigned c = * (const unsigned char *) p;
83 if ((c & 0xfe ) == 0xfc) {
87 if ((c & 0xfc ) == 0xf8) {
91 if ((c & 0xf8 ) == 0xf0) {
95 if ((c & 0xf0 ) == 0xe0) {
99 if ((c & 0xe0 ) == 0xc0) {
103 if ((c & 0x80 ) == 0x80) {
111 /* conv UTF-8 to UCS-4, useful for comparisons */
112 ber_int_t ldap_utf8_to_ucs4( const char * p )
114 const unsigned char *c = p;
117 static unsigned char mask[] = {
118 0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 };
120 len = LDAP_UTF8_CHARLEN(p);
122 if( len == 0 ) return UCS4_INVALID;
124 ch = c[0] & mask[len];
126 for(i=1; i < len; i++) {
127 if ((c[i] & 0xc0) != 0x80) {
138 /* conv UCS-4 to UTF-8, not used */
139 int ldap_ucs4_to_utf8( ber_int_t c, char *buf )
142 unsigned char* p = buf;
143 if(buf == NULL) return 0;
146 /* not a valid Unicode character */
148 } else if( c < 0x80 ) {
151 } else if( c < 0x800 ) {
152 p[len++] = 0xc0 | ( c >> 6 );
153 p[len++] = 0x80 | ( c & 0x3f );
155 } else if( c < 0x10000 ) {
156 p[len++] = 0xe0 | ( c >> 12 );
157 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
158 p[len++] = 0x80 | ( c & 0x3f );
160 } else if( c < 0x200000 ) {
161 p[len++] = 0xf0 | ( c >> 18 );
162 p[len++] = 0x80 | ( (c >> 12) & 0x3f );
163 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
164 p[len++] = 0x80 | ( c & 0x3f );
166 } else if( c < 0x400000 ) {
167 p[len++] = 0xf8 | ( c >> 24 );
168 p[len++] = 0x80 | ( (c >> 18) & 0x3f );
169 p[len++] = 0x80 | ( (c >> 12) & 0x3f );
170 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
171 p[len++] = 0x80 | ( c & 0x3f );
173 } else /* if( c < 0x80000000 ) */ {
174 p[len++] = 0xfc | ( c >> 30 );
175 p[len++] = 0x80 | ( (c >> 24) & 0x3f );
176 p[len++] = 0x80 | ( (c >> 18) & 0x3f );
177 p[len++] = 0x80 | ( (c >> 12) & 0x3f );
178 p[len++] = 0x80 | ( (c >> 6) & 0x3f );
179 p[len++] = 0x80 | ( c & 0x3f );
187 * Advance to the next UTF-8 character
189 * Ignores length of multibyte character, instead rely on
190 * continuation markers to find start of next character.
191 * This allows for "resyncing" of when invalid characters
192 * are provided provided the start of the next character
193 * is appears within the 6 bytes examined.
195 char* ldap_utf8_next( const char * p )
198 const unsigned char *u = p;
200 if( LDAP_UTF8_ISASCII(u) ) {
201 return (char *) &p[1];
204 for( i=1; i<6; i++ ) {
205 if ( u[i] & 0xc0 != 0x80 ) {
206 return (char *) &p[i];
210 return (char *) &p[i];
214 * Advance to the previous UTF-8 character
216 * Ignores length of multibyte character, instead rely on
217 * continuation markers to find start of next character.
218 * This allows for "resyncing" of when invalid characters
219 * are provided provided the start of the next character
220 * is appears within the 6 bytes examined.
222 char* ldap_utf8_prev( const char * p )
225 const unsigned char *u = p;
227 for( i=-1; i>-6 ; i-- ) {
228 if ( u[i] & 0xc0 != 0x80 ) {
229 return (char *) &p[i];
233 return (char *) &p[i];
237 * Copy one UTF-8 character from src to dst returning
238 * number of bytes copied.
240 * Ignores length of multibyte character, instead rely on
241 * continuation markers to find start of next character.
242 * This allows for "resyncing" of when invalid characters
243 * are provided provided the start of the next character
244 * is appears within the 6 bytes examined.
246 int ldap_utf8_copy( char* dst, const char *src )
249 const unsigned char *u = src;
253 if( LDAP_UTF8_ISASCII(u) ) {
257 for( i=1; i<6; i++ ) {
258 if ( u[i] & 0xc0 != 0x80 ) {
268 * UTF-8 ctype routines
269 * Only deals with characters < 0x100 (ie: US-ASCII)
272 int ldap_utf8_isascii( const char * p )
274 unsigned c = * (const unsigned char *) p;
278 int ldap_utf8_isdigit( const char * p )
280 unsigned c = * (const unsigned char *) p;
282 if(!ISASCII(c)) return 0;
284 return c >= '0' && c <= '9';
287 int ldap_utf8_isxdigit( const char * p )
289 unsigned c = * (const unsigned char *) p;
291 if(!ISASCII(c)) return 0;
293 return ( c >= '0' && c <= '9' )
294 || ( c >= 'A' && c <= 'F' )
295 || ( c >= 'a' && c <= 'f' );
298 int ldap_utf8_isspace( const char * p )
300 unsigned c = * (const unsigned char *) p;
302 if(!ISASCII(c)) return 0;
317 #ifndef UTF8_ALPHA_CTYPE
319 * These are not needed by the C SDK and are
320 * not "good enough" for general use.
322 int ldap_utf8_isalpha( const char * p )
324 unsigned c = * (const unsigned char *) p;
326 if(!ISASCII(c)) return 0;
328 return ( c >= 'A' && c <= 'Z' )
329 || ( c >= 'a' && c <= 'z' );
332 int ldap_utf8_isalnum( const char * p )
334 unsigned c = * (const unsigned char *) p;
336 if(!ISASCII(c)) return 0;
338 return ( c >= '0' && c <= '9' )
339 || ( c >= 'A' && c <= 'Z' )
340 || ( c >= 'a' && c <= 'z' );
343 int ldap_utf8_islower( const char * p )
345 unsigned c = * (const unsigned char *) p;
347 if(!ISASCII(c)) return 0;
349 return ( c >= 'a' && c <= 'z' );
352 int ldap_utf8_isupper( const char * p )
354 unsigned c = * (const unsigned char *) p;
356 if(!ISASCII(c)) return 0;
358 return ( c >= 'A' && c <= 'Z' );
364 * UTF-8 string routines
368 char * (ldap_utf8_strchr)( const char *str, const char *chr )
370 for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
371 if( ldap_utf8_to_ucs4( str ) == ldap_utf8_to_ucs4( chr ) ) {
379 /* like strcspn() but returns number of bytes, not characters */
380 ber_len_t (ldap_utf8_strcspn)( const char *str, const char *set )
385 for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
386 for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
387 if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) {
396 /* like strspn() but returns number of bytes, not characters */
397 ber_len_t (ldap_utf8_strspn)( const char *str, const char *set )
402 for( cstr = str; *cstr != '\0'; LDAP_UTF8_INCR(cstr) ) {
404 for( cset = set; ; LDAP_UTF8_INCR(cset) ) {
405 if( *cset == '\0' ) {
409 if( ldap_utf8_to_ucs4( cstr ) == ldap_utf8_to_ucs4( cset ) ) {
418 /* like strpbrk(), replaces strchr() as well */
419 char *(ldap_utf8_strpbrk)( const char *str, const char *set )
421 for( ; *str != '\0'; LDAP_UTF8_INCR(str) ) {
424 for( cset = set; *cset != '\0'; LDAP_UTF8_INCR(cset) ) {
425 if( ldap_utf8_to_ucs4( str ) == ldap_utf8_to_ucs4( cset ) ) {
434 /* like strtok_r(), not strtok() */
435 char *(ldap_utf8_strtok)(char *str, const char *sep, char **last)
440 if( last == NULL ) return NULL;
442 begin = str ? str : *last;
444 begin += ldap_utf8_strspn( begin, sep );
446 if( *begin == '\0' ) {
451 end = &begin[ ldap_utf8_strcspn( begin, sep ) ];
454 char *next = LDAP_UTF8_NEXT( end );