From: Hallvard Furuseth Date: Sun, 2 Dec 2007 16:10:23 +0000 (+0000) Subject: Integer indexing - handle huge values X-Git-Tag: OPENLDAP_REL_ENG_2_4_9~20^2~365 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=bddba8ae1762e9fbd64bab3308b914174bb9f9b9;p=openldap Integer indexing - handle huge values --- diff --git a/doc/man/man5/slapd-config.5 b/doc/man/man5/slapd-config.5 index 5e3b4bfd81..96aeffd4c8 100644 --- a/doc/man/man5/slapd-config.5 +++ b/doc/man/man5/slapd-config.5 @@ -456,7 +456,8 @@ feature. The default is 0. .B olcIndexIntLen: Specify the key length for ordered integer indices. The most significant bytes of the binary integer will be used for index keys. The default -value is 4, which provides exact indexing for 32 bit values. +value is 4, which provides exact indexing for 31 bit values. +A floating point representation is used to index too large values. .TP .B olcIndexSubstrIfMaxlen: Specify the maximum length for subinitial and subfinal indices. Only diff --git a/doc/man/man5/slapd.conf.5 b/doc/man/man5/slapd.conf.5 index 6bc5b58127..c20df8b688 100644 --- a/doc/man/man5/slapd.conf.5 +++ b/doc/man/man5/slapd.conf.5 @@ -473,7 +473,8 @@ continuing with the next line of the current file. .B index_intlen Specify the key length for ordered integer indices. The most significant bytes of the binary integer will be used for index keys. The default -value is 4, which provides exact indexing for 32 bit values. +value is 4, which provides exact indexing for 31 bit values. +A floating point representation is used to index too large values. .TP .B index_substr_if_minlen Specify the minimum length for subinitial and subfinal indices. An diff --git a/servers/slapd/bconfig.c b/servers/slapd/bconfig.c index 0a952f91bd..10fec37525 100644 --- a/servers/slapd/bconfig.c +++ b/servers/slapd/bconfig.c @@ -1160,6 +1160,8 @@ config_generic(ConfigArgs *c) { case CFG_IX_INTLEN: index_intlen = SLAP_INDEX_INTLEN_DEFAULT; + index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN( + SLAP_INDEX_INTLEN_DEFAULT ); break; case CFG_ACL: @@ -1511,6 +1513,8 @@ config_generic(ConfigArgs *c) { else if ( c->value_int > 255 ) c->value_int = 255; index_intlen = c->value_int; + index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN( + index_intlen ); break; case CFG_SORTVALS: { diff --git a/servers/slapd/proto-slap.h b/servers/slapd/proto-slap.h index 6bceda7575..4e1684235a 100644 --- a/servers/slapd/proto-slap.h +++ b/servers/slapd/proto-slap.h @@ -1877,6 +1877,10 @@ LDAP_SLAPD_V (unsigned int) index_substr_if_maxlen; LDAP_SLAPD_V (unsigned int) index_substr_any_len; LDAP_SLAPD_V (unsigned int) index_substr_any_step; LDAP_SLAPD_V (unsigned int) index_intlen; +/* all signed integers from strings of this size need more than intlen bytes */ +/* i.e. log(10)*(index_intlen_strlen-2) > log(2)*(8*(index_intlen)-1) */ +LDAP_SLAPD_V (unsigned int) index_intlen_strlen; +#define SLAP_INDEX_INTLEN_STRLEN(intlen) ((8*(intlen)-1) * 146/485 + 3) LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming; LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming_auth; diff --git a/servers/slapd/schema_init.c b/servers/slapd/schema_init.c index fa75175acc..501b9d6067 100644 --- a/servers/slapd/schema_init.c +++ b/servers/slapd/schema_init.c @@ -63,6 +63,8 @@ unsigned int index_substr_any_len = SLAP_INDEX_SUBSTR_ANY_LEN_DEFAULT; unsigned int index_substr_any_step = SLAP_INDEX_SUBSTR_ANY_STEP_DEFAULT; unsigned int index_intlen = SLAP_INDEX_INTLEN_DEFAULT; +unsigned int index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN( + SLAP_INDEX_INTLEN_DEFAULT ); ldap_pvt_thread_mutex_t ad_undef_mutex; ldap_pvt_thread_mutex_t oc_undef_mutex; @@ -2114,49 +2116,69 @@ integerMatch( static int integerVal2Key( - struct berval *val, + struct berval val, struct berval *key, - struct berval *tmp + struct berval itmp ) { - struct berval iv; - int neg; - - iv = *tmp; - if ( lutil_str2bin( val, &iv )) { - return LDAP_INVALID_SYNTAX; + /* index format: + * only if too large: one's complement , + * two's complement value (sign-extended or chopped as needed), + * with 1st byte of the above adjusted as follows: + * inverse sign in the top bits, + * then the sign bit as delimiter. + */ + ber_slen_t k = index_intlen_strlen, chop = 0; + unsigned char neg = 0xff, signmask = 0x80; + unsigned char lenbuf[sizeof(k) + 2], *lenp; + + if ( val.bv_val[0] != '-' ) { + neg = 0; + --k; + } + + /* Chop least significant digits, increase length instead */ + if ( val.bv_len > k ) { + chop = (val.bv_len - k + 2) / 7; /* 2 fewer digits */ + val.bv_len -= chop * 7; /* #digits chopped */ + chop *= 3; /* >#key bytes chopped: 256**3 > 10**7 */ + if ( chop > 0x7fffffff ) { + memset( key->bv_val, neg ^ 0xff, index_intlen ); + return 0; + } } - neg = iv.bv_val[0] & 0x80; - - /* Omit leading 0 pad byte */ - if ( !iv.bv_val[0] ) { - iv.bv_val++; - iv.bv_len--; + if ( lutil_str2bin( &val, &itmp )) { + return LDAP_INVALID_SYNTAX; } - /* If too small, sign-extend */ - if ( iv.bv_len < index_intlen ) { - int j, k, pad; - key->bv_val[0] = index_intlen; - k = index_intlen - iv.bv_len + 1; - if ( neg ) - pad = 0xff; - else - pad = 0; - for ( j=1; jbv_val[j] = pad; - for ( j = 0; jbv_val[j+k] = iv.bv_val[j]; - } else { - key->bv_val[0] = iv.bv_len; - memcpy( key->bv_val+1, iv.bv_val, index_intlen ); - } - if ( neg ) { - key->bv_val[0] = -key->bv_val[0]; - } - /* convert signed to unsigned */ - key->bv_val[0] ^= 0x80; + /* Omit leading sign byte */ + if ( itmp.bv_val[0] == neg ) { + itmp.bv_val++; + itmp.bv_len--; + } + + k = (ber_slen_t) index_intlen - ((ber_slen_t) itmp.bv_len + chop); + if ( k > 0 ) { + assert( chop == 0 ); + memset( key->bv_val, neg, k ); /* sign-extend */ + } else if ( k != 0 || ((itmp.bv_val[0] ^ neg) & 0xc0) ) { + lenp = lenbuf + sizeof(lenbuf); + k = -k; + do { + *--lenp = k ^ neg; + signmask >>= 1; + } while ( (k >>= 8) != 0 || (signmask >> 1) <= (*lenp ^ neg) ); + /* With n bytes used in lenbuf, the top n+1 bits of signmask + * are 1, and the top n+2 bits of *lenp are the sign bit. */ + k = (lenbuf + sizeof(lenbuf)) - lenp; + if ( k > index_intlen ) + k = index_intlen; + memcpy( key->bv_val, lenp, k ); + itmp.bv_len = index_intlen - k; + } + memcpy( key->bv_val + k, itmp.bv_val, itmp.bv_len ); + key->bv_val[0] ^= (unsigned char) -signmask & 0xff; /* invert sign */ return 0; } @@ -2177,34 +2199,35 @@ integerIndexer( BerVarray keys; int i, rc; - for( i=0; !BER_BVISNULL( &values[i] ); i++ ) { - /* just count them */ + /* count the values and find max needed length */ + itmp.bv_len = 0; + for( i = 0; !BER_BVISNULL( &values[i] ); i++ ) { + if ( itmp.bv_len < values[i].bv_len ) + itmp.bv_len = values[i].bv_len; } + if ( itmp.bv_len > index_intlen_strlen ) + itmp.bv_len > index_intlen_strlen; /* we should have at least one value at this point */ assert( i > 0 ); keys = slap_sl_malloc( sizeof( struct berval ) * (i+1), ctx ); for ( i = 0; !BER_BVISNULL( &values[i] ); i++ ) { - keys[i].bv_len = index_intlen+1; - keys[i].bv_val = slap_sl_malloc( index_intlen+1, ctx ); + keys[i].bv_len = index_intlen; + keys[i].bv_val = slap_sl_malloc( index_intlen, ctx ); } keys[i].bv_len = 0; keys[i].bv_val = NULL; - itmp.bv_val = ibuf; - itmp.bv_len = sizeof(ibuf); + if ( itmp.bv_len > (int) sizeof(ibuf) ) { + itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx ); + } else { + itmp.bv_val = ibuf; + itmp.bv_len = sizeof(ibuf); + } for ( i=0; !BER_BVISNULL( &values[i] ); i++ ) { - if ( values[i].bv_len > itmp.bv_len ) { - itmp.bv_len = values[i].bv_len; - if ( itmp.bv_val == ibuf ) { - itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx ); - } else { - itmp.bv_val = slap_sl_realloc( itmp.bv_val, itmp.bv_len, ctx ); - } - } - rc = integerVal2Key( &values[i], &keys[i], &itmp ); + rc = integerVal2Key( values[i], &keys[i], itmp ); if ( rc ) goto leave; } @@ -2238,18 +2261,19 @@ integerFilter( keys = slap_sl_malloc( sizeof( struct berval ) * 2, ctx ); - keys[0].bv_len = index_intlen + 1; - keys[0].bv_val = slap_sl_malloc( index_intlen+1, ctx ); + keys[0].bv_len = index_intlen; + keys[0].bv_val = slap_sl_malloc( index_intlen, ctx ); - if ( value->bv_len > sizeof( ibuf )) { - iv.bv_val = slap_sl_malloc( value->bv_len, ctx ); - iv.bv_len = value->bv_len; + iv.bv_len = value->bv_len < index_intlen_strlen + ? value->bv_len : index_intlen_strlen; + if ( iv.bv_len > (int) sizeof(ibuf) ) { + iv.bv_val = slap_sl_malloc( iv.bv_len, ctx ); } else { iv.bv_val = ibuf; iv.bv_len = sizeof(ibuf); } - rc = integerVal2Key( value, keys, &iv ); + rc = integerVal2Key( *value, keys, iv ); if ( rc == 0 ) *keysp = keys;