Integer indexing - handle huge values

diff --git a/doc/man/man5/slapd-config.5 b/doc/man/man5/slapd-config.5
index 5e3b4bfd815c0501544e2025e29f6d30b9e3161a..96aeffd4c81bb19fcd206b1839ee4b58d5d0543e 100644 (file)
--- a/doc/man/man5/slapd-config.5
+++ b/doc/man/man5/slapd-config.5
@@ -456,7 +456,8 @@ feature.  The default is 0.
 .B olcIndexIntLen: <integer>
 Specify the key length for ordered integer indices. The most significant
 bytes of the binary integer will be used for index keys. The default
-value is 4, which provides exact indexing for 32 bit values.
+value is 4, which provides exact indexing for 31 bit values.
+A floating point representation is used to index too large values.
 .TP
 .B olcIndexSubstrIfMaxlen: <integer>
 Specify the maximum length for subinitial and subfinal indices. Only

diff --git a/doc/man/man5/slapd.conf.5 b/doc/man/man5/slapd.conf.5
index 6bc5b5812733f1e201ebbe137dc04742a9c84492..c20df8b68843995a87787ee37d7c9d03d7d04f15 100644 (file)
--- a/doc/man/man5/slapd.conf.5
+++ b/doc/man/man5/slapd.conf.5
@@ -473,7 +473,8 @@ continuing with the next line of the current file.
 .B index_intlen <integer>
 Specify the key length for ordered integer indices. The most significant
 bytes of the binary integer will be used for index keys. The default
-value is 4, which provides exact indexing for 32 bit values.
+value is 4, which provides exact indexing for 31 bit values.
+A floating point representation is used to index too large values.
 .TP
 .B index_substr_if_minlen <integer>
 Specify the minimum length for subinitial and subfinal indices. An

diff --git a/servers/slapd/bconfig.c b/servers/slapd/bconfig.c
index 0a952f91bddce74719f5a5935b4bcc40a2e1db87..10fec37525e931f6dd2290cd49e62220ae20f2d8 100644 (file)
--- a/servers/slapd/bconfig.c
+++ b/servers/slapd/bconfig.c
@@ -1160,6 +1160,8 @@ config_generic(ConfigArgs *c) {
 
                case CFG_IX_INTLEN:
                        index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+                       index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+                               SLAP_INDEX_INTLEN_DEFAULT );
                        break;
 
                case CFG_ACL:
@@ -1511,6 +1513,8 @@ config_generic(ConfigArgs *c) {
                        else if ( c->value_int > 255 )
                                c->value_int = 255;
                        index_intlen = c->value_int;
+                       index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+                               index_intlen );
                        break;
                        
                case CFG_SORTVALS: {

diff --git a/servers/slapd/proto-slap.h b/servers/slapd/proto-slap.h
index 6bceda757583841e673c817975899b5c4b5bf779..4e1684235a9d36700272278b4756cd42bc06b3d9 100644 (file)
--- a/servers/slapd/proto-slap.h
+++ b/servers/slapd/proto-slap.h
@@ -1877,6 +1877,10 @@ LDAP_SLAPD_V (unsigned int) index_substr_if_maxlen;
 LDAP_SLAPD_V (unsigned int) index_substr_any_len;
 LDAP_SLAPD_V (unsigned int) index_substr_any_step;
 LDAP_SLAPD_V (unsigned int) index_intlen;
+/* all signed integers from strings of this size need more than intlen bytes */
+/* i.e. log(10)*(index_intlen_strlen-2) > log(2)*(8*(index_intlen)-1) */
+LDAP_SLAPD_V (unsigned int) index_intlen_strlen;
+#define SLAP_INDEX_INTLEN_STRLEN(intlen) ((8*(intlen)-1) * 146/485 + 3)
 
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming;
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming_auth;

diff --git a/servers/slapd/schema_init.c b/servers/slapd/schema_init.c
index fa75175acce52af1ff36b722156c7ffef939ea27..501b9d60673cbad1a184b1acd396fc2a42404d19 100644 (file)
--- a/servers/slapd/schema_init.c
+++ b/servers/slapd/schema_init.c
@@ -63,6 +63,8 @@ unsigned int index_substr_any_len = SLAP_INDEX_SUBSTR_ANY_LEN_DEFAULT;
 unsigned int index_substr_any_step = SLAP_INDEX_SUBSTR_ANY_STEP_DEFAULT;
 
 unsigned int index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+unsigned int index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+       SLAP_INDEX_INTLEN_DEFAULT );
 
 ldap_pvt_thread_mutex_t        ad_undef_mutex;
 ldap_pvt_thread_mutex_t        oc_undef_mutex;
@@ -2114,49 +2116,69 @@ integerMatch(
 
 static int
 integerVal2Key(
-       struct berval *val,
+       struct berval val,
        struct berval *key,
-       struct berval *tmp
+       struct berval itmp
 )
 {
-       struct berval iv;
-       int neg;
-
-       iv = *tmp;
-       if ( lutil_str2bin( val, &iv )) {
-               return LDAP_INVALID_SYNTAX;
+       /* index format:
+        * only if too large: one's complement <sign*length of chopped bytes>,
+        * two's complement value (sign-extended or chopped as needed),
+        * with 1st byte of the above adjusted as follows:
+        *   inverse sign in the top <number of length-bytes + 1> bits,
+        *   then the sign bit as delimiter.
+        */
+       ber_slen_t k = index_intlen_strlen, chop = 0;
+       unsigned char neg = 0xff, signmask = 0x80;
+       unsigned char lenbuf[sizeof(k) + 2], *lenp;
+
+       if ( val.bv_val[0] != '-' ) {
+               neg = 0;
+               --k;
+       }
+
+       /* Chop least significant digits, increase length instead */
+       if ( val.bv_len > k ) {
+               chop = (val.bv_len - k + 2) / 7;        /* 2 fewer digits */
+               val.bv_len -= chop * 7; /* #digits chopped */
+               chop *= 3;      /* >#key bytes chopped: 256**3 > 10**7 */
+               if ( chop > 0x7fffffff ) {
+                       memset( key->bv_val, neg ^ 0xff, index_intlen );
+                       return 0;
+               }
        }
 
-       neg = iv.bv_val[0] & 0x80;
-
-       /* Omit leading 0 pad byte */
-       if ( !iv.bv_val[0] ) {
-               iv.bv_val++;
-               iv.bv_len--;
+       if ( lutil_str2bin( &val, &itmp )) {
+               return LDAP_INVALID_SYNTAX;
        }
 
-       /* If too small, sign-extend */
-       if ( iv.bv_len < index_intlen ) {
-               int j, k, pad;
-               key->bv_val[0] = index_intlen;
-               k = index_intlen - iv.bv_len + 1;
-               if ( neg )
-                       pad = 0xff;
-               else
-                       pad = 0;
-               for ( j=1; j<k; j++)
-                       key->bv_val[j] = pad;
-               for ( j = 0; j<iv.bv_len; j++ )
-                       key->bv_val[j+k] = iv.bv_val[j];
-       } else {
-               key->bv_val[0] = iv.bv_len;
-               memcpy( key->bv_val+1, iv.bv_val, index_intlen );
-       }
-       if ( neg ) {
-               key->bv_val[0] = -key->bv_val[0];
-       }
-       /* convert signed to unsigned */
-       key->bv_val[0] ^= 0x80;
+       /* Omit leading sign byte */
+       if ( itmp.bv_val[0] == neg ) {
+               itmp.bv_val++;
+               itmp.bv_len--;
+       }
+
+       k = (ber_slen_t) index_intlen - ((ber_slen_t) itmp.bv_len + chop);
+       if ( k > 0 ) {
+               assert( chop == 0 );
+               memset( key->bv_val, neg, k );  /* sign-extend */
+       } else if ( k != 0 || ((itmp.bv_val[0] ^ neg) & 0xc0) ) {
+               lenp = lenbuf + sizeof(lenbuf);
+               k = -k;
+               do {
+                       *--lenp = k ^ neg;
+                       signmask >>= 1;
+               } while ( (k >>= 8) != 0 || (signmask >> 1) <= (*lenp ^ neg) );
+               /* With n bytes used in lenbuf, the top n+1 bits of signmask
+                * are 1, and the top n+2 bits of *lenp are the sign bit. */
+               k = (lenbuf + sizeof(lenbuf)) - lenp;
+               if ( k > index_intlen )
+                       k = index_intlen;
+               memcpy( key->bv_val, lenp, k );
+               itmp.bv_len = index_intlen - k;
+       }
+       memcpy( key->bv_val + k, itmp.bv_val, itmp.bv_len );
+       key->bv_val[0] ^= (unsigned char) -signmask & 0xff; /* invert sign */
        return 0;
 }
 
@@ -2177,34 +2199,35 @@ integerIndexer(
        BerVarray keys;
        int i, rc;
 
-       for( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
-               /* just count them */
+       /* count the values and find max needed length */
+       itmp.bv_len = 0;
+       for( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
+               if ( itmp.bv_len < values[i].bv_len )
+                       itmp.bv_len = values[i].bv_len;
        }
+       if ( itmp.bv_len > index_intlen_strlen )
+               itmp.bv_len > index_intlen_strlen;
 
        /* we should have at least one value at this point */
        assert( i > 0 );
 
        keys = slap_sl_malloc( sizeof( struct berval ) * (i+1), ctx );
        for ( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
-               keys[i].bv_len = index_intlen+1;
-               keys[i].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+               keys[i].bv_len = index_intlen;
+               keys[i].bv_val = slap_sl_malloc( index_intlen, ctx );
        }
        keys[i].bv_len = 0;
        keys[i].bv_val = NULL;
 
-       itmp.bv_val = ibuf;
-       itmp.bv_len = sizeof(ibuf);
+       if ( itmp.bv_len > (int) sizeof(ibuf) ) {
+               itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx );
+       } else {
+               itmp.bv_val = ibuf;
+               itmp.bv_len = sizeof(ibuf);
+       }
 
        for ( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
-               if ( values[i].bv_len > itmp.bv_len ) {
-                       itmp.bv_len = values[i].bv_len;
-                       if ( itmp.bv_val == ibuf ) {
-                               itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx );
-                       } else {
-                               itmp.bv_val = slap_sl_realloc( itmp.bv_val, itmp.bv_len, ctx );
-                       }
-               }
-               rc = integerVal2Key( &values[i], &keys[i], &itmp );
+               rc = integerVal2Key( values[i], &keys[i], itmp );
                if ( rc )
                        goto leave;
        }
@@ -2238,18 +2261,19 @@ integerFilter(
 
        keys = slap_sl_malloc( sizeof( struct berval ) * 2, ctx );
 
-       keys[0].bv_len = index_intlen + 1;
-       keys[0].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+       keys[0].bv_len = index_intlen;
+       keys[0].bv_val = slap_sl_malloc( index_intlen, ctx );
 
-       if ( value->bv_len > sizeof( ibuf )) {
-               iv.bv_val = slap_sl_malloc( value->bv_len, ctx );
-               iv.bv_len = value->bv_len;
+       iv.bv_len = value->bv_len < index_intlen_strlen
+               ? value->bv_len : index_intlen_strlen;
+       if ( iv.bv_len > (int) sizeof(ibuf) ) {
+               iv.bv_val = slap_sl_malloc( iv.bv_len, ctx );
        } else {
                iv.bv_val = ibuf;
                iv.bv_len = sizeof(ibuf);
        }
 
-       rc = integerVal2Key( value, keys, &iv );
+       rc = integerVal2Key( *value, keys, iv );
        if ( rc == 0 )
                *keysp = keys;