Integer indexing, sync with HEAD

diff --git a/include/lutil.h b/include/lutil.h
index ad8fc62a1eec5484a043d31c6c4d8de338c6931b..6c0b2f158034b968caaec189712a76025385b3e9 100644 (file)
--- a/include/lutil.h
+++ b/include/lutil.h
@@ -304,9 +304,8 @@ lutil_atoulx( unsigned long *v, const char *s, int x );
 #define lutil_atol(v, s)       lutil_atolx((v), (s), 10)
 #define lutil_atoul(v, s)      lutil_atoulx((v), (s), 10)
 
-/* Parse and unparse time intervals */
 LDAP_LUTIL_F (int)
-lutil_str2bin( struct berval *in, struct berval *out );
+lutil_str2bin( struct berval *in, struct berval *out, void *ctx );
 
 /* Parse and unparse time intervals */
 LDAP_LUTIL_F (int)

diff --git a/libraries/liblutil/utils.c b/libraries/liblutil/utils.c
index f14e6c877de679b20fdf96d2b4c1dd3f6212e26c..62db930f4437f7e75e5b818e2c8f88f356bdf701 100644 (file)
--- a/libraries/liblutil/utils.c
+++ b/libraries/liblutil/utils.c
@@ -35,6 +35,7 @@
 #include "lutil.h"
 #include "ldap_defaults.h"
 #include "ldap_pvt.h"
+#include "lber_pvt.h"
 
 #ifdef HAVE_EBCDIC
 int _trans_argv = 1;
@@ -671,7 +672,7 @@ scale( int new, lutil_int_decnum *prev, unsigned char *tmp )
  * any hex input.
  */
 int
-lutil_str2bin( struct berval *in, struct berval *out )
+lutil_str2bin( struct berval *in, struct berval *out, void *ctx )
 {
        char *pin, *pout, ctmp;
        char *end;
@@ -747,7 +748,7 @@ lutil_str2bin( struct berval *in, struct berval *out )
 
                /* tmp must be at least as large as outbuf */
                if ( out->bv_len > sizeof(tmpbuf)) {
-                       tmp = ber_memalloc( out->bv_len );
+                       tmp = ber_memalloc_x( out->bv_len, ctx );
                } else {
                        tmp = tmpbuf;
                }
@@ -795,7 +796,7 @@ lutil_str2bin( struct berval *in, struct berval *out )
                out->bv_len = num.len;
 decfail:
                if ( tmp != tmpbuf ) {
-                       ber_memfree( tmp );
+                       ber_memfree_x( tmp, ctx );
                }
        }
        return rc;

diff --git a/servers/slapd/bconfig.c b/servers/slapd/bconfig.c
index 0a952f91bddce74719f5a5935b4bcc40a2e1db87..10fec37525e931f6dd2290cd49e62220ae20f2d8 100644 (file)
--- a/servers/slapd/bconfig.c
+++ b/servers/slapd/bconfig.c
@@ -1160,6 +1160,8 @@ config_generic(ConfigArgs *c) {
 
                case CFG_IX_INTLEN:
                        index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+                       index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+                               SLAP_INDEX_INTLEN_DEFAULT );
                        break;
 
                case CFG_ACL:
@@ -1511,6 +1513,8 @@ config_generic(ConfigArgs *c) {
                        else if ( c->value_int > 255 )
                                c->value_int = 255;
                        index_intlen = c->value_int;
+                       index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+                               index_intlen );
                        break;
                        
                case CFG_SORTVALS: {

diff --git a/servers/slapd/proto-slap.h b/servers/slapd/proto-slap.h
index 6bceda757583841e673c817975899b5c4b5bf779..4e1684235a9d36700272278b4756cd42bc06b3d9 100644 (file)
--- a/servers/slapd/proto-slap.h
+++ b/servers/slapd/proto-slap.h
@@ -1877,6 +1877,10 @@ LDAP_SLAPD_V (unsigned int) index_substr_if_maxlen;
 LDAP_SLAPD_V (unsigned int) index_substr_any_len;
 LDAP_SLAPD_V (unsigned int) index_substr_any_step;
 LDAP_SLAPD_V (unsigned int) index_intlen;
+/* all signed integers from strings of this size need more than intlen bytes */
+/* i.e. log(10)*(index_intlen_strlen-2) > log(2)*(8*(index_intlen)-1) */
+LDAP_SLAPD_V (unsigned int) index_intlen_strlen;
+#define SLAP_INDEX_INTLEN_STRLEN(intlen) ((8*(intlen)-1) * 146/485 + 3)
 
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming;
 LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming_auth;

diff --git a/servers/slapd/schema_init.c b/servers/slapd/schema_init.c
index fa75175acce52af1ff36b722156c7ffef939ea27..ac0a73f68e7dab5badd029074ccc57845db7730f 100644 (file)
--- a/servers/slapd/schema_init.c
+++ b/servers/slapd/schema_init.c
@@ -63,6 +63,8 @@ unsigned int index_substr_any_len = SLAP_INDEX_SUBSTR_ANY_LEN_DEFAULT;
 unsigned int index_substr_any_step = SLAP_INDEX_SUBSTR_ANY_STEP_DEFAULT;
 
 unsigned int index_intlen = SLAP_INDEX_INTLEN_DEFAULT;
+unsigned int index_intlen_strlen = SLAP_INDEX_INTLEN_STRLEN(
+       SLAP_INDEX_INTLEN_DEFAULT );
 
 ldap_pvt_thread_mutex_t        ad_undef_mutex;
 ldap_pvt_thread_mutex_t        oc_undef_mutex;
@@ -2112,51 +2114,72 @@ integerMatch(
        return LDAP_SUCCESS;
 }
 
+/* 10**Chop < 256**Chopbytes and Chop > Chopbytes<<1 (for sign bit and itmp) */
+#define INDEX_INTLEN_CHOP 7
+#define INDEX_INTLEN_CHOPBYTES 3
+
 static int
 integerVal2Key(
-       struct berval *val,
+       struct berval *in,
        struct berval *key,
-       struct berval *tmp
-)
+       struct berval *tmp,
+       void *ctx )
 {
-       struct berval iv;
-       int neg;
+       /* index format:
+        * only if too large: one's complement <sign*exponent (chopped bytes)>,
+        * two's complement value (sign-extended or chopped as needed),
+        * however the top <number of exponent-bytes + 1> bits of first byte
+        * above is the inverse sign.   The next bit is the sign as delimiter.
+        */
+       ber_slen_t k = index_intlen_strlen;
+       ber_len_t chop = 0;
+       unsigned signmask = ~0x7fU;
+       unsigned char lenbuf[sizeof(k) + 2], *lenp, neg = 0xff;
+       struct berval val = *in, itmp = *tmp;
 
-       iv = *tmp;
-       if ( lutil_str2bin( val, &iv )) {
-               return LDAP_INVALID_SYNTAX;
+       if ( val.bv_val[0] != '-' ) {
+               neg = 0;
+               --k;
        }
 
-       neg = iv.bv_val[0] & 0x80;
-
-       /* Omit leading 0 pad byte */
-       if ( !iv.bv_val[0] ) {
-               iv.bv_val++;
-               iv.bv_len--;
+       /* Chop least significant digits, increase length instead */
+       if ( val.bv_len > (ber_len_t) k ) {
+               chop = (val.bv_len-k+2)/INDEX_INTLEN_CHOP; /* 2 fewer digits */
+               val.bv_len -= chop * INDEX_INTLEN_CHOP; /* #digits chopped */
+               chop *= INDEX_INTLEN_CHOPBYTES;         /* #bytes added */
        }
 
-       /* If too small, sign-extend */
-       if ( iv.bv_len < index_intlen ) {
-               int j, k, pad;
-               key->bv_val[0] = index_intlen;
-               k = index_intlen - iv.bv_len + 1;
-               if ( neg )
-                       pad = 0xff;
-               else
-                       pad = 0;
-               for ( j=1; j<k; j++)
-                       key->bv_val[j] = pad;
-               for ( j = 0; j<iv.bv_len; j++ )
-                       key->bv_val[j+k] = iv.bv_val[j];
-       } else {
-               key->bv_val[0] = iv.bv_len;
-               memcpy( key->bv_val+1, iv.bv_val, index_intlen );
-       }
-       if ( neg ) {
-               key->bv_val[0] = -key->bv_val[0];
+       if ( lutil_str2bin( &val, &itmp, ctx )) {
+               return LDAP_INVALID_SYNTAX;
        }
-       /* convert signed to unsigned */
-       key->bv_val[0] ^= 0x80;
+
+       /* Omit leading sign byte */
+       if ( itmp.bv_val[0] == neg ) {
+               itmp.bv_val++;
+               itmp.bv_len--;
+       }
+
+       k = (ber_slen_t) index_intlen - (ber_slen_t) (itmp.bv_len + chop);
+       if ( k > 0 ) {
+               assert( chop == 0 );
+               memset( key->bv_val, neg, k );  /* sign-extend */
+       } else if ( k != 0 || ((itmp.bv_val[0] ^ neg) & 0xc0) ) {
+               lenp = lenbuf + sizeof(lenbuf);
+               chop = - (ber_len_t) k;
+               do {
+                       *--lenp = ((unsigned char) chop & 0xff) ^ neg;
+                       signmask >>= 1;
+               } while ( (chop >>= 8) != 0 || (signmask >> 1) & (*lenp ^ neg) );
+               /* With n bytes in lenbuf, the top n+1 bits of (signmask&0xff)
+                * are 1, and the top n+2 bits of lenp[] are the sign bit. */
+               k = (lenbuf + sizeof(lenbuf)) - lenp;
+               if ( k > (ber_slen_t) index_intlen )
+                       k = index_intlen;
+               memcpy( key->bv_val, lenp, k );
+               itmp.bv_len = index_intlen - k;
+       }
+       memcpy( key->bv_val + k, itmp.bv_val, itmp.bv_len );
+       key->bv_val[0] ^= (unsigned char) signmask & 0xff; /* invert sign */
        return 0;
 }
 
@@ -2175,36 +2198,46 @@ integerIndexer(
        char ibuf[64];
        struct berval itmp;
        BerVarray keys;
+       ber_len_t vlen;
        int i, rc;
+       unsigned maxstrlen = index_intlen_strlen + INDEX_INTLEN_CHOP-1;
 
-       for( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
-               /* just count them */
+       /* count the values and find max needed length */
+       vlen = 0;
+       for( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
+               if ( vlen < values[i].bv_len )
+                       vlen = values[i].bv_len;
        }
+       if ( vlen > maxstrlen )
+               vlen = maxstrlen;
 
        /* we should have at least one value at this point */
        assert( i > 0 );
 
        keys = slap_sl_malloc( sizeof( struct berval ) * (i+1), ctx );
        for ( i = 0; !BER_BVISNULL( &values[i] ); i++ ) {
-               keys[i].bv_len = index_intlen+1;
-               keys[i].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+               keys[i].bv_len = index_intlen;
+               keys[i].bv_val = slap_sl_malloc( index_intlen, ctx );
        }
        keys[i].bv_len = 0;
        keys[i].bv_val = NULL;
 
-       itmp.bv_val = ibuf;
+       if ( vlen > sizeof(ibuf) ) {
+               itmp.bv_val = slap_sl_malloc( vlen, ctx );
+       } else {
+               itmp.bv_val = ibuf;
+       }
        itmp.bv_len = sizeof(ibuf);
 
        for ( i=0; !BER_BVISNULL( &values[i] ); i++ ) {
-               if ( values[i].bv_len > itmp.bv_len ) {
+               if ( itmp.bv_val != ibuf ) {
                        itmp.bv_len = values[i].bv_len;
-                       if ( itmp.bv_val == ibuf ) {
-                               itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx );
-                       } else {
-                               itmp.bv_val = slap_sl_realloc( itmp.bv_val, itmp.bv_len, ctx );
-                       }
+                       if ( itmp.bv_len <= sizeof(ibuf) )
+                               itmp.bv_len = sizeof(ibuf);
+                       else if ( itmp.bv_len > maxstrlen )
+                               itmp.bv_len = maxstrlen;
                }
-               rc = integerVal2Key( &values[i], &keys[i], &itmp );
+               rc = integerVal2Key( &values[i], &keys[i], &itmp, ctx );
                if ( rc )
                        goto leave;
        }
@@ -2238,18 +2271,19 @@ integerFilter(
 
        keys = slap_sl_malloc( sizeof( struct berval ) * 2, ctx );
 
-       keys[0].bv_len = index_intlen + 1;
-       keys[0].bv_val = slap_sl_malloc( index_intlen+1, ctx );
+       keys[0].bv_len = index_intlen;
+       keys[0].bv_val = slap_sl_malloc( index_intlen, ctx );
 
-       if ( value->bv_len > sizeof( ibuf )) {
-               iv.bv_val = slap_sl_malloc( value->bv_len, ctx );
-               iv.bv_len = value->bv_len;
+       iv.bv_len = value->bv_len < index_intlen_strlen + INDEX_INTLEN_CHOP-1
+               ? value->bv_len : index_intlen_strlen + INDEX_INTLEN_CHOP-1;
+       if ( iv.bv_len > (int) sizeof(ibuf) ) {
+               iv.bv_val = slap_sl_malloc( iv.bv_len, ctx );
        } else {
                iv.bv_val = ibuf;
                iv.bv_len = sizeof(ibuf);
        }
 
-       rc = integerVal2Key( value, keys, &iv );
+       rc = integerVal2Key( value, keys, &iv, ctx );
        if ( rc == 0 )
                *keysp = keys;
 
@@ -3106,7 +3140,7 @@ serialNumberAndIssuerNormalize(
        }
        sn2.bv_val = stmp;
        sn2.bv_len = sn.bv_len;
-       if ( lutil_str2bin( &sn, &sn2 )) {
+       if ( lutil_str2bin( &sn, &sn2, ctx )) {
                rc = LDAP_INVALID_SYNTAX;
                goto leave;
        }