From 8a1e7a4b21a171600622311932ac792d9643d515 Mon Sep 17 00:00:00 2001 From: Quanah Gibson-Mount Date: Tue, 27 Nov 2007 19:59:57 +0000 Subject: [PATCH] ITS#5239 -- Ordered indexing for integer attributes --- CHANGES | 1 + doc/man/man5/slapd-config.5 | 13 ++- doc/man/man5/slapd.conf.5 | 11 ++- servers/slapd/bconfig.c | 23 +++++- servers/slapd/proto-slap.h | 20 ++++- servers/slapd/schema_init.c | 152 ++++++++++++++++++++++++++++++++++-- 6 files changed, 205 insertions(+), 15 deletions(-) diff --git a/CHANGES b/CHANGES index 9151703aff..21aabbdf5c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ OpenLDAP 2.4 Change Log OpenLDAP 2.4.7 Engineering + Added slapd ordered indexing of integer attributes (ITS#5239) Fixed slapd paged results control handling (ITS#5191) Fixed slapd sasl-host parsing (ITS#5209) Fixed slapd filter normalization (ITS#5212) diff --git a/doc/man/man5/slapd-config.5 b/doc/man/man5/slapd-config.5 index 8064c8cf30..5e3b4bfd81 100644 --- a/doc/man/man5/slapd-config.5 +++ b/doc/man/man5/slapd-config.5 @@ -445,7 +445,7 @@ server .B with another database, without disrupting the currently active clients. The default is FALSE. You may wish to use -.B olcIdletTmeout +.B olcIdleTimeout along with this option. .TP .B olcIdleTimeout: @@ -453,6 +453,11 @@ Specify the number of seconds to wait before forcibly closing an idle client connection. A setting of 0 disables this feature. The default is 0. .TP +.B olcIndexIntLen: +Specify the key length for ordered integer indices. The most significant +bytes of the binary integer will be used for index keys. The default +value is 4, which provides exact indexing for 32 bit values. +.TP .B olcIndexSubstrIfMaxlen: Specify the maximum length for subinitial and subfinal indices. Only this many characters of an attribute value will be processed by the @@ -479,7 +484,11 @@ lookup. The default is 2. For example, with the default values, a search using this filter "cn=*abcdefgh*" would generate index lookups for "abcd", "cdef", and "efgh". -Note: Indexing support depends on the particular backend in use. +.LP +Note: Indexing support depends on the particular backend in use. Also, +changing these settings will generally require deleting any indices that +depend on these parameters and recreating them with +.BR slapindex (8). .TP .B olcLocalSSF: diff --git a/doc/man/man5/slapd.conf.5 b/doc/man/man5/slapd.conf.5 index c59aaf7fc1..6bc5b58127 100644 --- a/doc/man/man5/slapd.conf.5 +++ b/doc/man/man5/slapd.conf.5 @@ -470,6 +470,11 @@ feature. The default is 0. Read additional configuration information from the given file before continuing with the next line of the current file. .TP +.B index_intlen +Specify the key length for ordered integer indices. The most significant +bytes of the binary integer will be used for index keys. The default +value is 4, which provides exact indexing for 32 bit values. +.TP .B index_substr_if_minlen Specify the minimum length for subinitial and subfinal indices. An attribute value must have at least this many characters in order to be @@ -496,7 +501,11 @@ lookup. The default is 2. For example, with the default values, a search using this filter "cn=*abcdefgh*" would generate index lookups for "abcd", "cdef", and "efgh". -Note: Indexing support depends on the particular backend in use. +.LP +Note: Indexing support depends on the particular backend in use. Also, +changing these settings will generally require deleting any indices that +depend on these parameters and recreating them with +.BR slapindex (8). .TP .B localSSF diff --git a/servers/slapd/bconfig.c b/servers/slapd/bconfig.c index 7aa86fbbad..0a952f91bd 100644 --- a/servers/slapd/bconfig.c +++ b/servers/slapd/bconfig.c @@ -179,6 +179,7 @@ enum { CFG_MONITORING, CFG_SERVERID, CFG_SORTVALS, + CFG_IX_INTLEN, CFG_LAST }; @@ -372,6 +373,9 @@ static ConfigTable config_back_cf_table[] = { { "index_substr_any_step", "step", 2, 2, 0, ARG_INT|ARG_NONZERO, &index_substr_any_step, "( OLcfgGlAt:23 NAME 'olcIndexSubstrAnyStep' " "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, + { "index_intlen", "len", 2, 2, 0, ARG_INT|ARG_MAGIC|CFG_IX_INTLEN, + &config_generic, "( OLcfgGlAt:84 NAME 'olcIndexIntLen' " + "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, { "lastmod", "on|off", 2, 2, 0, ARG_DB|ARG_ON_OFF|ARG_MAGIC|CFG_LASTMOD, &config_generic, "( OLcfgDbAt:0.4 NAME 'olcLastMod' " "SYNTAX OMsBoolean SINGLE-VALUE )", NULL, NULL }, @@ -709,8 +713,8 @@ static ConfigOCs cf_ocs[] = { "olcConnMaxPending $ olcConnMaxPendingAuth $ " "olcDisallows $ olcGentleHUP $ olcIdleTimeout $ " "olcIndexSubstrIfMaxLen $ olcIndexSubstrIfMinLen $ " - "olcIndexSubstrAnyLen $ olcIndexSubstrAnyStep $ olcLocalSSF $ " - "olcLogLevel $ " + "olcIndexSubstrAnyLen $ olcIndexSubstrAnyStep $ olcIndexIntLen $ " + "olcLocalSSF $ olcLogLevel $ " "olcPasswordCryptSaltFormat $ olcPasswordHash $ olcPidFile $ " "olcPluginLogFile $ olcReadOnly $ olcReferral $ " "olcReplogFile $ olcRequires $ olcRestrict $ olcReverseLookup $ " @@ -1014,6 +1018,9 @@ config_generic(ConfigArgs *c) { case CFG_SSTR_IF_MIN: c->value_int = index_substr_if_minlen; break; + case CFG_IX_INTLEN: + c->value_int = index_intlen; + break; case CFG_SORTVALS: { ADlist *sv; rc = 1; @@ -1151,6 +1158,10 @@ config_generic(ConfigArgs *c) { c->be->be_flags &= ~SLAP_DBFLAG_HIDDEN; break; + case CFG_IX_INTLEN: + index_intlen = SLAP_INDEX_INTLEN_DEFAULT; + break; + case CFG_ACL: if ( c->valx < 0 ) { AccessControl *end; @@ -1494,6 +1505,14 @@ config_generic(ConfigArgs *c) { return(1); break; + case CFG_IX_INTLEN: + if ( c->value_int < SLAP_INDEX_INTLEN_DEFAULT ) + c->value_int = SLAP_INDEX_INTLEN_DEFAULT; + else if ( c->value_int > 255 ) + c->value_int = 255; + index_intlen = c->value_int; + break; + case CFG_SORTVALS: { ADlist *svnew = NULL, *svtail, *sv; diff --git a/servers/slapd/proto-slap.h b/servers/slapd/proto-slap.h index d53fd93cb0..6bceda7575 100644 --- a/servers/slapd/proto-slap.h +++ b/servers/slapd/proto-slap.h @@ -707,12 +707,12 @@ LDAP_SLAPD_F (int) connections_shutdown LDAP_P((void)); LDAP_SLAPD_F (int) connections_destroy LDAP_P((void)); LDAP_SLAPD_F (int) connections_timeout_idle LDAP_P((time_t)); -LDAP_SLAPD_F (int) connection_client_setup LDAP_P(( +LDAP_SLAPD_F (Connection *) connection_client_setup LDAP_P(( ber_socket_t s, ldap_pvt_thread_start_t *func, void *arg )); -LDAP_SLAPD_F (void) connection_client_enable LDAP_P(( ber_socket_t s )); -LDAP_SLAPD_F (void) connection_client_stop LDAP_P(( ber_socket_t s )); +LDAP_SLAPD_F (void) connection_client_enable LDAP_P(( Connection *c )); +LDAP_SLAPD_F (void) connection_client_stop LDAP_P(( Connection *c )); #ifdef LDAP_PF_LOCAL_SENDMSG #define LDAP_PF_LOCAL_SENDMSG_ARG(arg) , arg @@ -822,6 +822,19 @@ LDAP_SLAPD_V (const char *) slapd_slp_attrs; LDAP_SLAPD_V (slap_ssf_t) local_ssf; LDAP_SLAPD_V (struct runqueue_s) slapd_rq; +#ifdef HAVE_WINSOCK +LDAP_SLAPD_F (ber_socket_t) slapd_socknew(ber_socket_t s); +LDAP_SLAPD_F (ber_socket_t) slapd_sock2fd(ber_socket_t s); +LDAP_SLAPD_V (SOCKET *) slapd_ws_sockets; +#define SLAP_FD2SOCK(s) slapd_ws_sockets[s] +#define SLAP_SOCK2FD(s) slapd_sock2fd(s) +#define SLAP_SOCKNEW(s) slapd_socknew(s) +#else +#define SLAP_FD2SOCK(s) s +#define SLAP_SOCK2FD(s) s +#define SLAP_SOCKNEW(s) s +#endif + /* * dn.c */ @@ -1863,6 +1876,7 @@ LDAP_SLAPD_V (unsigned int) index_substr_if_minlen; LDAP_SLAPD_V (unsigned int) index_substr_if_maxlen; LDAP_SLAPD_V (unsigned int) index_substr_any_len; LDAP_SLAPD_V (unsigned int) index_substr_any_step; +LDAP_SLAPD_V (unsigned int) index_intlen; LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming; LDAP_SLAPD_V (ber_len_t) sockbuf_max_incoming_auth; diff --git a/servers/slapd/schema_init.c b/servers/slapd/schema_init.c index f318a75564..ee96f4e686 100644 --- a/servers/slapd/schema_init.c +++ b/servers/slapd/schema_init.c @@ -62,6 +62,8 @@ unsigned int index_substr_if_maxlen = SLAP_INDEX_SUBSTR_IF_MAXLEN_DEFAULT; unsigned int index_substr_any_len = SLAP_INDEX_SUBSTR_ANY_LEN_DEFAULT; unsigned int index_substr_any_step = SLAP_INDEX_SUBSTR_ANY_STEP_DEFAULT; +unsigned int index_intlen = SLAP_INDEX_INTLEN_DEFAULT; + ldap_pvt_thread_mutex_t ad_undef_mutex; ldap_pvt_thread_mutex_t oc_undef_mutex; @@ -2109,7 +2111,143 @@ integerMatch( *matchp = match; return LDAP_SUCCESS; } - + +/* Index generation function */ +static int +integerIndexer( + slap_mask_t use, + slap_mask_t flags, + Syntax *syntax, + MatchingRule *mr, + struct berval *prefix, + BerVarray values, + BerVarray *keysp, + void *ctx ) +{ + char ibuf[64]; + struct berval iv, itmp; + BerVarray keys; + int i, rc; + + for( i=0; !BER_BVISNULL( &values[i] ); i++ ) { + /* just count them */ + } + + /* we should have at least one value at this point */ + assert( i > 0 ); + + keys = slap_sl_malloc( sizeof( struct berval ) * (i+1), ctx ); + for ( i = 0; !BER_BVISNULL( &values[i] ); i++ ) { + keys[i].bv_len = index_intlen+1; + keys[i].bv_val = slap_sl_malloc( index_intlen+1, ctx ); + } + keys[i].bv_len = 0; + keys[i].bv_val = NULL; + + itmp.bv_val = ibuf; + itmp.bv_len = sizeof(ibuf); + + for ( i=0; !BER_BVISNULL( &values[i] ); i++ ) { + if ( values[i].bv_len > itmp.bv_len ) { + itmp.bv_len = values[i].bv_len; + if ( itmp.bv_val == ibuf ) { + itmp.bv_val = slap_sl_malloc( itmp.bv_len, ctx ); + } else { + itmp.bv_val = slap_sl_realloc( itmp.bv_val, itmp.bv_len, ctx ); + } + } + iv = itmp; + if ( lutil_str2bin( &values[i], &iv )) { + rc = LDAP_INVALID_SYNTAX; + goto leave; + } + /* If too small, pad with zeros */ + if ( iv.bv_len < index_intlen ) { + int j, k; + keys[i].bv_val[0] = index_intlen; + k = index_intlen - iv.bv_len + 1; + for ( j=1; jbv_len > sizeof( ibuf )) { + iv.bv_val = slap_sl_malloc( value->bv_len, ctx ); + iv.bv_len = value->bv_len; + } else { + iv.bv_val = ibuf; + iv.bv_len = sizeof(ibuf); + } + + if ( lutil_str2bin( value, &iv )) { + rc = LDAP_INVALID_SYNTAX; + goto leave; + } + /* If too small, pad with zeros */ + if ( iv.bv_len < index_intlen ) { + int j, k; + keys[0].bv_val[0] = index_intlen; + k = index_intlen - iv.bv_len + 1; + for ( j=1; jbv_val[n], sn.bv_val, sn.bv_len ); { int j; - unsigned char *v = sn2.bv_val; + unsigned char *v = (unsigned char *)sn2.bv_val; out->bv_val[n++] = '\''; for ( j = 0; j < sn2.bv_len; j++ ) { - sprintf( &out->bv_val[n], "%02X", v[j] ); + snprintf( &out->bv_val[n], out->bv_len - n + 1, + "%02X", v[j] ); n += 2; } out->bv_val[n++] = '\''; @@ -4650,14 +4788,14 @@ static slap_mrule_defs_rec mrule_defs[] = { {"( 2.5.13.14 NAME 'integerMatch' " "SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 )", - SLAP_MR_EQUALITY | SLAP_MR_EXT, NULL, + SLAP_MR_EQUALITY | SLAP_MR_EXT | SLAP_MR_ORDERED_INDEX, NULL, NULL, NULL, integerMatch, - octetStringIndexer, octetStringFilter, + integerIndexer, integerFilter, NULL }, {"( 2.5.13.15 NAME 'integerOrderingMatch' " "SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 )", - SLAP_MR_ORDERING, NULL, + SLAP_MR_ORDERING | SLAP_MR_ORDERED_INDEX, NULL, NULL, NULL, integerMatch, NULL, NULL, "integerMatch" }, -- 2.39.5