From 986bcd52a9909387eeb7cf640380d5df4532beb8 Mon Sep 17 00:00:00 2001 From: Jong Hyuk Choi Date: Fri, 9 May 2003 06:50:44 +0000 Subject: [PATCH] -syncUUID search in syncrepl -test017-syncreplication --- servers/slapd/config.c | 46 +----- servers/slapd/proto-slap.h | 2 + servers/slapd/slap.h | 1 + servers/slapd/syncrepl.c | 176 +++++++++++++++------ servers/slapd/tools/mimic.c | 5 + tests/data/slapd-syncrepl-master.conf | 30 ++++ tests/data/slapd-syncrepl-slave.conf | 45 ++++++ tests/scripts/defines.sh | 2 + tests/scripts/test017-syncreplication | 212 ++++++++++++++++++++++++++ 9 files changed, 429 insertions(+), 90 deletions(-) create mode 100644 tests/data/slapd-syncrepl-master.conf create mode 100644 tests/data/slapd-syncrepl-slave.conf create mode 100755 tests/scripts/test017-syncreplication diff --git a/servers/slapd/config.c b/servers/slapd/config.c index 0a0def3979..95a611335c 100644 --- a/servers/slapd/config.c +++ b/servers/slapd/config.c @@ -2896,51 +2896,9 @@ parse_syncrepl_line( si->attrsonly = 1; } else if ( !strncasecmp( cargv[ i ], ATTRSSTR, sizeof( ATTRSSTR ) - 1 ) ) { - char **tmp; val = cargv[ i ] + sizeof( ATTRSSTR ); - for ( ; hp = strchr( val, ' ' ); val = ++hp ) { - *hp = '\0'; - if ( *val != '\0' ) { - nr_attr++; - tmp = (char **) ch_realloc( si->attrs, nr_attr * sizeof( char * )); - if ( tmp == NULL ) { -#ifdef NEW_LOGGING - LDAP_LOG( CONFIG, ERR, "out of memory\n", 0,0,0 ); -#else - Debug( LDAP_DEBUG_ANY, "out of memory\n", 0,0,0 ); -#endif - return -1; - } - si->attrs = tmp; - si->attrs[ nr_attr - 1 ] = ch_strdup( val ); - } - } - if ( *val != '\0' ) { - nr_attr++; - tmp = (char **) ch_realloc( si->attrs, nr_attr * sizeof( char * )); - if ( tmp == NULL ) { -#ifdef NEW_LOGGING - LDAP_LOG( CONFIG, ERR, "out of memory\n", 0,0,0 ); -#else - Debug( LDAP_DEBUG_ANY, "out of memory\n", 0,0,0 ); -#endif - return -1; - } - si->attrs = tmp; - si->attrs[ nr_attr - 1 ] = ch_strdup( val ); - } - nr_attr++; - tmp = (char **) ch_realloc( si->attrs, nr_attr * sizeof( char * )); - if ( tmp == NULL ) { -#ifdef NEW_LOGGING - LDAP_LOG( CONFIG, ERR, "out of memory\n", 0,0,0 ); -#else - Debug( LDAP_DEBUG_ANY, "out of memory\n", 0,0,0 ); -#endif - return -1; - } - si->attrs = tmp; - si->attrs[ nr_attr - 1 ] = NULL; + si->attrs = NULL; + si->attrs = str2clist( si->attrs, val, "," ); gots |= GOT_ATTRS; } else if ( !strncasecmp( cargv[ i ], TYPESTR, sizeof( TYPESTR ) - 1 ) ) { diff --git a/servers/slapd/proto-slap.h b/servers/slapd/proto-slap.h index 91d81bc459..9d26c17ebc 100644 --- a/servers/slapd/proto-slap.h +++ b/servers/slapd/proto-slap.h @@ -1127,6 +1127,8 @@ LDAP_SLAPD_V (struct runqueue_s) syncrepl_rq; LDAP_SLAPD_F (void) init_syncrepl LDAP_P(()); LDAP_SLAPD_F (void*) do_syncrepl LDAP_P((void *, void *)); + +LDAP_SLAPD_F (char **) str2clist( char **, char *, const char * ); #endif LDAP_END_DECL diff --git a/servers/slapd/slap.h b/servers/slapd/slap.h index 5daba74a03..6b8fb8fa0d 100644 --- a/servers/slapd/slap.h +++ b/servers/slapd/slap.h @@ -1323,6 +1323,7 @@ typedef struct syncinfo_s { int tls; int found; struct berval *syncUUID; + struct berval *syncUUID_ndn; struct berval *syncCookie; Avlnode *presentlist; LDAP_LIST_HEAD(np, nonpresent_entry) nonpresentlist; diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c index 3794f00cd2..923d4ecb72 100644 --- a/servers/slapd/syncrepl.c +++ b/servers/slapd/syncrepl.c @@ -69,6 +69,7 @@ slap_mods2entry_syncrepl( Modifications *, Entry **, int, /* callback functions */ static int cookie_callback( struct slap_op *, struct slap_rep * ); +static int dn_callback( struct slap_op *, struct slap_rep * ); static int nonpresent_callback( struct slap_op *, struct slap_rep * ); static int null_callback( struct slap_op *, struct slap_rep * ); @@ -891,13 +892,12 @@ syncrepl_entry( char csnbuf[ LDAP_LUTIL_CSNSTR_BUFSIZE ]; SlapReply rs = {REP_RESULT}; - int rc; + int rc = LDAP_SUCCESS; + + struct berval base_bv = {0, NULL}; -#if 0 /* FIXME : UUID search required first */ char *filterstr; - struct berval filterstr_bv; Filter *filter; -#endif Attribute *a; @@ -919,74 +919,103 @@ syncrepl_entry( attr_merge_one( e, slap_schema.si_ad_entryUUID, syncUUID, syncUUID ); } -#if 0 /* FIXME : UUID search required first */ filterstr = (char *) sl_malloc( strlen("entryUUID=") + syncUUID->bv_len + 1, op->o_tmpmemctx ); strcpy( filterstr, "entryUUID=" ); strcat( filterstr, syncUUID->bv_val ); -#endif si->e = e; si->syncUUID = syncUUID; + si->syncUUID_ndn = NULL; -#if 0 /* FIXME : UUID search required first */ filter = str2filter( filterstr ); - ber_str2bv( filterstr, strlen(filterstr), 1, &filterstr_bv ); + ber_str2bv( filterstr, strlen(filterstr), 1, &op->ors_filterstr ); ch_free( filterstr ); -#endif + op->ors_filter = filter; + op->ors_scope = LDAP_SCOPE_SUBTREE; - op->o_req_dn = e->e_name; - op->o_req_ndn = e->e_nname; + /* get syncrepl cookie of shadow replica from subentry */ + ber_str2bv( si->base, strlen(si->base), 1, &base_bv ); + dnPrettyNormal( 0, &base_bv, &op->o_req_dn, &op->o_req_ndn, op->o_tmpmemctx ); + ch_free( base_bv.bv_val ); + /* set callback function */ op->o_callback = &cb; - cb.sc_response = null_callback; + cb.sc_response = dn_callback; cb.sc_private = si; + be->be_search( op, &rs ); + + ch_free( op->o_req_dn.bv_val ); + ch_free( op->o_req_ndn.bv_val ); + filter_free( op->ors_filter ); + ch_free( op->ors_filterstr.bv_val ); + + cb.sc_response = null_callback; + + if ( si->syncUUID_ndn ) + printf("syncUUID_ndn = %s\n", si->syncUUID_ndn ); + switch ( syncstate ) { case LDAP_SYNC_ADD : case LDAP_SYNC_MODIFY : -sync_add_retry: - op->o_tag = LDAP_REQ_MODIFY; - op->orm_modlist = modlist; - rc = be->be_modify( op, &rs ); - if ( rc != LDAP_SUCCESS ) { - if ( rc == LDAP_REFERRAL || - rc == LDAP_NO_SUCH_OBJECT || - rc == DB_NOTFOUND ) { - op->o_tag = LDAP_REQ_ADD; - op->ora_e = e; - rc = be->be_add( op, &rs ); - if ( rc != LDAP_SUCCESS ) { - if ( rc == LDAP_ALREADY_EXISTS ) { - goto sync_add_retry; - } else if ( rc == LDAP_REFERRAL || - rc == LDAP_NO_SUCH_OBJECT || - rc == DB_NOTFOUND ) { - syncrepl_add_glue(ld, op, e, - modlist, syncstate, - syncUUID, syncCookie); - } else { + + rc = LDAP_SUCCESS; + + if ( si->syncUUID_ndn ) { + op->o_req_dn = *si->syncUUID_ndn; + op->o_req_ndn = *si->syncUUID_ndn; + op->o_tag = LDAP_REQ_DELETE; + rc = be->be_delete( op, &rs ); + } + + if ( rc == LDAP_SUCCESS || + rc == LDAP_REFERRAL || + rc == LDAP_NO_SUCH_OBJECT || + rc == DB_NOTFOUND ) { + op->o_tag = LDAP_REQ_ADD; + op->ora_e = e; + op->o_req_dn = e->e_name; + op->o_req_ndn = e->e_nname; + rc = be->be_add( op, &rs ); + if ( rc != LDAP_SUCCESS ) { + if ( rc == LDAP_ALREADY_EXISTS ) { #ifdef NEW_LOGGING - LDAP_LOG( OPERATION, ERR, - "be_add failed (%d)\n", - rc, 0, 0 ); + LDAP_LOG( OPERATION, ERR, + "be_add failed : already exists (%d)\n", + rc, 0, 0 ); #else - Debug( LDAP_DEBUG_ANY, - "be_add failed (%d)\n", - rc, 0, 0 ); + Debug( LDAP_DEBUG_ANY, + "be_add failed : already exists (%d)\n", + rc, 0, 0 ); #endif - } + } else if ( rc == LDAP_REFERRAL || + rc == LDAP_NO_SUCH_OBJECT || + rc == DB_NOTFOUND ) { + syncrepl_add_glue(ld, op, e, + modlist, syncstate, + syncUUID, syncCookie); } else { - return 0; +#ifdef NEW_LOGGING + LDAP_LOG( OPERATION, ERR, + "be_add failed (%d)\n", + rc, 0, 0 ); +#else + Debug( LDAP_DEBUG_ANY, + "be_add failed (%d)\n", + rc, 0, 0 ); +#endif } } else { + return 0; + } + } else { #ifdef NEW_LOGGING - LDAP_LOG( OPERATION, ERR, - "be_modify failed (%d)\n", rc, 0, 0 ); + LDAP_LOG( OPERATION, ERR, + "be_modify/be_delete failed (%d)\n", rc, 0, 0 ); #else - Debug( LDAP_DEBUG_ANY, - "be_modify failed (%d)\n", rc, 0, 0 ); + Debug( LDAP_DEBUG_ANY, + "be_modify/be_delete failed (%d)\n", rc, 0, 0 ); #endif - } } si->e = NULL; @@ -1916,6 +1945,21 @@ cookie_callback( return LDAP_SUCCESS; } +static int +dn_callback( + Operation* op, + SlapReply* rs +) +{ + syncinfo_t *si = op->o_callback->sc_private; + + if ( rs->sr_type == REP_SEARCH ) { + si->syncUUID_ndn = &rs->sr_entry->e_nname; + } + + return LDAP_SUCCESS; +} + static int nonpresent_callback( Operation* op, @@ -1982,4 +2026,44 @@ null_callback( return LDAP_SUCCESS; } + +char ** +str2clist( char **out, char *in, const char *brkstr ) +{ + char *str; + char *s; + char *lasts; + int i, j; + const char *text; + char **new; + + /* find last element in list */ + for (i = 0; out && out[i]; i++); + + /* protect the input string from strtok */ + str = ch_strdup( in ); + + /* Count words in string */ + j=1; + for ( s = str; *s; s++ ) { + if ( strchr( brkstr, *s ) != NULL ) { + j++; + } + } + + out = ch_realloc( out, ( i + j + 1 ) * sizeof( char * ) ); + new = out + i; + for ( s = ldap_pvt_strtok( str, brkstr, &lasts ); + s != NULL; + s = ldap_pvt_strtok( NULL, brkstr, &lasts ) ) + { + *new = ch_strdup( s ); + new++; + } + + *new = NULL; + free( str ); + return( out ); +} + #endif diff --git a/servers/slapd/tools/mimic.c b/servers/slapd/tools/mimic.c index a9325af5f2..bebe7f9dd5 100644 --- a/servers/slapd/tools/mimic.c +++ b/servers/slapd/tools/mimic.c @@ -263,4 +263,9 @@ void* do_syncrepl( void *ctx, void *arg ) { return NULL; } + +char** str2clist( char **out, char *in, const char *brkstr ) +{ + return NULL; +} #endif diff --git a/tests/data/slapd-syncrepl-master.conf b/tests/data/slapd-syncrepl-master.conf new file mode 100644 index 0000000000..ba3eb66ab6 --- /dev/null +++ b/tests/data/slapd-syncrepl-master.conf @@ -0,0 +1,30 @@ +# $OpenLDAP$ +# +# master slapd config -- for testing of SYNC replication +# +ucdata-path ./ucdata +include ./schema/core.schema +include ./schema/cosine.schema +include ./schema/inetorgperson.schema +include ./schema/openldap.schema +# +pidfile ./test-db/slapd.pid +argsfile ./test-db/slapd.args + +modulepath ../servers/slapd/back-@BACKEND@/ +@MODULELOAD@ + +####################################################################### +# ldbm database definitions +####################################################################### + +database @BACKEND@ +#ldbm#cachesize 0 +suffix "o=University of Michigan,c=US" +directory ./test-db +rootdn "cn=Manager,o=University of Michigan,c=US" +rootpw secret +#ldbm#index objectClass eq +#ldbm#index cn,sn,uid pres,eq,sub +#bdb#index objectClass eq +#bdb#index cn,sn,uid pres,eq,sub diff --git a/tests/data/slapd-syncrepl-slave.conf b/tests/data/slapd-syncrepl-slave.conf new file mode 100644 index 0000000000..82a4a52e99 --- /dev/null +++ b/tests/data/slapd-syncrepl-slave.conf @@ -0,0 +1,45 @@ +# $OpenLDAP$ +# +# slave slapd config -- for testing of SYNC replication +# +ucdata-path ./ucdata +include ./schema/core.schema +include ./schema/cosine.schema +include ./schema/inetorgperson.schema +include ./schema/openldap.schema +# +pidfile ./test-repl/slapd.pid +argsfile ./test-repl/slapd.args + +modulepath ../servers/slapd/back-@BACKEND@/ +@MODULELOAD@ + +####################################################################### +# ldbm database definitions +####################################################################### + +database @BACKEND@ +#ldbm#cachesize 0 +suffix "o=University of Michigan,c=US" +directory ./test-repl +rootdn "cn=Replica,o=University of Michigan,c=US" +rootpw secret +#ldbm#index objectClass eq +#ldbm#index cn,sn,uid pres,eq,sub +#bdb#index objectClass eq +#bdb#index cn,sn,uid pres,eq,sub + +# Don't change syncrepl spec yet +syncrepl id=1 + master=ldap://localhost:9009 + updatedn="cn=Replica,o=University of Michigan,c=US" + binddn="cn=Manager,o=University of Michigan,c=US" + bindmethod=simple + credentials=secret + searchbase="o=University of Michigan,c=US" + filter="objectClass=*" + attrs="*" + lastmod=req + scope=sub + type=refreshOnly + interval=10 diff --git a/tests/scripts/defines.sh b/tests/scripts/defines.sh index 4af2e03e43..22b91ea874 100755 --- a/tests/scripts/defines.sh +++ b/tests/scripts/defines.sh @@ -12,7 +12,9 @@ PWCONF=$DATADIR/slapd-pw.conf ACLCONF=$DATADIR/slapd-acl.conf RCONF=$DATADIR/slapd-referrals.conf MASTERCONF=$DATADIR/slapd-repl-master.conf +SRMASTERCONF=$DATADIR/slapd-syncrepl-master.conf SLAVECONF=$DATADIR/slapd-repl-slave.conf +SRSLAVECONF=$DATADIR/slapd-syncrepl-slave.conf REFSLAVECONF=$DATADIR/slapd-ref-slave.conf SUBMASTERCONF=$DATADIR/slapd-repl-submaster.conf SUBSLAVECONF=$DATADIR/slapd-repl-subslave.conf diff --git a/tests/scripts/test017-syncreplication b/tests/scripts/test017-syncreplication new file mode 100755 index 0000000000..cff97015d8 --- /dev/null +++ b/tests/scripts/test017-syncreplication @@ -0,0 +1,212 @@ +#! /bin/sh +# $OpenLDAP$ + +SRCDIR="." +if test $# -ge 1 ; then + SRCDIR=$1; shift +fi + +. $SRCDIR/scripts/args.sh + +echo "running defines.sh" +. $SRCDIR/scripts/defines.sh + +# +# Test replication: +# - start master +# - start slave +# - populate over ldap +# - perform some modifies and deleted +# - retrieve database over ldap and compare against expected results +# + +echo "Cleaning up in $DBDIR..." +rm -f $DBDIR/[!C]* +echo "Cleaning up in $REPLDIR..." +rm -rf $REPLDIR/[!C]* + +echo "Starting master slapd on TCP/IP port $PORT..." +. $CONFFILTER $BACKEND $MONITORDB < $SRMASTERCONF > $DBCONF +$SLAPD -f $DBCONF -h $MASTERURI -d $LVL $TIMING > $MASTERLOG 2>&1 & +PID=$! +if test $WAIT != 0 ; then + echo PID $PID + read foo +fi + +echo "Starting slave slapd on TCP/IP port $SLAVEPORT..." +. $CONFFILTER $BACKEND $MONITORDB < $SRSLAVECONF > $REPLCONF +$SLAPD -f $REPLCONF -h $SLAVEURI -d $LVL $TIMING > $SLAVELOG 2>&1 & +SLAVEPID=$! +if test $WAIT != 0 ; then + echo SLAVEPID $SLAVEPID + read foo +fi + +echo "Using ldapsearch to check that master slapd is running..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 5 seconds for slapd to start..." + sleep 5 +done + +echo "Using ldapsearch to check that slave slapd is running..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 5 seconds for slapd to start..." + sleep 5 +done + +echo "Using ldapadd to populate the master directory..." +$LDAPADD -D "$MANAGERDN" -h $LOCALHOST -p $PORT -w $PASSWD < \ + $LDIFORDERED > /dev/null 2>&1 +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed ($RC)!" + kill -HUP $PID $SLAVEPID + kill -HUP $SLURPPID + exit $RC +fi + +echo "Waiting 30 seconds for syncrepl to receive changes..." +sleep 30 + +echo "Using ldapmodify to modify master directory..." + +# +# Do some modifications +# + +$LDAPMODIFY -v -D "$MANAGERDN" -h $LOCALHOST -p $PORT -w $PASSWD > \ + $TESTOUT 2>&1 << EOMODS +dn: cn=James A Jones 1, ou=Alumni Association, ou=People, o=University of Michigan, c=US +changetype: modify +add: drink +drink: Orange Juice +- +delete: sn +sn: Jones +- +add: sn +sn: Jones + +dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +changetype: modify +replace: drink +drink: Iced Tea +drink: Mad Dog 20/20 + +dn: cn=ITD Staff,ou=Groups,o=University of Michigan,c=US +changetype: modify +delete: member +member: cn=James A Jones 2, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +member: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +- +add: member +member: cn=Dorothy Stevens, ou=Alumni Association, ou=People, o=University of Michigan, c=US +member: cn=James A Jones 1, ou=Alumni Association, ou=People, o=University of Michigan, c=US + +dn: cn=All Staff,ou=Groups,o=University of Michigan,c=US +changetype: modify +delete: description + +dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +changetype: add +objectclass: OpenLDAPperson +cn: Gern Jensen +sn: Jensen +uid: gjensen +title: Chief Investigator, ITD +postaladdress: ITD $ 535 W. William St $ Ann Arbor, MI 48103 +seealso: cn=All Staff, ou=Groups, o=University of Michigan, c=US +drink: Coffee +homepostaladdress: 844 Brown St. Apt. 4 $ Ann Arbor, MI 48104 +description: Very odd +facsimiletelephonenumber: +1 313 555 7557 +telephonenumber: +1 313 555 8343 +mail: gjensen@mailgw.example.com +homephone: +1 313 555 8844 + +dn: ou=Retired, ou=People, o=University of Michigan, c=US +changetype: add +objectclass: organizationalUnit +ou: Retired + +dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +changetype: add +objectclass: OpenLDAPperson +cn: Rosco P. Coltrane +sn: Coltrane +uid: rosco + +dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +changetype: modrdn +newrdn: cn=Rosco P. Coltrane +deleteoldrdn: 1 +newsuperior: ou=Retired, ou=People, o=University of Michigan, c=US + +dn: cn=James A Jones 2, ou=Information Technology Division, ou=People, o=University of Michigan, c=US +changetype: delete + +EOMODS + +echo "Waiting 30 seconds for syncrepl to receive changes..." +sleep 30 + +echo "Using ldapsearch to read all the entries from the master..." +$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT \ + 'objectclass=*' > $MASTEROUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at master ($RC)!" + kill -HUP $PID $SLAVEPID + kill -HUP $SLURPPID + exit $RC +fi + +echo "Using ldapsearch to read all the entries from the slave..." +$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $SLAVEPORT \ + 'objectclass=*' > $SLAVEOUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at slave ($RC)!" + kill -HUP $PID $SLAVEPID + kill -HUP $SLURPPID + exit $RC +fi + +kill -HUP $PID $SLAVEPID +kill -HUP $SLURPPID + +SEARCHOUT=$MASTEROUT +LDIF=$SLAVEOUT + +echo "Filtering ldapsearch results..." +. $LDIFFILTER < $SEARCHOUT > $SEARCHFLT +echo "Filtering original ldif used to create database..." +. $LDIFFILTER < $LDIF > $LDIFFLT + +echo "Comparing retrieved entries from master and slave..." +$CMP $SEARCHFLT $LDIFFLT > $CMPOUT + +if test $? != 0 ; then + echo "test failed - master and slave databases differ" + exit 1 +fi + +echo ">>>>> Test succeeded" + + +exit 0 -- 2.39.5