1 /* centipede.c - generate and install indexing information (view w/tabstop=4) */
9 #include <ldapconfig.h>
12 #define DEFAULT_LDAPFILTER "(objectclass=*)"
14 #define CENTROID_VALUE 1
15 #define CENTROID_WORD 2
17 #define CENTROID_RELATIVE 1
18 #define CENTROID_FULL 2
20 #define WORD_BREAKS " -',.()!;:&$%*\"/\\+_<>=?[]|^~"
33 int srcldapauthmethod;
34 int destldapauthmethod;
38 static LDAP *start_ldap_search();
39 static LDAP *bind_to_destination_ldap();
40 static int create_tmp_files();
41 static int generate_new_centroids();
42 static LDAPMod **diff_centroids();
43 static LDAPMod **full_centroid();
44 static char **charray_add_dup();
46 static void usage( char *name )
48 fprintf( stderr, "usage: %s [options] -s url -d url attributes\n", name );
49 fprintf( stderr, "where:\n" );
50 fprintf( stderr, "\t-s url\t\t[[ldap://][host[:port]]/]searchbasedn\n");
51 fprintf( stderr, "\t-d url\t\t[[ldap://][host[:port]]/]centroidentrydn\n");
52 fprintf( stderr, "options:\n" );
53 fprintf( stderr, "\t-v \t\tturn on verbose mode\n" );
54 fprintf( stderr, "\t-n \t\tgenerate, but do not install index info\n" );
55 fprintf( stderr, "\t-f filter\tentry selection filter\n" );
56 fprintf( stderr, "\t-F \t\tgenerate a full centroid\n" );
57 fprintf( stderr, "\t-R \t\tgenerate a relative centroid\n" );
58 fprintf( stderr, "\t-w \t\tgenerate a word-based centroid\n" );
59 fprintf( stderr, "\t-t directory\tcentroid directory\n" );
60 fprintf( stderr, "\t-b binddn\tsource bind dn\n" );
61 fprintf( stderr, "\t-p passwd\tsource bind passwd (for simple auth)\n" );
62 fprintf( stderr, "\t-m authmethod\tsource authmethod \"simple\" or \"kerberos\"\n" );
63 fprintf( stderr, "\t-B binddn\tdestination bind dn\n" );
64 fprintf( stderr, "\t-P passwd\tdestination bind passwd (for simple auth)\n" );
65 fprintf( stderr, "\t-M authmethod\tdestination authmethod \"simple\" or \"kerberos\"\n" );
66 fprintf( stderr, "\t-c size\t\tldbm cache size\n" );
69 main( int argc, char **argv )
71 char *ldapfilter, *ldapref;
72 char *ldapsrcurl, *ldapdesturl;
88 ldapbase = DEFAULT_BASE;
89 srcldapauthmethod = LDAP_AUTH_SIMPLE;
90 destldapauthmethod = LDAP_AUTH_SIMPLE;
93 destldapbinddn = NULL;
94 destldappasswd = NULL;
95 ldapfilter = DEFAULT_LDAPFILTER;
96 centroidvalues = CENTROID_VALUE;
97 centroidtype = CENTROID_RELATIVE;
102 while ( (i = getopt( argc, argv, "s:d:c:b:B:f:FRWp:P:m:M:t:vwn" ))
105 case 's': /* source url [[ldap://][host[:port]]/]basedn */
106 ldapsrcurl = strdup( optarg );
109 case 'd': /* destination url [[ldap://][host[:port]]/]entrydn */
110 ldapdesturl = strdup( optarg );
113 case 'f': /* specify a filter */
114 ldapfilter = strdup( optarg );
117 case 'F': /* generate full centroid */
118 centroidtype = CENTROID_FULL;
121 case 'R': /* generate relative centroid */
122 centroidtype = CENTROID_RELATIVE;
125 case 'w': /* generate word centroid */
126 centroidvalues = CENTROID_WORD;
129 case 'W': /* generate weights */
133 case 't': /* temp file directory */
134 centdir = strdup( optarg );
137 case 'b': /* src bind dn */
138 srcldapbinddn = strdup( optarg );
141 case 'p': /* src bind password */
142 srcldappasswd = strdup( optarg );
147 case 'B': /* dest bind dn */
148 destldapbinddn = strdup( optarg );
151 case 'P': /* dest bind password */
152 destldappasswd = strdup( optarg );
157 case 'm': /* src bind method */
158 if ( strcasecmp( optarg, "simple" ) == 0 ) {
159 srcldapauthmethod = LDAP_AUTH_SIMPLE;
160 } else if ( strcasecmp( optarg, "kerberos" ) == 0 ) {
161 srcldapauthmethod = LDAP_AUTH_KRBV4;
163 fprintf( stderr, "%s: unknown auth method\n", optarg );
164 fprintf( stderr, "expecting \"simple\" or \"kerberos\"\n",
170 case 'M': /* dest bind method */
171 if ( strcasecmp( optarg, "simple" ) == 0 ) {
172 destldapauthmethod = LDAP_AUTH_SIMPLE;
173 } else if ( strcasecmp( optarg, "kerberos" ) == 0 ) {
174 destldapauthmethod = LDAP_AUTH_KRBV4;
176 fprintf( stderr, "%s: unknown auth method\n", optarg );
177 fprintf( stderr, "expecting \"simple\" or \"kerberos\"\n",
183 case 'c': /* ldbm cache size */
184 ldbmcachesize = atoi( optarg );
187 case 'v': /* turn on verbose mode */
191 case 'n': /* don't actually install index info */
200 if ( optind == argc || ldapsrcurl == NULL || ldapdesturl == NULL ) {
204 attrs = &argv[optind];
207 * open the ldap connection and start searching for the entries
208 * we will use to generate the centroids.
211 if ( (ld = start_ldap_search( ldapsrcurl, ldapfilter, attrs )) == NULL ) {
212 fprintf( stderr, "could not initiate ldap search\n" );
216 if ( create_tmp_files( attrs, &tmpfile, &ldbm ) != 0 ) {
217 fprintf( stderr, "could not create temp files\n" );
222 * go through the entries returned, building a centroid for each
223 * attribute as we go.
226 if ( (count = generate_new_centroids( ld, attrs, ldbm )) < 1 ) {
228 fprintf( stderr, "no entries matched\n" );
231 fprintf( stderr, "could not generate new centroid\n" );
237 * for each centroid we generated above, compare to the existing
238 * centroid, if any, and produce adds and deletes, or produce
239 * an entirely new centroid. in either case, update the "current"
240 * centroid version with the new one we just generated.
243 if ( (ld = bind_to_destination_ldap( ldapsrcurl, ldapdesturl )) == NULL ) {
245 "could not bind to index server, or could not create index entry\n" );
249 for ( i = 0; ldbm[i] != NULL; i++ ) {
250 /* generate the name of the existing centroid, if any */
251 s = strrchr( tmpfile[i], '/' );
253 sprintf( buf, "%s/cent.%s", tmpfile[i], attrs[i] );
256 /* generate the full centroid changes */
257 if ( centroidtype == CENTROID_FULL || (oldbm = ldbm_open( buf,
258 LDBM_WRITER, 0, ldbmcachesize )) == NULL ) {
259 if ( (mods = full_centroid( attrs[i], ldbm[i], count )) == NULL ) {
260 fprintf( stderr, "could not produce full centroid for %s\n",
265 /* generate the differential centroid changes */
267 if ( (mods = diff_centroids( attrs[i], oldbm, ldbm[i], count ))
269 fprintf( stderr, "could not diff centroids\n" );
277 printf("changes:\n");
278 for ( j = 0; mods[j] != NULL; j++ ) {
279 switch( mods[j]->mod_op ) {
281 printf( "\tadd: %s\n",mods[j]->mod_type );
283 case LDAP_MOD_DELETE:
284 printf( "\tdelete: %s\n",mods[j]->mod_type );
286 case LDAP_MOD_REPLACE:
287 printf( "\treplace: %s\n",mods[j]->mod_type );
290 if ( mods[j]->mod_values != NULL ) {
291 for ( k = 0; mods[j]->mod_values[k] != NULL; k++ ) {
292 printf( "\t\t%s\n", mods[j]->mod_values[k] );
296 printf("end changes:\n");
300 printf( "%sModifying centroid...", not ? "Not " : "" );
304 /* attempt to make the changes to the index server entry */
305 if ( !not && ldap_modify_s( ld, ldapbase, mods ) != LDAP_SUCCESS ) {
306 fprintf( stderr, "could not apply centroid modification for %s\n",
308 ldap_perror( ld, ldapbase );
310 ldap_mods_free( mods, 1 );
317 /* move the new centroid into the old one's place */
319 (void) unlink( buf );
320 if ( link( tmpfile[i], buf ) != 0 ) {
322 fprintf( stderr, "could not rename %s to %s\n", buf,
327 (void) unlink( tmpfile[i] );
331 for ( i = 0; attrs[i] != NULL; i++ ) {
332 ldbm_close( ldbm[i] );
342 * open an ldap connection, bind, and initiate the search
356 if ( strncmp( ldapsrcurl, "ldap://", 7 ) == 0 ) {
359 if ( (s2 = strchr( s, '/' )) == NULL ) {
360 ldapbase = strdup( s );
364 ldaphost = strdup( s );
367 ldapbase = strdup( s2 + 1 );
371 printf( "Base: %s\n", ldapbase );
372 printf( "Attributes:" );
373 for ( i = 0; attrs[i] != NULL; i++ ) {
374 printf( " %s", attrs[i] );
377 printf( "Binding to source LDAP server..." );
381 if ( (ld = ldap_open( ldaphost, LDAP_PORT )) == NULL ) {
382 perror( "ldap_open" );
386 if ( ldap_bind_s( ld, srcldapbinddn, srcldappasswd, srcldapauthmethod )
388 ldap_perror( ld, "ldap_bind_s" );
393 printf( "\nInitiating search..." );
394 if ( ldap_search( ld, ldapbase, LDAP_SCOPE_SUBTREE, ldapfilter, attrs, 0 )
396 ldap_perror( ld, "ldap_search" );
409 * create the temporary ldbm files we will use to hold the new centroids
421 for ( i = 0; attrs[i] != NULL; i++ )
425 if ( (*tmpfile = (char **) malloc( i * sizeof(char *) )) == NULL ) {
429 if ( (*ldbm = (LDBM *) malloc( i * sizeof(LDBM) )) == NULL ) {
433 for ( i = 0; attrs[i] != NULL; i++ ) {
434 if ( ((*tmpfile)[i] = tempnam( centdir, NULL )) == NULL ) {
439 if ( ((*ldbm)[i] = ldbm_open( (*tmpfile)[i], LDBM_WRCREAT, 0600,
440 ldbmcachesize )) == NULL ) {
441 fprintf( stderr, "ldbm_open of \"%s\" failed\n", (*tmpfile)[i] );
442 perror( "ldbm_open" );
446 (*tmpfile)[i] = NULL;
453 * step through each entry returned from the search and generate
454 * the appropriate centroid values.
458 generate_new_centroids(
466 LDAPMessage *res, *e;
472 printf( "Generating new centroids for..." );
479 while ( (rc = ldap_result( ld, LDAP_RES_ANY, 0, NULL, &res ))
480 == LDAP_RES_SEARCH_ENTRY ) {
482 e = ldap_first_entry( ld, res );
483 dn = ldap_get_dn( ld, e );
485 /* for each attr we want to generate a centroid for */
486 for ( i = 0; attrs[i] != NULL; i++ ) {
487 if ( (val = ldap_get_values( ld, e, attrs[i] )) == NULL ) {
492 for ( j = 0; val[j] != NULL; j++ ) {
493 /* normalize the value */
494 for ( s = val[j]; *s; s++ ) {
495 if ( isascii( *s ) ) {
500 if ( isascii( last ) && isdigit( last ) ) {
504 /* generate a value-based centroid */
505 if ( centroidvalues == CENTROID_VALUE ) {
507 key.dsize = strlen( key.dptr ) + 1;
508 (void) ldbm_store( ldbm[i], key, data, LDBM_INSERT );
510 /* generate a word-based centroid */
512 for ( w = strtok( val[j], WORD_BREAKS ); w != NULL;
513 w = strtok( NULL, WORD_BREAKS ) ) {
515 key.dsize = strlen( key.dptr ) + 1;
516 (void) ldbm_store( ldbm[i], key, data, LDBM_INSERT );
520 ldap_value_free( val );
529 printf( "%d entries\n", count );
536 * compare the old and new centroids, generating the appropriate add
537 * and delete operations. if the underlying database is ordered, we
538 * can do this more efficiently.
555 char **avals, **dvals;
556 int amax, acur, dmax, dcur;
560 printf( "Generating mods for differential %s centroid...", attr );
564 if ( (mods = (LDAPMod **) malloc( sizeof(LDAPMod *) * 4 )) == NULL ||
565 (mods[0] = (LDAPMod *) malloc( sizeof(LDAPMod) )) == NULL ||
566 (mods[1] = (LDAPMod *) malloc( sizeof(LDAPMod) )) == NULL ||
567 (mods[2] = (LDAPMod *) malloc( sizeof(LDAPMod) )) == NULL ||
568 (vals = (char **) malloc( 2 * sizeof(char *) )) == NULL ||
569 (vals[0] = (char *) malloc( 20 )) == NULL )
574 /* add values in mods[0] */
575 mods[0]->mod_op = LDAP_MOD_ADD;
576 mods[0]->mod_type = attr;
577 mods[0]->mod_values = NULL;
580 /* delete values in mods[1] */
581 mods[1]->mod_op = LDAP_MOD_DELETE;
582 mods[1]->mod_type = attr;
583 mods[1]->mod_values = NULL;
586 /* number of entries in mods[2] */
587 sprintf( vals[0], "%d", nentries );
589 mods[2]->mod_op = LDAP_MOD_REPLACE;
590 mods[2]->mod_type = "nentries";
591 mods[2]->mod_values = vals;
592 /* null terminate list of mods */
597 * if the underlying database is ordered, we can do a more efficient
598 * dual traversal, yielding O(N) performance.
603 for ( okey = ldbm_firstkey( oldbm ), nkey = ldbm_firstkey( nldbm );
604 okey.dptr != NULL && nkey.dptr != NULL; )
606 rc = strcmp( okey.dptr, nkey.dptr );
609 /* value is in both places - leave it */
610 if ( olast.dptr != NULL ) {
611 ldbm_datum_free( oldbm, olast );
614 if ( nlast.dptr != NULL ) {
615 ldbm_datum_free( nldbm, nlast );
619 okey = ldbm_nextkey( oldbm, olast );
620 nkey = ldbm_nextkey( nldbm, nlast );
621 } else if ( rc > 0 ) {
622 /* new value is not in old centroid - add it */
623 if ( charray_add_dup( &avals, &acur, &amax, nkey.dptr ) == NULL ) {
624 ldap_mods_free( mods, 1 );
628 if ( nlast.dptr != NULL ) {
629 ldbm_datum_free( nldbm, nlast );
632 nkey = ldbm_nextkey( nldbm, nlast );
634 /* old value is not in new centroid - delete it */
635 if ( charray_add_dup( &dvals, &dcur, &dmax, okey.dptr ) == NULL ) {
636 ldap_mods_free( mods, 1 );
640 if ( olast.dptr != NULL ) {
641 ldbm_datum_free( oldbm, olast );
644 okey = ldbm_nextkey( oldbm, olast );
648 while ( okey.dptr != NULL ) {
649 if ( charray_add_dup( &dvals, &dcur, &dmax, okey.dptr ) == NULL ) {
650 ldap_mods_free( mods, 1 );
654 okey = ldbm_nextkey( oldbm, olast );
655 if ( olast.dptr != NULL ) {
656 ldbm_datum_free( oldbm, olast );
660 if ( olast.dptr != NULL ) {
661 ldbm_datum_free( oldbm, olast );
663 while ( nkey.dptr != NULL ) {
664 if ( charray_add_dup( &avals, &acur, &amax, nkey.dptr ) == NULL ) {
665 ldap_mods_free( mods, 1 );
669 nkey = ldbm_nextkey( nldbm, nlast );
670 if ( nlast.dptr != NULL ) {
671 ldbm_datum_free( nldbm, nlast );
675 if ( nlast.dptr != NULL ) {
676 ldbm_datum_free( nldbm, nlast );
680 * if the underlying database is not ordered, we have to
681 * generate list of values to add by stepping through all new
682 * values and looking them up in the old centroid (not there => add),
683 * then stepping through all old values and looking them up in the
684 * new centroid (not there => delete). this yields O(Nf(N)) performance,
685 * where f(N) is the order to retrieve a single item.
688 /* generate list of values to add */
690 for ( key = ldbm_firstkey( nldbm ); key.dptr != NULL;
691 key = ldbm_nextkey( nldbm, lastkey ) ) {
692 /* see if it's in the old one */
693 data = ldbm_fetch( oldbm, key );
695 /* not there - add it */
696 if ( data.dptr == NULL ) {
697 if ( charray_add_dup( &avals, &acur, &amax, key.dptr ) == NULL ) {
698 ldap_mods_free( mods, 1 );
702 ldbm_datum_free( oldbm, data );
704 if ( lastkey.dptr != NULL ) {
705 ldbm_datum_free( nldbm, lastkey );
709 if ( lastkey.dptr != NULL ) {
710 ldbm_datum_free( nldbm, lastkey );
713 /* generate list of values to delete */
715 for ( key = ldbm_firstkey( oldbm ); key.dptr != NULL;
716 key = ldbm_nextkey( oldbm, lastkey ) ) {
717 /* see if it's in the new one */
718 data = ldbm_fetch( nldbm, key );
720 /* not there - delete it */
721 if ( data.dptr == NULL ) {
722 if ( charray_add_dup( &dvals, &dcur, &dmax, key.dptr ) == NULL ) {
723 ldap_mods_free( mods, 1 );
727 ldbm_datum_free( nldbm, data );
729 if ( lastkey.dptr != NULL ) {
730 ldbm_datum_free( oldbm, lastkey );
734 if ( lastkey.dptr != NULL ) {
735 ldbm_datum_free( oldbm, lastkey );
739 mods[0]->mod_values = avals;
740 mods[1]->mod_values = dvals;
747 if ( mods[1]->mod_values == NULL ) {
748 free( (char *) mods[1] );
751 if ( mods[0]->mod_values == NULL ) {
752 free( (char *) mods[0] );
756 if ( mods[0] == NULL ) {
757 free( (char *) mods );
777 printf( "Generating mods for full %s centroid...", attr );
781 if ( (mods = (LDAPMod **) malloc( sizeof(LDAPMod *) * 3 )) == NULL ||
782 (mods[0] = (LDAPMod *) malloc( sizeof(LDAPMod) )) == NULL ||
783 (mods[1] = (LDAPMod *) malloc( sizeof(LDAPMod) )) == NULL ||
784 (vals = (char **) malloc( 2 * sizeof(char *) )) == NULL ||
785 (vals[0] = (char *) malloc( 20 )) == NULL )
790 mods[0]->mod_op = LDAP_MOD_REPLACE;
791 mods[0]->mod_type = attr;
792 mods[0]->mod_values = NULL;
793 sprintf( vals[0], "%d", nentries );
795 mods[1]->mod_op = LDAP_MOD_REPLACE;
796 mods[1]->mod_type = "nentries";
797 mods[1]->mod_values = vals;
803 for ( key = ldbm_firstkey( ldbm ); key.dptr != NULL;
804 key = ldbm_nextkey( ldbm, lastkey ) ) {
805 if ( charray_add_dup( &vals, &vcur, &vmax, key.dptr ) == NULL ) {
806 ldap_mods_free( mods, 1 );
810 if ( lastkey.dptr != NULL ) {
811 ldbm_datum_free( ldbm, lastkey );
815 if ( lastkey.dptr != NULL ) {
816 ldbm_datum_free( ldbm, lastkey );
818 mods[0]->mod_values = vals;
825 if ( mods[0]->mod_values == NULL ) {
826 free( (char *) mods[0] );
827 free( (char *) mods );
835 * extract the destination ldap host, port, and base object for the
836 * server to receive the index information. then, open a connection,
837 * bind, and see if the entry exists. if not, create it and set things
838 * up so the centroid full and diff routines can modify it to contain
839 * the new centroid information.
843 bind_to_destination_ldap(
852 char *attrs[2], *refvalues[2], *ocvalues[2];
858 printf( "Binding to destination LDAP server..." );
862 /* first, pick out the destination ldap server info */
863 if ( ldapbase != NULL ) {
866 if ( strncmp( ldapdesturl, "ldap://", 7 ) == 0 ) {
869 if ( (s2 = strchr( s, '/' )) == NULL ) {
870 ldapbase = strdup( s );
874 if ( ldaphost != NULL )
876 ldaphost = strdup( s );
879 ldapbase = strdup( s2 + 1 );
881 strcpy( buf, "ref=" );
882 if ( strpbrk( ldapsrcurl, " ,;" ) != NULL ) {
885 for ( s = d = ldapsrcurl; *s; s++ ) {
891 strcat( buf, ldapsrcurl );
892 if ( strpbrk( ldapsrcurl, " ,;" ) != NULL ) {
896 strcat( buf, ldapbase );
898 ldapbase = strdup( buf );
900 if ( (ld = ldap_open( ldaphost, LDAP_PORT )) == NULL ) {
901 perror( "ldap_open" );
905 if ( ldap_bind_s( ld, destldapbinddn, destldappasswd, destldapauthmethod )
907 ldap_perror( ld, "ldap_bind_s" );
917 rc = ldap_search_s( ld, ldapbase, LDAP_SCOPE_BASE, "(objectclass=*)",
921 if ( rc == LDAP_NO_SUCH_OBJECT ) {
923 printf( "%sCreating centroid entry...", not ? "Not " : "" );
927 /* create the centroid index entry */
929 m[0].mod_type = "ref";
930 refvalues[0] = ldapsrcurl;
932 m[0].mod_values = refvalues;
934 m[1].mod_type = "objectclass";
935 ocvalues[0] = "indexentry";
937 m[1].mod_values = ocvalues;
942 if ( !not && ldap_add_s( ld, ldapbase, mp ) != LDAP_SUCCESS ) {
943 ldap_perror( ld, ldapbase );
952 } else if ( rc != LDAP_SUCCESS ) {
953 ldap_perror( ld, "ldap_search_s" );
972 *a = (char **) malloc( (BUFSIZ + 1) * sizeof(char *) );
975 } else if ( *cur >= *max ) {
977 *a = (char **) realloc( *a, (*max + 1) * sizeof(char *) );
983 (*a)[(*cur)++] = strdup( s );