From 6cafdfa8d82134f78e68325c4b9c10dd37935d7a Mon Sep 17 00:00:00 2001 From: Nadezhda Ivanova Date: Sat, 30 Jan 2016 20:14:29 +0200 Subject: [PATCH] ITS#8303 Asynchronous meta back-end for OpenLDAP --- configure.in | 23 + doc/man/man5/slapd-asyncmeta.5 | 487 +++ servers/slapd/back-asyncmeta/Makefile.in | 50 + servers/slapd/back-asyncmeta/abandon.c | 52 + servers/slapd/back-asyncmeta/add.c | 368 ++ servers/slapd/back-asyncmeta/back-asyncmeta.h | 847 +++++ servers/slapd/back-asyncmeta/bind.c | 1978 ++++++++++ servers/slapd/back-asyncmeta/candidates.c | 289 ++ servers/slapd/back-asyncmeta/compare.c | 293 ++ servers/slapd/back-asyncmeta/config.c | 3191 +++++++++++++++++ servers/slapd/back-asyncmeta/conn.c | 1327 +++++++ servers/slapd/back-asyncmeta/delete.c | 240 ++ servers/slapd/back-asyncmeta/dncache.c | 228 ++ servers/slapd/back-asyncmeta/init.c | 509 +++ servers/slapd/back-asyncmeta/map.c | 874 +++++ servers/slapd/back-asyncmeta/message_queue.c | 545 +++ servers/slapd/back-asyncmeta/meta_result.c | 1779 +++++++++ servers/slapd/back-asyncmeta/modify.c | 369 ++ servers/slapd/back-asyncmeta/modrdn.c | 315 ++ .../slapd/back-asyncmeta/proto-asyncmeta.h | 54 + servers/slapd/back-asyncmeta/search.c | 680 ++++ servers/slapd/back-asyncmeta/suffixmassage.c | 112 + servers/slapd/back-asyncmeta/unbind.c | 55 + servers/slapd/backover.c | 32 +- tests/progs/slapd-mtread.c | 2 +- 25 files changed, 14692 insertions(+), 7 deletions(-) create mode 100644 doc/man/man5/slapd-asyncmeta.5 create mode 100644 servers/slapd/back-asyncmeta/Makefile.in create mode 100644 servers/slapd/back-asyncmeta/abandon.c create mode 100644 servers/slapd/back-asyncmeta/add.c create mode 100644 servers/slapd/back-asyncmeta/back-asyncmeta.h create mode 100644 servers/slapd/back-asyncmeta/bind.c create mode 100644 servers/slapd/back-asyncmeta/candidates.c create mode 100644 servers/slapd/back-asyncmeta/compare.c create mode 100644 servers/slapd/back-asyncmeta/config.c create mode 100644 servers/slapd/back-asyncmeta/conn.c create mode 100644 servers/slapd/back-asyncmeta/delete.c create mode 100644 servers/slapd/back-asyncmeta/dncache.c create mode 100644 servers/slapd/back-asyncmeta/init.c create mode 100644 servers/slapd/back-asyncmeta/map.c create mode 100644 servers/slapd/back-asyncmeta/message_queue.c create mode 100644 servers/slapd/back-asyncmeta/meta_result.c create mode 100644 servers/slapd/back-asyncmeta/modify.c create mode 100644 servers/slapd/back-asyncmeta/modrdn.c create mode 100644 servers/slapd/back-asyncmeta/proto-asyncmeta.h create mode 100644 servers/slapd/back-asyncmeta/search.c create mode 100644 servers/slapd/back-asyncmeta/suffixmassage.c create mode 100644 servers/slapd/back-asyncmeta/unbind.c diff --git a/configure.in b/configure.in index b97307d604..dcf25e3f83 100644 --- a/configure.in +++ b/configure.in @@ -316,6 +316,8 @@ OL_ARG_ENABLE(mdb,[ --enable-mdb enable mdb database backend], yes, [no yes mod], ol_enable_backends)dnl OL_ARG_ENABLE(meta,[ --enable-meta enable metadirectory backend], no, [no yes mod], ol_enable_backends)dnl +OL_ARG_ENABLE(asyncmeta,[ --enable-asyncmeta enable asynchronous metadirectory backend], + no, [no yes mod], ol_enable_backends)dnl OL_ARG_ENABLE(monitor,[ --enable-monitor enable monitor backend], yes, [no yes mod], ol_enable_backends)dnl OL_ARG_ENABLE(ndb,[ --enable-ndb enable MySQL NDB Cluster backend], @@ -504,6 +506,10 @@ if test $ol_enable_meta/$ol_enable_ldap = yes/no ; then AC_MSG_ERROR([--enable-meta requires --enable-ldap]) fi +if test $ol_enable_meta_async/$ol_enable_ldap = yes/no ; then + AC_MSG_ERROR([--enable-asyncmeta requires --enable-ldap]) +fi + if test $ol_enable_lmpasswd = yes ; then if test $ol_with_tls = no ; then AC_MSG_ERROR([LAN Manager passwords require OpenSSL]) @@ -543,6 +549,7 @@ BUILD_HDB=no BUILD_LDAP=no BUILD_MDB=no BUILD_META=no +BUILD_ASYNCMETA=no BUILD_MONITOR=no BUILD_NDB=no BUILD_NULL=no @@ -2736,6 +2743,20 @@ if test "$ol_enable_meta" != no ; then AC_DEFINE_UNQUOTED(SLAPD_META,$MFLAG,[define to support LDAP Metadirectory backend]) fi +if test "$ol_enable_meta_async" != no ; then + BUILD_SLAPD=yes + BUILD_ASYNCMETA=$ol_enable_meta_async + BUILD_REWRITE=yes + if test "$ol_enable_meta_async" = mod ; then + SLAPD_DYNAMIC_BACKENDS="$SLAPD_DYNAMIC_BACKENDS back-asyncmeta" + MFLAG=SLAPD_MOD_DYNAMIC + else + SLAPD_STATIC_BACKENDS="$SLAPD_STATIC_BACKENDS back-asyncmeta" + MFLAG=SLAPD_MOD_STATIC + fi + AC_DEFINE_UNQUOTED(SLAPD_ASYNCMETA,$MFLAG,[define to support LDAP Async Metadirectory backend]) +fi + if test "$ol_enable_ndb" != no ; then BUILD_SLAPD=yes BUILD_NDB=$ol_enable_ndb @@ -3142,6 +3163,7 @@ dnl backends AC_SUBST(BUILD_LDAP) AC_SUBST(BUILD_MDB) AC_SUBST(BUILD_META) + AC_SUBST(BUILD_ASYNCMETA) AC_SUBST(BUILD_MONITOR) AC_SUBST(BUILD_NDB) AC_SUBST(BUILD_NULL) @@ -3253,6 +3275,7 @@ AC_CONFIG_FILES([Makefile:build/top.mk:Makefile.in:build/dir.mk] [servers/slapd/back-ldif/Makefile:build/top.mk:servers/slapd/back-ldif/Makefile.in:build/mod.mk] [servers/slapd/back-mdb/Makefile:build/top.mk:servers/slapd/back-mdb/Makefile.in:build/mod.mk] [servers/slapd/back-meta/Makefile:build/top.mk:servers/slapd/back-meta/Makefile.in:build/mod.mk] +[servers/slapd/back-asyncmeta/Makefile:build/top.mk:servers/slapd/back-asyncmeta/Makefile.in:build/mod.mk] [servers/slapd/back-monitor/Makefile:build/top.mk:servers/slapd/back-monitor/Makefile.in:build/mod.mk] [servers/slapd/back-ndb/Makefile:build/top.mk:servers/slapd/back-ndb/Makefile.in:build/mod.mk] [servers/slapd/back-null/Makefile:build/top.mk:servers/slapd/back-null/Makefile.in:build/mod.mk] diff --git a/doc/man/man5/slapd-asyncmeta.5 b/doc/man/man5/slapd-asyncmeta.5 new file mode 100644 index 0000000000..20f986a151 --- /dev/null +++ b/doc/man/man5/slapd-asyncmeta.5 @@ -0,0 +1,487 @@ +.TH SLAPD-ASYNCMETA 5 "RELEASEDATE" "OpenLDAP LDVERSION" +.\" Copyright 2016 The OpenLDAP Foundation. +.\" Portions Copyright 2016 Symas Corporation. +.\" Copying restrictions apply. See the COPYRIGHT file. +.\" $OpenLDAP$ +.\" + +.SH NAME +slapd\-asyncmeta \- asynchronous metadirectory backend to slapd +.SH SYNOPSIS +ETCDIR/slapd.conf +.SH DESCRIPTION +The +.B asyncmeta +backend to +.BR slapd (8) +performs basic LDAP proxying with respect to a set of remote LDAP +servers, called "targets". +The information contained in these servers can be presented as +belonging to a single Directory Information Tree (DIT). + +.LP +A good knowledge of the functionality of the +.BR slapd\-meta(5) +backend is recommended. This backend has been designed as +an asynchronous version of the +.B meta +backend. Unlike +.B meta +, the operation handling threads are no longer pending +on the response from the remote server, thus decreasing the +number of threads necessary to handle the same load. While +.B asyncmeta +maintains the functionality of +.B meta +and has a largely similar codebase, +some changes in operation and some new configuration directives have been +added. Some configuration options, such as +.B conn-ttl, +.B single-conn +and +.B use-temporary-conn +have been removed, as they are no longer relevant. +.LP +.B New connection handling: +.LP + +Unlike +.B meta, +which caches bound connections, the +.B asyncmeta +works with a configured maximum number of connections per target. +For each request redirected to a target, a different connection is selected. +Each connection has a queue, to which the request is added before it is sent to the +remote server, and is removed after the last response for that request is received. + For each new request, the connection with the smallest number of pending requests +is selected, or using round\-robin if the numbers are equal. +.LP +.B Overlays: +.LP +Due to implementation specifics, there is no guarantee that any of the existing OpenLDAP overlays will work with +.B asyncmeta +backend. + +.SH EXAMPLES +Refer to +.B slapd\-meta(5) +for configuration examples. + +.SH CONFIGURATION +These +.B slapd.conf +options apply to the ASYNCMETA backend database. +That is, they must follow a "database asyncmeta" line and come before any +subsequent "backend" or "database" lines. +Other database options are described in the +.BR slapd.conf (5) +manual page. + +.SH SPECIAL CONFIGURATION DIRECTIVES +Target configuration starts with the "uri" directive. +All the configuration directives that are not specific to targets +should be defined first for clarity, including those that are common +to all backends. +They are: + +.TP +.B default\-target none +This directive forces the backend to reject all those operations +that must resolve to a single target in case none or multiple +targets are selected. +They include: add, delete, modify, modrdn; compare is not included, as +well as bind since, as they don't alter entries, in case of multiple +matches an attempt is made to perform the operation on any candidate +target, with the constraint that at most one must succeed. +This directive can also be used when processing targets to mark a +specific target as default. + +.TP +.B dncache\-ttl {DISABLED|forever|} +This directive sets the time-to-live of the DN cache. +This caches the target that holds a given DN to speed up target +selection in case multiple targets would result from an uncached +search; forever means cache never expires; disabled means no DN +caching; otherwise a valid ( > 0 ) ttl is required, in the format +illustrated for the +.B idle\-timeout +directive. + +.TP +.B onerr {CONTINUE|report|stop} +This directive allows to select the behavior in case an error is returned +by one target during a search. +The default, \fBcontinue\fP, consists in continuing the operation, +trying to return as much data as possible. +If the value is set to \fBstop\fP, the search is terminated as soon +as an error is returned by one target, and the error is immediately +propagated to the client. +If the value is set to \fBreport\fP, the search is continuated to the end +but, in case at least one target returned an error code, the first +non-success error code is returned. + +.TP +.B max\-timeout\-ops +Specify the number of consecutive timed out requests, +after which the connection will be considered faulty and dropped. + +.TP +.B max-pending-ops +The maximum number of pending requests stored in a connection's queue. +The default is 128. When this number is exceeded, +.B LDAP_BUSY +will be returned to the client. + +.TP +.B max-target-conns +The maximum number of connections per target. Unlike +.B slapd\-meta(5), +no new connections will be created +once this number is reached. The default value is 255. + +.TP +.B norefs +If +.BR yes , +do not return search reference responses. +By default, they are returned unless request is LDAPv2. +If set before any target specification, it affects all targets, unless +overridden by any per-target directive. + +.TP +.B noundeffilter +If +.BR yes , +return success instead of searching if a filter is undefined or contains +undefined portions. +By default, the search is propagated after replacing undefined portions +with +.BR (!(objectClass=*)) , +which corresponds to the empty result set. +If set before any target specification, it affects all targets, unless +overridden by any per-target directive. + +.TP +.B protocol\-version {0,2,3} +This directive indicates what protocol version must be used to contact +the remote server. +If set to 0 (the default), the proxy uses the same protocol version +used by the client, otherwise the requested protocol is used. +The proxy returns \fIunwillingToPerform\fP if an operation that is +incompatible with the requested protocol is attempted. +If set before any target specification, it affects all targets, unless +overridden by any per-target directive. + +.TP +.B pseudoroot\-bind\-defer {YES|no} +This directive, when set to +.BR yes , +causes the authentication to the remote servers with the pseudo-root +identity (the identity defined in each +.B idassert-bind +directive) to be deferred until actually needed by subsequent operations. +Otherwise, all binds as the rootdn are propagated to the targets. + +.TP +.B quarantine ,[;,[...]] +Turns on quarantine of URIs that returned +.IR LDAP_UNAVAILABLE , +so that an attempt to reconnect only occurs at given intervals instead +of any time a client requests an operation. +The pattern is: retry only after at least +.I interval +seconds elapsed since last attempt, for exactly +.I num +times; then use the next pattern. +If +.I num +for the last pattern is "\fB+\fP", it retries forever; otherwise, +no more retries occur. +This directive must appear before any target specification; +it affects all targets with the same pattern. + +.TP +.B rebind\-as\-user {NO|yes} +If this option is given, the client's bind credentials are remembered +for rebinds, when trying to re-establish a broken connection, +or when chasing a referral, if +.B chase\-referrals +is set to +.IR yes . + +.TP +.B session\-tracking\-request {NO|yes} +Adds session tracking control for all requests. +The client's IP and hostname, and the identity associated to each request, +if known, are sent to the remote server for informational purposes. +This directive is incompatible with setting \fIprotocol\-version\fP to 2. +If set before any target specification, it affects all targets, unless +overridden by any per-target directive. + +.SH TARGET SPECIFICATION +Target specification starts with a "uri" directive: + +.TP +.B uri ://[]/ [...] +Identical to +.B meta. +See +.B slapd\-meta(5) +for details. + +.TP +.B acl\-authcDN "" +DN which is used to query the target server for acl checking, +as in the LDAP backend; it is supposed to have read access +on the target server to attributes used on the proxy for acl checking. +There is no risk of giving away such values; they are only used to +check permissions. +.B The acl\-authcDN identity is by no means implicitly used by the proxy +.B when the client connects anonymously. + +.TP +.B acl\-passwd +Password used with the +.B +acl\-authcDN +above. + +.TP +.B bind\-timeout +This directive defines the timeout, in microseconds, used when polling +for response after an asynchronous bind connection. See +.B slapd\-meta(5) +for details. + +.TP +.B chase\-referrals {YES|no} +enable/disable automatic referral chasing, which is delegated to the +underlying libldap, with rebinding eventually performed if the +\fBrebind\-as\-user\fP directive is used. The default is to chase referrals. +If set before any target specification, it affects all targets, unless +overridden by any per-target directive. + +.TP +.B client\-pr {accept-unsolicited|DISABLE|} +This feature allows to use RFC 2696 Paged Results control when performing +search operations with a specific target, +irrespective of the client's request. See +.B slapd\-meta(5) +for details. + +.TP +.B default\-target [] +The "default\-target" directive can also be used during target specification. +With no arguments it marks the current target as the default. +The optional number marks target as the default one, starting +from 1. +Target must be defined. + +.TP +.B filter +This directive allows specifying a +.BR regex (5) +pattern to indicate what search filter terms are actually served by a target. + +In a search request, if the search filter matches the \fIpattern\fP +the target is considered while fulfilling the request; otherwise +the target is ignored. There may be multiple occurrences of +the +.B filter +directive for each target. + +.TP +.B idassert\-authzFrom +if defined, selects what +.I local +identities are authorized to exploit the identity assertion feature. +The string +.B +follows the rules defined for the +.I authzFrom +attribute. +See +.BR slapd.conf (5), +section related to +.BR authz\-policy , +for details on the syntax of this field. + +.HP +.hy 0 +.B idassert\-bind +.B bindmethod=none|simple|sasl [binddn=] [credentials=] +.B [saslmech=] [secprops=] [realm=] +.B [authcId=] [authzId=] +.B [authz={native|proxyauthz}] [mode=] [flags=] +.B [starttls=no|yes|critical] +.B [tls_cert=] +.B [tls_key=] +.B [tls_cacert=] +.B [tls_cacertdir=] +.B [tls_reqcert=never|allow|try|demand] +.B [tls_ciphersuite=] +.B [tls_protocol_min=[.]] +.B [tls_crlcheck=none|peer|all] +Allows to define the parameters of the authentication method that is +internally used by the proxy to authorize connections that are +authenticated by other databases. See +.B slapd\-meta(5) +for details. + +.TP +.B idle\-timeout