From: Gavin Henry Date: Thu, 7 Feb 2008 22:35:07 +0000 (+0000) Subject: Update to tuning section using todays e-mail thread (note: performant isn't a word... X-Git-Tag: OPENLDAP_REL_ENG_2_4_9~20^2~194 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=8f1d222edb029dbfbc4cf03d556f937449a054e9;p=openldap Update to tuning section using todays e-mail thread (note: performant isn't a word Quanah ;-) ). --- diff --git a/doc/guide/admin/aspell.en.pws b/doc/guide/admin/aspell.en.pws index 3c201dd2a8..b8829abda7 100644 --- a/doc/guide/admin/aspell.en.pws +++ b/doc/guide/admin/aspell.en.pws @@ -1,4 +1,4 @@ -personal_ws-1.1 en 1491 +personal_ws-1.1 en 1492 nattrsets inappropriateAuthentication api @@ -8,8 +8,8 @@ reqEnd olcOverlayConfig shoesize olcTLSCACertificateFile -CGI cdx +CGI DCE DAP attributename @@ -20,8 +20,8 @@ kurt authzID authzid authzId -DAs ddd +DAs userApplications BNF attrs @@ -32,8 +32,8 @@ ldapport hallvard ASN acknowledgements -Chu ava +Chu monitorCounter del DDR @@ -84,13 +84,13 @@ olcModulePath maxentries authc seeAlso -searchbase searchBase +searchbase realnamingcontext -dn's -DNs -DN's dns +DN's +DNs +dn's dereference sortKey authzTo @@ -159,8 +159,8 @@ compareDN sizelimit unixODBC notAllowedOnNonLeaf -APIs blen +APIs attrsOnly attrsonly slappasswd @@ -193,13 +193,13 @@ args caseExactOrderingMatch olcDbQuarantine RELEASEDATE -baseDN basedn +baseDN argv gss schemachecking -whoami WhoAmI +whoami syslogd dataflow subentries @@ -244,8 +244,8 @@ ldd localstatedir sockbuf PENs -ipv IPv +ipv ghenry hyc multimaster @@ -280,8 +280,8 @@ intermediateResponse myOID structuralObjectClass integerMatch -openldap OpenLDAP +openldap moddn rewriteEngine AVAs @@ -300,8 +300,8 @@ bool logins jts memberAttr -newpasswdfile newPasswdFile +newpasswdfile ucdata LLL confdir @@ -324,16 +324,16 @@ modme refreshOnly PIII pwdPolicySubentry -supportedSASLmechanism supportedSASLMechanism +supportedSASLmechanism FIXME realanonymous caseExactMatch olcSizeLimit Bourne attr -objectidentifier objectIdentifier +objectidentifier refint msgtype OBJEXT @@ -384,8 +384,8 @@ Autoconf alloc PDU OLF -inetorgperson inetOrgPerson +inetorgperson deleteoldrdn monitorCounterObject pid @@ -445,9 +445,9 @@ OTP entrylimit attrdescN logold -pos -sbi PRD +sbi +pos reqEntries pre bvals @@ -473,8 +473,8 @@ telephonenumber telephoneNumber DLDAP peernamestyle -Sep SHA +Sep filename rpath argsfile @@ -504,8 +504,8 @@ olcDbIDLcacheSize ostring toolsets mwrscdx -SMD UCD +SMD cancelled crit organizationalUnit @@ -517,8 +517,8 @@ TGT modulepath quickstart mySNMP -tgz UDP +tgz RDBMs rdbms Matic @@ -538,9 +538,9 @@ olcDbConfig refreshDone ssf replogfile -rwm -TOC vec +TOC +rwm LDAPDN compareAttrDN endmacro @@ -548,15 +548,15 @@ tls repl monitoringslapd referralsp -tmp SRP +tmp olcDbNosync conns SSL PDkzODdASFxOQ SRV -rwx sss +rwx deallocators Contribware URLlist @@ -675,17 +675,18 @@ groupstyle ldapsearch cp displayName -eg bv +eg olcBackendConfig -dn fd +dn LDAPSync olcReplicationInterval fG gidNumber fi Instanstantiation +du eq FIPS dx @@ -812,8 +813,8 @@ ZZ entryCSNs dlopen continuated -newsuperior newSuperior +newsuperior Preprocessor XXLIBS deallocate @@ -865,8 +866,8 @@ pwdSafeModify contrib FQDNs bjorn -myldap myLDAP +myldap peercred SNMP myObjectClass @@ -886,8 +887,8 @@ ldapmodrdn ldapbis attributeoptions serverID -memberOf memberof +memberOf pseudorootpw allmail CFLAGS @@ -906,8 +907,8 @@ modifyAttrDN dcedn olcOverlay exop -berelement BerElement +berelement olcRootDN octetString SampleLDAP @@ -917,8 +918,8 @@ PostgreSQL bvstr filesystem pathtest -objectClass objectclass +objectClass submatches newrdn armijo @@ -933,8 +934,8 @@ jane syncuser Masarati LDAPSyntax -oldpasswdfile oldPasswdFile +oldpasswdfile reqDN SSFs ietf @@ -958,8 +959,8 @@ reqId setspec scanf TLSv -distinguishedname distinguishedName +distinguishedname BerVarray caseIgnoreSubstrin ldapwhoami @@ -987,8 +988,8 @@ slaptest zeilenga WebUpdate numericoid -changelog ChangeLog +changelog creatorsName ascii wahl @@ -1008,8 +1009,8 @@ simplebinddn authcDN TLSCipherSuite supportedSASLMechanisms -rootdse rootDSE +rootdse dsaparam cachefree UMich's @@ -1018,10 +1019,10 @@ schemadir attribute's extern varchar -olcDbCacheSize olcDbCachesize -authcid +olcDbCacheSize authcID +authcid POSIX hnPk ldapext @@ -1042,8 +1043,8 @@ sasldb somevalue LIBRELEASE randkey -starttls StartTLS +starttls LDAPSchemaExtensionItem reqReferral shtool @@ -1055,8 +1056,8 @@ subjectAltName errObject gsskrb valsort -bervals berval's +bervals derefFindingBaseObj checkpointed keytab @@ -1079,8 +1080,8 @@ README memcalloc inet saslargs -givenname givenName +givenname olcDbMode pidfile olcLimits @@ -1089,8 +1090,8 @@ tuple superset directoryString ktadd -proxyTemplate proxytemplate +proxyTemplate wildcards monitoredObject TTLs @@ -1104,8 +1105,8 @@ reqResult impl strongerAuthRequired outvalue -returnCode returncode +returnCode attributeDescription attrval dnssrv @@ -1129,20 +1130,20 @@ uncached ldapapiinfo groupOfUniqueNames dhparam -slapd's slapds +slapd's inputfile RDBMSes wildcard Locator -errAbsObject errABsObject +errAbsObject SASL's html searchResultDone olcBdbConfig -ldapmod LDAPMod +ldapmod olcHidden userPassword TLSRandFile @@ -1170,10 +1171,10 @@ cacertdir queryid Warper XDEFS -urls URL's -postalAddress +urls postaladdress +postalAddress passwd plugins george @@ -1189,16 +1190,16 @@ LDAPModifying slapdconfig sysconfig dnSubtreeMatch -olcSaslSecProps olcSaslSecprops +olcSaslSecProps auditModify groupOfNames jensen reloadHint prepending olcGlobal -matchingRule matchingrule +matchingRule SmVuc MSSQL nisMailAlias @@ -1213,9 +1214,9 @@ whsp realusers dnstyle suffixalias -proxyAttrset -proxyAttrSet proxyattrset +proxyAttrSet +proxyAttrset pwdMustChange ldif bvfree @@ -1229,8 +1230,8 @@ chown PRNGD LDAPRDN entryUUIDs -proxycache proxyCache +proxycache SERATGCgaGBYWGDEjJR noanonymous accessee @@ -1283,8 +1284,8 @@ passwdfile errMatchedDN everytime mkdep -olcDbindex olcDbIndex +olcDbindex syntaxOID reqData databasetype @@ -1336,8 +1337,8 @@ ACLs berptr olcModuleLoad namingViolation -attributetype attributeType +attributetype auditModRDN cacert memberUid @@ -1389,24 +1390,24 @@ preallocated syntaxes memberURL monitorRuntimeConfig -bindDn -bindDN binddn +bindDN +bindDn methodp -timeLimitExceeded timelimitExceeded +timeLimitExceeded pwdInHistory LTSTATIC -requestors requestor's +requestors LDAPCONF saslauthd MKDEPFLAG gecos entryUUID -gnutls -GNUtls GnuTLS +GNUtls +gnutls postread timeval DHAVE @@ -1429,8 +1430,8 @@ entryTtl LDAPControl pwdMinLength ldapcompare -readonly readOnly +readonly RANDFILE attrlist aci @@ -1456,8 +1457,8 @@ Kumar AES bdb attributeOrValueExists -manageDSAit ManageDsaIT +manageDSAit bindpw monitorContainer pEntry @@ -1469,8 +1470,8 @@ Blowfish mkln numericStringSubstringsMatch testgroup -openssl OpenSSL +openssl ModName cacheable freeit @@ -1479,8 +1480,8 @@ ber ali mandir changetype -CAs CA's +CAs typeA bvecfree ODBC diff --git a/doc/guide/admin/tuning.sdf b/doc/guide/admin/tuning.sdf index 28baa9624f..39c5a8f44c 100644 --- a/doc/guide/admin/tuning.sdf +++ b/doc/guide/admin/tuning.sdf @@ -27,14 +27,19 @@ H3: Memory Scale your cache to use available memory and increase system memory if you can. -More info here. +See {{SECT:Caching}} H3: Disks -Use fast subsystems. Put each database and logs on separate disks. +Use fast subsystems. Put each database and logs on separate disks configurable +via {{DB_CONFIG}}: -Example showing config settings +> # Data Directory +> set_data_dir /data/db +> +> # Transaction Log settings +> set_lg_dir /logs H3: Network Topology @@ -119,7 +124,7 @@ H3: What to watch out for The most common message you'll see that you should pay attention to is: -> "<= bdb_equality_candidates: (foo) index_param failed (18)" +> "<= bdb_equality_candidates: (foo) index_param failed (18)" That means that some application tried to use an equality filter ({{foo=}}) and attribute {{foo}} does not have an equality index. If you see a lot of these @@ -141,17 +146,17 @@ to sync the file system with every write ({{man syslogd/syslog.conf}}). In Linux you can prepend the log file name with a "-" in {{syslog.conf}}. For example, if you are using the default LOCAL4 logging you could try: -> # LDAP logs -> LOCAL4.* -/var/log/ldap +> # LDAP logs +> LOCAL4.* -/var/log/ldap For syslog-ng, add or modify the following line in {{syslog-ng.conf}}: -> options { sync(n); }; +> options { sync(n); }; where n is the number of lines which will be buffered before a write. -H2: BDB/HDB Database Caching +H2: Caching We all know what caching is, don't we? @@ -164,7 +169,24 @@ entry cache and {{TERM:IDL}} (IDL) cache. H3: Berkeley DB Cache -BerkeleyDB's own data cache operates on page-sized blocks of raw data. +There are two ways to tune for the BDB cachesize: + +(a) BDB cache size necessary to load the database via slapadd in optimal time + +(b) BDB cache size necessary to have a high performing running slapd once the data is loaded + +For (a), the optimal cachesize is the size of the entire database. If you +already have the database loaded, this is simply a + +> du -c -h *.bdb + +in the directory containing the OpenLDAP ({{/usr/local/var/openldap-data}}) data. + +For (b), the optimal cachesize is just the size of the {{id2entry.bdb}} file, +plus about 10% for growth. + +The tuning of {{DB_CONFIG}} should be done for each BDB type database +instantiated (back-bdb, back-hdb). Note that while the {{TERM:BDB}} cache is just raw chunks of memory and configured as a memory size, the {{slapd}}(8) entry cache holds parsed entries, @@ -186,7 +208,7 @@ that's large enough for your "working set." That means, large enough to hold all of the most frequently accessed data, plus a few less-frequently accessed items. -ORACLE LINKS HERE +For more information, please see: {{URL:http://www.oracle.com/technology/documentation/berkeley-db/db/ref/am_conf/cachesize.html}} H4: Calculating Cachesize @@ -206,7 +228,7 @@ along the path from the root of the tree down to the particular data item you're accessing. That's enough cache for a single search. For the general case, you want enough cache to contain all the internal nodes in the database. -> db_stat -d +> db_stat -d will tell you how many internal pages are present in a database. You should check this number for both dn2id and id2entry. @@ -224,7 +246,7 @@ and an {{id2entry}} that's 800MB. db_stat tells me that {{dn2id}} uses 4KB pages internal pages, and 45912 leaf pages. In order to efficiently retrieve any single entry in this database, the cache should be at least -> (433+1) * 4KB + (52+1) * 16KB in size: 1736KB + 848KB =~ 2.5MB. +> (433+1) * 4KB + (52+1) * 16KB in size: 1736KB + 848KB =~ 2.5MB. This doesn't take into account other library overhead, so this is even lower than the barest minimum. The default cache size, when nothing is configured, @@ -263,8 +285,9 @@ id2entry data, so 4MB is good enough. With back-bdb and back-hdb you can use "db_stat -m" to check how well the database cache is performing. +For more information on {{db_stat}}: {{URL:http://www.oracle.com/technology/documentation/berkeley-db/db/utility/db_stat.html}} -H3: {{slapd}}(8) Entry Cache +H3: {{slapd}}(8) Entry Cache (cachesize) The {{slapd}}(8) entry cache operates on decoded entries. The rationale - entries in the entry cache can be used directly, giving the fastest response. If an entry @@ -275,6 +298,10 @@ If the entry is in neither cache then BDB will have to flush some of its current cached pages and bring in the needed pages, resulting in a couple of expensive I/Os as well as parsing. +The most optimal value is of course, the entire number of entries in the database. +However, most directory servers don't consistently serve out their entire database, so setting this to a lesser number that more closely matches the believed working set of data is +sufficient. This is the second most important parameter for the DB. + As far as balancing the entry cache vs the BDB cache - parsed entries in memory are generally about twice as large as they are on disk. @@ -284,62 +311,27 @@ occurring in the database. It is merely the fact that the cache is thrashing itself that causes performance/response time to slowdown. -MOVE BELOW AROUND: - - -If you want to setup the cache size, please read: - - (Xref) How do I configure the BDB backend? - (Xref) What are the DB_CONFIG configuration directives? - http://www.sleepycat.com/docs/utility/db_recover.html - -A default config can be found in the answer: - - (Xref) What are the DB_CONFIG configuration directives? - -just change the set_lg_dir to point to your .log directory or comment that line. - -Quick guide: -* Create a DB_CONFIG file in your ldap home directory (/var/lib/ldap/DB_CONFIG) with the correct "set_cachesize" value -* stop your ldap server and run db_recover -h /var/lib/ldap -* start your ldap server and check the new cache size with: - - db_stat -h /var/lib/ldap -m | head -n 2 - -* this procedure is only needed if you use OpenLDAP 2.2 with the BDB or HDB backends; In OpenLDAP 2.3 DB recovery is performed automatically whenever the DB_CONFIG file is changed or when an unclean shutdown is detected. - - ---On Tuesday, February 22, 2005 12:15 PM -0500 Dusty Doris wrote: - - Few questions, if you change the cachesize and idlecachesize entries, do - you have to do anything special aside from restarting slapd, such as run - slapindex or db_recover? - - - Also, is there any way to tell how much memory these caches are taking up - to make sure they are not set too large? What happens if you set your - cachesize too large and you don't have enough available memory to store - these? Will that cause an issue with openldap, or will it just not cache - those entries that would make it exceed its available memory. Will it - just use some sort of FIFO on those caches? - - -It will consume the memory resources of your system, and likely cause issues. - - Finally, what do most people try to achieve with these values? Would the - goal be to make these as big as the directory? So, if I have 400,000 dn's - in my directory, would it be safe to set these at 400000 or would - something like 20,000 be good enough to get a nice performance increase? - +H3: {{TERM:IDL}} Cache (idlcachesize) -I try to cache the most actively used entries. Unless you expect all 400,000 entries of your DB to be accessed regularly, there is no need to cache that many entries. My entry cache is set to 20,000 (out of a little over 400,000 entries). +Each IDL holds the search results from a given query, so the IDL cache will +end up holding the most frequently requested search results. For back-bdb, +it is generally recommended to match the "cachesize" setting. For back-hdb, +it is generally recommended to be 3x"cachesize". -The idlcache has to do with how many unique result sets of searches you want to store in memory. Setting up this cache will allow your most frequently placed searches to get results much faster, but I doubt you want to try and cache the results of every search that hits your system. ;) +{NOTE: The idlcachesize setting directly affects search performance} ---Quanah +H3: {{slapd}}(8) Threads -H3: {{TERM:IDL}} Cache +{{slapd}}(8) can process requests via a configurable number of thread, which +in turn affects the in/out rate of connections. +This value should generally be a function of the number of "real" cores on +the system, for example on a server with 2 CPUs with one core each, set this +to 8, or 4 threads per real core. This is a "read" maximized value. The more +threads that are configured per core, the slower {{slapd}}(8) responds for +"read" operations. On the flip side, it appears to handle write operations +faster in a heavy write/low read scenario. -http://www.openldap.org/faq/data/cache/1076.html +The upper bound for good read performance appears to be 16 threads (which +also happens to be the default setting).