From 5af98b83701e67876eecd78cedd7bd0db9ccb656 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Thu, 3 Feb 2011 11:44:35 +0100 Subject: [PATCH] Add code to trim heap after big mallocs --- bacula/autoconf/config.h.in | 3 +++ bacula/autoconf/configure.in | 2 ++ bacula/src/dird/ua_cmds.c | 1 + bacula/src/dird/ua_restore.c | 11 +++++++++-- bacula/src/lib/bnet.c | 4 ++-- bacula/src/lib/bsock.c | 2 +- bacula/src/lib/htable.c | 3 ++- bacula/src/lib/jcr.c | 1 - bacula/src/lib/mem_pool.c | 30 +++++++++++++++++++++++++----- bacula/src/lib/mem_pool.h | 6 ++++-- bacula/src/lib/tree.c | 1 + 11 files changed, 50 insertions(+), 14 deletions(-) diff --git a/bacula/autoconf/config.h.in b/bacula/autoconf/config.h.in index ecb138732d..2fb4214085 100644 --- a/bacula/autoconf/config.h.in +++ b/bacula/autoconf/config.h.in @@ -547,6 +547,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MALLOC_H +/* Set if have malloc_trim */ +#undef HAVE_MALLOC_TRIM + /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H diff --git a/bacula/autoconf/configure.in b/bacula/autoconf/configure.in index af0a07cb59..302a47cd8f 100644 --- a/bacula/autoconf/configure.in +++ b/bacula/autoconf/configure.in @@ -2398,6 +2398,8 @@ AC_CHECK_FUNCS( \ [echo 'configure: cannot find needed function.'; exit 1] ) +AC_CHECK_FUNCS(malloc_trim, [AC_DEFINE(HAVE_MALLOC_TRIM, 1, [Set if have malloc_trim])]) + AC_CHECK_FUNCS(fchdir, [AC_DEFINE(HAVE_FCHDIR)]) AC_CHECK_FUNCS(strtoll, [AC_DEFINE(HAVE_STRTOLL)]) AC_CHECK_FUNCS(posix_fadvise) diff --git a/bacula/src/dird/ua_cmds.c b/bacula/src/dird/ua_cmds.c index e1a9cb11ce..1988ff10db 100644 --- a/bacula/src/dird/ua_cmds.c +++ b/bacula/src/dird/ua_cmds.c @@ -1553,6 +1553,7 @@ static int delete_pool(UAContext *ua) int memory_cmd(UAContext *ua, const char *cmd) { + garbage_collect_memory(); list_dir_status_header(ua); sm_dump(false, true); return 1; diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index e460ac5c25..49176f92fd 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -294,7 +294,7 @@ int restore_cmd(UAContext *ua, const char *cmd) parse_ua_args(ua); run_cmd(ua, ua->cmd); free_rx(&rx); - close_memory_pool(); /* release freed pool memory */ + garbage_collect_memory(); /* release unused memory */ return 1; bail_out: @@ -311,7 +311,7 @@ bail_out: } free_rx(&rx); - close_memory_pool(); /* release freed pool memory */ + garbage_collect_memory(); /* release unused memory */ return 0; } @@ -1165,6 +1165,13 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) } } #endif + /* + * At this point, the tree is built, so we can garbage collect + * any memory released by the SQL engine that RedHat has + * not returned to the OS :-( + */ + garbage_collect_memory(); + /* * Look at the first JobId on the list (presumably the oldest) and * if it is marked purged, don't do the manual selection because diff --git a/bacula/src/lib/bnet.c b/bacula/src/lib/bnet.c index 851197a981..e141767134 100644 --- a/bacula/src/lib/bnet.c +++ b/bacula/src/lib/bnet.c @@ -681,7 +681,7 @@ BSOCK *init_bsock(JCR * jcr, int sockfd, const char *who, const char *host, int bsock->tls = NULL; bsock->errors = 0; bsock->m_blocking = 1; - bsock->msg = get_pool_memory(PM_MESSAGE); + bsock->msg = get_pool_memory(PM_BSOCK); bsock->errmsg = get_pool_memory(PM_MESSAGE); bsock->set_who(bstrdup(who)); bsock->set_host(bstrdup(host)); @@ -701,7 +701,7 @@ BSOCK *dup_bsock(BSOCK *osock) { BSOCK *bsock = (BSOCK *)malloc(sizeof(BSOCK)); memcpy(bsock, osock, sizeof(BSOCK)); - bsock->msg = get_pool_memory(PM_MESSAGE); + bsock->msg = get_pool_memory(PM_BSOCK); bsock->errmsg = get_pool_memory(PM_MESSAGE); if (osock->who()) { bsock->set_who(bstrdup(osock->who())); diff --git a/bacula/src/lib/bsock.c b/bacula/src/lib/bsock.c index f9fa8fbccd..b1550c14b3 100644 --- a/bacula/src/lib/bsock.c +++ b/bacula/src/lib/bsock.c @@ -66,7 +66,7 @@ void BSOCK::init() { memset(this, 0, sizeof(BSOCK)); m_blocking = 1; - msg = get_pool_memory(PM_MESSAGE); + msg = get_pool_memory(PM_BSOCK); errmsg = get_pool_memory(PM_MESSAGE); /* * ****FIXME**** reduce this to a few hours once diff --git a/bacula/src/lib/htable.c b/bacula/src/lib/htable.c index a42ec022d0..eaa057d0d9 100644 --- a/bacula/src/lib/htable.c +++ b/bacula/src/lib/htable.c @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2003-2010 Free Software Foundation Europe e.V. + Copyright (C) 2003-2011 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -371,6 +371,7 @@ void htable::destroy() free(table); table = NULL; + garbage_collect_memory(); Dmsg0(100, "Done destroy.\n"); } diff --git a/bacula/src/lib/jcr.c b/bacula/src/lib/jcr.c index 5b88efdc35..0f63c0807e 100644 --- a/bacula/src/lib/jcr.c +++ b/bacula/src/lib/jcr.c @@ -576,7 +576,6 @@ void free_jcr(JCR *jcr) free_common_jcr(jcr); close_msg(NULL); /* flush any daemon messages */ - garbage_collect_memory_pool(); Dmsg0(dbglvl, "Exit free_jcr\n"); } diff --git a/bacula/src/lib/mem_pool.c b/bacula/src/lib/mem_pool.c index a371fdd737..575b4868e2 100644 --- a/bacula/src/lib/mem_pool.c +++ b/bacula/src/lib/mem_pool.c @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2010 Free Software Foundation Europe e.V. + Copyright (C) 2000-2011 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -47,6 +47,9 @@ */ #include "bacula.h" +#ifdef HAVE_MALLOC_TRIM +#include +#endif struct s_pool_ctl { int32_t size; /* default size */ @@ -69,7 +72,8 @@ static struct s_pool_ctl pool_ctl[] = { { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */ { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */ { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */ - { 1024, 1024, 0, 0, NULL } /* PM_EMSG error message buffer */ + { 1024, 1024, 0, 0, NULL }, /* PM_EMSG error message buffer */ + { 4096, 4096, 0, 0, NULL } /* PM_BSOCK message buffer */ }; #else @@ -79,7 +83,8 @@ static struct s_pool_ctl pool_ctl[] = { { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */ { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */ { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */ - { 20, 20, 0, 0, NULL } /* PM_EMSG error message buffer */ + { 20, 20, 0, 0, NULL }, /* PM_EMSG error message buffer */ + { 20, 20, 0, 0, NULL } /* PM_BSOCK message buffer */ }; #endif @@ -377,13 +382,13 @@ void garbage_collect_memory_pool() if (now >= last_garbage_collection + garbage_interval) { last_garbage_collection = now; V(mutex); - close_memory_pool(); + garbage_collect_memory(); } else { V(mutex); } } -/* Release all pooled memory */ +/* Release all freed pooled memory */ void close_memory_pool() { struct abufhead *buf, *next; @@ -412,6 +417,21 @@ void close_memory_pool() } +/* + * Garbage collect and trim memory if possible + * This should be called after all big memory usages + * if possible. + */ +void garbage_collect_memory() +{ + close_memory_pool(); /* release free chain */ +#ifdef HAVE_MALLOC_TRIM + P(mutex); + malloc_trim(8192); + V(mutex); +#endif +} + #ifdef DEBUG static const char *pool_name(int pool) { diff --git a/bacula/src/lib/mem_pool.h b/bacula/src/lib/mem_pool.h index 626bf22fc1..c1a2f4d998 100644 --- a/bacula/src/lib/mem_pool.h +++ b/bacula/src/lib/mem_pool.h @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2010 Free Software Foundation Europe e.V. + Copyright (C) 2000-2011 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -76,6 +76,7 @@ extern void garbage_collect_memory_pool(); extern void close_memory_pool(); extern void print_memory_pool_stats(); +extern void garbage_collect_memory(); #define PM_NOPOOL 0 /* nonpooled memory */ @@ -83,7 +84,8 @@ extern void print_memory_pool_stats(); #define PM_FNAME 2 /* file name buffer */ #define PM_MESSAGE 3 /* daemon message */ #define PM_EMSG 4 /* error message */ -#define PM_MAX PM_EMSG /* Number of types */ +#define PM_BSOCK 5 /* BSOCK buffer */ +#define PM_MAX PM_BSOCK /* Number of types */ class POOL_MEM { char *mem; diff --git a/bacula/src/lib/tree.c b/bacula/src/lib/tree.c index f9fc4ac886..fda5b0a72c 100644 --- a/bacula/src/lib/tree.c +++ b/bacula/src/lib/tree.c @@ -184,6 +184,7 @@ void free_tree(TREE_ROOT *root) } Dmsg3(100, "Total size=%u blocks=%u freed_blocks=%u\n", root->total_size, root->blocks, freed_blocks); free(root); + garbage_collect_memory(); return; } -- 2.39.5