X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Flib%2Fmem_pool.c;h=61dc1bf52d3404c1e1c06ac4fe6590b81204f702;hb=3f8a3a045ea058657030f588a10f786449d00e0d;hp=251d5dcc103b477c206f350383470a2504f2a0a0;hpb=1ef84681d13d0c355d6ae78579d2afe1c8d8d0e2;p=bacula%2Fbacula diff --git a/bacula/src/lib/mem_pool.c b/bacula/src/lib/mem_pool.c index 251d5dcc10..61dc1bf52d 100644 --- a/bacula/src/lib/mem_pool.c +++ b/bacula/src/lib/mem_pool.c @@ -1,10 +1,10 @@ /* - * Bacula memory pool routines. + * Bacula memory pool routines. * * The idea behind these routines is that there will be * pools of memory that are pre-allocated for quick * access. The pools will have a fixed memory size on allocation - * but if need be, the size can be increased. This is + * but if need be, the size can be increased. This is * particularly useful for filename * buffers where 256 bytes should be sufficient in 99.99% * of the cases, but when it isn't we want to be able to @@ -15,51 +15,69 @@ * there is enough memory, simply call the check_pool_memory_size() * with the desired size and it will adjust only if necessary. * - * Kern E. Sibbald + * Kern E. Sibbald + * + * Version $Id$ */ /* - Copyright (C) 2000, 2001, 2002 Kern Sibbald and John Walker + Copyright (C) 2000-2006 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as amended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ #include "bacula.h" struct s_pool_ctl { - size_t size; /* default size */ - size_t max_size; /* max allocated */ - size_t max_used; /* max buffers used */ - size_t in_use; /* number in use */ - struct abufhead *free_buf; /* pointer to free buffers */ + int32_t size; /* default size */ + int32_t max_allocated; /* max allocated */ + int32_t max_used; /* max buffers used */ + int32_t in_use; /* number in use */ + struct abufhead *free_buf; /* pointer to free buffers */ }; +/* Bacula Name length plus extra */ +#define NLEN (MAX_NAME_LENGTH+2) + +/* #define STRESS_TEST_POOL */ +#ifndef STRESS_TEST_POOL +/* + * Define default Pool buffer sizes + */ static struct s_pool_ctl pool_ctl[] = { - { 256, 256, 0, 0, NULL }, /* PM_NOPOOL no pooling */ - { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */ - { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */ - { 1024, 1024, 0, 0, NULL } /* PM_EMSG error message buffer */ + { 256, 256, 0, 0, NULL }, /* PM_NOPOOL no pooling */ + { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */ + { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */ + { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */ + { 1024, 1024, 0, 0, NULL } /* PM_EMSG error message buffer */ }; +#else + +/* This is used ONLY when stress testing the code */ +static struct s_pool_ctl pool_ctl[] = { + { 20, 20, 0, 0, NULL }, /* PM_NOPOOL no pooling */ + { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */ + { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */ + { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */ + { 20, 20, 0, 0, NULL } /* PM_EMSG error message buffer */ +}; +#endif + /* Memory allocation control structures and storage. */ struct abufhead { - size_t ablen; /* Buffer length in bytes */ - int32_t pool; /* pool */ - struct abufhead *next; /* pointer to next free buffer */ + int32_t ablen; /* Buffer length in bytes */ + int32_t pool; /* pool */ + struct abufhead *next; /* pointer to next free buffer */ }; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; @@ -69,15 +87,12 @@ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; #define HEAD_SIZE BALIGN(sizeof(struct abufhead)) -extern void *sm_malloc(char *fname, int lineno, int nbytes); - -void *sm_get_pool_memory(char *fname, int lineno, int pool) +POOLMEM *sm_get_pool_memory(const char *fname, int lineno, int pool) { struct abufhead *buf; - sm_check(fname, lineno, True); if (pool > PM_MAX) { - Emsg2(M_ABORT, 0, "MemPool index %d larger than max %d\n", pool, PM_MAX); + Emsg2(M_ABORT, 0, _("MemPool index %d larger than max %d\n"), pool, PM_MAX); } P(mutex); if (pool_ctl[pool].free_buf) { @@ -85,17 +100,17 @@ void *sm_get_pool_memory(char *fname, int lineno, int pool) pool_ctl[pool].free_buf = buf->next; pool_ctl[pool].in_use++; if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) { - pool_ctl[pool].max_used = pool_ctl[pool].in_use; + pool_ctl[pool].max_used = pool_ctl[pool].in_use; } V(mutex); - Dmsg3(150, "sm_get_pool_memory reuse %x to %s:%d\n", buf, fname, lineno); + Dmsg3(1800, "sm_get_pool_memory reuse %p to %s:%d\n", buf, fname, lineno); sm_new_owner(fname, lineno, (char *)buf); - return (void *)((char *)buf+HEAD_SIZE); + return (POOLMEM *)((char *)buf+HEAD_SIZE); } - - if ((buf = (struct abufhead *) sm_malloc(fname, lineno, pool_ctl[pool].size+HEAD_SIZE)) == NULL) { + + if ((buf = (struct abufhead *)sm_malloc(fname, lineno, pool_ctl[pool].size+HEAD_SIZE)) == NULL) { V(mutex); - Emsg1(M_ABORT, 0, "Out of memory requesting %d bytes\n", pool_ctl[pool].size); + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size); } buf->ablen = pool_ctl[pool].size; buf->pool = pool; @@ -104,19 +119,18 @@ void *sm_get_pool_memory(char *fname, int lineno, int pool) pool_ctl[pool].max_used = pool_ctl[pool].in_use; } V(mutex); - Dmsg3(150, "sm_get_pool_memory give %x to %s:%d\n", buf, fname, lineno); - return (void *)((char *)buf+HEAD_SIZE); + Dmsg3(1800, "sm_get_pool_memory give %p to %s:%d\n", buf, fname, lineno); + return (POOLMEM *)((char *)buf+HEAD_SIZE); } /* Get nonpool memory of size requested */ -void *sm_get_memory(char *fname, int lineno, size_t size) +POOLMEM *sm_get_memory(const char *fname, int lineno, int32_t size) { struct abufhead *buf; int pool = 0; - sm_check(fname, lineno, True); - if ((buf = (struct abufhead *) sm_malloc(fname, lineno, size+HEAD_SIZE)) == NULL) { - Emsg1(M_ABORT, 0, "Out of memory requesting %d bytes\n", size); + if ((buf = (struct abufhead *)sm_malloc(fname, lineno, size+HEAD_SIZE)) == NULL) { + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size); } buf->ablen = size; buf->pool = pool; @@ -124,12 +138,92 @@ void *sm_get_memory(char *fname, int lineno, size_t size) pool_ctl[pool].in_use++; if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) pool_ctl[pool].max_used = pool_ctl[pool].in_use; - return (void *)(((char *)buf)+HEAD_SIZE); + return (POOLMEM *)(((char *)buf)+HEAD_SIZE); +} + + +/* Return the size of a memory buffer */ +int32_t sm_sizeof_pool_memory(const char *fname, int lineno, POOLMEM *obuf) +{ + char *cp = (char *)obuf; + + ASSERT(obuf); + cp -= HEAD_SIZE; + return ((struct abufhead *)cp)->ablen; +} + +/* Realloc pool memory buffer */ +POOLMEM *sm_realloc_pool_memory(const char *fname, int lineno, POOLMEM *obuf, int32_t size) +{ + char *cp = (char *)obuf; + void *buf; + int pool; + + ASSERT(obuf); + P(mutex); + cp -= HEAD_SIZE; + buf = sm_realloc(fname, lineno, cp, size+HEAD_SIZE); + if (buf == NULL) { + V(mutex); + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size); + } + ((struct abufhead *)buf)->ablen = size; + pool = ((struct abufhead *)buf)->pool; + if (size > pool_ctl[pool].max_allocated) { + pool_ctl[pool].max_allocated = size; + } + V(mutex); + return (POOLMEM *)(((char *)buf)+HEAD_SIZE); +} + +POOLMEM *sm_check_pool_memory_size(const char *fname, int lineno, POOLMEM *obuf, int32_t size) +{ + ASSERT(obuf); + if (size <= sizeof_pool_memory(obuf)) { + return obuf; + } + return realloc_pool_memory(obuf, size); +} + +/* Free a memory buffer */ +void sm_free_pool_memory(const char *fname, int lineno, POOLMEM *obuf) +{ + struct abufhead *buf; + int pool; + + ASSERT(obuf); + P(mutex); + buf = (struct abufhead *)((char *)obuf - HEAD_SIZE); + pool = buf->pool; + pool_ctl[pool].in_use--; + if (pool == 0) { + free((char *)buf); /* free nonpooled memory */ + } else { /* otherwise link it to the free pool chain */ +#ifdef DEBUG + struct abufhead *next; + /* Don't let him free the same buffer twice */ + for (next=pool_ctl[pool].free_buf; next; next=next->next) { + if (next == buf) { + Dmsg4(1800, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno); + Dmsg4(1800, "bad free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno); + V(mutex); /* unblock the pool */ + ASSERT(next != buf); /* attempt to free twice */ + } + } +#endif + buf->next = pool_ctl[pool].free_buf; + pool_ctl[pool].free_buf = buf; + } + Dmsg4(1800, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno); + V(mutex); } + #else -void *get_pool_memory(int pool) +/* ========= NO SMARTALLOC ========================================= */ + +POOLMEM *get_pool_memory(int pool) { struct abufhead *buf; @@ -138,12 +232,12 @@ void *get_pool_memory(int pool) buf = pool_ctl[pool].free_buf; pool_ctl[pool].free_buf = buf->next; V(mutex); - return (void *)((char *)buf+HEAD_SIZE); + return (POOLMEM *)((char *)buf+HEAD_SIZE); } - + if ((buf=malloc(pool_ctl[pool].size+HEAD_SIZE)) == NULL) { V(mutex); - Emsg1(M_ABORT, 0, "Out of memory requesting %d bytes\n", pool_ctl[pool].size); + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size); } buf->ablen = pool_ctl[pool].size; buf->pool = pool; @@ -153,17 +247,17 @@ void *get_pool_memory(int pool) pool_ctl[pool].max_used = pool_ctl[pool].in_use; } V(mutex); - return (void *)(((char *)buf)+HEAD_SIZE); + return (POOLMEM *)(((char *)buf)+HEAD_SIZE); } /* Get nonpool memory of size requested */ -void *get_memory(size_t size) +POOLMEM *get_memory(int32_t size) { struct abufhead *buf; int pool = 0; if ((buf=malloc(size+HEAD_SIZE)) == NULL) { - Emsg1(M_ABORT, 0, "Out of memory requesting %d bytes\n", size); + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size); } buf->ablen = size; buf->pool = pool; @@ -172,75 +266,49 @@ void *get_memory(size_t size) if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) { pool_ctl[pool].max_used = pool_ctl[pool].in_use; } - return (void *)(((char *)buf)+HEAD_SIZE); -} -#endif /* SMARTALLOC */ - - - -/* Free a memory buffer */ -void free_pool_memory(void *obuf) -{ - struct abufhead *buf; - int pool; - - sm_check(__FILE__, __LINE__, False); - ASSERT(obuf); - P(mutex); - buf = (struct abufhead *)((char *)obuf - HEAD_SIZE); - pool = buf->pool; - pool_ctl[pool].in_use--; - if (pool == 0) { - free((char *)buf); /* free nonpooled memory */ - } else { /* otherwise link it to the free pool chain */ - buf->next = pool_ctl[pool].free_buf; - pool_ctl[pool].free_buf = buf; - } - Dmsg2(150, "free_pool_memory %x pool=%d\n", buf, pool); - V(mutex); + return (POOLMEM *)(((char *)buf)+HEAD_SIZE); } /* Return the size of a memory buffer */ -size_t sizeof_pool_memory(void *obuf) +int32_t sizeof_pool_memory(POOLMEM *obuf) { char *cp = (char *)obuf; - sm_check(__FILE__, __LINE__, False); ASSERT(obuf); cp -= HEAD_SIZE; return ((struct abufhead *)cp)->ablen; } + + /* Realloc pool memory buffer */ -void *realloc_pool_memory(void *obuf, size_t size) +POOLMEM *realloc_pool_memory(POOLMEM *obuf, int32_t size) { char *cp = (char *)obuf; void *buf; int pool; - sm_check(__FILE__, __LINE__, False); ASSERT(obuf); P(mutex); cp -= HEAD_SIZE; buf = realloc(cp, size+HEAD_SIZE); if (buf == NULL) { V(mutex); - Emsg1(M_ABORT, 0, "Out of memory requesting %d bytes\n", size); + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size); } ((struct abufhead *)buf)->ablen = size; pool = ((struct abufhead *)buf)->pool; - if (size > pool_ctl[pool].max_size) { - pool_ctl[pool].max_size = size; + if (size > pool_ctl[pool].max_allocated) { + pool_ctl[pool].max_allocated = size; } V(mutex); - sm_check(__FILE__, __LINE__, False); - return (void *)(((char *)buf)+HEAD_SIZE); + return (POOLMEM *)(((char *)buf)+HEAD_SIZE); } -void *check_pool_memory_size(void *obuf, size_t size) + +POOLMEM *check_pool_memory_size(POOLMEM *obuf, int32_t size) { - sm_check(__FILE__, __LINE__, False); ASSERT(obuf); if (size <= sizeof_pool_memory(obuf)) { return obuf; @@ -248,30 +316,102 @@ void *check_pool_memory_size(void *obuf, size_t size) return realloc_pool_memory(obuf, size); } +/* Free a memory buffer */ +void free_pool_memory(POOLMEM *obuf) +{ + struct abufhead *buf; + int pool; + + ASSERT(obuf); + P(mutex); + buf = (struct abufhead *)((char *)obuf - HEAD_SIZE); + pool = buf->pool; + pool_ctl[pool].in_use--; + if (pool == 0) { + free((char *)buf); /* free nonpooled memory */ + } else { /* otherwise link it to the free pool chain */ +#ifdef DEBUG + struct abufhead *next; + /* Don't let him free the same buffer twice */ + for (next=pool_ctl[pool].free_buf; next; next=next->next) { + if (next == buf) { + V(mutex); + ASSERT(next != buf); /* attempt to free twice */ + } + } +#endif + buf->next = pool_ctl[pool].free_buf; + pool_ctl[pool].free_buf = buf; + } + Dmsg2(1800, "free_pool_memory %p pool=%d\n", buf, pool); + V(mutex); +} + +#endif /* SMARTALLOC */ + + +/* + * Clean up memory pool periodically + * + */ +static time_t last_garbage_collection = 0; +const int garbage_interval = 24 * 60 * 60; /* garbage collect every 24 hours */ + +void garbage_collect_memory_pool() +{ + time_t now; + + Dmsg0(200, "garbage collect memory pool\n"); + P(mutex); + if (last_garbage_collection == 0) { + last_garbage_collection = time(NULL); + V(mutex); + return; + } + now = time(NULL); + if (now >= last_garbage_collection + garbage_interval) { + last_garbage_collection = now; + V(mutex); + close_memory_pool(); + } else { + V(mutex); + } +} + + + + /* Release all pooled memory */ void close_memory_pool() { struct abufhead *buf, *next; - int i; + int count = 0; + uint64_t bytes = 0; + char ed1[50]; - sm_check(__FILE__, __LINE__, False); + sm_check(__FILE__, __LINE__, false); P(mutex); - for (i=1; i<=PM_MAX; i++) { + for (int i=1; i<=PM_MAX; i++) { buf = pool_ctl[i].free_buf; while (buf) { - next = buf->next; - free((char *)buf); - buf = next; + next = buf->next; + count++; + bytes += sizeof_pool_memory((char *)buf); + free((char *)buf); + buf = next; } + pool_ctl[i].free_buf = NULL; } + Dmsg2(100, "Freed mem_pool count=%d size=%s\n", count, edit_uint64_with_commas(bytes, ed1)); V(mutex); + } #ifdef DEBUG -static char *pool_name(int pool) +static const char *pool_name(int pool) { - static char *name[] = {"NoPool", "FNAME ", "MSG ", "EMSG "}; + static const char *name[] = {"NoPool", "NAME ", "FNAME ", "MSG ", "EMSG "}; static char buf[30]; if (pool >= 0 && pool <= PM_MAX) { @@ -280,21 +420,162 @@ static char *pool_name(int pool) sprintf(buf, "%-6d", pool); return buf; } - -/* Print staticstics on memory pool usage - */ + +/* Print staticstics on memory pool usage + */ void print_memory_pool_stats() { - int i; + Pmsg0(-1, "Pool Maxsize Maxused Inuse\n"); + for (int i=0; i<=PM_MAX; i++) + Pmsg4(-1, "%5s %7d %7d %5d\n", pool_name(i), pool_ctl[i].max_allocated, + pool_ctl[i].max_used, pool_ctl[i].in_use); - Dmsg0(-1, "Pool Maxsize Maxused Inuse\n"); - for (i=0; i<=PM_MAX; i++) - Dmsg4(-1, "%5s %7d %7d %5d\n", pool_name(i), pool_ctl[i].max_size, - pool_ctl[i].max_used, pool_ctl[i].in_use); - - Dmsg0(-1, "\n"); + Pmsg0(-1, "\n"); } #else -void print_memory_pool_stats() {} +void print_memory_pool_stats() {} #endif /* DEBUG */ + + +/* + * Concatenate a string (str) onto a pool memory buffer pm + * Returns: length of concatenated string + */ +int pm_strcat(POOLMEM **pm, const char *str) +{ + int pmlen = strlen(*pm); + int len = strlen(str) + 1; + + *pm = check_pool_memory_size(*pm, pmlen + len); + memcpy(*pm+pmlen, str, len); + return pmlen + len - 1; +} + +int pm_strcat(POOLMEM *&pm, const char *str) +{ + int pmlen = strlen(pm); + int len = strlen(str) + 1; + + pm = check_pool_memory_size(pm, pmlen + len); + memcpy(pm+pmlen, str, len); + return pmlen + len - 1; +} + + +int pm_strcat(POOLMEM *&pm, POOL_MEM &str) +{ + int pmlen = strlen(pm); + int len = strlen(str.c_str()) + 1; + + pm = check_pool_memory_size(pm, pmlen + len); + memcpy(pm+pmlen, str.c_str(), len); + return pmlen + len - 1; +} + +int pm_strcat(POOL_MEM &pm, const char *str) +{ + int pmlen = strlen(pm.c_str()); + int len = strlen(str) + 1; + + pm.check_size(pmlen + len); + memcpy(pm.c_str()+pmlen, str, len); + return pmlen + len - 1; +} + + +/* + * Copy a string (str) into a pool memory buffer pm + * Returns: length of string copied + */ +int pm_strcpy(POOLMEM **pm, const char *str) +{ + int len = strlen(str) + 1; + + *pm = check_pool_memory_size(*pm, len); + memcpy(*pm, str, len); + return len - 1; +} + +int pm_strcpy(POOLMEM *&pm, const char *str) +{ + int len = strlen(str) + 1; + + pm = check_pool_memory_size(pm, len); + memcpy(pm, str, len); + return len - 1; +} + +int pm_strcpy(POOLMEM *&pm, POOL_MEM &str) +{ + int len = strlen(str.c_str()) + 1; + + pm = check_pool_memory_size(pm, len); + memcpy(pm, str.c_str(), len); + return len - 1; +} + + +int pm_strcpy(POOL_MEM &pm, const char *str) +{ + int len = strlen(str) + 1; + pm.check_size(len); + memcpy(pm.c_str(), str, len); + return len - 1; +} + +/* ============== CLASS POOL_MEM ============== */ + +/* Return the size of a memory buffer */ +int32_t POOL_MEM::max_size() +{ + int32_t size; + char *cp = mem; + cp -= HEAD_SIZE; + size = ((struct abufhead *)cp)->ablen; + Dmsg1(900, "max_size=%d\n", size); + return size; +} + +void POOL_MEM::realloc_pm(int32_t size) +{ + char *cp = mem; + char *buf; + int pool; + + P(mutex); + cp -= HEAD_SIZE; + buf = (char *)realloc(cp, size+HEAD_SIZE); + if (buf == NULL) { + V(mutex); + Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size); + } + Dmsg2(900, "Old buf=%p new buf=%p\n", cp, buf); + ((struct abufhead *)buf)->ablen = size; + pool = ((struct abufhead *)buf)->pool; + if (size > pool_ctl[pool].max_allocated) { + pool_ctl[pool].max_allocated = size; + } + mem = buf+HEAD_SIZE; + V(mutex); + Dmsg3(900, "Old buf=%p new buf=%p mem=%p\n", cp, buf, mem); +} + +int POOL_MEM::strcat(const char *str) +{ + int pmlen = strlen(mem); + int len = strlen(str) + 1; + + check_size(pmlen + len); + memcpy(mem+pmlen, str, len); + return pmlen + len - 1; +} + + +int POOL_MEM::strcpy(const char *str) +{ + int len = strlen(str) + 1; + check_size(len); + memcpy(mem, str, len); + return len - 1; +}