2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula memory pool routines.
31 * The idea behind these routines is that there will be
32 * pools of memory that are pre-allocated for quick
33 * access. The pools will have a fixed memory size on allocation
34 * but if need be, the size can be increased. This is
35 * particularly useful for filename
36 * buffers where 256 bytes should be sufficient in 99.99%
37 * of the cases, but when it isn't we want to be able to
40 * A major advantage of the pool memory aside from the speed
41 * is that the buffer carrys around its size, so to ensure that
42 * there is enough memory, simply call the check_pool_memory_size()
43 * with the desired size and it will adjust only if necessary.
51 #ifdef HAVE_MALLOC_TRIM
52 extern "C" int malloc_trim (size_t pad);
56 int32_t size; /* default size */
57 int32_t max_allocated; /* max allocated */
58 int32_t max_used; /* max buffers used */
59 int32_t in_use; /* number in use */
60 struct abufhead *free_buf; /* pointer to free buffers */
63 /* Bacula Name length plus extra */
64 #define NLEN (MAX_NAME_LENGTH+2)
66 /* #define STRESS_TEST_POOL */
67 #ifndef STRESS_TEST_POOL
69 * Define default Pool buffer sizes
71 static struct s_pool_ctl pool_ctl[] = {
72 { 256, 256, 0, 0, NULL }, /* PM_NOPOOL no pooling */
73 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
74 { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */
75 { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */
76 { 1024, 1024, 0, 0, NULL }, /* PM_EMSG error message buffer */
77 { 4096, 4096, 0, 0, NULL } /* PM_BSOCK message buffer */
81 /* This is used ONLY when stress testing the code */
82 static struct s_pool_ctl pool_ctl[] = {
83 { 20, 20, 0, 0, NULL }, /* PM_NOPOOL no pooling */
84 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
85 { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */
86 { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */
87 { 20, 20, 0, 0, NULL }, /* PM_EMSG error message buffer */
88 { 20, 20, 0, 0, NULL } /* PM_BSOCK message buffer */
93 /* Memory allocation control structures and storage. */
95 int32_t ablen; /* Buffer length in bytes */
96 int32_t pool; /* pool */
97 struct abufhead *next; /* pointer to next free buffer */
98 int32_t bnet_size; /* dummy for bnet_send() */
101 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
105 #define HEAD_SIZE BALIGN(sizeof(struct abufhead))
107 POOLMEM *sm_get_pool_memory(const char *fname, int lineno, int pool)
109 struct abufhead *buf;
112 Emsg2(M_ABORT, 0, _("MemPool index %d larger than max %d\n"), pool, PM_MAX);
115 if (pool_ctl[pool].free_buf) {
116 buf = pool_ctl[pool].free_buf;
117 pool_ctl[pool].free_buf = buf->next;
118 pool_ctl[pool].in_use++;
119 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
120 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
123 Dmsg3(1800, "sm_get_pool_memory reuse %p to %s:%d\n", buf, fname, lineno);
124 sm_new_owner(fname, lineno, (char *)buf);
125 return (POOLMEM *)((char *)buf+HEAD_SIZE);
128 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
130 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
132 buf->ablen = pool_ctl[pool].size;
134 pool_ctl[pool].in_use++;
135 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
136 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
139 Dmsg3(1800, "sm_get_pool_memory give %p to %s:%d\n", buf, fname, lineno);
140 return (POOLMEM *)((char *)buf+HEAD_SIZE);
143 /* Get nonpool memory of size requested */
144 POOLMEM *sm_get_memory(const char *fname, int lineno, int32_t size)
146 struct abufhead *buf;
149 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, size+HEAD_SIZE)) == NULL) {
150 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
155 pool_ctl[pool].in_use++;
156 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used)
157 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
158 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
161 /* Return the size of a memory buffer */
162 int32_t sm_sizeof_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
164 char *cp = (char *)obuf;
167 Emsg0(M_ABORT, 0, _("obuf is NULL\n"));
170 return ((struct abufhead *)cp)->ablen;
173 /* Realloc pool memory buffer */
174 POOLMEM *sm_realloc_pool_memory(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
176 char *cp = (char *)obuf;
183 buf = sm_realloc(fname, lineno, cp, size+HEAD_SIZE);
186 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
188 ((struct abufhead *)buf)->ablen = size;
189 pool = ((struct abufhead *)buf)->pool;
190 if (size > pool_ctl[pool].max_allocated) {
191 pool_ctl[pool].max_allocated = size;
194 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
197 POOLMEM *sm_check_pool_memory_size(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
200 if (size <= sizeof_pool_memory(obuf)) {
203 return realloc_pool_memory(obuf, size);
206 /* Free a memory buffer */
207 void sm_free_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
209 struct abufhead *buf;
214 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
216 pool_ctl[pool].in_use--;
218 free((char *)buf); /* free nonpooled memory */
219 } else { /* otherwise link it to the free pool chain */
221 struct abufhead *next;
222 /* Don't let him free the same buffer twice */
223 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
225 Dmsg4(1800, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
226 Dmsg4(1800, "bad free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
227 V(mutex); /* unblock the pool */
228 ASSERT(next != buf); /* attempt to free twice */
232 buf->next = pool_ctl[pool].free_buf;
233 pool_ctl[pool].free_buf = buf;
235 Dmsg4(1800, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
241 /* ========= NO SMARTALLOC ========================================= */
243 POOLMEM *get_pool_memory(int pool)
245 struct abufhead *buf;
248 if (pool_ctl[pool].free_buf) {
249 buf = pool_ctl[pool].free_buf;
250 pool_ctl[pool].free_buf = buf->next;
252 return (POOLMEM *)((char *)buf+HEAD_SIZE);
255 if ((buf=malloc(pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
257 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
259 buf->ablen = pool_ctl[pool].size;
262 pool_ctl[pool].in_use++;
263 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
264 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
267 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
270 /* Get nonpool memory of size requested */
271 POOLMEM *get_memory(int32_t size)
273 struct abufhead *buf;
276 if ((buf=malloc(size+HEAD_SIZE)) == NULL) {
277 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
282 pool_ctl[pool].in_use++;
283 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
284 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
286 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
289 /* Return the size of a memory buffer */
290 int32_t sizeof_pool_memory(POOLMEM *obuf)
292 char *cp = (char *)obuf;
296 return ((struct abufhead *)cp)->ablen;
299 /* Realloc pool memory buffer */
300 POOLMEM *realloc_pool_memory(POOLMEM *obuf, int32_t size)
302 char *cp = (char *)obuf;
309 buf = realloc(cp, size+HEAD_SIZE);
312 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
314 ((struct abufhead *)buf)->ablen = size;
315 pool = ((struct abufhead *)buf)->pool;
316 if (size > pool_ctl[pool].max_allocated) {
317 pool_ctl[pool].max_allocated = size;
320 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
323 POOLMEM *check_pool_memory_size(POOLMEM *obuf, int32_t size)
326 if (size <= sizeof_pool_memory(obuf)) {
329 return realloc_pool_memory(obuf, size);
332 /* Free a memory buffer */
333 void free_pool_memory(POOLMEM *obuf)
335 struct abufhead *buf;
340 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
342 pool_ctl[pool].in_use--;
344 free((char *)buf); /* free nonpooled memory */
345 } else { /* otherwise link it to the free pool chain */
347 struct abufhead *next;
348 /* Don't let him free the same buffer twice */
349 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
352 ASSERT(next != buf); /* attempt to free twice */
356 buf->next = pool_ctl[pool].free_buf;
357 pool_ctl[pool].free_buf = buf;
359 Dmsg2(1800, "free_pool_memory %p pool=%d\n", buf, pool);
362 #endif /* SMARTALLOC */
365 * Clean up memory pool periodically
368 static time_t last_garbage_collection = 0;
369 const int garbage_interval = 24 * 60 * 60; /* garbage collect every 24 hours */
371 void garbage_collect_memory_pool()
375 Dmsg0(200, "garbage collect memory pool\n");
377 if (last_garbage_collection == 0) {
378 last_garbage_collection = time(NULL);
383 if (now >= last_garbage_collection + garbage_interval) {
384 last_garbage_collection = now;
386 garbage_collect_memory();
392 /* Release all freed pooled memory */
393 void close_memory_pool()
395 struct abufhead *buf, *next;
400 sm_check(__FILE__, __LINE__, false);
402 for (int i=1; i<=PM_MAX; i++) {
403 buf = pool_ctl[i].free_buf;
407 bytes += sizeof_pool_memory((char *)buf);
411 pool_ctl[i].free_buf = NULL;
413 Dmsg2(001, "Freed mem_pool count=%d size=%s\n", count, edit_uint64_with_commas(bytes, ed1));
414 if (debug_level >= 1) {
415 print_memory_pool_stats();
422 * Garbage collect and trim memory if possible
423 * This should be called after all big memory usages
426 void garbage_collect_memory()
428 close_memory_pool(); /* release free chain */
429 #ifdef HAVE_MALLOC_TRIM
437 static const char *pool_name(int pool)
439 static const char *name[] = {"NoPool", "NAME ", "FNAME ", "MSG ", "EMSG ", "BSOCK "};
442 if (pool >= 0 && pool <= PM_MAX) {
445 sprintf(buf, "%-6d", pool);
449 /* Print staticstics on memory pool usage
451 void print_memory_pool_stats()
453 Pmsg0(-1, "Pool Maxsize Maxused Inuse\n");
454 for (int i=0; i<=PM_MAX; i++)
455 Pmsg4(-1, "%5s %7d %7d %5d\n", pool_name(i), pool_ctl[i].max_allocated,
456 pool_ctl[i].max_used, pool_ctl[i].in_use);
462 void print_memory_pool_stats() {}
466 * Concatenate a string (str) onto a pool memory buffer pm
467 * Returns: length of concatenated string
469 int pm_strcat(POOLMEM **pm, const char *str)
471 int pmlen = strlen(*pm);
476 len = strlen(str) + 1;
477 *pm = check_pool_memory_size(*pm, pmlen + len);
478 memcpy(*pm+pmlen, str, len);
479 return pmlen + len - 1;
482 int pm_strcat(POOLMEM *&pm, const char *str)
484 int pmlen = strlen(pm);
489 len = strlen(str) + 1;
490 pm = check_pool_memory_size(pm, pmlen + len);
491 memcpy(pm+pmlen, str, len);
492 return pmlen + len - 1;
495 int pm_strcat(POOLMEM *&pm, POOL_MEM &str)
497 int pmlen = strlen(pm);
498 int len = strlen(str.c_str()) + 1;
500 pm = check_pool_memory_size(pm, pmlen + len);
501 memcpy(pm+pmlen, str.c_str(), len);
502 return pmlen + len - 1;
505 int pm_strcat(POOL_MEM &pm, const char *str)
507 int pmlen = strlen(pm.c_str());
512 len = strlen(str) + 1;
513 pm.check_size(pmlen + len);
514 memcpy(pm.c_str()+pmlen, str, len);
515 return pmlen + len - 1;
519 * Copy a string (str) into a pool memory buffer pm
520 * Returns: length of string copied
522 int pm_strcpy(POOLMEM **pm, const char *str)
528 len = strlen(str) + 1;
529 *pm = check_pool_memory_size(*pm, len);
530 memcpy(*pm, str, len);
534 int pm_strcpy(POOLMEM *&pm, const char *str)
540 len = strlen(str) + 1;
541 pm = check_pool_memory_size(pm, len);
542 memcpy(pm, str, len);
546 int pm_strcpy(POOLMEM *&pm, POOL_MEM &str)
548 int len = strlen(str.c_str()) + 1;
550 pm = check_pool_memory_size(pm, len);
551 memcpy(pm, str.c_str(), len);
555 int pm_strcpy(POOL_MEM &pm, const char *str)
561 len = strlen(str) + 1;
563 memcpy(pm.c_str(), str, len);
568 * Copy data into a pool memory buffer pm
569 * Returns: length of data copied
571 int pm_memcpy(POOLMEM **pm, const char *data, int32_t n)
573 *pm = check_pool_memory_size(*pm, n);
574 memcpy(*pm, data, n);
578 int pm_memcpy(POOLMEM *&pm, const char *data, int32_t n)
580 pm = check_pool_memory_size(pm, n);
585 int pm_memcpy(POOLMEM *&pm, POOL_MEM &data, int32_t n)
587 pm = check_pool_memory_size(pm, n);
588 memcpy(pm, data.c_str(), n);
592 int pm_memcpy(POOL_MEM &pm, const char *data, int32_t n)
595 memcpy(pm.c_str(), data, n);
599 /* ============== CLASS POOL_MEM ============== */
601 /* Return the size of a memory buffer */
602 int32_t POOL_MEM::max_size()
607 size = ((struct abufhead *)cp)->ablen;
608 Dmsg1(900, "max_size=%d\n", size);
612 void POOL_MEM::realloc_pm(int32_t size)
620 buf = (char *)realloc(cp, size+HEAD_SIZE);
623 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
625 Dmsg2(900, "Old buf=%p new buf=%p\n", cp, buf);
626 ((struct abufhead *)buf)->ablen = size;
627 pool = ((struct abufhead *)buf)->pool;
628 if (size > pool_ctl[pool].max_allocated) {
629 pool_ctl[pool].max_allocated = size;
633 Dmsg3(900, "Old buf=%p new buf=%p mem=%p\n", cp, buf, mem);
636 int POOL_MEM::strcat(const char *str)
638 int pmlen = strlen(mem);
643 len = strlen(str) + 1;
644 check_size(pmlen + len);
645 memcpy(mem+pmlen, str, len);
646 return pmlen + len - 1;
649 int POOL_MEM::strcpy(const char *str)
655 len = strlen(str) + 1;
657 memcpy(mem, str, len);