2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula memory pool routines.
31 * The idea behind these routines is that there will be
32 * pools of memory that are pre-allocated for quick
33 * access. The pools will have a fixed memory size on allocation
34 * but if need be, the size can be increased. This is
35 * particularly useful for filename
36 * buffers where 256 bytes should be sufficient in 99.99%
37 * of the cases, but when it isn't we want to be able to
40 * A major advantage of the pool memory aside from the speed
41 * is that the buffer carrys around its size, so to ensure that
42 * there is enough memory, simply call the check_pool_memory_size()
43 * with the desired size and it will adjust only if necessary.
50 #ifdef HAVE_MALLOC_TRIM
55 int32_t size; /* default size */
56 int32_t max_allocated; /* max allocated */
57 int32_t max_used; /* max buffers used */
58 int32_t in_use; /* number in use */
59 struct abufhead *free_buf; /* pointer to free buffers */
62 /* Bacula Name length plus extra */
63 #define NLEN (MAX_NAME_LENGTH+2)
65 /* #define STRESS_TEST_POOL */
66 #ifndef STRESS_TEST_POOL
68 * Define default Pool buffer sizes
70 static struct s_pool_ctl pool_ctl[] = {
71 { 256, 256, 0, 0, NULL }, /* PM_NOPOOL no pooling */
72 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
73 { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */
74 { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */
75 { 1024, 1024, 0, 0, NULL }, /* PM_EMSG error message buffer */
76 { 4096, 4096, 0, 0, NULL } /* PM_BSOCK message buffer */
80 /* This is used ONLY when stress testing the code */
81 static struct s_pool_ctl pool_ctl[] = {
82 { 20, 20, 0, 0, NULL }, /* PM_NOPOOL no pooling */
83 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
84 { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */
85 { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */
86 { 20, 20, 0, 0, NULL }, /* PM_EMSG error message buffer */
87 { 20, 20, 0, 0, NULL } /* PM_BSOCK message buffer */
92 /* Memory allocation control structures and storage. */
94 int32_t ablen; /* Buffer length in bytes */
95 int32_t pool; /* pool */
96 struct abufhead *next; /* pointer to next free buffer */
97 int32_t bnet_size; /* dummy for bnet_send() */
100 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
104 #define HEAD_SIZE BALIGN(sizeof(struct abufhead))
106 POOLMEM *sm_get_pool_memory(const char *fname, int lineno, int pool)
108 struct abufhead *buf;
111 Emsg2(M_ABORT, 0, _("MemPool index %d larger than max %d\n"), pool, PM_MAX);
114 if (pool_ctl[pool].free_buf) {
115 buf = pool_ctl[pool].free_buf;
116 pool_ctl[pool].free_buf = buf->next;
117 pool_ctl[pool].in_use++;
118 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
119 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
122 Dmsg3(1800, "sm_get_pool_memory reuse %p to %s:%d\n", buf, fname, lineno);
123 sm_new_owner(fname, lineno, (char *)buf);
124 return (POOLMEM *)((char *)buf+HEAD_SIZE);
127 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
129 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
131 buf->ablen = pool_ctl[pool].size;
133 pool_ctl[pool].in_use++;
134 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
135 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
138 Dmsg3(1800, "sm_get_pool_memory give %p to %s:%d\n", buf, fname, lineno);
139 return (POOLMEM *)((char *)buf+HEAD_SIZE);
142 /* Get nonpool memory of size requested */
143 POOLMEM *sm_get_memory(const char *fname, int lineno, int32_t size)
145 struct abufhead *buf;
148 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, size+HEAD_SIZE)) == NULL) {
149 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
154 pool_ctl[pool].in_use++;
155 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used)
156 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
157 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
160 /* Return the size of a memory buffer */
161 int32_t sm_sizeof_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
163 char *cp = (char *)obuf;
166 Emsg0(M_ABORT, 0, _("obuf is NULL\n"));
169 return ((struct abufhead *)cp)->ablen;
172 /* Realloc pool memory buffer */
173 POOLMEM *sm_realloc_pool_memory(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
175 char *cp = (char *)obuf;
182 buf = sm_realloc(fname, lineno, cp, size+HEAD_SIZE);
185 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
187 ((struct abufhead *)buf)->ablen = size;
188 pool = ((struct abufhead *)buf)->pool;
189 if (size > pool_ctl[pool].max_allocated) {
190 pool_ctl[pool].max_allocated = size;
193 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
196 POOLMEM *sm_check_pool_memory_size(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
199 if (size <= sizeof_pool_memory(obuf)) {
202 return realloc_pool_memory(obuf, size);
205 /* Free a memory buffer */
206 void sm_free_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
208 struct abufhead *buf;
213 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
215 pool_ctl[pool].in_use--;
217 free((char *)buf); /* free nonpooled memory */
218 } else { /* otherwise link it to the free pool chain */
220 struct abufhead *next;
221 /* Don't let him free the same buffer twice */
222 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
224 Dmsg4(1800, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
225 Dmsg4(1800, "bad free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
226 V(mutex); /* unblock the pool */
227 ASSERT(next != buf); /* attempt to free twice */
231 buf->next = pool_ctl[pool].free_buf;
232 pool_ctl[pool].free_buf = buf;
234 Dmsg4(1800, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
240 /* ========= NO SMARTALLOC ========================================= */
242 POOLMEM *get_pool_memory(int pool)
244 struct abufhead *buf;
247 if (pool_ctl[pool].free_buf) {
248 buf = pool_ctl[pool].free_buf;
249 pool_ctl[pool].free_buf = buf->next;
251 return (POOLMEM *)((char *)buf+HEAD_SIZE);
254 if ((buf=malloc(pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
256 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
258 buf->ablen = pool_ctl[pool].size;
261 pool_ctl[pool].in_use++;
262 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
263 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
266 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
269 /* Get nonpool memory of size requested */
270 POOLMEM *get_memory(int32_t size)
272 struct abufhead *buf;
275 if ((buf=malloc(size+HEAD_SIZE)) == NULL) {
276 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
281 pool_ctl[pool].in_use++;
282 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
283 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
285 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
288 /* Return the size of a memory buffer */
289 int32_t sizeof_pool_memory(POOLMEM *obuf)
291 char *cp = (char *)obuf;
295 return ((struct abufhead *)cp)->ablen;
298 /* Realloc pool memory buffer */
299 POOLMEM *realloc_pool_memory(POOLMEM *obuf, int32_t size)
301 char *cp = (char *)obuf;
308 buf = realloc(cp, size+HEAD_SIZE);
311 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
313 ((struct abufhead *)buf)->ablen = size;
314 pool = ((struct abufhead *)buf)->pool;
315 if (size > pool_ctl[pool].max_allocated) {
316 pool_ctl[pool].max_allocated = size;
319 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
322 POOLMEM *check_pool_memory_size(POOLMEM *obuf, int32_t size)
325 if (size <= sizeof_pool_memory(obuf)) {
328 return realloc_pool_memory(obuf, size);
331 /* Free a memory buffer */
332 void free_pool_memory(POOLMEM *obuf)
334 struct abufhead *buf;
339 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
341 pool_ctl[pool].in_use--;
343 free((char *)buf); /* free nonpooled memory */
344 } else { /* otherwise link it to the free pool chain */
346 struct abufhead *next;
347 /* Don't let him free the same buffer twice */
348 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
351 ASSERT(next != buf); /* attempt to free twice */
355 buf->next = pool_ctl[pool].free_buf;
356 pool_ctl[pool].free_buf = buf;
358 Dmsg2(1800, "free_pool_memory %p pool=%d\n", buf, pool);
361 #endif /* SMARTALLOC */
364 * Clean up memory pool periodically
367 static time_t last_garbage_collection = 0;
368 const int garbage_interval = 24 * 60 * 60; /* garbage collect every 24 hours */
370 void garbage_collect_memory_pool()
374 Dmsg0(200, "garbage collect memory pool\n");
376 if (last_garbage_collection == 0) {
377 last_garbage_collection = time(NULL);
382 if (now >= last_garbage_collection + garbage_interval) {
383 last_garbage_collection = now;
385 garbage_collect_memory();
391 /* Release all freed pooled memory */
392 void close_memory_pool()
394 struct abufhead *buf, *next;
399 sm_check(__FILE__, __LINE__, false);
401 for (int i=1; i<=PM_MAX; i++) {
402 buf = pool_ctl[i].free_buf;
406 bytes += sizeof_pool_memory((char *)buf);
410 pool_ctl[i].free_buf = NULL;
412 Dmsg2(001, "Freed mem_pool count=%d size=%s\n", count, edit_uint64_with_commas(bytes, ed1));
413 if (debug_level >= 1) {
414 print_memory_pool_stats();
421 * Garbage collect and trim memory if possible
422 * This should be called after all big memory usages
425 void garbage_collect_memory()
427 close_memory_pool(); /* release free chain */
428 #ifdef HAVE_MALLOC_TRIM
436 static const char *pool_name(int pool)
438 static const char *name[] = {"NoPool", "NAME ", "FNAME ", "MSG ", "EMSG ", "BSOCK "};
441 if (pool >= 0 && pool <= PM_MAX) {
444 sprintf(buf, "%-6d", pool);
448 /* Print staticstics on memory pool usage
450 void print_memory_pool_stats()
452 Pmsg0(-1, "Pool Maxsize Maxused Inuse\n");
453 for (int i=0; i<=PM_MAX; i++)
454 Pmsg4(-1, "%5s %7d %7d %5d\n", pool_name(i), pool_ctl[i].max_allocated,
455 pool_ctl[i].max_used, pool_ctl[i].in_use);
461 void print_memory_pool_stats() {}
465 * Concatenate a string (str) onto a pool memory buffer pm
466 * Returns: length of concatenated string
468 int pm_strcat(POOLMEM **pm, const char *str)
470 int pmlen = strlen(*pm);
475 len = strlen(str) + 1;
476 *pm = check_pool_memory_size(*pm, pmlen + len);
477 memcpy(*pm+pmlen, str, len);
478 return pmlen + len - 1;
481 int pm_strcat(POOLMEM *&pm, const char *str)
483 int pmlen = strlen(pm);
488 len = strlen(str) + 1;
489 pm = check_pool_memory_size(pm, pmlen + len);
490 memcpy(pm+pmlen, str, len);
491 return pmlen + len - 1;
494 int pm_strcat(POOLMEM *&pm, POOL_MEM &str)
496 int pmlen = strlen(pm);
497 int len = strlen(str.c_str()) + 1;
499 pm = check_pool_memory_size(pm, pmlen + len);
500 memcpy(pm+pmlen, str.c_str(), len);
501 return pmlen + len - 1;
504 int pm_strcat(POOL_MEM &pm, const char *str)
506 int pmlen = strlen(pm.c_str());
511 len = strlen(str) + 1;
512 pm.check_size(pmlen + len);
513 memcpy(pm.c_str()+pmlen, str, len);
514 return pmlen + len - 1;
518 * Copy a string (str) into a pool memory buffer pm
519 * Returns: length of string copied
521 int pm_strcpy(POOLMEM **pm, const char *str)
527 len = strlen(str) + 1;
528 *pm = check_pool_memory_size(*pm, len);
529 memcpy(*pm, str, len);
533 int pm_strcpy(POOLMEM *&pm, const char *str)
539 len = strlen(str) + 1;
540 pm = check_pool_memory_size(pm, len);
541 memcpy(pm, str, len);
545 int pm_strcpy(POOLMEM *&pm, POOL_MEM &str)
547 int len = strlen(str.c_str()) + 1;
549 pm = check_pool_memory_size(pm, len);
550 memcpy(pm, str.c_str(), len);
554 int pm_strcpy(POOL_MEM &pm, const char *str)
560 len = strlen(str) + 1;
562 memcpy(pm.c_str(), str, len);
567 * Copy data into a pool memory buffer pm
568 * Returns: length of data copied
570 int pm_memcpy(POOLMEM **pm, const char *data, int32_t n)
572 *pm = check_pool_memory_size(*pm, n);
573 memcpy(*pm, data, n);
577 int pm_memcpy(POOLMEM *&pm, const char *data, int32_t n)
579 pm = check_pool_memory_size(pm, n);
584 int pm_memcpy(POOLMEM *&pm, POOL_MEM &data, int32_t n)
586 pm = check_pool_memory_size(pm, n);
587 memcpy(pm, data.c_str(), n);
591 int pm_memcpy(POOL_MEM &pm, const char *data, int32_t n)
594 memcpy(pm.c_str(), data, n);
598 /* ============== CLASS POOL_MEM ============== */
600 /* Return the size of a memory buffer */
601 int32_t POOL_MEM::max_size()
606 size = ((struct abufhead *)cp)->ablen;
607 Dmsg1(900, "max_size=%d\n", size);
611 void POOL_MEM::realloc_pm(int32_t size)
619 buf = (char *)realloc(cp, size+HEAD_SIZE);
622 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
624 Dmsg2(900, "Old buf=%p new buf=%p\n", cp, buf);
625 ((struct abufhead *)buf)->ablen = size;
626 pool = ((struct abufhead *)buf)->pool;
627 if (size > pool_ctl[pool].max_allocated) {
628 pool_ctl[pool].max_allocated = size;
632 Dmsg3(900, "Old buf=%p new buf=%p mem=%p\n", cp, buf, mem);
635 int POOL_MEM::strcat(const char *str)
637 int pmlen = strlen(mem);
642 len = strlen(str) + 1;
643 check_size(pmlen + len);
644 memcpy(mem+pmlen, str, len);
645 return pmlen + len - 1;
648 int POOL_MEM::strcpy(const char *str)
654 len = strlen(str) + 1;
656 memcpy(mem, str, len);