2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2014 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from many
7 others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 Bacula® is a registered trademark of Kern Sibbald.
17 * Bacula memory pool routines.
19 * The idea behind these routines is that there will be
20 * pools of memory that are pre-allocated for quick
21 * access. The pools will have a fixed memory size on allocation
22 * but if need be, the size can be increased. This is
23 * particularly useful for filename
24 * buffers where 256 bytes should be sufficient in 99.99%
25 * of the cases, but when it isn't we want to be able to
28 * A major advantage of the pool memory aside from the speed
29 * is that the buffer carrys around its size, so to ensure that
30 * there is enough memory, simply call the check_pool_memory_size()
31 * with the desired size and it will adjust only if necessary.
38 #define dbglvl DT_MEMORY|800
40 #ifdef HAVE_MALLOC_TRIM
41 extern "C" int malloc_trim (size_t pad);
45 int32_t size; /* default size */
46 int32_t max_allocated; /* max allocated */
47 int32_t max_used; /* max buffers used */
48 int32_t in_use; /* number in use */
49 struct abufhead *free_buf; /* pointer to free buffers */
52 /* Bacula Name length plus extra */
53 #define NLEN (MAX_NAME_LENGTH+2)
55 /* #define STRESS_TEST_POOL */
56 #ifndef STRESS_TEST_POOL
58 * Define default Pool buffer sizes
60 static struct s_pool_ctl pool_ctl[] = {
61 { 256, 256, 0, 0, NULL }, /* PM_NOPOOL no pooling */
62 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
63 { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */
64 { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */
65 { 1024, 1024, 0, 0, NULL }, /* PM_EMSG error message buffer */
66 { 4096, 4096, 0, 0, NULL } /* PM_BSOCK message buffer */
70 /* This is used ONLY when stress testing the code */
71 static struct s_pool_ctl pool_ctl[] = {
72 { 20, 20, 0, 0, NULL }, /* PM_NOPOOL no pooling */
73 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
74 { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */
75 { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */
76 { 20, 20, 0, 0, NULL }, /* PM_EMSG error message buffer */
77 { 20, 20, 0, 0, NULL } /* PM_BSOCK message buffer */
82 /* Memory allocation control structures and storage. */
84 int32_t ablen; /* Buffer length in bytes */
85 int32_t pool; /* pool */
86 struct abufhead *next; /* pointer to next free buffer */
87 int32_t bnet_size; /* dummy for bnet_send() */
90 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
94 #define HEAD_SIZE BALIGN(sizeof(struct abufhead))
96 POOLMEM *sm_get_pool_memory(const char *fname, int lineno, int pool)
101 Emsg2(M_ABORT, 0, _("MemPool index %d larger than max %d\n"), pool, PM_MAX);
104 if (pool_ctl[pool].free_buf) {
105 buf = pool_ctl[pool].free_buf;
106 pool_ctl[pool].free_buf = buf->next;
107 pool_ctl[pool].in_use++;
108 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
109 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
112 Dmsg3(dbglvl, "sm_get_pool_memory reuse %p to %s:%d\n", buf, fname, lineno);
113 sm_new_owner(fname, lineno, (char *)buf);
114 return (POOLMEM *)((char *)buf+HEAD_SIZE);
117 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
119 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
121 buf->ablen = pool_ctl[pool].size;
123 pool_ctl[pool].in_use++;
124 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
125 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
128 Dmsg3(dbglvl, "sm_get_pool_memory give %p to %s:%d\n", buf, fname, lineno);
129 return (POOLMEM *)((char *)buf+HEAD_SIZE);
132 /* Get nonpool memory of size requested */
133 POOLMEM *sm_get_memory(const char *fname, int lineno, int32_t size)
135 struct abufhead *buf;
138 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, size+HEAD_SIZE)) == NULL) {
139 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
144 pool_ctl[pool].in_use++;
145 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used)
146 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
147 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
150 /* Return the size of a memory buffer */
151 int32_t sm_sizeof_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
153 char *cp = (char *)obuf;
156 Emsg0(M_ABORT, 0, _("obuf is NULL\n"));
159 return ((struct abufhead *)cp)->ablen;
162 /* Realloc pool memory buffer */
163 POOLMEM *sm_realloc_pool_memory(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
165 char *cp = (char *)obuf;
172 buf = sm_realloc(fname, lineno, cp, size+HEAD_SIZE);
175 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
177 ((struct abufhead *)buf)->ablen = size;
178 pool = ((struct abufhead *)buf)->pool;
179 if (size > pool_ctl[pool].max_allocated) {
180 pool_ctl[pool].max_allocated = size;
183 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
186 POOLMEM *sm_check_pool_memory_size(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
189 if (size <= sizeof_pool_memory(obuf)) {
192 return realloc_pool_memory(obuf, size);
195 /* Free a memory buffer */
196 void sm_free_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
198 struct abufhead *buf;
203 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
205 pool_ctl[pool].in_use--;
207 free((char *)buf); /* free nonpooled memory */
208 } else { /* otherwise link it to the free pool chain */
210 /* Disabled because it hangs in #5507 */
212 struct abufhead *next;
213 /* Don't let him free the same buffer twice */
214 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
216 Dmsg4(dbglvl, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
217 Dmsg4(dbglvl, "bad free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
218 V(mutex); /* unblock the pool */
219 ASSERT(next != buf); /* attempt to free twice */
223 buf->next = pool_ctl[pool].free_buf;
224 pool_ctl[pool].free_buf = buf;
226 Dmsg4(dbglvl, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
232 /* ========= NO SMARTALLOC ========================================= */
234 POOLMEM *get_pool_memory(int pool)
236 struct abufhead *buf;
239 if (pool_ctl[pool].free_buf) {
240 buf = pool_ctl[pool].free_buf;
241 pool_ctl[pool].free_buf = buf->next;
243 return (POOLMEM *)((char *)buf+HEAD_SIZE);
246 if ((buf=malloc(pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
248 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
250 buf->ablen = pool_ctl[pool].size;
253 pool_ctl[pool].in_use++;
254 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
255 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
258 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
261 /* Get nonpool memory of size requested */
262 POOLMEM *get_memory(int32_t size)
264 struct abufhead *buf;
267 if ((buf=malloc(size+HEAD_SIZE)) == NULL) {
268 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
273 pool_ctl[pool].in_use++;
274 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
275 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
277 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
280 /* Return the size of a memory buffer */
281 int32_t sizeof_pool_memory(POOLMEM *obuf)
283 char *cp = (char *)obuf;
287 return ((struct abufhead *)cp)->ablen;
290 /* Realloc pool memory buffer */
291 POOLMEM *realloc_pool_memory(POOLMEM *obuf, int32_t size)
293 char *cp = (char *)obuf;
300 buf = realloc(cp, size+HEAD_SIZE);
303 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
305 ((struct abufhead *)buf)->ablen = size;
306 pool = ((struct abufhead *)buf)->pool;
307 if (size > pool_ctl[pool].max_allocated) {
308 pool_ctl[pool].max_allocated = size;
311 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
314 POOLMEM *check_pool_memory_size(POOLMEM *obuf, int32_t size)
317 if (size <= sizeof_pool_memory(obuf)) {
320 return realloc_pool_memory(obuf, size);
323 /* Free a memory buffer */
324 void free_pool_memory(POOLMEM *obuf)
326 struct abufhead *buf;
331 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
333 pool_ctl[pool].in_use--;
335 free((char *)buf); /* free nonpooled memory */
336 } else { /* otherwise link it to the free pool chain */
338 struct abufhead *next;
339 /* Don't let him free the same buffer twice */
340 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
343 ASSERT(next != buf); /* attempt to free twice */
347 buf->next = pool_ctl[pool].free_buf;
348 pool_ctl[pool].free_buf = buf;
350 Dmsg2(dbglvl, "free_pool_memory %p pool=%d\n", buf, pool);
353 #endif /* SMARTALLOC */
356 * Clean up memory pool periodically
359 static time_t last_garbage_collection = 0;
360 const int garbage_interval = 24 * 60 * 60; /* garbage collect every 24 hours */
362 void garbage_collect_memory_pool()
366 Dmsg0(200, "garbage collect memory pool\n");
368 if (last_garbage_collection == 0) {
369 last_garbage_collection = time(NULL);
374 if (now >= last_garbage_collection + garbage_interval) {
375 last_garbage_collection = now;
377 garbage_collect_memory();
383 /* Release all freed pooled memory */
384 void close_memory_pool()
386 struct abufhead *buf, *next;
391 sm_check(__FILE__, __LINE__, false);
393 for (int i=1; i<=PM_MAX; i++) {
394 buf = pool_ctl[i].free_buf;
398 bytes += sizeof_pool_memory((char *)buf);
402 pool_ctl[i].free_buf = NULL;
404 Dmsg2(DT_MEMORY|001, "Freed mem_pool count=%d size=%s\n", count, edit_uint64_with_commas(bytes, ed1));
405 if (chk_dbglvl(DT_MEMORY|1)) {
406 print_memory_pool_stats();
413 * Garbage collect and trim memory if possible
414 * This should be called after all big memory usages
417 void garbage_collect_memory()
419 close_memory_pool(); /* release free chain */
420 #ifdef HAVE_MALLOC_TRIM
428 static const char *pool_name(int pool)
430 static const char *name[] = {"NoPool", "NAME ", "FNAME ", "MSG ", "EMSG ", "BSOCK "};
433 if (pool >= 0 && pool <= PM_MAX) {
436 sprintf(buf, "%-6d", pool);
440 /* Print staticstics on memory pool usage
442 void print_memory_pool_stats()
444 Pmsg0(-1, "Pool Maxsize Maxused Inuse\n");
445 for (int i=0; i<=PM_MAX; i++)
446 Pmsg4(-1, "%5s %7d %7d %5d\n", pool_name(i), pool_ctl[i].max_allocated,
447 pool_ctl[i].max_used, pool_ctl[i].in_use);
453 void print_memory_pool_stats() {}
457 * Concatenate a string (str) onto a pool memory buffer pm
458 * Returns: length of concatenated string
460 int pm_strcat(POOLMEM **pm, const char *str)
462 int pmlen = strlen(*pm);
467 len = strlen(str) + 1;
468 *pm = check_pool_memory_size(*pm, pmlen + len);
469 memcpy(*pm+pmlen, str, len);
470 return pmlen + len - 1;
473 int pm_strcat(POOLMEM *&pm, const char *str)
475 int pmlen = strlen(pm);
480 len = strlen(str) + 1;
481 pm = check_pool_memory_size(pm, pmlen + len);
482 memcpy(pm+pmlen, str, len);
483 return pmlen + len - 1;
486 int pm_strcat(POOLMEM *&pm, POOL_MEM &str)
488 int pmlen = strlen(pm);
489 int len = strlen(str.c_str()) + 1;
491 pm = check_pool_memory_size(pm, pmlen + len);
492 memcpy(pm+pmlen, str.c_str(), len);
493 return pmlen + len - 1;
496 int pm_strcat(POOL_MEM &pm, const char *str)
498 int pmlen = strlen(pm.c_str());
503 len = strlen(str) + 1;
504 pm.check_size(pmlen + len);
505 memcpy(pm.c_str()+pmlen, str, len);
506 return pmlen + len - 1;
510 * Copy a string (str) into a pool memory buffer pm
511 * Returns: length of string copied
513 int pm_strcpy(POOLMEM **pm, const char *str)
519 len = strlen(str) + 1;
520 *pm = check_pool_memory_size(*pm, len);
521 memcpy(*pm, str, len);
525 int pm_strcpy(POOLMEM *&pm, const char *str)
531 len = strlen(str) + 1;
532 pm = check_pool_memory_size(pm, len);
533 memcpy(pm, str, len);
537 int pm_strcpy(POOLMEM *&pm, POOL_MEM &str)
539 int len = strlen(str.c_str()) + 1;
541 pm = check_pool_memory_size(pm, len);
542 memcpy(pm, str.c_str(), len);
546 int pm_strcpy(POOL_MEM &pm, const char *str)
552 len = strlen(str) + 1;
554 memcpy(pm.c_str(), str, len);
559 * Copy data into a pool memory buffer pm
560 * Returns: length of data copied
562 int pm_memcpy(POOLMEM **pm, const char *data, int32_t n)
564 *pm = check_pool_memory_size(*pm, n);
565 memcpy(*pm, data, n);
569 int pm_memcpy(POOLMEM *&pm, const char *data, int32_t n)
571 pm = check_pool_memory_size(pm, n);
576 int pm_memcpy(POOLMEM *&pm, POOL_MEM &data, int32_t n)
578 pm = check_pool_memory_size(pm, n);
579 memcpy(pm, data.c_str(), n);
583 int pm_memcpy(POOL_MEM &pm, const char *data, int32_t n)
586 memcpy(pm.c_str(), data, n);
590 /* ============== CLASS POOL_MEM ============== */
592 /* Return the size of a memory buffer */
593 int32_t POOL_MEM::max_size()
598 size = ((struct abufhead *)cp)->ablen;
599 Dmsg1(900, "max_size=%d\n", size);
603 void POOL_MEM::realloc_pm(int32_t size)
611 buf = (char *)realloc(cp, size+HEAD_SIZE);
614 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
616 Dmsg2(900, "Old buf=%p new buf=%p\n", cp, buf);
617 ((struct abufhead *)buf)->ablen = size;
618 pool = ((struct abufhead *)buf)->pool;
619 if (size > pool_ctl[pool].max_allocated) {
620 pool_ctl[pool].max_allocated = size;
624 Dmsg3(900, "Old buf=%p new buf=%p mem=%p\n", cp, buf, mem);
627 int POOL_MEM::strcat(const char *str)
629 int pmlen = strlen(mem);
634 len = strlen(str) + 1;
635 check_size(pmlen + len);
636 memcpy(mem+pmlen, str, len);
637 return pmlen + len - 1;
640 int POOL_MEM::strcpy(const char *str)
646 len = strlen(str) + 1;
648 memcpy(mem, str, len);