2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
20 * Bacula memory pool routines.
22 * The idea behind these routines is that there will be
23 * pools of memory that are pre-allocated for quick
24 * access. The pools will have a fixed memory size on allocation
25 * but if need be, the size can be increased. This is
26 * particularly useful for filename
27 * buffers where 256 bytes should be sufficient in 99.99%
28 * of the cases, but when it isn't we want to be able to
31 * A major advantage of the pool memory aside from the speed
32 * is that the buffer carrys around its size, so to ensure that
33 * there is enough memory, simply call the check_pool_memory_size()
34 * with the desired size and it will adjust only if necessary.
41 #define dbglvl DT_MEMORY|800
43 #ifdef HAVE_MALLOC_TRIM
44 extern "C" int malloc_trim (size_t pad);
48 int32_t size; /* default size */
49 int32_t max_allocated; /* max allocated */
50 int32_t max_used; /* max buffers used */
51 int32_t in_use; /* number in use */
52 struct abufhead *free_buf; /* pointer to free buffers */
55 /* Bacula Name length plus extra */
56 #define NLEN (MAX_NAME_LENGTH+2)
58 /* #define STRESS_TEST_POOL */
59 #ifndef STRESS_TEST_POOL
61 * Define default Pool buffer sizes
63 static struct s_pool_ctl pool_ctl[] = {
64 { 256, 256, 0, 0, NULL }, /* PM_NOPOOL no pooling */
65 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
66 { 256, 256, 0, 0, NULL }, /* PM_FNAME filename buffers */
67 { 512, 512, 0, 0, NULL }, /* PM_MESSAGE message buffer */
68 { 1024, 1024, 0, 0, NULL }, /* PM_EMSG error message buffer */
69 { 4096, 4096, 0, 0, NULL } /* PM_BSOCK message buffer */
73 /* This is used ONLY when stress testing the code */
74 static struct s_pool_ctl pool_ctl[] = {
75 { 20, 20, 0, 0, NULL }, /* PM_NOPOOL no pooling */
76 { NLEN, NLEN,0, 0, NULL }, /* PM_NAME Bacula name */
77 { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */
78 { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */
79 { 20, 20, 0, 0, NULL }, /* PM_EMSG error message buffer */
80 { 20, 20, 0, 0, NULL } /* PM_BSOCK message buffer */
85 /* Memory allocation control structures and storage. */
87 int32_t ablen; /* Buffer length in bytes */
88 int32_t pool; /* pool */
89 struct abufhead *next; /* pointer to next free buffer */
90 int32_t bnet_size; /* dummy for bnet_send() */
91 int32_t bnet_extension; /* dummy for bnet extension */
94 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
96 #define HEAD_SIZE BALIGN(sizeof(struct abufhead))
100 POOLMEM *sm_get_pool_memory(const char *fname, int lineno, int pool)
102 struct abufhead *buf;
105 Emsg2(M_ABORT, 0, _("MemPool index %d larger than max %d\n"), pool, PM_MAX);
108 if (pool_ctl[pool].free_buf) {
109 buf = pool_ctl[pool].free_buf;
110 pool_ctl[pool].free_buf = buf->next;
111 pool_ctl[pool].in_use++;
112 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
113 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
116 Dmsg3(dbglvl, "sm_get_pool_memory reuse %p to %s:%d\n", buf, fname, lineno);
117 sm_new_owner(fname, lineno, (char *)buf);
118 return (POOLMEM *)((char *)buf+HEAD_SIZE);
121 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
123 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
125 buf->ablen = pool_ctl[pool].size;
127 pool_ctl[pool].in_use++;
128 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
129 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
132 Dmsg3(dbglvl, "sm_get_pool_memory give %p to %s:%d\n", buf, fname, lineno);
133 return (POOLMEM *)((char *)buf+HEAD_SIZE);
136 /* Get nonpool memory of size requested */
137 POOLMEM *sm_get_memory(const char *fname, int lineno, int32_t size)
139 struct abufhead *buf;
142 if ((buf = (struct abufhead *)sm_malloc(fname, lineno, size+HEAD_SIZE)) == NULL) {
143 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
148 pool_ctl[pool].in_use++;
149 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used)
150 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
151 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
154 /* Return the size of a memory buffer */
155 int32_t sm_sizeof_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
157 char *cp = (char *)obuf;
160 Emsg0(M_ABORT, 0, _("obuf is NULL\n"));
163 return ((struct abufhead *)cp)->ablen;
166 /* Realloc pool memory buffer */
167 POOLMEM *sm_realloc_pool_memory(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
169 char *cp = (char *)obuf;
176 buf = sm_realloc(fname, lineno, cp, size+HEAD_SIZE);
179 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
181 ((struct abufhead *)buf)->ablen = size;
182 pool = ((struct abufhead *)buf)->pool;
183 if (size > pool_ctl[pool].max_allocated) {
184 pool_ctl[pool].max_allocated = size;
187 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
190 POOLMEM *sm_check_pool_memory_size(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
193 if (size <= sizeof_pool_memory(obuf)) {
196 return realloc_pool_memory(obuf, size);
199 /* Free a memory buffer */
200 void sm_free_pool_memory(const char *fname, int lineno, POOLMEM *obuf)
202 struct abufhead *buf;
207 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
209 pool_ctl[pool].in_use--;
211 free((char *)buf); /* free nonpooled memory */
212 } else { /* otherwise link it to the free pool chain */
214 /* Disabled because it hangs in #5507 */
216 struct abufhead *next;
217 /* Don't let him free the same buffer twice */
218 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
220 Dmsg4(dbglvl, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
221 Dmsg4(dbglvl, "bad free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
222 V(mutex); /* unblock the pool */
223 ASSERT(next != buf); /* attempt to free twice */
227 buf->next = pool_ctl[pool].free_buf;
228 pool_ctl[pool].free_buf = buf;
230 Dmsg4(dbglvl, "free_pool_memory %p pool=%d from %s:%d\n", buf, pool, fname, lineno);
236 /* ========= NO SMARTALLOC ========================================= */
238 POOLMEM *get_pool_memory(int pool)
240 struct abufhead *buf;
243 if (pool_ctl[pool].free_buf) {
244 buf = pool_ctl[pool].free_buf;
245 pool_ctl[pool].free_buf = buf->next;
247 return (POOLMEM *)((char *)buf+HEAD_SIZE);
250 if ((buf=(struct abufhead*)malloc(pool_ctl[pool].size+HEAD_SIZE)) == NULL) {
252 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), pool_ctl[pool].size);
254 buf->ablen = pool_ctl[pool].size;
257 pool_ctl[pool].in_use++;
258 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
259 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
262 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
265 /* Get nonpool memory of size requested */
266 POOLMEM *get_memory(int32_t size)
268 struct abufhead *buf;
271 if ((buf=(struct abufhead *)malloc(size+HEAD_SIZE)) == NULL) {
272 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
277 pool_ctl[pool].in_use++;
278 if (pool_ctl[pool].in_use > pool_ctl[pool].max_used) {
279 pool_ctl[pool].max_used = pool_ctl[pool].in_use;
281 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
284 /* Return the size of a memory buffer */
285 int32_t sizeof_pool_memory(POOLMEM *obuf)
287 char *cp = (char *)obuf;
291 return ((struct abufhead *)cp)->ablen;
294 /* Realloc pool memory buffer */
295 POOLMEM *realloc_pool_memory(POOLMEM *obuf, int32_t size)
297 char *cp = (char *)obuf;
304 buf = realloc(cp, size+HEAD_SIZE);
307 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
309 ((struct abufhead *)buf)->ablen = size;
310 pool = ((struct abufhead *)buf)->pool;
311 if (size > pool_ctl[pool].max_allocated) {
312 pool_ctl[pool].max_allocated = size;
315 return (POOLMEM *)(((char *)buf)+HEAD_SIZE);
318 POOLMEM *check_pool_memory_size(POOLMEM *obuf, int32_t size)
321 if (size <= sizeof_pool_memory(obuf)) {
324 return realloc_pool_memory(obuf, size);
327 /* Free a memory buffer */
328 void free_pool_memory(POOLMEM *obuf)
330 struct abufhead *buf;
335 buf = (struct abufhead *)((char *)obuf - HEAD_SIZE);
337 pool_ctl[pool].in_use--;
339 free((char *)buf); /* free nonpooled memory */
340 } else { /* otherwise link it to the free pool chain */
342 struct abufhead *next;
343 /* Don't let him free the same buffer twice */
344 for (next=pool_ctl[pool].free_buf; next; next=next->next) {
347 ASSERT(next != buf); /* attempt to free twice */
351 buf->next = pool_ctl[pool].free_buf;
352 pool_ctl[pool].free_buf = buf;
354 Dmsg2(dbglvl, "free_pool_memory %p pool=%d\n", buf, pool);
357 #endif /* SMARTALLOC */
360 * Clean up memory pool periodically
363 static time_t last_garbage_collection = 0;
364 const int garbage_interval = 24 * 60 * 60; /* garbage collect every 24 hours */
366 void garbage_collect_memory_pool()
370 Dmsg0(200, "garbage collect memory pool\n");
372 if (last_garbage_collection == 0) {
373 last_garbage_collection = time(NULL);
378 if (now >= last_garbage_collection + garbage_interval ||
380 last_garbage_collection = now;
382 garbage_collect_memory();
388 /* Release all freed pooled memory */
389 void close_memory_pool()
391 struct abufhead *buf, *next;
396 sm_check(__FILE__, __LINE__, false);
398 for (int i=1; i<=PM_MAX; i++) {
399 buf = pool_ctl[i].free_buf;
403 bytes += sizeof_pool_memory((char *)buf);
407 pool_ctl[i].free_buf = NULL;
409 Dmsg2(DT_MEMORY|001, "Freed mem_pool count=%d size=%s\n", count, edit_uint64_with_commas(bytes, ed1));
410 if (chk_dbglvl(DT_MEMORY|1)) {
411 print_memory_pool_stats();
418 * Garbage collect and trim memory if possible
419 * This should be called after all big memory usages
422 void garbage_collect_memory()
424 close_memory_pool(); /* release free chain */
425 #ifdef HAVE_MALLOC_TRIM
433 static const char *pool_name(int pool)
435 static const char *name[] = {"NoPool", "NAME ", "FNAME ", "MSG ", "EMSG ", "BSOCK "};
438 if (pool >= 0 && pool <= PM_MAX) {
441 sprintf(buf, "%-6d", pool);
445 /* Print staticstics on memory pool usage
447 void print_memory_pool_stats()
449 Pmsg0(-1, "Pool Maxsize Maxused Inuse\n");
450 for (int i=0; i<=PM_MAX; i++)
451 Pmsg4(-1, "%5s %7d %7d %5d\n", pool_name(i), pool_ctl[i].max_allocated,
452 pool_ctl[i].max_used, pool_ctl[i].in_use);
458 void print_memory_pool_stats() {}
462 * Concatenate a string (str) onto a pool memory buffer pm
463 * Returns: length of concatenated string
465 int pm_strcat(POOLMEM **pm, const char *str)
467 int pmlen = strlen(*pm);
472 len = strlen(str) + 1;
473 *pm = check_pool_memory_size(*pm, pmlen + len);
474 memcpy(*pm+pmlen, str, len);
475 return pmlen + len - 1;
478 int pm_strcat(POOLMEM *&pm, const char *str)
480 int pmlen = strlen(pm);
485 len = strlen(str) + 1;
486 pm = check_pool_memory_size(pm, pmlen + len);
487 memcpy(pm+pmlen, str, len);
488 return pmlen + len - 1;
491 int pm_strcat(POOLMEM *&pm, POOL_MEM &str)
493 int pmlen = strlen(pm);
494 int len = strlen(str.c_str()) + 1;
496 pm = check_pool_memory_size(pm, pmlen + len);
497 memcpy(pm+pmlen, str.c_str(), len);
498 return pmlen + len - 1;
501 int pm_strcat(POOL_MEM &pm, const char *str)
503 int pmlen = strlen(pm.c_str());
508 len = strlen(str) + 1;
509 pm.check_size(pmlen + len);
510 memcpy(pm.c_str()+pmlen, str, len);
511 return pmlen + len - 1;
514 int pm_strcat(POOL_MEM &pm, POOL_MEM &str)
516 int pmlen = strlen(pm.c_str());
519 len = strlen(str.c_str()) + 1;
520 pm.check_size(pmlen + len);
521 memcpy(pm.c_str()+pmlen, str.c_str(), len);
522 return pmlen + len - 1;
526 * Copy a string (str) into a pool memory buffer pm
527 * Returns: length of string copied
529 int pm_strcpy(POOLMEM **pm, const char *str)
535 len = strlen(str) + 1;
536 *pm = check_pool_memory_size(*pm, len);
537 memcpy(*pm, str, len);
541 int pm_strcpy(POOLMEM *&pm, const char *str)
547 len = strlen(str) + 1;
548 pm = check_pool_memory_size(pm, len);
549 memcpy(pm, str, len);
553 int pm_strcpy(POOLMEM *&pm, POOL_MEM &str)
555 int len = strlen(str.c_str()) + 1;
557 pm = check_pool_memory_size(pm, len);
558 memcpy(pm, str.c_str(), len);
562 int pm_strcpy(POOL_MEM &pm, const char *str)
568 len = strlen(str) + 1;
570 memcpy(pm.c_str(), str, len);
575 * Copy data into a pool memory buffer pm
576 * Returns: length of data copied
578 int pm_memcpy(POOLMEM **pm, const char *data, int32_t n)
580 *pm = check_pool_memory_size(*pm, n);
581 memcpy(*pm, data, n);
585 int pm_memcpy(POOLMEM *&pm, const char *data, int32_t n)
587 pm = check_pool_memory_size(pm, n);
592 int pm_memcpy(POOLMEM *&pm, POOL_MEM &data, int32_t n)
594 pm = check_pool_memory_size(pm, n);
595 memcpy(pm, data.c_str(), n);
599 int pm_memcpy(POOL_MEM &pm, const char *data, int32_t n)
602 memcpy(pm.c_str(), data, n);
606 /* ============== CLASS POOL_MEM ============== */
608 /* Return the size of a memory buffer */
609 int32_t POOL_MEM::max_size()
614 size = ((struct abufhead *)cp)->ablen;
615 Dmsg1(900, "max_size=%d\n", size);
619 void POOL_MEM::realloc_pm(int32_t size)
627 buf = (char *)realloc(cp, size+HEAD_SIZE);
630 Emsg1(M_ABORT, 0, _("Out of memory requesting %d bytes\n"), size);
632 Dmsg2(900, "Old buf=%p new buf=%p\n", cp, buf);
633 ((struct abufhead *)buf)->ablen = size;
634 pool = ((struct abufhead *)buf)->pool;
635 if (size > pool_ctl[pool].max_allocated) {
636 pool_ctl[pool].max_allocated = size;
640 Dmsg3(900, "Old buf=%p new buf=%p mem=%p\n", cp, buf, mem);
643 int POOL_MEM::strcat(const char *str)
645 int pmlen = strlen(mem);
650 len = strlen(str) + 1;
651 check_size(pmlen + len);
652 memcpy(mem+pmlen, str, len);
653 return pmlen + len - 1;
656 int POOL_MEM::strcpy(const char *str)
662 len = strlen(str) + 1;
664 memcpy(mem, str, len);