1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2017 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
25 /* Get debugging help from Valgrind */
26 #include <valgrind/memcheck.h>
27 #define VGMEMP_MARK(m,s) VALGRIND_MAKE_MEM_NOACCESS(m,s)
28 #define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z)
29 #define VGMEMP_TRIM(h,a,s) VALGRIND_MEMPOOL_TRIM(h,a,s)
30 #define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s)
31 #define VGMEMP_CHANGE(h,a,b,s) VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
33 #define VGMEMP_MARK(m,s)
34 #define VGMEMP_CREATE(h,r,z)
35 #define VGMEMP_TRIM(h,a,s)
36 #define VGMEMP_ALLOC(h,a,s)
37 #define VGMEMP_CHANGE(h,a,b,s)
41 * This allocator returns temporary memory from a slab in a given memory
42 * context, aligned on a 2-int boundary. It cannot be used for data
43 * which will outlive the task allocating it.
45 * A new memory context attaches to the creator's thread context, if any.
46 * Threads cannot use other threads' memory contexts; there are no locks.
48 * The caller of slap_sl_malloc, usually a thread pool task, must
49 * slap_sl_free the memory before finishing: New tasks reuse the context
50 * and normally reset it, reclaiming memory left over from last task.
52 * The allocator helps memory fragmentation, speed and memory leaks.
53 * It is not (yet) reliable as a garbage collector:
55 * It falls back to context NULL - plain ber_memalloc() - when the
56 * context's slab is full. A reset does not reclaim such memory.
57 * Conversely, free/realloc of data not from the given context assumes
58 * context NULL. The data must not belong to another memory context.
60 * Code which has lost track of the current memory context can try
61 * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
63 * Allocations cannot yet return failure. Like ch_malloc, they succeed
64 * or abort slapd. This will change, do fix code which assumes success.
68 * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
69 * allocated blocks' head - and in freed blocks also at the tail, marked
70 * by ORing *next* block's head with 1. Freed blocks are only reclaimed
71 * from the last block forward. This is fast, but when a block is never
72 * freed, older blocks will not be reclaimed until the slab is reset...
75 #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
76 enum { No_sl_malloc = 1 };
78 enum { No_sl_malloc = 0 };
81 #define SLAP_SLAB_SOBLOCK 64
86 LDAP_LIST_ENTRY(slab_object) so_link;
95 unsigned char **sh_map;
96 LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
97 LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
101 Align = sizeof(ber_len_t) > 2*sizeof(int)
102 ? sizeof(ber_len_t) : 2*sizeof(int),
103 Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
104 order_start = Align_log2 - 1,
108 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
110 static void print_slheap(int level, void *ctx);
113 /* Keep memory context in a thread-local var, or in a global when no threads */
115 static struct slab_heap *slheap;
116 # define SET_MEMCTX(thrctx, memctx, sfree) ((void) (slheap = (memctx)))
117 # define GET_MEMCTX(thrctx, memctxp) (*(memctxp) = slheap)
119 # define memctx_key ((void *) slap_sl_mem_init)
120 # define SET_MEMCTX(thrctx, memctx, kfree) \
121 ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
122 # define GET_MEMCTX(thrctx, memctxp) \
123 ((void) (*(memctxp) = NULL), \
124 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
126 #endif /* NO_THREADS */
129 /* Destroy the context, or if key==NULL clean it up for reuse. */
136 struct slab_heap *sh = data;
137 struct slab_object *so;
144 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
145 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
147 struct slab_object *so_tmp = so;
148 so = LDAP_LIST_NEXT(so, so_link);
149 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
151 ch_free(sh->sh_map[i]);
153 ch_free(sh->sh_free);
156 so = LDAP_LIST_FIRST(&sh->sh_sopool);
158 struct slab_object *so_tmp = so;
159 so = LDAP_LIST_NEXT(so, so_link);
160 if (!so_tmp->so_blockhead) {
161 LDAP_LIST_REMOVE(so_tmp, so_link);
164 so = LDAP_LIST_FIRST(&sh->sh_sopool);
166 struct slab_object *so_tmp = so;
167 so = LDAP_LIST_NEXT(so, so_link);
173 ber_memfree_x(sh->sh_base, NULL);
174 ber_memfree_x(sh, NULL);
178 BerMemoryFunctions slap_sl_mfuncs =
179 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
184 assert( Align == 1 << Align_log2 );
186 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
189 /* Create, reset or just return the memory context of the current thread. */
199 struct slab_heap *sh;
200 ber_len_t size_shift;
201 struct slab_object *so;
203 enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
205 sh = GET_MEMCTX(thrctx, &memctx);
209 /* Round up to doubleword boundary, then make room for initial
210 * padding, preserving expected available size for pool version */
211 size = ((size + Align-1) & -Align) + Base_offset;
214 sh = ch_malloc(sizeof(struct slab_heap));
215 base = ch_malloc(size);
216 SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
217 VGMEMP_MARK(base, size);
218 VGMEMP_CREATE(sh, 0, 0);
220 slap_sl_mem_destroy(NULL, sh);
222 if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
223 newptr = ch_realloc(base, size);
224 if ( newptr == NULL ) return NULL;
225 VGMEMP_CHANGE(sh, base, newptr, size);
228 VGMEMP_TRIM(sh, base, 0);
231 sh->sh_end = base + size;
233 /* Align (base + head of first block) == first returned block */
237 sh->sh_stack = stack;
242 int i, order = -1, order_end = -1;
244 size_shift = size - 1;
247 } while (size_shift >>= 1);
248 order = order_end - order_start + 1;
249 sh->sh_maxorder = order_end;
251 sh->sh_free = (struct sh_freelist *)
252 ch_malloc(order * sizeof(struct sh_freelist));
253 for (i = 0; i < order; i++) {
254 LDAP_LIST_INIT(&sh->sh_free[i]);
257 LDAP_LIST_INIT(&sh->sh_sopool);
259 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
260 slap_replenish_sopool(sh);
262 so = LDAP_LIST_FIRST(&sh->sh_sopool);
263 LDAP_LIST_REMOVE(so, so_link);
266 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
268 sh->sh_map = (unsigned char **)
269 ch_malloc(order * sizeof(unsigned char *));
270 for (i = 0; i < order; i++) {
271 int shiftamt = order_start + 1 + i;
272 int nummaps = size >> shiftamt;
275 if (!nummaps) nummaps = 1;
276 sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
277 memset(sh->sh_map[i], 0, nummaps);
285 * Assign memory context to thread context. Use NULL to detach
286 * current memory context from thread. Future users must
287 * know the context, since ch_free/slap_sl_context() cannot find it.
295 SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
304 struct slab_heap *sh = ctx;
305 ber_len_t *ptr, *newptr;
307 /* ber_set_option calls us like this */
308 if (No_sl_malloc || !ctx) {
309 newptr = ber_memalloc_x( size, NULL );
310 if ( newptr ) return newptr;
311 Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
312 (unsigned long) size, 0, 0);
314 exit( EXIT_FAILURE );
317 /* Add room for head, ensure room for tail when freed, and
318 * round up to doubleword boundary. */
319 size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
322 if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
323 newptr = sh->sh_last;
324 sh->sh_last = (char *) sh->sh_last + size;
325 VGMEMP_ALLOC(sh, newptr, size);
327 return( (void *)newptr );
330 size -= sizeof(ber_len_t);
333 struct slab_object *so_new, *so_left, *so_right;
334 ber_len_t size_shift;
336 int i, j, order = -1;
338 size_shift = size - 1;
341 } while (size_shift >>= 1);
343 size -= sizeof(ber_len_t);
345 for (i = order; i <= sh->sh_maxorder &&
346 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
349 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
350 LDAP_LIST_REMOVE(so_new, so_link);
351 ptr = so_new->so_ptr;
352 diff = (unsigned long)((char*)ptr -
353 (char*)sh->sh_base) >> (order + 1);
354 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
356 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
358 } else if (i <= sh->sh_maxorder) {
359 for (j = i; j > order; j--) {
360 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
361 LDAP_LIST_REMOVE(so_left, so_link);
362 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
363 slap_replenish_sopool(sh);
365 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
366 LDAP_LIST_REMOVE(so_right, so_link);
367 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
368 if (j == order + 1) {
369 ptr = so_left->so_ptr;
370 diff = (unsigned long)((char*)ptr -
371 (char*)sh->sh_base) >> (order+1);
372 sh->sh_map[order-order_start][diff>>3] |=
375 LDAP_LIST_INSERT_HEAD(
376 &sh->sh_free[j-1-order_start], so_right, so_link);
377 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
380 LDAP_LIST_INSERT_HEAD(
381 &sh->sh_free[j-1-order_start], so_right, so_link);
382 LDAP_LIST_INSERT_HEAD(
383 &sh->sh_free[j-1-order_start], so_left, so_link);
387 /* FIXME: missing return; guessing we failed... */
390 Debug(LDAP_DEBUG_TRACE,
391 "sl_malloc %lu: ch_malloc\n",
392 (unsigned long) size, 0, 0);
393 return ch_malloc(size);
396 #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
397 ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
398 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
401 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
404 ber_len_t total = n * size;
406 /* The sqrt test is a slight optimization: often avoids the division */
407 if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
408 newptr = slap_sl_malloc( total, ctx );
409 memset( newptr, 0, n*size );
411 Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
412 (unsigned long) n, (unsigned long) size, 0);
420 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
422 struct slab_heap *sh = ctx;
423 ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
427 return slap_sl_malloc(size, ctx);
429 /* Not our memory? */
430 if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
431 /* Like ch_realloc(), except not trying a new context */
432 newptr = ber_memrealloc_x(ptr, size, NULL);
436 Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
437 (unsigned long) size, 0, 0);
439 exit( EXIT_FAILURE );
443 slap_sl_free(ptr, ctx);
450 /* Add room for head, round up to doubleword boundary */
451 size = (size + sizeof(ber_len_t) + Align-1) & -Align;
455 /* Never shrink blocks */
456 if (size <= oldsize) {
461 nextp = (ber_len_t *) ((char *) p + oldsize);
463 /* If reallocing the last block, try to grow it */
464 if (nextp == sh->sh_last) {
465 if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
466 sh->sh_last = (char *) p + size;
467 p[0] = (p[0] & 1) | size;
471 /* Nowhere to grow, need to alloc and copy */
473 /* Slight optimization of the final realloc variant */
474 newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
475 AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
476 /* Not last block, can just mark old region as free */
482 size -= sizeof(ber_len_t);
483 oldsize -= sizeof(ber_len_t);
485 } else if (oldsize > size) {
489 newptr = slap_sl_malloc(size, ctx);
490 AC_MEMCPY(newptr, ptr, oldsize);
491 slap_sl_free(ptr, ctx);
496 slap_sl_free(void *ptr, void *ctx)
498 struct slab_heap *sh = ctx;
500 ber_len_t *p = ptr, *nextp, *tmpp;
505 if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
506 ber_memfree_x(ptr, NULL);
514 nextp = (ber_len_t *) ((char *) p + size);
515 if (sh->sh_last != nextp) {
516 /* Mark it free: tail = size, head of next block |= 1 */
519 /* We can't tell Valgrind about it yet, because we
520 * still need read/write access to this block for
521 * when we eventually get to reclaim it.
524 /* Reclaim freed block(s) off tail */
526 p = (ber_len_t *) ((char *) p - p[-1]);
529 VGMEMP_TRIM(sh, sh->sh_base,
530 (char *) sh->sh_last - (char *) sh->sh_base);
534 int size_shift, order_size;
535 struct slab_object *so;
537 int i, inserted = 0, order = -1;
539 size_shift = size + sizeof(ber_len_t) - 1;
542 } while (size_shift >>= 1);
544 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
545 order_size = 1 << (i+1);
546 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
547 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
548 if (diff == ((diff>>1)<<1)) {
549 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
550 (1<<((diff+1)&0x7)))) {
551 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
553 if ((char*)so->so_ptr == (char*)tmpp) {
554 LDAP_LIST_REMOVE( so, so_link );
555 } else if ((char*)so->so_ptr ==
556 (char*)tmpp + order_size) {
557 LDAP_LIST_REMOVE(so, so_link);
560 so = LDAP_LIST_NEXT(so, so_link);
563 if (i < sh->sh_maxorder) {
566 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
571 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
572 slap_replenish_sopool(sh);
574 so = LDAP_LIST_FIRST(&sh->sh_sopool);
575 LDAP_LIST_REMOVE(so, so_link);
577 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
581 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
582 "free object not found while bit is clear.\n",
589 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
590 slap_replenish_sopool(sh);
592 so = LDAP_LIST_FIRST(&sh->sh_sopool);
593 LDAP_LIST_REMOVE(so, so_link);
595 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
601 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
602 (1<<((diff-1)&0x7)))) {
603 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
605 if ((char*)so->so_ptr == (char*)tmpp) {
606 LDAP_LIST_REMOVE(so, so_link);
607 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
608 LDAP_LIST_REMOVE(so, so_link);
612 so = LDAP_LIST_NEXT(so, so_link);
615 if (i < sh->sh_maxorder) {
617 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
621 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
622 slap_replenish_sopool(sh);
624 so = LDAP_LIST_FIRST(&sh->sh_sopool);
625 LDAP_LIST_REMOVE(so, so_link);
627 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
631 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
632 "free object not found while bit is clear.\n",
639 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
640 slap_replenish_sopool(sh);
642 so = LDAP_LIST_FIRST(&sh->sh_sopool);
643 LDAP_LIST_REMOVE(so, so_link);
645 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
656 slap_sl_release( void *ptr, void *ctx )
658 struct slab_heap *sh = ctx;
659 if ( sh && ptr >= sh->sh_base && ptr <= sh->sh_end )
664 slap_sl_mark( void *ctx )
666 struct slab_heap *sh = ctx;
671 * Return the memory context of the current thread if the given block of
672 * memory belongs to it, otherwise return NULL.
675 slap_sl_context( void *ptr )
678 struct slab_heap *sh;
680 if ( slapMode & SLAP_TOOL_MODE ) return NULL;
682 sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
683 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
689 static struct slab_object *
690 slap_replenish_sopool(
694 struct slab_object *so_block;
697 so_block = (struct slab_object *)ch_malloc(
698 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
700 if ( so_block == NULL ) {
704 so_block[0].so_blockhead = 1;
705 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
706 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
707 so_block[i].so_blockhead = 0;
708 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
716 print_slheap(int level, void *ctx)
718 struct slab_heap *sh = ctx;
719 struct slab_object *so;
723 Debug(level, "NULL memctx\n", 0, 0, 0);
727 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
729 for (i = order_start; i <= sh->sh_maxorder; i++) {
731 Debug(level, "order=%d\n", i, 0, 0);
732 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
733 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
737 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
739 Debug(level, "\n", 0, 0, 0);
740 Debug(level, "free list:\n", 0, 0, 0);
741 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
743 Debug(level, "%p\n", so->so_ptr, 0, 0);
744 so = LDAP_LIST_NEXT(so, so_link);