1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2010 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
25 * This allocator returns temporary memory from a slab in a given memory
26 * context, aligned on a 2-int boundary. It cannot be used for data
27 * which will outlive the task allocating it.
29 * A new memory context attaches to the creator's thread context, if any.
30 * Threads cannot use other threads' memory contexts; there are no locks.
32 * The caller of slap_sl_malloc, usually a thread pool task, must
33 * slap_sl_free the memory before finishing: New tasks reuse the context
34 * and normally reset it, reclaiming memory left over from last task.
36 * The allocator helps memory fragmentation, speed and memory leaks.
37 * It is not (yet) reliable as a garbage collector:
39 * It falls back to context NULL - plain ber_memalloc() - when the
40 * context's slab is full. A reset does not reclaim such memory.
41 * Conversely, free/realloc of data not from the given context assumes
42 * context NULL. The data must not belong to another memory context.
44 * Code which has lost track of the current memory context can try
45 * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
47 * Allocations cannot yet return failure. Like ch_malloc, they succeed
48 * or abort slapd. This will change, do fix code which assumes success.
52 * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
53 * the head and tail of each allocated block. The tail length of a freed
54 * block is ORed with 1 to mark it free. Freed blocks are only reclaimed
55 * from the last block forward. This is fast, but when a block is never
56 * freed, older blocks will not be reclaimed until the slab is reset...
60 Align = sizeof(ber_len_t) > 2*sizeof(int)
61 ? sizeof(ber_len_t) : 2*sizeof(int),
62 Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
63 order_start = Align_log2 - 1,
67 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
69 static void print_slheap(int level, void *ctx);
72 /* Keep memory context in a thread-local var, or in a global when no threads */
74 static struct slab_heap *slheap;
75 # define SET_MEMCTX(thrctx, memctx, sfree) ((void) (slheap = (memctx)))
76 # define GET_MEMCTX(thrctx, memctxp) (*(memctxp) = slheap))
78 # define memctx_key ((void *) slap_sl_mem_init)
79 # define SET_MEMCTX(thrctx, memctx, kfree) \
80 ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
81 # define GET_MEMCTX(thrctx, memctxp) \
82 ((void) (*(memctxp) = NULL), \
83 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
85 #endif /* NO_THREADS */
88 /* Destroy the context, or if key==NULL clean it up for reuse. */
95 struct slab_heap *sh = data;
96 struct slab_object *so;
100 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
101 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
103 struct slab_object *so_tmp = so;
104 so = LDAP_LIST_NEXT(so, so_link);
105 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
107 ch_free(sh->sh_map[i]);
109 ch_free(sh->sh_free);
112 so = LDAP_LIST_FIRST(&sh->sh_sopool);
114 struct slab_object *so_tmp = so;
115 so = LDAP_LIST_NEXT(so, so_link);
116 if (!so_tmp->so_blockhead) {
117 LDAP_LIST_REMOVE(so_tmp, so_link);
120 so = LDAP_LIST_FIRST(&sh->sh_sopool);
122 struct slab_object *so_tmp = so;
123 so = LDAP_LIST_NEXT(so, so_link);
129 ber_memfree_x(sh->sh_base, NULL);
130 ber_memfree_x(sh, NULL);
134 BerMemoryFunctions slap_sl_mfuncs =
135 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
140 assert( Align == 1 << Align_log2 );
141 /* Adding head+tail preserves alignment */
142 assert( 2*sizeof(ber_len_t) % Align == 0 );
144 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
147 /* Create, reset or just return the memory context of the current thread. */
157 struct slab_heap *sh;
158 ber_len_t size_shift;
159 struct slab_object *so;
161 sh = GET_MEMCTX(thrctx, &memctx);
165 /* round up to doubleword boundary */
166 size = (size + Align-1) & -Align;
169 sh = ch_malloc(sizeof(struct slab_heap));
170 sh->sh_base = ch_malloc(size);
171 SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
173 slap_sl_mem_destroy(NULL, sh);
174 if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
177 newptr = ch_realloc( sh->sh_base, size );
178 if ( newptr == NULL ) return NULL;
179 sh->sh_base = newptr;
182 sh->sh_end = (char *) sh->sh_base + size;
184 sh->sh_stack = stack;
186 /* insert dummy len */
188 ber_len_t *i = sh->sh_base;
193 int i, order = -1, order_end = -1;
195 size_shift = size - 1;
198 } while (size_shift >>= 1);
199 order = order_end - order_start + 1;
200 sh->sh_maxorder = order_end;
202 sh->sh_free = (struct sh_freelist *)
203 ch_malloc(order * sizeof(struct sh_freelist));
204 for (i = 0; i < order; i++) {
205 LDAP_LIST_INIT(&sh->sh_free[i]);
208 LDAP_LIST_INIT(&sh->sh_sopool);
210 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
211 slap_replenish_sopool(sh);
213 so = LDAP_LIST_FIRST(&sh->sh_sopool);
214 LDAP_LIST_REMOVE(so, so_link);
215 so->so_ptr = sh->sh_base;
217 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
219 sh->sh_map = (unsigned char **)
220 ch_malloc(order * sizeof(unsigned char *));
221 for (i = 0; i < order; i++) {
222 int shiftamt = order_start + 1 + i;
223 int nummaps = size >> shiftamt;
226 if (!nummaps) nummaps = 1;
227 sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
228 memset(sh->sh_map[i], 0, nummaps);
235 * Separate memory context from thread context. Future users must
236 * know the context, since ch_free/slap_sl_context() cannot find it.
244 SET_MEMCTX(thrctx, NULL, 0);
253 struct slab_heap *sh = ctx;
254 ber_len_t *ptr, *newptr;
256 #ifdef SLAP_NO_SL_MALLOC
257 newptr = ber_memalloc_x( size, NULL );
258 if ( newptr ) return newptr;
260 exit( EXIT_FAILURE );
263 /* ber_set_option calls us like this */
265 newptr = ber_memalloc_x( size, NULL );
266 if ( newptr ) return newptr;
267 Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
268 (unsigned long) size, 0, 0);
270 exit( EXIT_FAILURE );
273 /* round up to doubleword boundary, plus space for len at head and tail */
274 size = (size + 2*sizeof(ber_len_t) + Align-1) & -Align;
277 if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
278 newptr = sh->sh_last;
279 sh->sh_last = (char *) sh->sh_last + size;
280 size -= sizeof(ber_len_t);
282 ((ber_len_t *) sh->sh_last)[-1] = size;
283 return( (void *)newptr );
286 size -= 2*sizeof(ber_len_t);
289 struct slab_object *so_new, *so_left, *so_right;
290 ber_len_t size_shift;
292 int i, j, order = -1;
294 size_shift = size - 1;
297 } while (size_shift >>= 1);
299 size -= sizeof(ber_len_t);
301 for (i = order; i <= sh->sh_maxorder &&
302 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
305 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
306 LDAP_LIST_REMOVE(so_new, so_link);
307 ptr = so_new->so_ptr;
308 diff = (unsigned long)((char*)ptr -
309 (char*)sh->sh_base) >> (order + 1);
310 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
312 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
314 } else if (i <= sh->sh_maxorder) {
315 for (j = i; j > order; j--) {
316 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
317 LDAP_LIST_REMOVE(so_left, so_link);
318 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
319 slap_replenish_sopool(sh);
321 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
322 LDAP_LIST_REMOVE(so_right, so_link);
323 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
324 if (j == order + 1) {
325 ptr = so_left->so_ptr;
326 diff = (unsigned long)((char*)ptr -
327 (char*)sh->sh_base) >> (order+1);
328 sh->sh_map[order-order_start][diff>>3] |=
331 LDAP_LIST_INSERT_HEAD(
332 &sh->sh_free[j-1-order_start], so_right, so_link);
333 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
336 LDAP_LIST_INSERT_HEAD(
337 &sh->sh_free[j-1-order_start], so_right, so_link);
338 LDAP_LIST_INSERT_HEAD(
339 &sh->sh_free[j-1-order_start], so_left, so_link);
343 /* FIXME: missing return; guessing we failed... */
346 Debug(LDAP_DEBUG_TRACE,
347 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
348 (unsigned long) size, 0, 0);
349 return ch_malloc(size);
352 #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
353 ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
354 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
357 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
360 ber_len_t total = n * size;
362 /* The sqrt test is a slight optimization: often avoids the division */
363 if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
364 newptr = slap_sl_malloc( total, ctx );
365 memset( newptr, 0, n*size );
367 Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
368 (unsigned long) n, (unsigned long) size, 0);
376 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
378 struct slab_heap *sh = ctx;
379 ber_len_t oldsize, *p = (ber_len_t *) ptr;
383 return slap_sl_malloc(size, ctx);
385 #ifdef SLAP_NO_SL_MALLOC
386 newptr = ber_memrealloc_x( ptr, size, NULL );
387 if ( newptr ) return newptr;
389 exit( EXIT_FAILURE );
392 /* Not our memory? */
393 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
394 /* Like ch_realloc(), except not trying a new context */
395 newptr = ber_memrealloc_x(ptr, size, NULL);
399 Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
400 (unsigned long) size, 0, 0);
402 exit( EXIT_FAILURE );
406 slap_sl_free(ptr, ctx);
413 /* Round up to doubleword boundary, add room for head */
414 size = ((size + Align-1) & -Align) + sizeof( ber_len_t );
418 /* Never shrink blocks */
419 if (size <= oldsize) {
422 /* If reallocing the last block, try to grow it */
423 } else if ((char *) ptr + oldsize == sh->sh_last) {
424 if (size < (char *) sh->sh_end - (char *) ptr) {
425 sh->sh_last = (char *) ptr + size;
427 p[size/sizeof(ber_len_t)] = size;
431 /* Nowhere to grow, need to alloc and copy */
433 /* Slight optimization of the final realloc variant */
434 size -= sizeof(ber_len_t);
435 oldsize -= sizeof(ber_len_t);
436 newptr = slap_sl_malloc(size, ctx);
437 AC_MEMCPY(newptr, ptr, oldsize);
438 /* Not last block, can just mark old region as free */
439 p[p[0]/sizeof(ber_len_t)] |= 1;
443 size -= sizeof(ber_len_t);
444 oldsize -= sizeof(ber_len_t);
446 } else if (oldsize > size) {
450 newptr = slap_sl_malloc(size, ctx);
451 AC_MEMCPY(newptr, ptr, oldsize);
452 slap_sl_free(ptr, ctx);
457 slap_sl_free(void *ptr, void *ctx)
459 struct slab_heap *sh = ctx;
461 ber_len_t *p = (ber_len_t *)ptr, *tmpp;
466 #ifdef SLAP_NO_SL_MALLOC
467 ber_memfree_x( ptr, NULL );
471 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
472 ber_memfree_x(ptr, NULL);
474 } else if (sh->sh_stack) {
476 p = (ber_len_t *) ((char *) ptr + size);
479 /* reclaim free space off tail */
480 if (sh->sh_last == p) {
482 p = (ber_len_t *) ((char *) p - size + 1) - 1;
489 int size_shift, order_size;
490 struct slab_object *so;
492 int i, inserted = 0, order = -1;
495 size_shift = size + sizeof(ber_len_t) - 1;
498 } while (size_shift >>= 1);
500 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
501 order_size = 1 << (i+1);
502 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
503 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
504 if (diff == ((diff>>1)<<1)) {
505 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
506 (1<<((diff+1)&0x7)))) {
507 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
509 if ((char*)so->so_ptr == (char*)tmpp) {
510 LDAP_LIST_REMOVE( so, so_link );
511 } else if ((char*)so->so_ptr ==
512 (char*)tmpp + order_size) {
513 LDAP_LIST_REMOVE(so, so_link);
516 so = LDAP_LIST_NEXT(so, so_link);
519 if (i < sh->sh_maxorder) {
522 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
527 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
528 slap_replenish_sopool(sh);
530 so = LDAP_LIST_FIRST(&sh->sh_sopool);
531 LDAP_LIST_REMOVE(so, so_link);
533 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
537 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
538 "free object not found while bit is clear.\n",
545 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
546 slap_replenish_sopool(sh);
548 so = LDAP_LIST_FIRST(&sh->sh_sopool);
549 LDAP_LIST_REMOVE(so, so_link);
551 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
557 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
558 (1<<((diff-1)&0x7)))) {
559 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
561 if ((char*)so->so_ptr == (char*)tmpp) {
562 LDAP_LIST_REMOVE(so, so_link);
563 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
564 LDAP_LIST_REMOVE(so, so_link);
568 so = LDAP_LIST_NEXT(so, so_link);
571 if (i < sh->sh_maxorder) {
573 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
577 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
578 slap_replenish_sopool(sh);
580 so = LDAP_LIST_FIRST(&sh->sh_sopool);
581 LDAP_LIST_REMOVE(so, so_link);
583 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
587 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
588 "free object not found while bit is clear.\n",
595 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
596 slap_replenish_sopool(sh);
598 so = LDAP_LIST_FIRST(&sh->sh_sopool);
599 LDAP_LIST_REMOVE(so, so_link);
601 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
612 * Return the memory context of the current thread if the given block of
613 * memory belongs to it, otherwise return NULL.
616 slap_sl_context( void *ptr )
619 struct slab_heap *sh;
621 if ( slapMode & SLAP_TOOL_MODE ) return NULL;
623 sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
624 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
630 static struct slab_object *
631 slap_replenish_sopool(
635 struct slab_object *so_block;
638 so_block = (struct slab_object *)ch_malloc(
639 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
641 if ( so_block == NULL ) {
645 so_block[0].so_blockhead = 1;
646 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
647 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
648 so_block[i].so_blockhead = 0;
649 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
657 print_slheap(int level, void *ctx)
659 struct slab_heap *sh = ctx;
660 struct slab_object *so;
664 Debug(level, "NULL memctx\n", 0, 0, 0);
668 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
670 for (i = order_start; i <= sh->sh_maxorder; i++) {
672 Debug(level, "order=%d\n", i, 0, 0);
673 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
674 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
678 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
680 Debug(level, "\n", 0, 0, 0);
681 Debug(level, "free list:\n", 0, 0, 0);
682 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
684 Debug(level, "%p\n", so->so_ptr, 0, 0);
685 so = LDAP_LIST_NEXT(so, so_link);