1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2009 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
24 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
26 static void print_slheap(int level, void *ctx);
29 /* Destroy the context, or if key==NULL clean it up for reuse. */
36 struct slab_heap *sh = data;
37 int pad = 2*sizeof(int)-1, pad_shift;
38 int order_start = -1, i;
39 struct slab_object *so;
45 } while (pad_shift >>= 1);
47 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
48 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
50 struct slab_object *so_tmp = so;
51 so = LDAP_LIST_NEXT(so, so_link);
52 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
54 ch_free(sh->sh_map[i]);
59 so = LDAP_LIST_FIRST(&sh->sh_sopool);
61 struct slab_object *so_tmp = so;
62 so = LDAP_LIST_NEXT(so, so_link);
63 if (!so_tmp->so_blockhead) {
64 LDAP_LIST_REMOVE(so_tmp, so_link);
67 so = LDAP_LIST_FIRST(&sh->sh_sopool);
69 struct slab_object *so_tmp = so;
70 so = LDAP_LIST_NEXT(so, so_link);
76 ber_memfree_x(sh->sh_base, NULL);
77 ber_memfree_x(sh, NULL);
81 BerMemoryFunctions slap_sl_mfuncs =
82 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
87 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
91 static struct slab_heap *slheap;
94 /* This allocator always returns memory aligned on a 2-int boundary.
96 * The stack-based allocator stores the size as a ber_len_t at both
97 * the head and tail of the allocated block. When freeing a block, the
98 * tail length is ORed with 1 to mark it as free. Freed space can only
99 * be reclaimed from the tail forward. If the tail block is never freed,
100 * nothing else will be reclaimed until the slab is reset...
110 struct slab_heap *sh;
111 ber_len_t size_shift;
112 int pad = 2*sizeof(int)-1, pad_shift;
113 int order = -1, order_start = -1, order_end = -1;
115 struct slab_object *so;
121 ldap_pvt_thread_pool_getkey(
122 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL );
129 /* round up to doubleword boundary */
134 sh = ch_malloc(sizeof(struct slab_heap));
135 sh->sh_base = ch_malloc(size);
139 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
140 (void *)sh, slap_sl_mem_destroy, NULL, NULL);
143 slap_sl_mem_destroy(NULL, sh);
144 if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
147 newptr = ch_realloc( sh->sh_base, size );
148 if ( newptr == NULL ) return NULL;
149 sh->sh_base = newptr;
152 sh->sh_end = (char *) sh->sh_base + size;
154 sh->sh_stack = stack;
156 /* insert dummy len */
158 ber_len_t *i = sh->sh_base;
163 size_shift = size - 1;
166 } while (size_shift >>= 1);
171 } while (pad_shift >>= 1);
172 order = order_end - order_start + 1;
173 sh->sh_maxorder = order_end;
175 sh->sh_free = (struct sh_freelist *)
176 ch_malloc(order * sizeof(struct sh_freelist));
177 for (i = 0; i < order; i++) {
178 LDAP_LIST_INIT(&sh->sh_free[i]);
181 LDAP_LIST_INIT(&sh->sh_sopool);
183 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
184 slap_replenish_sopool(sh);
186 so = LDAP_LIST_FIRST(&sh->sh_sopool);
187 LDAP_LIST_REMOVE(so, so_link);
188 so->so_ptr = sh->sh_base;
190 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
192 sh->sh_map = (unsigned char **)
193 ch_malloc(order * sizeof(unsigned char *));
194 for (i = 0; i < order; i++) {
195 int shiftamt = order_start + 1 + i;
196 int nummaps = size >> shiftamt;
199 if (!nummaps) nummaps = 1;
200 sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
201 memset(sh->sh_map[i], 0, nummaps);
216 /* separate from context */
217 ldap_pvt_thread_pool_setkey( ctx, (void *)slap_sl_mem_init,
218 NULL, 0, NULL, NULL );
228 struct slab_heap *sh = ctx;
229 int pad = 2*sizeof(int)-1, pad_shift;
230 ber_len_t *ptr, *newptr;
232 #ifdef SLAP_NO_SL_MALLOC
233 newptr = ber_memalloc_x( size, NULL );
234 if ( newptr ) return newptr;
236 exit( EXIT_FAILURE );
239 /* ber_set_option calls us like this */
241 newptr = ber_memalloc_x( size, NULL );
242 if ( newptr ) return newptr;
244 exit( EXIT_FAILURE );
247 /* round up to doubleword boundary, plus space for len at head and tail */
248 size += 2*sizeof(ber_len_t) + pad;
252 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
253 Debug(LDAP_DEBUG_TRACE,
254 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
256 return ch_malloc(size);
258 newptr = sh->sh_last;
259 sh->sh_last = (char *) sh->sh_last + size;
260 size -= sizeof(ber_len_t);
262 *(ber_len_t *)((char *)sh->sh_last - sizeof(ber_len_t)) = size;
263 return( (void *)newptr );
265 struct slab_object *so_new, *so_left, *so_right;
266 ber_len_t size_shift;
267 int order = -1, order_start = -1;
271 size_shift = size - 1;
274 } while (size_shift >>= 1);
279 } while (pad_shift >>= 1);
281 for (i = order; i <= sh->sh_maxorder &&
282 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
285 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
286 LDAP_LIST_REMOVE(so_new, so_link);
287 ptr = so_new->so_ptr;
288 diff = (unsigned long)((char*)ptr -
289 (char*)sh->sh_base) >> (order + 1);
290 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
291 *ptr++ = size - sizeof(ber_len_t);
292 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
294 } else if (i <= sh->sh_maxorder) {
295 for (j = i; j > order; j--) {
296 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
297 LDAP_LIST_REMOVE(so_left, so_link);
298 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
299 slap_replenish_sopool(sh);
301 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
302 LDAP_LIST_REMOVE(so_right, so_link);
303 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
304 if (j == order + 1) {
305 ptr = so_left->so_ptr;
306 diff = (unsigned long)((char*)ptr -
307 (char*)sh->sh_base) >> (order+1);
308 sh->sh_map[order-order_start][diff>>3] |=
310 *ptr++ = size - sizeof(ber_len_t);
311 LDAP_LIST_INSERT_HEAD(
312 &sh->sh_free[j-1-order_start], so_right, so_link);
313 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
316 LDAP_LIST_INSERT_HEAD(
317 &sh->sh_free[j-1-order_start], so_right, so_link);
318 LDAP_LIST_INSERT_HEAD(
319 &sh->sh_free[j-1-order_start], so_left, so_link);
323 Debug( LDAP_DEBUG_TRACE,
324 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
326 return (void*)ch_malloc(size);
330 /* FIXME: missing return; guessing... */
335 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
339 newptr = slap_sl_malloc( n*size, ctx );
341 memset( newptr, 0, n*size );
347 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
349 struct slab_heap *sh = ctx;
350 int pad = 2*sizeof(int) -1;
351 ber_len_t *p = (ber_len_t *)ptr, *newptr;
354 return slap_sl_malloc(size, ctx);
356 #ifdef SLAP_NO_SL_MALLOC
357 newptr = ber_memrealloc_x( ptr, size, NULL );
358 if ( newptr ) return newptr;
360 exit( EXIT_FAILURE );
363 /* Not our memory? */
364 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
365 /* duplicate of realloc behavior, oh well */
366 newptr = ber_memrealloc_x(ptr, size, NULL);
370 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
373 exit( EXIT_FAILURE );
377 slap_sl_free(ptr, ctx);
382 /* round up to doubleword boundary */
383 size += pad + sizeof( ber_len_t );
388 /* Never shrink blocks */
392 /* If reallocing the last block, we can grow it */
393 } else if ((char *)ptr + p[0] == sh->sh_last &&
394 (char *)ptr + size < (char *)sh->sh_end ) {
396 sh->sh_last = (char *)ptr + size;
398 p[size/sizeof(ber_len_t)] = size;
400 /* Nowhere to grow, need to alloc and copy */
402 newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
403 AC_MEMCPY(newptr, ptr, p[0]-sizeof(ber_len_t));
404 /* mark old region as free */
405 p[p[0]/sizeof(ber_len_t)] |= 1;
411 newptr2 = slap_sl_malloc(size, ctx);
413 AC_MEMCPY(newptr2, ptr, size);
415 AC_MEMCPY(newptr2, ptr, p[-1]);
417 slap_sl_free(ptr, ctx);
423 slap_sl_free(void *ptr, void *ctx)
425 struct slab_heap *sh = ctx;
427 ber_len_t *p = (ber_len_t *)ptr, *tmpp;
432 #ifdef SLAP_NO_SL_MALLOC
433 ber_memfree_x( ptr, NULL );
437 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
438 ber_memfree_x(ptr, NULL);
439 } else if (sh->sh_stack) {
440 tmpp = (ber_len_t *)((char *)ptr + p[-1]);
443 /* reclaim free space off tail */
444 while ( tmpp == sh->sh_last ) {
445 if ( tmpp[-1] & 1 ) {
447 ptr = (char *)tmpp - size;
448 p = (ber_len_t *)ptr;
457 int size_shift, order_size;
458 int pad = 2*sizeof(int)-1, pad_shift;
459 int order_start = -1, order = -1;
460 struct slab_object *so;
465 size_shift = size + sizeof(ber_len_t) - 1;
468 } while (size_shift >>= 1);
473 } while (pad_shift >>= 1);
475 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
476 order_size = 1 << (i+1);
477 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
478 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
479 if (diff == ((diff>>1)<<1)) {
480 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
481 (1<<((diff+1)&0x7)))) {
482 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
484 if ((char*)so->so_ptr == (char*)tmpp) {
485 LDAP_LIST_REMOVE( so, so_link );
486 } else if ((char*)so->so_ptr ==
487 (char*)tmpp + order_size) {
488 LDAP_LIST_REMOVE(so, so_link);
491 so = LDAP_LIST_NEXT(so, so_link);
494 if (i < sh->sh_maxorder) {
497 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
502 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
503 slap_replenish_sopool(sh);
505 so = LDAP_LIST_FIRST(&sh->sh_sopool);
506 LDAP_LIST_REMOVE(so, so_link);
508 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
512 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
513 "free object not found while bit is clear.\n",
520 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
521 slap_replenish_sopool(sh);
523 so = LDAP_LIST_FIRST(&sh->sh_sopool);
524 LDAP_LIST_REMOVE(so, so_link);
526 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
532 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
533 (1<<((diff-1)&0x7)))) {
534 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
536 if ((char*)so->so_ptr == (char*)tmpp) {
537 LDAP_LIST_REMOVE(so, so_link);
538 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
539 LDAP_LIST_REMOVE(so, so_link);
543 so = LDAP_LIST_NEXT(so, so_link);
546 if (i < sh->sh_maxorder) {
548 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
552 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
553 slap_replenish_sopool(sh);
555 so = LDAP_LIST_FIRST(&sh->sh_sopool);
556 LDAP_LIST_REMOVE(so, so_link);
558 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
562 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
563 "free object not found while bit is clear.\n",
570 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
571 slap_replenish_sopool(sh);
573 so = LDAP_LIST_FIRST(&sh->sh_sopool);
574 LDAP_LIST_REMOVE(so, so_link);
576 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
587 slap_sl_context( void *ptr )
589 struct slab_heap *sh;
592 if ( slapMode & SLAP_TOOL_MODE ) return NULL;
597 ctx = ldap_pvt_thread_pool_context();
600 ldap_pvt_thread_pool_getkey(
601 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL);
605 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
611 static struct slab_object *
612 slap_replenish_sopool(
616 struct slab_object *so_block;
619 so_block = (struct slab_object *)ch_malloc(
620 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
622 if ( so_block == NULL ) {
626 so_block[0].so_blockhead = 1;
627 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
628 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
629 so_block[i].so_blockhead = 0;
630 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
638 print_slheap(int level, void *ctx)
640 struct slab_heap *sh = ctx;
641 int order_start = -1;
642 int pad = 2*sizeof(int)-1, pad_shift;
643 struct slab_object *so;
647 Debug(level, "NULL memctx\n", 0, 0, 0);
654 } while (pad_shift >>= 1);
656 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
658 for (i = order_start; i <= sh->sh_maxorder; i++) {
660 Debug(level, "order=%d\n", i, 0, 0);
661 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
662 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
666 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
668 Debug(level, "\n", 0, 0, 0);
669 Debug(level, "free list:\n", 0, 0, 0);
670 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
672 Debug(level, "%lx\n", (unsigned long) so->so_ptr, 0, 0);
673 so = LDAP_LIST_NEXT(so, so_link);