1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2010 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
25 Align = 2 * sizeof(int),
26 Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
27 order_start = Align_log2 - 1,
31 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
33 static void print_slheap(int level, void *ctx);
36 /* Keep memory context in a thread-local var, or in a global when no threads */
38 static struct slab_heap *slheap;
39 # define SET_MEMCTX(thrctx, memctx, sfree) ((void) (slheap = (memctx)))
40 # define GET_MEMCTX(thrctx, memctxp) (*(memctxp) = slheap))
42 # define memctx_key ((void *) slap_sl_mem_init)
43 # define SET_MEMCTX(thrctx, memctx, kfree) \
44 ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
45 # define GET_MEMCTX(thrctx, memctxp) \
46 ((void) (*(memctxp) = NULL), \
47 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
49 #endif /* NO_THREADS */
52 /* Destroy the context, or if key==NULL clean it up for reuse. */
59 struct slab_heap *sh = data;
60 struct slab_object *so;
64 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
65 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
67 struct slab_object *so_tmp = so;
68 so = LDAP_LIST_NEXT(so, so_link);
69 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
71 ch_free(sh->sh_map[i]);
76 so = LDAP_LIST_FIRST(&sh->sh_sopool);
78 struct slab_object *so_tmp = so;
79 so = LDAP_LIST_NEXT(so, so_link);
80 if (!so_tmp->so_blockhead) {
81 LDAP_LIST_REMOVE(so_tmp, so_link);
84 so = LDAP_LIST_FIRST(&sh->sh_sopool);
86 struct slab_object *so_tmp = so;
87 so = LDAP_LIST_NEXT(so, so_link);
93 ber_memfree_x(sh->sh_base, NULL);
94 ber_memfree_x(sh, NULL);
98 BerMemoryFunctions slap_sl_mfuncs =
99 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
104 assert( Align == 1 << Align_log2 );
105 /* Adding head+tail preserves alignment */
106 assert( 2*sizeof(ber_len_t) % Align == 0 );
108 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
111 /* This allocator always returns memory aligned on a 2-int boundary.
113 * The stack-based allocator stores the size as a ber_len_t at both
114 * the head and tail of the allocated block. When freeing a block, the
115 * tail length is ORed with 1 to mark it as free. Freed space can only
116 * be reclaimed from the tail forward. If the tail block is never freed,
117 * nothing else will be reclaimed until the slab is reset...
128 struct slab_heap *sh;
129 ber_len_t size_shift;
130 struct slab_object *so;
132 sh = GET_MEMCTX(thrctx, &memctx);
136 /* round up to doubleword boundary */
137 size = (size + Align-1) & -Align;
140 sh = ch_malloc(sizeof(struct slab_heap));
141 sh->sh_base = ch_malloc(size);
142 SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
144 slap_sl_mem_destroy(NULL, sh);
145 if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
148 newptr = ch_realloc( sh->sh_base, size );
149 if ( newptr == NULL ) return NULL;
150 sh->sh_base = newptr;
153 sh->sh_end = (char *) sh->sh_base + size;
155 sh->sh_stack = stack;
157 /* insert dummy len */
159 ber_len_t *i = sh->sh_base;
164 int i, order = -1, order_end = -1;
166 size_shift = size - 1;
169 } while (size_shift >>= 1);
170 order = order_end - order_start + 1;
171 sh->sh_maxorder = order_end;
173 sh->sh_free = (struct sh_freelist *)
174 ch_malloc(order * sizeof(struct sh_freelist));
175 for (i = 0; i < order; i++) {
176 LDAP_LIST_INIT(&sh->sh_free[i]);
179 LDAP_LIST_INIT(&sh->sh_sopool);
181 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
182 slap_replenish_sopool(sh);
184 so = LDAP_LIST_FIRST(&sh->sh_sopool);
185 LDAP_LIST_REMOVE(so, so_link);
186 so->so_ptr = sh->sh_base;
188 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
190 sh->sh_map = (unsigned char **)
191 ch_malloc(order * sizeof(unsigned char *));
192 for (i = 0; i < order; i++) {
193 int shiftamt = order_start + 1 + i;
194 int nummaps = size >> shiftamt;
197 if (!nummaps) nummaps = 1;
198 sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
199 memset(sh->sh_map[i], 0, nummaps);
211 /* separate from context */
212 SET_MEMCTX(thrctx, NULL, 0);
221 struct slab_heap *sh = ctx;
222 ber_len_t *ptr, *newptr;
224 #ifdef SLAP_NO_SL_MALLOC
225 newptr = ber_memalloc_x( size, NULL );
226 if ( newptr ) return newptr;
228 exit( EXIT_FAILURE );
231 /* ber_set_option calls us like this */
233 newptr = ber_memalloc_x( size, NULL );
234 if ( newptr ) return newptr;
235 Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
236 (unsigned long) size, 0, 0);
238 exit( EXIT_FAILURE );
241 /* round up to doubleword boundary, plus space for len at head and tail */
242 size = (size + 2*sizeof(ber_len_t) + Align-1) & -Align;
245 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
246 size -= 2*sizeof(ber_len_t);
247 Debug(LDAP_DEBUG_TRACE,
248 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
249 (unsigned long) size, 0, 0);
250 return ch_malloc(size);
252 newptr = sh->sh_last;
253 sh->sh_last = (char *) sh->sh_last + size;
254 size -= sizeof(ber_len_t);
256 *(ber_len_t *)((char *)sh->sh_last - sizeof(ber_len_t)) = size;
257 return( (void *)newptr );
259 struct slab_object *so_new, *so_left, *so_right;
260 ber_len_t size_shift;
262 int i, j, order = -1;
264 size_shift = size - 1;
267 } while (size_shift >>= 1);
269 size -= sizeof(ber_len_t);
271 for (i = order; i <= sh->sh_maxorder &&
272 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
275 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
276 LDAP_LIST_REMOVE(so_new, so_link);
277 ptr = so_new->so_ptr;
278 diff = (unsigned long)((char*)ptr -
279 (char*)sh->sh_base) >> (order + 1);
280 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
282 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
284 } else if (i <= sh->sh_maxorder) {
285 for (j = i; j > order; j--) {
286 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
287 LDAP_LIST_REMOVE(so_left, so_link);
288 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
289 slap_replenish_sopool(sh);
291 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
292 LDAP_LIST_REMOVE(so_right, so_link);
293 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
294 if (j == order + 1) {
295 ptr = so_left->so_ptr;
296 diff = (unsigned long)((char*)ptr -
297 (char*)sh->sh_base) >> (order+1);
298 sh->sh_map[order-order_start][diff>>3] |=
301 LDAP_LIST_INSERT_HEAD(
302 &sh->sh_free[j-1-order_start], so_right, so_link);
303 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
306 LDAP_LIST_INSERT_HEAD(
307 &sh->sh_free[j-1-order_start], so_right, so_link);
308 LDAP_LIST_INSERT_HEAD(
309 &sh->sh_free[j-1-order_start], so_left, so_link);
313 /* FIXME: missing return; guessing we failed... */
316 Debug(LDAP_DEBUG_TRACE,
317 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
318 (unsigned long) size, 0, 0);
319 return ch_malloc(size);
322 #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
323 ((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
324 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
327 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
330 ber_len_t total = n * size;
332 /* The sqrt test is a slight optimization: often avoids the division */
333 if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
334 newptr = slap_sl_malloc( total, ctx );
335 memset( newptr, 0, n*size );
337 Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
338 (unsigned long) n, (unsigned long) size, 0);
346 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
348 struct slab_heap *sh = ctx;
349 ber_len_t oldsize, *p = (ber_len_t *) ptr;
353 return slap_sl_malloc(size, ctx);
355 #ifdef SLAP_NO_SL_MALLOC
356 newptr = ber_memrealloc_x( ptr, size, NULL );
357 if ( newptr ) return newptr;
359 exit( EXIT_FAILURE );
362 /* Not our memory? */
363 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
364 /* Like ch_realloc(), except not trying a new context */
365 newptr = ber_memrealloc_x(ptr, size, NULL);
369 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
370 (unsigned long) size, 0, 0);
372 exit( EXIT_FAILURE );
376 slap_sl_free(ptr, ctx);
383 /* Round up to doubleword boundary, add room for head */
384 size = ((size + Align-1) & -Align) + sizeof( ber_len_t );
388 /* Never shrink blocks */
389 if (size <= oldsize) {
392 /* If reallocing the last block, try to grow it */
393 } else if ((char *) ptr + oldsize == sh->sh_last) {
394 if (size < (char *) sh->sh_end - (char *) ptr) {
395 sh->sh_last = (char *) ptr + size;
397 p[size/sizeof(ber_len_t)] = size;
401 /* Nowhere to grow, need to alloc and copy */
403 /* Slight optimization of the final realloc variant */
404 size -= sizeof(ber_len_t);
405 oldsize -= sizeof(ber_len_t);
406 newptr = slap_sl_malloc(size, ctx);
407 AC_MEMCPY(newptr, ptr, oldsize);
408 /* Not last block, can just mark old region as free */
409 p[p[0]/sizeof(ber_len_t)] |= 1;
413 size -= sizeof(ber_len_t);
414 oldsize -= sizeof(ber_len_t);
416 } else if (oldsize > size) {
420 newptr = slap_sl_malloc(size, ctx);
421 AC_MEMCPY(newptr, ptr, oldsize);
422 slap_sl_free(ptr, ctx);
427 slap_sl_free(void *ptr, void *ctx)
429 struct slab_heap *sh = ctx;
431 ber_len_t *p = (ber_len_t *)ptr, *tmpp;
436 #ifdef SLAP_NO_SL_MALLOC
437 ber_memfree_x( ptr, NULL );
441 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
442 ber_memfree_x(ptr, NULL);
444 } else if (sh->sh_stack) {
446 p = (ber_len_t *) ((char *) ptr + size);
449 /* reclaim free space off tail */
450 if (sh->sh_last == p) {
452 p = (ber_len_t *) ((char *) p - size + 1) - 1;
459 int size_shift, order_size;
460 struct slab_object *so;
462 int i, inserted = 0, order = -1;
465 size_shift = size + sizeof(ber_len_t) - 1;
468 } while (size_shift >>= 1);
470 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
471 order_size = 1 << (i+1);
472 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
473 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
474 if (diff == ((diff>>1)<<1)) {
475 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
476 (1<<((diff+1)&0x7)))) {
477 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
479 if ((char*)so->so_ptr == (char*)tmpp) {
480 LDAP_LIST_REMOVE( so, so_link );
481 } else if ((char*)so->so_ptr ==
482 (char*)tmpp + order_size) {
483 LDAP_LIST_REMOVE(so, so_link);
486 so = LDAP_LIST_NEXT(so, so_link);
489 if (i < sh->sh_maxorder) {
492 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
497 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
498 slap_replenish_sopool(sh);
500 so = LDAP_LIST_FIRST(&sh->sh_sopool);
501 LDAP_LIST_REMOVE(so, so_link);
503 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
507 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
508 "free object not found while bit is clear.\n",
515 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
516 slap_replenish_sopool(sh);
518 so = LDAP_LIST_FIRST(&sh->sh_sopool);
519 LDAP_LIST_REMOVE(so, so_link);
521 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
527 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
528 (1<<((diff-1)&0x7)))) {
529 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
531 if ((char*)so->so_ptr == (char*)tmpp) {
532 LDAP_LIST_REMOVE(so, so_link);
533 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
534 LDAP_LIST_REMOVE(so, so_link);
538 so = LDAP_LIST_NEXT(so, so_link);
541 if (i < sh->sh_maxorder) {
543 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
547 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
548 slap_replenish_sopool(sh);
550 so = LDAP_LIST_FIRST(&sh->sh_sopool);
551 LDAP_LIST_REMOVE(so, so_link);
553 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
557 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
558 "free object not found while bit is clear.\n",
565 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
566 slap_replenish_sopool(sh);
568 so = LDAP_LIST_FIRST(&sh->sh_sopool);
569 LDAP_LIST_REMOVE(so, so_link);
571 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
582 slap_sl_context( void *ptr )
585 struct slab_heap *sh;
587 if ( slapMode & SLAP_TOOL_MODE ) return NULL;
589 sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
590 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
596 static struct slab_object *
597 slap_replenish_sopool(
601 struct slab_object *so_block;
604 so_block = (struct slab_object *)ch_malloc(
605 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
607 if ( so_block == NULL ) {
611 so_block[0].so_blockhead = 1;
612 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
613 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
614 so_block[i].so_blockhead = 0;
615 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
623 print_slheap(int level, void *ctx)
625 struct slab_heap *sh = ctx;
626 struct slab_object *so;
630 Debug(level, "NULL memctx\n", 0, 0, 0);
634 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
636 for (i = order_start; i <= sh->sh_maxorder; i++) {
638 Debug(level, "order=%d\n", i, 0, 0);
639 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
640 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
644 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
646 Debug(level, "\n", 0, 0, 0);
647 Debug(level, "free list:\n", 0, 0, 0);
648 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
650 Debug(level, "%p\n", so->so_ptr, 0, 0);
651 so = LDAP_LIST_NEXT(so, so_link);