1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2009 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
24 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
26 static void print_slheap(int level, void *ctx);
35 struct slab_heap *sh = data;
36 int pad = 2*sizeof(int)-1, pad_shift;
37 int order_start = -1, i;
38 struct slab_object *so;
41 ber_memfree_x(sh->sh_base, NULL);
42 ber_memfree_x(sh, NULL);
47 } while (pad_shift >>= 1);
49 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
50 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
52 struct slab_object *so_tmp = so;
53 so = LDAP_LIST_NEXT(so, so_link);
54 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
56 ch_free(sh->sh_map[i]);
61 so = LDAP_LIST_FIRST(&sh->sh_sopool);
63 struct slab_object *so_tmp = so;
64 so = LDAP_LIST_NEXT(so, so_link);
65 if (!so_tmp->so_blockhead) {
66 LDAP_LIST_REMOVE(so_tmp, so_link);
69 so = LDAP_LIST_FIRST(&sh->sh_sopool);
71 struct slab_object *so_tmp = so;
72 so = LDAP_LIST_NEXT(so, so_link);
75 ber_memfree_x(sh->sh_base, NULL);
76 ber_memfree_x(sh, NULL);
80 BerMemoryFunctions slap_sl_mfuncs =
81 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
86 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
90 static struct slab_heap *slheap;
101 struct slab_heap *sh;
102 ber_len_t size_shift;
103 int pad = 2*sizeof(int)-1, pad_shift;
104 int order = -1, order_start = -1, order_end = -1;
106 struct slab_object *so;
112 ldap_pvt_thread_pool_getkey(
113 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL );
120 /* round up to doubleword boundary */
126 sh = ch_malloc(sizeof(struct slab_heap));
127 sh->sh_base = ch_malloc(size);
131 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
132 (void *)sh, slap_sl_mem_destroy, NULL, NULL);
134 } else if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
137 newptr = ch_realloc( sh->sh_base, size );
138 if ( newptr == NULL ) return NULL;
139 sh->sh_base = newptr;
141 /* insert dummy len */
143 ber_len_t *i = sh->sh_base;
147 sh->sh_end = (char *) sh->sh_base + size;
148 sh->sh_stack = stack;
151 size_shift = size - 1;
154 } while (size_shift >>= 1);
159 } while (pad_shift >>= 1);
161 order = order_end - order_start + 1;
164 sh = (struct slab_heap *) ch_malloc(sizeof(struct slab_heap));
165 sh->sh_base = ch_malloc(size);
169 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
170 (void *)sh, slap_sl_mem_destroy, NULL, NULL);
173 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
174 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
176 struct slab_object *so_tmp = so;
177 so = LDAP_LIST_NEXT(so, so_link);
178 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
180 ch_free(sh->sh_map[i]);
182 ch_free(sh->sh_free);
185 so = LDAP_LIST_FIRST(&sh->sh_sopool);
187 struct slab_object *so_tmp = so;
188 so = LDAP_LIST_NEXT(so, so_link);
189 if (!so_tmp->so_blockhead) {
190 LDAP_LIST_REMOVE(so_tmp, so_link);
193 so = LDAP_LIST_FIRST(&sh->sh_sopool);
195 struct slab_object *so_tmp = so;
196 so = LDAP_LIST_NEXT(so, so_link);
200 if (size > (char *)sh->sh_end - (char *)sh->sh_base) {
203 newptr = ch_realloc( sh->sh_base, size );
204 if ( newptr == NULL ) return NULL;
205 sh->sh_base = newptr;
208 sh->sh_end = (char *)sh->sh_base + size;
209 sh->sh_maxorder = order_end;
211 sh->sh_free = (struct sh_freelist *)
212 ch_malloc(order * sizeof(struct sh_freelist));
213 for (i = 0; i < order; i++) {
214 LDAP_LIST_INIT(&sh->sh_free[i]);
217 LDAP_LIST_INIT(&sh->sh_sopool);
219 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
220 slap_replenish_sopool(sh);
222 so = LDAP_LIST_FIRST(&sh->sh_sopool);
223 LDAP_LIST_REMOVE(so, so_link);
224 so->so_ptr = sh->sh_base;
226 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
228 sh->sh_map = (unsigned char **)
229 ch_malloc(order * sizeof(unsigned char *));
230 for (i = 0; i < order; i++) {
231 int shiftamt = order_start + 1 + i;
232 int nummaps = size >> shiftamt;
235 if (!nummaps) nummaps = 1;
236 sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
237 memset(sh->sh_map[i], 0, nummaps);
239 sh->sh_stack = stack;
253 /* separate from context */
254 ldap_pvt_thread_pool_setkey( ctx, (void *)slap_sl_mem_init,
255 NULL, 0, NULL, NULL );
265 struct slab_heap *sh = ctx;
266 int pad = 2*sizeof(int)-1, pad_shift;
267 ber_len_t *ptr, *newptr;
269 #ifdef SLAP_NO_SL_MALLOC
270 newptr = ber_memalloc_x( size, NULL );
271 if ( newptr ) return newptr;
273 exit( EXIT_FAILURE );
276 /* ber_set_option calls us like this */
278 newptr = ber_memalloc_x( size, NULL );
279 if ( newptr ) return newptr;
281 exit( EXIT_FAILURE );
284 /* round up to doubleword boundary, plus space for len at head and tail */
285 size += 2*sizeof(ber_len_t) + pad;
289 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
290 Debug(LDAP_DEBUG_TRACE,
291 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
293 return ch_malloc(size);
295 newptr = sh->sh_last;
296 *newptr++ = size - sizeof(ber_len_t);
297 sh->sh_last = (char *) sh->sh_last + size;
299 ptr[-1] = size - sizeof(ber_len_t);
300 return( (void *)newptr );
302 struct slab_object *so_new, *so_left, *so_right;
303 ber_len_t size_shift;
304 int order = -1, order_start = -1;
308 size_shift = size - 1;
311 } while (size_shift >>= 1);
316 } while (pad_shift >>= 1);
318 for (i = order; i <= sh->sh_maxorder &&
319 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
322 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
323 LDAP_LIST_REMOVE(so_new, so_link);
324 ptr = so_new->so_ptr;
325 diff = (unsigned long)((char*)ptr -
326 (char*)sh->sh_base) >> (order + 1);
327 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
328 *ptr++ = size - sizeof(ber_len_t);
329 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
331 } else if (i <= sh->sh_maxorder) {
332 for (j = i; j > order; j--) {
333 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
334 LDAP_LIST_REMOVE(so_left, so_link);
335 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
336 slap_replenish_sopool(sh);
338 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
339 LDAP_LIST_REMOVE(so_right, so_link);
340 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
341 if (j == order + 1) {
342 ptr = so_left->so_ptr;
343 diff = (unsigned long)((char*)ptr -
344 (char*)sh->sh_base) >> (order+1);
345 sh->sh_map[order-order_start][diff>>3] |=
347 *ptr++ = size - sizeof(ber_len_t);
348 LDAP_LIST_INSERT_HEAD(
349 &sh->sh_free[j-1-order_start], so_right, so_link);
350 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
353 LDAP_LIST_INSERT_HEAD(
354 &sh->sh_free[j-1-order_start], so_right, so_link);
355 LDAP_LIST_INSERT_HEAD(
356 &sh->sh_free[j-1-order_start], so_left, so_link);
360 Debug( LDAP_DEBUG_TRACE,
361 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
363 return (void*)ch_malloc(size);
367 /* FIXME: missing return; guessing... */
372 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
376 newptr = slap_sl_malloc( n*size, ctx );
378 memset( newptr, 0, n*size );
384 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
386 struct slab_heap *sh = ctx;
387 int pad = 2*sizeof(int) -1;
388 ber_len_t *p = (ber_len_t *)ptr, *newptr;
391 return slap_sl_malloc(size, ctx);
393 #ifdef SLAP_NO_SL_MALLOC
394 newptr = ber_memrealloc_x( ptr, size, NULL );
395 if ( newptr ) return newptr;
397 exit( EXIT_FAILURE );
400 /* Not our memory? */
401 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
402 /* duplicate of realloc behavior, oh well */
403 newptr = ber_memrealloc_x(ptr, size, NULL);
407 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
410 exit( EXIT_FAILURE );
414 slap_sl_free(ptr, ctx);
419 /* round up to doubleword boundary */
420 size += pad + sizeof( ber_len_t );
423 /* Never shrink blocks */
427 /* If reallocing the last block, we can grow it */
428 } else if ((char *)ptr + p[-1] == sh->sh_last &&
429 (char *)ptr + size < (char *)sh->sh_end ) {
431 sh->sh_last = (char *)sh->sh_last + size - p[-1];
437 /* Nowhere to grow, need to alloc and copy */
439 newptr = slap_sl_malloc(size-2*sizeof(ber_len_t), ctx);
440 AC_MEMCPY(newptr, ptr, p[-1]-sizeof(ber_len_t));
441 /* mark old region as free */
442 p[p[-1]/sizeof(ber_len_t)-1] |= 1;
448 newptr2 = slap_sl_malloc(size, ctx);
450 AC_MEMCPY(newptr2, ptr, size);
452 AC_MEMCPY(newptr2, ptr, p[-1]);
454 slap_sl_free(ptr, ctx);
460 slap_sl_free(void *ptr, void *ctx)
462 struct slab_heap *sh = ctx;
463 int size, size_shift, order_size;
464 int pad = 2*sizeof(int)-1, pad_shift;
465 ber_len_t *p = (ber_len_t *)ptr, *tmpp;
466 int order_start = -1, order = -1;
467 struct slab_object *so;
474 #ifdef SLAP_NO_SL_MALLOC
475 ber_memfree_x( ptr, NULL );
479 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
480 ber_memfree_x(ptr, NULL);
481 } else if (sh->sh_stack) {
482 tmpp = (ber_len_t *)((char *)ptr + p[-1]);
485 /* reclaim free space off tail */
486 while ( tmpp == sh->sh_last ) {
487 if ( tmpp[-1] & 1 ) {
489 ptr = (char *)tmpp - size;
490 p = (ber_len_t *)ptr;
500 size_shift = size + sizeof(ber_len_t) - 1;
503 } while (size_shift >>= 1);
508 } while (pad_shift >>= 1);
510 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
511 order_size = 1 << (i+1);
512 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
513 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
514 if (diff == ((diff>>1)<<1)) {
515 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
516 (1<<((diff+1)&0x7)))) {
517 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
519 if ((char*)so->so_ptr == (char*)tmpp) {
520 LDAP_LIST_REMOVE( so, so_link );
521 } else if ((char*)so->so_ptr ==
522 (char*)tmpp + order_size) {
523 LDAP_LIST_REMOVE(so, so_link);
526 so = LDAP_LIST_NEXT(so, so_link);
529 if (i < sh->sh_maxorder) {
532 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
537 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
538 slap_replenish_sopool(sh);
540 so = LDAP_LIST_FIRST(&sh->sh_sopool);
541 LDAP_LIST_REMOVE(so, so_link);
543 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
547 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
548 "free object not found while bit is clear.\n",
555 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
556 slap_replenish_sopool(sh);
558 so = LDAP_LIST_FIRST(&sh->sh_sopool);
559 LDAP_LIST_REMOVE(so, so_link);
561 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
567 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
568 (1<<((diff-1)&0x7)))) {
569 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
571 if ((char*)so->so_ptr == (char*)tmpp) {
572 LDAP_LIST_REMOVE(so, so_link);
573 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
574 LDAP_LIST_REMOVE(so, so_link);
578 so = LDAP_LIST_NEXT(so, so_link);
581 if (i < sh->sh_maxorder) {
583 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
587 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
588 slap_replenish_sopool(sh);
590 so = LDAP_LIST_FIRST(&sh->sh_sopool);
591 LDAP_LIST_REMOVE(so, so_link);
593 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
597 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
598 "free object not found while bit is clear.\n",
605 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
606 slap_replenish_sopool(sh);
608 so = LDAP_LIST_FIRST(&sh->sh_sopool);
609 LDAP_LIST_REMOVE(so, so_link);
611 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
622 slap_sl_context( void *ptr )
624 struct slab_heap *sh;
627 if ( slapMode & SLAP_TOOL_MODE ) return NULL;
632 ctx = ldap_pvt_thread_pool_context();
635 ldap_pvt_thread_pool_getkey(
636 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL);
640 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
646 static struct slab_object *
647 slap_replenish_sopool(
651 struct slab_object *so_block;
654 so_block = (struct slab_object *)ch_malloc(
655 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
657 if ( so_block == NULL ) {
661 so_block[0].so_blockhead = 1;
662 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
663 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
664 so_block[i].so_blockhead = 0;
665 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
673 print_slheap(int level, void *ctx)
675 struct slab_heap *sh = ctx;
676 int order_start = -1;
677 int pad = 2*sizeof(int)-1, pad_shift;
678 struct slab_object *so;
682 Debug(level, "NULL memctx\n", 0, 0, 0);
689 } while (pad_shift >>= 1);
691 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
693 for (i = order_start; i <= sh->sh_maxorder; i++) {
695 Debug(level, "order=%d\n", i, 0, 0);
696 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
697 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
701 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
703 Debug(level, "\n", 0, 0, 0);
704 Debug(level, "free list:\n", 0, 0, 0);
705 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
707 Debug(level, "%lx\n", (unsigned long) so->so_ptr, 0, 0);
708 so = LDAP_LIST_NEXT(so, so_link);