1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2004 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
24 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
25 static void print_slheap(int level, void *ctx);
33 struct slab_heap *sh = data;
34 int pad = 2*sizeof(int)-1, pad_shift;
35 int order_start = -1, i;
36 struct slab_object *so;
39 ber_memfree_x(sh->sh_base, NULL);
40 ber_memfree_x(sh, NULL);
45 } while (pad_shift >>= 1);
47 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
48 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
50 struct slab_object *so_tmp = so;
51 so = LDAP_LIST_NEXT(so, so_link);
52 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
54 ch_free(sh->sh_map[i]);
59 so = LDAP_LIST_FIRST(&sh->sh_sopool);
61 struct slab_object *so_tmp = so;
62 so = LDAP_LIST_NEXT(so, so_link);
63 if (!so_tmp->so_blockhead) {
64 LDAP_LIST_REMOVE(so_tmp, so_link);
67 so = LDAP_LIST_FIRST(&sh->sh_sopool);
69 struct slab_object *so_tmp = so;
70 so = LDAP_LIST_NEXT(so, so_link);
73 ber_memfree_x(sh->sh_base, NULL);
74 ber_memfree_x(sh, NULL);
78 BerMemoryFunctions slap_sl_mfuncs =
79 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
84 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
88 static struct slab_heap *slheap;
98 struct slab_heap *sh = NULL;
100 int pad = 2*sizeof(int)-1, pad_shift;
101 int order = -1, order_start = -1, order_end = -1;
103 struct slab_object *so;
108 ldap_pvt_thread_pool_getkey(
109 ctx, (void *)slap_sl_mem_init, (void **)&sh, NULL );
112 /* round up to doubleword boundary */
118 sh = ch_malloc(sizeof(struct slab_heap));
119 sh->sh_base = ch_malloc(size);
123 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
124 (void *)sh, slap_sl_mem_destroy);
126 } else if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
127 sh->sh_base = ch_realloc(sh->sh_base, size);
129 sh->sh_last = sh->sh_base;
130 sh->sh_end = (char *) sh->sh_base + size;
131 sh->sh_stack = stack;
134 size_shift = size - 1;
137 } while (size_shift >>= 1);
142 } while (pad_shift >>= 1);
144 order = order_end - order_start + 1;
147 sh = (struct slab_heap *) ch_malloc(sizeof(struct slab_heap));
148 sh->sh_base = ch_malloc(size);
152 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
153 (void *)sh, slap_sl_mem_destroy);
156 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
157 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
159 struct slab_object *so_tmp = so;
160 so = LDAP_LIST_NEXT(so, so_link);
161 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
163 ch_free(sh->sh_map[i]);
165 ch_free(sh->sh_free);
168 so = LDAP_LIST_FIRST(&sh->sh_sopool);
170 struct slab_object *so_tmp = so;
171 so = LDAP_LIST_NEXT(so, so_link);
172 if (!so_tmp->so_blockhead) {
173 LDAP_LIST_REMOVE(so_tmp, so_link);
176 so = LDAP_LIST_FIRST(&sh->sh_sopool);
178 struct slab_object *so_tmp = so;
179 so = LDAP_LIST_NEXT(so, so_link);
183 if (size > (char *)sh->sh_end - (char *)sh->sh_base) {
184 sh->sh_base = realloc(sh->sh_base, size);
187 sh->sh_end = (char *)sh->sh_base + size;
188 sh->sh_maxorder = order_end;
190 sh->sh_free = (struct sh_freelist *)
191 ch_malloc(order * sizeof(struct sh_freelist));
192 for (i = 0; i < order; i++) {
193 LDAP_LIST_INIT(&sh->sh_free[i]);
196 LDAP_LIST_INIT(&sh->sh_sopool);
198 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
199 slap_replenish_sopool(sh);
201 so = LDAP_LIST_FIRST(&sh->sh_sopool);
202 LDAP_LIST_REMOVE(so, so_link);
203 so->so_ptr = sh->sh_base;
205 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
207 sh->sh_map = (unsigned char **)
208 ch_malloc(order * sizeof(unsigned char *));
209 for (i = 0; i < order; i++) {
210 sh->sh_map[i] = (unsigned char *)
211 ch_malloc(size >> (1 << (order_start + i + 3)));
212 memset(sh->sh_map[i], 0, size >> (1 << (order_start + i + 3)));
214 sh->sh_stack = stack;
228 /* separate from context */
229 ldap_pvt_thread_pool_setkey( ctx, (void *)slap_sl_mem_init, NULL, NULL );
239 struct slab_heap *sh = ctx;
240 ber_len_t size_shift;
241 int pad = 2*sizeof(int)-1, pad_shift;
242 int order = -1, order_start = -1;
243 struct slab_object *so_new, *so_left, *so_right;
244 ber_len_t *ptr, *new;
248 /* ber_set_option calls us like this */
249 if (!ctx) return ber_memalloc_x(size, NULL);
251 Debug(LDAP_DEBUG_TRACE, "==> slap_sl_malloc (%d)\n", size, 0, 0);
253 /* round up to doubleword boundary */
254 size += sizeof(ber_len_t) + pad;
258 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
259 Debug(LDAP_DEBUG_TRACE,
260 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
262 return ch_malloc(size);
265 *new++ = size - sizeof(ber_len_t);
266 sh->sh_last = (char *) sh->sh_last + size;
267 return( (void *)new );
269 size_shift = size - 1;
272 } while (size_shift >>= 1);
277 } while (pad_shift >>= 1);
279 for (i = order; i <= sh->sh_maxorder &&
280 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
283 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
284 LDAP_LIST_REMOVE(so_new, so_link);
285 ptr = so_new->so_ptr;
286 diff = (unsigned long)((char*)ptr -
287 (char*)sh->sh_base) >> (order + 1);
288 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
289 *ptr++ = size - sizeof(ber_len_t);
290 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
292 } else if (i <= sh->sh_maxorder) {
293 for (j = i; j > order; j--) {
294 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
295 LDAP_LIST_REMOVE(so_left, so_link);
296 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
297 slap_replenish_sopool(sh);
299 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
300 LDAP_LIST_REMOVE(so_right, so_link);
301 so_right->so_ptr = so_left->so_ptr + (1 << j);
302 if (j == order + 1) {
303 ptr = so_left->so_ptr;
304 diff = (unsigned long)((char*)ptr -
305 (char*)sh->sh_base) >> (order+1);
306 sh->sh_map[order-order_start][diff>>3] |=
308 *ptr++ = size - sizeof(ber_len_t);
309 LDAP_LIST_INSERT_HEAD(
310 &sh->sh_free[j-1-order_start], so_right, so_link);
311 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
314 LDAP_LIST_INSERT_HEAD(
315 &sh->sh_free[j-1-order_start], so_right, so_link);
316 LDAP_LIST_INSERT_HEAD(
317 &sh->sh_free[j-1-order_start], so_left, so_link);
321 Debug( LDAP_DEBUG_TRACE,
322 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
324 return (void*)ch_malloc(size);
330 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
334 new = slap_sl_malloc( n*size, ctx );
336 memset( new, 0, n*size );
342 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
344 struct slab_heap *sh = ctx;
346 int pad = 2*sizeof(int)-1, pad_shift;
347 int order_start = -1, order = -1;
348 struct slab_object *so;
349 ber_len_t *p = (ber_len_t *)ptr, *new;
353 return slap_sl_malloc(size, ctx);
355 /* Not our memory? */
356 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
357 /* duplicate of realloc behavior, oh well */
358 new = ber_memrealloc_x(ptr, size, NULL);
362 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
365 exit( EXIT_FAILURE );
369 slap_sl_free(ptr, ctx);
374 /* Never shrink blocks */
378 /* If reallocing the last block, we can grow it */
379 } else if ((char *)ptr + p[-1] == sh->sh_last) {
381 sh->sh_last = (char *)sh->sh_last + size - p[-1];
384 /* Nowhere to grow, need to alloc and copy */
386 new = slap_sl_malloc(size, ctx);
387 AC_MEMCPY(new, ptr, p[-1]);
392 newptr = slap_sl_malloc(size, ctx);
394 AC_MEMCPY(newptr, ptr, size);
396 AC_MEMCPY(newptr, ptr, p[-1]);
398 slap_sl_free(ptr, ctx);
404 slap_sl_free(void *ptr, void *ctx)
406 struct slab_heap *sh = ctx;
407 int size, size_shift, order_size;
408 int pad = 2*sizeof(int)-1, pad_shift;
409 ber_len_t *p = (ber_len_t *)ptr, *tmpp;
410 int order_start = -1, order = -1;
411 struct slab_object *so;
413 int i, k, inserted = 0;
415 Debug( LDAP_DEBUG_TRACE, "==> slap_sl_free \n", 0, 0, 0);
417 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
418 ber_memfree_x(ptr, NULL);
419 } else if (sh->sh_stack && (char *)ptr + p[-1] == sh->sh_last) {
422 } else if (!sh->sh_stack) {
424 size_shift = size + sizeof(ber_len_t) - 1;
427 } while (size_shift >>= 1);
432 } while (pad_shift >>= 1);
434 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
435 order_size = 1 << (i+1);
436 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
437 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
438 if (diff == ((diff>>1)<<1)) {
439 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
440 (1<<((diff+1)&0x7)))) {
441 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
443 if ((char*)so->so_ptr == (char*)tmpp) {
444 LDAP_LIST_REMOVE( so, so_link );
445 } else if ((char*)so->so_ptr ==
446 (char*)tmpp + order_size) {
447 LDAP_LIST_REMOVE(so, so_link);
450 so = LDAP_LIST_NEXT(so, so_link);
453 if (i < sh->sh_maxorder) {
456 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
461 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
462 slap_replenish_sopool(sh);
464 so = LDAP_LIST_FIRST(&sh->sh_sopool);
465 LDAP_LIST_REMOVE(so, so_link);
467 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
471 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
472 "free object not found while bit is clear.\n",
479 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
480 slap_replenish_sopool(sh);
482 so = LDAP_LIST_FIRST(&sh->sh_sopool);
483 LDAP_LIST_REMOVE(so, so_link);
485 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
491 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
492 (1<<((diff-1)&0x7)))) {
493 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
495 if ((char*)so->so_ptr == (char*)tmpp) {
496 LDAP_LIST_REMOVE(so, so_link);
497 } else if ((char*)tmpp == so->so_ptr + order_size) {
498 LDAP_LIST_REMOVE(so, so_link);
502 so = LDAP_LIST_NEXT(so, so_link);
505 if (i < sh->sh_maxorder) {
507 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
511 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
512 slap_replenish_sopool(sh);
514 so = LDAP_LIST_FIRST(&sh->sh_sopool);
515 LDAP_LIST_REMOVE(so, so_link);
517 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
521 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
522 "free object not found while bit is clear.\n",
529 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
530 slap_replenish_sopool(sh);
532 so = LDAP_LIST_FIRST(&sh->sh_sopool);
533 LDAP_LIST_REMOVE(so, so_link);
535 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
546 slap_sl_context( void *ptr )
548 struct slab_heap *sh = NULL;
554 ctx = ldap_pvt_thread_pool_context();
556 ldap_pvt_thread_pool_getkey(ctx, (void *)slap_sl_mem_init,
560 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
566 static struct slab_object *
567 slap_replenish_sopool(
571 struct slab_object *so_block;
574 so_block = (struct slab_object *)ch_malloc(
575 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
577 if ( so_block == NULL ) {
581 so_block[0].so_blockhead = 1;
582 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
583 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
584 so_block[i].so_blockhead = 0;
585 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
592 print_slheap(int level, void *ctx)
594 struct slab_heap *sh = ctx;
595 int order_start = -1;
596 int pad = 2*sizeof(int)-1, pad_shift;
597 struct slab_object *so;
601 Debug(level, "NULL memctx\n", 0, 0, 0);
608 } while (pad_shift >>= 1);
610 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
612 for (i = order_start; i <= sh->sh_maxorder; i++) {
614 Debug(level, "order=%d\n", i, 0, 0);
615 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
616 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
620 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
622 Debug(level, "\n", 0, 0, 0);
623 Debug(level, "free list:\n", 0, 0, 0);
624 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
626 Debug(level, "%x\n",so->so_ptr, 0, 0);
627 so = LDAP_LIST_NEXT(so, so_link);