1 /* sl_malloc.c - malloc routines using a per-thread slab */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2005 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
20 #include <ac/string.h>
24 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
26 static void print_slheap(int level, void *ctx);
35 struct slab_heap *sh = data;
36 int pad = 2*sizeof(int)-1, pad_shift;
37 int order_start = -1, i;
38 struct slab_object *so;
41 ber_memfree_x(sh->sh_base, NULL);
42 ber_memfree_x(sh, NULL);
47 } while (pad_shift >>= 1);
49 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
50 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
52 struct slab_object *so_tmp = so;
53 so = LDAP_LIST_NEXT(so, so_link);
54 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
56 ch_free(sh->sh_map[i]);
61 so = LDAP_LIST_FIRST(&sh->sh_sopool);
63 struct slab_object *so_tmp = so;
64 so = LDAP_LIST_NEXT(so, so_link);
65 if (!so_tmp->so_blockhead) {
66 LDAP_LIST_REMOVE(so_tmp, so_link);
69 so = LDAP_LIST_FIRST(&sh->sh_sopool);
71 struct slab_object *so_tmp = so;
72 so = LDAP_LIST_NEXT(so, so_link);
75 ber_memfree_x(sh->sh_base, NULL);
76 ber_memfree_x(sh, NULL);
80 BerMemoryFunctions slap_sl_mfuncs =
81 { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
86 ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
90 static struct slab_heap *slheap;
100 struct slab_heap *sh;
101 ber_len_t size_shift;
102 int pad = 2*sizeof(int)-1, pad_shift;
103 int order = -1, order_start = -1, order_end = -1;
105 struct slab_object *so;
111 ldap_pvt_thread_pool_getkey(
112 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL );
116 /* round up to doubleword boundary */
122 sh = ch_malloc(sizeof(struct slab_heap));
123 sh->sh_base = ch_malloc(size);
127 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
128 (void *)sh, slap_sl_mem_destroy);
130 } else if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
131 sh->sh_base = ch_realloc(sh->sh_base, size);
133 sh->sh_last = sh->sh_base;
134 sh->sh_end = (char *) sh->sh_base + size;
135 sh->sh_stack = stack;
138 size_shift = size - 1;
141 } while (size_shift >>= 1);
146 } while (pad_shift >>= 1);
148 order = order_end - order_start + 1;
151 sh = (struct slab_heap *) ch_malloc(sizeof(struct slab_heap));
152 sh->sh_base = ch_malloc(size);
156 ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
157 (void *)sh, slap_sl_mem_destroy);
160 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
161 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
163 struct slab_object *so_tmp = so;
164 so = LDAP_LIST_NEXT(so, so_link);
165 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
167 ch_free(sh->sh_map[i]);
169 ch_free(sh->sh_free);
172 so = LDAP_LIST_FIRST(&sh->sh_sopool);
174 struct slab_object *so_tmp = so;
175 so = LDAP_LIST_NEXT(so, so_link);
176 if (!so_tmp->so_blockhead) {
177 LDAP_LIST_REMOVE(so_tmp, so_link);
180 so = LDAP_LIST_FIRST(&sh->sh_sopool);
182 struct slab_object *so_tmp = so;
183 so = LDAP_LIST_NEXT(so, so_link);
187 if (size > (char *)sh->sh_end - (char *)sh->sh_base) {
188 sh->sh_base = realloc(sh->sh_base, size);
191 sh->sh_end = (char *)sh->sh_base + size;
192 sh->sh_maxorder = order_end;
194 sh->sh_free = (struct sh_freelist *)
195 ch_malloc(order * sizeof(struct sh_freelist));
196 for (i = 0; i < order; i++) {
197 LDAP_LIST_INIT(&sh->sh_free[i]);
200 LDAP_LIST_INIT(&sh->sh_sopool);
202 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
203 slap_replenish_sopool(sh);
205 so = LDAP_LIST_FIRST(&sh->sh_sopool);
206 LDAP_LIST_REMOVE(so, so_link);
207 so->so_ptr = sh->sh_base;
209 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
211 sh->sh_map = (unsigned char **)
212 ch_malloc(order * sizeof(unsigned char *));
213 for (i = 0; i < order; i++) {
214 int shiftamt = order_start + 1 + i;
215 int nummaps = size >> shiftamt;
218 if (!nummaps) nummaps = 1;
219 sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
220 memset(sh->sh_map[i], 0, nummaps);
222 sh->sh_stack = stack;
236 /* separate from context */
237 ldap_pvt_thread_pool_setkey( ctx, (void *)slap_sl_mem_init, NULL, NULL );
247 struct slab_heap *sh = ctx;
248 ber_len_t size_shift;
249 int pad = 2*sizeof(int)-1, pad_shift;
250 int order = -1, order_start = -1;
251 struct slab_object *so_new, *so_left, *so_right;
252 ber_len_t *ptr, *new;
256 /* ber_set_option calls us like this */
257 if (!ctx) return ber_memalloc_x(size, NULL);
259 /* round up to doubleword boundary */
260 size += sizeof(ber_len_t) + pad;
264 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
265 Debug(LDAP_DEBUG_TRACE,
266 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
268 return ch_malloc(size);
271 *new++ = size - sizeof(ber_len_t);
272 sh->sh_last = (char *) sh->sh_last + size;
273 return( (void *)new );
275 size_shift = size - 1;
278 } while (size_shift >>= 1);
283 } while (pad_shift >>= 1);
285 for (i = order; i <= sh->sh_maxorder &&
286 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
289 so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
290 LDAP_LIST_REMOVE(so_new, so_link);
291 ptr = so_new->so_ptr;
292 diff = (unsigned long)((char*)ptr -
293 (char*)sh->sh_base) >> (order + 1);
294 sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
295 *ptr++ = size - sizeof(ber_len_t);
296 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
298 } else if (i <= sh->sh_maxorder) {
299 for (j = i; j > order; j--) {
300 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
301 LDAP_LIST_REMOVE(so_left, so_link);
302 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
303 slap_replenish_sopool(sh);
305 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
306 LDAP_LIST_REMOVE(so_right, so_link);
307 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
308 if (j == order + 1) {
309 ptr = so_left->so_ptr;
310 diff = (unsigned long)((char*)ptr -
311 (char*)sh->sh_base) >> (order+1);
312 sh->sh_map[order-order_start][diff>>3] |=
314 *ptr++ = size - sizeof(ber_len_t);
315 LDAP_LIST_INSERT_HEAD(
316 &sh->sh_free[j-1-order_start], so_right, so_link);
317 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
320 LDAP_LIST_INSERT_HEAD(
321 &sh->sh_free[j-1-order_start], so_right, so_link);
322 LDAP_LIST_INSERT_HEAD(
323 &sh->sh_free[j-1-order_start], so_left, so_link);
327 Debug( LDAP_DEBUG_TRACE,
328 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
330 return (void*)ch_malloc(size);
334 /* FIXME: missing return; guessing... */
339 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
343 new = slap_sl_malloc( n*size, ctx );
345 memset( new, 0, n*size );
351 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
353 struct slab_heap *sh = ctx;
354 int pad = 2*sizeof(int) -1;
355 ber_len_t *p = (ber_len_t *)ptr, *new;
358 return slap_sl_malloc(size, ctx);
360 /* Not our memory? */
361 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
362 /* duplicate of realloc behavior, oh well */
363 new = ber_memrealloc_x(ptr, size, NULL);
367 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
370 exit( EXIT_FAILURE );
374 slap_sl_free(ptr, ctx);
379 /* round up to doubleword boundary */
380 size += pad + sizeof( ber_len_t );
383 /* Never shrink blocks */
387 /* If reallocing the last block, we can grow it */
388 } else if ((char *)ptr + p[-1] == sh->sh_last &&
389 (char *)ptr + size < (char *)sh->sh_end ) {
391 sh->sh_last = (char *)sh->sh_last + size - p[-1];
394 /* Nowhere to grow, need to alloc and copy */
396 new = slap_sl_malloc(size, ctx);
397 AC_MEMCPY(new, ptr, p[-1]);
402 newptr = slap_sl_malloc(size, ctx);
404 AC_MEMCPY(newptr, ptr, size);
406 AC_MEMCPY(newptr, ptr, p[-1]);
408 slap_sl_free(ptr, ctx);
414 slap_sl_free(void *ptr, void *ctx)
416 struct slab_heap *sh = ctx;
417 int size, size_shift, order_size;
418 int pad = 2*sizeof(int)-1, pad_shift;
419 ber_len_t *p = (ber_len_t *)ptr, *tmpp;
420 int order_start = -1, order = -1;
421 struct slab_object *so;
425 if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
426 ber_memfree_x(ptr, NULL);
427 } else if (sh->sh_stack && (char *)ptr + p[-1] == sh->sh_last) {
430 } else if (!sh->sh_stack) {
432 size_shift = size + sizeof(ber_len_t) - 1;
435 } while (size_shift >>= 1);
440 } while (pad_shift >>= 1);
442 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
443 order_size = 1 << (i+1);
444 diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
445 sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
446 if (diff == ((diff>>1)<<1)) {
447 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
448 (1<<((diff+1)&0x7)))) {
449 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
451 if ((char*)so->so_ptr == (char*)tmpp) {
452 LDAP_LIST_REMOVE( so, so_link );
453 } else if ((char*)so->so_ptr ==
454 (char*)tmpp + order_size) {
455 LDAP_LIST_REMOVE(so, so_link);
458 so = LDAP_LIST_NEXT(so, so_link);
461 if (i < sh->sh_maxorder) {
464 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
469 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
470 slap_replenish_sopool(sh);
472 so = LDAP_LIST_FIRST(&sh->sh_sopool);
473 LDAP_LIST_REMOVE(so, so_link);
475 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
479 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
480 "free object not found while bit is clear.\n",
487 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
488 slap_replenish_sopool(sh);
490 so = LDAP_LIST_FIRST(&sh->sh_sopool);
491 LDAP_LIST_REMOVE(so, so_link);
493 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
499 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
500 (1<<((diff-1)&0x7)))) {
501 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
503 if ((char*)so->so_ptr == (char*)tmpp) {
504 LDAP_LIST_REMOVE(so, so_link);
505 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
506 LDAP_LIST_REMOVE(so, so_link);
510 so = LDAP_LIST_NEXT(so, so_link);
513 if (i < sh->sh_maxorder) {
515 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1], so, so_link);
519 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
520 slap_replenish_sopool(sh);
522 so = LDAP_LIST_FIRST(&sh->sh_sopool);
523 LDAP_LIST_REMOVE(so, so_link);
525 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
529 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
530 "free object not found while bit is clear.\n",
537 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
538 slap_replenish_sopool(sh);
540 so = LDAP_LIST_FIRST(&sh->sh_sopool);
541 LDAP_LIST_REMOVE(so, so_link);
543 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
554 slap_sl_context( void *ptr )
556 struct slab_heap *sh;
559 if ( slapMode & SLAP_TOOL_MODE ) return NULL;
564 ctx = ldap_pvt_thread_pool_context();
567 ldap_pvt_thread_pool_getkey(
568 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL);
572 if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
578 static struct slab_object *
579 slap_replenish_sopool(
583 struct slab_object *so_block;
586 so_block = (struct slab_object *)ch_malloc(
587 SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
589 if ( so_block == NULL ) {
593 so_block[0].so_blockhead = 1;
594 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
595 for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
596 so_block[i].so_blockhead = 0;
597 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
605 print_slheap(int level, void *ctx)
607 struct slab_heap *sh = ctx;
608 int order_start = -1;
609 int pad = 2*sizeof(int)-1, pad_shift;
610 struct slab_object *so;
614 Debug(level, "NULL memctx\n", 0, 0, 0);
621 } while (pad_shift >>= 1);
623 Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
625 for (i = order_start; i <= sh->sh_maxorder; i++) {
627 Debug(level, "order=%d\n", i, 0, 0);
628 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
629 Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
633 Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
635 Debug(level, "\n", 0, 0, 0);
636 Debug(level, "free list:\n", 0, 0, 0);
637 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
639 Debug(level, "%lx\n", (unsigned long) so->so_ptr, 0, 0);
640 so = LDAP_LIST_NEXT(so, so_link);