]> git.sur5r.net Git - openldap/blob - servers/slapd/sl_malloc.c
2e25abfd1ed22e984e40af0367b8cb5ced17e4a8
[openldap] / servers / slapd / sl_malloc.c
1 /* sl_malloc.c - malloc routines using a per-thread slab */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4  *
5  * Copyright 2003-2010 The OpenLDAP Foundation.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted only as authorized by the OpenLDAP
10  * Public License.
11  *
12  * A copy of this license is available in the file LICENSE in the
13  * top-level directory of the distribution or, alternatively, at
14  * <http://www.OpenLDAP.org/license.html>.
15  */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21
22 #include "slap.h"
23
24 enum {
25         Align = 2 * sizeof(int),
26         Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
27         order_start = Align_log2 - 1,
28         pad = Align - 1
29 };
30
31 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
32 #ifdef SLAPD_UNUSED
33 static void print_slheap(int level, void *ctx);
34 #endif
35
36 /* Destroy the context, or if key==NULL clean it up for reuse. */
37 void
38 slap_sl_mem_destroy(
39         void *key,
40         void *data
41 )
42 {
43         struct slab_heap *sh = data;
44         struct slab_object *so;
45         int i;
46
47         if (!sh->sh_stack) {
48                 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
49                         so = LDAP_LIST_FIRST(&sh->sh_free[i]);
50                         while (so) {
51                                 struct slab_object *so_tmp = so;
52                                 so = LDAP_LIST_NEXT(so, so_link);
53                                 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
54                         }
55                         ch_free(sh->sh_map[i]);
56                 }
57                 ch_free(sh->sh_free);
58                 ch_free(sh->sh_map);
59
60                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
61                 while (so) {
62                         struct slab_object *so_tmp = so;
63                         so = LDAP_LIST_NEXT(so, so_link);
64                         if (!so_tmp->so_blockhead) {
65                                 LDAP_LIST_REMOVE(so_tmp, so_link);
66                         }
67                 }
68                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
69                 while (so) {
70                         struct slab_object *so_tmp = so;
71                         so = LDAP_LIST_NEXT(so, so_link);
72                         ch_free(so_tmp);
73                 }
74         }
75
76         if (key != NULL) {
77                 ber_memfree_x(sh->sh_base, NULL);
78                 ber_memfree_x(sh, NULL);
79         }
80 }
81
82 BerMemoryFunctions slap_sl_mfuncs =
83         { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
84
85 void
86 slap_sl_mem_init()
87 {
88         assert( Align == 1 << Align_log2 );
89         /* Adding head+tail preserves alignment */
90         assert( 2*sizeof(ber_len_t) % Align == 0 );
91
92         ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
93 }
94
95 #ifdef NO_THREADS
96 static struct slab_heap *slheap;
97 #endif
98
99 /* This allocator always returns memory aligned on a 2-int boundary.
100  *
101  * The stack-based allocator stores the size as a ber_len_t at both
102  * the head and tail of the allocated block. When freeing a block, the
103  * tail length is ORed with 1 to mark it as free. Freed space can only
104  * be reclaimed from the tail forward. If the tail block is never freed,
105  * nothing else will be reclaimed until the slab is reset...
106  */
107 void *
108 slap_sl_mem_create(
109         ber_len_t size,
110         int stack,
111         void *ctx,
112         int new
113 )
114 {
115         struct slab_heap *sh;
116         ber_len_t size_shift;
117         struct slab_object *so;
118
119 #ifdef NO_THREADS
120         sh = slheap;
121 #else
122         void *sh_tmp = NULL;
123         ldap_pvt_thread_pool_getkey(
124                 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL );
125         sh = sh_tmp;
126 #endif
127
128         if ( sh && !new )
129                 return sh;
130
131         /* round up to doubleword boundary */
132         size = (size + Align-1) & -Align;
133
134         if (!sh) {
135                         sh = ch_malloc(sizeof(struct slab_heap));
136                         sh->sh_base = ch_malloc(size);
137 #ifdef NO_THREADS
138                         slheap = sh;
139 #else
140                         ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
141                                 (void *)sh, slap_sl_mem_destroy, NULL, NULL);
142 #endif
143         } else {
144                 slap_sl_mem_destroy(NULL, sh);
145                 if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
146                         void    *newptr;
147
148                         newptr = ch_realloc( sh->sh_base, size );
149                         if ( newptr == NULL ) return NULL;
150                         sh->sh_base = newptr;
151                 }
152         }
153         sh->sh_end = (char *) sh->sh_base + size;
154
155         sh->sh_stack = stack;
156         if (stack) {
157                 /* insert dummy len */
158                 {
159                         ber_len_t *i = sh->sh_base;
160                         *i++ = 0;
161                         sh->sh_last = i;
162                 }
163         } else {
164                 int i, order = -1, order_end = -1;
165
166                 size_shift = size - 1;
167                 do {
168                         order_end++;
169                 } while (size_shift >>= 1);
170                 order = order_end - order_start + 1;
171                 sh->sh_maxorder = order_end;
172
173                 sh->sh_free = (struct sh_freelist *)
174                                                 ch_malloc(order * sizeof(struct sh_freelist));
175                 for (i = 0; i < order; i++) {
176                         LDAP_LIST_INIT(&sh->sh_free[i]);
177                 }
178
179                 LDAP_LIST_INIT(&sh->sh_sopool);
180
181                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
182                         slap_replenish_sopool(sh);
183                 }
184                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
185                 LDAP_LIST_REMOVE(so, so_link);
186                 so->so_ptr = sh->sh_base;
187
188                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
189
190                 sh->sh_map = (unsigned char **)
191                                         ch_malloc(order * sizeof(unsigned char *));
192                 for (i = 0; i < order; i++) {
193                         int shiftamt = order_start + 1 + i;
194                         int nummaps = size >> shiftamt;
195                         assert(nummaps);
196                         nummaps >>= 3;
197                         if (!nummaps) nummaps = 1;
198                         sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
199                         memset(sh->sh_map[i], 0, nummaps);
200                 }
201         }
202         return sh;
203 }
204
205 void
206 slap_sl_mem_detach(
207         void *ctx,
208         void *memctx
209 )
210 {
211 #ifdef NO_THREADS
212         slheap = NULL;
213 #else
214         /* separate from context */
215         ldap_pvt_thread_pool_setkey( ctx, (void *)slap_sl_mem_init,
216                 NULL, 0, NULL, NULL );
217 #endif
218 }
219
220 void *
221 slap_sl_malloc(
222     ber_len_t   size,
223     void *ctx
224 )
225 {
226         struct slab_heap *sh = ctx;
227         ber_len_t *ptr, *newptr;
228
229 #ifdef SLAP_NO_SL_MALLOC
230         newptr = ber_memalloc_x( size, NULL );
231         if ( newptr ) return newptr;
232         assert( 0 );
233         exit( EXIT_FAILURE );
234 #endif
235
236         /* ber_set_option calls us like this */
237         if (!ctx) {
238                 newptr = ber_memalloc_x( size, NULL );
239                 if ( newptr ) return newptr;
240                 assert( 0 );
241                 exit( EXIT_FAILURE );
242         }
243
244         /* round up to doubleword boundary, plus space for len at head and tail */
245         size = (size + 2*sizeof(ber_len_t) + Align-1) & -Align;
246
247         if (sh->sh_stack) {
248                 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
249                         Debug(LDAP_DEBUG_TRACE,
250                                 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
251                                 (long)size, 0, 0);
252                         return ch_malloc(size);
253                 }
254                 newptr = sh->sh_last;
255                 sh->sh_last = (char *) sh->sh_last + size;
256                 size -= sizeof(ber_len_t);
257                 *newptr++ = size;
258                 *(ber_len_t *)((char *)sh->sh_last - sizeof(ber_len_t)) = size;
259                 return( (void *)newptr );
260         } else {
261                 struct slab_object *so_new, *so_left, *so_right;
262                 ber_len_t size_shift;
263                 unsigned long diff;
264                 int i, j, order = -1;
265
266                 size_shift = size - 1;
267                 do {
268                         order++;
269                 } while (size_shift >>= 1);
270
271                 for (i = order; i <= sh->sh_maxorder &&
272                                 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
273
274                 if (i == order) {
275                         so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
276                         LDAP_LIST_REMOVE(so_new, so_link);
277                         ptr = so_new->so_ptr;
278                         diff = (unsigned long)((char*)ptr -
279                                         (char*)sh->sh_base) >> (order + 1);
280                         sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
281                         *ptr++ = size - sizeof(ber_len_t);
282                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
283                         return((void*)ptr);
284                 } else if (i <= sh->sh_maxorder) {
285                         for (j = i; j > order; j--) {
286                                 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
287                                 LDAP_LIST_REMOVE(so_left, so_link);
288                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
289                                         slap_replenish_sopool(sh);
290                                 }
291                                 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
292                                 LDAP_LIST_REMOVE(so_right, so_link);
293                                 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
294                                 if (j == order + 1) {
295                                         ptr = so_left->so_ptr;
296                                         diff = (unsigned long)((char*)ptr -
297                                                         (char*)sh->sh_base) >> (order+1);
298                                         sh->sh_map[order-order_start][diff>>3] |=
299                                                         (1 << (diff & 0x7));
300                                         *ptr++ = size - sizeof(ber_len_t);
301                                         LDAP_LIST_INSERT_HEAD(
302                                                         &sh->sh_free[j-1-order_start], so_right, so_link);
303                                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
304                                         return((void*)ptr);
305                                 } else {
306                                         LDAP_LIST_INSERT_HEAD(
307                                                         &sh->sh_free[j-1-order_start], so_right, so_link);
308                                         LDAP_LIST_INSERT_HEAD(
309                                                         &sh->sh_free[j-1-order_start], so_left, so_link);
310                                 }
311                         }
312                 } else {
313                         Debug( LDAP_DEBUG_TRACE,
314                                 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
315                                 (long)size, 0, 0);
316                         return (void*)ch_malloc(size);
317                 }
318         }
319
320         /* FIXME: missing return; guessing... */
321         return NULL;
322 }
323
324 void *
325 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
326 {
327         void *newptr;
328
329         newptr = slap_sl_malloc( n*size, ctx );
330         if ( newptr ) {
331                 memset( newptr, 0, n*size );
332         }
333         return newptr;
334 }
335
336 void *
337 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
338 {
339         struct slab_heap *sh = ctx;
340         ber_len_t oldsize, *p = (ber_len_t *) ptr;
341         void *newptr;
342
343         if (ptr == NULL)
344                 return slap_sl_malloc(size, ctx);
345
346 #ifdef SLAP_NO_SL_MALLOC
347         newptr = ber_memrealloc_x( ptr, size, NULL );
348         if ( newptr ) return newptr;
349         assert( 0 );
350         exit( EXIT_FAILURE );
351 #endif
352
353         /* Not our memory? */
354         if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
355                 /* duplicate of realloc behavior, oh well */
356                 newptr = ber_memrealloc_x(ptr, size, NULL);
357                 if (newptr) {
358                         return newptr;
359                 }
360                 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
361                                 (long) size, 0, 0);
362                 assert(0);
363                 exit( EXIT_FAILURE );
364         }
365
366         if (size == 0) {
367                 slap_sl_free(ptr, ctx);
368                 return NULL;
369         }
370
371         oldsize = p[-1];
372
373         if (sh->sh_stack) {
374                 /* Round up to doubleword boundary, add room for head */
375                 size = ((size + Align-1) & -Align) + sizeof( ber_len_t );
376
377                 p--;
378
379                 /* Never shrink blocks */
380                 if (size <= oldsize) {
381                         return ptr;
382         
383                 /* If reallocing the last block, try to grow it */
384                 } else if ((char *) ptr + oldsize == sh->sh_last) {
385                         if (size < (char *) sh->sh_end - (char *) ptr) {
386                                 sh->sh_last = (char *) ptr + size;
387                                 p[0] = size;
388                                 p[size/sizeof(ber_len_t)] = size;
389                                 return ptr;
390                         }
391
392                 /* Nowhere to grow, need to alloc and copy */
393                 } else {
394                         /* Slight optimization of the final realloc variant */
395                         size -= sizeof(ber_len_t);
396                         oldsize -= sizeof(ber_len_t);
397                         newptr = slap_sl_malloc(size, ctx);
398                         AC_MEMCPY(newptr, ptr, oldsize);
399                         /* Not last block, can just mark old region as free */
400                         p[p[0]/sizeof(ber_len_t)] |= 1;
401                         return newptr;
402                 }
403
404                 size -= sizeof(ber_len_t);
405                 oldsize -= sizeof(ber_len_t);
406
407         } else if (oldsize > size) {
408                 oldsize = size;
409         }
410
411         newptr = slap_sl_malloc(size, ctx);
412         AC_MEMCPY(newptr, ptr, oldsize);
413         slap_sl_free(ptr, ctx);
414         return newptr;
415 }
416
417 void
418 slap_sl_free(void *ptr, void *ctx)
419 {
420         struct slab_heap *sh = ctx;
421         ber_len_t size;
422         ber_len_t *p = (ber_len_t *)ptr, *tmpp;
423
424         if (!ptr)
425                 return;
426
427 #ifdef SLAP_NO_SL_MALLOC
428         ber_memfree_x( ptr, NULL );
429         return;
430 #endif
431
432         if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
433                 ber_memfree_x(ptr, NULL);
434
435         } else if (sh->sh_stack) {
436                 size = p[-1];
437                 p = (ber_len_t *) ((char *) ptr + size);
438                 /* mark it free */
439                 p[-1] = size |= 1;
440                 /* reclaim free space off tail */
441                 if (sh->sh_last == p) {
442                         do {
443                                 p = (ber_len_t *) ((char *) p - size + 1) - 1;
444                                 size = p[-1];
445                         } while (size & 1);
446                         sh->sh_last = p;
447                 }
448
449         } else {
450                 int size_shift, order_size;
451                 struct slab_object *so;
452                 unsigned long diff;
453                 int i, inserted = 0, order = -1;
454
455                 size = *(--p);
456                 size_shift = size + sizeof(ber_len_t) - 1;
457                 do {
458                         order++;
459                 } while (size_shift >>= 1);
460
461                 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
462                         order_size = 1 << (i+1);
463                         diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
464                         sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
465                         if (diff == ((diff>>1)<<1)) {
466                                 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
467                                                 (1<<((diff+1)&0x7)))) {
468                                         so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
469                                         while (so) {
470                                                 if ((char*)so->so_ptr == (char*)tmpp) {
471                                                         LDAP_LIST_REMOVE( so, so_link );
472                                                 } else if ((char*)so->so_ptr ==
473                                                                 (char*)tmpp + order_size) {
474                                                         LDAP_LIST_REMOVE(so, so_link);
475                                                         break;
476                                                 }
477                                                 so = LDAP_LIST_NEXT(so, so_link);
478                                         }
479                                         if (so) {
480                                                 if (i < sh->sh_maxorder) {
481                                                         inserted = 1;
482                                                         so->so_ptr = tmpp;
483                                                         LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
484                                                                         so, so_link);
485                                                 }
486                                                 continue;
487                                         } else {
488                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
489                                                         slap_replenish_sopool(sh);
490                                                 }
491                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
492                                                 LDAP_LIST_REMOVE(so, so_link);
493                                                 so->so_ptr = tmpp;
494                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
495                                                                 so, so_link);
496                                                 break;
497
498                                                 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
499                                                         "free object not found while bit is clear.\n",
500                                                         0, 0, 0);
501                                                 assert(so != NULL);
502
503                                         }
504                                 } else {
505                                         if (!inserted) {
506                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
507                                                         slap_replenish_sopool(sh);
508                                                 }
509                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
510                                                 LDAP_LIST_REMOVE(so, so_link);
511                                                 so->so_ptr = tmpp;
512                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
513                                                                 so, so_link);
514                                         }
515                                         break;
516                                 }
517                         } else {
518                                 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
519                                                 (1<<((diff-1)&0x7)))) {
520                                         so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
521                                         while (so) {
522                                                 if ((char*)so->so_ptr == (char*)tmpp) {
523                                                         LDAP_LIST_REMOVE(so, so_link);
524                                                 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
525                                                         LDAP_LIST_REMOVE(so, so_link);
526                                                         tmpp = so->so_ptr;
527                                                         break;
528                                                 }
529                                                 so = LDAP_LIST_NEXT(so, so_link);
530                                         }
531                                         if (so) {
532                                                 if (i < sh->sh_maxorder) {
533                                                         inserted = 1;
534                                                         LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],                                                                    so, so_link);
535                                                         continue;
536                                                 }
537                                         } else {
538                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
539                                                         slap_replenish_sopool(sh);
540                                                 }
541                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
542                                                 LDAP_LIST_REMOVE(so, so_link);
543                                                 so->so_ptr = tmpp;
544                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
545                                                                 so, so_link);
546                                                 break;
547
548                                                 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
549                                                         "free object not found while bit is clear.\n",
550                                                         0, 0, 0 );
551                                                 assert(so != NULL);
552
553                                         }
554                                 } else {
555                                         if ( !inserted ) {
556                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
557                                                         slap_replenish_sopool(sh);
558                                                 }
559                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
560                                                 LDAP_LIST_REMOVE(so, so_link);
561                                                 so->so_ptr = tmpp;
562                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
563                                                                 so, so_link);
564                                         }
565                                         break;
566                                 }
567                         }
568                 }
569         }
570 }
571
572 void *
573 slap_sl_context( void *ptr )
574 {
575         struct slab_heap *sh;
576         void *ctx, *sh_tmp;
577
578         if ( slapMode & SLAP_TOOL_MODE ) return NULL;
579
580 #ifdef NO_THREADS
581         sh = slheap;
582 #else
583         ctx = ldap_pvt_thread_pool_context();
584
585         sh_tmp = NULL;
586         ldap_pvt_thread_pool_getkey(
587                 ctx, (void *)slap_sl_mem_init, &sh_tmp, NULL);
588         sh = sh_tmp;
589 #endif
590
591         if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
592                 return sh;
593         }
594         return NULL;
595 }
596
597 static struct slab_object *
598 slap_replenish_sopool(
599     struct slab_heap* sh
600 )
601 {
602     struct slab_object *so_block;
603     int i;
604
605     so_block = (struct slab_object *)ch_malloc(
606                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
607
608     if ( so_block == NULL ) {
609         return NULL;
610     }
611
612     so_block[0].so_blockhead = 1;
613     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
614     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
615         so_block[i].so_blockhead = 0;
616         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
617     }
618
619     return so_block;
620 }
621
622 #ifdef SLAPD_UNUSED
623 static void
624 print_slheap(int level, void *ctx)
625 {
626         struct slab_heap *sh = ctx;
627         struct slab_object *so;
628         int i, j, once = 0;
629
630         if (!ctx) {
631                 Debug(level, "NULL memctx\n", 0, 0, 0);
632                 return;
633         }
634
635         Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
636
637         for (i = order_start; i <= sh->sh_maxorder; i++) {
638                 once = 0;
639                 Debug(level, "order=%d\n", i, 0, 0);
640                 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
641                         Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
642                         once = 1;
643                 }
644                 if (!once) {
645                         Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
646                 }
647                 Debug(level, "\n", 0, 0, 0);
648                 Debug(level, "free list:\n", 0, 0, 0);
649                 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
650                 while (so) {
651                         Debug(level, "%lx\n", (unsigned long) so->so_ptr, 0, 0);
652                         so = LDAP_LIST_NEXT(so, so_link);
653                 }
654         }
655 }
656 #endif