]> git.sur5r.net Git - openldap/blob - servers/slapd/sl_malloc.c
restructuring and cleanup
[openldap] / servers / slapd / sl_malloc.c
1 /* sl_malloc.c - malloc routines using a per-thread slab */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4  *
5  * Copyright 2003-2004 The OpenLDAP Foundation.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted only as authorized by the OpenLDAP
10  * Public License.
11  *
12  * A copy of this license is available in the file LICENSE in the
13  * top-level directory of the distribution or, alternatively, at
14  * <http://www.OpenLDAP.org/license.html>.
15  */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21
22 #include "slap.h"
23
24 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
25 static void print_slheap(int level, void *ctx);
26
27 void
28 slap_sl_mem_destroy(
29         void *key,
30         void *data
31 )
32 {
33         struct slab_heap *sh = data;
34         int pad = 2*sizeof(int)-1, pad_shift;
35         int order_start = -1, i;
36         struct slab_object *so;
37
38         if (sh->sh_stack) {
39                 ber_memfree_x(sh->sh_base, NULL);
40                 ber_memfree_x(sh, NULL);
41         } else {
42                 pad_shift = pad - 1;
43                 do {
44                         order_start++;
45                 } while (pad_shift >>= 1);
46
47                 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
48                         so = LDAP_LIST_FIRST(&sh->sh_free[i]);
49                         while (so) {
50                                 struct slab_object *so_tmp = so;
51                                 so = LDAP_LIST_NEXT(so, so_link);
52                                 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
53                         }
54                         ch_free(sh->sh_map[i]);
55                 }
56                 ch_free(sh->sh_free);
57                 ch_free(sh->sh_map);
58
59                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
60                 while (so) {
61                         struct slab_object *so_tmp = so;
62                         so = LDAP_LIST_NEXT(so, so_link);
63                         if (!so_tmp->so_blockhead) {
64                                 LDAP_LIST_REMOVE(so_tmp, so_link);
65                         }
66                 }
67                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
68                 while (so) {
69                         struct slab_object *so_tmp = so;
70                         so = LDAP_LIST_NEXT(so, so_link);
71                         ch_free(so_tmp);
72                 }
73                 ber_memfree_x(sh->sh_base, NULL);
74                 ber_memfree_x(sh, NULL);
75         }
76 }
77
78 BerMemoryFunctions slap_sl_mfuncs =
79         { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
80
81 void
82 slap_sl_mem_init()
83 {
84         ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
85 }
86
87 #ifdef NO_THREADS
88 static struct slab_heap *slheap;
89 #endif
90
91 void *
92 slap_sl_mem_create(
93         ber_len_t size,
94         int stack,
95         void *ctx
96 )
97 {
98         struct slab_heap *sh = NULL;
99         ber_len_t size_shift;
100         int pad = 2*sizeof(int)-1, pad_shift;
101         int order = -1, order_start = -1, order_end = -1;
102         int i, k;
103         struct slab_object *so;
104
105 #ifdef NO_THREADS
106         sh = slheap;
107 #else
108         ldap_pvt_thread_pool_getkey(
109                 ctx, (void *)slap_sl_mem_init, (void **)&sh, NULL );
110 #endif
111
112         /* round up to doubleword boundary */
113         size += pad;
114         size &= ~pad;
115
116         if (stack) {
117                 if (!sh) {
118                         sh = ch_malloc(sizeof(struct slab_heap));
119                         sh->sh_base = ch_malloc(size);
120 #ifdef NO_THREADS
121                         slheap = sh;
122 #else
123                         ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
124                                 (void *)sh, slap_sl_mem_destroy);
125 #endif
126                 } else if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
127                         sh->sh_base = ch_realloc(sh->sh_base, size);
128                 }
129                 sh->sh_last = sh->sh_base;
130                 sh->sh_end = (char *) sh->sh_base + size;
131                 sh->sh_stack = stack;
132                 return sh;
133         } else {
134                 size_shift = size - 1;
135                 do {
136                         order_end++;
137                 } while (size_shift >>= 1);
138
139                 pad_shift = pad - 1;
140                 do {
141                         order_start++;
142                 } while (pad_shift >>= 1);
143
144                 order = order_end - order_start + 1;
145
146                 if (!sh) {
147                         sh = (struct slab_heap *) ch_malloc(sizeof(struct slab_heap));
148                         sh->sh_base = ch_malloc(size);
149 #ifdef NO_THREADS
150                         slheap = sh;
151 #else
152                         ldap_pvt_thread_pool_setkey(ctx, (void *)slap_sl_mem_init,
153                                 (void *)sh, slap_sl_mem_destroy);
154 #endif
155                 } else {
156                         for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
157                                 so = LDAP_LIST_FIRST(&sh->sh_free[i]);
158                                 while (so) {
159                                         struct slab_object *so_tmp = so;
160                                         so = LDAP_LIST_NEXT(so, so_link);
161                                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
162                                 }
163                                 ch_free(sh->sh_map[i]);
164                         }
165                         ch_free(sh->sh_free);
166                         ch_free(sh->sh_map);
167
168                         so = LDAP_LIST_FIRST(&sh->sh_sopool);
169                         while (so) {
170                                 struct slab_object *so_tmp = so;
171                                 so = LDAP_LIST_NEXT(so, so_link);
172                                 if (!so_tmp->so_blockhead) {
173                                         LDAP_LIST_REMOVE(so_tmp, so_link);
174                                 }
175                         }
176                         so = LDAP_LIST_FIRST(&sh->sh_sopool);
177                         while (so) {
178                                 struct slab_object *so_tmp = so;
179                                 so = LDAP_LIST_NEXT(so, so_link);
180                                 ch_free(so_tmp);
181                         }
182
183                         if (size > (char *)sh->sh_end - (char *)sh->sh_base) {
184                                 sh->sh_base = realloc(sh->sh_base, size);
185                         }
186                 }
187                 sh->sh_end = (char *)sh->sh_base + size;
188                 sh->sh_maxorder = order_end;
189
190                 sh->sh_free = (struct sh_freelist *)
191                                                 ch_malloc(order * sizeof(struct sh_freelist));
192                 for (i = 0; i < order; i++) {
193                         LDAP_LIST_INIT(&sh->sh_free[i]);
194                 }
195
196                 LDAP_LIST_INIT(&sh->sh_sopool);
197
198                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
199                         slap_replenish_sopool(sh);
200                 }
201                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
202                 LDAP_LIST_REMOVE(so, so_link);
203                 so->so_ptr = sh->sh_base;
204
205                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
206
207                 sh->sh_map = (unsigned char **)
208                                         ch_malloc(order * sizeof(unsigned char *));
209                 for (i = 0; i < order; i++) {
210                         sh->sh_map[i] = (unsigned char *)
211                                                         ch_malloc(size >> (1 << (order_start + i + 3)));
212                         memset(sh->sh_map[i], 0, size >> (1 << (order_start + i + 3)));
213                 }
214                 sh->sh_stack = stack;
215                 return sh;
216         }
217 }
218
219 void
220 slap_sl_mem_detach(
221         void *ctx,
222         void *memctx
223 )
224 {
225 #ifdef NO_THREADS
226         slheap = NULL;
227 #else
228         /* separate from context */
229         ldap_pvt_thread_pool_setkey( ctx, (void *)slap_sl_mem_init, NULL, NULL );
230 #endif
231 }
232
233 void *
234 slap_sl_malloc(
235     ber_len_t   size,
236     void *ctx
237 )
238 {
239         struct slab_heap *sh = ctx;
240         ber_len_t size_shift;
241         int pad = 2*sizeof(int)-1, pad_shift;
242         int order = -1, order_start = -1;
243         struct slab_object *so_new, *so_left, *so_right;
244         ber_len_t *ptr, *new;
245         unsigned long diff;
246         int i, j, k;
247
248         /* ber_set_option calls us like this */
249         if (!ctx) return ber_memalloc_x(size, NULL);
250
251         Debug(LDAP_DEBUG_TRACE, "==> slap_sl_malloc (%d)\n", size, 0, 0);
252
253         /* round up to doubleword boundary */
254         size += sizeof(ber_len_t) + pad;
255         size &= ~pad;
256
257         if (sh->sh_stack) {
258                 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
259                         Debug(LDAP_DEBUG_TRACE,
260                                 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
261                                 (long)size, 0, 0);
262                         return ch_malloc(size);
263                 }
264                 new = sh->sh_last;
265                 *new++ = size - sizeof(ber_len_t);
266                 sh->sh_last = (char *) sh->sh_last + size;
267                 return( (void *)new );
268         } else {
269                 size_shift = size - 1;
270                 do {
271                         order++;
272                 } while (size_shift >>= 1);
273
274                 pad_shift = pad - 1;
275                 do {
276                         order_start++;
277                 } while (pad_shift >>= 1);
278
279                 for (i = order; i <= sh->sh_maxorder &&
280                                 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
281
282                 if (i == order) {
283                         so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
284                         LDAP_LIST_REMOVE(so_new, so_link);
285                         ptr = so_new->so_ptr;
286                         diff = (unsigned long)((char*)ptr -
287                                         (char*)sh->sh_base) >> (order + 1);
288                         sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
289                         *ptr++ = size - sizeof(ber_len_t);
290                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
291                         return((void*)ptr);
292                 } else if (i <= sh->sh_maxorder) {
293                         for (j = i; j > order; j--) {
294                                 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
295                                 LDAP_LIST_REMOVE(so_left, so_link);
296                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
297                                         slap_replenish_sopool(sh);
298                                 }
299                                 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
300                                 LDAP_LIST_REMOVE(so_right, so_link);
301                                 so_right->so_ptr = so_left->so_ptr + (1 << j);
302                                 if (j == order + 1) {
303                                         ptr = so_left->so_ptr;
304                                         diff = (unsigned long)((char*)ptr -
305                                                         (char*)sh->sh_base) >> (order+1);
306                                         sh->sh_map[order-order_start][diff>>3] |=
307                                                         (1 << (diff & 0x7));
308                                         *ptr++ = size - sizeof(ber_len_t);
309                                         LDAP_LIST_INSERT_HEAD(
310                                                         &sh->sh_free[j-1-order_start], so_right, so_link);
311                                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
312                                         return((void*)ptr);
313                                 } else {
314                                         LDAP_LIST_INSERT_HEAD(
315                                                         &sh->sh_free[j-1-order_start], so_right, so_link);
316                                         LDAP_LIST_INSERT_HEAD(
317                                                         &sh->sh_free[j-1-order_start], so_left, so_link);
318                                 }
319                         }
320                 } else {
321                         Debug( LDAP_DEBUG_TRACE,
322                                 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
323                                 (long)size, 0, 0);
324                         return (void*)ch_malloc(size);
325                 }
326         }
327 }
328
329 void *
330 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
331 {
332         void *new;
333
334         new = slap_sl_malloc( n*size, ctx );
335         if ( new ) {
336                 memset( new, 0, n*size );
337         }
338         return new;
339 }
340
341 void *
342 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
343 {
344         struct slab_heap *sh = ctx;
345         int size_shift;
346         int pad = 2*sizeof(int)-1, pad_shift;
347         int order_start = -1, order = -1;
348         struct slab_object *so;
349         ber_len_t *p = (ber_len_t *)ptr, *new;
350         unsigned long diff;
351
352         if (ptr == NULL)
353                 return slap_sl_malloc(size, ctx);
354
355         /* Not our memory? */
356         if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
357                 /* duplicate of realloc behavior, oh well */
358                 new = ber_memrealloc_x(ptr, size, NULL);
359                 if (new) {
360                         return new;
361                 }
362                 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
363                                 (long) size, 0, 0);
364                 assert(0);
365                 exit( EXIT_FAILURE );
366         }
367
368         if (size == 0) {
369                 slap_sl_free(ptr, ctx);
370                 return NULL;
371         }
372
373         if (sh->sh_stack) {
374                 /* Never shrink blocks */
375                 if (size <= p[-1]) {
376                         new = p;
377         
378                 /* If reallocing the last block, we can grow it */
379                 } else if ((char *)ptr + p[-1] == sh->sh_last) {
380                         new = p;
381                         sh->sh_last = (char *)sh->sh_last + size - p[-1];
382                         p[-1] = size;
383         
384                 /* Nowhere to grow, need to alloc and copy */
385                 } else {
386                         new = slap_sl_malloc(size, ctx);
387                         AC_MEMCPY(new, ptr, p[-1]);
388                 }
389                 return new;
390         } else {
391                 void *newptr;
392                 newptr = slap_sl_malloc(size, ctx);
393                 if (size < p[-1]) {
394                         AC_MEMCPY(newptr, ptr, size);
395                 } else {
396                         AC_MEMCPY(newptr, ptr, p[-1]);
397                 }
398                 slap_sl_free(ptr, ctx);
399                 return newptr;
400         }
401 }
402
403 void
404 slap_sl_free(void *ptr, void *ctx)
405 {
406         struct slab_heap *sh = ctx;
407         int size, size_shift, order_size;
408         int pad = 2*sizeof(int)-1, pad_shift;
409         ber_len_t *p = (ber_len_t *)ptr, *tmpp;
410         int order_start = -1, order = -1;
411         struct slab_object *so;
412         unsigned long diff;
413         int i, k, inserted = 0;
414
415         Debug( LDAP_DEBUG_TRACE, "==> slap_sl_free \n", 0, 0, 0);
416
417         if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
418                 ber_memfree_x(ptr, NULL);
419         } else if (sh->sh_stack && (char *)ptr + p[-1] == sh->sh_last) {
420                 p--;
421                 sh->sh_last = p;
422         } else if (!sh->sh_stack) {
423                 size = *(--p);
424                 size_shift = size + sizeof(ber_len_t) - 1;
425                 do {
426                         order++;
427                 } while (size_shift >>= 1);
428
429                 pad_shift = pad - 1;
430                 do {
431                         order_start++;
432                 } while (pad_shift >>= 1);
433
434                 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
435                         order_size = 1 << (i+1);
436                         diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
437                         sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
438                         if (diff == ((diff>>1)<<1)) {
439                                 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
440                                                 (1<<((diff+1)&0x7)))) {
441                                         so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
442                                         while (so) {
443                                                 if ((char*)so->so_ptr == (char*)tmpp) {
444                                                         LDAP_LIST_REMOVE( so, so_link );
445                                                 } else if ((char*)so->so_ptr ==
446                                                                 (char*)tmpp + order_size) {
447                                                         LDAP_LIST_REMOVE(so, so_link);
448                                                         break;
449                                                 }
450                                                 so = LDAP_LIST_NEXT(so, so_link);
451                                         }
452                                         if (so) {
453                                                 if (i < sh->sh_maxorder) {
454                                                         inserted = 1;
455                                                         so->so_ptr = tmpp;
456                                                         LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
457                                                                         so, so_link);
458                                                 }
459                                                 continue;
460                                         } else {
461                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
462                                                         slap_replenish_sopool(sh);
463                                                 }
464                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
465                                                 LDAP_LIST_REMOVE(so, so_link);
466                                                 so->so_ptr = tmpp;
467                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
468                                                                 so, so_link);
469                                                 break;
470
471                                                 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
472                                                         "free object not found while bit is clear.\n",
473                                                         0, 0, 0);
474                                                 assert(so);
475
476                                         }
477                                 } else {
478                                         if (!inserted) {
479                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
480                                                         slap_replenish_sopool(sh);
481                                                 }
482                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
483                                                 LDAP_LIST_REMOVE(so, so_link);
484                                                 so->so_ptr = tmpp;
485                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
486                                                                 so, so_link);
487                                         }
488                                         break;
489                                 }
490                         } else {
491                                 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
492                                                 (1<<((diff-1)&0x7)))) {
493                                         so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
494                                         while (so) {
495                                                 if ((char*)so->so_ptr == (char*)tmpp) {
496                                                         LDAP_LIST_REMOVE(so, so_link);
497                                                 } else if ((char*)tmpp == so->so_ptr + order_size) {
498                                                         LDAP_LIST_REMOVE(so, so_link);
499                                                         tmpp = so->so_ptr;
500                                                         break;
501                                                 }
502                                                 so = LDAP_LIST_NEXT(so, so_link);
503                                         }
504                                         if (so) {
505                                                 if (i < sh->sh_maxorder) {
506                                                         inserted = 1;
507                                                         LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],                                                                    so, so_link);
508                                                         continue;
509                                                 }
510                                         } else {
511                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
512                                                         slap_replenish_sopool(sh);
513                                                 }
514                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
515                                                 LDAP_LIST_REMOVE(so, so_link);
516                                                 so->so_ptr = tmpp;
517                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
518                                                                 so, so_link);
519                                                 break;
520
521                                                 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
522                                                         "free object not found while bit is clear.\n",
523                                                         0, 0, 0 );
524                                                 assert( so );
525
526                                         }
527                                 } else {
528                                         if ( !inserted ) {
529                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
530                                                         slap_replenish_sopool(sh);
531                                                 }
532                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
533                                                 LDAP_LIST_REMOVE(so, so_link);
534                                                 so->so_ptr = tmpp;
535                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
536                                                                 so, so_link);
537                                         }
538                                         break;
539                                 }
540                         }
541                 }
542         }
543 }
544
545 void *
546 slap_sl_context( void *ptr )
547 {
548         struct slab_heap *sh = NULL;
549         void *ctx;
550
551 #ifdef NO_THREADS
552         sh = slheap;
553 #else
554         ctx = ldap_pvt_thread_pool_context();
555
556         ldap_pvt_thread_pool_getkey(ctx, (void *)slap_sl_mem_init,
557                         (void **)&sh, NULL);
558 #endif
559
560         if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
561                 return sh;
562         }
563         return NULL;
564 }
565
566 static struct slab_object *
567 slap_replenish_sopool(
568     struct slab_heap* sh
569 )
570 {
571     struct slab_object *so_block;
572     int i;
573
574     so_block = (struct slab_object *)ch_malloc(
575                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
576
577     if ( so_block == NULL ) {
578         return NULL;
579     }
580
581     so_block[0].so_blockhead = 1;
582     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
583     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
584         so_block[i].so_blockhead = 0;
585         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
586     }
587
588     return so_block;
589 }
590
591 static void
592 print_slheap(int level, void *ctx)
593 {
594         struct slab_heap *sh = ctx;
595         int order_start = -1;
596         int pad = 2*sizeof(int)-1, pad_shift;
597         struct slab_object *so;
598         int i, j, once = 0;
599
600         if (!ctx) {
601                 Debug(level, "NULL memctx\n", 0, 0, 0);
602                 return;
603         }
604
605         pad_shift = pad - 1;
606         do {
607                 order_start++;
608         } while (pad_shift >>= 1);
609
610         Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
611
612         for (i = order_start; i <= sh->sh_maxorder; i++) {
613                 once = 0;
614                 Debug(level, "order=%d\n", i, 0, 0);
615                 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
616                         Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
617                         once = 1;
618                 }
619                 if (!once) {
620                         Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
621                 }
622                 Debug(level, "\n", 0, 0, 0);
623                 Debug(level, "free list:\n", 0, 0, 0);
624                 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
625                 while (so) {
626                         Debug(level, "%x\n",so->so_ptr, 0, 0);
627                         so = LDAP_LIST_NEXT(so, so_link);
628                 }
629         }
630 }