]> git.sur5r.net Git - openldap/blob - servers/slapd/sl_malloc.c
ITS#6437, failure/fallback/debug handling:
[openldap] / servers / slapd / sl_malloc.c
1 /* sl_malloc.c - malloc routines using a per-thread slab */
2 /* $OpenLDAP$ */
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
4  *
5  * Copyright 2003-2011 The OpenLDAP Foundation.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted only as authorized by the OpenLDAP
10  * Public License.
11  *
12  * A copy of this license is available in the file LICENSE in the
13  * top-level directory of the distribution or, alternatively, at
14  * <http://www.OpenLDAP.org/license.html>.
15  */
16
17 #include "portable.h"
18
19 #include <stdio.h>
20 #include <ac/string.h>
21
22 #include "slap.h"
23
24 enum {
25         Align = 2 * sizeof(int),
26         Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
27         order_start = Align_log2 - 1,
28         pad = Align - 1
29 };
30
31 static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
32 #ifdef SLAPD_UNUSED
33 static void print_slheap(int level, void *ctx);
34 #endif
35
36 /* Keep memory context in a thread-local var, or in a global when no threads */
37 #ifdef NO_THREADS
38 static struct slab_heap *slheap;
39 # define SET_MEMCTX(thrctx, memctx, sfree)      ((void) (slheap = (memctx)))
40 # define GET_MEMCTX(thrctx, memctxp)            (*(memctxp) = slheap))
41 #else
42 # define memctx_key ((void *) slap_sl_mem_init)
43 # define SET_MEMCTX(thrctx, memctx, kfree) \
44         ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
45 # define GET_MEMCTX(thrctx, memctxp) \
46         ((void) (*(memctxp) = NULL), \
47          (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
48          *(memctxp))
49 #endif /* NO_THREADS */
50
51
52 /* Destroy the context, or if key==NULL clean it up for reuse. */
53 void
54 slap_sl_mem_destroy(
55         void *key,
56         void *data
57 )
58 {
59         struct slab_heap *sh = data;
60         struct slab_object *so;
61         int i;
62
63         if (!sh->sh_stack) {
64                 for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
65                         so = LDAP_LIST_FIRST(&sh->sh_free[i]);
66                         while (so) {
67                                 struct slab_object *so_tmp = so;
68                                 so = LDAP_LIST_NEXT(so, so_link);
69                                 LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
70                         }
71                         ch_free(sh->sh_map[i]);
72                 }
73                 ch_free(sh->sh_free);
74                 ch_free(sh->sh_map);
75
76                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
77                 while (so) {
78                         struct slab_object *so_tmp = so;
79                         so = LDAP_LIST_NEXT(so, so_link);
80                         if (!so_tmp->so_blockhead) {
81                                 LDAP_LIST_REMOVE(so_tmp, so_link);
82                         }
83                 }
84                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
85                 while (so) {
86                         struct slab_object *so_tmp = so;
87                         so = LDAP_LIST_NEXT(so, so_link);
88                         ch_free(so_tmp);
89                 }
90         }
91
92         if (key != NULL) {
93                 ber_memfree_x(sh->sh_base, NULL);
94                 ber_memfree_x(sh, NULL);
95         }
96 }
97
98 BerMemoryFunctions slap_sl_mfuncs =
99         { slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
100
101 void
102 slap_sl_mem_init()
103 {
104         assert( Align == 1 << Align_log2 );
105         /* Adding head+tail preserves alignment */
106         assert( 2*sizeof(ber_len_t) % Align == 0 );
107
108         ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
109 }
110
111 /* This allocator always returns memory aligned on a 2-int boundary.
112  *
113  * The stack-based allocator stores the size as a ber_len_t at both
114  * the head and tail of the allocated block. When freeing a block, the
115  * tail length is ORed with 1 to mark it as free. Freed space can only
116  * be reclaimed from the tail forward. If the tail block is never freed,
117  * nothing else will be reclaimed until the slab is reset...
118  */
119 void *
120 slap_sl_mem_create(
121         ber_len_t size,
122         int stack,
123         void *thrctx,
124         int new
125 )
126 {
127         void *memctx;
128         struct slab_heap *sh;
129         ber_len_t size_shift;
130         struct slab_object *so;
131
132         sh = GET_MEMCTX(thrctx, &memctx);
133         if ( sh && !new )
134                 return sh;
135
136         /* round up to doubleword boundary */
137         size = (size + Align-1) & -Align;
138
139         if (!sh) {
140                 sh = ch_malloc(sizeof(struct slab_heap));
141                 sh->sh_base = ch_malloc(size);
142                 SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
143         } else {
144                 slap_sl_mem_destroy(NULL, sh);
145                 if ( size > (char *)sh->sh_end - (char *)sh->sh_base ) {
146                         void    *newptr;
147
148                         newptr = ch_realloc( sh->sh_base, size );
149                         if ( newptr == NULL ) return NULL;
150                         sh->sh_base = newptr;
151                 }
152         }
153         sh->sh_end = (char *) sh->sh_base + size;
154
155         sh->sh_stack = stack;
156         if (stack) {
157                 /* insert dummy len */
158                 {
159                         ber_len_t *i = sh->sh_base;
160                         *i++ = 0;
161                         sh->sh_last = i;
162                 }
163         } else {
164                 int i, order = -1, order_end = -1;
165
166                 size_shift = size - 1;
167                 do {
168                         order_end++;
169                 } while (size_shift >>= 1);
170                 order = order_end - order_start + 1;
171                 sh->sh_maxorder = order_end;
172
173                 sh->sh_free = (struct sh_freelist *)
174                                                 ch_malloc(order * sizeof(struct sh_freelist));
175                 for (i = 0; i < order; i++) {
176                         LDAP_LIST_INIT(&sh->sh_free[i]);
177                 }
178
179                 LDAP_LIST_INIT(&sh->sh_sopool);
180
181                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
182                         slap_replenish_sopool(sh);
183                 }
184                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
185                 LDAP_LIST_REMOVE(so, so_link);
186                 so->so_ptr = sh->sh_base;
187
188                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
189
190                 sh->sh_map = (unsigned char **)
191                                         ch_malloc(order * sizeof(unsigned char *));
192                 for (i = 0; i < order; i++) {
193                         int shiftamt = order_start + 1 + i;
194                         int nummaps = size >> shiftamt;
195                         assert(nummaps);
196                         nummaps >>= 3;
197                         if (!nummaps) nummaps = 1;
198                         sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
199                         memset(sh->sh_map[i], 0, nummaps);
200                 }
201         }
202         return sh;
203 }
204
205 void
206 slap_sl_mem_detach(
207         void *thrctx,
208         void *memctx
209 )
210 {
211         /* separate from context */
212         SET_MEMCTX(thrctx, NULL, 0);
213 }
214
215 void *
216 slap_sl_malloc(
217     ber_len_t   size,
218     void *ctx
219 )
220 {
221         struct slab_heap *sh = ctx;
222         ber_len_t *ptr, *newptr;
223
224 #ifdef SLAP_NO_SL_MALLOC
225         newptr = ber_memalloc_x( size, NULL );
226         if ( newptr ) return newptr;
227         assert( 0 );
228         exit( EXIT_FAILURE );
229 #endif
230
231         /* ber_set_option calls us like this */
232         if (!ctx) {
233                 newptr = ber_memalloc_x( size, NULL );
234                 if ( newptr ) return newptr;
235                 Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
236                         (unsigned long) size, 0, 0);
237                 assert( 0 );
238                 exit( EXIT_FAILURE );
239         }
240
241         /* round up to doubleword boundary, plus space for len at head and tail */
242         size = (size + 2*sizeof(ber_len_t) + Align-1) & -Align;
243
244         if (sh->sh_stack) {
245                 if ((char *)sh->sh_last + size >= (char *)sh->sh_end) {
246                         size -= 2*sizeof(ber_len_t);
247                         Debug(LDAP_DEBUG_TRACE,
248                                 "slap_sl_malloc of %lu bytes failed, using ch_malloc\n",
249                                 (unsigned long) size, 0, 0);
250                         return ch_malloc(size);
251                 }
252                 newptr = sh->sh_last;
253                 sh->sh_last = (char *) sh->sh_last + size;
254                 size -= sizeof(ber_len_t);
255                 *newptr++ = size;
256                 *(ber_len_t *)((char *)sh->sh_last - sizeof(ber_len_t)) = size;
257                 return( (void *)newptr );
258         } else {
259                 struct slab_object *so_new, *so_left, *so_right;
260                 ber_len_t size_shift;
261                 unsigned long diff;
262                 int i, j, order = -1;
263
264                 size_shift = size - 1;
265                 do {
266                         order++;
267                 } while (size_shift >>= 1);
268
269                 size -= sizeof(ber_len_t);
270
271                 for (i = order; i <= sh->sh_maxorder &&
272                                 LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
273
274                 if (i == order) {
275                         so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
276                         LDAP_LIST_REMOVE(so_new, so_link);
277                         ptr = so_new->so_ptr;
278                         diff = (unsigned long)((char*)ptr -
279                                         (char*)sh->sh_base) >> (order + 1);
280                         sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
281                         *ptr++ = size;
282                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
283                         return((void*)ptr);
284                 } else if (i <= sh->sh_maxorder) {
285                         for (j = i; j > order; j--) {
286                                 so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
287                                 LDAP_LIST_REMOVE(so_left, so_link);
288                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
289                                         slap_replenish_sopool(sh);
290                                 }
291                                 so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
292                                 LDAP_LIST_REMOVE(so_right, so_link);
293                                 so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
294                                 if (j == order + 1) {
295                                         ptr = so_left->so_ptr;
296                                         diff = (unsigned long)((char*)ptr -
297                                                         (char*)sh->sh_base) >> (order+1);
298                                         sh->sh_map[order-order_start][diff>>3] |=
299                                                         (1 << (diff & 0x7));
300                                         *ptr++ = size;
301                                         LDAP_LIST_INSERT_HEAD(
302                                                         &sh->sh_free[j-1-order_start], so_right, so_link);
303                                         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
304                                         return((void*)ptr);
305                                 } else {
306                                         LDAP_LIST_INSERT_HEAD(
307                                                         &sh->sh_free[j-1-order_start], so_right, so_link);
308                                         LDAP_LIST_INSERT_HEAD(
309                                                         &sh->sh_free[j-1-order_start], so_left, so_link);
310                                 }
311                         }
312                 }
313                 /* FIXME: missing return; guessing we failed... */
314         }
315
316         Debug(LDAP_DEBUG_TRACE,
317                 "sl_malloc %lu: ch_malloc\n",
318                 (unsigned long) size, 0, 0);
319         return ch_malloc(size);
320 }
321
322 void *
323 slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
324 {
325         void *newptr;
326
327         newptr = slap_sl_malloc( n*size, ctx );
328         if ( newptr ) {
329                 memset( newptr, 0, n*size );
330         }
331         return newptr;
332 }
333
334 void *
335 slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
336 {
337         struct slab_heap *sh = ctx;
338         ber_len_t oldsize, *p = (ber_len_t *) ptr;
339         void *newptr;
340
341         if (ptr == NULL)
342                 return slap_sl_malloc(size, ctx);
343
344 #ifdef SLAP_NO_SL_MALLOC
345         newptr = ber_memrealloc_x( ptr, size, NULL );
346         if ( newptr ) return newptr;
347         assert( 0 );
348         exit( EXIT_FAILURE );
349 #endif
350
351         /* Not our memory? */
352         if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
353                 /* Like ch_realloc(), except not trying a new context */
354                 newptr = ber_memrealloc_x(ptr, size, NULL);
355                 if (newptr) {
356                         return newptr;
357                 }
358                 Debug(LDAP_DEBUG_ANY, "ch_realloc of %lu bytes failed\n",
359                         (unsigned long) size, 0, 0);
360                 assert(0);
361                 exit( EXIT_FAILURE );
362         }
363
364         if (size == 0) {
365                 slap_sl_free(ptr, ctx);
366                 return NULL;
367         }
368
369         oldsize = p[-1];
370
371         if (sh->sh_stack) {
372                 /* Round up to doubleword boundary, add room for head */
373                 size = ((size + Align-1) & -Align) + sizeof( ber_len_t );
374
375                 p--;
376
377                 /* Never shrink blocks */
378                 if (size <= oldsize) {
379                         return ptr;
380         
381                 /* If reallocing the last block, try to grow it */
382                 } else if ((char *) ptr + oldsize == sh->sh_last) {
383                         if (size < (char *) sh->sh_end - (char *) ptr) {
384                                 sh->sh_last = (char *) ptr + size;
385                                 p[0] = size;
386                                 p[size/sizeof(ber_len_t)] = size;
387                                 return ptr;
388                         }
389
390                 /* Nowhere to grow, need to alloc and copy */
391                 } else {
392                         /* Slight optimization of the final realloc variant */
393                         size -= sizeof(ber_len_t);
394                         oldsize -= sizeof(ber_len_t);
395                         newptr = slap_sl_malloc(size, ctx);
396                         AC_MEMCPY(newptr, ptr, oldsize);
397                         /* Not last block, can just mark old region as free */
398                         p[p[0]/sizeof(ber_len_t)] |= 1;
399                         return newptr;
400                 }
401
402                 size -= sizeof(ber_len_t);
403                 oldsize -= sizeof(ber_len_t);
404
405         } else if (oldsize > size) {
406                 oldsize = size;
407         }
408
409         newptr = slap_sl_malloc(size, ctx);
410         AC_MEMCPY(newptr, ptr, oldsize);
411         slap_sl_free(ptr, ctx);
412         return newptr;
413 }
414
415 void
416 slap_sl_free(void *ptr, void *ctx)
417 {
418         struct slab_heap *sh = ctx;
419         ber_len_t size;
420         ber_len_t *p = (ber_len_t *)ptr, *tmpp;
421
422         if (!ptr)
423                 return;
424
425 #ifdef SLAP_NO_SL_MALLOC
426         ber_memfree_x( ptr, NULL );
427         return;
428 #endif
429
430         if (!sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
431                 ber_memfree_x(ptr, NULL);
432
433         } else if (sh->sh_stack) {
434                 size = p[-1];
435                 p = (ber_len_t *) ((char *) ptr + size);
436                 /* mark it free */
437                 p[-1] = size |= 1;
438                 /* reclaim free space off tail */
439                 if (sh->sh_last == p) {
440                         do {
441                                 p = (ber_len_t *) ((char *) p - size + 1) - 1;
442                                 size = p[-1];
443                         } while (size & 1);
444                         sh->sh_last = p;
445                 }
446
447         } else {
448                 int size_shift, order_size;
449                 struct slab_object *so;
450                 unsigned long diff;
451                 int i, inserted = 0, order = -1;
452
453                 size = *(--p);
454                 size_shift = size + sizeof(ber_len_t) - 1;
455                 do {
456                         order++;
457                 } while (size_shift >>= 1);
458
459                 for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
460                         order_size = 1 << (i+1);
461                         diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
462                         sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
463                         if (diff == ((diff>>1)<<1)) {
464                                 if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
465                                                 (1<<((diff+1)&0x7)))) {
466                                         so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
467                                         while (so) {
468                                                 if ((char*)so->so_ptr == (char*)tmpp) {
469                                                         LDAP_LIST_REMOVE( so, so_link );
470                                                 } else if ((char*)so->so_ptr ==
471                                                                 (char*)tmpp + order_size) {
472                                                         LDAP_LIST_REMOVE(so, so_link);
473                                                         break;
474                                                 }
475                                                 so = LDAP_LIST_NEXT(so, so_link);
476                                         }
477                                         if (so) {
478                                                 if (i < sh->sh_maxorder) {
479                                                         inserted = 1;
480                                                         so->so_ptr = tmpp;
481                                                         LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
482                                                                         so, so_link);
483                                                 }
484                                                 continue;
485                                         } else {
486                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
487                                                         slap_replenish_sopool(sh);
488                                                 }
489                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
490                                                 LDAP_LIST_REMOVE(so, so_link);
491                                                 so->so_ptr = tmpp;
492                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
493                                                                 so, so_link);
494                                                 break;
495
496                                                 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
497                                                         "free object not found while bit is clear.\n",
498                                                         0, 0, 0);
499                                                 assert(so != NULL);
500
501                                         }
502                                 } else {
503                                         if (!inserted) {
504                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
505                                                         slap_replenish_sopool(sh);
506                                                 }
507                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
508                                                 LDAP_LIST_REMOVE(so, so_link);
509                                                 so->so_ptr = tmpp;
510                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
511                                                                 so, so_link);
512                                         }
513                                         break;
514                                 }
515                         } else {
516                                 if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
517                                                 (1<<((diff-1)&0x7)))) {
518                                         so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
519                                         while (so) {
520                                                 if ((char*)so->so_ptr == (char*)tmpp) {
521                                                         LDAP_LIST_REMOVE(so, so_link);
522                                                 } else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
523                                                         LDAP_LIST_REMOVE(so, so_link);
524                                                         tmpp = so->so_ptr;
525                                                         break;
526                                                 }
527                                                 so = LDAP_LIST_NEXT(so, so_link);
528                                         }
529                                         if (so) {
530                                                 if (i < sh->sh_maxorder) {
531                                                         inserted = 1;
532                                                         LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],                                                                    so, so_link);
533                                                         continue;
534                                                 }
535                                         } else {
536                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
537                                                         slap_replenish_sopool(sh);
538                                                 }
539                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
540                                                 LDAP_LIST_REMOVE(so, so_link);
541                                                 so->so_ptr = tmpp;
542                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
543                                                                 so, so_link);
544                                                 break;
545
546                                                 Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
547                                                         "free object not found while bit is clear.\n",
548                                                         0, 0, 0 );
549                                                 assert(so != NULL);
550
551                                         }
552                                 } else {
553                                         if ( !inserted ) {
554                                                 if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
555                                                         slap_replenish_sopool(sh);
556                                                 }
557                                                 so = LDAP_LIST_FIRST(&sh->sh_sopool);
558                                                 LDAP_LIST_REMOVE(so, so_link);
559                                                 so->so_ptr = tmpp;
560                                                 LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
561                                                                 so, so_link);
562                                         }
563                                         break;
564                                 }
565                         }
566                 }
567         }
568 }
569
570 void *
571 slap_sl_context( void *ptr )
572 {
573         void *memctx;
574         struct slab_heap *sh;
575
576         if ( slapMode & SLAP_TOOL_MODE ) return NULL;
577
578         sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
579         if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
580                 return sh;
581         }
582         return NULL;
583 }
584
585 static struct slab_object *
586 slap_replenish_sopool(
587     struct slab_heap* sh
588 )
589 {
590     struct slab_object *so_block;
591     int i;
592
593     so_block = (struct slab_object *)ch_malloc(
594                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
595
596     if ( so_block == NULL ) {
597         return NULL;
598     }
599
600     so_block[0].so_blockhead = 1;
601     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
602     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
603         so_block[i].so_blockhead = 0;
604         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
605     }
606
607     return so_block;
608 }
609
610 #ifdef SLAPD_UNUSED
611 static void
612 print_slheap(int level, void *ctx)
613 {
614         struct slab_heap *sh = ctx;
615         struct slab_object *so;
616         int i, j, once = 0;
617
618         if (!ctx) {
619                 Debug(level, "NULL memctx\n", 0, 0, 0);
620                 return;
621         }
622
623         Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder, 0, 0);
624
625         for (i = order_start; i <= sh->sh_maxorder; i++) {
626                 once = 0;
627                 Debug(level, "order=%d\n", i, 0, 0);
628                 for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
629                         Debug(level, "%02x ", sh->sh_map[i-order_start][j], 0, 0);
630                         once = 1;
631                 }
632                 if (!once) {
633                         Debug(level, "%02x ", sh->sh_map[i-order_start][0], 0, 0);
634                 }
635                 Debug(level, "\n", 0, 0, 0);
636                 Debug(level, "free list:\n", 0, 0, 0);
637                 so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
638                 while (so) {
639                         Debug(level, "%p\n", so->so_ptr, 0, 0);
640                         so = LDAP_LIST_NEXT(so, so_link);
641                 }
642         }
643 }
644 #endif