From: Kurt Zeilenga Date: Thu, 4 Feb 1999 18:00:50 +0000 (+0000) Subject: Fix dbcache/entry lock deadlock. If dbcache lock is held, it's X-Git-Tag: OPENLDAP_SLAPD_BACK_LDAP~636 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=366701bdf7dd6b2d78f46f710ebec1c9fde2899d;p=openldap Fix dbcache/entry lock deadlock. If dbcache lock is held, it's okay to read and write LDBM specific fields (state, refcnt, LRU. The id field, though is read-only once set. cache_find_entry_dn2id(), hence, does not require any entry locks. cache_find_entry_id() must do a entry_rdwr_trylock() and back off if busy. Add new rdwr lock code with trylock() functionality. Implement entry_rdwr_trylock(). --- diff --git a/include/ldap_pvt_thread.h b/include/ldap_pvt_thread.h index 2f8a9229f5..6024713bbc 100644 --- a/include/ldap_pvt_thread.h +++ b/include/ldap_pvt_thread.h @@ -180,6 +180,9 @@ LDAP_F int ldap_pvt_thread_set_concurrency LDAP_P(( int )); #endif +#define LDAP_PVT_THREAD_CREATE_JOINABLE 0 +#define LDAP_PVT_THREAD_CREATE_DETACHED 1 + LDAP_F int ldap_pvt_thread_create LDAP_P(( ldap_pvt_thread_t * thread, @@ -232,15 +235,17 @@ LDAP_F int ldap_pvt_thread_mutex_unlock LDAP_P(( ldap_pvt_thread_mutex_t *mutex )); typedef struct ldap_pvt_thread_rdwr_var { - int lt_readers_reading; - int lt_writer_writing; - ldap_pvt_thread_mutex_t lt_mutex; - ldap_pvt_thread_cond_t lt_lock_free; + ldap_pvt_thread_mutex_t ltrw_mutex; + ldap_pvt_thread_cond_t ltrw_read; /* wait for read */ + ldap_pvt_thread_cond_t ltrw_write; /* wait for write */ + int ltrw_valid; +#define LDAP_PVT_THREAD_RDWR_VALUE 0x0bad + int ltrw_r_active; + int ltrw_w_active; + int ltrw_r_wait; + int ltrw_w_wait; } ldap_pvt_thread_rdwr_t; -#define LDAP_PVT_THREAD_CREATE_DETACHED 1 -#define LDAP_PVT_THREAD_CREATE_JOINABLE 0 - LDAP_F int ldap_pvt_thread_rdwr_init LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int @@ -248,21 +253,28 @@ ldap_pvt_thread_rdwr_destroy LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int ldap_pvt_thread_rdwr_rlock LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int +ldap_pvt_thread_rdwr_rtrylock LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); +LDAP_F int ldap_pvt_thread_rdwr_runlock LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int ldap_pvt_thread_rdwr_wlock LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int +ldap_pvt_thread_rdwr_wtrylock LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); +LDAP_F int ldap_pvt_thread_rdwr_wunlock LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); #ifdef LDAP_DEBUG LDAP_F int -ldap_pvt_thread_rdwr_rchk LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); +ldap_pvt_thread_rdwr_readers LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int -ldap_pvt_thread_rdwr_wchk LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); +ldap_pvt_thread_rdwr_writers LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); LDAP_F int -ldap_pvt_thread_rdwr_rwchk LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); +ldap_pvt_thread_rdwr_active LDAP_P((ldap_pvt_thread_rdwr_t *rdwrp)); #endif /* LDAP_DEBUG */ +#define LDAP_PVT_THREAD_EINVAL EINVAL +#define LDAP_PVT_THREAD_EBUSY EINVAL + LDAP_END_DECL #endif /* _LDAP_THREAD_H */ diff --git a/libraries/libldap_r/rdwr.c b/libraries/libldap_r/rdwr.c index 56de51cf94..b4dabbc134 100644 --- a/libraries/libldap_r/rdwr.c +++ b/libraries/libldap_r/rdwr.c @@ -1,100 +1,197 @@ /* -** This basic implementation of Reader/Writer locks does not -** protect writers from starvation. That is, if a writer is +** This is an improved implementation of Reader/Writer locks does +** not protect writers from starvation. That is, if a writer is ** currently waiting on a reader, any new reader will get ** the lock before the writer. +** +** Does not support cancellation nor does any status checking. */ /******************************************************** - * An example source module to accompany... - * - * "Using POSIX Threads: Programming with Pthreads" - * by Brad nichols, Dick Buttlar, Jackie Farrell - * O'Reilly & Associates, Inc. - * + * Adapted from: + * "Programming with Posix Threads" + * by David R Butenhof + * Addison-Wesley ******************************************************** - * rdwr.c -- - * - * Library of functions implementing reader/writer locks */ #include "portable.h" #include +#include #include "ldap_pvt_thread.h" int -ldap_pvt_thread_rdwr_init(ldap_pvt_thread_rdwr_t *rdwrp ) +ldap_pvt_thread_rdwr_init( ldap_pvt_thread_rdwr_t *rw ) { - rdwrp->lt_readers_reading = 0; - rdwrp->lt_writer_writing = 0; - ldap_pvt_thread_mutex_init(&(rdwrp->lt_mutex) ); - ldap_pvt_thread_cond_init(&(rdwrp->lt_lock_free) ); + memset( rw, 0, sizeof(ldap_pvt_thread_rdwr_t) ); + + /* we should check return results */ + ldap_pvt_thread_mutex_init( &rw->ltrw_mutex ); + ldap_pvt_thread_cond_init( &rw->ltrw_read ); + ldap_pvt_thread_cond_init( &rw->ltrw_write ); + + rw->ltrw_valid = LDAP_PVT_THREAD_RDWR_VALUE; return 0; } int -ldap_pvt_thread_rdwr_destroy(ldap_pvt_thread_rdwr_t *rdwrp ) +ldap_pvt_thread_rdwr_destroy( ldap_pvt_thread_rdwr_t *rw ) { - ldap_pvt_thread_mutex_destroy(&(rdwrp->lt_mutex) ); - ldap_pvt_thread_cond_destroy(&(rdwrp->lt_lock_free) ); + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + /* active threads? */ + if( rw->ltrw_r_active > 0 || rw->ltrw_w_active > 1) { + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + return LDAP_PVT_THREAD_EBUSY; + } + + /* waiting threads? */ + if( rw->ltrw_r_wait > 0 || rw->ltrw_w_wait > 0) { + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + return LDAP_PVT_THREAD_EBUSY; + } + + rw->ltrw_valid = 0; + + ldap_pvt_thread_mutex_destroy( &rw->ltrw_mutex ); + ldap_pvt_thread_cond_destroy( &rw->ltrw_read ); + ldap_pvt_thread_cond_destroy( &rw->ltrw_write ); + return 0; } -int ldap_pvt_thread_rdwr_rlock(ldap_pvt_thread_rdwr_t *rdwrp){ - ldap_pvt_thread_mutex_lock(&(rdwrp->lt_mutex)); - while(rdwrp->lt_writer_writing) { - ldap_pvt_thread_cond_wait(&(rdwrp->lt_lock_free), - &(rdwrp->lt_mutex)); +int ldap_pvt_thread_rdwr_rlock( ldap_pvt_thread_rdwr_t *rw ) +{ + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + if( rw->ltrw_w_active > 1 ) { + /* writer is active */ + + rw->ltrw_r_wait++; + + do { + ldap_pvt_thread_cond_wait( + &rw->ltrw_read, &rw->ltrw_mutex ); + } while( rw->ltrw_w_active > 1 ); + + rw->ltrw_r_wait--; } - rdwrp->lt_readers_reading++; - ldap_pvt_thread_mutex_unlock(&(rdwrp->lt_mutex)); + + rw->ltrw_r_active++; + + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + return 0; } -int ldap_pvt_thread_rdwr_runlock(ldap_pvt_thread_rdwr_t *rdwrp) +int ldap_pvt_thread_rdwr_rtrylock( ldap_pvt_thread_rdwr_t *rw ) { - ldap_pvt_thread_mutex_lock(&(rdwrp->lt_mutex)); - if (rdwrp->lt_readers_reading == 0) { - ldap_pvt_thread_mutex_unlock(&(rdwrp->lt_mutex)); - return -1; + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + if( rw->ltrw_w_active > 1) { + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + return LDAP_PVT_THREAD_EBUSY; } - else { - rdwrp->lt_readers_reading--; - if (rdwrp->lt_readers_reading == 0) { - ldap_pvt_thread_cond_signal(&(rdwrp->lt_lock_free)); - } - ldap_pvt_thread_mutex_unlock(&(rdwrp->lt_mutex)); - return 0; + + rw->ltrw_r_active++; + + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + + return 0; +} + +int ldap_pvt_thread_rdwr_runlock( ldap_pvt_thread_rdwr_t *rw ) +{ + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + rw->ltrw_r_active--; + + if (rw->ltrw_r_active == 0 && rw->ltrw_w_wait > 0 ) { + ldap_pvt_thread_cond_signal( &rw->ltrw_write ); } + + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + + return 0; } -int ldap_pvt_thread_rdwr_wlock(ldap_pvt_thread_rdwr_t *rdwrp) +int ldap_pvt_thread_rdwr_wlock( ldap_pvt_thread_rdwr_t *rw ) { - ldap_pvt_thread_mutex_lock(&(rdwrp->lt_mutex)); - while(rdwrp->lt_writer_writing || rdwrp->lt_readers_reading) { - ldap_pvt_thread_cond_wait(&(rdwrp->lt_lock_free), - &(rdwrp->lt_mutex)); + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + if ( rw->ltrw_w_active > 0 || rw->ltrw_r_active > 0 ) { + rw->ltrw_w_wait++; + + do { + ldap_pvt_thread_cond_wait( + &rw->ltrw_write, &rw->ltrw_mutex ); + } while ( rw->ltrw_w_active > 0 || rw->ltrw_r_active > 0 ); + + rw->ltrw_w_wait--; } - rdwrp->lt_writer_writing++; - ldap_pvt_thread_mutex_unlock(&(rdwrp->lt_mutex)); + + rw->ltrw_w_active++; + + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + return 0; } -int ldap_pvt_thread_rdwr_wunlock(ldap_pvt_thread_rdwr_t *rdwrp) +int ldap_pvt_thread_rdwr_wtrylock( ldap_pvt_thread_rdwr_t *rw ) { - ldap_pvt_thread_mutex_lock(&(rdwrp->lt_mutex)); - if (rdwrp->lt_writer_writing == 0) { - ldap_pvt_thread_mutex_unlock(&(rdwrp->lt_mutex)); - return -1; + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + if ( rw->ltrw_w_active > 0 || rw->ltrw_r_active > 0 ) { + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + return LDAP_PVT_THREAD_EBUSY; } - else { - rdwrp->lt_writer_writing = 0; - ldap_pvt_thread_cond_broadcast(&(rdwrp->lt_lock_free)); - ldap_pvt_thread_mutex_unlock(&(rdwrp->lt_mutex)); - return 0; + + rw->ltrw_w_active++; + + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + + return 0; +} + +int ldap_pvt_thread_rdwr_wunlock( ldap_pvt_thread_rdwr_t *rw ) +{ + if( rw->ltrw_valid != LDAP_PVT_THREAD_RDWR_VALUE ) + return LDAP_PVT_THREAD_EINVAL; + + ldap_pvt_thread_mutex_lock( &rw->ltrw_mutex ); + + rw->ltrw_w_active--; + + if (rw->ltrw_r_wait > 0) { + ldap_pvt_thread_cond_broadcast( &rw->ltrw_read ); + + } else if (rw->ltrw_w_wait > 0) { + ldap_pvt_thread_cond_signal( &rw->ltrw_write ); } + + ldap_pvt_thread_mutex_unlock( &rw->ltrw_mutex ); + + return 0; } #ifdef LDAP_DEBUG @@ -109,19 +206,20 @@ int ldap_pvt_thread_rdwr_wunlock(ldap_pvt_thread_rdwr_t *rdwrp) * a lock are caught. */ -int ldap_pvt_thread_rdwr_rchk(ldap_pvt_thread_rdwr_t *rdwrp) +int ldap_pvt_thread_rdwr_readers(ldap_pvt_thread_rdwr_t *rw) { - return(rdwrp->lt_readers_reading!=0); + return( rw->ltrw_r_active ); } -int ldap_pvt_thread_rdwr_wchk(ldap_pvt_thread_rdwr_t *rdwrp) +int ldap_pvt_thread_rdwr_writers(ldap_pvt_thread_rdwr_t *rw) { - return(rdwrp->lt_writer_writing!=0); + return( rw->ltrw_w_active ); } -int ldap_pvt_thread_rdwr_rwchk(ldap_pvt_thread_rdwr_t *rdwrp) + +int ldap_pvt_thread_rdwr_active(ldap_pvt_thread_rdwr_t *rw) { - return(ldap_pvt_thread_rdwr_rchk(rdwrp) || - ldap_pvt_thread_rdwr_wchk(rdwrp)); + return(ldap_pvt_thread_rdwr_readers(rw) + + ldap_pvt_thread_rdwr_writers(rw)); } #endif /* LDAP_DEBUG */ diff --git a/servers/slapd/back-ldbm/cache.c b/servers/slapd/back-ldbm/cache.c index add8693393..58ea06f47e 100644 --- a/servers/slapd/back-ldbm/cache.c +++ b/servers/slapd/back-ldbm/cache.c @@ -4,6 +4,7 @@ #include +#include #include #include @@ -58,6 +59,7 @@ cache_set_state( struct cache *cache, Entry *e, int state ) ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); } +#ifdef not_used static void cache_return_entry( struct cache *cache, Entry *e ) { @@ -71,14 +73,25 @@ cache_return_entry( struct cache *cache, Entry *e ) /* free cache mutex */ ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); } +#endif static void cache_return_entry_rw( struct cache *cache, Entry *e, int rw ) { Debug( LDAP_DEBUG_TRACE, "====> cache_return_entry_%s\n", rw ? "w" : "r", 0, 0); + + /* set cache mutex */ + ldap_pvt_thread_mutex_lock( &cache->c_mutex ); + entry_rdwr_unlock(e, rw);; - cache_return_entry(cache, e); + + if ( --e->e_refcnt == 0 && e->e_state == ENTRY_STATE_DELETED ) { + entry_free( e ); + } + + /* free cache mutex */ + ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); } void @@ -202,7 +215,7 @@ cache_add_entry_lock( /* XXX check for writer lock - should also check no readers pending */ #ifdef LDAP_DEBUG - assert(!ldap_pvt_thread_rdwr_rwchk(&e->e_rdwr)); + assert(!ldap_pvt_thread_rdwr_active( &e->e_rdwr )); #endif /* delete from cache and lru q */ @@ -241,6 +254,11 @@ cache_find_entry_dn2id( if ( (ep = (Entry *) avl_find( cache->c_dntree, (caddr_t) &e, cache_entrydn_cmp )) != NULL ) { + /* + * ep now points to an unlocked entry + * we do not need to lock the entry if we only + * check the state, refcnt, LRU, and id. + */ free(e.e_ndn); Debug(LDAP_DEBUG_TRACE, "====> cache_find_entry_dn2id: found dn: %s\n", @@ -257,42 +275,16 @@ cache_find_entry_dn2id( return( NOID ); } - /* XXX is this safe without writer lock? */ - ep->e_refcnt++; - /* lru */ LRU_DELETE( cache, ep ); LRU_ADD( cache, ep ); - - /* acquire reader lock */ - entry_rdwr_lock(ep, 0); - - /* re-check */ - if ( ep->e_state == ENTRY_STATE_DELETED || - ep->e_state == ENTRY_STATE_CREATING ) - { - /* XXX check that is is required */ - ep->e_refcnt--; - - /* free reader lock */ - entry_rdwr_unlock(ep, 0); - /* free cache mutex */ - ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); - - return( NOID ); - } - + /* save id */ id = ep->e_id; - /* free reader lock */ - entry_rdwr_unlock(ep, 0); - /* free cache mutex */ ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); - cache_return_entry( &li->li_cache, ep ); - return( id ); } @@ -318,11 +310,12 @@ cache_find_entry_id( Entry e; Entry *ep; + e.e_id = id; + +try_again: /* set cache mutex */ ldap_pvt_thread_mutex_lock( &cache->c_mutex ); - e.e_id = id; - if ( (ep = (Entry *) avl_find( cache->c_idtree, (caddr_t) &e, cache_entryid_cmp )) != NULL ) { @@ -340,35 +333,25 @@ cache_find_entry_id( ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); return( NULL ); } - /* XXX is this safe without writer lock? */ - ep->e_refcnt++; - /* lru */ - LRU_DELETE( cache, ep ); - LRU_ADD( cache, ep ); - /* acquire reader lock */ - entry_rdwr_lock(ep, 0); - - /* re-check */ - if ( ep->e_state == ENTRY_STATE_DELETED || - ep->e_state == ENTRY_STATE_CREATING ) { - - /* XXX check that is is required */ - ep->e_refcnt--; - - /* free reader lock */ - entry_rdwr_unlock(ep, 0); + if ( entry_rdwr_trylock(ep, rw) == LDAP_PVT_THREAD_EBUSY ) { + /* could not acquire entry lock... + * owner cannot free as we have the cache locked. + * so, unlock the cache, yield, and try again. + */ /* free cache mutex */ ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); - return( NULL ); + ldap_pvt_thread_yield(); + goto try_again; } - if ( rw ) { - entry_rdwr_unlock(ep, 0); - entry_rdwr_lock(ep, 1); - } + /* lru */ + LRU_DELETE( cache, ep ); + LRU_ADD( cache, ep ); + + ep->e_refcnt++; /* free cache mutex */ ldap_pvt_thread_mutex_unlock( &cache->c_mutex ); @@ -405,7 +388,7 @@ cache_delete_entry( /* XXX check for writer lock - should also check no readers pending */ #ifdef LDAP_DEBUG - assert(ldap_pvt_thread_rdwr_wchk(&e->e_rdwr)); + assert(ldap_pvt_thread_rdwr_writers(&e->e_rdwr)); #endif /* set cache mutex */ diff --git a/servers/slapd/back-ldbm/id2entry.c b/servers/slapd/back-ldbm/id2entry.c index 2aee65d523..ae5e18868e 100644 --- a/servers/slapd/back-ldbm/id2entry.c +++ b/servers/slapd/back-ldbm/id2entry.c @@ -64,18 +64,13 @@ id2entry_delete( Backend *be, Entry *e ) Debug(LDAP_DEBUG_TRACE, "=> id2entry_delete( %lu, \"%s\" )\n", e->e_id, e->e_dn, 0 ); - /* XXX - check for writer lock - should also check no reader pending */ #ifdef LDAP_DEBUG - assert(ldap_pvt_thread_rdwr_wchk(&e->e_rdwr)); + /* check for writer lock */ + assert(ldap_pvt_thread_rdwr_writers(&e->e_rdwr) == 1); #endif ldbm_datum_init( key ); - /* XXX - check for writer lock - should also check no reader pending */ - Debug (LDAP_DEBUG_TRACE, - "rdwr_Xchk: readers_reading: %d writer_writing: %d\n", - e->e_rdwr.lt_readers_reading, e->e_rdwr.lt_writer_writing, 0); - if ( (db = ldbm_cache_open( be, "id2entry", LDBM_SUFFIX, LDBM_WRCREAT )) == NULL ) { Debug( LDAP_DEBUG_ANY, "Could not open/create id2entry%s\n", diff --git a/servers/slapd/entry.c b/servers/slapd/entry.c index 239b97434b..1e7a7c6b55 100644 --- a/servers/slapd/entry.c +++ b/servers/slapd/entry.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -218,11 +219,20 @@ entry_free( Entry *e ) int i; Attribute *a, *next; - /* XXX check that no reader/writer locks exist */ + /* check that no reader/writer locks exist */ + + if ( ldap_pvt_thread_rdwr_wtrylock( &e->e_rdwr ) == + LDAP_PVT_THREAD_EBUSY ) + { + Debug( LDAP_DEBUG_ANY, "entry_free(%ld): active (%d, %d)\n", + e->e_id, + ldap_pvt_thread_rdwr_readers( &e->e_rdwr ), + ldap_pvt_thread_rdwr_writers( &e->e_rdwr )); + #ifdef LDAP_DEBUG - assert( !ldap_pvt_thread_rdwr_wchk(&e->e_rdwr) && - !ldap_pvt_thread_rdwr_rchk(&e->e_rdwr) ); + assert(!ldap_pvt_thread_rdwr_active( &e->e_rdwr )); #endif + } if ( e->e_dn != NULL ) { free( e->e_dn ); @@ -260,6 +270,17 @@ entry_rdwr_wlock(Entry *e) return entry_rdwr_lock( e, 1 ); } +int +entry_rdwr_trylock(Entry *e, int rw) +{ + Debug( LDAP_DEBUG_ARGS, "entry_rdwr_%strylock: ID: %ld\n", + rw ? "w" : "r", e->e_id, 0); + if (rw) + return ldap_pvt_thread_rdwr_wtrylock(&e->e_rdwr); + else + return ldap_pvt_thread_rdwr_rtrylock(&e->e_rdwr); +} + int entry_rdwr_unlock(Entry *e, int rw) {