From: Kern Sibbald Date: Fri, 2 Feb 2007 10:42:11 +0000 (+0000) Subject: kes Fix memory leak with storage ids in cats/sql_get.c X-Git-Tag: Release-7.0.0~6969 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=4014f987b40bc24f31aecedaf09d78a3781d41b5;p=bacula%2Fbacula kes Fix memory leak with storage ids in cats/sql_get.c kes Terminate watchdog earlier to avoid reference to released memory -- reported by Jason Austin. kes Move closing the database from jobq.c to the director daemon termination routine. This fixes memory leaks for shadow jobs (i.e. migration jobs). kes Free up the unique jobid chain items in migrate.c. This fixes a memory leak problem. kes Convert some ugly looking for statements to use foreach_alist in findlib/find.c. This will facilitate converting the structures to use dlist (for large include/exclude lists). kes Fix a bug in the btree.c and btree.h routines, then rename them rblist and add them to be built in src/lib. Include some new methods written by Rudolf Cejka that make the code more readable (hides some of the ugly casting). kes Add set_next() and set_prev() methods which make the code much more readable. Also add a new dlistString class that facilitates storing strings in dlists. To be used in the large include/exclude lists. kes Make some trivial modifications to lib/tree.h that use rblist rather than dlist for storing the tree links. This was suggested by Rudolf Cejka. The result of this is that the restore tree now uses red-black binary trees rather than simple linked lists. This should give rather dramatic speed improvements for directories contining large numbers of directories/files (more than 10000). git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4088 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/kernstodo b/bacula/kernstodo index 9a1c54eb52..ef54bfd5db 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -41,6 +41,7 @@ Document: Priority: +- Why doesn't @"xxx abc" work in a conf file? - Figure out some way to "automatically" backup conf changes. - Look at using posix_fadvise(2) for backups -- see bug #751. Possibly add the code at findlib/bfile.c:795 diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index b2cc9734e5..9176853496 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -412,12 +412,10 @@ int db_get_job_volume_parameters(JCR *jcr, B_DB *mdb, JobId_t JobId, VOL_PARAMS stat = 0; } else { stat = mdb->num_rows; - DBId_t *SId; + DBId_t *SId = NULL; if (stat > 0) { *VolParams = Vols = (VOL_PARAMS *)malloc(stat * sizeof(VOL_PARAMS)); SId = (DBId_t *)malloc(stat * sizeof(DBId_t)); - } else { - SId = NULL; } for (i=0; i < stat; i++) { if ((row = sql_fetch_row(mdb)) == NULL) { @@ -453,6 +451,9 @@ int db_get_job_volume_parameters(JCR *jcr, B_DB *mdb, JobId_t JobId, VOL_PARAMS } } } + if (SId) { + free(SId); + } } sql_free_result(mdb); } diff --git a/bacula/src/dird/dird.c b/bacula/src/dird/dird.c index 6d48a4f236..5081f08561 100644 --- a/bacula/src/dird/dird.c +++ b/bacula/src/dird/dird.c @@ -301,6 +301,7 @@ void terminate_dird(int sig) exit(1); } already_here = true; + stop_watchdog(); generate_daemon_event(NULL, "Exit"); write_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); delete_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); @@ -319,7 +320,6 @@ void terminate_dird(int sig) free_config_resources(); term_ua_server(); term_msg(); /* terminate message handler */ - stop_watchdog(); cleanup_crypto(); close_memory_pool(); /* release free memory in pool */ sm_dump(false); diff --git a/bacula/src/dird/job.c b/bacula/src/dird/job.c index 1ca59f857f..83d06fcaed 100644 --- a/bacula/src/dird/job.c +++ b/bacula/src/dird/job.c @@ -882,6 +882,10 @@ void dird_free_jcr(JCR *jcr) pthread_cond_destroy(&jcr->term_wait); jcr->term_wait_inited = false; } + if (jcr->db) { + db_close_database(jcr, jcr->db); + jcr->db = NULL; + } if (jcr->stime) { Dmsg0(200, "Free JCR stime\n"); free_pool_memory(jcr->stime); diff --git a/bacula/src/dird/jobq.c b/bacula/src/dird/jobq.c index 03230dae10..780b2f258f 100755 --- a/bacula/src/dird/jobq.c +++ b/bacula/src/dird/jobq.c @@ -558,10 +558,6 @@ void *jobq_server(void *arg) Dmsg0(2300, "Back from running new job.\n"); } /* Clean up and release old jcr */ - if (jcr->db) { - db_close_database(jcr, jcr->db); - jcr->db = NULL; - } Dmsg2(2300, "====== Termination job=%d use_cnt=%d\n", jcr->JobId, jcr->use_count()); jcr->SDJobStatus = 0; V(jq->mutex); /* release internal lock */ diff --git a/bacula/src/dird/migrate.c b/bacula/src/dird/migrate.c index 11d497b546..6d21bd8a4a 100644 --- a/bacula/src/dird/migrate.c +++ b/bacula/src/dird/migrate.c @@ -1002,6 +1002,9 @@ static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1, bail_out: Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list); + foreach_dlist(item, item_chain) { + free(item->item); + } delete item_chain; return ok; } diff --git a/bacula/src/findlib/find.c b/bacula/src/findlib/find.c index c4bd014adf..75da814b49 100644 --- a/bacula/src/findlib/find.c +++ b/bacula/src/findlib/find.c @@ -119,14 +119,14 @@ get_win32_driveletters(FF_PKT *ff, char* szDrives) findFILESET *fileset = ff->fileset; if (fileset) { - int i, j; + int i; + char *fname; for (i=0; iinclude_list.size(); i++) { findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i); /* look through all files and check */ - for (j=0; jname_list.size(); j++) { - char *fname = (char *)incexe->name_list.get(j); + foreach_alist(fname, &incexe->name_list) { /* fname should match x:/ */ if (strlen(fname) >= 2 && B_ISALPHA(fname[0]) && fname[1] == ':') { @@ -190,9 +190,10 @@ find_files(JCR *jcr, FF_PKT *ff, int callback(FF_PKT *ff_pkt, void *hpkt, bool t ff->drivetypes = fo->drivetype; bstrncat(ff->VerifyOpts, fo->VerifyOpts, sizeof(ff->VerifyOpts)); } - for (j=0; jname_list.size(); j++) { - Dmsg1(100, "F %s\n", (char *)incexe->name_list.get(j)); - ff->top_fname = (char *)incexe->name_list.get(j); + char *fname; + foreach_alist(fname, &incexe->name_list) { + Dmsg1(100, "F %s\n", fname); + ff->top_fname = fname; if (find_one_file(jcr, ff, our_callback, his_pkt, ff->top_fname, (dev_t)-1, true) == 0) { return 0; /* error return */ } @@ -339,8 +340,9 @@ static bool accept_file(FF_PKT *ff) } fnm_flags = (incexe->current_opts != NULL && incexe->current_opts->flags & FO_IGNORECASE) ? FNM_CASEFOLD : 0; - for (j=0; jname_list.size(); j++) { - if (fnmatch((char *)incexe->name_list.get(j), ff->fname, fnmode|fnm_flags) == 0) { + char *fname; + foreach_alist(fname, &incexe->name_list) { + if (fnmatch(fname, ff->fname, fnmode|fnm_flags) == 0) { Dmsg1(100, "Reject wild2: %s\n", ff->fname); return false; /* reject file */ } diff --git a/bacula/src/lib/Makefile.in b/bacula/src/lib/Makefile.in index 74fc4cef70..e8c54a17e5 100644 --- a/bacula/src/lib/Makefile.in +++ b/bacula/src/lib/Makefile.in @@ -30,7 +30,7 @@ LIBSRCS = attr.c base64.c berrno.c bsys.c bget_msg.c \ md5.c message.c mem_pool.c openssl.c parse_conf.c \ queue.c bregex.c \ res.c rwlock.c scan.c serial.c sha1.c \ - signal.c smartall.c tls.c tree.c \ + signal.c smartall.c rblist.c tls.c tree.c \ util.c var.c watchdog.c workq.c btimers.c \ address_conf.c pythonlib.c @@ -43,7 +43,7 @@ LIBOBJS = attr.o base64.o berrno.o bsys.o bget_msg.o \ md5.o message.o mem_pool.o openssl.o parse_conf.o \ queue.o bregex.o \ res.o rwlock.o scan.o serial.o sha1.o \ - signal.o smartall.o tls.o tree.o \ + signal.o smartall.o rblist.o tls.o tree.o \ util.o var.o watchdog.o workq.o btimers.o \ address_conf.o pythonlib.o diff --git a/bacula/src/lib/btree.c b/bacula/src/lib/btree.c index c17fe143a2..c861f23ddd 100644 --- a/bacula/src/lib/btree.c +++ b/bacula/src/lib/btree.c @@ -430,6 +430,10 @@ int main() printf("%d items appended\n", CNT*CNT*CNT); printf("num_items=%d\n", jcr_chain->size()); + foreach_btree(jcr, jcr_chain) { +// printf("%s\n", jcr->buf); /* turn on if you want lots of output */ + } + jcr = (MYJCR *)malloc(sizeof(MYJCR)); memset(jcr, 0, sizeof(MYJCR)); diff --git a/bacula/src/lib/btree.h b/bacula/src/lib/btree.h index 23e79a9be9..344eb69cfc 100644 --- a/bacula/src/lib/btree.h +++ b/bacula/src/lib/btree.h @@ -47,12 +47,13 @@ * * Loop var through each member of list */ -#define foreach_btree(var, tree) \ - for(*((bnode **)&(var))=(tree)->first(); (*((bnode **)&(var))=(tree)->next((bnode *)var)); ) -#ifdef the_old_way +#ifdef HAVE_TYPEOF +#define foreach_btree(var, tree) \ + for((var)=(typeof(var))(tree)->first(); (var); (var)=(typeof(var))(tree)->next((bnode *)var) ) +#else #define foreach_btree(var, tree) \ - for((var)=(tree)->first(); (((bnode *)(var))=(tree)->next((bnode *)var)); ) + for(*((bnode **)&(var))=(tree)->first(); (var); (*((bnode **)&(var))=(tree)->next((bnode *)var)) ) #endif struct bnode; diff --git a/bacula/src/lib/dlist.c b/bacula/src/lib/dlist.c index 5cb8be2a7f..1e00346047 100644 --- a/bacula/src/lib/dlist.c +++ b/bacula/src/lib/dlist.c @@ -1,18 +1,7 @@ -/* - * Bacula doubly linked list routines. - * - * dlist is a doubly linked list with the links being in the - * list data item. - * - * Kern Sibbald, July MMIII - * - * Version $Id$ - * - */ /* Bacula® - The Network Backup Solution - Copyright (C) 2003-2006 Free Software Foundation Europe e.V. + Copyright (C) 2003-2007 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -36,6 +25,17 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * Bacula doubly linked list routines. + * + * dlist is a doubly linked list with the links being in the + * list data item. + * + * Kern Sibbald, July MMIII + * + * Version $Id$ + * + */ #include "bacula.h" @@ -48,10 +48,10 @@ */ void dlist::append(void *item) { - ((dlink *)(((char *)item)+loffset))->next = NULL; - ((dlink *)(((char *)item)+loffset))->prev = tail; + set_next(item, NULL); + set_prev(item, tail); if (tail) { - ((dlink *)(((char *)tail)+loffset))->next = item; + set_next(tail, item); } tail = item; if (head == NULL) { /* if empty list, */ @@ -65,10 +65,10 @@ void dlist::append(void *item) */ void dlist::prepend(void *item) { - ((dlink *)(((char *)item)+loffset))->next = head; - ((dlink *)(((char *)item)+loffset))->prev = NULL; + set_next(item, head); + set_prev(item, NULL); if (head) { - ((dlink *)(((char *)head)+loffset))->prev = item; + set_prev(head, item); } head = item; if (tail == NULL) { /* if empty list, */ @@ -81,11 +81,11 @@ void dlist::insert_before(void *item, void *where) { dlink *where_link = (dlink *)((char *)where+loffset); - ((dlink *)(((char *)item)+loffset))->next = where; - ((dlink *)(((char *)item)+loffset))->prev = where_link->prev; + set_next(item, where); + set_prev(item, where_link->prev); if (where_link->prev) { - ((dlink *)(((char *)(where_link->prev))+loffset))->next = item; + set_next(where_link->prev, item); } where_link->prev = item; if (head == where) { @@ -98,11 +98,11 @@ void dlist::insert_after(void *item, void *where) { dlink *where_link = (dlink *)((char *)where+loffset); - ((dlink *)(((char *)item)+loffset))->next = where_link->next; - ((dlink *)(((char *)item)+loffset))->prev = where; + set_next(item, where_link->next); + set_prev(item, where); if (where_link->next) { - ((dlink *)(((char *)(where_link->next))+loffset))->prev = item; + set_prev(where_link->next, item); } where_link->next = item; if (tail == where) { @@ -294,7 +294,7 @@ void dlist::remove(void *item) if (item == head) { head = ilink->next; if (head) { - ((dlink *)(((char *)head)+loffset))->prev = NULL; + set_prev(head, NULL); } if (item == tail) { tail = ilink->prev; @@ -302,13 +302,13 @@ void dlist::remove(void *item) } else if (item == tail) { tail = ilink->prev; if (tail) { - ((dlink *)(((char *)tail)+loffset))->next = NULL; + set_next(tail, NULL); } } else { xitem = ilink->next; - ((dlink *)(((char *)xitem)+loffset))->prev = ilink->prev; + set_prev(xitem, ilink->prev); xitem = ilink->prev; - ((dlink *)(((char *)xitem)+loffset))->next = ilink->next; + set_next(xitem, ilink->next); } num_items--; if (num_items == 0) { @@ -345,7 +345,20 @@ void dlist::destroy() head = tail = NULL; } +/* String helpers for dlist usage */ +dlistString *new_dlistString(const char *str) +{ + return new_dlistString(str, strlen(str)); +} + +dlistString *new_dlistString(const char *str, int len) +{ + dlistString *node; + node = (dlistString *)malloc(sizeof(dlistString) + len); + bstrncpy(node->c_str(), str, len); + return node; +} #ifdef TEST_PROGRAM @@ -373,6 +386,7 @@ int main() MYJCR *jcr1; MYJCR *save_jcr = NULL; MYJCR *next_jcr; + int count; jcr_chain = (dlist *)malloc(sizeof(dlist)); jcr_chain->init(jcr, &jcr->link); @@ -406,6 +420,10 @@ int main() jcr_chain->destroy(); free(jcr_chain); + /* The following may seem a bit odd, but we create a chaing + * of jcr objects. Within a jcr object, there is a buf + * that points to a malloced string containing data + */ jcr_chain = New(dlist(jcr, &jcr->link)); printf("append 20 items 0-19\n"); for (int i=0; i<20; i++) { @@ -442,7 +460,7 @@ int main() #define CNT 26 printf("append %d items\n", CNT*CNT*CNT); strcpy(buf, "ZZZ"); - int count = 0; + count = 0; for (int i=0; ic_str()); + } + printf("destroy dlistString chain\n"); + chain.destroy(); sm_dump(false); diff --git a/bacula/src/lib/dlist.h b/bacula/src/lib/dlist.h index 66934ce7b8..189f02231d 100644 --- a/bacula/src/lib/dlist.h +++ b/bacula/src/lib/dlist.h @@ -1,12 +1,7 @@ -/* - * Written by Kern Sibbald MMIV - * - * Version $Id$ - */ /* Bacula® - The Network Backup Solution - Copyright (C) 2004-2006 Free Software Foundation Europe e.V. + Copyright (C) 2004-2007 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -30,13 +25,21 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * Written by Kern Sibbald MMIV + * + * Version $Id$ + */ /* ======================================================================== * * Doubly linked list -- dlist + * + * See the end of the file for the dlistString class which + * facilitates storing strings in a dlist. * - * Kern Sibbald, MMIV + * Kern Sibbald, MMIV and MMVII * */ @@ -59,8 +62,6 @@ for((var)=NULL; (*((void **)&(var))=(void*)((list)->next(var))); ) #endif - - struct dlink { void *next; void *prev; @@ -78,6 +79,8 @@ public: void init(void *item, dlink *link); void prepend(void *item); void append(void *item); + void set_prev(void *item, void *prev); + void set_next(void *item, void *next); void insert_before(void *item, void *where); void insert_after(void *item, void *where); void *binary_insert(void *item, int compare(void *item1, void *item2)); @@ -127,6 +130,18 @@ inline dlist::dlist(void) : head(0), tail(0), loffset(0), num_items(0) { } +inline void dlist::set_prev(void *item, void *prev) +{ + ((dlink *)(((char *)item)+loffset))->prev = prev; +} + +inline void dlist::set_next(void *item, void *next) +{ + ((dlink *)(((char *)item)+loffset))->next = next; +} + + + inline bool dlist::empty() const { return head == NULL; @@ -148,3 +163,25 @@ inline void * dlist::last() const { return tail; } + +/* + * C string helper routines for dlist + * The string (char *) is kept in the node + * + * Kern Sibbald, February 2007 + * + */ +class dlistString : public dlink +{ +public: + char *c_str() { return m_str; }; + +private: + char m_str[1]; + /* !!! Don't put anything after this as this space is used + * to hold the string in inline + */ +}; + +extern dlistString *new_dlistString(const char *str, int len); +extern dlistString *new_dlistString(const char *str); diff --git a/bacula/src/lib/lib.h b/bacula/src/lib/lib.h index 671e066b41..37a8c2b4bc 100644 --- a/bacula/src/lib/lib.h +++ b/bacula/src/lib/lib.h @@ -10,7 +10,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2006 Free Software Foundation Europe e.V. + Copyright (C) 2000-2007 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -38,6 +38,7 @@ #include "smartall.h" #include "alist.h" #include "dlist.h" +#include "rblist.h" #include "base64.h" #include "bits.h" #include "btime.h" diff --git a/bacula/src/lib/rblist.c b/bacula/src/lib/rblist.c new file mode 100644 index 0000000000..9673d747a9 --- /dev/null +++ b/bacula/src/lib/rblist.c @@ -0,0 +1,496 @@ +/* + * Bacula red-black binary tree routines. + * + * rblist is a binary tree with the links being in the data item. + * + * Developped in part from ideas obtained from several online University + * courses. + * + * Kern Sibbald, November MMV + * + * Version $Id$ + * + */ +/* + Bacula® - The Network Backup Solution + + Copyright (C) 2005-2007 Free Software Foundation Europe e.V. + + The main author of Bacula is Kern Sibbald, with contributions from + many others, a complete list can be found in the file AUTHORS. + This program is Free Software; you can redistribute it and/or + modify it under the terms of version two of the GNU General Public + License as published by the Free Software Foundation plus additions + that are listed in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + + Bacula® is a registered trademark of John Walker. + The licensor of Bacula is the Free Software Foundation Europe + (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, + Switzerland, email:ftf@fsfeurope.org. +*/ + +#include "bacula.h" + +/* =================================================================== + * rblist + */ + +/* + * Insert an item in the tree, but only if it is unique + * otherwise, the item is returned non inserted + * The big trick is keeping the tree balanced after the + * insert. We use a parent pointer to make it simpler and + * to avoid recursion. + * + * Returns: item if item inserted + * other_item if same value already exists (item not inserted) + */ +void *rblist::insert(void *item, int compare(void *item1, void *item2)) +{ + void *x, *y; + void *last = NULL; /* last leaf if not found */ + void *found = NULL; + int comp = 0; + + /* Search */ + x = head; + while (x && !found) { + last = x; + comp = compare(item, x); + if (comp < 0) { + x = left(x); + } else if (comp > 0) { + x = right(x); + } else { + found = x; + } + } + + if (found) { /* found? */ + return found; /* yes, return item found */ + } + set_left(item, NULL); + set_right(item, NULL); + set_parent(item, NULL); + set_red(item, false); + /* Handle empty tree */ + if (num_items == 0) { + head = item; + num_items++; + return item; + } + x = last; + /* Not found, so insert it on appropriate side of tree */ + if (comp < 0) { + set_left(last, item); + } else { + set_right(last, item); + } + set_red(last, true); + set_parent(item, last); + num_items++; + + /* Now we must walk up the tree balancing it */ + x = last; + while (x != head && red(parent(x))) { + if (parent(x) == left(parent(parent(x)))) { + /* Look at the right side of our grandparent */ + y = right(parent(parent(x))); + if (y && red(y)) { + /* our parent must be black */ + set_red(parent(x), false); + set_red(y, false); + set_red(parent(parent(x)), true); + x = parent(parent(x)); /* move up to grandpa */ + } else { + if (x == right(parent(x))) { /* right side of parent? */ + x = parent(x); + left_rotate(x); + } + /* make parent black too */ + set_red(parent(x), false); + set_red(parent(parent(x)), true); + right_rotate(parent(parent(x))); + } + } else { + /* Look at left side of our grandparent */ + y = left(parent(parent(x))); + if (y && red(y)) { + set_red(parent(x), false); + set_red(y, false); + set_red(parent(parent(x)), true); + x = parent(parent(x)); /* move up to grandpa */ + } else { + if (x == left(parent(x))) { + x = parent(x); + right_rotate(x); + } + /* make parent black too */ + set_red(parent(x), false); + set_red(parent(parent(x)), true); + left_rotate(parent(parent(x))); + } + } + } + /* Make sure the head is always black */ + set_red(head, false); + return item; +} + +/* + * Search for item + */ +void *rblist::search(void *item, int compare(void *item1, void *item2)) +{ + void *found = NULL; + void *x; + int comp; + + x = head; + while (x) { + comp = compare(item, x); + if (comp < 0) { + x = left(x); + } else if (comp > 0) { + x = right(x); + } else { + found = x; + break; + } + } + return found; +} + +/* + * Get first item (i.e. lowest value) + */ +void *rblist::first(void) +{ + void *x; + + x = head; + down = true; + while (x) { + if (left(x)) { + x = left(x); + continue; + } + return x; + } + /* Tree is empty */ + return NULL; +} + +/* + * This is a non-recursive btree walk routine that returns + * the items one at a time in order. I've never seen a + * non-recursive tree walk routine published that returns + * one item at a time rather than doing a callback. + * + * Return the next item in sorted order. We assume first() + * was called once before calling this routine. + * We always go down as far as we can to the left, then up, and + * down one to the right, and again down as far as we can to the + * left. etc. + * + * Returns: pointer to next larger item + * NULL when no more items in tree + */ +void *rblist::next(void *item) +{ + void *x; + + if (!item) { + return first(); + } + + x = item; + if ((down && !left(x) && right(x)) || (!down && right(x))) { + /* Move down to right one */ + down = true; + x = right(x); + /* Then all the way down left */ + while (left(x)) { + x = left(x); + } + return x; + } + + /* We have gone down all we can, so now go up */ + for ( ;; ) { + /* If at head, we are done */ + if (!parent(x)) { + return NULL; + } + /* Move up in tree */ + down = false; + /* if coming from right, continue up */ + if (right(parent(x)) == x) { + x = parent(x); + continue; + } + /* Coming from left, go up one -- ie. return parent */ + return parent(x); + } +} + +/* + * Similer to next(), but visits all right nodes when + * coming up the tree. + */ +void *rblist::any(void *item) +{ + void *x; + + x = item; + if ((down && !left(x) && right(x)) || (!down && right(x))) { + /* Move down to right one */ + down = true; + x = right(x); + /* Then all the way down left */ + while (left(x)) { + x = left(x); + } + return x; + } + + /* We have gone down all we can, so now go up */ + for ( ;; ) { + /* If at head, we are done */ + if (!parent(x)) { + return NULL; + } + down = false; + /* Go up one and return parent */ + return parent(x); + } +} + + +/* x is item, y is below and to right, then rotated to below left */ +void rblist::left_rotate(void *item) +{ + void *y; + void *x; + + x = item; + y = right(x); + set_right(x, left(y)); + if (left(y)) { + set_parent(left(y), x); + } + set_parent(y, parent(x)); + /* if no parent then we have a new head */ + if (!parent(x)) { + head = y; + } else if (x == left(parent(x))) { + set_left(parent(x), y); + } else { + set_right(parent(x), y); + } + set_left(y, x); + set_parent(x, y); +} + +void rblist::right_rotate(void *item) +{ + void *x, *y; + + y = item; + x = left(y); + set_left(y, right(x)); + if (right(x)) { + set_parent(right(x), y); + } + set_parent(x, parent(y)); + /* if no parent then we have a new head */ + if (!parent(y)) { + head = x; + } else if (y == left(parent(y))) { + set_left(parent(y), x); + } else { + set_right(parent(y), x); + } + set_right(x, y); + set_parent(y, x); +} + + +void rblist::remove(void *item) +{ +} + +/* Destroy the tree contents. Not totally working */ +void rblist::destroy() +{ + void *x, *y = NULL; + + x = first(); +// printf("head=%p first=%p left=%p right=%p\n", head, x, left(x), right(x)); + + for ( ; (y=any(x)); ) { + /* Prune the last item */ + if (parent(x)) { + if (x == left(parent(x))) { + set_left(parent(x), NULL); + } else if (x == right(parent(x))) { + set_right(parent(x), NULL); + } + } + if (!left(x) && !right(x)) { + if (head == x) { + head = NULL; + } +// if (num_items<30) { +// printf("free nitems=%d item=%p left=%p right=%p\n", num_items, x, left(x), right(x)); +// } + free((void *)x); /* free previous node */ + num_items--; + } + x = y; /* save last node */ + } + if (x) { + if (x == head) { + head = NULL; + } +// printf("free nitems=%d item=%p left=%p right=%p\n", num_items, x, left(x), right(x)); + free((void *)x); + num_items--; + } + if (head) { +// printf("Free head\n"); + free((void *)head); + } +// printf("free nitems=%d\n", num_items); + + head = NULL; +} + + + +#ifdef TEST_PROGRAM + +struct MYJCR { + void link; + char *buf; +}; + +static int my_compare(void *item1, void *item2) +{ + MYJCR *jcr1, *jcr2; + int comp; + jcr1 = (MYJCR *)item1; + jcr2 = (MYJCR *)item2; + comp = strcmp(jcr1->buf, jcr2->buf); + //Dmsg3(000, "compare=%d: %s to %s\n", comp, jcr1->buf, jcr2->buf); + return comp; +} + +int main() +{ + char buf[30]; + rblist *jcr_chain; + MYJCR *jcr = NULL; + MYJCR *jcr1; + + + /* Now do a binary insert for the tree */ + jcr_chain = New(rblist()); +#define CNT 26 + printf("append %d items\n", CNT*CNT*CNT); + strcpy(buf, "ZZZ"); + int count = 0; + for (int i=0; ibuf = bstrdup(buf); +// printf("buf=%p %s\n", jcr, jcr->buf); + jcr1 = (MYJCR *)jcr_chain->insert((void *)jcr, my_compare); + if (jcr != jcr1) { + Dmsg2(000, "Insert of %s vs %s failed.\n", jcr->buf, jcr1->buf); + } + buf[1]--; + } + buf[1] = 'Z'; + buf[2]--; + } + buf[2] = 'Z'; + buf[0]--; + } + printf("%d items appended\n", CNT*CNT*CNT); + printf("num_items=%d\n", jcr_chain->size()); + + jcr = (MYJCR *)malloc(sizeof(MYJCR)); + memset(jcr, 0, sizeof(MYJCR)); + + jcr->buf = bstrdup("a"); + if ((jcr1=(MYJCR *)jcr_chain->search((void *)jcr, my_compare))) { + printf("One less failed!!!! Got: %s\n", jcr1->buf); + } else { + printf("One less: OK\n"); + } + free(jcr->buf); + + jcr->buf = bstrdup("ZZZZZZZZZZZZZZZZ"); + if ((jcr1=(MYJCR *)jcr_chain->search((void *)jcr, my_compare))) { + printf("One greater failed!!!! Got:%s\n", jcr1->buf); + } else { + printf("One greater: OK\n"); + } + free(jcr->buf); + + jcr->buf = bstrdup("AAA"); + if ((jcr1=(MYJCR *)jcr_chain->search((void *)jcr, my_compare))) { + printf("Search for AAA got %s\n", jcr1->buf); + } else { + printf("Search for AAA not found\n"); + } + free(jcr->buf); + + jcr->buf = bstrdup("ZZZ"); + if ((jcr1 = (MYJCR *)jcr_chain->search((void *)jcr, my_compare))) { + printf("Search for ZZZ got %s\n", jcr1->buf); + } else { + printf("Search for ZZZ not found\n"); + } + free(jcr->buf); + free(jcr); + + + printf("Find each of %d items in tree.\n", count); + for (jcr=(MYJCR *)jcr_chain->first(); jcr; (jcr=(MYJCR *)jcr_chain->next((void *)jcr)) ) { +// printf("Got: %s\n", jcr->buf); + if (!jcr_chain->search((void *)jcr, my_compare)) { + printf("rblist binary_search item not found = %s\n", jcr->buf); + } + } + printf("Free each of %d items in tree.\n", count); + for (jcr=(MYJCR *)jcr_chain->first(); jcr; (jcr=(MYJCR *)jcr_chain->next((void *)jcr)) ) { +// printf("Free: %p %s\n", jcr, jcr->buf); + free(jcr->buf); + jcr->buf = NULL; + } + printf("num_items=%d\n", jcr_chain->size()); + delete jcr_chain; + + + sm_dump(true); + +} +#endif diff --git a/bacula/src/lib/rblist.h b/bacula/src/lib/rblist.h new file mode 100644 index 0000000000..39d19bc008 --- /dev/null +++ b/bacula/src/lib/rblist.h @@ -0,0 +1,169 @@ +/* + Bacula® - The Network Backup Solution + + Copyright (C) 2005-20076 Free Software Foundation Europe e.V. + + The main author of Bacula is Kern Sibbald, with contributions from + many others, a complete list can be found in the file AUTHORS. + This program is Free Software; you can redistribute it and/or + modify it under the terms of version two of the GNU General Public + License as published by the Free Software Foundation plus additions + that are listed in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + + Bacula® is a registered trademark of John Walker. + The licensor of Bacula is the Free Software Foundation Europe + (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, + Switzerland, email:ftf@fsfeurope.org. +*/ + +/* + * Version $Id$ + */ + +/* ======================================================================== + * + * red-black binary tree routines -- rblist.h + * + * Kern Sibbald, MMV + * + */ + +#define M_ABORT 1 + +/* + * There is a lot of extra casting here to work around the fact + * that some compilers (Sun and Visual C++) do not accept + * (bnode *) as an lvalue on the left side of an equal. + * + * Loop var through each member of list + */ +#ifdef HAVE_TYPEOF +#define foreach_rblist(var, tree) \ + for (((var)=(typeof(var))(tree)->first()); (var); ((var)=(typeof(var))(tree)->next(var))) +#else +#define foreach_rblist(var, tree) \ + for ((*((void **)&(var))=(void*)((tree)->first())); (var); (*((void **)&(var))=(void*)((tree)->next(var)))) +#endif + +struct rblink { + void *parent; + void *left; + void *right; + bool red; +}; + +class rblist : public SMARTALLOC { + void *head; + int16_t loffset; + uint32_t num_items; + bool down; + void left_rotate(void *item); + void right_rotate(void *item); +public: + rblist(void *item, rblink *link); + rblist(void); + ~rblist(void) { destroy(); } + void init(void *item, rblink *link); + void set_parent(void *item, void *parent); + void set_left(void *item, void *left); + void set_right(void *item, void *right); + void set_red(void *item, bool red); + void *parent(const void *item) const; + void *left(const void *item) const; + void *right(const void *item) const; + bool red(const void *item) const; + void *insert(void *item, int compare(void *item1, void *item2)); + void *search(void *item, int compare(void *item1, void *item2)); + void *first(void); + void *next(void *item); + void *any(void *item); + void remove(void *item); + bool empty(void) const; + int size(void) const; + void destroy(void); +}; + +inline rblist::rblist(void *item, rblink *link) +{ + init(item, link); +} + +/* Constructor with link at head of item */ +inline rblist::rblist(void): head(0), loffset(0), num_items(0) +{ +} + +/* + * This allows us to do explicit initialization, + * allowing us to mix C++ classes inside malloc'ed + * C structures. Define before called in constructor. + */ +inline void rblist::init(void *item, rblink *link) +{ + head = NULL; + loffset = (int)((char *)link - (char *)item); + if (loffset < 0 || loffset > 5000) { + Emsg0(M_ABORT, 0, "Improper rblist initialization.\n"); + } + num_items = 0; +} + +inline void rblist::set_parent(void *item, void *parent) +{ + ((rblink *)(((char *)item)+loffset))->parent = parent; +} + +inline void rblist::set_left(void *item, void *left) +{ + ((rblink *)(((char *)item)+loffset))->left = left; +} + +inline void rblist::set_right(void *item, void *right) +{ + ((rblink *)(((char *)item)+loffset))->right = right; +} + +inline void rblist::set_red(void *item, bool red) +{ + ((rblink *)(((char *)item)+loffset))->red = red; +} + +inline bool rblist::empty(void) const +{ + return head == NULL; +} + +inline int rblist::size() const +{ + return num_items; +} + +inline void *rblist::parent(const void *item) const +{ + return ((rblink *)(((char *)item)+loffset))->parent; +} + +inline void *rblist::left(const void *item) const +{ + return ((rblink *)(((char *)item)+loffset))->left; +} + +inline void *rblist::right(const void *item) const +{ + return ((rblink *)(((char *)item)+loffset))->right; +} + +inline bool rblist::red(const void *item) const +{ + return ((rblink *)(((char *)item)+loffset))->red; +} diff --git a/bacula/src/lib/tree.c b/bacula/src/lib/tree.c index 708e524c53..f4dea3a199 100755 --- a/bacula/src/lib/tree.c +++ b/bacula/src/lib/tree.c @@ -294,7 +294,7 @@ static TREE_NODE *search_and_insert_tree_node(char *fname, int type, TREE_NODE *node, *found_node; node = new_tree_node(root); node->fname = fname; - found_node = (TREE_NODE *)parent->child.binary_insert(node, node_compare); + found_node = (TREE_NODE *)parent->child.insert(node, node_compare); if (found_node != node) { /* already in list */ free_tree_node(root); /* free node allocated above */ found_node->inserted = false; diff --git a/bacula/src/lib/tree.h b/bacula/src/lib/tree.h index fbe89b02fd..6ee6fdb642 100644 --- a/bacula/src/lib/tree.h +++ b/bacula/src/lib/tree.h @@ -58,8 +58,8 @@ struct s_mem { struct s_tree_node { /* KEEP sibling as the first member to avoid having to * do initialization of child */ - dlink sibling; - dlist child; + rblink sibling; + rblist child; char *fname; /* file name */ int32_t FileIndex; /* file index */ uint32_t JobId; /* JobId */ @@ -79,8 +79,8 @@ typedef struct s_tree_node TREE_NODE; struct s_tree_root { /* KEEP sibling as the first member to avoid having to * do initialization of child */ - dlink sibling; - dlist child; + rblink sibling; + rblist child; const char *fname; /* file name */ int32_t FileIndex; /* file index */ uint32_t JobId; /* JobId */ diff --git a/bacula/src/version.h b/bacula/src/version.h index 78cd11f5c2..335e6355bd 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -3,9 +3,9 @@ */ #undef VERSION -#define VERSION "2.1.1" -#define BDATE "28 January 2007" -#define LSMDATE "28Jan07" +#define VERSION "2.1.2" +#define BDATE "02 February 2007" +#define LSMDATE "02Feb07" #define PROG_COPYRIGHT "Copyright (C) %d-2007 Free Software Foundation Europe e.V.\n" #define BYEAR "2007" /* year for copyright messages in progs */ diff --git a/bacula/technotes-2.1 b/bacula/technotes-2.1 index bac697dbf4..6bdb1c64fc 100644 --- a/bacula/technotes-2.1 +++ b/bacula/technotes-2.1 @@ -1,7 +1,32 @@ Technical notes on version 2.1 General: -28Jan08 +02Feb07 +kes Fix memory leak with storage ids in cats/sql_get.c +kes Terminate watchdog earlier to avoid reference to released + memory -- reported by Jason Austin. +kes Move closing the database from jobq.c to the director daemon + termination routine. This fixes memory leaks for shadow jobs + (i.e. migration jobs). +kes Free up the unique jobid chain items in migrate.c. This fixes + a memory leak problem. +kes Convert some ugly looking for statements to use foreach_alist + in findlib/find.c. This will facilitate converting the structures + to use dlist (for large include/exclude lists). +kes Fix a bug in the btree.c and btree.h routines, then rename them + rblist and add them to be built in src/lib. Include some new + methods written by Rudolf Cejka that make the code more readable + (hides some of the ugly casting). +kes Add set_next() and set_prev() methods which make the code much more + readable. Also add a new dlistString class that facilitates storing + strings in dlists. To be used in the large include/exclude lists. +kes Make some trivial modifications to lib/tree.h that use rblist + rather than dlist for storing the tree links. This was suggested + by Rudolf Cejka. The result of this is that the restore tree now + uses red-black binary trees rather than simple linked lists. This + should give rather dramatic speed improvements for directories + contining large numbers of directories/files (more than 10000). +28Jan07 kes Fix maxruntime bug #621. 26Jan07 ebl Implement the include JobID in spool file name project.