From: Kern Sibbald Date: Sat, 25 Oct 2003 10:34:34 +0000 (+0000) Subject: Optimize tree.c a bit, take Scott's spec & patch X-Git-Tag: Release-7.0.0~9941 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=eaa0197fb34ad77bcd72b5a7a7d0e83bb9de9591;p=bacula%2Fbacula Optimize tree.c a bit, take Scott's spec & patch git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@773 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/kernstodo b/bacula/kernstodo index 12d9065149..9770ef2b3d 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -45,6 +45,7 @@ For 1.33 - Finish implementation of Verify=DiskToCatalog - Change console to bconsole. - Change smtp to bsmtp. +- Possibly up network buffers to 65K. - Add device name to "Current Volume not acceptable because ..." - Make sure that Bacula rechecks the tape after the 20 min wait. - Set IO_NOWAIT on Bacula TCP/IP packets. @@ -1069,4 +1070,3 @@ Done: (see kernsdone for more) - Document need to put LabelFormat in quotes. - Implement job in VerifyToCatalog - Eliminate ua_retention.c (retentioncmd) if possible. - diff --git a/bacula/platforms/redhat/bacula.spec.in b/bacula/platforms/redhat/bacula.spec.in index f69b37527f..c49d55c777 100644 --- a/bacula/platforms/redhat/bacula.spec.in +++ b/bacula/platforms/redhat/bacula.spec.in @@ -38,7 +38,8 @@ Version: @VERSION@ Release: 1 Group: System Environment/Daemons Copyright: GPL v2 -Source: http://www.prdownloads.sourceforge.net/bacula/%{name}-%{version}.tar.gz +Source0:http://www.prdownloads.sourceforge.net/bacula/%{name}-%{version}.tar.gz +Source1:http://www.prdownloads.sourceforge.net/bacula/depkgs-24Jul03.tar.gz BuildRoot: %{_tmppath}/%{name}-root URL: http://www.bacula.org/ Vendor: The Bacula Team @@ -116,7 +117,7 @@ This is the File daemon (Client) only package. %prep -%setup +%setup -b 1 %build @@ -131,6 +132,9 @@ cd ${cwd} # patch the make_sqlite_tables script for installation bindir patch src/cats/make_sqlite_tables.in src/cats/make_sqlite_tables.in.patch +# patch the make_catalog_backup script for installation bindir +patch src/cats/make_catalog_backup.in src/cats/make_catalog_backup.in.patch + %configure \ --prefix=/usr \ --sbindir=/usr/sbin \ @@ -369,6 +373,9 @@ chmod 0755 /usr/sbin/gnome-console /sbin/chkconfig --del bacula-fd %changelog +* Fri Oct 24 2003 D. Scott Barninger +- Added separate Source declaration for depkgs +- added patch for make_catalog_backup script * Mon May 11 2003 D. Scott Barninger - Misc changes to mysql/sqlite build and rh7/8 menu differences - Added rh_version to sub-package names diff --git a/bacula/src/cats/make_catalog_backup.in b/bacula/src/cats/make_catalog_backup.in index fe9c774fc3..19d1a30d49 100755 --- a/bacula/src/cats/make_catalog_backup.in +++ b/bacula/src/cats/make_catalog_backup.in @@ -13,7 +13,6 @@ fi # # To read back a MySQL database use: # cd @working_dir@ -# rm -f @SQL_BINDIR@/../var/bacula/* # mysql bacula.sql +--- +> echo ".dump" | /usr/lib/sqlite/sqlite bacula.db >bacula.sql +11c11 +< @SQL_BINDIR@/mysqldump $* -f --opt bacula >bacula.sql +--- +> /usr/bin/mysqldump $* -f --opt bacula >bacula.sql +16c16 +< # rm -f @SQL_BINDIR@/../var/bacula/* +--- +> # rm -f /var/lib/mysql/bacula/* +22c22 +< # sqlite bacula.db # /usr/lib/sqlite/sqlite bacula.db FileIndex = atoi(row[2]); new_node->JobId = atoi(row[3]); new_node->type = type; - new_node->extract = 1; /* extract all by default */ + new_node->extract = true; /* extract all by default */ tree->cnt++; return 0; } @@ -174,19 +174,19 @@ int insert_tree_handler(void *ctx, int num_fields, char **row) * down the tree setting all children if the * node is a directory. */ -static void set_extract(UAContext *ua, TREE_NODE *node, TREE_CTX *tree, int value) +static void set_extract(UAContext *ua, TREE_NODE *node, TREE_CTX *tree, bool extract) { TREE_NODE *n; FILE_DBR fdbr; struct stat statp; - node->extract = value; + node->extract = extract; /* For a non-file (i.e. directory), we see all the children */ if (node->type != TN_FILE) { for (n=node->child; n; n=n->sibling) { - set_extract(ua, n, tree, value); + set_extract(ua, n, tree, extract); } - } else if (value) { + } else if (extract) { char cwd[2000]; /* Ordinary file, we get the full path, look up the * attributes, decode them, and if we are hard linked to @@ -206,7 +206,7 @@ static void set_extract(UAContext *ua, TREE_NODE *node, TREE_CTX *tree, int valu if (LinkFI) { for (n=first_tree_node(tree->root); n; n=next_tree_node(n)) { if (n->FileIndex == LinkFI && n->JobId == node->JobId) { - n->extract = 1; + n->extract = true; break; } } @@ -226,7 +226,7 @@ static int markcmd(UAContext *ua, TREE_CTX *tree) } for (node = tree->node->child; node; node=node->sibling) { if (fnmatch(ua->argk[1], node->fname, 0) == 0) { - set_extract(ua, node, tree, 1); + set_extract(ua, node, tree, true); } } return 1; @@ -234,18 +234,18 @@ static int markcmd(UAContext *ua, TREE_CTX *tree) static int countcmd(UAContext *ua, TREE_CTX *tree) { - int total, extract; + int total, num_extract; - total = extract = 0; + total = num_extract = 0; for (TREE_NODE *node=first_tree_node(tree->root); node; node=next_tree_node(node)) { if (node->type != TN_NEWDIR) { total++; if (node->extract) { - extract++; + num_extract++; } } } - bsendmsg(ua, "%d total files. %d marked for restoration.\n", total, extract); + bsendmsg(ua, "%d total files. %d marked for restoration.\n", total, num_extract); return 1; } @@ -293,7 +293,7 @@ extern char *getgroup(gid_t gid); /* * This is actually the long form used for "dir" */ -static void ls_output(char *buf, char *fname, int extract, struct stat *statp) +static void ls_output(char *buf, char *fname, bool extract, struct stat *statp) { char *p, *f; char ec1[30]; @@ -421,7 +421,7 @@ static int unmarkcmd(UAContext *ua, TREE_CTX *tree) } for (node = tree->node->child; node; node=node->sibling) { if (fnmatch(ua->argk[1], node->fname, 0) == 0) { - set_extract(ua, node, tree, 0); + set_extract(ua, node, tree, false); } } return 1; diff --git a/bacula/src/lib/tree.c b/bacula/src/lib/tree.c index 1927f26ba7..3ca5b14692 100755 --- a/bacula/src/lib/tree.c +++ b/bacula/src/lib/tree.c @@ -32,6 +32,15 @@ #define MAXPATHLEN 1000 #endif +#undef Dmsg0 +#undef Dmsg1 +#undef Dmsg2 +#undef Dmsg3 +#define Dmsg0(n,f) +#define Dmsg1(n,f,a1) +#define Dmsg2(n,f,a1,a2) +#define Dmsg3(n,f,a1,a2,a3) + /* * This subroutine gets a big buffer. */ @@ -40,11 +49,13 @@ static void malloc_buf(TREE_ROOT *root, int size) struct s_mem *mem; mem = (struct s_mem *)malloc(size); + root->total_size += size; + root->blocks++; mem->next = root->mem; root->mem = mem; mem->mem = mem->first; mem->rem = (char *)mem + size - mem->mem; - Dmsg2(400, "malloc buf size=%d rem=%d\n", size, mem->rem); + Dmsg2(200, "malloc buf size=%d rem=%d\n", size, mem->rem); } @@ -65,9 +76,9 @@ TREE_ROOT *new_tree(int count) root = (TREE_ROOT *)malloc(sizeof(TREE_ROOT)); memset(root, 0, sizeof(TREE_ROOT)); root->type = TN_ROOT; - /* Assume filename = 20 characters average length */ - size = count * (BALIGN(sizeof(TREE_NODE)) + 20); - if (size > 10000000) { + /* Assume filename + node = 40 characters average length */ + size = count * (BALIGN(sizeof(TREE_NODE)) + 40); + if (count > 1000000 || size > 10000000) { size = 10000000; } Dmsg2(400, "count=%d size=%d\n", count, size); @@ -82,15 +93,20 @@ TREE_ROOT *new_tree(int count) TREE_NODE *new_tree_node(TREE_ROOT *root, int type) { TREE_NODE *node; - int size = BALIGN(sizeof(TREE_NODE)); + int asize = BALIGN(sizeof(TREE_NODE)); - if (root->mem->rem < size) { - malloc_buf(root, 20000); + if (root->mem->rem < asize) { + uint32_t mb_size; + if (root->total_size >= 1000000) { + mb_size = 1000000; + } else { + mb_size = 100000; + } + malloc_buf(root, mb_size); } - - root->mem->rem -= size; + root->mem->rem -= asize; node = (TREE_NODE *)root->mem->mem; - root->mem->mem += size; + root->mem->mem += asize; memset(node, 0, sizeof(TREE_NODE)); node->type = type; return node; @@ -108,7 +124,13 @@ static char *tree_alloc(TREE_ROOT *root, int size) int asize = BALIGN(size); if (root->mem->rem < asize) { - malloc_buf(root, 20000+asize); + uint32_t mb_size; + if (root->total_size >= 1000000) { + mb_size = 1000000; + } else { + mb_size = 100000; + } + malloc_buf(root, mb_size); } root->mem->rem -= asize; buf = root->mem->mem; @@ -130,6 +152,7 @@ void free_tree(TREE_ROOT *root) if (root->cached_path) { free_pool_memory(root->cached_path); } + Dmsg2(400, "Total size=%u blocks=%d\n", root->total_size, root->blocks); free(root); return; } @@ -137,7 +160,8 @@ void free_tree(TREE_ROOT *root) /* - * Insert a node in the tree + * Insert a node in the tree. This is the main subroutine + * called when building a tree. * */ TREE_NODE *insert_tree_node(char *path, TREE_NODE *node, @@ -145,14 +169,14 @@ TREE_NODE *insert_tree_node(char *path, TREE_NODE *node, { TREE_NODE *sibling; char *p, *q, *fname; - int len = strlen(path); + int path_len = strlen(path); Dmsg1(100, "insert_tree_node: %s\n", path); /* * If trailing slash, strip it */ - if (len > 0) { - q = path + len - 1; + if (path_len > 0) { + q = path + path_len - 1; if (*q == '/') { *q = 0; /* strip trailing slash */ } else { @@ -167,11 +191,12 @@ TREE_NODE *insert_tree_node(char *path, TREE_NODE *node, if (!parent) { /* if no parent, we need to make one */ *p = 0; /* terminate path */ Dmsg1(100, "make_tree_path for %s\n", path); - if ((int)strlen(path) == root->cached_path_len && + path_len = strlen(path); /* get new length */ + if (path_len == root->cached_path_len && strcmp(path, root->cached_path) == 0) { parent = root->cached_parent; } else { - root->cached_path_len = strlen(path); + root->cached_path_len = path_len; pm_strcpy(&root->cached_path, path); parent = make_tree_path(path, root); root->cached_parent = parent; @@ -188,9 +213,11 @@ TREE_NODE *insert_tree_node(char *path, TREE_NODE *node, Dmsg1(100, "No / found: %s\n", path); } + uint16_t fname_len = strlen(fname); for (sibling=parent->child; sibling; sibling=sibling->sibling) { Dmsg2(100, "sibling->fname=%s fname=%s\n", sibling->fname, fname); - if (strcmp(sibling->fname, fname) == 0) { + if (sibling->fname_len == fname_len && + strcmp(sibling->fname, fname) == 0) { Dmsg1(100, "make_tree_path: found parent=%s\n", parent->fname); if (q) { /* if trailing slash on entry */ *q = '/'; /* restore it */ @@ -234,9 +261,11 @@ TREE_NODE *make_tree_path(char *path, TREE_ROOT *root) type = TN_DIR_NLS; } /* Is it already a sibling? */ + uint16_t fname_len = strlen(fname); for (sibling=parent->child; sibling; sibling=sibling->sibling) { Dmsg2(100, "sibling->fname=%s fname=%s\n", sibling->fname, fname); - if (strcmp(sibling->fname, fname) == 0) { + if (sibling->fname_len == fname_len && + strcmp(sibling->fname, fname) == 0) { Dmsg1(100, "make_tree_path: found parent=%s\n", parent->fname); return sibling; } @@ -256,7 +285,8 @@ void append_tree_node(char *fname, TREE_NODE *node, TREE_ROOT *root, TREE_NODE * TREE_NODE *child; Dmsg1(100, "append_tree_node: %s\n", fname); - node->fname = tree_alloc(root, strlen(fname) + 1); + node->fname_len = strlen(fname); + node->fname = tree_alloc(root, node->fname_len + 1); strcpy(node->fname, fname); node->parent = parent; if (!parent->child) { @@ -435,6 +465,7 @@ int main(int argc, char *argv[]) root = new_tree(); root->fname = tree_alloc(root, 1); *root->fname = 0; + root->fname_len = 0; FillDirectoryTree("/home/kern/bacula/k", root, NULL); diff --git a/bacula/src/lib/tree.h b/bacula/src/lib/tree.h index 2fd3386b02..608dd3b90b 100644 --- a/bacula/src/lib/tree.h +++ b/bacula/src/lib/tree.h @@ -31,12 +31,17 @@ struct s_mem { char first[1]; /* first byte */ }; +/* + * Keep this node as small as possible because + * there is one for each file. + */ struct s_tree_node { char *fname; /* file name */ int32_t FileIndex; /* file index */ uint32_t JobId; /* JobId */ - short type; /* node type */ - short extract; /* set if extracting */ + uint16_t fname_len; /* length of string */ + uint8_t type; /* node type */ + bool extract; /* set if extracting */ struct s_tree_node *parent; struct s_tree_node *sibling; struct s_tree_node *child; @@ -48,8 +53,9 @@ struct s_tree_root { char *fname; /* file name */ int32_t FileIndex; /* file index */ uint32_t JobId; /* JobId */ - short type; /* node type */ - short extract; /* set if extracting */ + uint16_t fname_len; /* length of string */ + uint8_t type; /* node type */ + bool extract; /* set if extracting */ struct s_tree_node *parent; struct s_tree_node *sibling; struct s_tree_node *child; @@ -59,6 +65,8 @@ struct s_tree_root { struct s_tree_node *first; /* first entry in the tree */ struct s_tree_node *last; /* last entry in tree */ struct s_mem *mem; /* tree memory */ + uint32_t total_size; /* total bytes allocated */ + uint32_t blocks; /* total mallocs */ int cached_path_len; /* length of cached path */ char *cached_path; /* cached current path */ TREE_NODE *cached_parent; /* cached parent for above path */