From 1d1d4ab652baf58b3c92c5d7688765c5a46d0b54 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Mon, 13 Jan 2003 19:11:18 +0000 Subject: [PATCH] Fix smartall to be thread safe git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@285 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/kernstodo | 13 ++-- bacula/src/findlib/find.h | 108 +++++++++++++++++----------------- bacula/src/findlib/find_one.c | 58 ++++++++++-------- bacula/src/lib/mem_pool.c | 9 +-- bacula/src/lib/smartall.c | 50 ++++++++++------ 5 files changed, 131 insertions(+), 107 deletions(-) diff --git a/bacula/kernstodo b/bacula/kernstodo index 666a5d3271..a2eb615de5 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -16,13 +16,16 @@ Testing to do: (painful) - test and fix < code and | code. For 1.29 release: +- Add include list to end of chain in findlib +- Look into Pruning/purging problems or why there seem to + be so many files listed each night. +- Check out DB errors seen during recover. - Write Unix emulator for Windows. - Why is catreq.c:111 Find vol called twice for a job? -- Add include list to end of chain in findlib - Fix cancel in find_one -- need jcr. +- Cancel does not work for restore in FD. - Write SetJobStatus() function so cancel status not lost. - Find out why Full saves run slower and slower (hashing?) -- Test size of hash table needed in find_one using testfind. - Make 1.28c release - Rewrite find_one.c to use only pool_memory instead of alloca and malloc. @@ -33,9 +36,7 @@ For 1.29 release: - Why are save/restore of device different sizes (sparse?) Yup! Fix it. -- Implement some why for the Console to dynamically create - a job. -- Cancel does not work for restore in FD. +- Implement some why for the Console to dynamically create a job. - Restore to a particular time -- e.g. before date, after date. - Implement SHA1 - Implement disk spooling @@ -725,4 +726,4 @@ Done: (see kernsdone for more) - Figure out how to restore the catalog. - Make sure differential handled correctly - Look at ua_prune.c in detail. Why did JobType work at all?????? - +- Test size of hash table needed in find_one using testfind (about 1300). diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h index a16a3f2a2c..a696e5156f 100755 --- a/bacula/src/findlib/find.h +++ b/bacula/src/findlib/find.h @@ -35,8 +35,6 @@ #define MODE_RALL (S_IRUSR|S_IRGRP|S_IROTH) -#define DEFAULT_NAMEBUF_LEN 150 /* default filename buffer length */ - #ifdef HAVE_FNMATCH #include #else @@ -48,42 +46,42 @@ /* * Status codes returned by create_file() */ -#define CF_SKIP 1 /* skip file (not newer or something) */ -#define CF_ERROR 2 /* error creating file */ -#define CF_EXTRACT 3 /* file created, data to extract */ -#define CF_CREATED 4 /* file created, no data to extract */ +#define CF_SKIP 1 /* skip file (not newer or something) */ +#define CF_ERROR 2 /* error creating file */ +#define CF_EXTRACT 3 /* file created, data to extract */ +#define CF_CREATED 4 /* file created, no data to extract */ /* * NOTE!!! These go on the tape, so don't change them. If * need be, add to them. */ -#define FT_LNKSAVED 1 /* hard link to file already saved */ -#define FT_REGE 2 /* Regular file but empty */ -#define FT_REG 3 /* Regular file */ -#define FT_LNK 4 /* Soft Link */ -#define FT_DIR 5 /* Directory */ -#define FT_SPEC 6 /* Special file -- chr, blk, fifo, sock */ -#define FT_NOACCESS 7 /* Not able to access */ -#define FT_NOFOLLOW 8 /* Could not follow link */ -#define FT_NOSTAT 9 /* Could not stat file */ -#define FT_NOCHG 10 /* Incremental option, file not changed */ -#define FT_DIRNOCHG 11 /* Incremental option, directory not changed */ -#define FT_ISARCH 12 /* Trying to save archive file */ -#define FT_NORECURSE 13 /* No recursion into directory */ -#define FT_NOFSCHG 14 /* Different file system, prohibited */ -#define FT_NOOPEN 15 /* Could not open directory */ -#define FT_RAW 16 /* Raw block device */ -#define FT_FIFO 17 /* Raw fifo device */ +#define FT_LNKSAVED 1 /* hard link to file already saved */ +#define FT_REGE 2 /* Regular file but empty */ +#define FT_REG 3 /* Regular file */ +#define FT_LNK 4 /* Soft Link */ +#define FT_DIR 5 /* Directory */ +#define FT_SPEC 6 /* Special file -- chr, blk, fifo, sock */ +#define FT_NOACCESS 7 /* Not able to access */ +#define FT_NOFOLLOW 8 /* Could not follow link */ +#define FT_NOSTAT 9 /* Could not stat file */ +#define FT_NOCHG 10 /* Incremental option, file not changed */ +#define FT_DIRNOCHG 11 /* Incremental option, directory not changed */ +#define FT_ISARCH 12 /* Trying to save archive file */ +#define FT_NORECURSE 13 /* No recursion into directory */ +#define FT_NOFSCHG 14 /* Different file system, prohibited */ +#define FT_NOOPEN 15 /* Could not open directory */ +#define FT_RAW 16 /* Raw block device */ +#define FT_FIFO 17 /* Raw fifo device */ /* Options saved in "flag" of ff packet */ -#define FO_MD5 0x01 /* Do MD5 checksum */ -#define FO_GZIP 0x02 /* Do Zlib compression */ -#define FO_NO_RECURSION 0x04 /* no recursion in directories */ -#define FO_MULTIFS 0x08 /* multiple file systems */ -#define FO_SPARSE 0x10 /* do sparse file checking */ -#define FO_IF_NEWER 0x20 /* replace if newer */ -#define FO_NOREPLACE 0x40 /* never replace */ -#define FO_READFIFO 0x80 /* read data from fifo */ +#define FO_MD5 0x01 /* Do MD5 checksum */ +#define FO_GZIP 0x02 /* Do Zlib compression */ +#define FO_NO_RECURSION 0x04 /* no recursion in directories */ +#define FO_MULTIFS 0x08 /* multiple file systems */ +#define FO_SPARSE 0x10 /* do sparse file checking */ +#define FO_IF_NEWER 0x20 /* replace if newer */ +#define FO_NOREPLACE 0x40 /* never replace */ +#define FO_READFIFO 0x80 /* read data from fifo */ /* * Options saved in "options" of include list @@ -95,21 +93,21 @@ #define OPT_compute_MD5 0x01 /* compute MD5 of file's data */ #define OPT_GZIP_compression 0x02 /* use GZIP compression */ #define OPT_no_recursion 0x04 /* no recursion in directories */ -#define OPT_multifs 0x08 /* multiple file systems */ -#define OPT_sparse 0x10 /* do sparse file checking */ +#define OPT_multifs 0x08 /* multiple file systems */ +#define OPT_sparse 0x10 /* do sparse file checking */ #define OPT_replace_if_newer 0x20 /* replace file if newer */ #define OPT_never_replace 0x40 /* never replace */ -#define OPT_read_fifo 0x80 /* read data from fifo (named pipe) */ +#define OPT_read_fifo 0x80 /* read data from fifo (named pipe) */ struct s_included_file { struct s_included_file *next; - int options; /* backup options */ - int level; /* compression level */ - int len; /* length of fname */ - int pattern; /* set if pattern */ - char VerifyOpts[20]; /* Options for verify */ + int options; /* backup options */ + int level; /* compression level */ + int len; /* length of fname */ + int pattern; /* set if pattern */ + char VerifyOpts[20]; /* Options for verify */ char fname[1]; }; @@ -125,27 +123,27 @@ struct s_excluded_file { * first argument to the find_files callback subroutine. */ typedef struct ff { - char *fname; /* filename */ - char *link; /* link if file linked */ - POOLMEM *sys_fname; /* system filename */ - struct stat statp; /* stat packet */ - int type; /* FT_ type from above */ - int fid; /* file id if opened */ - int flags; /* control flags */ - int ff_errno; /* errno */ - int incremental; /* do incremental save */ - time_t save_time; /* start of incremental time */ - int mtime_only; /* incremental on mtime_only */ - int dereference; /* follow links */ - int GZIP_level; /* compression level */ - int atime_preserve; /* preserve access times */ - int null_output_device; /* using null output device */ + char *fname; /* filename */ + char *link; /* link if file linked */ + POOLMEM *sys_fname; /* system filename */ + struct stat statp; /* stat packet */ + int type; /* FT_ type from above */ + int fid; /* file id if opened */ + int flags; /* control flags */ + int ff_errno; /* errno */ + int incremental; /* do incremental save */ + time_t save_time; /* start of incremental time */ + int mtime_only; /* incremental on mtime_only */ + int dereference; /* follow links */ + int GZIP_level; /* compression level */ + int atime_preserve; /* preserve access times */ + int null_output_device; /* using null output device */ char VerifyOpts[20]; struct s_included_file *included_files_list; struct s_excluded_file *excluded_files_list; struct s_excluded_file *excluded_paths_list; - struct f_link *linklist; /* hard linked files */ + struct f_link *linklist; /* hard linked files */ } FF_PKT; #include "protos.h" diff --git a/bacula/src/findlib/find_one.c b/bacula/src/findlib/find_one.c index 5f2f3468ea..132de5a40b 100755 --- a/bacula/src/findlib/find_one.c +++ b/bacula/src/findlib/find_one.c @@ -178,9 +178,9 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt } else if (S_ISLNK(ff_pkt->statp.st_mode)) { int size; - char *buffer = (char *)alloca(PATH_MAX + 1); + char *buffer = (char *)alloca(path_max + name_max + 2); - size = readlink(fname, buffer, PATH_MAX + 1); + size = readlink(fname, buffer, path_max + name_max + 1); if (size < 0) { /* Could not follow link */ ff_pkt->type = FT_NOFOLLOW; @@ -195,9 +195,9 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt } else if (S_ISDIR(ff_pkt->statp.st_mode)) { DIR *directory; struct dirent *entry, *result; - char *namebuf; - size_t namebuf_len; - size_t len; + char *link; + int link_len; + int len; int status; dev_t our_device = ff_pkt->statp.st_dev; @@ -210,15 +210,16 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt /* Build a canonical directory name with a trailing slash. */ len = strlen(fname); - namebuf_len = len + DEFAULT_NAMEBUF_LEN; - namebuf = (char *)bmalloc(namebuf_len + 2); - bstrncpy(namebuf, fname, namebuf_len); - while (len >= 1 && namebuf[len - 1] == '/') + link_len = len + 200; + link = (char *)bmalloc(link_len + 2); + bstrncpy(link, fname, link_len); + /* Strip all trailing slashes */ + while (len >= 1 && link[len - 1] == '/') len--; - namebuf[len++] = '/'; - namebuf[len] = '\0'; + link[len++] = '/'; /* add back one */ + link[len] = 0; - ff_pkt->link = namebuf; + ff_pkt->link = link; if (ff_pkt->incremental && (ff_pkt->statp.st_mtime < ff_pkt->save_time && ff_pkt->statp.st_ctime < ff_pkt->save_time)) { @@ -236,7 +237,7 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt * user has turned it off for this directory. */ if (ff_pkt->flags & FO_NO_RECURSION) { - free(namebuf); + free(link); /* No recursion into this directory */ ff_pkt->type = FT_NORECURSE; return handle_file(ff_pkt, pkt); @@ -248,7 +249,7 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt */ if (!top_level && !(ff_pkt->flags & FO_MULTIFS) && parent_device != ff_pkt->statp.st_dev) { - free(namebuf); + free(link); /* returning here means we do not handle this directory */ ff_pkt->type = FT_NOFSCHG; return handle_file(ff_pkt, pkt); @@ -258,7 +259,7 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt */ errno = 0; if ((directory = opendir(fname)) == NULL) { - free(namebuf); + free(link); ff_pkt->type = FT_NOOPEN; ff_pkt->ff_errno = errno; return handle_file(ff_pkt, pkt); @@ -269,16 +270,19 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt * before traversing it. */ rtn_stat = 1; - entry = (struct dirent *)malloc(sizeof(struct dirent) + name_max + 10); + entry = (struct dirent *)malloc(sizeof(struct dirent) + name_max + 100); for ( ;; ) { - char *p; + char *p, *q; + int i; status = readdir_r(directory, entry, &result); + sm_check(__FILE__, __LINE__, False); Dmsg3(200, "readdir stat=%d result=%x name=%s\n", status, result, entry->d_name); if (status != 0 || result == NULL) { break; } + ASSERT(name_max+1 > sizeof(struct dirent) + (int)NAMELEN(entry)); p = entry->d_name; /* Skip `.', `..', and excluded file names. */ if (p[0] == '\0' || (p[0] == '.' && (p[1] == '\0' || @@ -286,18 +290,24 @@ find_one_file(FF_PKT *ff_pkt, int handle_file(FF_PKT *ff, void *hpkt), void *pkt continue; } - if ((int)NAMELEN(entry) + len >= namebuf_len) { - namebuf_len = len + NAMELEN(entry); - namebuf = (char *)brealloc(namebuf, namebuf_len + 2); + if ((int)NAMELEN(entry) + len >= link_len) { + link_len = len + NAMELEN(entry) + 1; + link = (char *)brealloc(link, link_len + 1); } - strcpy(namebuf + len, entry->d_name); - if (!file_is_excluded(ff_pkt, namebuf)) { - rtn_stat = find_one_file(ff_pkt, handle_file, pkt, namebuf, our_device, 0); + q = link + len; + for (i=0; i < (int)NAMELEN(entry); i++) { + *q++ = *p++; + } + *q = 0; + sm_check(__FILE__, __LINE__, False); + if (!file_is_excluded(ff_pkt, link)) { + rtn_stat = find_one_file(ff_pkt, handle_file, pkt, link, our_device, 0); } } closedir(directory); - free(namebuf); + free(link); free(entry); + if (ff_pkt->atime_preserve) { utime(fname, &restore_times); } diff --git a/bacula/src/lib/mem_pool.c b/bacula/src/lib/mem_pool.c index 1c47b41441..4f928deb69 100644 --- a/bacula/src/lib/mem_pool.c +++ b/bacula/src/lib/mem_pool.c @@ -50,6 +50,7 @@ struct s_pool_ctl { struct abufhead *free_buf; /* pointer to free buffers */ }; +/* #define STRESS_TEST_POOL */ #ifndef STRESS_TEST_POOL /* * Define default Pool buffer sizes @@ -64,10 +65,10 @@ static struct s_pool_ctl pool_ctl[] = { /* This is used ONLY when stress testing the code */ static struct s_pool_ctl pool_ctl[] = { - { 10, 10, 0, 0, NULL }, /* PM_NOPOOL no pooling */ - { 10, 10, 0, 0, NULL }, /* PM_FNAME filename buffers */ - { 10, 10, 0, 0, NULL }, /* PM_MESSAGE message buffer */ - { 10, 10, 0, 0, NULL } /* PM_EMSG error message buffer */ + { 20, 20, 0, 0, NULL }, /* PM_NOPOOL no pooling */ + { 20, 20, 0, 0, NULL }, /* PM_FNAME filename buffers */ + { 20, 20, 0, 0, NULL }, /* PM_MESSAGE message buffer */ + { 20, 20, 0, 0, NULL } /* PM_EMSG error message buffer */ }; #endif diff --git a/bacula/src/lib/smartall.c b/bacula/src/lib/smartall.c index a5b4beed73..5b7bda5795 100644 --- a/bacula/src/lib/smartall.c +++ b/bacula/src/lib/smartall.c @@ -19,7 +19,7 @@ */ /* - Copyright (C) 2000, 2001, 2002 Kern Sibbald and John Walker + Copyright (C) 2000-2003 Kern Sibbald and John Walker This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -45,11 +45,12 @@ #undef malloc #undef free -/*LINTLIBRARY*/ #ifdef SMARTALLOC +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + extern char my_name[]; /* daemon name */ typedef unsigned short sm_ushort; @@ -95,7 +96,8 @@ static void *smalloc(char *fname, int lineno, unsigned int nbytes) ASSERT(nbytes > 0); nbytes += HEAD_SIZE + 1; - if ((buf = (char *) malloc(nbytes)) != NULL) { + if ((buf = (char *)malloc(nbytes)) != NULL) { + P(mutex); /* Enqueue buffer on allocated list */ qinsert(&abqueue, (struct b_queue *) buf); ((struct abufhead *) buf)->ablen = nbytes; @@ -104,10 +106,11 @@ static void *smalloc(char *fname, int lineno, unsigned int nbytes) /* Emplace end-clobber detector at end of buffer */ buf[nbytes - 1] = (((long) buf) & 0xFF) ^ 0xC5; buf += HEAD_SIZE; /* Increment to user data start */ + V(mutex); } sm_check(fname, lineno, True); Dmsg4(1150, "smalloc %d at %x from %s:%d\n", nbytes, buf, fname, lineno); - return (void *) buf; + return (void *)buf; } /* SM_NEW_OWNER -- Update the File and line number for a buffer @@ -116,8 +119,8 @@ static void *smalloc(char *fname, int lineno, unsigned int nbytes) void sm_new_owner(char *fname, int lineno, char *buf) { buf -= HEAD_SIZE; /* Decrement to header */ - ((struct abufhead *) buf)->abfname = bufimode ? NULL : fname; - ((struct abufhead *) buf)->ablineno = (sm_ushort) lineno; + ((struct abufhead *)buf)->abfname = bufimode ? NULL : fname; + ((struct abufhead *)buf)->ablineno = (sm_ushort) lineno; return; } @@ -139,6 +142,7 @@ void sm_free(char *file, int line, void *fp) cp -= HEAD_SIZE; qp = (struct b_queue *) cp; + P(mutex); Dmsg4(1150, "sm_free %d at %x from %s:%d\n", ((struct abufhead *)cp)->ablen, fp, ((struct abufhead *)cp)->abfname, ((struct abufhead *)cp)->ablineno); @@ -146,9 +150,11 @@ void sm_free(char *file, int line, void *fp) /* The following assertions will catch virtually every release of an address which isn't an allocated buffer. */ if (qp->qnext->qprev != qp) { + V(mutex); Emsg2(M_ABORT, 0, "qp->qnext->qprev != qp called from %s:%d\n", file, line); } if (qp->qprev->qnext != qp) { + V(mutex); Emsg2(M_ABORT, 0, "qp->qprev->qnext != qp called from %s:%d\n", file, line); } @@ -158,11 +164,13 @@ void sm_free(char *file, int line, void *fp) if (((unsigned char *) cp)[((struct abufhead *) cp)->ablen - 1] != ((((long) cp) & 0xFF) ^ 0xC5)) { + V(mutex); Emsg2(M_ABORT, 0, "Buffer overrun called from %s:%d\n", file, line); } qdchain(qp); + V(mutex); /* Now we wipe the contents of the just-released buffer with "designer garbage" (Duff Kurland's phrase) of alternating @@ -238,8 +246,7 @@ void *sm_realloc(char *fname, int lineno, void *ptr, unsigned int size) return the buffer passed in. */ cp -= HEAD_SIZE; - osize = ((struct abufhead *) cp)->ablen - - (HEAD_SIZE + 1); + osize = ((struct abufhead *) cp)->ablen - (HEAD_SIZE + 1); if (size == osize) { return ptr; } @@ -312,7 +319,11 @@ void actuallyfree(void *cp) */ void sm_dump(Boolean bufdump) { - struct abufhead *ap = (struct abufhead *) abqueue.qnext; + struct abufhead *ap; + + P(mutex); + + ap = (struct abufhead *)abqueue.qnext; while (ap != (struct abufhead *) &abqueue) { @@ -359,6 +370,7 @@ void sm_dump(Boolean bufdump) } ap = (struct abufhead *) ap->abq.qnext; } + V(mutex); } #undef sm_check @@ -375,9 +387,11 @@ void sm_check(char *fname, int lineno, Boolean bufdump) /* SM_CHECK_RTN -- Check the buffers and return 1 if OK otherwise 0 */ int sm_check_rtn(char *fname, int lineno, Boolean bufdump) { - struct abufhead *ap = (struct abufhead *) abqueue.qnext; + struct abufhead *ap; int bad, badbuf = 0; + P(mutex); + ap = (struct abufhead *) abqueue.qnext; while (ap != (struct abufhead *) &abqueue) { bad = 0; if ((ap == NULL) || @@ -393,30 +407,29 @@ int sm_check_rtn(char *fname, int lineno, Boolean bufdump) } badbuf |= bad; if (bad) { - Emsg2(M_FATAL, 0, + fprintf(stderr, "\nDamaged buffers found at %s:%d\n", fname, lineno); if (bad & 0x1) { - Emsg0(M_FATAL, 0, " discovery of bad prev link.\n"); + fprintf(stderr, " discovery of bad prev link.\n"); } if (bad & 0x2) { - Emsg0(M_FATAL, 0, " discovery of bad next link.\n"); + fprintf(stderr, " discovery of bad next link.\n"); } if (bad & 0x4) { - Emsg0(M_FATAL, 0, " discovery of data overrun.\n"); + fprintf(stderr, " discovery of data overrun.\n"); } - Emsg1(M_FATAL, 0, " Buffer address: %lx\n", (long) ap); + fprintf(stderr, " Buffer address: %lx\n", (long) ap); if (ap->abfname != NULL) { unsigned memsize = ap->ablen - (HEAD_SIZE + 1); char errmsg[80]; - sprintf(errmsg, + fprintf(stderr, "Damaged buffer: %6u bytes allocated at line %d of %s %s\n", memsize, ap->ablineno, my_name, ap->abfname ); - Emsg1(M_FATAL, 0, "%s", errmsg); if (bufdump) { unsigned llen = 0; char *cp = ((char *) ap) + HEAD_SIZE; @@ -439,12 +452,13 @@ int sm_check_rtn(char *fname, int lineno, Boolean bufdump) llen++; memsize--; } - Emsg1(M_FATAL, 0, "%s\n", errmsg); + fprintf(stderr, "%s\n", errmsg); } } } ap = (struct abufhead *) ap->abq.qnext; } + V(mutex); return badbuf ? 0 : 1; } -- 2.39.5