From c66d0785ff68244b8ce0b7ebb5817492448786c1 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sun, 12 Nov 2006 11:08:07 +0000 Subject: [PATCH] kes Change error message 'illegal' to 'invalid' -- bug #707 kes Add rather primitive device resource deadlock detection in the job queue handler. It detects the same read and write device for migration and cancels the job. kes Start adding a unique_dbid_hander() routine in migration to avoid duplicate mediaids and duplicate jobids. kes Add patch from bug #708 to permit relative paths such as ../xxx in the restore tree routine. kes Add Eric Bollengier's patch to reduce locking time in the SD after despooling, and thus many jobs finish faster. kes Correct locking order of reservations lock and device mutex in reserve.c -- this should correct Arno's deadlock that occurred when doing mount/unmount at the same time a job is reserving a drive. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@3606 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/.cvsignore | 5 + bacula/kernstodo | 16 +++ bacula/src/cats/make_catalog_backup.in | 2 +- bacula/src/console/console.c | 2 +- bacula/src/dird/jobq.c | 23 ++- bacula/src/dird/migrate.c | 22 +++ bacula/src/lib/serial.c | 191 ++++++++++++------------- bacula/src/lib/tree.c | 10 +- bacula/src/stored/.cvsignore | 5 + bacula/src/stored/acquire.c | 9 +- bacula/src/stored/append.c | 6 +- bacula/src/stored/block.c | 8 +- bacula/src/stored/dircmd.c | 1 + bacula/src/stored/reserve.c | 3 + bacula/src/stored/spool.c | 7 +- bacula/src/win32/scripts/.cvsignore | 1 + bacula/technotes-1.39 | 21 ++- 17 files changed, 210 insertions(+), 122 deletions(-) create mode 100644 bacula/src/win32/scripts/.cvsignore diff --git a/bacula/.cvsignore b/bacula/.cvsignore index d447ee6f7a..ef7abf4a81 100644 --- a/bacula/.cvsignore +++ b/bacula/.cvsignore @@ -63,3 +63,8 @@ zapall zaptapes kerns-pgsql-config fake-mtx +filter +label +relabel +sed +weof diff --git a/bacula/kernstodo b/bacula/kernstodo index 5b9deb7295..61433f8bfb 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -41,6 +41,22 @@ Document: Priority: +- Arno's reservation deadlock. +- Doc items +- Bug reports +- Eric's SD patch +- Add encryption regression tests +- Test Volume compatibility between machine architectures +- Encryption documentation +- Migration Volume span bug +- Wrong jobbytes with query 12 (todo) +- bacula-1.38.2-ssl.patch +- Bare-metal recovery Windows (todo) +- Rescue release +- Test FIFO backup/restore -- make regression +- Document need for UTF-8 format + + For 1.39: - Implement Python event for backing up/restoring a file. diff --git a/bacula/src/cats/make_catalog_backup.in b/bacula/src/cats/make_catalog_backup.in index 8bc30a582b..fcd060d4df 100755 --- a/bacula/src/cats/make_catalog_backup.in +++ b/bacula/src/cats/make_catalog_backup.in @@ -27,7 +27,7 @@ else if test xpostgresql = x@DB_NAME@ ; then if test $# -gt 2; then PGPASSWORD=$3 - export PGPASSWORD + export PGPASSWORD fi exec @SQL_BINDIR@/pg_dump -c -U $2 $1 >$1.sql else diff --git a/bacula/src/console/console.c b/bacula/src/console/console.c index 331f50ccc8..26ff1ed6bc 100644 --- a/bacula/src/console/console.c +++ b/bacula/src/console/console.c @@ -185,7 +185,7 @@ static int do_a_command(FILE *input, BSOCK *UA_sock) } } if (!found) { - pm_strcat(&UA_sock->msg, _(": is an illegal command\n")); + pm_strcat(&UA_sock->msg, _(": is an invalid command\n")); UA_sock->msglen = strlen(UA_sock->msg); sendit(UA_sock->msg); } diff --git a/bacula/src/dird/jobq.c b/bacula/src/dird/jobq.c index 7440fd74d3..078cd9419a 100755 --- a/bacula/src/dird/jobq.c +++ b/bacula/src/dird/jobq.c @@ -592,12 +592,18 @@ void *jobq_server(void *arg) } if (!acquire_resources(jcr)) { - je = jn; /* point to next waiting job */ - continue; + /* If resource conflict, job is canceled */ + if (!job_canceled(jcr)) { + je = jn; /* point to next waiting job */ + continue; + } } - /* Got all locks, now remove it from wait queue and append it - * to the ready queue + /* + * Got all locks, now remove it from wait queue and append it + * to the ready queue. Note, we may also get here if the + * job was canceled. Once it is "run", it will quickly + * terminate. */ jq->waiting_jobs->remove(je); jq->ready_jobs->append(je); @@ -683,6 +689,12 @@ static bool acquire_resources(JCR *jcr) } if (jcr->wstore) { + if (jcr->rstore == jcr->wstore) { /* deadlock */ + jcr->rstore->NumConcurrentJobs = 0; /* back out rstore */ + Jmsg(jcr, M_FATAL, 0, _("Job canceled. Attempt to read and write same device.\n")); + set_jcr_job_status(jcr, JS_Canceled); + return false; + } if (jcr->wstore->NumConcurrentJobs == 0 && jcr->wstore->NumConcurrentJobs < jcr->wstore->MaxConcurrentJobs) { /* Simple case, first job */ @@ -736,9 +748,6 @@ static bool acquire_resources(JCR *jcr) set_jcr_job_status(jcr, JS_WaitJobRes); return false; } - /* Check actual device availability */ - /* ***FIXME****/ - jcr->acquired_resource_locks = true; return true; diff --git a/bacula/src/dird/migrate.c b/bacula/src/dird/migrate.c index 0fe24e0a67..cbc4bbb5e5 100644 --- a/bacula/src/dird/migrate.c +++ b/bacula/src/dird/migrate.c @@ -354,6 +354,28 @@ static int unique_name_handler(void *ctx, int num_fields, char **row) return 0; } +static int unique_dbid_handler(void *ctx, int num_fields, char **row) +{ + dlist *list = (dlist *)ctx; + + uitem *new_item = (uitem *)malloc(sizeof(uitem)); + uitem *item; + + memset(new_item, 0, sizeof(uitem)); + new_item->item = bstrdup(row[0]); + Dmsg1(dbglevel, "Item=%s\n", row[0]); + item = (uitem *)list->binary_insert((void *)new_item, item_compare); + if (item != new_item) { /* already in list */ + free(new_item->item); + free((char *)new_item); + return 0; + } + return 0; +} + + + + /* Get Job names in Pool */ const char *sql_job = "SELECT DISTINCT Job.Name from Job,Pool" diff --git a/bacula/src/lib/serial.c b/bacula/src/lib/serial.c index a042324108..93855b6de7 100644 --- a/bacula/src/lib/serial.c +++ b/bacula/src/lib/serial.c @@ -1,28 +1,23 @@ /* - Serialisation Support Functions - John Walker + Serialisation Support Functions + John Walker Version $Id$ */ /* - Copyright (C) 2000-2004 Kern Sibbald and John Walker + Copyright (C) 2000-2006 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as amended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ @@ -32,11 +27,11 @@ /* - NOTE: The following functions should work on any - vaguely contemporary platform. Production - builds should use optimised macros (void - on platforms with network byte order and IEEE - floating point format as native. + NOTE: The following functions should work on any + vaguely contemporary platform. Production + builds should use optimised macros (void + on platforms with network byte order and IEEE + floating point format as native. */ @@ -85,16 +80,16 @@ void serial_uint32(uint8_t * * const ptr, const uint32_t v) void serial_int64(uint8_t * * const ptr, const int64_t v) { if (htonl(1) == 1L) { - memcpy(*ptr, &v, sizeof(int64_t)); + memcpy(*ptr, &v, sizeof(int64_t)); } else { - int i; - uint8_t rv[sizeof(int64_t)]; - uint8_t *pv = (uint8_t *) &v; - - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(*ptr, &rv, sizeof(int64_t)); + int i; + uint8_t rv[sizeof(int64_t)]; + uint8_t *pv = (uint8_t *) &v; + + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(*ptr, &rv, sizeof(int64_t)); } *ptr += sizeof(int64_t); } @@ -105,16 +100,16 @@ void serial_int64(uint8_t * * const ptr, const int64_t v) void serial_uint64(uint8_t * * const ptr, const uint64_t v) { if (htonl(1) == 1L) { - memcpy(*ptr, &v, sizeof(uint64_t)); + memcpy(*ptr, &v, sizeof(uint64_t)); } else { - int i; - uint8_t rv[sizeof(uint64_t)]; - uint8_t *pv = (uint8_t *) &v; - - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(*ptr, &rv, sizeof(uint64_t)); + int i; + uint8_t rv[sizeof(uint64_t)]; + uint8_t *pv = (uint8_t *) &v; + + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(*ptr, &rv, sizeof(uint64_t)); } *ptr += sizeof(uint64_t); } @@ -125,42 +120,42 @@ void serial_uint64(uint8_t * * const ptr, const uint64_t v) void serial_btime(uint8_t * * const ptr, const btime_t v) { if (htonl(1) == 1L) { - memcpy(*ptr, &v, sizeof(btime_t)); + memcpy(*ptr, &v, sizeof(btime_t)); } else { - int i; - uint8_t rv[sizeof(btime_t)]; - uint8_t *pv = (uint8_t *) &v; - - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(*ptr, &rv, sizeof(btime_t)); + int i; + uint8_t rv[sizeof(btime_t)]; + uint8_t *pv = (uint8_t *) &v; + + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(*ptr, &rv, sizeof(btime_t)); } *ptr += sizeof(btime_t); } -/* serial_float64 -- Serialise a 64 bit IEEE floating point number. - This code assumes that the host floating point - format is IEEE and that floating point quantities - are stored in IEEE format either LSB first or MSB - first. More creative host formats will require - additional transformations here. */ +/* serial_float64 -- Serialise a 64 bit IEEE floating point number. + This code assumes that the host floating point + format is IEEE and that floating point quantities + are stored in IEEE format either LSB first or MSB + first. More creative host formats will require + additional transformations here. */ void serial_float64(uint8_t * * const ptr, const float64_t v) { if (htonl(1) == 1L) { - memcpy(*ptr, &v, sizeof(float64_t)); + memcpy(*ptr, &v, sizeof(float64_t)); } else { - int i; - uint8_t rv[sizeof(float64_t)]; - uint8_t *pv = (uint8_t *) &v; - - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(*ptr, &rv, sizeof(float64_t)); + int i; + uint8_t rv[sizeof(float64_t)]; + uint8_t *pv = (uint8_t *) &v; + + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(*ptr, &rv, sizeof(float64_t)); } *ptr += sizeof(float64_t); } @@ -174,7 +169,7 @@ void serial_string(uint8_t * * const ptr, const char * const str) } -/* unserial_int16 -- Unserialise a signed 16 bit integer. */ +/* unserial_int16 -- Unserialise a signed 16 bit integer. */ int16_t unserial_int16(uint8_t * * const ptr) { @@ -196,7 +191,7 @@ uint16_t unserial_uint16(uint8_t * * const ptr) return ntohs(vo); } -/* unserial_int32 -- Unserialise a signed 32 bit integer. */ +/* unserial_int32 -- Unserialise a signed 32 bit integer. */ int32_t unserial_int32(uint8_t * * const ptr) { @@ -225,40 +220,40 @@ uint64_t unserial_uint64(uint8_t * * const ptr) uint64_t v; if (htonl(1) == 1L) { - memcpy(&v, *ptr, sizeof(uint64_t)); + memcpy(&v, *ptr, sizeof(uint64_t)); } else { - int i; - uint8_t rv[sizeof(uint64_t)]; - uint8_t *pv = (uint8_t *) &v; - - memcpy(&v, *ptr, sizeof(uint64_t)); - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(&v, &rv, sizeof(uint64_t)); + int i; + uint8_t rv[sizeof(uint64_t)]; + uint8_t *pv = (uint8_t *) &v; + + memcpy(&v, *ptr, sizeof(uint64_t)); + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(&v, &rv, sizeof(uint64_t)); } *ptr += sizeof(uint64_t); return v; } -/* unserial_btime -- Unserialise a btime_t 64 bit integer. */ +/* unserial_btime -- Unserialise a btime_t 64 bit integer. */ btime_t unserial_btime(uint8_t * * const ptr) { btime_t v; if (htonl(1) == 1L) { - memcpy(&v, *ptr, sizeof(btime_t)); + memcpy(&v, *ptr, sizeof(btime_t)); } else { - int i; - uint8_t rv[sizeof(btime_t)]; - uint8_t *pv = (uint8_t *) &v; - - memcpy(&v, *ptr, sizeof(btime_t)); - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(&v, &rv, sizeof(btime_t)); + int i; + uint8_t rv[sizeof(btime_t)]; + uint8_t *pv = (uint8_t *) &v; + + memcpy(&v, *ptr, sizeof(btime_t)); + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(&v, &rv, sizeof(btime_t)); } *ptr += sizeof(btime_t); return v; @@ -267,28 +262,28 @@ btime_t unserial_btime(uint8_t * * const ptr) /* unserial_float64 -- Unserialise a 64 bit IEEE floating point number. - This code assumes that the host floating point - format is IEEE and that floating point quantities - are stored in IEEE format either LSB first or MSB - first. More creative host formats will require - additional transformations here. */ + This code assumes that the host floating point + format is IEEE and that floating point quantities + are stored in IEEE format either LSB first or MSB + first. More creative host formats will require + additional transformations here. */ float64_t unserial_float64(uint8_t * * const ptr) { float64_t v; if (htonl(1) == 1L) { - memcpy(&v, *ptr, sizeof(float64_t)); + memcpy(&v, *ptr, sizeof(float64_t)); } else { - int i; - uint8_t rv[sizeof(float64_t)]; - uint8_t *pv = (uint8_t *) &v; - - memcpy(&v, *ptr, sizeof(float64_t)); - for (i = 0; i < 8; i++) { - rv[i] = pv[7 - i]; - } - memcpy(&v, &rv, sizeof(float64_t)); + int i; + uint8_t rv[sizeof(float64_t)]; + uint8_t *pv = (uint8_t *) &v; + + memcpy(&v, *ptr, sizeof(float64_t)); + for (i = 0; i < 8; i++) { + rv[i] = pv[7 - i]; + } + memcpy(&v, &rv, sizeof(float64_t)); } *ptr += sizeof(float64_t); return v; diff --git a/bacula/src/lib/tree.c b/bacula/src/lib/tree.c index b223ef8a94..899b237a98 100755 --- a/bacula/src/lib/tree.c +++ b/bacula/src/lib/tree.c @@ -342,11 +342,13 @@ TREE_NODE *tree_cwd(char *path, TREE_ROOT *root, TREE_NODE *node) if (strcmp(path, ".") == 0) { return node; } - if (strcmp(path, "..") == 0) { - if (node->parent) { - return node->parent; + /* Handle relative path */ + if (strncmp(path, "..", 2) == 0 && (path[2] == '/' || path[2] == 0)) { + TREE_NODE *parent = node->parent ? node->parent : node; + if (path[2] == 0) { + return parent; } else { - return node; + return tree_cwd(path+3, root, parent); } } if (path[0] == '/') { diff --git a/bacula/src/stored/.cvsignore b/bacula/src/stored/.cvsignore index 1eb6193da2..38454fcc11 100644 --- a/bacula/src/stored/.cvsignore +++ b/bacula/src/stored/.cvsignore @@ -21,3 +21,8 @@ changer.out mtx-changer stored.conf test.conf +dvd-simulator +mount-simulator +mounted +slots +unmount-simulator diff --git a/bacula/src/stored/acquire.c b/bacula/src/stored/acquire.c index 728e7cc378..fd9e53fb38 100644 --- a/bacula/src/stored/acquire.c +++ b/bacula/src/stored/acquire.c @@ -417,6 +417,9 @@ get_out: * This job is done, so release the device. From a Unix standpoint, * the device remains open. * + * Note, if we are spooling, we may enter with the device locked. + * However, in all cases, unlock the device when leaving. + * */ bool release_device(DCR *dcr) { @@ -424,7 +427,10 @@ bool release_device(DCR *dcr) DEVICE *dev = dcr->dev; bool ok = true; - lock_device(dev); + /* lock only if not already locked by this thread */ + if (!dcr->dev_locked) { + lock_device(dev); + } Dmsg2(100, "release_device device %s is %s\n", dev->print_name(), dev->is_tape()?"tape":"disk"); /* if device is reserved, job never started, so release the reserve here */ @@ -508,6 +514,7 @@ bool release_device(DCR *dcr) Dmsg1(400, "alert status=%d\n", status); free_pool_memory(alert); } + dcr->dev_locked = false; /* set no longer locked */ unlock_device(dev); if (jcr->read_dcr == dcr) { jcr->read_dcr = NULL; diff --git a/bacula/src/stored/append.c b/bacula/src/stored/append.c index bbecc716e5..89953b466b 100644 --- a/bacula/src/stored/append.c +++ b/bacula/src/stored/append.c @@ -294,6 +294,7 @@ bool do_append_data(JCR *jcr) if (!ok) { discard_data_spool(dcr); } else { + /* Note: if commit is OK, the device will remain locked */ commit_data_spool(dcr); } @@ -301,7 +302,10 @@ bool do_append_data(JCR *jcr) ok = dvd_close_job(dcr); /* do DVD cleanup if any */ } - /* Release the device -- and send final Vol info to DIR */ + /* + * Release the device -- and send final Vol info to DIR + * and unlock it. + */ release_device(dcr); if (!ok || job_canceled(jcr)) { diff --git a/bacula/src/stored/block.c b/bacula/src/stored/block.c index 55ca5d8454..5e380b7a30 100644 --- a/bacula/src/stored/block.c +++ b/bacula/src/stored/block.c @@ -332,8 +332,8 @@ bool write_block_to_device(DCR *dcr) return stat; } - if (!dcr->dev_locked) { - lock_device(dev); + if (!dcr->dev_locked) { /* device already locked? */ + lock_device(dev); /* no, lock it */ } /* @@ -374,8 +374,8 @@ bool write_block_to_device(DCR *dcr) } bail_out: - if (!dcr->dev_locked) { - unlock_device(dev); + if (!dcr->dev_locked) { /* did we lock dev above? */ + unlock_device(dev); /* unlock it now */ } return stat; } diff --git a/bacula/src/stored/dircmd.c b/bacula/src/stored/dircmd.c index 134b784b49..9a8a2bd46d 100644 --- a/bacula/src/stored/dircmd.c +++ b/bacula/src/stored/dircmd.c @@ -763,6 +763,7 @@ static bool unmount_cmd(JCR *jcr) Dmsg2(90, "%d waiter dev_block=%d. doing unmount\n", dev->num_waiting, dev->dev_blocked); if (!unload_autochanger(dcr, -1)) { + /* ***FIXME**** what is this ???? */ dev->close(); } if (dev->is_dvd() && !dev->unmount(0)) { diff --git a/bacula/src/stored/reserve.c b/bacula/src/stored/reserve.c index a7763839bf..eca2ea3d9b 100644 --- a/bacula/src/stored/reserve.c +++ b/bacula/src/stored/reserve.c @@ -824,7 +824,10 @@ static bool reserve_device_for_append(DCR *dcr, RCTX &rctx) ASSERT(dcr); + /* Get locks in correct order */ + unlock_reservations(); P(dev->mutex); + lock_reservations(); /* If device is being read, we cannot write it */ if (dev->can_read()) { diff --git a/bacula/src/stored/spool.c b/bacula/src/stored/spool.c index d79b0f0857..a896cbae03 100644 --- a/bacula/src/stored/spool.c +++ b/bacula/src/stored/spool.c @@ -309,10 +309,13 @@ static bool despool_data(DCR *dcr, bool commit) rdcr->jcr = NULL; free_dcr(rdcr); free(rdev); - dcr->dev_locked = false; dcr->spooling = true; /* turn on spooling again */ dcr->despooling = false; - unlock_device(dcr->dev); + /* If doing a commit, leave the device locked -- unlocked in release_device() */ + if (!commit) { + dcr->dev_locked = false; + unlock_device(dcr->dev); + } return ok; } diff --git a/bacula/src/win32/scripts/.cvsignore b/bacula/src/win32/scripts/.cvsignore new file mode 100644 index 0000000000..a4383358ec --- /dev/null +++ b/bacula/src/win32/scripts/.cvsignore @@ -0,0 +1 @@ +*.d diff --git a/bacula/technotes-1.39 b/bacula/technotes-1.39 index 177d732d08..8b3efe6519 100644 --- a/bacula/technotes-1.39 +++ b/bacula/technotes-1.39 @@ -2,6 +2,21 @@ General: +12Nov06 +kes Change error message 'illegal' to 'invalid' -- bug #707 +kes Add rather primitive device resource deadlock detection in + the job queue handler. It detects the same read and write device + for migration and cancels the job. +kes Start adding a unique_dbid_hander() routine in migration to avoid + duplicate mediaids and duplicate jobids. +kes Add patch from bug #708 to permit relative paths such as ../xxx + in the restore tree routine. +kes Add Eric Bollengier's patch to reduce locking time in the SD after + despooling, and thus many jobs finish faster. +kes Correct locking order of reservations lock and device mutex in + reserve.c -- this should correct Arno's deadlock that occurred + when doing mount/unmount at the same time a job is reserving a + drive. 05Nov06 rbn Fixed problems with encryption when combined with compression or sparse files. Unfortunately this means that all previous @@ -31,10 +46,10 @@ rbn Reorganize Start menu so that documentation and configuration are 21Oct06 rbn Change daemons and utilities so that if -c is omitted from the command line AND there is no configuration file in the current - directory then the "standard" configuration file (eg + directory then the 'standard' configuration file (eg /etc/bacula/bacula-*.conf) will be used. If the argument to -c doesn't contain any path separators AND there is no configuration - file by that name in the current directory then the "standard" + file by that name in the current directory then the 'standard' configuration file directory will be searched for a file by that name. rbn Fixed restore on Windows so that backslashes as path separators works. @@ -95,7 +110,7 @@ kes Turn on heap reporting in Dir with zero debug level. kes Send a message to the sys log when Bacula forces a SEG FAULT, and send the same message to stdout. 07Oct06 -kes Begin work on new GUI "console". +kes Begin work on new GUI 'console'. kes Make configure look in non-standard MySQL library directory for Solaris. kes Make mtx-changer automatically configure the autochanger wait -- 2.39.5