From a2b923b23ff097ce48121dcb24b365e7c741e3cb Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sat, 4 Jun 2005 13:27:10 +0000 Subject: [PATCH] - Implement attribute caching to put Signature into database at the same time as the file attributes thus eliminating a number of database accesses. - Correct a reservation problem. - Implement full Dir Storage use. - Reduce a bit of TLS #ifdeffing. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@2104 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/kes-1.37 | 8 +++ bacula/src/cats/cats.h | 40 ++++++------- bacula/src/cats/sql.c | 11 +++- bacula/src/cats/sql_create.c | 3 +- bacula/src/dird/backup.c | 23 +++---- bacula/src/dird/catreq.c | 61 +++++++++++++------ bacula/src/dird/dird_conf.c | 2 +- bacula/src/dird/dird_conf.h | 20 ++----- bacula/src/dird/job.c | 8 +++ bacula/src/dird/msgchan.c | 113 ++++++++++++++++------------------- bacula/src/jcr.h | 8 ++- bacula/src/stored/acquire.c | 11 ++-- bacula/src/stored/append.c | 2 +- bacula/src/stored/dev.h | 18 ++---- bacula/src/stored/job.c | 8 +-- bacula/src/version.h | 2 +- 16 files changed, 176 insertions(+), 162 deletions(-) diff --git a/bacula/kes-1.37 b/bacula/kes-1.37 index 1caa9c3ddf..f19a2db570 100644 --- a/bacula/kes-1.37 +++ b/bacula/kes-1.37 @@ -3,6 +3,14 @@ General: +Changes to 1.37.21: +- Implement attribute caching to put Signature into database + at the same time as the file attributes thus eliminating a + number of database accesses. +- Correct a reservation problem. +- Implement full Dir Storage use. +- Reduce a bit of TLS #ifdeffing. + Changes to 1.37.20: 04Jun05 - Minor changes diff --git a/bacula/src/cats/cats.h b/bacula/src/cats/cats.h index 02b30d5f7c..65b0a51281 100644 --- a/bacula/src/cats/cats.h +++ b/bacula/src/cats/cats.h @@ -13,24 +13,18 @@ * * Version $Id$ */ - /* Copyright (C) 2000-2005 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as ammended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ @@ -99,7 +93,7 @@ typedef struct s_sql_field { * subroutines. * S Q L I T E */ -typedef struct s_db { +struct B_DB { BQUEUE bq; /* queue control */ brwlock_t lock; /* transaction lock */ struct sqlite *db; @@ -135,7 +129,7 @@ typedef struct s_db { POOLMEM *esc_name; /* Escaped file/path name */ int fnl; /* file name length */ int pnl; /* path name length */ -} B_DB; +}; /* @@ -207,7 +201,7 @@ typedef struct s_sql_field { * subroutines. * S Q L I T E */ -typedef struct s_db { +struct B_DB { BQUEUE bq; /* queue control */ brwlock_t lock; /* transaction lock */ struct sqlite3 *db; @@ -243,7 +237,7 @@ typedef struct s_db { POOLMEM *esc_name; /* Escaped file/path name */ int fnl; /* file name length */ int pnl; /* path name length */ -} B_DB; +}; /* * Conversion of sqlite 2 names to sqlite3 @@ -307,7 +301,7 @@ SQL_FIELD *my_sqlite_fetch_field(B_DB *mdb); * * M Y S Q L */ -typedef struct s_db { +struct B_DB { BQUEUE bq; /* queue control */ brwlock_t lock; /* transaction lock */ MYSQL mysql; @@ -335,7 +329,7 @@ typedef struct s_db { POOLMEM *esc_name; /* Escaped file/path name */ int fnl; /* file name length */ int pnl; /* path name length */ -} B_DB; +}; #define DB_STATUS int @@ -384,7 +378,7 @@ typedef struct pg_field { * * P O S T G R E S Q L */ -typedef struct s_db { +struct B_DB { BQUEUE bq; /* queue control */ brwlock_t lock; /* transaction lock */ PGconn *db; @@ -418,7 +412,7 @@ typedef struct s_db { POOLMEM *esc_name; /* Escaped file/path name */ int fnl; /* file name length */ int pnl; /* path name length */ -} B_DB; +}; void my_postgresql_free_result(B_DB *mdb); POSTGRESQL_ROW my_postgresql_fetch_row (B_DB *mdb); @@ -470,7 +464,7 @@ struct s_control { /* This is the REAL definition for using the * Bacula internal DB */ -typedef struct s_db { +struct B_DB { BQUEUE bq; /* queue control */ /* pthread_mutex_t mutex; */ /* single thread lock */ brwlock_t lock; /* transaction lock */ @@ -489,7 +483,7 @@ typedef struct s_db { POOLMEM *cached_path; int cached_path_len; /* length of cached path */ uint32_t cached_path_id; -} B_DB; +}; #endif /* HAVE_SQLITE3 */ #endif /* HAVE_MYSQL */ @@ -507,9 +501,9 @@ typedef struct s_db { /* This is a "dummy" definition for use outside of sql.c */ -typedef struct s_db { +struct B_DB { int dummy; /* for SunOS compiler */ -} B_DB; +}; #endif /* __SQL_C */ diff --git a/bacula/src/cats/sql.c b/bacula/src/cats/sql.c index 0c1e728d49..c962803767 100644 --- a/bacula/src/cats/sql.c +++ b/bacula/src/cats/sql.c @@ -267,6 +267,10 @@ void db_start_transaction(JCR *jcr, B_DB *mdb) if (!jcr->attr) { jcr->attr = get_pool_memory(PM_FNAME); } + if (!jcr->ar) { + jcr->ar = (ATTR_DBR *)malloc(sizeof(ATTR_DBR)); + } + #ifdef HAVE_SQLITE if (!mdb->allow_transactions) { return; @@ -315,14 +319,15 @@ void db_end_transaction(JCR *jcr, B_DB *mdb) if (!mdb) { return; } -#ifdef xxx + if (jcr && jcr->cached_attribute) { - if (!db_create_file_attributes_record(jcr, jcr->db, &jcr->ar)) { + Dmsg0(400, "Flush last cached attribute.\n"); + if (!db_create_file_attributes_record(jcr, mdb, jcr->ar)) { Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db)); } jcr->cached_attribute = false; } -#endif + #ifdef HAVE_SQLITE if (!mdb->allow_transactions) { return; diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index bef5a43b43..0d159b5fa9 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -672,7 +672,8 @@ int db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) */ if (!(ar->Stream == STREAM_UNIX_ATTRIBUTES || ar->Stream == STREAM_UNIX_ATTRIBUTES_EX)) { - Mmsg0(&mdb->errmsg, _("Attempt to put non-attributes into catalog\n")); + Mmsg1(&mdb->errmsg, _("Attempt to put non-attributes into catalog. Stream=%d\n"), + ar->Stream); Jmsg(jcr, M_ERROR, 0, "%s", mdb->errmsg); goto bail_out; } diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 6bf16f6edb..f4eba70dd5 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -13,24 +13,18 @@ * * Version $Id$ */ - /* Copyright (C) 2000-2005 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as ammended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ @@ -176,7 +170,10 @@ bool do_backup(JCR *jcr) return false; } /* - * Now start a Storage daemon message thread + * Now start a Storage daemon message thread. Note, + * this thread is used to provide the catalog services + * for the backup job, including inserting the attributes + * into the catalog. See catalog_update() in catreq.c */ if (!start_storage_daemon_message_thread(jcr)) { return false; @@ -211,7 +208,6 @@ bool do_backup(JCR *jcr) store->SDDport = store->SDport; } -#ifdef HAVE_TLS /* TLS Requirement */ if (store->tls_enable) { if (store->tls_require) { @@ -220,7 +216,6 @@ bool do_backup(JCR *jcr) tls_need = BNET_TLS_OK; } } -#endif bnet_fsend(fd, storaddr, store->address, store->SDDport, tls_need); diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index 45f0a18fda..d9c65fab4e 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -13,22 +13,17 @@ * Version $Id$ */ /* - Copyright (C) 2000-2005 Kern Sibbald + Copyright (C) 2001-2005 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as ammended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ @@ -326,15 +321,20 @@ void catalog_update(JCR *jcr, BSOCK *bs, char *msg) char *p; int len; char *fname, *attr; - ATTR_DBR *ar = &jcr->ar; + ATTR_DBR *ar = NULL; if (!jcr->pool->catalog_files) { - return; + return; /* user disabled cataloging */ } + /* Start transaction allocates jcr->attr and jcr->ar if needed */ db_start_transaction(jcr, jcr->db); /* start transaction if not already open */ - jcr->attr = check_pool_memory_size(jcr->attr, bs->msglen); - memcpy(jcr->attr, bs->msg, bs->msglen); - p = jcr->attr; + ar = jcr->ar; + + /* Start by scanning directly in the message buffer to get Stream + * there may be a cached attr so we cannot yet write into + * jcr->attr or jcr->ar + */ + p = bs->msg; skip_nonspaces(&p); /* UpdCat */ skip_spaces(&p); skip_nonspaces(&p); /* Job=nnn */ @@ -354,6 +354,16 @@ void catalog_update(JCR *jcr, BSOCK *bs, char *msg) VolSessionId, VolSessionTime, FileIndex, Stream, data_len); if (Stream == STREAM_UNIX_ATTRIBUTES || Stream == STREAM_UNIX_ATTRIBUTES_EX) { + if (jcr->cached_attribute) { + Dmsg2(400, "Cached attr. Stream=%d fname=%s\n", ar->Stream, ar->fname); + if (!db_create_file_attributes_record(jcr, jcr->db, ar)) { + Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db)); + } + } + /* Any cached attr is flushed so we can reuse jcr->attr and jcr->ar */ + jcr->attr = check_pool_memory_size(jcr->attr, bs->msglen); + memcpy(jcr->attr, bs->msg, bs->msglen); + p = jcr->attr - bs->msg + p; /* point p into jcr->attr */ skip_nonspaces(&p); /* skip FileIndex */ skip_spaces(&p); skip_nonspaces(&p); /* skip FileType */ @@ -372,13 +382,16 @@ void catalog_update(JCR *jcr, BSOCK *bs, char *msg) ar->JobId = jcr->JobId; ar->Sig = NULL; ar->SigType = 0; + jcr->cached_attribute = true; Dmsg2(400, "dirddb, ar)) { Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db)); } +#endif } else if (Stream == STREAM_MD5_SIGNATURE || Stream == STREAM_SHA1_SIGNATURE) { fname = p; if (ar->FileIndex != FileIndex) { @@ -396,9 +409,19 @@ void catalog_update(JCR *jcr, BSOCK *bs, char *msg) } bin_to_base64(SIGbuf, fname, len); Dmsg3(400, "SIGlen=%d SIG=%s type=%d\n", strlen(SIGbuf), SIGbuf, Stream); - if (!db_add_SIG_to_file_record(jcr, jcr->db, ar->FileId, SIGbuf, type)) { - Jmsg(jcr, M_ERROR, 0, _("Catalog error updating MD5/SHA1. %s"), - db_strerror(jcr->db)); + if (jcr->cached_attribute) { + ar->Sig = SIGbuf; + ar->SigType = type; + Dmsg2(400, "Cached attr with SIG. Stream=%d fname=%s\n", ar->Stream, ar->fname); + if (!db_create_file_attributes_record(jcr, jcr->db, ar)) { + Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db)); + } + jcr->cached_attribute = false; + } else { + if (!db_add_SIG_to_file_record(jcr, jcr->db, ar->FileId, SIGbuf, type)) { + Jmsg(jcr, M_ERROR, 0, _("Catalog error updating MD5/SHA1. %s"), + db_strerror(jcr->db)); + } } } } diff --git a/bacula/src/dird/dird_conf.c b/bacula/src/dird/dird_conf.c index a75eeaa420..2c425cbc78 100644 --- a/bacula/src/dird/dird_conf.c +++ b/bacula/src/dird/dird_conf.c @@ -269,7 +269,7 @@ RES_ITEM job_items[] = { {"spoolattributes",store_yesno, ITEM(res_job.SpoolAttributes), 1, ITEM_DEFAULT, 0}, {"spooldata", store_yesno, ITEM(res_job.spool_data), 1, ITEM_DEFAULT, 0}, {"rerunfailedlevels", store_yesno, ITEM(res_job.rerun_failed_levels), 1, ITEM_DEFAULT, 0}, - {"newvolumeeachjob", store_yesno, ITEM(res_job.NewVolEachJob), 1, ITEM_DEFAULT, 0}, + {"prefermountedvolumes", store_yesno, ITEM(res_job.PreferMountedVolumes), 1, ITEM_DEFAULT, 1}, {"runbeforejob", store_str, ITEM(res_job.RunBeforeJob), 0, 0, 0}, {"runafterjob", store_str, ITEM(res_job.RunAfterJob), 0, 0, 0}, {"runafterfailedjob", store_str, ITEM(res_job.RunAfterFailedJob), 0, 0, 0}, diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index 57b4d92810..b029df0e44 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -107,9 +107,8 @@ public: uint32_t MaxConcurrentJobs; /* Max concurrent jobs for whole director */ utime_t FDConnectTimeout; /* timeout for connect in seconds */ utime_t SDConnectTimeout; /* timeout in seconds */ -#ifdef HAVE_TLS int tls_enable; /* Enable TLS */ - int tls_require; /* Require TLS */ + int tls_require; /* Require TLS */ int tls_verify_peer; /* TLS Verify Client Certificate */ char *tls_ca_certfile; /* TLS CA Certificate File */ char *tls_ca_certdir; /* TLS CA Certificate Directory */ @@ -119,7 +118,6 @@ public: alist *tls_allowed_cns; /* TLS Allowed Clients */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ -#endif /* HAVE_TLS */ }; /* @@ -176,9 +174,8 @@ public: RES hdr; char *password; /* UA server password */ alist *ACL_lists[Num_ACL]; /* pointers to ACLs */ -#ifdef HAVE_TLS - int tls_enable; /* Enable TLS */ - int tls_require; /* Require TLS */ + int tls_enable; /* Enable TLS */ + int tls_require; /* Require TLS */ int tls_verify_peer; /* TLS Verify Client Certificate */ char *tls_ca_certfile; /* TLS CA Certificate File */ char *tls_ca_certdir; /* TLS CA Certificate Directory */ @@ -188,7 +185,6 @@ public: alist *tls_allowed_cns; /* TLS Allowed Clients */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ -#endif /* HAVE_TLS */ }; @@ -227,16 +223,14 @@ public: CAT *catalog; /* Catalog resource */ uint32_t MaxConcurrentJobs; /* Maximume concurrent jobs */ uint32_t NumConcurrentJobs; /* number of concurrent jobs running */ -#ifdef HAVE_TLS int tls_enable; /* Enable TLS */ - int tls_require; /* Require TLS */ + int tls_require; /* Require TLS */ char *tls_ca_certfile; /* TLS CA Certificate File */ char *tls_ca_certdir; /* TLS CA Certificate Directory */ char *tls_certfile; /* TLS Client Certificate File */ char *tls_keyfile; /* TLS Client Key File */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ -#endif /* HAVE_TLS */ }; /* @@ -256,16 +250,14 @@ public: int autochanger; /* set if autochanger */ uint32_t MaxConcurrentJobs; /* Maximume concurrent jobs */ uint32_t NumConcurrentJobs; /* number of concurrent jobs running */ -#ifdef HAVE_TLS int tls_enable; /* Enable TLS */ - int tls_require; /* Require TLS */ + int tls_require; /* Require TLS */ char *tls_ca_certfile; /* TLS CA Certificate File */ char *tls_ca_certdir; /* TLS CA Certificate Directory */ char *tls_certfile; /* TLS Client Certificate File */ char *tls_keyfile; /* TLS Client Key File */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ -#endif /* HAVE_TLS */ int64_t StorageId; /* Set from Storage DB record */ @@ -316,7 +308,7 @@ public: int SpoolAttributes; /* Set to spool attributes in SD */ int spool_data; /* Set to spool data in SD */ int rerun_failed_levels; /* Upgrade to rerun failed levels */ - int NewVolEachJob; /* Mount new volume each Job */ + int PreferMountedVolumes; /* Prefer vols mounted rather than new one */ uint32_t MaxConcurrentJobs; /* Maximume concurrent jobs */ int RescheduleOnError; /* Set to reschedule on error */ int RescheduleTimes; /* Number of times to reschedule job */ diff --git a/bacula/src/dird/job.c b/bacula/src/dird/job.c index 9c6ba26e34..7fa9fb58e0 100644 --- a/bacula/src/dird/job.c +++ b/bacula/src/dird/job.c @@ -791,6 +791,14 @@ void dird_free_jcr_pointers(JCR *jcr) pthread_cond_destroy(&jcr->term_wait); jcr->term_wait_inited = false; } + if (jcr->attr) { + free_pool_memory(jcr->attr); + jcr->attr = NULL; + } + if (jcr->ar) { + free(jcr->ar); + jcr->ar = NULL; + } } /* diff --git a/bacula/src/dird/msgchan.c b/bacula/src/dird/msgchan.c index 117543028e..dd55fab3ae 100644 --- a/bacula/src/dird/msgchan.c +++ b/bacula/src/dird/msgchan.c @@ -9,9 +9,9 @@ * * Basic tasks done here: * Open a message channel with the Storage daemon - * to authenticate ourself and to pass the JobId. + * to authenticate ourself and to pass the JobId. * Create a thread to interact with the Storage daemon - * who returns a job status and requests Catalog services, etc. + * who returns a job status and requests Catalog services, etc. * * Version $Id$ */ @@ -19,19 +19,14 @@ Copyright (C) 2000-2005 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as ammended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ @@ -41,7 +36,7 @@ /* Commands sent to Storage daemon */ static char jobcmd[] = "JobId=%d job=%s job_name=%s client_name=%s " "type=%d level=%d FileSet=%s NoAttr=%d SpoolAttr=%d FileSetMD5=%s " - "SpoolData=%d WritePartAfterJob=%d NewVol=%d\n"; + "SpoolData=%d WritePartAfterJob=%d PreferMountedVols=%d\n"; static char use_storage[] = "use storage=%s media_type=%s pool_name=%s " "pool_type=%s append=%d copy=%d stripe=%d\n"; static char use_device[] = "use device=%s\n"; @@ -53,7 +48,7 @@ static char OK_device[] = "3000 OK use device device=%s\n"; /* Storage Daemon requests */ static char Job_start[] = "3010 Job %127s start\n"; -static char Job_end[] = +static char Job_end[] = "3099 Job %127s end JobStatus=%d JobFiles=%d JobBytes=%" lld "\n"; /* Forward referenced functions */ @@ -64,13 +59,13 @@ extern "C" void *msg_thread(void *arg); * and perform authentication. */ bool connect_to_storage_daemon(JCR *jcr, int retry_interval, - int max_retry_time, int verbose) + int max_retry_time, int verbose) { BSOCK *sd; STORE *store; if (jcr->store_bsock) { - return true; /* already connected */ + return true; /* already connected */ } store = (STORE *)jcr->storage->first(); @@ -81,11 +76,11 @@ bool connect_to_storage_daemon(JCR *jcr, int retry_interval, store->SDport); sd = bnet_connect(jcr, retry_interval, max_retry_time, _("Storage daemon"), store->address, - NULL, store->SDport, verbose); + NULL, store->SDport, verbose); if (sd == NULL) { return false; } - sd->res = (RES *)store; /* save pointer to other end */ + sd->res = (RES *)store; /* save pointer to other end */ jcr->store_bsock = sd; if (!authenticate_storage_daemon(jcr, store)) { @@ -146,10 +141,10 @@ int start_storage_daemon_job(JCR *jcr, alist *store, int append) bstrncpy(jcr->fileset->MD5, "**Dummy**", sizeof(jcr->fileset->MD5)); } bnet_fsend(sd, jobcmd, jcr->JobId, jcr->Job, jcr->job->hdr.name, - jcr->client->hdr.name, jcr->JobType, jcr->JobLevel, - jcr->fileset->hdr.name, !jcr->pool->catalog_files, - jcr->job->SpoolAttributes, jcr->fileset->MD5, jcr->spool_data, - jcr->write_part_after_job, jcr->job->NewVolEachJob); + jcr->client->hdr.name, jcr->JobType, jcr->JobLevel, + jcr->fileset->hdr.name, !jcr->pool->catalog_files, + jcr->job->SpoolAttributes, jcr->fileset->MD5, jcr->spool_data, + jcr->write_part_after_job, jcr->job->PreferMountedVolumes); Dmsg1(100, ">stored: %s\n", sd->msg); unbash_spaces(jcr->job->hdr.name); unbash_spaces(jcr->client->hdr.name); @@ -157,17 +152,17 @@ int start_storage_daemon_job(JCR *jcr, alist *store, int append) if (bget_dirmsg(sd) > 0) { Dmsg1(100, "msg); if (sscanf(sd->msg, OKjob, &jcr->VolSessionId, - &jcr->VolSessionTime, &auth_key) != 3) { + &jcr->VolSessionTime, &auth_key) != 3) { Dmsg1(100, "BadJob=%s\n", sd->msg); Jmsg(jcr, M_FATAL, 0, _("Storage daemon rejected Job command: %s\n"), sd->msg); - return 0; + return 0; } else { - jcr->sd_auth_key = bstrdup(auth_key); + jcr->sd_auth_key = bstrdup(auth_key); Dmsg1(150, "sd_auth_key=%s\n", jcr->sd_auth_key); } } else { Jmsg(jcr, M_FATAL, 0, _("first(); + foreach_alist(storage, store) { +// storage = (STORE *)store->first(); pm_strcpy(store_name, storage->hdr.name); bash_spaces(store_name); pm_strcpy(media_type, storage->media_type); bash_spaces(media_type); bnet_fsend(sd, use_storage, store_name.c_str(), media_type.c_str(), - pool_name.c_str(), pool_type.c_str(), append, copy, stripe); + pool_name.c_str(), pool_type.c_str(), append, copy, stripe); DEVICE *dev; /* Loop over alternative storage Devices until one is OK */ foreach_alist(dev, storage->device) { - pm_strcpy(device_name, dev->hdr.name); - bash_spaces(device_name); - bnet_fsend(sd, use_device, device_name.c_str()); + pm_strcpy(device_name, dev->hdr.name); + bash_spaces(device_name); + bnet_fsend(sd, use_device, device_name.c_str()); Dmsg1(100, ">stored: %s", sd->msg); } - bnet_sig(sd, BNET_EOD); /* end of Devices */ - bnet_sig(sd, BNET_EOD); /* end of Storages */ + bnet_sig(sd, BNET_EOD); /* end of Devices */ + bnet_sig(sd, BNET_EOD); /* end of Storages */ if (bget_dirmsg(sd) > 0) { Dmsg1(100, "msg); - /* ****FIXME**** save actual device name */ - ok = sscanf(sd->msg, OK_device, device_name.c_str()) == 1; + /* ****FIXME**** save actual device name */ + ok = sscanf(sd->msg, OK_device, device_name.c_str()) == 1; } else { - POOL_MEM err_msg; - pm_strcpy(err_msg, sd->msg); /* save message */ - Jmsg(jcr, M_WARNING, 0, _("\n" + POOL_MEM err_msg; + pm_strcpy(err_msg, sd->msg); /* save message */ + Jmsg(jcr, M_FATAL, 0, _("\n" " Storage daemon didn't accept Device \"%s\" because:\n %s"), - device_name.c_str(), err_msg.c_str()/* sd->msg */); + device_name.c_str(), err_msg.c_str()/* sd->msg */); } -// if (!ok) { -// break; -// } -// } + break; + } if (ok) { ok = bnet_fsend(sd, "run"); Dmsg1(100, ">stored: %s\n", sd->msg); @@ -238,7 +231,7 @@ int start_storage_daemon_message_thread(JCR *jcr) pthread_t thid; P(jcr->mutex); - jcr->use_count++; /* mark in use by msg thread */ + jcr->use_count++; /* mark in use by msg thread */ jcr->sd_msg_thread_done = false; jcr->SD_msg_chan = 0; V(jcr->mutex); @@ -259,13 +252,13 @@ extern "C" void msg_thread_cleanup(void *arg) { JCR *jcr = (JCR *)arg; Dmsg0(200, "End msg_thread\n"); - db_end_transaction(jcr, jcr->db); /* terminate any open transaction */ + db_end_transaction(jcr, jcr->db); /* terminate any open transaction */ P(jcr->mutex); jcr->sd_msg_thread_done = true; pthread_cond_broadcast(&jcr->term_wait); /* wakeup any waiting threads */ jcr->SD_msg_chan = 0; V(jcr->mutex); - free_jcr(jcr); /* release jcr */ + free_jcr(jcr); /* release jcr */ } /* @@ -294,20 +287,20 @@ extern "C" void *msg_thread(void *arg) while ((stat=bget_dirmsg(sd)) >= 0) { Dmsg1(200, "msg); if (sscanf(sd->msg, Job_start, &Job) == 1) { - continue; + continue; } if (sscanf(sd->msg, Job_end, &Job, &JobStatus, &JobFiles, - &JobBytes) == 4) { - jcr->SDJobStatus = JobStatus; /* termination status */ - jcr->SDJobFiles = JobFiles; - jcr->SDJobBytes = JobBytes; - break; + &JobBytes) == 4) { + jcr->SDJobStatus = JobStatus; /* termination status */ + jcr->SDJobFiles = JobFiles; + jcr->SDJobBytes = JobBytes; + break; } } if (is_bnet_error(sd)) { jcr->SDJobStatus = JS_ErrorTerminated; } - pthread_cleanup_pop(1); /* remove and execute the handler */ + pthread_cleanup_pop(1); /* remove and execute the handler */ return NULL; } @@ -328,11 +321,11 @@ void wait_for_storage_daemon_termination(JCR *jcr) Dmsg0(300, "I'm waiting for message thread termination.\n"); pthread_cond_timedwait(&jcr->term_wait, &jcr->mutex, &timeout); if (job_canceled(jcr)) { - cancel_count++; + cancel_count++; } /* Give SD 30 seconds to clean up after cancel */ if (cancel_count == 3) { - break; + break; } } V(jcr->mutex); @@ -354,15 +347,15 @@ extern "C" void *device_thread(void *arg) for (i=0; i < MAX_TRIES; i++) { if (!connect_to_storage_daemon(jcr, 10, 30, 1)) { Dmsg0(000, "Failed connecting to SD.\n"); - continue; + continue; } LockRes(); foreach_res(dev, R_DEVICE) { - if (!update_device_res(jcr, dev)) { + if (!update_device_res(jcr, dev)) { Dmsg1(900, "Error updating device=%s\n", dev->hdr.name); - } else { + } else { Dmsg1(900, "Updated Device=%s\n", dev->hdr.name); - } + } } UnlockRes(); bnet_close(jcr->store_bsock); diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index 45faa3ef01..2182be7606 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -88,6 +88,8 @@ /* Forward referenced structures */ class JCR; struct FF_PKT; +struct B_DB; +struct ATTR_DBR; typedef void (JCR_free_HANDLER)(JCR *jcr); @@ -146,6 +148,8 @@ public: bool cached_attribute; /* set if attribute is cached */ POOLMEM *attr; /* Attribute string from SD */ + B_DB *db; /* database pointer */ + ATTR_DBR *ar; /* DB attribute record */ /* Daemon specific part of JCR */ /* This should be empty in the library */ @@ -175,7 +179,6 @@ public: volatile int SDJobStatus; /* Storage Job Status */ volatile int FDJobStatus; /* File daemon Job Status */ uint32_t ExpectedFiles; /* Expected restore files */ - B_DB *db; /* database pointer */ uint32_t MediaId; /* DB record IDs associated with this job */ uint32_t PoolId; /* Pool record id */ FileId_t FileId; /* Last file id inserted */ @@ -198,7 +201,6 @@ public: bool needs_sd; /* set if SD needed by Job */ bool cloned; /* set if cloned */ bool unlink_bsr; /* Unlink bsr file created */ - ATTR_DBR ar; /* DB attribute record */ #endif /* DIRECTOR_DAEMON */ @@ -255,7 +257,7 @@ public: int CurVol; /* Current Volume count */ DIRRES* director; /* Director resource */ bool write_part_after_job; /* Set to write part after job */ - bool NewVolEachJob; /* Allow using new volume */ + bool PreferMountedVols; /* Prefer mounted vols rather than new */ uint32_t FileId; /* Last file id inserted */ diff --git a/bacula/src/stored/acquire.c b/bacula/src/stored/acquire.c index 31569cf193..4b9122644c 100644 --- a/bacula/src/stored/acquire.c +++ b/bacula/src/stored/acquire.c @@ -89,17 +89,14 @@ void free_dcr(DCR *dcr) JCR *jcr = dcr->jcr; DEVICE *dev = dcr->dev; - /* - * If we reserved the device, we must decrement the - * number of writers. - */ if (dcr->reserved_device) { lock_device(dev); - dev->num_writers--; + dev->reserved_device--; + Dmsg1(200, "=========== Dec reserve=%d\n", dev->reserved_device); + dcr->reserved_device = false; if (dev->num_writers < 0) { Jmsg1(dcr->jcr, M_ERROR, 0, _("Hey! num_writers=%d!!!!\n"), dev->num_writers); dev->num_writers = 0; - dcr->reserved_device = false; } unlock_device(dev); } @@ -322,6 +319,7 @@ DCR *acquire_device_for_append(DCR *dcr) if (dcr->reserved_device) { dev->reserved_device--; + Dmsg1(200, "============ Dec reserve=%d\n", dev->reserved_device); dcr->reserved_device = false; } @@ -430,6 +428,7 @@ bool release_device(DCR *dcr) /* if device is reserved, job never started, so release the reserve here */ if (dcr->reserved_device) { dev->reserved_device--; + Dmsg1(200, "========= Dec reserve=%d\n", dev->reserved_device); dcr->reserved_device = false; } diff --git a/bacula/src/stored/append.c b/bacula/src/stored/append.c index 3c5cd66e61..284e9fa718 100644 --- a/bacula/src/stored/append.c +++ b/bacula/src/stored/append.c @@ -49,7 +49,7 @@ bool do_append_data(JCR *jcr) DEVICE *dev = dcr->dev; - Dmsg0(10, "Start append data.\n"); + Dmsg0(100, "Start append data.\n"); memset(&rec, 0, sizeof(rec)); diff --git a/bacula/src/stored/dev.h b/bacula/src/stored/dev.h index 1e00eff79f..11c5bb9890 100644 --- a/bacula/src/stored/dev.h +++ b/bacula/src/stored/dev.h @@ -11,19 +11,14 @@ Copyright (C) 2000-2005 Kern Sibbald This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of - the License, or (at your option) any later version. + modify it under the terms of the GNU General Public License + version 2 as ammended with additional clauses defined in the + file LICENSE in the main source directory. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public - License along with this program; if not, write to the Free - Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, - MA 02111-1307, USA. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + the file LICENSE for additional details. */ @@ -259,7 +254,7 @@ public: int is_open() const { return state & ST_OPENED; } int is_offline() const { return state & ST_OFFLINE; } int is_labeled() const { return state & ST_LABEL; } - int is_busy() const; /* either reading or writing */ + int is_busy() const { return state & ST_READ || num_writers || reserved_device; } int at_eof() const { return state & ST_EOF; } int at_eot() const { return state & ST_EOT; } int at_weot() const { return state & ST_WEOT; } @@ -304,7 +299,6 @@ public: }; /* Note, these return int not bool! */ -inline int DEVICE::is_busy() const { return state & ST_READ || num_writers || reserved_device; } inline const char *DEVICE::strerror() const { return errmsg; } inline const char *DEVICE::archive_name() const { return dev_name; } inline const char *DEVICE::print_name() const { return prt_name; } diff --git a/bacula/src/stored/job.c b/bacula/src/stored/job.c index 65b65729ad..e4300d8dbd 100644 --- a/bacula/src/stored/job.c +++ b/bacula/src/stored/job.c @@ -33,7 +33,7 @@ extern uint32_t newVolSessionId(); /* Requests from the Director daemon */ static char jobcmd[] = "JobId=%d job=%127s job_name=%127s client_name=%127s " "type=%d level=%d FileSet=%127s NoAttr=%d SpoolAttr=%d FileSetMD5=%127s " - "SpoolData=%d WritePartAfterJob=%d NewVol=%d\n"; + "SpoolData=%d WritePartAfterJob=%d PreferMountedVols=%d\n"; /* Responses sent to Director daemon */ @@ -60,7 +60,7 @@ bool job_cmd(JCR *jcr) BSOCK *dir = jcr->dir_bsock; POOL_MEM job_name, client_name, job, fileset_name, fileset_md5; int JobType, level, spool_attributes, no_attributes, spool_data; - int write_part_after_job, NewVol; + int write_part_after_job, PreferMountedVols; JCR *ojcr; @@ -72,7 +72,7 @@ bool job_cmd(JCR *jcr) client_name.c_str(), &JobType, &level, fileset_name.c_str(), &no_attributes, &spool_attributes, fileset_md5.c_str(), &spool_data, - &write_part_after_job, &NewVol) != 13) { + &write_part_after_job, &PreferMountedVols) != 13) { pm_strcpy(jcr->errmsg, dir->msg); bnet_fsend(dir, BAD_job, jcr->errmsg); Dmsg1(100, ">dird: %s\n", dir->msg); @@ -111,7 +111,7 @@ bool job_cmd(JCR *jcr) jcr->write_part_after_job = write_part_after_job; jcr->fileset_md5 = get_pool_memory(PM_NAME); pm_strcpy(jcr->fileset_md5, fileset_md5); - jcr->NewVolEachJob = NewVol; + jcr->PreferMountedVols = PreferMountedVols; jcr->authenticated = false; diff --git a/bacula/src/version.h b/bacula/src/version.h index e15ed40847..554a47836a 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -1,6 +1,6 @@ /* */ #undef VERSION -#define VERSION "1.37.20" +#define VERSION "1.37.21" #define BDATE "04 June 2005" #define LSMDATE "04Jun05" -- 2.39.5