From f5c84d71c548c3996d0073166dbc59b08151e3e3 Mon Sep 17 00:00:00 2001 From: Eric Bollengier Date: Thu, 30 Jul 2009 16:48:42 +0000 Subject: [PATCH] Allow restore from multiple storage git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@9128 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/dird/bsr.c | 2 +- bacula/src/dird/fd_cmds.c | 4 +- bacula/src/dird/msgchan.c | 1 + bacula/src/dird/restore.c | 474 ++++++++++++++++++++++++------- bacula/src/dird/verify.c | 11 +- bacula/src/filed/authenticate.c | 3 +- bacula/src/filed/job.c | 94 ++++-- bacula/src/filed/verify.c | 24 +- bacula/src/jcr.h | 2 + bacula/src/stored/parse_bsr.c | 13 + bacula/technotes | 1 + regress/DartTestfile.txt.in | 1 + regress/all-disk-tests | 1 + regress/scripts/functions | 8 + regress/tests/multi-storage-test | 97 +++++++ 15 files changed, 591 insertions(+), 145 deletions(-) create mode 100644 regress/tests/multi-storage-test diff --git a/bacula/src/dird/bsr.c b/bacula/src/dird/bsr.c index a924ff1367..f8b7598930 100644 --- a/bacula/src/dird/bsr.c +++ b/bacula/src/dird/bsr.c @@ -347,7 +347,7 @@ static uint32_t write_bsr_item(RBSR *bsr, UAContext *ua, find_storage_resource(ua, rx, bsr->VolParams[i].Storage, bsr->VolParams[i].MediaType); } -// fprintf(fd, "Storage=\"%s\"\n", bsr->VolParams[i].Storage); + fprintf(fd, "Storage=\"%s\"\n", bsr->VolParams[i].Storage); fprintf(fd, "Volume=\"%s\"\n", bsr->VolParams[i].VolumeName); fprintf(fd, "MediaType=\"%s\"\n", bsr->VolParams[i].MediaType); if (bsr->fileregex) { diff --git a/bacula/src/dird/fd_cmds.c b/bacula/src/dird/fd_cmds.c index b24cff0eae..71d9fc3b61 100644 --- a/bacula/src/dird/fd_cmds.c +++ b/bacula/src/dird/fd_cmds.c @@ -123,8 +123,8 @@ int connect_to_file_daemon(JCR *jcr, int retry_interval, int max_retry_time, * Now send JobId and authorization key */ fd->fsend(jobcmd, edit_int64(jcr->JobId, ed1), jcr->Job, jcr->VolSessionId, - jcr->VolSessionTime, jcr->sd_auth_key); - if (strcmp(jcr->sd_auth_key, "dummy") != 0) { + jcr->VolSessionTime, jcr->sd_auth_key); + if (!jcr->keep_sd_auth_key && strcmp(jcr->sd_auth_key, "dummy")) { memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key)); } Dmsg1(100, ">filed: %s", fd->msg); diff --git a/bacula/src/dird/msgchan.c b/bacula/src/dird/msgchan.c index c00f81b9d7..0305056713 100644 --- a/bacula/src/dird/msgchan.c +++ b/bacula/src/dird/msgchan.c @@ -204,6 +204,7 @@ bool start_storage_daemon_job(JCR *jcr, alist *rstore, alist *wstore, bool send_ Jmsg(jcr, M_FATAL, 0, _("Storage daemon rejected Job command: %s\n"), sd->msg); return false; } else { + bfree_and_null(jcr->sd_auth_key); jcr->sd_auth_key = bstrdup(auth_key); Dmsg1(150, "sd_auth_key=%s\n", jcr->sd_auth_key); } diff --git a/bacula/src/dird/restore.c b/bacula/src/dird/restore.c index 434842255f..82f798021a 100644 --- a/bacula/src/dird/restore.c +++ b/bacula/src/dird/restore.c @@ -52,163 +52,421 @@ /* Commands sent to File daemon */ static char restorecmd[] = "restore replace=%c prelinks=%d where=%s\n"; static char restorecmdR[] = "restore replace=%c prelinks=%d regexwhere=%s\n"; -static char storaddr[] = "storage address=%s port=%d ssl=0\n"; +static char storaddr[] = "storage address=%s port=%d ssl=0 Authorization=%s\n"; /* Responses received from File daemon */ static char OKrestore[] = "2000 OK restore\n"; static char OKstore[] = "2000 OK storage\n"; +static char OKstoreend[] = "2000 OK storage end\n"; /* Responses received from the Storage daemon */ static char OKbootstrap[] = "3000 OK bootstrap\n"; -/* - * Do a restore of the specified files - * - * Returns: 0 on failure - * 1 on success - */ -bool do_restore(JCR *jcr) +static void build_restore_command(JCR *jcr, POOL_MEM &ret) { - BSOCK *fd, *sd; - JOB_DBR rjr; /* restore job record */ char replace, *where, *cmd; char empty = '\0'; - int stat; - free_wstorage(jcr); /* we don't write */ + /* Build the restore command */ - if (!allow_duplicate_job(jcr)) { - goto bail_out; + if (jcr->replace != 0) { + replace = jcr->replace; + } else if (jcr->job->replace != 0) { + replace = jcr->job->replace; + } else { + replace = REPLACE_ALWAYS; /* always replace */ } + + if (jcr->RegexWhere) { + where = jcr->RegexWhere; /* override */ + cmd = restorecmdR; + } else if (jcr->job->RegexWhere) { + where = jcr->job->RegexWhere; /* no override take from job */ + cmd = restorecmdR; - memset(&rjr, 0, sizeof(rjr)); - jcr->jr.JobLevel = L_FULL; /* Full restore */ - if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { - Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); - goto bail_out; + } else if (jcr->where) { + where = jcr->where; /* override */ + cmd = restorecmd; + } else if (jcr->job->RestoreWhere) { + where = jcr->job->RestoreWhere; /* no override take from job */ + cmd = restorecmd; + + } else { /* nothing was specified */ + where = ∅ /* use default */ + cmd = restorecmd; } - Dmsg0(20, "Updated job start record\n"); + + jcr->prefix_links = jcr->job->PrefixLinks; - Dmsg1(20, "RestoreJobId=%d\n", jcr->job->RestoreJobId); + bash_spaces(where); + Mmsg(ret, cmd, replace, jcr->prefix_links, where); + unbash_spaces(where); +} + +struct bootstrap_info +{ + FILE *bs; + UAContext *ua; + char storage[MAX_NAME_LENGTH+1]; +}; + +#define UA_CMD_SIZE 1000 + +/* Open the bootstrap file and find the first Storage= + * Returns ok if able to open + * It fills the storage name (should be the first line) + * and the file descriptor to the bootstrap file, + * it should be used for next operations, and need to be closed + * at the end. + */ +static bool open_bootstrap_file(JCR *jcr, struct bootstrap_info &info) +{ + FILE *bs; + UAContext *ua; + info.bs = NULL; + info.ua = NULL; if (!jcr->RestoreBootstrap) { - Jmsg0(jcr, M_FATAL, 0, _("Cannot restore without a bootstrap file.\n" - "You probably ran a restore job directly. All restore jobs must\n" - "be run using the restore command.\n")); - goto bail_out; + return false; + } + strncpy(info.storage, jcr->rstore->name(), MAX_NAME_LENGTH); + + bs = fopen(jcr->RestoreBootstrap, "rb"); + if (!bs) { + berrno be; + Jmsg(jcr, M_FATAL, 0, _("Could not open bootstrap file %s: ERR=%s\n"), + jcr->RestoreBootstrap, be.bstrerror()); + set_jcr_job_status(jcr, JS_ErrorTerminated); + return false; } + ua = new_ua_context(jcr); + ua->cmd = check_pool_memory_size(ua->cmd, UA_CMD_SIZE+1); + while (!fgets(ua->cmd, UA_CMD_SIZE, bs)) { + parse_ua_args(ua); + if (ua->argc != 1) { + continue; + } + if (!strcasecmp(ua->argk[0], "Storage")) { + strncpy(info.storage, ua->argv[0], MAX_NAME_LENGTH); + break; + } + } + info.bs = bs; + info.ua = ua; + fseek(bs, 0, SEEK_SET); /* return to the top of the file */ + return true; +} - /* Print Job Start message */ - Jmsg(jcr, M_INFO, 0, _("Start Restore Job %s\n"), jcr->Job); +/* + * This function compare the given storage name with the + * the current one. We compare the name and the address:port. + * Returns true if we use the same storage. + */ +static bool is_on_same_storage(JCR *jcr, char *new_one) +{ + STORE *new_store; - /* - * Open a message channel connection with the Storage - * daemon. This is to let him know that our client - * will be contacting him for a backup session. - * - */ - Dmsg0(10, "Open connection with storage daemon\n"); - set_jcr_job_status(jcr, JS_WaitSD); - /* - * Start conversation with Storage daemon - */ - if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) { - goto bail_out; + /* with old FD, we send the whole bootstrap to the storage */ + if (jcr->FDVersion < 2) { + return true; + } + /* we are in init loop ? shoudn't fall here */ + if (!*new_one) { + return true; } - sd = jcr->store_bsock; - /* - * Now start a job with the Storage daemon + /* same name */ + if (!strcmp(new_one, jcr->rstore->name())) { + return true; + } + new_store = (STORE *)GetResWithName(R_STORAGE, new_one); + if (!new_store) { + Jmsg(jcr, M_FATAL, 0, + _("Could not get storage resource '%s'.\n"), new_one); + set_jcr_job_status(jcr, JS_ErrorTerminated); + return false; + } + /* if Port and Hostname/IP are same, we are talking to the same + * Storage Daemon */ - if (!start_storage_daemon_job(jcr, jcr->rstorage, NULL)) { - goto bail_out; + if (jcr->rstore->SDport != new_store->SDport || + strcmp(jcr->rstore->address, new_store->address)) + { + return false; } + return true; +} - /* - * Send the bootstrap file -- what Volumes/files to restore - */ - if (!send_bootstrap_file(jcr, sd) || - !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) { - goto bail_out; +/* + * Check if the current line contains Storage="xxx", and compare the + * result to the current storage. We use UAContext to analyse the bsr + * string. + * + * Returns true if we need to change the storage, and it set the new + * Storage resource name in "storage" arg. + */ +static bool check_for_new_storage(JCR *jcr, struct bootstrap_info &info) +{ + UAContext *ua = info.ua; + parse_ua_args(ua); + if (ua->argc != 1) { + return false; } + if (!strcasecmp(ua->argk[0], "Storage")) { + /* Continue if this is a volume from the same storage. */ + if (is_on_same_storage(jcr, ua->argv[0])) { + return false; + } + /* note the next storage name */ + strncpy(info.storage, ua->argv[0], MAX_NAME_LENGTH); + Dmsg1(5, "Change storage to %s\n", info.storage); + return true; + } + return false; +} - if (!sd->fsend("run")) { - goto bail_out; +/* + * Send bootstrap file to Storage daemon section by section. + */ +static bool send_bootstrap_file(JCR *jcr, BSOCK *sock, + struct bootstrap_info &info) +{ + uint64_t pos; + const char *bootstrap = "bootstrap\n"; + UAContext *ua = info.ua; + FILE *bs = info.bs; + + Dmsg1(400, "send_bootstrap_file: %s\n", jcr->RestoreBootstrap); + if (!jcr->RestoreBootstrap) { + return false; } - /* - * Now start a Storage daemon message thread - */ - if (!start_storage_daemon_message_thread(jcr)) { - goto bail_out; + sock->fsend(bootstrap); + pos = ftell(bs); + while(fgets(ua->cmd, UA_CMD_SIZE, bs)) { + if (check_for_new_storage(jcr, info)) { + /* Otherwise, we need to contact another storage daemon. + * Reset bs to the beginning of the current segment. + */ + fseek(bs, pos, SEEK_SET); + break; + } + sock->fsend("%s", ua->cmd); + pos = ftell(bs); } - Dmsg0(50, "Storage daemon connection OK\n"); + sock->signal(BNET_EOD); + return true; +} +/* + * Change the read storage resource for the current job. + */ +static void select_rstore(JCR *jcr, struct bootstrap_info &info) +{ + USTORE ustore; + if (!strcmp(jcr->rstore->name(), info.storage)) { + return; + } - /* - * Start conversation with File daemon - */ - set_jcr_job_status(jcr, JS_WaitFD); - if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) { + if (!(ustore.store = (STORE *)GetResWithName(R_STORAGE,info.storage))) { + Jmsg(jcr, M_FATAL, 0, + _("Could not get storage resource '%s'.\n"), info.storage); + set_jcr_job_status(jcr, JS_ErrorTerminated); + return; + } + + if (jcr->store_bsock) { + jcr->store_bsock->destroy(); + jcr->store_bsock = NULL; + } + + free_rstorage(jcr); + set_rstorage(jcr, &ustore); +} + +/* + * Clean the struct bootstrap_info struct + */ +static void close_bootstrap_file(struct bootstrap_info &info) +{ + if (info.bs) { + fclose(info.bs); + } + if (info.ua) { + free_ua_context(info.ua); + } +} + +/* + * Take a bootstrap and for each different storage, we change the storage + * resource and start a new restore session between the client and the storage + * + */ +bool restore_bootstrap(JCR *jcr) +{ + BSOCK *fd=NULL, *sd; + bool end_loop=false; + bool first_time=true; + struct bootstrap_info info; + POOL_MEM restore_cmd(PM_MESSAGE); + bool ret=false; + + /* this command is used for each part */ + build_restore_command(jcr, restore_cmd); + + if (!open_bootstrap_file(jcr, info)) { goto bail_out; } + while (!end_loop && !feof(info.bs)) { + + select_rstore(jcr, info); + + /* + * Open a message channel connection with the Storage + * daemon. This is to let him know that our client + * will be contacting him for a backup session. + * + */ + Dmsg0(10, "Open connection with storage daemon\n"); + set_jcr_job_status(jcr, JS_WaitSD); + /* + * Start conversation with Storage daemon + */ + if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) { + goto bail_out; + } + sd = jcr->store_bsock; + /* + * Now start a job with the Storage daemon + */ + if (!start_storage_daemon_job(jcr, jcr->rstorage, NULL)) { + goto bail_out; + } - fd = jcr->file_bsock; - set_jcr_job_status(jcr, JS_Running); + if (first_time) { + /* + * Start conversation with File daemon + */ + set_jcr_job_status(jcr, JS_WaitFD); + jcr->keep_sd_auth_key = true; /* don't clear the sd_auth_key now */ + if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) { + goto bail_out; + } - /* - * send Storage daemon address to the File daemon, - * then wait for File daemon to make connection - * with Storage daemon. - */ - if (jcr->rstore->SDDport == 0) { - jcr->rstore->SDDport = jcr->rstore->SDport; + fd = jcr->file_bsock; + } + + set_jcr_job_status(jcr, JS_WaitSD); + + /* + * Send the bootstrap file -- what Volumes/files to restore + */ + if (!send_bootstrap_file(jcr, sd, info) || + !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) { + goto bail_out; + } + + if (!sd->fsend("run")) { + goto bail_out; + } + /* + * Now start a Storage daemon message thread + */ + if (!start_storage_daemon_message_thread(jcr)) { + goto bail_out; + } + Dmsg0(50, "Storage daemon connection OK\n"); + + /* + * send Storage daemon address to the File daemon, + * then wait for File daemon to make connection + * with Storage daemon. + */ + if (jcr->rstore->SDDport == 0) { + jcr->rstore->SDDport = jcr->rstore->SDport; + } + fd->fsend(storaddr, jcr->rstore->address, jcr->rstore->SDDport, + jcr->sd_auth_key); + memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key)); + + Dmsg1(6, "dird>filed: %s\n", fd->msg); + if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) { + goto bail_out; + } + + if (first_time) { + if (!send_runscripts_commands(jcr)) { + goto bail_out; + } + first_time=false; + } + + fd->fsend("%s", restore_cmd.c_str()); + + if (!response(jcr, fd, OKrestore, "Restore", DISPLAY_ERROR)) { + goto bail_out; + } + + if (jcr->FDVersion < 2) { /* Old FD */ + end_loop=true; /* we do only one loop */ + + } else { + if (!response(jcr, fd, OKstoreend, "Store end", DISPLAY_ERROR)) { + goto bail_out; + } + wait_for_storage_daemon_termination(jcr); + } + } /* the whole boostrap has been send */ + + if (fd && jcr->FDVersion >= 2) { + fd->fsend("endrestore"); } - fd->fsend(storaddr, jcr->rstore->address, jcr->rstore->SDDport); - Dmsg1(6, "dird>filed: %s\n", fd->msg); - if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) { + + ret = true; + +bail_out: + close_bootstrap_file(info); + return ret; +} + +/* + * Do a restore of the specified files + * + * Returns: 0 on failure + * 1 on success + */ +bool do_restore(JCR *jcr) +{ + JOB_DBR rjr; /* restore job record */ + int stat; + + free_wstorage(jcr); /* we don't write */ + + if (!allow_duplicate_job(jcr)) { goto bail_out; } - if (!send_runscripts_commands(jcr)) { + memset(&rjr, 0, sizeof(rjr)); + jcr->jr.JobLevel = L_FULL; /* Full restore */ + if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { + Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); goto bail_out; } + Dmsg0(20, "Updated job start record\n"); - /* Send restore command */ + Dmsg1(20, "RestoreJobId=%d\n", jcr->job->RestoreJobId); - if (jcr->replace != 0) { - replace = jcr->replace; - } else if (jcr->job->replace != 0) { - replace = jcr->job->replace; - } else { - replace = REPLACE_ALWAYS; /* always replace */ + if (!jcr->RestoreBootstrap) { + Jmsg(jcr, M_FATAL, 0, _("Cannot restore without a bootstrap file.\n" + "You probably ran a restore job directly. All restore jobs must\n" + "be run using the restore command.\n")); + goto bail_out; } - - if (jcr->RegexWhere) { - where = jcr->RegexWhere; /* override */ - cmd = restorecmdR; - } else if (jcr->job->RegexWhere) { - where = jcr->job->RegexWhere; /* no override take from job */ - cmd = restorecmdR; - } else if (jcr->where) { - where = jcr->where; /* override */ - cmd = restorecmd; - } else if (jcr->job->RestoreWhere) { - where = jcr->job->RestoreWhere; /* no override take from job */ - cmd = restorecmd; - - } else { /* nothing was specified */ - where = ∅ /* use default */ - cmd = restorecmd; - } - - jcr->prefix_links = jcr->job->PrefixLinks; - bash_spaces(where); - fd->fsend(cmd, replace, jcr->prefix_links, where); - unbash_spaces(where); + /* Print Job Start message */ + Jmsg(jcr, M_INFO, 0, _("Start Restore Job %s\n"), jcr->Job); - if (!response(jcr, fd, OKrestore, "Restore", DISPLAY_ERROR)) { + if (!restore_bootstrap(jcr)) { goto bail_out; } diff --git a/bacula/src/dird/verify.c b/bacula/src/dird/verify.c index 2f0eaa6588..4373f23a20 100644 --- a/bacula/src/dird/verify.c +++ b/bacula/src/dird/verify.c @@ -47,12 +47,12 @@ #include "findlib/find.h" /* Commands sent to File daemon */ -static char verifycmd[] = "verify level=%s\n"; -static char storaddr[] = "storage address=%s port=%d ssl=0\n"; +static char verifycmd[] = "verify level=%s\n"; +static char storaddr[] = "storage address=%s port=%d ssl=0 Authorization=%s\n"; /* Responses received from File daemon */ -static char OKverify[] = "2000 OK verify\n"; -static char OKstore[] = "2000 OK storage\n"; +static char OKverify[] = "2000 OK verify\n"; +static char OKstore[] = "2000 OK storage\n"; /* Responses received from the Storage daemon */ static char OKbootstrap[] = "3000 OK bootstrap\n"; @@ -278,7 +278,8 @@ bool do_verify(JCR *jcr) if (jcr->rstore->SDDport == 0) { jcr->rstore->SDDport = jcr->rstore->SDport; } - bnet_fsend(fd, storaddr, jcr->rstore->address, jcr->rstore->SDDport); + bnet_fsend(fd, storaddr, jcr->rstore->address, + jcr->rstore->SDDport, jcr->sd_auth_key); if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) { goto bail_out; } diff --git a/bacula/src/filed/authenticate.c b/bacula/src/filed/authenticate.c index 5c62eb408a..34da5fa440 100644 --- a/bacula/src/filed/authenticate.c +++ b/bacula/src/filed/authenticate.c @@ -42,8 +42,9 @@ const int dbglvl = 50; /* Version at end of Hello * prior to 10Mar08 no version * 1 10Mar08 + * 2 13Mar09 - added the ability to restore from multiple storages */ -static char OK_hello[] = "2000 OK Hello 1\n"; +static char OK_hello[] = "2000 OK Hello 2\n"; static char Dir_sorry[] = "2999 Authentication failed.\n"; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index c91f2135fa..af59139c5f 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -79,6 +79,7 @@ static int fileset_cmd(JCR *jcr); static int level_cmd(JCR *jcr); static int verify_cmd(JCR *jcr); static int restore_cmd(JCR *jcr); +static int end_restore_cmd(JCR *jcr); static int storage_cmd(JCR *jcr); static int session_cmd(JCR *jcr); static int response(JCR *jcr, BSOCK *sd, char *resp, const char *cmd); @@ -90,7 +91,7 @@ static int runbefore_cmd(JCR *jcr); static int runafter_cmd(JCR *jcr); static int runbeforenow_cmd(JCR *jcr); static void set_options(findFOPTS *fo, const char *opts); - +static void set_storage_auth_key(JCR *jcr, char *key); /* Exported functions */ @@ -113,6 +114,7 @@ static struct s_cmds cmds[] = { {"JobId=", job_cmd, 0}, {"level = ", level_cmd, 0}, {"restore", restore_cmd, 0}, + {"endrestore", end_restore_cmd, 0}, {"session", session_cmd, 0}, {"status", status_cmd, 1}, {".status", qstatus_cmd, 1}, @@ -129,7 +131,8 @@ static struct s_cmds cmds[] = { /* Commands received from director that need scanning */ static char jobcmd[] = "JobId=%d Job=%127s SDid=%d SDtime=%d Authorization=%100s"; -static char storaddr[] = "storage address=%s port=%d ssl=%d"; +static char storaddr[] = "storage address=%s port=%d ssl=%d Authorization=%100s"; +static char storaddr_v1[] = "storage address=%s port=%d ssl=%d"; static char sessioncmd[] = "session %127s %ld %ld %ld %ld %ld %ld\n"; static char restorecmd[] = "restore replace=%c prelinks=%d where=%s\n"; static char restorecmd1[] = "restore replace=%c prelinks=%d where=\n"; @@ -153,6 +156,7 @@ static char OKverify[] = "2000 OK verify\n"; static char OKrestore[] = "2000 OK restore\n"; static char OKsession[] = "2000 OK session\n"; static char OKstore[] = "2000 OK storage\n"; +static char OKstoreend[] = "2000 OK storage end\n"; static char OKjob[] = "2000 OK Job %s (%s) %s,%s,%s"; static char OKsetdebug[] = "2000 OK setdebug=%d\n"; static char BADjob[] = "2901 Bad Job\n"; @@ -473,20 +477,18 @@ static int estimate_cmd(JCR *jcr) static int job_cmd(JCR *jcr) { BSOCK *dir = jcr->dir_bsock; - POOLMEM *sd_auth_key; + POOL_MEM sd_auth_key(PM_MESSAGE); + sd_auth_key.check_size(dir->msglen); - sd_auth_key = get_memory(dir->msglen); if (sscanf(dir->msg, jobcmd, &jcr->JobId, jcr->Job, - &jcr->VolSessionId, &jcr->VolSessionTime, - sd_auth_key) != 5) { + &jcr->VolSessionId, &jcr->VolSessionTime, + sd_auth_key.c_str()) != 5) { pm_strcpy(jcr->errmsg, dir->msg); Jmsg(jcr, M_FATAL, 0, _("Bad Job Command: %s"), jcr->errmsg); dir->fsend(BADjob); - free_pool_memory(sd_auth_key); return 0; } - jcr->sd_auth_key = bstrdup(sd_auth_key); - free_pool_memory(sd_auth_key); + set_storage_auth_key(jcr, sd_auth_key.c_str()); Dmsg2(120, "JobId=%d Auth=%s\n", jcr->JobId, jcr->sd_auth_key); Mmsg(jcr->errmsg, "JobId=%d Job=%s", jcr->JobId, jcr->Job); new_plugins(jcr); /* instantiate plugins for this jcr */ @@ -1378,6 +1380,36 @@ static int session_cmd(JCR *jcr) return dir->fsend(OKsession); } +static void set_storage_auth_key(JCR *jcr, char *key) +{ + /* if no key don't update anything */ + if (!*key) { + return; + } + + /* We can be contacting multiple storage daemons. + * So, make sure that any old jcr->store_bsock is cleaned up. + */ + if (jcr->store_bsock) { + jcr->store_bsock->destroy(); + jcr->store_bsock = NULL; + } + + /* We can be contacting multiple storage daemons. + * So, make sure that any old jcr->sd_auth_key is cleaned up. + */ + if (jcr->sd_auth_key) { + /* If we already have a Authorization key, director can do multi + * storage restore + */ + Dmsg0(5, "set multi_restore=true\n"); + jcr->multi_restore = true; + bfree(jcr->sd_auth_key); + } + + jcr->sd_auth_key = bstrdup(key); +} + /* * Get address of storage daemon from Director * @@ -1386,16 +1418,29 @@ static int storage_cmd(JCR *jcr) { int stored_port; /* storage daemon port */ int enable_ssl; /* enable ssl to sd */ + POOL_MEM sd_auth_key(PM_MESSAGE); BSOCK *dir = jcr->dir_bsock; BSOCK *sd = new_bsock(); /* storage daemon bsock */ + Dmsg1(100, "StorageCmd: %s", dir->msg); - if (sscanf(dir->msg, storaddr, &jcr->stored_addr, &stored_port, &enable_ssl) != 3) { - pm_strcpy(jcr->errmsg, dir->msg); - Jmsg(jcr, M_FATAL, 0, _("Bad storage command: %s"), jcr->errmsg); - goto bail_out; + sd_auth_key.check_size(dir->msglen); + if (sscanf(dir->msg, storaddr, &jcr->stored_addr, &stored_port, + &enable_ssl, sd_auth_key.c_str()) != 4) + { + if (sscanf(dir->msg, storaddr_v1, &jcr->stored_addr, + &stored_port, &enable_ssl) != 3) + { + pm_strcpy(jcr->errmsg, dir->msg); + Jmsg(jcr, M_FATAL, 0, _("Bad storage command: %s"), jcr->errmsg); + goto bail_out; + } } - Dmsg3(110, "Open storage: %s:%d ssl=%d\n", jcr->stored_addr, stored_port, enable_ssl); + + set_storage_auth_key(jcr, sd_auth_key.c_str()); + + Dmsg3(110, "Open storage: %s:%d ssl=%d\n", jcr->stored_addr, stored_port, + enable_ssl); /* Open command communications with Storage daemon */ /* Try to connect for 1 hour at 10 second intervals */ @@ -1428,8 +1473,8 @@ static int storage_cmd(JCR *jcr) return dir->fsend(OKstore); bail_out: - dir->fsend(BADcmd, "storage"); - return 0; + dir->fsend(BADcmd, "storage"); + return 0; } @@ -1807,12 +1852,29 @@ static int restore_cmd(JCR *jcr) sd->signal(BNET_TERMINATE); bail_out: + bfree_and_null(jcr->where); if (jcr->JobErrors) { set_jcr_job_status(jcr, JS_ErrorTerminated); } Dmsg0(130, "Done in job.c\n"); + + int ret; + if (jcr->multi_restore) { + dir->fsend(OKstoreend); + ret = 1; /* we continue the loop, waiting for next part */ + } else { + end_restore_cmd(jcr); + ret = 0; /* we stop here */ + } + + return ret; +} + +static int end_restore_cmd(JCR *jcr) +{ + Dmsg0(5, "end_restore_cmd\n"); generate_plugin_event(jcr, bEventEndRestoreJob); return 0; /* return and terminate command loop */ } diff --git a/bacula/src/filed/verify.c b/bacula/src/filed/verify.c index 8f50f78cdb..f4da34b666 100644 --- a/bacula/src/filed/verify.c +++ b/bacula/src/filed/verify.c @@ -187,18 +187,18 @@ static int verify_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) /* Send file attributes to Director (note different format than for Storage) */ Dmsg2(400, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, ff_pkt->fname); if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) { - stat = bnet_fsend(dir, "%d %d %s %s%c%s%c%s%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, - 0, attribs, 0, ff_pkt->link, 0); + stat = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, + 0, attribs, 0, ff_pkt->link, 0); } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) { - /* Here link is the canonical filename (i.e. with trailing slash) */ - stat = bnet_fsend(dir,"%d %d %s %s%c%s%c%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link, - 0, attribs, 0, 0); + /* Here link is the canonical filename (i.e. with trailing slash) */ + stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link, + 0, attribs, 0, 0); } else { - stat = bnet_fsend(dir,"%d %d %s %s%c%s%c%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, - 0, attribs, 0, 0); + stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, + 0, attribs, 0, 0); } Dmsg2(20, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg); if (!stat) { @@ -260,10 +260,10 @@ static int verify_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true); Dmsg3(400, "send inx=%d %s=%s\n", jcr->JobFiles, digest_name, digest_buf); - bnet_fsend(dir, "%d %d %s *%s-%d*", jcr->JobFiles, digest_stream, digest_buf, + dir->fsend("%d %d %s *%s-%d*", jcr->JobFiles, digest_stream, digest_buf, digest_name, jcr->JobFiles); Dmsg3(20, "bfiled>bdird: %s len=%d: msg=%s\n", digest_name, - dir->msglen, dir->msg); + dir->msglen, dir->msg); free(digest_buf); } diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index 7a321672ff..151a5224cf 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -336,6 +336,7 @@ public: bool Encrypt; /* Encryption used by FD */ bool stats_enabled; /* Keep all job records in a table for long term statistics */ bool no_maxtime; /* Don't check Max*Time for this JCR */ + bool keep_sd_auth_key; /* Clear or not the SD auth key after connection*/ #endif /* DIRECTOR_DAEMON */ @@ -372,6 +373,7 @@ public: CRYPTO_CTX crypto; /* Crypto ctx */ DIRRES* director; /* Director resource */ bool VSS; /* VSS used by FD */ + bool multi_restore; /* Dir can do multiple storage restore */ htable *file_list; /* Previous file list (accurate mode) */ #endif /* FILE_DAEMON */ diff --git a/bacula/src/stored/parse_bsr.c b/bacula/src/stored/parse_bsr.c index f5e0108de2..f4c631b398 100644 --- a/bacula/src/stored/parse_bsr.c +++ b/bacula/src/stored/parse_bsr.c @@ -59,6 +59,7 @@ static BSR *store_exclude(LEX *lc, BSR *bsr); static BSR *store_stream(LEX *lc, BSR *bsr); static BSR *store_slot(LEX *lc, BSR *bsr); static BSR *store_fileregex(LEX *lc, BSR *bsr); +static BSR *store_nothing(LEX *lc, BSR *bsr); static bool is_fast_rejection_ok(BSR *bsr); static bool is_positioning_ok(BSR *bsr); @@ -91,6 +92,7 @@ struct kw_items items[] = { {"slot", store_slot}, {"device", store_device}, {"fileregex", store_fileregex}, + {"storage", store_nothing}, {NULL, NULL} }; @@ -286,6 +288,17 @@ static BSR *store_mediatype(LEX *lc, BSR *bsr) return bsr; } +static BSR *store_nothing(LEX *lc, BSR *bsr) +{ + int token; + + token = lex_get_token(lc, T_STRING); + if (token == T_ERROR) { + return NULL; + } + return bsr; +} + /* Shove the Device name in each Volume in the current bsr */ static BSR *store_device(LEX *lc, BSR *bsr) { diff --git a/bacula/technotes b/bacula/technotes index 839133e1e8..562dc78bb2 100644 --- a/bacula/technotes +++ b/bacula/technotes @@ -3,6 +3,7 @@ General: 30Jul09 +ebl Add restore from multiple storage functionnality kes Add 'show disabled' command that lists the disabled jobs. kes Modify enable/disable commands to show only appropriate Jobs. 29Jul09 diff --git a/regress/DartTestfile.txt.in b/regress/DartTestfile.txt.in index 1069df0c4b..916852a428 100644 --- a/regress/DartTestfile.txt.in +++ b/regress/DartTestfile.txt.in @@ -38,6 +38,7 @@ ADD_TEST(disk:migration-jobspan-test "@regressdir@/tests/migration-jobspan-test" ADD_TEST(disk:migration-job-test "@regressdir@/tests/migration-job-test") ADD_TEST(disk:migration-time-test "@regressdir@/tests/migration-time-test") ADD_TEST(disk:migration-volume-test "@regressdir@/tests/migration-volume-test") +ADD_TEST(disk:multiple-storage-test "@regressdir@/tests/multiple-storage-test") ADD_TEST(disk:query-test "@regressdir@/tests/query-test") ADD_TEST(disk:recycle-test "@regressdir@/tests/recycle-test") ADD_TEST(disk:regexwhere-test "@regressdir@/tests/regexwhere-test") diff --git a/regress/all-disk-tests b/regress/all-disk-tests index b5912d76f9..371a3a3261 100755 --- a/regress/all-disk-tests +++ b/regress/all-disk-tests @@ -64,6 +64,7 @@ nice tests/migration-job-test nice tests/migration-jobspan-test nice tests/migration-volume-test nice tests/migration-time-test +nice tests/multiple-storage-test nice tests/hardlink-test nice tests/tls-test nice tests/virtual-changer-test diff --git a/regress/scripts/functions b/regress/scripts/functions index 329403c9e0..fe1b9630be 100644 --- a/regress/scripts/functions +++ b/regress/scripts/functions @@ -350,6 +350,14 @@ copy_test_confs() ${rscripts}/cleanup } +disable_pluguins() +{ + for i in ${conf}/bacula-fd.conf; do + sed 's/Plugin/#Plugin/i' $i > $tmp/1 + cp -f $tmp/1 $i + done +} + debug_wait() { if test "x${REGRESS_WAIT}" = "x1"; then diff --git a/regress/tests/multi-storage-test b/regress/tests/multi-storage-test new file mode 100644 index 0000000000..ea90f07f77 --- /dev/null +++ b/regress/tests/multi-storage-test @@ -0,0 +1,97 @@ +#!/bin/sh +# +# This script uses the virtual disk autochanger +# +TestName="multi-storage-test" +JobName=backup +. scripts/functions + +scripts/cleanup +scripts/copy-2disk-confs +scripts/prepare-disk-changer + +perl -ne ' +if (/^Storage {/) { $in=1; $nb++; } +if (/^}/) { $in=0 } +if (/Address / && $in) {$_ = "Address = 127.0.0.$nb\n"; } +print; +' $conf/bacula-dir.conf > $tmp/1 +cp $tmp/1 $conf/bacula-dir.conf + +disable_pluguins + +echo "${cwd}/tmp/build" >${cwd}/tmp/file-list +if test ! -d ${cwd}/tmp/build ; then + mkdir ${cwd}/tmp/build +fi +cp -p ${cwd}/build/src/dird/*.c ${cwd}/tmp/build +cd ${cwd}/tmp +echo "${cwd}/tmp/build/ficheriro1.txt" >restore-list +echo "${cwd}/tmp/build/ficheriro2.txt" >>restore-list +cd ${cwd} + +change_jobname $JobName +start_test + +# Write out bconsole commands +cat <${cwd}/tmp/bconcmds +@$out /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=DDS-4 volume=TestVolume001 Pool=Default slot=1 drive=0 +label storage=File volume=TestVolume002 Pool=Default +run job=$JobName storage=DDS-4 yes +wait +messages +quit +END_OF_DATA + +run_bacula + +echo "ficheriro1.txt" >${cwd}/tmp/build/ficheriro1.txt +echo "ficheriro2.txt" >${cwd}/tmp/build/ficheriro2.txt + + +cat <${cwd}/tmp/bconcmds +@$out /dev/null +messages +@$out ${cwd}/tmp/log1.out +@# Force Incremental on the second Volume +run level=Incremental storage=File job=$JobName yes +wait +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +setdebug trace=1 level=110 client +setdebug trace=1 level=110 director +restore where=${cwd}/tmp/bacula-restores select all done yes +wait +messages +@$out +quit +END_OF_DATA + +run_bconsole + +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_tmp_build_diff + +# +# This script seems to more or less randomly fail, so we +# add extra code here to produce a "dump" in the event of +# an error. +# +if [ $dstat != 0 -o $bstat != 0 -o $rstat != 0 ] ; then + cat ${cwd}/tmp/log1.out + echo " " + cat ${cwd}/tmp/log2.out + echo " " + diff -r ${cwd}/tmp/build ${cwd}/tmp/bacula-restores${cwd}/tmp/build +fi + +end_test -- 2.39.5