X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Fdird%2Fbackup.c;h=77a3845e2d9a87b83f7e9c93bbbbb06589faf8ff;hb=6ab87b8f29631612d4150c37475d5ddb2d367d80;hp=91658ecb46357cca9a5e074c2014530114e7edbe;hpb=5aa415328704b532273117dd5d4efb04b041c406;p=bacula%2Fbacula diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 91658ecb46..77a3845e2d 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -1,12 +1,12 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2009 Free Software Foundation Europe e.V. + Copyright (C) 2000-2012 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public + modify it under the terms of version three of the GNU Affero General Public License as published by the Free Software Foundation and included in the file LICENSE. @@ -15,7 +15,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Affero General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. @@ -38,7 +38,6 @@ * to do the backup. * When the File daemon finishes the job, update the DB. * - * Version $Id$ */ #include "bacula.h" @@ -46,7 +45,7 @@ #include "ua.h" /* Commands sent to File daemon */ -static char backupcmd[] = "backup\n"; +static char backupcmd[] = "backup FileIndex=%ld\n"; static char storaddr[] = "storage address=%s port=%d ssl=%d\n"; /* Responses received from File daemon */ @@ -65,7 +64,7 @@ static char OldEndJob[] = "2800 End Job TermCode=%d JobFiles=%u " bool do_backup_init(JCR *jcr) { - if (jcr->get_JobLevel() == L_VIRTUAL_FULL) { + if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { return do_vbackup_init(jcr); } free_rstorage(jcr); /* we don't read so release */ @@ -106,7 +105,7 @@ bool do_backup_init(JCR *jcr) /* Take all base jobs from job resource and find the * last L_BASE jobid. */ -static bool get_base_jobids(JCR *jcr, POOLMEM *jobids) +static bool get_base_jobids(JCR *jcr, db_list_ctx *jobids) { JOB_DBR jr; JOB *job; @@ -125,18 +124,21 @@ static bool get_base_jobids(JCR *jcr, POOLMEM *jobids) db_get_base_jobid(jcr, jcr->db, &jr, &id); if (id) { - if (jobids[0]) { - pm_strcat(jobids, ","); + if (jobids->count) { + pm_strcat(jobids->list, ","); } - pm_strcat(jobids, edit_uint64(id, str_jobid)); + pm_strcat(jobids->list, edit_uint64(id, str_jobid)); + jobids->count++; } } - return *jobids != '\0'; + return jobids->count > 0; } /* - * Foreach files in currrent list, send "/path/fname\0LStat" to FD + * Foreach files in currrent list, send "/path/fname\0LStat\0MD5\0Delta" to FD + * row[0]=Path, row[1]=Filename, row[2]=FileIndex + * row[3]=JobId row[4]=LStat row[5]=DeltaSeq row[6]=MD5 */ static int accurate_list_handler(void *ctx, int num_fields, char **row) { @@ -146,92 +148,160 @@ static int accurate_list_handler(void *ctx, int num_fields, char **row) return 1; } - if (row[2] > 0) { /* discard when file_index == 0 */ - jcr->file_bsock->fsend("%s%s%c%s", row[0], row[1], 0, row[4]); + if (row[2][0] == '0') { /* discard when file_index == 0 */ + return 0; + } + + /* sending with checksum */ + if (jcr->use_accurate_chksum + && num_fields == 7 + && row[6][0] /* skip checksum = '0' */ + && row[6][1]) + { + jcr->file_bsock->fsend("%s%s%c%s%c%s%c%s", + row[0], row[1], 0, row[4], 0, row[6], 0, row[5]); + } else { + jcr->file_bsock->fsend("%s%s%c%s%c%c%s", + row[0], row[1], 0, row[4], 0, 0, row[5]); } return 0; } +/* In this procedure, we check if the current fileset is using checksum + * FileSet-> Include-> Options-> Accurate/Verify/BaseJob=checksum + * This procedure uses jcr->HasBase, so it must be call after the initialization + */ +static bool is_checksum_needed_by_fileset(JCR *jcr) +{ + FILESET *f; + INCEXE *inc; + FOPTS *fopts; + bool in_block=false; + bool have_basejob_option=false; + if (!jcr->job || !jcr->job->fileset) { + return false; + } + + f = jcr->job->fileset; + + for (int i=0; i < f->num_includes; i++) { /* Parse all Include {} */ + inc = f->include_items[i]; + + for (int j=0; j < inc->num_opts; j++) { /* Parse all Options {} */ + fopts = inc->opts_list[j]; + + for (char *k=fopts->opts; *k ; k++) { /* Try to find one request */ + switch (*k) { + case 'V': /* verify */ + in_block = (jcr->getJobType() == JT_VERIFY); /* not used now */ + break; + case 'J': /* Basejob keyword */ + have_basejob_option = in_block = jcr->HasBase; + break; + case 'C': /* Accurate keyword */ + in_block = !jcr->is_JobLevel(L_FULL); + break; + case ':': /* End of keyword */ + in_block = false; + break; + case '5': /* MD5 */ + case '1': /* SHA1 */ + if (in_block) { + Dmsg0(50, "Checksum will be sent to FD\n"); + return true; + } + break; + default: + break; + } + } + } + } + + /* By default for BaseJobs, we send the checksum */ + if (!have_basejob_option && jcr->HasBase) { + return true; + } + + Dmsg0(50, "Checksum will be sent to FD\n"); + return false; +} + /* * Send current file list to FD * DIR -> FD : accurate files=xxxx - * DIR -> FD : /path/to/file\0Lstat - * DIR -> FD : /path/to/dir/\0Lstat + * DIR -> FD : /path/to/file\0Lstat\0MD5\0Delta + * DIR -> FD : /path/to/dir/\0Lstat\0MD5\0Delta * ... * DIR -> FD : EOD */ bool send_accurate_current_files(JCR *jcr) { POOL_MEM buf; - bool ret=true; + db_list_ctx jobids; + db_list_ctx nb; - if (!jcr->accurate || job_canceled(jcr)) { + /* In base level, no previous job is used and no restart incomplete jobs */ + if (jcr->is_canceled() || jcr->is_JobLevel(L_BASE)) { return true; } - /* In base level, no previous job is used */ - if (jcr->get_JobLevel() == L_BASE) { + if (!jcr->accurate) { return true; } - POOLMEM *nb = get_pool_memory(PM_FNAME); - POOLMEM *jobids = get_pool_memory(PM_FNAME); - nb[0] = jobids[0] = '\0'; - - if (jcr->get_JobLevel() == L_FULL) { + if (jcr->is_JobLevel(L_FULL)) { /* On Full mode, if no previous base job, no accurate things */ - if (!get_base_jobids(jcr, jobids)) { - goto bail_out; + if (get_base_jobids(jcr, &jobids)) { + jcr->HasBase = true; + Jmsg(jcr, M_INFO, 0, _("Using BaseJobId(s): %s\n"), jobids.list); + } else { + return true; } - jcr->HasBase = true; - Jmsg(jcr, M_INFO, 0, _("Using BaseJobId(s): %s\n"), jobids); - } else { /* For Incr/Diff level, we search for older jobs */ - db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, jobids); + db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, &jobids); /* We are in Incr/Diff, but no Full to build the accurate list... */ - if (*jobids == 0) { - ret=false; + if (jobids.count == 0) { Jmsg(jcr, M_FATAL, 0, _("Cannot find previous jobids.\n")); - goto bail_out; + return false; /* fail */ } } + /* Don't send and store the checksum if fileset doesn't require it */ + jcr->use_accurate_chksum = is_checksum_needed_by_fileset(jcr); + if (jcr->JobId) { /* display the message only for real jobs */ Jmsg(jcr, M_INFO, 0, _("Sending Accurate information.\n")); } /* to be able to allocate the right size for htable */ - Mmsg(buf, "SELECT sum(JobFiles) FROM Job WHERE JobId IN (%s)",jobids); - db_sql_query(jcr->db, buf.c_str(), db_get_int_handler, nb); - Dmsg2(200, "jobids=%s nb=%s\n", jobids, nb); - jcr->file_bsock->fsend("accurate files=%s\n", nb); + Mmsg(buf, "SELECT sum(JobFiles) FROM Job WHERE JobId IN (%s)", jobids.list); + db_sql_query(jcr->db, buf.c_str(), db_list_handler, &nb); + Dmsg2(200, "jobids=%s nb=%s\n", jobids.list, nb.list); + jcr->file_bsock->fsend("accurate files=%s\n", nb.list); if (!db_open_batch_connexion(jcr, jcr->db)) { - ret = false; - Jmsg0(jcr, M_FATAL, 0, "Can't get dedicate sql connexion"); - goto bail_out; + Jmsg0(jcr, M_FATAL, 0, "Can't get batch sql connexion"); + return false; /* Fail */ } if (jcr->HasBase) { - db_create_base_file_list(jcr, jcr->db_batch, jobids); - db_get_base_file_list(jcr, jcr->db_batch, + jcr->nb_base_files = str_to_int64(nb.list); + db_create_base_file_list(jcr, jcr->db, jobids.list); + db_get_base_file_list(jcr, jcr->db, jcr->use_accurate_chksum, accurate_list_handler, (void *)jcr); } else { - db_get_file_list(jcr, jcr->db_batch, jobids, + db_get_file_list(jcr, jcr->db_batch, + jobids.list, jcr->use_accurate_chksum, false /* no delta */, accurate_list_handler, (void *)jcr); } - /* TODO: close the batch connexion ? (can be used very soon) */ + /* TODO: close the batch connection ? (can be used very soon) */ jcr->file_bsock->signal(BNET_EOD); - -bail_out: - free_pool_memory(jobids); - free_pool_memory(nb); - - return ret; + return true; } /* @@ -247,8 +317,10 @@ bool do_backup(JCR *jcr) BSOCK *fd; STORE *store; char ed1[100]; + db_int64_ctx job; + POOL_MEM buf; - if (jcr->get_JobLevel() == L_VIRTUAL_FULL) { + if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { return do_vbackup(jcr); } @@ -256,7 +328,7 @@ bool do_backup(JCR *jcr) Jmsg(jcr, M_INFO, 0, _("Start Backup JobId %s, Job=%s\n"), edit_uint64(jcr->JobId, ed1), jcr->Job); - set_jcr_job_status(jcr, JS_Running); + jcr->setJobStatus(JS_Running); Dmsg2(100, "JobId=%d JobLevel=%c\n", jcr->jr.JobId, jcr->jr.JobLevel); if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); @@ -270,7 +342,7 @@ bool do_backup(JCR *jcr) * */ Dmsg0(110, "Open connection with storage daemon\n"); - set_jcr_job_status(jcr, JS_WaitSD); + jcr->setJobStatus(JS_WaitSD); /* * Start conversation with Storage daemon */ @@ -289,7 +361,7 @@ bool do_backup(JCR *jcr) * to avoid two threads from using the BSOCK structure at * the same time. */ - if (!bnet_fsend(jcr->store_bsock, "run")) { + if (!jcr->store_bsock->fsend("run")) { return false; } @@ -304,23 +376,23 @@ bool do_backup(JCR *jcr) } Dmsg0(150, "Storage daemon connection OK\n"); - set_jcr_job_status(jcr, JS_WaitFD); + jcr->setJobStatus(JS_WaitFD); if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) { goto bail_out; } - set_jcr_job_status(jcr, JS_Running); + jcr->setJobStatus(JS_Running); fd = jcr->file_bsock; - if (!send_include_list(jcr)) { + if (!send_level_command(jcr)) { goto bail_out; } - if (!send_exclude_list(jcr)) { + if (!send_include_list(jcr)) { goto bail_out; } - if (!send_level_command(jcr)) { + if (!send_exclude_list(jcr)) { goto bail_out; } @@ -371,11 +443,12 @@ bool do_backup(JCR *jcr) * all files to FD. */ if (!send_accurate_current_files(jcr)) { - goto bail_out; + goto bail_out; /* error */ } /* Send backup command */ - fd->fsend(backupcmd); + fd->fsend(backupcmd, jcr->JobFiles); + Dmsg1(100, ">filed: %s", fd->msg); if (!response(jcr, fd, OKbackup, "backup", DISPLAY_ERROR)) { goto bail_out; } @@ -384,10 +457,8 @@ bool do_backup(JCR *jcr) stat = wait_for_job_termination(jcr); db_write_batch_file_records(jcr); /* used by bulk batch file insert */ - if (jcr->HasBase && - !db_commit_base_file_attributes_record(jcr, jcr->db_batch)) - { - Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db_batch)); + if (jcr->HasBase && !db_commit_base_file_attributes_record(jcr, jcr->db)) { + Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); } if (stat == JS_Terminated) { @@ -398,7 +469,7 @@ bool do_backup(JCR *jcr) /* Come here only after starting SD thread */ bail_out: - set_jcr_job_status(jcr, JS_ErrorTerminated); + jcr->setJobStatus(JS_ErrorTerminated); Dmsg1(400, "wait for sd. use=%d\n", jcr->use_count()); /* Cancel SD */ wait_for_job_termination(jcr, FDConnectTimeout); @@ -426,7 +497,7 @@ int wait_for_job_termination(JCR *jcr, int timeout) int Encrypt = 0; btimer_t *tid=NULL; - set_jcr_job_status(jcr, JS_Running); + jcr->setJobStatus(JS_Running); if (fd) { if (timeout) { @@ -440,7 +511,7 @@ int wait_for_job_termination(JCR *jcr, int timeout) sscanf(fd->msg, OldEndJob, &jcr->FDJobStatus, &JobFiles, &ReadBytes, &JobBytes, &JobErrors) == 5)) { fd_ok = true; - set_jcr_job_status(jcr, jcr->FDJobStatus); + jcr->setJobStatus(jcr->FDJobStatus); Dmsg1(100, "FDStatus=%c\n", (char)jcr->JobStatus); } else { Jmsg(jcr, M_WARNING, 0, _("Unexpected Client Job message: %s\n"), @@ -455,14 +526,26 @@ int wait_for_job_termination(JCR *jcr, int timeout) } if (is_bnet_error(fd)) { + int i = 0; Jmsg(jcr, M_FATAL, 0, _("Network error with FD during %s: ERR=%s\n"), - job_type_to_str(jcr->get_JobType()), fd->bstrerror()); + job_type_to_str(jcr->getJobType()), fd->bstrerror()); + while (i++ < 10 && jcr->job->RescheduleIncompleteJobs && jcr->is_canceled()) { + bmicrosleep(3, 0); + } + } fd->signal(BNET_TERMINATE); /* tell Client we are terminating */ } - /* Force cancel in SD if failing */ - if (job_canceled(jcr) || !fd_ok) { + /* + * Force cancel in SD if failing, but not for Incomplete jobs + * so that we let the SD despool. + */ + Dmsg5(100, "cancel=%d fd_ok=%d FDJS=%d JS=%d SDJS=%d\n", jcr->is_canceled(), fd_ok, jcr->FDJobStatus, + jcr->JobStatus, jcr->SDJobStatus); + if (jcr->is_canceled() || (!jcr->job->RescheduleIncompleteJobs && !fd_ok)) { + Dmsg4(100, "fd_ok=%d FDJS=%d JS=%d SDJS=%d\n", fd_ok, jcr->FDJobStatus, + jcr->JobStatus, jcr->SDJobStatus); cancel_storage_daemon_job(jcr); } @@ -513,22 +596,34 @@ void backup_cleanup(JCR *jcr, int TermCode) CLIENT_DBR cr; double kbps, compression; utime_t RunTime; + POOL_MEM base_info; - if (jcr->get_JobLevel() == L_VIRTUAL_FULL) { + if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { vbackup_cleanup(jcr, TermCode); return; } Dmsg2(100, "Enter backup_cleanup %d %c\n", TermCode, TermCode); - memset(&mr, 0, sizeof(mr)); memset(&cr, 0, sizeof(cr)); +#ifdef xxxx + /* The current implementation of the JS_Warning status is not + * completed. SQL part looks to be ok, but the code is using + * JS_Terminated almost everywhere instead of (JS_Terminated || JS_Warning) + * as we do with is_canceled() + */ + if (jcr->getJobStatus() == JS_Terminated && + (jcr->JobErrors || jcr->SDErrors || jcr->JobWarnings)) { + TermCode = JS_Warnings; + } +#endif + update_job_end(jcr, TermCode); if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) { Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"), db_strerror(jcr->db)); - set_jcr_job_status(jcr, JS_ErrorTerminated); + jcr->setJobStatus(JS_ErrorTerminated); } bstrncpy(cr.Name, jcr->client->name(), sizeof(cr.Name)); @@ -541,7 +636,7 @@ void backup_cleanup(JCR *jcr, int TermCode) if (!db_get_media_record(jcr, jcr->db, &mr)) { Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"), mr.VolumeName, db_strerror(jcr->db)); - set_jcr_job_status(jcr, JS_ErrorTerminated); + jcr->setJobStatus(JS_ErrorTerminated); } update_bootstrap_file(jcr); @@ -554,6 +649,9 @@ void backup_cleanup(JCR *jcr, int TermCode) term_msg = _("Backup OK"); } break; + case JS_Incomplete: + term_msg = _("Backup failed -- incomplete"); + break; case JS_Warnings: term_msg = _("Backup OK -- with warnings"); break; @@ -617,9 +715,15 @@ void backup_cleanup(JCR *jcr, int TermCode) jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg)); jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); + if (jcr->HasBase) { + Mmsg(base_info, " Base files/Used files: %lld/%lld (%.2f%%)\n", + jcr->nb_base_files, + jcr->nb_base_files_used, + jcr->nb_base_files_used*100.0/jcr->nb_base_files); + } // bmicrosleep(15, 0); /* for debugging SIGHUP */ - Jmsg(jcr, msg_type, 0, _("%s %s %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n" " Build OS: %s %s %s\n" " JobId: %d\n" " Job: %s\n" @@ -640,6 +744,7 @@ void backup_cleanup(JCR *jcr, int TermCode) " SD Bytes Written: %s (%sB)\n" " Rate: %.1f KB/s\n" " Software Compression: %s\n" +"%s" /* Basefile info */ " VSS: %s\n" " Encryption: %s\n" " Accurate: %s\n" @@ -652,11 +757,11 @@ void backup_cleanup(JCR *jcr, int TermCode) " FD termination status: %s\n" " SD termination status: %s\n" " Termination: %s\n\n"), - BACULA, my_name, VERSION, LSMDATE, edt, + BACULA, my_name, VERSION, LSMDATE, HOST_OS, DISTNAME, DISTVER, jcr->jr.JobId, jcr->jr.Job, - level_to_str(jcr->get_JobLevel()), jcr->since, + level_to_str(jcr->getJobLevel()), jcr->since, jcr->client->name(), cr.Uname, jcr->fileset->name(), jcr->FSCreateTime, jcr->pool->name(), jcr->pool_source, @@ -675,6 +780,7 @@ void backup_cleanup(JCR *jcr, int TermCode) edit_uint64_with_suffix(jcr->SDJobBytes, ec6), kbps, compress, + base_info.c_str(), jcr->VSS?_("yes"):_("no"), jcr->Encrypt?_("yes"):_("no"), jcr->accurate?_("yes"):_("no"), @@ -713,7 +819,7 @@ void update_bootstrap_file(JCR *jcr) fd = bpipe ? bpipe->wfd : NULL; } else { /* ***FIXME*** handle BASE */ - fd = fopen(fname, jcr->get_JobLevel()==L_FULL?"w+b":"a+b"); + fd = fopen(fname, jcr->is_JobLevel(L_FULL)?"w+b":"a+b"); } if (fd) { VolCount = db_get_job_volume_parameters(jcr, jcr->db, jcr->JobId, @@ -722,14 +828,14 @@ void update_bootstrap_file(JCR *jcr) Jmsg(jcr, M_ERROR, 0, _("Could not get Job Volume Parameters to " "update Bootstrap file. ERR=%s\n"), db_strerror(jcr->db)); if (jcr->SDJobFiles != 0) { - set_jcr_job_status(jcr, JS_ErrorTerminated); + jcr->setJobStatus(JS_ErrorTerminated); } } /* Start output with when and who wrote it */ bstrftimes(edt, sizeof(edt), time(NULL)); fprintf(fd, "# %s - %s - %s%s\n", edt, jcr->jr.Job, - level_to_str(jcr->get_JobLevel()), jcr->since); + level_to_str(jcr->getJobLevel()), jcr->since); for (int i=0; i < VolCount; i++) { /* Write the record */ fprintf(fd, "Volume=\"%s\"\n", VolParams[i].VolumeName); @@ -757,7 +863,7 @@ void update_bootstrap_file(JCR *jcr) berrno be; Jmsg(jcr, M_ERROR, 0, _("Could not open WriteBootstrap file:\n" "%s: ERR=%s\n"), fname, be.bstrerror()); - set_jcr_job_status(jcr, JS_ErrorTerminated); + jcr->setJobStatus(JS_ErrorTerminated); } free_pool_memory(fname); }