From: Eric Bollengier Date: Fri, 18 Dec 2015 10:20:53 +0000 (+0100) Subject: Implement level=Data to the Verify job X-Git-Tag: Release-7.4.0~119 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=4d04bb5463c0eedc11b87b5bc277a51f38c7ab32;p=bacula%2Fbacula Implement level=Data to the Verify job --- diff --git a/bacula/src/cats/sql_find.c b/bacula/src/cats/sql_find.c index 7c83b2e663..1c41c9cc04 100644 --- a/bacula/src/cats/sql_find.c +++ b/bacula/src/cats/sql_find.c @@ -260,6 +260,7 @@ bool BDB::bdb_find_last_jobid(JCR *jcr, const char *Name, JOB_DBR *jr) edit_int64(jr->ClientId, ed1)); } else if (jr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG || jr->JobLevel == L_VERIFY_DISK_TO_CATALOG || + jr->JobLevel == L_VERIFY_DATA || jr->JobType == JT_BACKUP) { if (Name) { bdb_escape_string(jcr, esc_name, (char*)Name, diff --git a/bacula/src/dird/verify.c b/bacula/src/dird/verify.c index fecdbce7e0..b9f1c4f84e 100644 --- a/bacula/src/dird/verify.c +++ b/bacula/src/dird/verify.c @@ -65,11 +65,10 @@ bool do_verify_init(JCR *jcr) free_rstorage(jcr); free_wstorage(jcr); break; + case L_VERIFY_DATA: case L_VERIFY_VOLUME_TO_CATALOG: free_wstorage(jcr); break; - case L_VERIFY_DATA: - break; default: Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->getJobLevel(), jcr->getJobLevel()); @@ -115,11 +114,13 @@ bool do_verify(JCR *jcr) */ if (jcr->getJobLevel() == L_VERIFY_CATALOG || jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || - jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG) { + jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG || + jcr->getJobLevel() == L_VERIFY_DATA) { memcpy(&jr, &jcr->jr, sizeof(jr)); if (jcr->verify_job && (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || - jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG)) { + jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG || + jcr->getJobLevel() == L_VERIFY_DATA)) { Name = jcr->verify_job->name(); } else { Name = NULL; @@ -154,7 +155,8 @@ bool do_verify(JCR *jcr) */ if (jcr->getJobLevel() == L_VERIFY_CATALOG || jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || - jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG) { + jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG || + jcr->getJobLevel() == L_VERIFY_DATA) { jcr->previous_jr.JobId = verify_jobid; if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) { Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"), @@ -177,7 +179,7 @@ bool do_verify(JCR *jcr) * create a dummy authorization key (passed to * File daemon but not used). */ - if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) { + if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || jcr->getJobLevel() == L_VERIFY_DATA) { int stat; /* * Note: negative status is an error, zero status, means @@ -195,10 +197,43 @@ bool do_verify(JCR *jcr) jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */ } + /* Pass the original fileset to the client */ + if (jcr->getJobLevel() == L_VERIFY_DATA) { + FILESET_DBR fdbr; + memset(&fdbr, 0, sizeof(fdbr)); + fdbr.FileSetId = jcr->previous_jr.FileSetId; + if (!db_get_fileset_record(jcr, jcr->db, &fdbr)) { + Jmsg(jcr, M_FATAL, 0, + _("Could not get fileset record from previous Job. ERR=%s"), + db_strerror(jcr->db)); + return false; + } + + jcr->fileset = (FILESET *)GetResWithName(R_FILESET, fdbr.FileSet); + if (!jcr->fileset) { + if (jcr->verify_job) { + jcr->fileset = jcr->verify_job->fileset; + Jmsg(jcr, M_WARNING, 0, + _("Could not find FileSet resource \"%s\" from previous Job\n"), + fdbr.FileSet); + Jmsg(jcr, M_INFO, 0, + _("Using FileSet \"%\"\n"), jcr->fileset->name()); + + } else { + Jmsg(jcr, M_FATAL, 0, + _("Could not get FileSet resource for verify Job.")); + return false; + } + } + Dmsg1(50, "FileSet = %s\n", jcr->fileset->name()); + } + + /* Pass the current fileset to the client */ if (jcr->getJobLevel() == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) { jcr->fileset = jcr->verify_job->fileset; } - Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->previous_jr.ClientId, jcr->getJobLevel()); + Dmsg2(100, "ClientId=%u JobLevel=%c\n", + jcr->previous_jr.ClientId, jcr->getJobLevel()); if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); @@ -209,7 +244,9 @@ bool do_verify(JCR *jcr) Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%s Level=%s Job=%s\n"), edit_uint64(jcr->JobId, ed1), level_to_str(jcr->getJobLevel()), jcr->Job); - if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) { + if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || + jcr->getJobLevel() == L_VERIFY_DATA) + { /* * Start conversation with Storage daemon */ @@ -272,6 +309,7 @@ bool do_verify(JCR *jcr) case L_VERIFY_CATALOG: level = "catalog"; break; + case L_VERIFY_DATA: case L_VERIFY_VOLUME_TO_CATALOG: if (jcr->sd_calls_client) { if (jcr->FDVersion < 10) { @@ -308,18 +346,19 @@ bool do_verify(JCR *jcr) Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n")); goto bail_out; } - - level = "volume"; - break; - case L_VERIFY_DATA: - level = "data"; + if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) { + level = "volume"; + } else { + level = "data"; + } break; case L_VERIFY_DISK_TO_CATALOG: level="disk_to_catalog"; break; default: - Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->getJobLevel(), - jcr->getJobLevel()); + Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), + jcr->getJobLevel(), + jcr->getJobLevel()); goto bail_out; } @@ -371,6 +410,10 @@ bool do_verify(JCR *jcr) db_write_batch_file_records(jcr); break; + case L_VERIFY_DATA: + /* Nothing special to do */ + bget_dirmsg(fd); /* eat EOD */ + break; default: Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->getJobLevel()); goto bail_out; @@ -402,7 +445,7 @@ void verify_cleanup(JCR *jcr, int TermCode) Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->getJobLevel(), jcr->ExpectedFiles, jcr->JobFiles); - if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG && + if ((jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || jcr->getJobLevel() == L_VERIFY_DATA) && jcr->ExpectedFiles != jcr->JobFiles) { TermCode = JS_ErrorTerminated; } @@ -421,7 +464,11 @@ void verify_cleanup(JCR *jcr, int TermCode) msg_type = M_INFO; /* by default INFO message */ switch (TermCode) { case JS_Terminated: - term_msg = _("Verify OK"); + if (jcr->JobErrors || jcr->SDErrors) { + term_msg = _("Verify OK -- with warnings"); + } else { + term_msg = _("Verify OK"); + } break; case JS_FatalError: case JS_ErrorTerminated: @@ -452,7 +499,7 @@ void verify_cleanup(JCR *jcr, int TermCode) } jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg)); - if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) { + if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || jcr->getJobLevel() == L_VERIFY_DATA) { jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n" " Build OS: %s %s %s\n" @@ -468,6 +515,7 @@ void verify_cleanup(JCR *jcr, int TermCode) " Files Expected: %s\n" " Files Examined: %s\n" " Non-fatal FD errors: %d\n" +" SD Errors: %d\n" " FD termination status: %s\n" " SD termination status: %s\n" " Termination: %s\n\n"), @@ -485,6 +533,7 @@ void verify_cleanup(JCR *jcr, int TermCode) edit_uint64_with_commas(jcr->ExpectedFiles, ec1), edit_uint64_with_commas(jcr->JobFiles, ec2), jcr->JobErrors, + jcr->SDErrors, fd_term_msg, sd_term_msg, term_msg); diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index 48ade9f95e..af8c623049 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -341,7 +341,6 @@ static void *handle_director_request(BSOCK *dir) dir->fsend(EndJob, jcr->JobStatus, jcr->JobFiles, jcr->ReadBytes, jcr->JobBytes, jcr->JobErrors, vss, encrypt, 0, 0); - //Dmsg0(0/*110*/, dir->msg); } generate_daemon_event(jcr, "JobEnd"); @@ -2153,6 +2152,7 @@ static int verify_cmd(JCR *jcr) case L_VERIFY_CATALOG: do_verify(jcr); break; + case L_VERIFY_DATA: case L_VERIFY_VOLUME_TO_CATALOG: if (!open_sd_read_session(jcr)) { return 0; diff --git a/bacula/src/filed/protos.h b/bacula/src/filed/protos.h index 96a836c95a..e97bcb70e6 100644 --- a/bacula/src/filed/protos.h +++ b/bacula/src/filed/protos.h @@ -27,6 +27,9 @@ extern void do_verify_volume(JCR *jcr); extern void do_restore(JCR *jcr); extern int make_estimate(JCR *jcr); +/* From restore.c */ +bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length); + /* From authenticate.c */ bool authenticate_director(JCR *jcr); bool authenticate_storagedaemon(JCR *jcr); diff --git a/bacula/src/filed/verify_vol.c b/bacula/src/filed/verify_vol.c index e29a7dcfa5..b05ae65ade 100644 --- a/bacula/src/filed/verify_vol.c +++ b/bacula/src/filed/verify_vol.c @@ -27,28 +27,222 @@ #include "bacula.h" #include "filed.h" +#include "findlib/win32filter.h" + +#if defined(HAVE_LIBZ) +const bool have_libz = true; +#else +const bool have_libz = false; +#endif + +#ifdef HAVE_LZO +const bool have_lzo = true; +#else +const bool have_lzo = false; +#endif + +class v_ctx { +public: + JCR *jcr; + int32_t stream; /* stream less new bits */ + int32_t prev_stream; /* previous stream */ + int32_t full_stream; /* full stream including new bits */ + int32_t type; /* file type FT_ */ + int64_t size; /* current file size */ + ATTR *attr; /* Pointer to attributes */ + + bool check_size; /* Check or not the size attribute */ + bool check_chksum; /* Check the checksum */ + + crypto_digest_t digesttype; + Win32Filter win32filter; + char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)]; /* current digest */ + + v_ctx(JCR *ajcr) : + jcr(ajcr), stream(0), prev_stream(0), full_stream(0), type(0), size(-1), + attr(new_attr(jcr)), check_size(false), check_chksum(false), + digesttype(CRYPTO_DIGEST_NONE), win32filter() + { + *digest = 0; + scan_fileset(); + }; + ~v_ctx() { + free_attr(attr); + }; + /* Call this function when we change the file + * We check the st_size and we compute the digest + */ + bool close_previous_stream(); + + /* Call when we have a sparse record */ + void skip_sparse_header(char **data, uint32_t *length); + + /* Scan the fileset to know if we want to check checksums or st_size */ + void scan_fileset(); + + /* In cleanup, we reset the current file size to -1 */ + void reset_size() { + size = -1; + }; + + /* Used for sparse files */ + void set_size(int64_t val) { + size = MAX(size, val); + }; + + void update_size(int64_t val) { + if (size == -1) { + size = 0; + } + size += val; + }; + + void update_checksum(char *wbuf, int32_t wsize) { + if (wsize > 0 && check_chksum) { + if (!jcr->crypto.digest) { + jcr->crypto.digest = crypto_digest_new(jcr, digesttype); + } + crypto_digest_update(jcr->crypto.digest, (uint8_t *)wbuf, wsize); + } + }; +}; /* Data received from Storage Daemon */ static char rec_header[] = "rechdr %ld %ld %ld %ld %ld"; /* Forward referenced functions */ +/* We don't know in advance which digest mode is needed, we do not + * want to store files on disk either to check afterward. So, we read + * the fileset definition and we try to guess the digest that will be + * used. If the FileSet uses multiple digests, it will not work. + */ +void v_ctx::scan_fileset() +{ + findFILESET *fileset; + + check_size = check_chksum = false; + digesttype = CRYPTO_DIGEST_NONE; + + if (!jcr->ff || !jcr->ff->fileset) { + return; + } + + fileset = jcr->ff->fileset; + + for (int i=0; iinclude_list.size(); i++) { + findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i); + + for (int j=0; jopts_list.size(); j++) { + findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j); + check_size = (strchr(fo->VerifyOpts, 's') != NULL); + if ((strchr(fo->VerifyOpts, '1') != NULL) || + (strchr(fo->VerifyOpts, '5') != NULL)) + { + check_chksum = true; + } + + if (fo->flags & FO_MD5) { + digesttype = CRYPTO_DIGEST_MD5; + return; + } + if (fo->flags & FO_SHA1) { + digesttype = CRYPTO_DIGEST_SHA1; + return; + } + if (fo->flags & FO_SHA256) { + digesttype = CRYPTO_DIGEST_SHA256; + return; + } + if (fo->flags & FO_SHA512) { + digesttype = CRYPTO_DIGEST_SHA512; + return; + } + } + } + digesttype = CRYPTO_DIGEST_NONE; + if (check_chksum) { + Jmsg(jcr, M_WARNING, 0, _("Checksum verification required in Verify FileSet option, but no Signature found in the FileSet\n")); + check_chksum = false; + } +} + +/* Compute the file size for sparse records and adjust the data */ +void v_ctx::skip_sparse_header(char **data, uint32_t *length) +{ + unser_declare; + uint64_t faddr; + unser_begin(*data, OFFSET_FADDR_SIZE); + unser_uint64(faddr); + + /* For sparse, we assume that the file is at least big as faddr */ + set_size(faddr); + + *data += OFFSET_FADDR_SIZE; + *length -= OFFSET_FADDR_SIZE; +} + +/* + * If extracting, close any previous stream + */ +bool v_ctx::close_previous_stream() +{ + bool rtn = true; + uint8_t buf[CRYPTO_DIGEST_MAX_SIZE]; + uint32_t len = CRYPTO_DIGEST_MAX_SIZE; + char ed1[50], ed2[50]; + + /* Reset the win32 filter that strips header stream out of the file */ + win32filter.init(); + + /* Check the size if possible */ + if (check_size && size >= 0) { + if (attr->type == FT_REG && size != attr->statp.st_size) { + Dmsg1(50, "Size comparison failed for %s\n", jcr->last_fname); + Jmsg(jcr, M_INFO, 0, + _(" st_size differs on \"%s\". Vol: %s File: %s\n"), + jcr->last_fname, + edit_int64(size, ed1), + edit_int64((int64_t)attr->statp.st_size, ed2)); + jcr->setJobStatus(JS_Differences); + } + reset_size(); + } + + /* Compute the digest and store it */ + *digest = 0; + if (jcr->crypto.digest) { + if (!crypto_digest_finalize(jcr->crypto.digest, buf, &len)) { + Dmsg1(50, "Unable to finalize digest for %s\n", jcr->last_fname); + rtn = false; + + } else { + bin_to_base64(digest, sizeof(digest), (char *)buf, len, true); + } + crypto_digest_free(jcr->crypto.digest); + jcr->crypto.digest = NULL; + } + return rtn; +} /* - * Verify attributes of the requested files on the Volume + * Verify attributes or data of the requested files on the Volume * */ void do_verify_volume(JCR *jcr) { BSOCK *sd, *dir; - POOLMEM *fname; /* original file name */ - POOLMEM *lname; /* link name */ - int32_t stream, full_stream; uint32_t size; uint32_t VolSessionId, VolSessionTime, file_index; - uint32_t record_file_index; char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)]; - int type, stat; + int stat; + char *wbuf; /* write buffer */ + uint32_t wsize; /* write size */ + uint32_t rsize; /* read size */ + bool msg_encrypt = false; + bool do_chksum; + v_ctx vctx(jcr); + ATTR *attr = vctx.attr; sd = jcr->store_bsock; if (!sd) { @@ -74,8 +268,12 @@ void do_verify_volume(JCR *jcr) } jcr->buf_size = sd->msglen; - fname = get_pool_memory(PM_FNAME); - lname = get_pool_memory(PM_FNAME); + /* use the same buffer size to decompress both gzip and lzo */ + if (have_libz || have_lzo) { + uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100; + jcr->compress_buf = get_memory(compress_buf_size); + jcr->compress_buf_size = compress_buf_size; + } GetMsg *fdmsg = New(GetMsg(jcr, sd, rec_header, GETMSG_MAX_MSG_SIZE)); fdmsg->start_read_sock(); @@ -85,16 +283,20 @@ void do_verify_volume(JCR *jcr) * Get a record from the Storage daemon */ while (fdmsg->bget_msg(&bmsg) >= 0 && !job_canceled(jcr)) { + /* Remember previous stream type */ + vctx.prev_stream = vctx.stream; + /* * First we expect a Stream Record Header */ if (sscanf(bmsg->rbuf, rec_header, &VolSessionId, &VolSessionTime, &file_index, - &full_stream, &size) != 5) { + &vctx.full_stream, &size) != 5) { Jmsg1(jcr, M_FATAL, 0, _("Record header scan error: %s\n"), bmsg->rbuf); goto bail_out; } - stream = full_stream & STREAMMASK_TYPE; - Dmsg4(30, "Got hdr: FilInx=%d FullStream=%d Stream=%d size=%d.\n", file_index, full_stream, stream, size); + vctx.stream = vctx.full_stream & STREAMMASK_TYPE; + Dmsg4(30, "Got hdr: FilInx=%d FullStream=%d Stream=%d size=%d.\n", + file_index, vctx.full_stream, vctx.stream, size); /* * Now we expect the Stream Data @@ -107,152 +309,239 @@ void do_verify_volume(JCR *jcr) Jmsg2(jcr, M_FATAL, 0, _("Actual data size %d not same as header %d\n"), bmsg->origlen, size); goto bail_out; } - Dmsg2(30, "Got stream data %s, len=%d\n", stream_to_ascii(stream), bmsg->rbuflen); + Dmsg2(30, "Got stream data %s, len=%d\n", stream_to_ascii(vctx.stream), bmsg->rbuflen); /* File Attributes stream */ - switch (stream) { + switch (vctx.stream) { case STREAM_UNIX_ATTRIBUTES: case STREAM_UNIX_ATTRIBUTES_EX: - char *ap, *lp, *fp; - Dmsg0(400, "Stream=Unix Attributes.\n"); - - if ((int)sizeof_pool_memory(fname) < bmsg->rbuflen) { - fname = realloc_pool_memory(fname, bmsg->rbuflen + 1); - } - - if ((int)sizeof_pool_memory(lname) < bmsg->rbuflen) { - lname = realloc_pool_memory(lname, bmsg->rbuflen + 1); + if (!vctx.close_previous_stream()) { + goto bail_out; } - *fname = 0; - *lname = 0; - /* - * An Attributes record consists of: - * File_index - * Type (FT_types) - * Filename - * Attributes - * Link name (if file linked i.e. FT_LNK) - * Extended Attributes (if Win32) + * Unpack attributes and do sanity check them */ - if (sscanf(bmsg->rbuf, "%d %d", &record_file_index, &type) != 2) { - Jmsg(jcr, M_FATAL, 0, _("Error scanning record header: %s\n"), bmsg->rbuf); - Dmsg0(0, "\nError scanning header\n"); + if (!unpack_attributes_record(jcr, vctx.stream, + bmsg->rbuf, bmsg->rbuflen, attr)) { goto bail_out; } - Dmsg2(30, "Got Attr: FilInx=%d type=%d\n", record_file_index, type); - ap = bmsg->rbuf; - while (*ap++ != ' ') /* skip record file index */ - ; - while (*ap++ != ' ') /* skip type */ - ; - /* Save filename and position to attributes */ - fp = fname; - while (*ap != 0) { - *fp++ = *ap++; /* copy filename to fname */ - } - *fp = *ap++; /* terminate filename & point to attribs */ - - Dmsg2(100, "File=%s Attr=%s\n", fname, ap); - /* Skip to Link name */ - if (type == FT_LNK || type == FT_LNKSAVED) { - lp = ap; - while (*lp++ != 0) { - ; - } - pm_strcat(lname, lp); /* "save" link name */ - } else { - *lname = 0; - } + + attr->data_stream = decode_stat(attr->attr, &attr->statp, + sizeof(attr->statp), &attr->LinkFI); + jcr->lock(); jcr->JobFiles++; jcr->num_files_examined++; - pm_strcpy(jcr->last_fname, fname); /* last file examined */ + pm_strcpy(jcr->last_fname, attr->fname); /* last file examined */ jcr->unlock(); + if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) { + /* + * Send file attributes to Director + * File_index + * Stream + * Verify Options + * Filename (full path) + * Encoded attributes + * Link name (if type==FT_LNK) + * For a directory, link is the same as fname, but with trailing + * slash. For a linked file, link is the link. + */ + /* Send file attributes to Director */ + Dmsg2(200, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, attr->fname); + if (attr->type == FT_LNK || attr->type == FT_LNKSAVED) { + stat = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, "pinsug5", attr->fname, + 0, attr->attr, 0, attr->lname, 0); + /* for a deleted record, we set fileindex=0 */ + } else if (attr->type == FT_DELETED) { + stat = dir->fsend("%d %d %s %s%c%s%c%c", 0, + STREAM_UNIX_ATTRIBUTES, "pinsug5", attr->fname, + 0, attr->attr, 0, 0); + } else { + stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, "pinsug5", attr->fname, + 0, attr->attr, 0, 0); + } + Dmsg2(200, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg); + if (!stat) { + Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), dir->bstrerror()); + goto bail_out; + } + } + break; + /* - * Send file attributes to Director - * File_index - * Stream - * Verify Options - * Filename (full path) - * Encoded attributes - * Link name (if type==FT_LNK) - * For a directory, link is the same as fname, but with trailing - * slash. For a linked file, link is the link. + * Restore stream object is counted, but not restored here */ - /* Send file attributes to Director */ - Dmsg2(200, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, fname); - if (type == FT_LNK || type == FT_LNKSAVED) { - stat = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, "pinsug5", fname, - 0, ap, 0, lname, 0); - /* for a deleted record, we set fileindex=0 */ - } else if (type == FT_DELETED) { - stat = dir->fsend("%d %d %s %s%c%s%c%c", 0, - STREAM_UNIX_ATTRIBUTES, "pinsug5", fname, - 0, ap, 0, 0); - } else { - stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, "pinsug5", fname, - 0, ap, 0, 0); - } - Dmsg2(200, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg); - if (!stat) { - Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), dir->bstrerror()); - goto bail_out; - } + case STREAM_RESTORE_OBJECT: + jcr->lock(); + jcr->JobFiles++; + jcr->num_files_examined++; + jcr->unlock(); break; + default: + break; + } + + const char *digest_code = NULL; + + switch(vctx.stream) { case STREAM_MD5_DIGEST: bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_MD5_SIZE, true); - Dmsg2(400, "send inx=%d MD5=%s\n", jcr->JobFiles, digest); - dir->fsend("%d %d %s *MD5-%d*", jcr->JobFiles, STREAM_MD5_DIGEST, digest, - jcr->JobFiles); - Dmsg2(20, "bfiled>bdird: MD5 len=%d: msg=%s\n", dir->msglen, dir->msg); + digest_code = "MD5"; break; case STREAM_SHA1_DIGEST: bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_SHA1_SIZE, true); - Dmsg2(400, "send inx=%d SHA1=%s\n", jcr->JobFiles, digest); - dir->fsend("%d %d %s *SHA1-%d*", jcr->JobFiles, STREAM_SHA1_DIGEST, - digest, jcr->JobFiles); - Dmsg2(20, "bfiled>bdird: SHA1 len=%d: msg=%s\n", dir->msglen, dir->msg); + digest_code = "SHA1"; break; case STREAM_SHA256_DIGEST: bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_SHA256_SIZE, true); - Dmsg2(400, "send inx=%d SHA256=%s\n", jcr->JobFiles, digest); - dir->fsend("%d %d %s *SHA256-%d*", jcr->JobFiles, STREAM_SHA256_DIGEST, - digest, jcr->JobFiles); - Dmsg2(20, "bfiled>bdird: SHA256 len=%d: msg=%s\n", dir->msglen, dir->msg); + digest_code = "SHA256"; break; case STREAM_SHA512_DIGEST: bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_SHA512_SIZE, true); - Dmsg2(400, "send inx=%d SHA512=%s\n", jcr->JobFiles, digest); - dir->fsend("%d %d %s *SHA512-%d*", jcr->JobFiles, STREAM_SHA512_DIGEST, - digest, jcr->JobFiles); - Dmsg2(20, "bfiled>bdird: SHA512 len=%d: msg=%s\n", dir->msglen, dir->msg); - break; - - /* - * Restore stream object is counted, but not restored here - */ - case STREAM_RESTORE_OBJECT: - jcr->lock(); - jcr->JobFiles++; - jcr->num_files_examined++; - jcr->unlock(); + digest_code = "SHA512"; break; - /* Ignore everything else */ default: + *digest = 0; break; + } + + if (digest_code && jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) { + dir->fsend("%d %d %s *%s-%d*", jcr->JobFiles, vctx.stream, + digest, digest_code, jcr->JobFiles); + + } else if (jcr->getJobLevel() == L_VERIFY_DATA) { + + /* Compare digest */ + if (vctx.check_chksum && *digest) { + /* probably an empty file, we can create an empty crypto session */ + if (!jcr->crypto.digest) { + jcr->crypto.digest = crypto_digest_new(jcr, vctx.digesttype); + } + vctx.close_previous_stream(); + if (strncmp(digest, vctx.digest, + MIN(sizeof(digest), sizeof(vctx.digest))) != 0) + { + Jmsg(jcr, M_INFO, 0, + _(" %s differs on \"%s\". File=%s Vol=%s\n"), + stream_to_ascii(vctx.stream), jcr->last_fname, + vctx.digest, digest); + jcr->setJobStatus(JS_Differences); + Dmsg3(50, "Signature verification failed for %s %s != %s\n", + jcr->last_fname, digest, vctx.digest); + } + } + + /* Compute size and checksum for level=Data */ + switch (vctx.stream) { + case STREAM_ENCRYPTED_FILE_DATA: + case STREAM_ENCRYPTED_WIN32_DATA: + case STREAM_ENCRYPTED_FILE_GZIP_DATA: + case STREAM_ENCRYPTED_WIN32_GZIP_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: + if (!msg_encrypt) { + Jmsg(jcr, M_WARNING, 0, + _("Verification of encrypted file data is not supported.\n")); + msg_encrypt = true; + } + break; + + case STREAM_PLUGIN_DATA: + case STREAM_FILE_DATA: + case STREAM_SPARSE_DATA: + case STREAM_WIN32_DATA: + case STREAM_GZIP_DATA: + case STREAM_SPARSE_GZIP_DATA: + case STREAM_WIN32_GZIP_DATA: + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: + do_chksum=true; + if (!(attr->type == FT_RAW || attr->type == FT_FIFO || attr->type == FT_REG)) { + break; + } + wbuf = bmsg->rbuf; + rsize = bmsg->rbuflen; + jcr->ReadBytes += rsize; + wsize = rsize; + + if (vctx.stream == STREAM_SPARSE_DATA + || vctx.stream == STREAM_SPARSE_COMPRESSED_DATA + || vctx.stream == STREAM_SPARSE_GZIP_DATA) { + vctx.skip_sparse_header(&wbuf, &wsize); + } + + /* On Windows, the checksum is computed after the compression + * On Unix, the checksum is computed before the compression + */ + if (vctx.stream == STREAM_WIN32_GZIP_DATA + || vctx.stream == STREAM_WIN32_DATA + || vctx.stream == STREAM_WIN32_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) + { + do_chksum = false; + vctx.update_checksum(wbuf, wsize); + } + + if (vctx.stream == STREAM_GZIP_DATA + || vctx.stream == STREAM_SPARSE_GZIP_DATA + || vctx.stream == STREAM_WIN32_GZIP_DATA + || vctx.stream == STREAM_ENCRYPTED_FILE_GZIP_DATA + || vctx.stream == STREAM_COMPRESSED_DATA + || vctx.stream == STREAM_SPARSE_COMPRESSED_DATA + || vctx.stream == STREAM_WIN32_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) { + + if (!decompress_data(jcr, vctx.stream, &wbuf, &wsize)) { + dequeue_messages(jcr); + goto bail_out; + } + } + + /* Unix way to deal with checksums */ + if (do_chksum) { + vctx.update_checksum(wbuf, wsize); + } + + if (vctx.stream == STREAM_WIN32_GZIP_DATA + || vctx.stream == STREAM_WIN32_DATA + || vctx.stream == STREAM_WIN32_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA + || vctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) { + + int64_t wbuf_len = wsize; + int64_t wsize64 = 0; + if (vctx.win32filter.have_data(&wbuf, &wbuf_len, &wsize64)) { + wsize = wsize64; + } + } + jcr->JobBytes += wsize; + vctx.update_size(wsize); + break; + + /* TODO: Handle data to compute checksums */ + /* Ignore everything else */ + default: + break; + } } /* end switch */ } /* end while bnet_get */ + if (!vctx.close_previous_stream()) { + goto bail_out; + } jcr->setJobStatus(JS_Terminated); goto ok_out; @@ -264,13 +553,11 @@ ok_out: fdmsg->wait_read_sock(); delete bmsg; free_GetMsg(fdmsg); - if (jcr->compress_buf) { - free(jcr->compress_buf); + free_pool_memory(jcr->compress_buf); jcr->compress_buf = NULL; } - free_pool_memory(fname); - free_pool_memory(lname); - Dmsg2(050, "End Verify-Vol. Files=%d Bytes=%" lld "\n", jcr->JobFiles, + /* TODO: We probably want to mark the job as failed if we have errors */ + Dmsg2(50, "End Verify-Vol. Files=%d Bytes=%" lld "\n", jcr->JobFiles, jcr->JobBytes); } diff --git a/bacula/src/findlib/Makefile.in b/bacula/src/findlib/Makefile.in index 7370881b39..1b47d900e9 100644 --- a/bacula/src/findlib/Makefile.in +++ b/bacula/src/findlib/Makefile.in @@ -25,12 +25,12 @@ dummy: # # include files installed when using libtool # -INCLUDE_FILES = bfile.h find.h protos.h +INCLUDE_FILES = bfile.h find.h protos.h win32filter.h # LIBBACFIND_SRCS = find.c match.c find_one.c attribs.c create_file.c \ bfile.c drivetype.c enable_priv.c fstype.c mkpath.c \ - savecwd.c + savecwd.c win32filter.c LIBBACFIND_OBJS = $(LIBBACFIND_SRCS:.c=.o) LIBBACFIND_LOBJS = $(LIBBACFIND_SRCS:.c=.lo) diff --git a/bacula/src/findlib/win32filter.c b/bacula/src/findlib/win32filter.c new file mode 100644 index 0000000000..f6533f3c6c --- /dev/null +++ b/bacula/src/findlib/win32filter.c @@ -0,0 +1,96 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2015 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#include "win32filter.h" + +#define WIN32_STREAM_HEADER_SIZE 20 /* the size of the WIN32_STREAM_ID header without the name */ + +/* search in a record of a STREAM_WIN32_DATA for the true data + * when found: return true, '*raw' is set at the beginning of the data + * and *use_len is the length of data to read. + * *raw_len is decremented and contains the amount of data that as not + * been filtered yet. + * For this STREAM_WIN32_DATA, you can call have_data() only one + * per record. + * If the stream where the data is can be spread all around the stream + * you must call have_data() until *raw_len is zero and increment + * *data before the next call. + */ +bool Win32Filter::have_data(char **raw, int64_t *raw_len, int64_t *use_len) +{ + int64_t size; + char *orig=*raw; + Dmsg1(100, "have_data(%lld)\n", *raw_len); + while (*raw_len > 0) { + /* In this rec, we could have multiple streams of data and headers + * to handle before to reach the data, then we must iterate + */ + + Dmsg4(100, "s off=%lld len=%lld skip_size=%lld data_size=%lld\n", *raw-orig, *raw_len, skip_size, data_size); + if (skip_size > 0) { + /* skip what the previous header told us to skip */ + size = *raw_len < skip_size ? *raw_len : skip_size; + skip_size -= size; + *raw_len -= size; + *raw += size; + } + + Dmsg4(100, "h off=%lld len=%lld skip_size=%lld data_size=%lld\n", *raw-orig, *raw_len, skip_size, data_size); + if (data_size == 0 && skip_size == 0 && *raw_len > 0) { + /* read a WIN32_STREAM header, merge it with the part that was read + * from the previous record, if any, if the header was split across + * 2 records. + */ + size = WIN32_STREAM_HEADER_SIZE - header_pos; + if (*raw_len < size) { + size = *raw_len; + } + memcpy((char *)&header + header_pos, *raw, size); + header_pos += size; + *raw_len -= size; + *raw += size; + if (header_pos == WIN32_STREAM_HEADER_SIZE) { + Dmsg5(100, "header pos=%d size=%lld name_size=%d len=%lld StreamId=0x%x\n", header_pos, size, + header.dwStreamNameSize, header.Size, header.dwStreamId); + header_pos = 0; + skip_size = header.dwStreamNameSize; /* skip the name of the stream */ + if (header.dwStreamId == WIN32_BACKUP_DATA) { + data_size = header.Size; + } else { + skip_size += header.Size; /* skip the all stream */ + } + } + Dmsg4(100, "H off=%lld len=%lld skip_size=%lld data_size=%lld\n", *raw-orig, *raw_len, skip_size, data_size); + } + + Dmsg4(100, "d off=%lld len=%lld skip_size=%lld data_size=%lld\n", *raw - orig, *raw_len, skip_size, data_size); + if (data_size > 0 && skip_size == 0 && *raw_len > 0) { + /* some data to read */ + size = *raw_len < data_size ? *raw_len : data_size; + data_size -= size; + *raw_len -= size; + *use_len = size; + Dmsg5(100, "D off=%lld len=%lld use_len=%lld skip_size=%lld data_size=%lld\n", *raw-orig, *raw_len, + *use_len, skip_size, data_size); + return true; + } + } + + return false; +} diff --git a/bacula/src/findlib/win32filter.h b/bacula/src/findlib/win32filter.h new file mode 100644 index 0000000000..048ce9406a --- /dev/null +++ b/bacula/src/findlib/win32filter.h @@ -0,0 +1,59 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2015 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef WIN32FILTER_H +#define WIN32FILTER_H + +#include "bacula.h" +#include "bfile.h" /* for BWIN32_STREAM_ID */ + +class Win32Filter +{ +public: + int64_t skip_size; /* how many bytes we have to skip before next header */ + int64_t data_size; /* how many data are expected in the stream */ + int header_pos; /* the part of the header that was filled in by previous record */ + + BWIN32_STREAM_ID header; + + Win32Filter() { + init(); + }; + + void init() { + skip_size = 0; + data_size = 0; + header_pos = 0; + }; + + void copy(Win32Filter *f) + { + skip_size = f->skip_size; + data_size = f->data_size; + header_pos = f->header_pos; + header = f->header; + }; + + /* If the stream is HHHDDDDD, you can call have_data("HHHDDDDD", 8, ) + * and it will return "DDDDD", 0, 5 + */ + bool have_data(char **raw, int64_t *raw_len, int64_t *data_len); +}; + +#endif