X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Ffiled%2Fbackup.c;h=60beddb80c8e233f3062f3e8ee3d0e0825f02545;hb=7fc5fe1b10047210db9c20ccb415d9ac87f6eee1;hp=9525a0579cfce14c424272dbcbd68897cefdf2f5;hpb=f4a719b5d8214c03b12e962adc7558ffdbdc4edb;p=bacula%2Fbacula diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 9525a0579c..60beddb80c 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -1,12 +1,12 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2008 Free Software Foundation Europe e.V. + Copyright (C) 2000-2010 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public + modify it under the terms of version three of the GNU Affero General Public License as published by the Free Software Foundation and included in the file LICENSE. @@ -15,7 +15,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Affero General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. @@ -25,45 +25,45 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ -/* +/** * Bacula File Daemon backup.c send file attributes and data * to the Storage daemon. * * Kern Sibbald, March MM * - * Version $Id$ - * */ #include "bacula.h" #include "filed.h" +#ifdef HAVE_DARWIN_OS +const bool have_darwin_os = true; +#else +const bool have_darwin_os = false; +#endif + +#if defined(HAVE_ACL) +const bool have_acl = true; +#else +const bool have_acl = false; +#endif + +#if defined(HAVE_XATTR) +const bool have_xattr = true; +#else +const bool have_xattr = false; +#endif + /* Forward referenced functions */ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level); static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest); bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream); -static bool read_and_send_acl(JCR *jcr, int acltype, int stream); static bool crypto_session_start(JCR *jcr); static void crypto_session_end(JCR *jcr); static bool crypto_session_send(JCR *jcr, BSOCK *sd); +static void close_vss_backup_session(JCR *jcr); -/* - * check for BSD nodump flag - */ -static bool no_dump(JCR *jcr, FF_PKT *ff_pkt) -{ -#if defined(HAVE_CHFLAGS) && defined(UF_NODUMP) - if ( (ff_pkt->flags & FO_HONOR_NODUMP) && - (ff_pkt->statp.st_flags & UF_NODUMP) ) { - Jmsg(jcr, M_INFO, 1, _(" NODUMP flag set - will not process %s\n"), - ff_pkt->fname); - return true; /* do not backup this file */ - } -#endif - return false; /* do backup */ -} - -/* +/** * Find all the requested files and send them * to the Storage daemon. * @@ -103,7 +103,8 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) } jcr->buf_size = sd->msglen; - /* Adjust for compression so that output buffer is + /** + * Adjust for compression so that output buffer is * 12 bytes + 0.1% larger than input buffer plus 18 bytes. * This gives a bit extra plus room for the sparse addr if any. * Note, we adjust the read size to be smaller so that the @@ -138,29 +139,58 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime); - /* in accurate mode, we overwrite the find_one check function */ + /** in accurate mode, we overwrite the find_one check function */ if (jcr->accurate) { set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file); } start_heartbeat_monitor(jcr); - jcr->acl_text = get_pool_memory(PM_MESSAGE); + if (have_acl) { + jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t)); + memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t)); + jcr->acl_data->content = get_pool_memory(PM_MESSAGE); + } + + if (have_xattr) { + jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t)); + memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t)); + jcr->xattr_data->content = get_pool_memory(PM_MESSAGE); + } - /* Subroutine save_file() is called for each file */ + /** Subroutine save_file() is called for each file */ if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) { ok = false; /* error */ set_jcr_job_status(jcr, JS_ErrorTerminated); } - accurate_send_deleted_list(jcr); /* send deleted list to SD */ + if (have_acl && jcr->acl_data->nr_errors > 0) { + Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"), + jcr->acl_data->nr_errors); + } + if (have_xattr && jcr->xattr_data->nr_errors > 0) { + Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"), + jcr->xattr_data->nr_errors); + } + + close_vss_backup_session(jcr); - free_pool_memory(jcr->acl_text); + accurate_finish(jcr); /* send deleted or base file list to SD */ stop_heartbeat_monitor(jcr); sd->signal(BNET_EOD); /* end of sending data */ + if (have_acl && jcr->acl_data) { + free_pool_memory(jcr->acl_data->content); + free(jcr->acl_data); + jcr->acl_data = NULL; + } + if (have_xattr && jcr->xattr_data) { + free_pool_memory(jcr->xattr_data->content); + free(jcr->xattr_data); + jcr->xattr_data = NULL; + } if (jcr->big_buf) { free(jcr->big_buf); jcr->big_buf = NULL; @@ -188,7 +218,7 @@ static bool crypto_session_start(JCR *jcr) { crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC; - /* + /** * Create encryption session data and a cached, DER-encoded session data * structure. We use a single session key for each backup, so we'll encode * the session data only once. @@ -196,28 +226,28 @@ static bool crypto_session_start(JCR *jcr) if (jcr->crypto.pki_encrypt) { uint32_t size = 0; - /* Create per-job session encryption context */ + /** Create per-job session encryption context */ jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients); - /* Get the session data size */ + /** Get the session data size */ if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n")); return false; } - /* Allocate buffer */ + /** Allocate buffer */ jcr->crypto.pki_session_encoded = get_memory(size); - /* Encode session data */ + /** Encode session data */ if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n")); return false; } - /* ... and store the encoded size */ + /** ... and store the encoded size */ jcr->crypto.pki_session_encoded_size = size; - /* Allocate the encryption/decryption buffer */ + /** Allocate the encryption/decryption buffer */ jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE); } return true; @@ -242,7 +272,7 @@ static bool crypto_session_send(JCR *jcr, BSOCK *sd) { POOLMEM *msgsave; - /* Send our header */ + /** Send our header */ Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA); sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA); @@ -259,7 +289,7 @@ static bool crypto_session_send(JCR *jcr, BSOCK *sd) } -/* +/** * Called here by find() for each file included. * This is a callback. The original is find_files() above. * @@ -272,6 +302,7 @@ static bool crypto_session_send(JCR *jcr, BSOCK *sd) int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) { bool do_read = false; + bool plugin_started = false; int stat, data_stream; int rtnstat = 0; DIGEST *digest = NULL; @@ -287,7 +318,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) #endif BSOCK *sd = jcr->store_bsock; - if (job_canceled(jcr)) { + if (jcr->is_job_canceled()) { return 0; } @@ -299,23 +330,20 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) break; case FT_REGE: Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname); - if (no_dump(jcr, ff_pkt)) - return 1; has_file_data = true; break; case FT_REG: Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname); - if (no_dump(jcr, ff_pkt)) - return 1; has_file_data = true; break; case FT_LNK: Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link); break; + case FT_RESTORE_FIRST: + Dmsg1(100, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname); + break; case FT_DIRBEGIN: jcr->num_files_examined--; /* correct file count */ - if (no_dump(jcr, ff_pkt)) /* disable recursion on nodump directories */ - ff_pkt->flags |= FO_NO_RECURSION; return 1; /* not used */ case FT_NORECURSE: Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"), @@ -325,8 +353,13 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) case FT_NOFSCHG: /* Suppress message for /dev filesystems */ if (!is_in_fileset(ff_pkt)) { - Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"), - ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname); +#ifdef HAVE_WIN32 + Jmsg(jcr, M_INFO, 1, _(" %s is a junction point or a different filesystem. Will not descend from %s into it.\n"), + ff_pkt->fname, ff_pkt->top_fname); +#else + Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into it.\n"), + ff_pkt->fname, ff_pkt->top_fname); +#endif } ff_pkt->type = FT_DIREND; /* Backup only the directory entry */ break; @@ -361,21 +394,21 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) berrno be; Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } case FT_NOFOLLOW: { berrno be; Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } case FT_NOSTAT: { berrno be; Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } case FT_DIRNOCHG: @@ -389,21 +422,21 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) berrno be; Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } default: Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname); - jcr->Errors++; + jcr->JobErrors++; return 1; } Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname); - /* Digests and encryption are only useful if there's file data */ + /** Digests and encryption are only useful if there's file data */ if (has_file_data) { - /* + /** * Setup for digest handling. If this fails, the digest will be set to NULL * and not used. Note, the digest (file hash) can be any one of the four * algorithms below. @@ -431,60 +464,68 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) digest_stream = STREAM_SHA512_DIGEST; } - /* Did digest initialization fail? */ + /** Did digest initialization fail? */ if (digest_stream != STREAM_NONE && digest == NULL) { Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"), stream_to_ascii(digest_stream)); } - /* - * Set up signature digest handling. If this fails, the signature digest will be set to - * NULL and not used. + /** + * Set up signature digest handling. If this fails, the signature digest + * will be set to NULL and not used. + */ + /* TODO landonf: We should really only calculate the digest once, for + * both verification and signing. */ - // TODO landonf: We should really only calculate the digest once, for both verification and signing. if (jcr->crypto.pki_sign) { signing_digest = crypto_digest_new(jcr, signing_algorithm); - /* Full-stop if a failure occurred initializing the signature digest */ + /** Full-stop if a failure occurred initializing the signature digest */ if (signing_digest == NULL) { Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"), stream_to_ascii(signing_algorithm)); - jcr->Errors++; + jcr->JobErrors++; goto good_rtn; } } - /* Enable encryption */ + /** Enable encryption */ if (jcr->crypto.pki_encrypt) { ff_pkt->flags |= FO_ENCRYPT; } } - /* Initialize the file descriptor we use for data and other streams. */ + /** Initialize the file descriptor we use for data and other streams. */ binit(&ff_pkt->bfd); if (ff_pkt->flags & FO_PORTABLE) { set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */ } if (ff_pkt->cmd_plugin) { + /* Tell bfile that it needs to call plugin */ if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) { goto bail_out; } send_plugin_name(jcr, sd, true); /* signal start of plugin data */ + plugin_started = true; } - /* Send attributes -- must be done after binit() */ + /** Send attributes -- must be done after binit() */ if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) { goto bail_out; } + /** Meta data only for restore object */ + if (ff_pkt->type == FT_RESTORE_FIRST) { + goto good_rtn; + } - /* Set up the encryption context and send the session data to the SD */ + /** Set up the encryption context and send the session data to the SD */ if (has_file_data && jcr->crypto.pki_encrypt) { if (!crypto_session_send(jcr, sd)) { goto bail_out; } } - /* + /** * Open any file with data that we intend to save, then save it. * * Note, if is_win32_backup, we must open the Directory so that @@ -501,11 +542,12 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) { do_read = true; } + if (ff_pkt->cmd_plugin) { do_read = true; } - Dmsg1(400, "do_read=%d\n", do_read); + Dmsg2(150, "type=%d do_read=%d\n", ff_pkt->type, do_read); if (do_read) { btimer_t *tid; @@ -521,7 +563,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) berrno be; Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname, be.bstrerror()); - jcr->Errors++; + jcr->JobErrors++; if (tid) { stop_thread_timer(tid); tid = NULL; @@ -546,69 +588,104 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) } } -#ifdef HAVE_DARWIN_OS - /* Regular files can have resource forks and Finder Info */ - if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && - ff_pkt->flags & FO_HFSPLUS)) { - if (ff_pkt->hfsinfo.rsrclength > 0) { - int flags; - int rsrc_stream; - if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { - ff_pkt->ff_errno = errno; - berrno be; - Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"), - ff_pkt->fname, be.bstrerror()); - jcr->Errors++; - if (is_bopen(&ff_pkt->bfd)) { - bclose(&ff_pkt->bfd); + if (have_darwin_os) { + /** Regular files can have resource forks and Finder Info */ + if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && + ff_pkt->flags & FO_HFSPLUS)) { + if (ff_pkt->hfsinfo.rsrclength > 0) { + int flags; + int rsrc_stream; + if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { + ff_pkt->ff_errno = errno; + berrno be; + Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"), + ff_pkt->fname, be.bstrerror()); + jcr->JobErrors++; + if (is_bopen(&ff_pkt->bfd)) { + bclose(&ff_pkt->bfd); + } + goto good_rtn; + } + flags = ff_pkt->flags; + ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE); + if (flags & FO_ENCRYPT) { + rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA; + } else { + rsrc_stream = STREAM_MACOS_FORK_DATA; + } + stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest); + ff_pkt->flags = flags; + bclose(&ff_pkt->bfd); + if (!stat) { + goto bail_out; } - goto good_rtn; } - flags = ff_pkt->flags; - ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE); - if (flags & FO_ENCRYPT) { - rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA; - } else { - rsrc_stream = STREAM_MACOS_FORK_DATA; + + Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname); + sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES); + Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); + pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32); + sd->msglen = 32; + if (digest) { + crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen); } - stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest); - ff_pkt->flags = flags; - bclose(&ff_pkt->bfd); - if (!stat) { - goto bail_out; + if (signing_digest) { + crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen); } + sd->send(); + sd->signal(BNET_EOD); } + } - Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname); - sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES); - Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); - memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32); - sd->msglen = 32; - if (digest) { - crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen); - } - if (signing_digest) { - crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen); + /** + * Save ACLs when requested and available for anything not being a symlink and not being a plugin. + */ + if (have_acl) { + if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) { + switch (build_acl_streams(jcr, ff_pkt)) { + case bacl_exit_fatal: + goto bail_out; + case bacl_exit_error: + /** + * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB + * print the error message set by the lower level routine in jcr->errmsg. + */ + if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) { + Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg); + } + jcr->acl_data->nr_errors++; + break; + case bacl_exit_ok: + break; + } } - sd->send(); - sd->signal(BNET_EOD); } -#endif - if (ff_pkt->flags & FO_ACL) { - /* Read access ACLs for files, dirs and links */ - if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ACCESS_ACL)) { - goto bail_out; - } - /* Directories can have default ACLs too */ - if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) { - if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_DEFAULT_ACL)) { + /** + * Save Extended Attributes when requested and available for all files not being a plugin. + */ + if (have_xattr) { + if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) { + switch (build_xattr_streams(jcr, ff_pkt)) { + case bxattr_exit_fatal: goto bail_out; + case bxattr_exit_error: + /** + * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB + * print the error message set by the lower level routine in jcr->errmsg. + */ + if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) { + Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg); + } + jcr->xattr_data->nr_errors++; + break; + case bxattr_exit_ok: + break; } } } - /* Terminate the signing digest and send it to the Storage daemon */ + /** Terminate the signing digest and send it to the Storage daemon */ if (signing_digest) { uint32_t size = 0; @@ -622,22 +699,22 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) goto bail_out; } - /* Get signature size */ + /** Get signature size */ if (!crypto_sign_encode(sig, NULL, &size)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n")); goto bail_out; } - /* Grow the bsock buffer to fit our message if necessary */ + /** Grow the bsock buffer to fit our message if necessary */ if (sizeof_pool_memory(sd->msg) < (int32_t)size) { sd->msg = realloc_pool_memory(sd->msg, size); } - /* Send our header */ - sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST); + /** Send our header */ + sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST); Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); - /* Encode signature data */ + /** Encode signature data */ if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n")); goto bail_out; @@ -648,7 +725,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) sd->signal(BNET_EOD); /* end of checksum */ } - /* Terminate any digest and send it to Storage daemon */ + /** Terminate any digest and send it to Storage daemon */ if (digest) { uint32_t size; @@ -657,7 +734,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) size = CRYPTO_DIGEST_MAX_SIZE; - /* Grow the bsock buffer to fit our message if necessary */ + /** Grow the bsock buffer to fit our message if necessary */ if (sizeof_pool_memory(sd->msg) < (int32_t)size) { sd->msg = realloc_pool_memory(sd->msg, size); } @@ -671,14 +748,14 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) sd->send(); sd->signal(BNET_EOD); /* end of checksum */ } - if (ff_pkt->cmd_plugin) { - send_plugin_name(jcr, sd, false); /* signal end of plugin data */ - } good_rtn: rtnstat = 1; /* good return */ bail_out: + if (ff_pkt->cmd_plugin && plugin_started) { + send_plugin_name(jcr, sd, false); /* signal end of plugin data */ + } if (digest) { crypto_digest_free(digest); } @@ -691,7 +768,7 @@ bail_out: return rtnstat; } -/* +/** * Send data read from an already open file descriptor. * * We return 1 on sucess and 0 on errors. @@ -702,8 +779,8 @@ bail_out: * Currently this is not a problem as the only other stream, resource forks, * are not handled as sparse files. */ -int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, - DIGEST *signing_digest) +static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, + DIGEST *signing_digest) { BSOCK *sd = jcr->store_bsock; uint64_t fileAddr = 0; /* file address */ @@ -743,14 +820,14 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, wbuf = jcr->compress_buf; /* compressed output here */ cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ - /* + /** * Only change zlib parameters if there is no pending operation. * This should never happen as deflatereset is called after each * deflate. */ if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) { - /* set gzip compression level - must be done per file */ + /** set gzip compression level - must be done per file */ if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset, ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); @@ -768,7 +845,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n")); goto err; } - /* Allocate the cipher context */ + /** Allocate the cipher context */ if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true, &cipher_block_size)) == NULL) { /* Shouldn't happen! */ @@ -776,7 +853,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, goto err; } - /* + /** * Grow the crypto buffer, if necessary. * crypto_cipher_update() will buffer up to (cipher_block_size - 1). * We grow crypto_buf to the maximum number of blocks that @@ -790,18 +867,20 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */ } - /* + /** * Send Data header to Storage daemon * */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } Dmsg1(300, ">stored: datahdr %s\n", sd->msg); - /* + /** * Make space at beginning of buffer for fileAddr because this * same buffer will be used for writing if compression is off. */ @@ -809,7 +888,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, rbuf += SPARSE_FADDR_SIZE; rsize -= SPARSE_FADDR_SIZE; #ifdef HAVE_FREEBSD_OS - /* + /** * To read FreeBSD partitions, the read size must be * a multiple of 512. */ @@ -817,18 +896,18 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, #endif } - /* a RAW device read on win32 only works if the buffer is a multiple of 512 */ + /** a RAW device read on win32 only works if the buffer is a multiple of 512 */ #ifdef HAVE_WIN32 if (S_ISBLK(ff_pkt->statp.st_mode)) rsize = (rsize/512) * 512; #endif - /* + /** * Read the file data */ while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) { - /* Check for sparse blocks */ + /** Check for sparse blocks */ if (ff_pkt->flags & FO_SPARSE) { ser_declare; bool allZeros = false; @@ -839,12 +918,12 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, allZeros = is_buf_zero(rbuf, rsize); } if (!allZeros) { - /* Put file address as first data in buffer */ + /** Put file address as first data in buffer */ ser_begin(wbuf, SPARSE_FADDR_SIZE); ser_uint64(fileAddr); /* store fileAddr in begin of buffer */ } fileAddr += sd->msglen; /* update file address */ - /* Skip block of all zeros */ + /** Skip block of all zeros */ if (allZeros) { continue; /* skip block of zeros */ } @@ -852,21 +931,21 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, jcr->ReadBytes += sd->msglen; /* count bytes read */ - /* Uncompressed cipher input length */ + /** Uncompressed cipher input length */ cipher_input_len = sd->msglen; - /* Update checksum if requested */ + /** Update checksum if requested */ if (digest) { crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen); } - /* Update signing digest if requested */ + /** Update signing digest if requested */ if (signing_digest) { crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen); } #ifdef HAVE_LIBZ - /* Do compression if turned on */ + /** Do compression if turned on */ if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) { Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen); @@ -881,7 +960,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, goto err; } compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out; - /* reset zlib stream to be able to begin from scratch again */ + /** reset zlib stream to be able to begin from scratch again */ if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); @@ -895,7 +974,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, cipher_input_len = compress_len; } #endif - /* + /** * Note, here we prepend the current record length to the beginning * of the encrypted data. This is because both sparse and compression * restore handling want records returned to them with exactly the @@ -916,7 +995,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, cipher_input_len += SPARSE_FADDR_SIZE; } - /* Encrypt the length of the input block */ + /** Encrypt the length of the input block */ uint8_t packet_len[sizeof(uint32_t)]; ser_begin(packet_len, sizeof(uint32_t)); @@ -925,23 +1004,23 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len), (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) { - /* Encryption failed. Shouldn't happen. */ + /** Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } - /* Encrypt the input block */ + /** Encrypt the input block */ if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len, (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) { if ((initial_len + encrypted_len) == 0) { - /* No full block of data available, read more data */ + /** No full block of data available, read more data */ continue; } Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len, sd->msglen); sd->msglen = initial_len + encrypted_len; /* set encrypted length */ } else { - /* Encryption failed. Shouldn't happen. */ + /** Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } @@ -953,8 +1032,10 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, } sd->msg = wbuf; /* set correct write buffer */ if (!sd->send()) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); @@ -968,11 +1049,11 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, berrno be; Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno)); - if (jcr->Errors++ > 1000) { /* insanity check */ - Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n")); + if (jcr->JobErrors++ > 1000) { /* insanity check */ + Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors); } } else if (ff_pkt->flags & FO_ENCRYPT) { - /* + /** * For encryption, we must call finalize to push out any * buffered data. */ @@ -983,13 +1064,15 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, goto err; } - /* Note, on SSL pre-0.9.7, there is always some output */ + /** Note, on SSL pre-0.9.7, there is always some output */ if (encrypted_len > 0) { sd->msglen = encrypted_len; /* set encrypted length */ sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */ if (!sd->send()) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); @@ -999,19 +1082,21 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, } if (!sd->signal(BNET_EOD)) { /* indicate end of file data */ - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } - /* Free the cipher context */ + /** Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } return 1; err: - /* Free the cipher context */ + /** Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } @@ -1021,83 +1106,35 @@ err: return 0; } -/* - * Read and send an ACL for the last encountered file. - */ -static bool read_and_send_acl(JCR *jcr, int acltype, int stream) -{ -#ifdef HAVE_ACL - BSOCK *sd = jcr->store_bsock; - POOLMEM *msgsave; - int len; -#ifdef FD_NO_SEND_TEST - return true; -#endif - - len = bacl_get(jcr, acltype); - if (len < 0) { - Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname); - return true; - } - if (len == 0) { - return true; /* no ACL */ - } - - /* Send header */ - if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); - return false; - } - - /* Send the buffer to the storage deamon */ - Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text); - msgsave = sd->msg; - sd->msg = jcr->acl_text; - sd->msglen = len + 1; - if (!sd->send()) { - sd->msg = msgsave; - sd->msglen = 0; - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); - return false; - } - - jcr->JobBytes += sd->msglen; - sd->msg = msgsave; - if (!sd->signal(BNET_EOD)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); - return false; - } - - Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname); -#endif - return true; -} - bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream) { BSOCK *sd = jcr->store_bsock; char attribs[MAXSTRING]; - char attribsEx[MAXSTRING]; + char attribsExBuf[MAXSTRING]; + char *attribsEx = NULL; int attr_stream; - int stat; + int comp_len; + bool stat; #ifdef FD_NO_SEND_TEST return true; #endif Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname); - /* Find what data stream we will use, then encode the attributes */ + /** Find what data stream we will use, then encode the attributes */ if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) { /* This should not happen */ Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n")); return false; } - encode_stat(attribs, ff_pkt, data_stream); + encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream); - /* Now possibly extend the attributes */ - attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt); + /** Now possibly extend the attributes */ + if (ff_pkt->type == FT_RESTORE_FIRST) { + attr_stream = STREAM_RESTORE_OBJECT; + } else { + attribsEx = attribsExBuf; + attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt); + } Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx); @@ -1107,18 +1144,20 @@ bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream) pm_strcpy(jcr->last_fname, ff_pkt->fname); jcr->unlock(); - /* + /** * Send Attributes header to Storage daemon * */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } return false; } Dmsg1(300, ">stored: attrhdr %s\n", sd->msg); - /* + /** * Send file attributes to Storage daemon * File_index * File type @@ -1127,40 +1166,87 @@ bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream) * Link name (if type==FT_LNK or FT_LNKSAVED) * Encoded extended-attributes (for Win32) * + * or send Restore Object to Storage daemon + * File_index + * File_type + * Object_index + * Object_len (possibly compressed) + * Object_full_len (not compressed) + * Object_compression + * Plugin_name + * Object_name + * Binary Object data + * * For a directory, link is the same as fname, but with trailing * slash. For a linked file, link is the link. */ if (ff_pkt->type != FT_DELETED) { /* already stripped */ strip_path(ff_pkt); } - if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) { + switch (ff_pkt->type) { + case FT_LNK: + case FT_LNKSAVED: Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link); stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0, attribsEx, 0); - } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) { + break; + case FT_DIREND: + case FT_REPARSE: /* Here link is the canonical filename (i.e. with trailing slash) */ stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0); - } else { + break; + case FT_RESTORE_FIRST: + comp_len = ff_pkt->object_len; + ff_pkt->object_compression = 0; + if (ff_pkt->object_len > 1000) { + /* Big object, compress it */ + int stat; + comp_len = ff_pkt->object_len + 1000; + POOLMEM *comp_obj = get_memory(comp_len); + stat = Zdeflate(ff_pkt->object, ff_pkt->object_len, comp_obj, comp_len); + if (comp_len < ff_pkt->object_len) { + ff_pkt->object = comp_obj; + ff_pkt->object_compression = 1; /* zlib level 9 compression */ + } else { + /* Uncompressed object smaller, use it */ + comp_len = ff_pkt->object_len; + } + Dmsg2(100, "Object compressed from %d to %d bytes\n", ff_pkt->object_len, comp_len); + } + sd->msglen = Mmsg(sd->msg, "%d %d %d %d %d %d %s%c%s%c", + jcr->JobFiles, ff_pkt->type, ff_pkt->object_index, + comp_len, ff_pkt->object_len, ff_pkt->object_compression, + ff_pkt->fname, 0, ff_pkt->object_name, 0); + sd->msg = check_pool_memory_size(sd->msg, sd->msglen + comp_len + 2); + memcpy(sd->msg + sd->msglen, ff_pkt->object, comp_len); + /* Note we send one extra byte so Dir can store zero after object */ + sd->msglen += comp_len + 1; + stat = sd->send(); + if (ff_pkt->object_compression) { + free_and_null_pool_memory(ff_pkt->object); + } + break; + default: stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0); + break; } if (ff_pkt->type != FT_DELETED) { unstrip_path(ff_pkt); } Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg); - if (!stat) { + if (!stat && !jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); - return false; } sd->signal(BNET_EOD); /* indicate end of attributes data */ - return true; + return stat; } -/* +/** * Do in place strip of path */ static bool do_strip(int count, char *in) @@ -1169,7 +1255,7 @@ static bool do_strip(int count, char *in) int stripped; int numsep = 0; - /* Copy to first path separator -- Win32 might have c: ... */ + /** Copy to first path separator -- Win32 might have c: ... */ while (*in && !IsPathSeparator(*in)) { out++; in++; } @@ -1197,7 +1283,7 @@ static bool do_strip(int count, char *in) return stripped==count && numsep>count; } -/* +/** * If requested strip leading components of the path so that we can * save file as if it came from a subdirectory. This is most useful * for dealing with snapshots, by removing the snapshot directory, or @@ -1222,7 +1308,7 @@ void strip_path(FF_PKT *ff_pkt) sm_check(__FILE__, __LINE__, true); } - /* + /** * Strip path. If it doesn't succeed put it back. If * it does, and there is a different link string, * attempt to strip the link. If it fails, back them @@ -1234,7 +1320,7 @@ void strip_path(FF_PKT *ff_pkt) unstrip_path(ff_pkt); goto rtn; } - /* Strip links but not symlinks */ + /** Strip links but not symlinks */ if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) { if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) { unstrip_path(ff_pkt); @@ -1261,3 +1347,35 @@ void unstrip_path(FF_PKT *ff_pkt) sm_check(__FILE__, __LINE__, true); } } + +static void close_vss_backup_session(JCR *jcr) +{ +#if defined(WIN32_VSS) + /* STOP VSS ON WIN32 */ + /* tell vss to close the backup session */ + if (jcr->VSS) { + if (g_pVSSClient->CloseBackup()) { + /* inform user about writer states */ + for (int i=0; i<(int)g_pVSSClient->GetWriterCount(); i++) { + int msg_type = M_INFO; + if (g_pVSSClient->GetWriterState(i) < 1) { + msg_type = M_WARNING; + jcr->JobErrors++; + } + Jmsg(jcr, msg_type, 0, _("VSS Writer (BackupComplete): %s\n"), g_pVSSClient->GetWriterInfo(i)); + } + } + WCHAR *metadata = g_pVSSClient->GetMetadata(); + if (metadata) { + FF_PKT *ff_pkt = jcr->ff; + ff_pkt->fname = "job"; + ff_pkt->type = FT_RESTORE_FIRST; + ff_pkt->LinkFI = 0; + ff_pkt->object_name = "job_metadata.xml"; + ff_pkt->object = (char *)metadata; + ff_pkt->object_len = (wcslen(metadata) + 1) * sizeof(WCHAR); + save_file(jcr, ff_pkt, true); + } + } +#endif +}