X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Ffiled%2Fbackup.c;h=60beddb80c8e233f3062f3e8ee3d0e0825f02545;hb=7fc5fe1b10047210db9c20ccb415d9ac87f6eee1;hp=f5fbd25ef2188ed00d25d0c56268b614536d7a6d;hpb=326b90cc815ca9ce77bcf360062e7873b65e6aab;p=bacula%2Fbacula diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index f5fbd25ef2..60beddb80c 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -1,12 +1,12 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2007 Free Software Foundation Europe e.V. + Copyright (C) 2000-2010 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public + modify it under the terms of version three of the GNU Affero General Public License as published by the Free Software Foundation and included in the file LICENSE. @@ -15,38 +15,55 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Affero General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - Bacula® is a registered trademark of John Walker. + Bacula® is a registered trademark of Kern Sibbald. The licensor of Bacula is the Free Software Foundation Europe (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ -/* +/** * Bacula File Daemon backup.c send file attributes and data * to the Storage daemon. * * Kern Sibbald, March MM * - * Version $Id$ - * */ #include "bacula.h" #include "filed.h" +#ifdef HAVE_DARWIN_OS +const bool have_darwin_os = true; +#else +const bool have_darwin_os = false; +#endif + +#if defined(HAVE_ACL) +const bool have_acl = true; +#else +const bool have_acl = false; +#endif + +#if defined(HAVE_XATTR) +const bool have_xattr = true; +#else +const bool have_xattr = false; +#endif + /* Forward referenced functions */ -static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level); -static void strip_path(FF_PKT *ff_pkt); -static void unstrip_path(FF_PKT *ff_pkt); +int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level); static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest); -static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream); -static bool read_and_send_acl(JCR *jcr, int acltype, int stream); +bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream); +static bool crypto_session_start(JCR *jcr); +static void crypto_session_end(JCR *jcr); +static bool crypto_session_send(JCR *jcr, BSOCK *sd); +static void close_vss_backup_session(JCR *jcr); -/* +/** * Find all the requested files and send them * to the Storage daemon. * @@ -63,8 +80,6 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) BSOCK *sd; bool ok = true; // TODO landonf: Allow user to specify encryption algorithm - crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC; - sd = jcr->store_bsock; @@ -81,25 +96,27 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) } else { buf_size = 0; /* use default */ } - if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) { + if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) { set_jcr_job_status(jcr, JS_ErrorTerminated); Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n")); return false; } jcr->buf_size = sd->msglen; - /* Adjust for compression so that output buffer is - * 12 bytes + 0.1% larger than input buffer plus 18 bytes. - * This gives a bit extra plus room for the sparse addr if any. - * Note, we adjust the read size to be smaller so that the - * same output buffer can be used without growing it. + /** + * Adjust for compression so that output buffer is + * 12 bytes + 0.1% larger than input buffer plus 18 bytes. + * This gives a bit extra plus room for the sparse addr if any. + * Note, we adjust the read size to be smaller so that the + * same output buffer can be used without growing it. * - * The zlib compression workset is initialized here to minimise - * the "per file" load. The jcr member is only set, if the init was successful. + * The zlib compression workset is initialized here to minimize + * the "per file" load. The jcr member is only set, if the init + * was successful. */ jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30; jcr->compress_buf = get_memory(jcr->compress_buf_size); - + #ifdef HAVE_LIBZ z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream)); if (pZlibStream) { @@ -108,64 +125,72 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) pZlibStream->opaque = Z_NULL; pZlibStream->state = Z_NULL; - if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) + if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) { jcr->pZLIB_compress_workset = pZlibStream; - else + } else { free (pZlibStream); + } } #endif - /* Create encryption session data and a cached, DER-encoded session data - * structure. We use a single session key for each backup, so we'll encode - * the session data only once. */ - if (jcr->pki_encrypt) { - uint32_t size = 0; - - /* Create per-job session encryption context */ - jcr->pki_session = crypto_session_new(cipher, jcr->pki_recipients); - - /* Get the session data size */ - if (crypto_session_encode(jcr->pki_session, (uint8_t *)0, &size) == false) { - Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n")); - return 0; - } - - /* Allocate buffer */ - jcr->pki_session_encoded = (uint8_t *)malloc(size); - - /* Encode session data */ - if (crypto_session_encode(jcr->pki_session, jcr->pki_session_encoded, &size) == false) { - Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n")); - return 0; - } - - /* ... and store the encoded size */ - jcr->pki_session_encoded_size = size; - - /* Allocate the encryption/decryption buffer */ - jcr->crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE); + if (!crypto_session_start(jcr)) { + return false; } - Dmsg1(300, "set_find_options ff=%p\n", jcr->ff); set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime); - Dmsg0(300, "start find files\n"); + /** in accurate mode, we overwrite the find_one check function */ + if (jcr->accurate) { + set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file); + } + start_heartbeat_monitor(jcr); - jcr->acl_text = get_pool_memory(PM_MESSAGE); + if (have_acl) { + jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t)); + memset((caddr_t)jcr->acl_data, 0, sizeof(acl_data_t)); + jcr->acl_data->content = get_pool_memory(PM_MESSAGE); + } + + if (have_xattr) { + jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t)); + memset((caddr_t)jcr->xattr_data, 0, sizeof(xattr_data_t)); + jcr->xattr_data->content = get_pool_memory(PM_MESSAGE); + } - /* Subroutine save_file() is called for each file */ - if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) { + /** Subroutine save_file() is called for each file */ + if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) { ok = false; /* error */ set_jcr_job_status(jcr, JS_ErrorTerminated); } - free_pool_memory(jcr->acl_text); + if (have_acl && jcr->acl_data->nr_errors > 0) { + Jmsg(jcr, M_ERROR, 0, _("Encountered %ld acl errors while doing backup\n"), + jcr->acl_data->nr_errors); + } + if (have_xattr && jcr->xattr_data->nr_errors > 0) { + Jmsg(jcr, M_ERROR, 0, _("Encountered %ld xattr errors while doing backup\n"), + jcr->xattr_data->nr_errors); + } + + close_vss_backup_session(jcr); + + accurate_finish(jcr); /* send deleted or base file list to SD */ stop_heartbeat_monitor(jcr); sd->signal(BNET_EOD); /* end of sending data */ + if (have_acl && jcr->acl_data) { + free_pool_memory(jcr->acl_data->content); + free(jcr->acl_data); + jcr->acl_data = NULL; + } + if (have_xattr && jcr->xattr_data) { + free_pool_memory(jcr->xattr_data->content); + free(jcr->xattr_data); + jcr->xattr_data = NULL; + } if (jcr->big_buf) { free(jcr->big_buf); jcr->big_buf = NULL; @@ -182,23 +207,89 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) free (jcr->pZLIB_compress_workset); jcr->pZLIB_compress_workset = NULL; } - if (jcr->crypto_buf) { - free_pool_memory(jcr->crypto_buf); - jcr->crypto_buf = NULL; + crypto_session_end(jcr); + + + Dmsg1(100, "end blast_data ok=%d\n", ok); + return ok; +} + +static bool crypto_session_start(JCR *jcr) +{ + crypto_cipher_t cipher = CRYPTO_CIPHER_AES_128_CBC; + + /** + * Create encryption session data and a cached, DER-encoded session data + * structure. We use a single session key for each backup, so we'll encode + * the session data only once. + */ + if (jcr->crypto.pki_encrypt) { + uint32_t size = 0; + + /** Create per-job session encryption context */ + jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients); + + /** Get the session data size */ + if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) { + Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n")); + return false; + } + + /** Allocate buffer */ + jcr->crypto.pki_session_encoded = get_memory(size); + + /** Encode session data */ + if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) { + Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n")); + return false; + } + + /** ... and store the encoded size */ + jcr->crypto.pki_session_encoded_size = size; + + /** Allocate the encryption/decryption buffer */ + jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE); + } + return true; +} + +static void crypto_session_end(JCR *jcr) +{ + if (jcr->crypto.crypto_buf) { + free_pool_memory(jcr->crypto.crypto_buf); + jcr->crypto.crypto_buf = NULL; } - if (jcr->pki_session) { - crypto_session_free(jcr->pki_session); + if (jcr->crypto.pki_session) { + crypto_session_free(jcr->crypto.pki_session); } - if (jcr->pki_session_encoded) { - free(jcr->pki_session_encoded); - jcr->pki_session_encoded = NULL; + if (jcr->crypto.pki_session_encoded) { + free_pool_memory(jcr->crypto.pki_session_encoded); + jcr->crypto.pki_session_encoded = NULL; } +} - Dmsg1(100, "end blast_data ok=%d\n", ok); - return ok; +static bool crypto_session_send(JCR *jcr, BSOCK *sd) +{ + POOLMEM *msgsave; + + /** Send our header */ + Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA); + sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA); + + msgsave = sd->msg; + sd->msg = jcr->crypto.pki_session_encoded; + sd->msglen = jcr->crypto.pki_session_encoded_size; + jcr->JobBytes += sd->msglen; + + Dmsg1(100, "Send data len=%d\n", sd->msglen); + sd->send(); + sd->msg = msgsave; + sd->signal(BNET_EOD); + return true; } -/* + +/** * Called here by find() for each file included. * This is a callback. The original is find_files() above. * @@ -208,9 +299,10 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) * 0 if error * -1 to ignore file/directory (not used here) */ -static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) +int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) { bool do_read = false; + bool plugin_started = false; int stat, data_stream; int rtnstat = 0; DIGEST *digest = NULL; @@ -224,10 +316,9 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) #else crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1; #endif - JCR *jcr = (JCR *)vjcr; BSOCK *sd = jcr->store_bsock; - if (job_canceled(jcr)) { + if (jcr->is_job_canceled()) { return 0; } @@ -248,6 +339,9 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) case FT_LNK: Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link); break; + case FT_RESTORE_FIRST: + Dmsg1(100, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname); + break; case FT_DIRBEGIN: jcr->num_files_examined--; /* correct file count */ return 1; /* not used */ @@ -258,9 +352,14 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) break; case FT_NOFSCHG: /* Suppress message for /dev filesystems */ - if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) { - Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"), - ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname); + if (!is_in_fileset(ff_pkt)) { +#ifdef HAVE_WIN32 + Jmsg(jcr, M_INFO, 1, _(" %s is a junction point or a different filesystem. Will not descend from %s into it.\n"), + ff_pkt->fname, ff_pkt->top_fname); +#else + Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into it.\n"), + ff_pkt->fname, ff_pkt->top_fname); +#endif } ff_pkt->type = FT_DIREND; /* Backup only the directory entry */ break; @@ -279,6 +378,10 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) break; case FT_SPEC: Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname); + if (S_ISSOCK(ff_pkt->statp.st_mode)) { + Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname); + return 1; + } break; case FT_RAW: Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname); @@ -289,23 +392,23 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) break; case FT_NOACCESS: { berrno be; - Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname, + Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } case FT_NOFOLLOW: { berrno be; - Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), + Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } case FT_NOSTAT: { berrno be; - Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname, + Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } case FT_DIRNOCHG: @@ -317,23 +420,23 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) return 1; case FT_NOOPEN: { berrno be; - Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), + Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno)); - jcr->Errors++; + jcr->JobErrors++; return 1; } default: Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname); - jcr->Errors++; + jcr->JobErrors++; return 1; } Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname); - /* Digests and encryption are only useful if there's file data */ + /** Digests and encryption are only useful if there's file data */ if (has_file_data) { - /* + /** * Setup for digest handling. If this fails, the digest will be set to NULL * and not used. Note, the digest (file hash) can be any one of the four * algorithms below. @@ -361,75 +464,68 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) digest_stream = STREAM_SHA512_DIGEST; } - /* Did digest initialization fail? */ + /** Did digest initialization fail? */ if (digest_stream != STREAM_NONE && digest == NULL) { Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"), stream_to_ascii(digest_stream)); } - /* - * Set up signature digest handling. If this fails, the signature digest will be set to - * NULL and not used. + /** + * Set up signature digest handling. If this fails, the signature digest + * will be set to NULL and not used. + */ + /* TODO landonf: We should really only calculate the digest once, for + * both verification and signing. */ - // TODO landonf: We should really only calculate the digest once, for both verification and signing. - if (jcr->pki_sign) { + if (jcr->crypto.pki_sign) { signing_digest = crypto_digest_new(jcr, signing_algorithm); - /* Full-stop if a failure occurred initializing the signature digest */ + /** Full-stop if a failure occurred initializing the signature digest */ if (signing_digest == NULL) { Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"), stream_to_ascii(signing_algorithm)); - jcr->Errors++; + jcr->JobErrors++; goto good_rtn; } } - /* Enable encryption */ - if (jcr->pki_encrypt) { + /** Enable encryption */ + if (jcr->crypto.pki_encrypt) { ff_pkt->flags |= FO_ENCRYPT; } } - /* Initialise the file descriptor we use for data and other streams. */ + /** Initialize the file descriptor we use for data and other streams. */ binit(&ff_pkt->bfd); if (ff_pkt->flags & FO_PORTABLE) { set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */ } - if (ff_pkt->reader) { - if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) { - Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"), - ff_pkt->reader); + if (ff_pkt->cmd_plugin) { + /* Tell bfile that it needs to call plugin */ + if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) { goto bail_out; } + send_plugin_name(jcr, sd, true); /* signal start of plugin data */ + plugin_started = true; } - /* Send attributes -- must be done after binit() */ + /** Send attributes -- must be done after binit() */ if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) { goto bail_out; } + /** Meta data only for restore object */ + if (ff_pkt->type == FT_RESTORE_FIRST) { + goto good_rtn; + } - /* Set up the encryption context and send the session data to the SD */ - if (has_file_data && jcr->pki_encrypt) { - /* Send our header */ - Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA); - sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA); - - /* Grow the bsock buffer to fit our message if necessary */ - if (sizeof_pool_memory(sd->msg) < jcr->pki_session_encoded_size) { - sd->msg = realloc_pool_memory(sd->msg, jcr->pki_session_encoded_size); + /** Set up the encryption context and send the session data to the SD */ + if (has_file_data && jcr->crypto.pki_encrypt) { + if (!crypto_session_send(jcr, sd)) { + goto bail_out; } - - /* Copy our message over and send it */ - memcpy(sd->msg, jcr->pki_session_encoded, jcr->pki_session_encoded_size); - sd->msglen = jcr->pki_session_encoded_size; - jcr->JobBytes += sd->msglen; - - Dmsg1(100, "Send data len=%d\n", sd->msglen); - sd->send(); - sd->signal(BNET_EOD); } - /* + /** * Open any file with data that we intend to save, then save it. * * Note, if is_win32_backup, we must open the Directory so that @@ -439,7 +535,7 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) #ifdef HAVE_WIN32 do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0; #else - do_read = ff_pkt->statp.st_size > 0; + do_read = ff_pkt->statp.st_size > 0; #endif } else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO || ff_pkt->type == FT_REPARSE || @@ -447,8 +543,14 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) do_read = true; } + if (ff_pkt->cmd_plugin) { + do_read = true; + } + + Dmsg2(150, "type=%d do_read=%d\n", ff_pkt->type, do_read); if (do_read) { btimer_t *tid; + if (ff_pkt->type == FT_FIFO) { tid = start_thread_timer(jcr, pthread_self(), 60); } else { @@ -459,9 +561,9 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) { ff_pkt->ff_errno = errno; berrno be; - Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname, + Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname, be.bstrerror()); - jcr->Errors++; + jcr->JobErrors++; if (tid) { stop_thread_timer(tid); tid = NULL; @@ -486,69 +588,104 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) } } -#ifdef HAVE_DARWIN_OS - /* Regular files can have resource forks and Finder Info */ - if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && - ff_pkt->flags & FO_HFSPLUS)) { - if (ff_pkt->hfsinfo.rsrclength > 0) { - int flags; - int rsrc_stream; - if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { - ff_pkt->ff_errno = errno; - berrno be; - Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), - ff_pkt->fname, be.bstrerror()); - jcr->Errors++; - if (is_bopen(&ff_pkt->bfd)) { - bclose(&ff_pkt->bfd); + if (have_darwin_os) { + /** Regular files can have resource forks and Finder Info */ + if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && + ff_pkt->flags & FO_HFSPLUS)) { + if (ff_pkt->hfsinfo.rsrclength > 0) { + int flags; + int rsrc_stream; + if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { + ff_pkt->ff_errno = errno; + berrno be; + Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"), + ff_pkt->fname, be.bstrerror()); + jcr->JobErrors++; + if (is_bopen(&ff_pkt->bfd)) { + bclose(&ff_pkt->bfd); + } + goto good_rtn; + } + flags = ff_pkt->flags; + ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE); + if (flags & FO_ENCRYPT) { + rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA; + } else { + rsrc_stream = STREAM_MACOS_FORK_DATA; + } + stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest); + ff_pkt->flags = flags; + bclose(&ff_pkt->bfd); + if (!stat) { + goto bail_out; } - goto good_rtn; } - flags = ff_pkt->flags; - ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE); - if (flags & FO_ENCRYPT) { - rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA; - } else { - rsrc_stream = STREAM_MACOS_FORK_DATA; + + Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname); + sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES); + Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); + pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32); + sd->msglen = 32; + if (digest) { + crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen); } - stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest); - ff_pkt->flags = flags; - bclose(&ff_pkt->bfd); - if (!stat) { - goto bail_out; + if (signing_digest) { + crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen); } + sd->send(); + sd->signal(BNET_EOD); } + } - Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname); - sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES); - Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); - memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32); - sd->msglen = 32; - if (digest) { - crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen); - } - if (signing_digest) { - crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen); + /** + * Save ACLs when requested and available for anything not being a symlink and not being a plugin. + */ + if (have_acl) { + if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) { + switch (build_acl_streams(jcr, ff_pkt)) { + case bacl_exit_fatal: + goto bail_out; + case bacl_exit_error: + /** + * Non-fatal errors, count them and when the number is under ACL_REPORT_ERR_MAX_PER_JOB + * print the error message set by the lower level routine in jcr->errmsg. + */ + if (jcr->acl_data->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) { + Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg); + } + jcr->acl_data->nr_errors++; + break; + case bacl_exit_ok: + break; + } } - sd->send(); - sd->signal(BNET_EOD); } -#endif - if (ff_pkt->flags & FO_ACL) { - /* Read access ACLs for files, dirs and links */ - if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) { - goto bail_out; - } - /* Directories can have default ACLs too */ - if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) { - if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) { + /** + * Save Extended Attributes when requested and available for all files not being a plugin. + */ + if (have_xattr) { + if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) { + switch (build_xattr_streams(jcr, ff_pkt)) { + case bxattr_exit_fatal: goto bail_out; + case bxattr_exit_error: + /** + * Non-fatal errors, count them and when the number is under XATTR_REPORT_ERR_MAX_PER_JOB + * print the error message set by the lower level routine in jcr->errmsg. + */ + if (jcr->xattr_data->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) { + Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg); + } + jcr->xattr_data->nr_errors++; + break; + case bxattr_exit_ok: + break; } } } - /* Terminate the signing digest and send it to the Storage daemon */ + /** Terminate the signing digest and send it to the Storage daemon */ if (signing_digest) { uint32_t size = 0; @@ -557,27 +694,27 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) goto bail_out; } - if (!crypto_sign_add_signer(sig, signing_digest, jcr->pki_keypair)) { + if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n")); goto bail_out; } - /* Get signature size */ + /** Get signature size */ if (!crypto_sign_encode(sig, NULL, &size)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n")); goto bail_out; } - /* Grow the bsock buffer to fit our message if necessary */ + /** Grow the bsock buffer to fit our message if necessary */ if (sizeof_pool_memory(sd->msg) < (int32_t)size) { sd->msg = realloc_pool_memory(sd->msg, size); } - /* Send our header */ - sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_SIGNED_DIGEST); + /** Send our header */ + sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST); Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); - /* Encode signature data */ + /** Encode signature data */ if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) { Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n")); goto bail_out; @@ -588,7 +725,7 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) sd->signal(BNET_EOD); /* end of checksum */ } - /* Terminate any digest and send it to Storage daemon and the Director */ + /** Terminate any digest and send it to Storage daemon */ if (digest) { uint32_t size; @@ -597,7 +734,7 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level) size = CRYPTO_DIGEST_MAX_SIZE; - /* Grow the bsock buffer to fit our message if necessary */ + /** Grow the bsock buffer to fit our message if necessary */ if (sizeof_pool_memory(sd->msg) < (int32_t)size) { sd->msg = realloc_pool_memory(sd->msg, size); } @@ -616,6 +753,9 @@ good_rtn: rtnstat = 1; /* good return */ bail_out: + if (ff_pkt->cmd_plugin && plugin_started) { + send_plugin_name(jcr, sd, false); /* signal end of plugin data */ + } if (digest) { crypto_digest_free(digest); } @@ -628,7 +768,7 @@ bail_out: return rtnstat; } -/* +/** * Send data read from an already open file descriptor. * * We return 1 on sucess and 0 on errors. @@ -639,8 +779,8 @@ bail_out: * Currently this is not a problem as the only other stream, resource forks, * are not handled as sparse files. */ -int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, - DIGEST *signing_digest) +static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, + DIGEST *signing_digest) { BSOCK *sd = jcr->store_bsock; uint64_t fileAddr = 0; /* file address */ @@ -680,14 +820,14 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, wbuf = jcr->compress_buf; /* compressed output here */ cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ - /* + /** * Only change zlib parameters if there is no pending operation. * This should never happen as deflatereset is called after each * deflate. */ if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) { - /* set gzip compression level - must be done per file */ + /** set gzip compression level - must be done per file */ if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset, ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); @@ -705,40 +845,42 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n")); goto err; } - /* Allocate the cipher context */ - if ((cipher_ctx = crypto_cipher_new(jcr->pki_session, true, + /** Allocate the cipher context */ + if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true, &cipher_block_size)) == NULL) { /* Shouldn't happen! */ Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n")); goto err; } - /* + /** * Grow the crypto buffer, if necessary. * crypto_cipher_update() will buffer up to (cipher_block_size - 1). * We grow crypto_buf to the maximum number of blocks that * could be returned for the given read buffer size. * (Using the larger of either rsize or max_compress_len) */ - jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf, + jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf, (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) + cipher_block_size - 1) / cipher_block_size * cipher_block_size); - wbuf = jcr->crypto_buf; /* Encrypted, possibly compressed output here. */ + wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */ } - /* + /** * Send Data header to Storage daemon * */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } Dmsg1(300, ">stored: datahdr %s\n", sd->msg); - /* + /** * Make space at beginning of buffer for fileAddr because this * same buffer will be used for writing if compression is off. */ @@ -746,7 +888,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, rbuf += SPARSE_FADDR_SIZE; rsize -= SPARSE_FADDR_SIZE; #ifdef HAVE_FREEBSD_OS - /* + /** * To read FreeBSD partitions, the read size must be * a multiple of 512. */ @@ -754,54 +896,56 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, #endif } - /* a RAW device read on win32 only works if the buffer is a multiple of 512 */ + /** a RAW device read on win32 only works if the buffer is a multiple of 512 */ #ifdef HAVE_WIN32 if (S_ISBLK(ff_pkt->statp.st_mode)) rsize = (rsize/512) * 512; #endif - /* + /** * Read the file data */ while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) { - /* Check for sparse blocks */ + /** Check for sparse blocks */ if (ff_pkt->flags & FO_SPARSE) { ser_declare; - bool haveBlock = true; - if (sd->msglen == rsize && - fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size || + bool allZeros = false; + if ((sd->msglen == rsize && + fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) || ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) && (uint64_t)ff_pkt->statp.st_size == 0)) { - haveBlock = !is_buf_zero(rbuf, rsize); + allZeros = is_buf_zero(rbuf, rsize); } - if (haveBlock) { + if (!allZeros) { + /** Put file address as first data in buffer */ ser_begin(wbuf, SPARSE_FADDR_SIZE); ser_uint64(fileAddr); /* store fileAddr in begin of buffer */ } fileAddr += sd->msglen; /* update file address */ - if (!haveBlock) { + /** Skip block of all zeros */ + if (allZeros) { continue; /* skip block of zeros */ } } jcr->ReadBytes += sd->msglen; /* count bytes read */ - /* Uncompressed cipher input length */ + /** Uncompressed cipher input length */ cipher_input_len = sd->msglen; - /* Update checksum if requested */ + /** Update checksum if requested */ if (digest) { crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen); } - /* Update signing digest if requested */ + /** Update signing digest if requested */ if (signing_digest) { crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen); } #ifdef HAVE_LIBZ - /* Do compression if turned on */ + /** Do compression if turned on */ if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) { Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen); @@ -816,7 +960,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, goto err; } compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out; - /* reset zlib stream to be able to begin from scratch again */ + /** reset zlib stream to be able to begin from scratch again */ if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); @@ -830,7 +974,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, cipher_input_len = compress_len; } #endif - /* + /** * Note, here we prepend the current record length to the beginning * of the encrypted data. This is because both sparse and compression * restore handling want records returned to them with exactly the @@ -851,7 +995,7 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, cipher_input_len += SPARSE_FADDR_SIZE; } - /* Encrypt the length of the input block */ + /** Encrypt the length of the input block */ uint8_t packet_len[sizeof(uint32_t)]; ser_begin(packet_len, sizeof(uint32_t)); @@ -859,24 +1003,24 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, Dmsg1(20, "Encrypt len=%d\n", cipher_input_len); if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len), - (u_int8_t *)jcr->crypto_buf, &initial_len)) { - /* Encryption failed. Shouldn't happen. */ + (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) { + /** Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } - /* Encrypt the input block */ + /** Encrypt the input block */ if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len, - (u_int8_t *)&jcr->crypto_buf[initial_len], &encrypted_len)) { + (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) { if ((initial_len + encrypted_len) == 0) { - /* No full block of data available, read more data */ + /** No full block of data available, read more data */ continue; } Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len, sd->msglen); sd->msglen = initial_len + encrypted_len; /* set encrypted length */ } else { - /* Encryption failed. Shouldn't happen. */ + /** Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } @@ -888,8 +1032,10 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, } sd->msg = wbuf; /* set correct write buffer */ if (!sd->send()) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); @@ -903,28 +1049,30 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, berrno be; Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno)); - if (jcr->Errors++ > 1000) { /* insanity check */ - Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n")); + if (jcr->JobErrors++ > 1000) { /* insanity check */ + Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors); } } else if (ff_pkt->flags & FO_ENCRYPT) { - /* + /** * For encryption, we must call finalize to push out any * buffered data. */ - if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf, + if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf, &encrypted_len)) { /* Padding failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n")); goto err; } - /* Note, on SSL pre-0.9.7, there is always some output */ + /** Note, on SSL pre-0.9.7, there is always some output */ if (encrypted_len > 0) { sd->msglen = encrypted_len; /* set encrypted length */ - sd->msg = jcr->crypto_buf; /* set correct write buffer */ + sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */ if (!sd->send()) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); @@ -934,19 +1082,21 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, } if (!sd->signal(BNET_EOD)) { /* indicate end of file data */ - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } goto err; } - /* Free the cipher context */ + /** Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } return 1; err: - /* Free the cipher context */ + /** Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } @@ -956,83 +1106,35 @@ err: return 0; } -/* - * Read and send an ACL for the last encountered file. - */ -static bool read_and_send_acl(JCR *jcr, int acltype, int stream) -{ -#ifdef HAVE_ACL - BSOCK *sd = jcr->store_bsock; - POOLMEM *msgsave; - int len; -#ifdef FD_NO_SEND_TEST - return true; -#endif - - len = bacl_get(jcr, acltype); - if (len < 0) { - Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname); - return true; - } - if (len == 0) { - return true; /* no ACL */ - } - - /* Send header */ - if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); - return false; - } - - /* Send the buffer to the storage deamon */ - Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text); - msgsave = sd->msg; - sd->msg = jcr->acl_text; - sd->msglen = len + 1; - if (!sd->send()) { - sd->msg = msgsave; - sd->msglen = 0; - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); - return false; - } - - jcr->JobBytes += sd->msglen; - sd->msg = msgsave; - if (!sd->signal(BNET_EOD)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); - return false; - } - - Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname); -#endif - return true; -} - -static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream) +bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream) { BSOCK *sd = jcr->store_bsock; char attribs[MAXSTRING]; - char attribsEx[MAXSTRING]; + char attribsExBuf[MAXSTRING]; + char *attribsEx = NULL; int attr_stream; - int stat; + int comp_len; + bool stat; #ifdef FD_NO_SEND_TEST return true; #endif Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname); - /* Find what data stream we will use, then encode the attributes */ + /** Find what data stream we will use, then encode the attributes */ if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) { /* This should not happen */ Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n")); return false; } - encode_stat(attribs, ff_pkt, data_stream); + encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream); - /* Now possibly extend the attributes */ - attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt); + /** Now possibly extend the attributes */ + if (ff_pkt->type == FT_RESTORE_FIRST) { + attr_stream = STREAM_RESTORE_OBJECT; + } else { + attribsEx = attribsExBuf; + attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt); + } Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx); @@ -1042,18 +1144,20 @@ static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_strea pm_strcpy(jcr->last_fname, ff_pkt->fname); jcr->unlock(); - /* + /** * Send Attributes header to Storage daemon * */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) { - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - sd->bstrerror()); + if (!jcr->is_job_canceled()) { + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + sd->bstrerror()); + } return false; } Dmsg1(300, ">stored: attrhdr %s\n", sd->msg); - /* + /** * Send file attributes to Storage daemon * File_index * File type @@ -1062,36 +1166,87 @@ static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_strea * Link name (if type==FT_LNK or FT_LNKSAVED) * Encoded extended-attributes (for Win32) * + * or send Restore Object to Storage daemon + * File_index + * File_type + * Object_index + * Object_len (possibly compressed) + * Object_full_len (not compressed) + * Object_compression + * Plugin_name + * Object_name + * Binary Object data + * * For a directory, link is the same as fname, but with trailing * slash. For a linked file, link is the link. */ - strip_path(ff_pkt); - if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) { + if (ff_pkt->type != FT_DELETED) { /* already stripped */ + strip_path(ff_pkt); + } + switch (ff_pkt->type) { + case FT_LNK: + case FT_LNKSAVED: Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link); stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0, attribsEx, 0); - } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) { + break; + case FT_DIREND: + case FT_REPARSE: /* Here link is the canonical filename (i.e. with trailing slash) */ stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0); - } else { + break; + case FT_RESTORE_FIRST: + comp_len = ff_pkt->object_len; + ff_pkt->object_compression = 0; + if (ff_pkt->object_len > 1000) { + /* Big object, compress it */ + int stat; + comp_len = ff_pkt->object_len + 1000; + POOLMEM *comp_obj = get_memory(comp_len); + stat = Zdeflate(ff_pkt->object, ff_pkt->object_len, comp_obj, comp_len); + if (comp_len < ff_pkt->object_len) { + ff_pkt->object = comp_obj; + ff_pkt->object_compression = 1; /* zlib level 9 compression */ + } else { + /* Uncompressed object smaller, use it */ + comp_len = ff_pkt->object_len; + } + Dmsg2(100, "Object compressed from %d to %d bytes\n", ff_pkt->object_len, comp_len); + } + sd->msglen = Mmsg(sd->msg, "%d %d %d %d %d %d %s%c%s%c", + jcr->JobFiles, ff_pkt->type, ff_pkt->object_index, + comp_len, ff_pkt->object_len, ff_pkt->object_compression, + ff_pkt->fname, 0, ff_pkt->object_name, 0); + sd->msg = check_pool_memory_size(sd->msg, sd->msglen + comp_len + 2); + memcpy(sd->msg + sd->msglen, ff_pkt->object, comp_len); + /* Note we send one extra byte so Dir can store zero after object */ + sd->msglen += comp_len + 1; + stat = sd->send(); + if (ff_pkt->object_compression) { + free_and_null_pool_memory(ff_pkt->object); + } + break; + default: stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0); + break; + } + if (ff_pkt->type != FT_DELETED) { + unstrip_path(ff_pkt); } - unstrip_path(ff_pkt); Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg); - if (!stat) { + if (!stat && !jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); - return false; } sd->signal(BNET_EOD); /* indicate end of attributes data */ - return true; + return stat; } -/* +/** * Do in place strip of path */ static bool do_strip(int count, char *in) @@ -1100,11 +1255,11 @@ static bool do_strip(int count, char *in) int stripped; int numsep = 0; - /* Copy to first path separator -- Win32 might have c: ... */ + /** Copy to first path separator -- Win32 might have c: ... */ while (*in && !IsPathSeparator(*in)) { - *out++ = *in++; + out++; in++; } - *out++ = *in++; + out++; in++; numsep++; /* one separator seen */ for (stripped=0; strippedcount; } -/* - * If requested strip leading components of the path +/** + * If requested strip leading components of the path so that we can + * save file as if it came from a subdirectory. This is most useful + * for dealing with snapshots, by removing the snapshot directory, or + * in handling vendor migrations where files have been restored with + * a vendor product into a subdirectory. */ -static void strip_path(FF_PKT *ff_pkt) +void strip_path(FF_PKT *ff_pkt) { if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) { Dmsg1(200, "No strip for %s\n", ff_pkt->fname); @@ -1142,35 +1301,81 @@ static void strip_path(FF_PKT *ff_pkt) ff_pkt->link_save = get_pool_memory(PM_FNAME); } pm_strcpy(ff_pkt->fname_save, ff_pkt->fname); + if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) { + pm_strcpy(ff_pkt->link_save, ff_pkt->link); + Dmsg2(500, "strcpy link_save=%d link=%d\n", strlen(ff_pkt->link_save), + strlen(ff_pkt->link)); + sm_check(__FILE__, __LINE__, true); + } - /* + /** * Strip path. If it doesn't succeed put it back. If * it does, and there is a different link string, - * attempt to strip the link. If it fails, but them + * attempt to strip the link. If it fails, back them * both back. + * Do not strip symlinks. * I.e. if either stripping fails don't strip anything. */ - if (do_strip(ff_pkt->strip_path, ff_pkt->fname)) { - if (ff_pkt->fname != ff_pkt->link) { - pm_strcpy(ff_pkt->link_save, ff_pkt->link); - if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) { - strcpy(ff_pkt->link, ff_pkt->link_save); - strcpy(ff_pkt->fname, ff_pkt->fname_save); - } - } - } else { - strcpy(ff_pkt->fname, ff_pkt->fname_save); + if (!do_strip(ff_pkt->strip_path, ff_pkt->fname)) { + unstrip_path(ff_pkt); + goto rtn; } - Dmsg2(200, "fname=%s stripped=%s\n", ff_pkt->fname_save, ff_pkt->fname); + /** Strip links but not symlinks */ + if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) { + if (!do_strip(ff_pkt->strip_path, ff_pkt->link)) { + unstrip_path(ff_pkt); + } + } + +rtn: + Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname, + ff_pkt->link); } -static void unstrip_path(FF_PKT *ff_pkt) +void unstrip_path(FF_PKT *ff_pkt) { if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) { return; } strcpy(ff_pkt->fname, ff_pkt->fname_save); - if (ff_pkt->fname != ff_pkt->link) { + if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) { + Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link, + ff_pkt->link_save); strcpy(ff_pkt->link, ff_pkt->link_save); + Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link), + strlen(ff_pkt->link_save)); + sm_check(__FILE__, __LINE__, true); } } + +static void close_vss_backup_session(JCR *jcr) +{ +#if defined(WIN32_VSS) + /* STOP VSS ON WIN32 */ + /* tell vss to close the backup session */ + if (jcr->VSS) { + if (g_pVSSClient->CloseBackup()) { + /* inform user about writer states */ + for (int i=0; i<(int)g_pVSSClient->GetWriterCount(); i++) { + int msg_type = M_INFO; + if (g_pVSSClient->GetWriterState(i) < 1) { + msg_type = M_WARNING; + jcr->JobErrors++; + } + Jmsg(jcr, msg_type, 0, _("VSS Writer (BackupComplete): %s\n"), g_pVSSClient->GetWriterInfo(i)); + } + } + WCHAR *metadata = g_pVSSClient->GetMetadata(); + if (metadata) { + FF_PKT *ff_pkt = jcr->ff; + ff_pkt->fname = "job"; + ff_pkt->type = FT_RESTORE_FIRST; + ff_pkt->LinkFI = 0; + ff_pkt->object_name = "job_metadata.xml"; + ff_pkt->object = (char *)metadata; + ff_pkt->object_len = (wcslen(metadata) + 1) * sizeof(WCHAR); + save_file(jcr, ff_pkt, true); + } + } +#endif +}