/*
- Bacula® - The Network Backup Solution
+ Bacula(R) - The Network Backup Solution
- Copyright (C) 2000-2014 Free Software Foundation Europe e.V.
+ Copyright (C) 2000-2017 Kern Sibbald
- The main author of Bacula is Kern Sibbald, with contributions from many
- others, a complete list can be found in the file AUTHORS.
+ The original author of Bacula is Kern Sibbald, with contributions
+ from many others, a complete list can be found in the file AUTHORS.
You may use this file and others of this release according to the
license defined in the LICENSE file, which includes the Affero General
Public License, v3.0 ("AGPLv3") and some additional permissions and
terms pursuant to its AGPLv3 Section 7.
- Bacula® is a registered trademark of Kern Sibbald.
-*/
-/**
+ This notice must be preserved when any source code is
+ conveyed and/or propagated.
+
+ Bacula(R) is a registered trademark of Kern Sibbald.
+ */
+/*
* Bacula File Daemon backup.c send file attributes and data
* to the Storage daemon.
*
#include "bacula.h"
#include "filed.h"
-#include "ch.h"
-
-#ifdef HAVE_DARWIN_OS
-const bool have_darwin_os = true;
-#else
-const bool have_darwin_os = false;
-#endif
+#include "backup.h"
-#if defined(HAVE_ACL)
-const bool have_acl = true;
+#ifdef HAVE_LZO
+const bool have_lzo = true;
#else
-const bool have_acl = false;
+const bool have_lzo = false;
#endif
-#if defined(HAVE_XATTR)
-const bool have_xattr = true;
+#ifdef HAVE_LIBZ
+const bool have_libz = true;
#else
-const bool have_xattr = false;
+const bool have_libz = false;
#endif
/* Forward referenced functions */
int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
-static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signature_digest);
-bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
-static bool crypto_session_start(JCR *jcr);
-static void crypto_session_end(JCR *jcr);
-static bool crypto_session_send(JCR *jcr, BSOCK *sd);
+static int send_data(bctx_t &bctx, int stream);
static void close_vss_backup_session(JCR *jcr);
+#ifdef HAVE_DARWIN_OS
+static bool send_resource_fork(bctx_t &bctx);
+#endif
+static bool setup_compression(bctx_t &bctx);
+static bool do_lzo_compression(bctx_t &bctx);
+static bool do_libz_compression(bctx_t &bctx);
/**
* Find all the requested files and send them
*
* For the same reason, lzo compression is initialized here.
*/
-#ifdef HAVE_LZO
- jcr->compress_buf_size = MAX(jcr->buf_size + (jcr->buf_size / 16) + 67 + (int)sizeof(comp_stream_header), jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30);
- jcr->compress_buf = get_memory(jcr->compress_buf_size);
-#else
- jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
- jcr->compress_buf = get_memory(jcr->compress_buf_size);
-#endif
+ if (have_lzo) {
+ jcr->compress_buf_size = MAX(jcr->buf_size + (jcr->buf_size / 16) + 67 + (int)sizeof(comp_stream_header), jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30);
+ jcr->compress_buf = get_memory(jcr->compress_buf_size);
+ } else {
+ jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
+ jcr->compress_buf = get_memory(jcr->compress_buf_size);
+ }
#ifdef HAVE_LIBZ
z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
return false;
}
- set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
+ set_find_options(jcr->ff, jcr->incremental, jcr->mtime);
+ set_find_snapshot_function(jcr->ff, snapshot_convert_path);
/** in accurate mode, we overload the find_one check function */
if (jcr->accurate) {
set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
}
-
start_heartbeat_monitor(jcr);
- if (have_acl) {
- jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
- memset(jcr->acl_data, 0, sizeof(acl_data_t));
- jcr->acl_data->u.build = (acl_build_data_t *)malloc(sizeof(acl_build_data_t));
- memset(jcr->acl_data->u.build, 0, sizeof(acl_build_data_t));
- jcr->acl_data->u.build->content = get_pool_memory(PM_MESSAGE);
- }
-
- if (have_xattr) {
- jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
- memset(jcr->xattr_data, 0, sizeof(xattr_data_t));
- jcr->xattr_data->u.build = (xattr_build_data_t *)malloc(sizeof(xattr_build_data_t));
- memset(jcr->xattr_data->u.build, 0, sizeof(xattr_build_data_t));
- jcr->xattr_data->u.build->content = get_pool_memory(PM_MESSAGE);
- }
+ jcr->xacl = (XACL*)new_xacl();
- /** Subroutine save_file() is called for each file */
+ /* Subroutine save_file() is called for each file */
if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
ok = false; /* error */
jcr->setJobStatus(JS_ErrorTerminated);
}
- if (have_acl && jcr->acl_data->u.build->nr_errors > 0) {
- Jmsg(jcr, M_WARNING, 0, _("Encountered %ld acl errors while doing backup\n"),
- jcr->acl_data->u.build->nr_errors);
- }
- if (have_xattr && jcr->xattr_data->u.build->nr_errors > 0) {
- Jmsg(jcr, M_WARNING, 0, _("Encountered %ld xattr errors while doing backup\n"),
- jcr->xattr_data->u.build->nr_errors);
+ if (jcr->xacl) {
+ if (jcr->xacl->get_acl_nr_errors() > 0) {
+ Jmsg(jcr, M_WARNING, 0, _("Had %ld acl errors while doing backup\n"),
+ jcr->xacl->get_acl_nr_errors());
+ }
+ if (jcr->xacl->get_xattr_nr_errors() > 0) {
+ Jmsg(jcr, M_WARNING, 0, _("Had %ld xattr errors while doing backup\n"),
+ jcr->xacl->get_xattr_nr_errors());
+ }
}
-
+ /* Delete or keep snapshots */
+ close_snapshot_backup_session(jcr);
close_vss_backup_session(jcr);
accurate_finish(jcr); /* send deleted or base file list to SD */
sd->signal(BNET_EOD); /* end of sending data */
- if (have_acl && jcr->acl_data) {
- free_pool_memory(jcr->acl_data->u.build->content);
- free(jcr->acl_data->u.build);
- free(jcr->acl_data);
- jcr->acl_data = NULL;
- }
- if (have_xattr && jcr->xattr_data) {
- free_pool_memory(jcr->xattr_data->u.build->content);
- free(jcr->xattr_data->u.build);
- free(jcr->xattr_data);
- jcr->xattr_data = NULL;
+ if (jcr->xacl) {
+ delete(jcr->xacl);
+ jcr->xacl = NULL;
}
if (jcr->big_buf) {
- free(jcr->big_buf);
- jcr->big_buf = NULL;
+ bfree_and_null(jcr->big_buf);
}
if (jcr->compress_buf) {
- free_pool_memory(jcr->compress_buf);
- jcr->compress_buf = NULL;
+ free_and_null_pool_memory(jcr->compress_buf);
}
if (jcr->pZLIB_compress_workset) {
/* Free the zlib stream */
#ifdef HAVE_LIBZ
deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
#endif
- free (jcr->pZLIB_compress_workset);
- jcr->pZLIB_compress_workset = NULL;
+ bfree_and_null(jcr->pZLIB_compress_workset);
}
if (jcr->LZO_compress_workset) {
- free (jcr->LZO_compress_workset);
- jcr->LZO_compress_workset = NULL;
+ bfree_and_null(jcr->LZO_compress_workset);
}
crypto_session_end(jcr);
return ok;
}
-static bool crypto_session_start(JCR *jcr)
-{
- crypto_cipher_t cipher = (crypto_cipher_t) me->pki_cipher;
-
- /**
- * Create encryption session data and a cached, DER-encoded session data
- * structure. We use a single session key for each backup, so we'll encode
- * the session data only once.
- */
- if (jcr->crypto.pki_encrypt) {
- uint32_t size = 0;
-
- /** Create per-job session encryption context */
- jcr->crypto.pki_session = crypto_session_new(cipher, jcr->crypto.pki_recipients);
- if (!jcr->crypto.pki_session) {
- Jmsg(jcr, M_FATAL, 0, _("Unsupported cipher on this system.\n"));
- return false;
- }
-
- /** Get the session data size */
- if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)0, &size)) {
- Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
- return false;
- }
-
- /** Allocate buffer */
- jcr->crypto.pki_session_encoded = get_memory(size);
-
- /** Encode session data */
- if (!crypto_session_encode(jcr->crypto.pki_session, (uint8_t *)jcr->crypto.pki_session_encoded, &size)) {
- Jmsg(jcr, M_FATAL, 0, _("An error occurred while encrypting the stream.\n"));
- return false;
- }
-
- /** ... and store the encoded size */
- jcr->crypto.pki_session_encoded_size = size;
-
- /** Allocate the encryption/decryption buffer */
- jcr->crypto.crypto_buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
- }
- return true;
-}
-
-static void crypto_session_end(JCR *jcr)
-{
- if (jcr->crypto.crypto_buf) {
- free_pool_memory(jcr->crypto.crypto_buf);
- jcr->crypto.crypto_buf = NULL;
- }
- if (jcr->crypto.pki_session) {
- crypto_session_free(jcr->crypto.pki_session);
- }
- if (jcr->crypto.pki_session_encoded) {
- free_pool_memory(jcr->crypto.pki_session_encoded);
- jcr->crypto.pki_session_encoded = NULL;
- }
-}
-
-static bool crypto_session_send(JCR *jcr, BSOCK *sd)
-{
- POOLMEM *msgsave;
-
- /** Send our header */
- Dmsg2(100, "Send hdr fi=%ld stream=%d\n", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA);
- sd->fsend("%ld %d %lld", jcr->JobFiles, STREAM_ENCRYPTED_SESSION_DATA,
- (int64_t)jcr->ff->statp.st_size);
- msgsave = sd->msg;
- sd->msg = jcr->crypto.pki_session_encoded;
- sd->msglen = jcr->crypto.pki_session_encoded_size;
- jcr->JobBytes += sd->msglen;
-
- Dmsg1(100, "Send data len=%d\n", sd->msglen);
- sd->send();
- sd->msg = msgsave;
- sd->signal(BNET_EOD);
- return true;
-}
-
/**
* Called here by find() for each file included.
bool do_read = false;
bool plugin_started = false;
bool do_plugin_set = false;
- int stat, data_stream;
+ int stat;
int rtnstat = 0;
- DIGEST *digest = NULL;
- DIGEST *signing_digest = NULL;
- int digest_stream = STREAM_NONE;
- SIGNATURE *sig = NULL;
bool has_file_data = false;
- struct save_pkt sp; /* use by option plugin */
+ struct save_pkt sp; /* used by option plugin */
+ BSOCK *sd = jcr->store_bsock;
+ bctx_t bctx; /* backup context */
+
+ memset(&bctx, 0, sizeof(bctx));
+ bctx.sd = sd;
+ bctx.ff_pkt = ff_pkt;
+ bctx.jcr = jcr;
- crypto_digest_t signing_algorithm = (crypto_digest_t) me->pki_digest;
- BSOCK *sd = jcr->store_bsock;
time_t now = time(NULL);
if (jcr->last_stat_time == 0) {
jcr->last_stat_time = now;
jcr->stat_interval = 30; /* Default 30 seconds */
} else if (now >= jcr->last_stat_time + jcr->stat_interval) {
- jcr->dir_bsock->fsend("Progress Job=x files=%ld bytes=%lld bps=%ld\n",
+ jcr->dir_bsock->fsend("Progress JobId=x files=%ld bytes=%lld bps=%ld\n",
jcr->JobFiles, jcr->JobBytes, jcr->LastRate);
jcr->last_stat_time = now;
}
- if (jcr->is_canceled()) {
- Dmsg0(100, "Job canceled by user.\n");
+ if (jcr->is_canceled() || jcr->is_incomplete()) {
+ Dmsg0(100, "Job canceled by user or marked incomplete.\n");
return 0;
}
Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
/** Digests and encryption are only useful if there's file data */
- if (has_file_data) {
- /**
- * Setup for digest handling. If this fails, the digest will be set to NULL
- * and not used. Note, the digest (file hash) can be any one of the four
- * algorithms below.
- *
- * The signing digest is a single algorithm depending on
- * whether or not we have SHA2.
- * ****FIXME**** the signing algoritm should really be
- * determined a different way!!!!!! What happens if
- * sha2 was available during backup but not restore?
- */
- if (ff_pkt->flags & FO_MD5) {
- digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
- digest_stream = STREAM_MD5_DIGEST;
-
- } else if (ff_pkt->flags & FO_SHA1) {
- digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
- digest_stream = STREAM_SHA1_DIGEST;
-
- } else if (ff_pkt->flags & FO_SHA256) {
- digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
- digest_stream = STREAM_SHA256_DIGEST;
-
- } else if (ff_pkt->flags & FO_SHA512) {
- digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
- digest_stream = STREAM_SHA512_DIGEST;
- }
-
- /** Did digest initialization fail? */
- if (digest_stream != STREAM_NONE && digest == NULL) {
- Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
- stream_to_ascii(digest_stream));
- }
-
- /**
- * Set up signature digest handling. If this fails, the signature digest
- * will be set to NULL and not used.
- */
- /* TODO landonf: We should really only calculate the digest once, for
- * both verification and signing.
- */
- if (jcr->crypto.pki_sign) {
- signing_digest = crypto_digest_new(jcr, signing_algorithm);
-
- /** Full-stop if a failure occurred initializing the signature digest */
- if (signing_digest == NULL) {
- Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
- stream_to_ascii(signing_algorithm));
- jcr->JobErrors++;
- goto good_rtn;
- }
- }
-
- /** Enable encryption */
- if (jcr->crypto.pki_encrypt) {
- ff_pkt->flags |= FO_ENCRYPT;
- }
+ if (has_file_data && !crypto_setup_digests(bctx)) {
+ goto good_rtn;
}
/** Initialize the file descriptor we use for data and other streams. */
}
/** Send attributes -- must be done after binit() */
- if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
+ if (!encode_and_send_attributes(bctx)) {
goto bail_out;
}
/** Meta data only for restore object */
int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
ff_pkt->bfd.reparse_point = (ff_pkt->type == FT_REPARSE ||
ff_pkt->type == FT_JUNCTION);
+ set_fattrs(&ff_pkt->bfd, &ff_pkt->statp);
if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
tid = NULL;
}
- stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
+ stat = send_data(bctx, bctx.data_stream);
if (ff_pkt->flags & FO_CHKCHANGES) {
has_file_changed(jcr, ff_pkt);
}
}
- if (have_darwin_os) {
- /** Regular files can have resource forks and Finder Info */
- if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
- ff_pkt->flags & FO_HFSPLUS)) {
- if (ff_pkt->hfsinfo.rsrclength > 0) {
- int flags;
- int rsrc_stream;
- if (bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
- ff_pkt->ff_errno = errno;
- berrno be;
- Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
- ff_pkt->fname, be.bstrerror());
- jcr->JobErrors++;
- if (is_bopen(&ff_pkt->bfd)) {
- bclose(&ff_pkt->bfd);
- }
- goto good_rtn;
- }
- flags = ff_pkt->flags;
- ff_pkt->flags &= ~(FO_COMPRESS|FO_SPARSE|FO_OFFSETS);
- if (flags & FO_ENCRYPT) {
- rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
- } else {
- rsrc_stream = STREAM_MACOS_FORK_DATA;
- }
- stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
- ff_pkt->flags = flags;
- bclose(&ff_pkt->bfd);
- if (!stat) {
- goto bail_out;
- }
- }
-
- Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
- sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
- Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
- pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
- sd->msglen = 32;
- if (digest) {
- crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
- }
- if (signing_digest) {
- crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
- }
- sd->send();
- sd->signal(BNET_EOD);
- }
- }
-
- /**
- * Save ACLs when requested and available for anything not being a symlink
- * and not being a plugin.
- */
- if (have_acl) {
- if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK && !ff_pkt->cmd_plugin) {
- switch (build_acl_streams(jcr, ff_pkt)) {
- case bacl_exit_fatal:
- goto bail_out;
- case bacl_exit_error:
- /**
- * Non-fatal errors, count them and when the number is under
- * ACL_REPORT_ERR_MAX_PER_JOB print the error message set by the
- * lower level routine in jcr->errmsg.
- */
- if (jcr->acl_data->u.build->nr_errors < ACL_REPORT_ERR_MAX_PER_JOB) {
- Jmsg(jcr, M_WARNING, 0, "%s", jcr->errmsg);
- }
- jcr->acl_data->u.build->nr_errors++;
- break;
- case bacl_exit_ok:
- break;
- }
- }
+#ifdef HAVE_DARWIN_OS
+ if (!send_resource_fork(bctx)) {
+ goto bail_out;
}
+#endif
- /**
- * Save Extended Attributes when requested and available for all files not
- * being a plugin.
+ /*
+ * Save ACLs and Extended Attributes when requested and available
+ * for anything not being a symlink and not being a plugin (why not?).
*/
- if (have_xattr) {
- if (ff_pkt->flags & FO_XATTR && !ff_pkt->cmd_plugin) {
- switch (build_xattr_streams(jcr, ff_pkt)) {
- case bxattr_exit_fatal:
- goto bail_out;
- case bxattr_exit_error:
- /**
- * Non-fatal errors, count them and when the number is under
- * XATTR_REPORT_ERR_MAX_PER_JOB print the error message set by the
- * lower level routine in jcr->errmsg.
- */
- if (jcr->xattr_data->u.build->nr_errors < XATTR_REPORT_ERR_MAX_PER_JOB) {
- Jmsg(jcr, M_WARNING, 0, "%s", jcr->errmsg);
- }
- jcr->xattr_data->u.build->nr_errors++;
- break;
- case bxattr_exit_ok:
- break;
- }
- }
- }
-
- /** Terminate the signing digest and send it to the Storage daemon */
- if (signing_digest) {
- uint32_t size = 0;
-
- if ((sig = crypto_sign_new(jcr)) == NULL) {
- Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
- goto bail_out;
- }
-
- if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
- Jmsg(jcr, M_FATAL, 0, _("An error occurred while adding signer the stream.\n"));
+ if (jcr->xacl){
+ if (jcr->xacl->backup_acl(jcr, ff_pkt) == bRC_XACL_error) {
goto bail_out;
}
-
- /** Get signature size */
- if (!crypto_sign_encode(sig, NULL, &size)) {
- Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
- goto bail_out;
- }
-
- /** Grow the bsock buffer to fit our message if necessary */
- if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
- sd->msg = realloc_pool_memory(sd->msg, size);
- }
-
- /** Send our header */
- sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
- Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
-
- /** Encode signature data */
- if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
- Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
+ if (jcr->xacl->backup_xattr(jcr, ff_pkt) == bRC_XACL_error) {
goto bail_out;
}
-
- sd->msglen = size;
- sd->send();
- sd->signal(BNET_EOD); /* end of checksum */
}
- /** Terminate any digest and send it to Storage daemon */
- if (digest) {
- uint32_t size;
-
- sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
- Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
-
- size = CRYPTO_DIGEST_MAX_SIZE;
-
- /** Grow the bsock buffer to fit our message if necessary */
- if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
- sd->msg = realloc_pool_memory(sd->msg, size);
- }
-
- if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
- Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
- goto bail_out;
- }
-
- /* Keep the checksum if this file is a hardlink */
- if (ff_pkt->linked) {
- ff_pkt_set_link_digest(ff_pkt, digest_stream, sd->msg, size);
- }
-
- sd->msglen = size;
- sd->send();
- sd->signal(BNET_EOD); /* end of checksum */
- }
-
- /* Check if original file has a digest, and send it */
- if (ff_pkt->type == FT_LNKSAVED && ff_pkt->digest) {
- Dmsg2(300, "Link %s digest %d\n", ff_pkt->fname, ff_pkt->digest_len);
- sd->fsend("%ld %d 0", jcr->JobFiles, ff_pkt->digest_stream);
-
- sd->msg = check_pool_memory_size(sd->msg, ff_pkt->digest_len);
- memcpy(sd->msg, ff_pkt->digest, ff_pkt->digest_len);
- sd->msglen = ff_pkt->digest_len;
- sd->send();
-
- sd->signal(BNET_EOD); /* end of hardlink record */
+ if (!crypto_terminate_digests(bctx)) {
+ goto bail_out;
}
good_rtn:
rtnstat = 1;
bail_out:
- if (jcr->is_canceled()) {
- Dmsg0(100, "Job canceled by user.\n");
+ if (jcr->is_incomplete() || jcr->is_canceled()) {
+ Dmsg0(100, "Job canceled by user or marked incomplete.\n");
rtnstat = 0;
}
if (plugin_started) {
jcr->plugin = NULL;
jcr->opt_plugin = false;
}
- if (digest) {
- crypto_digest_free(digest);
- }
- if (signing_digest) {
- crypto_digest_free(signing_digest);
- }
- if (sig) {
- crypto_sign_free(sig);
- }
+ crypto_free(bctx);
return rtnstat;
}
/**
* Send data read from an already open file descriptor.
*
- * We return 1 on sucess and 0 on errors.
+ * We return 1 on success and 0 on errors.
*
* ***FIXME***
* We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
* Currently this is not a problem as the only other stream, resource forks,
* are not handled as sparse files.
*/
-static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
- DIGEST *signing_digest)
+static int send_data(bctx_t &bctx, int stream)
{
+ JCR *jcr = bctx.jcr;
BSOCK *sd = jcr->store_bsock;
- uint64_t fileAddr = 0; /* file address */
- char *rbuf, *wbuf;
- int32_t rsize = jcr->buf_size; /* read buffer size */
- POOLMEM *msgsave;
- CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
- const uint8_t *cipher_input;
- uint32_t cipher_input_len;
- uint32_t cipher_block_size;
- uint32_t encrypted_len;
+
#ifdef FD_NO_SEND_TEST
return 1;
#endif
- msgsave = sd->msg;
- rbuf = sd->msg; /* read buffer */
- wbuf = sd->msg; /* write buffer */
- cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
-
- Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
-
-#if defined(HAVE_LIBZ) || defined(HAVE_LZO)
- uLong compress_len = 0;
- uLong max_compress_len = 0;
- const Bytef *cbuf = NULL;
- #ifdef HAVE_LIBZ
- int zstat;
+ bctx.rsize = jcr->buf_size;
+ bctx.fileAddr = 0;
+ bctx.cipher_ctx = NULL;
+ bctx.msgsave = sd->msg;
+ bctx.rbuf = sd->msg; /* read buffer */
+ bctx.wbuf = sd->msg; /* write buffer */
+ bctx.cipher_input = (uint8_t *)bctx.rbuf; /* encrypt uncompressed data */
- if ((ff_pkt->flags & FO_COMPRESS) && ff_pkt->Compress_algo == COMPRESS_GZIP) {
- if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
- cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
- max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
- } else {
- cbuf = (Bytef *)jcr->compress_buf;
- max_compress_len = jcr->compress_buf_size; /* set max length */
- }
- wbuf = jcr->compress_buf; /* compressed output here */
- cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
+ Dmsg1(300, "Saving data, type=%d\n", bctx.ff_pkt->type);
- /**
- * Only change zlib parameters if there is no pending operation.
- * This should never happen as deflatereset is called after each
- * deflate.
- */
-
- if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
- /** set gzip compression level - must be done per file */
- if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
- ff_pkt->Compress_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
- Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
- jcr->setJobStatus(JS_ErrorTerminated);
- goto err;
- }
- }
- }
- #endif
- #ifdef HAVE_LZO
- Bytef *cbuf2;
- int lzores;
- comp_stream_header ch;
-
- memset(&ch, 0, sizeof(comp_stream_header));
- cbuf2 = NULL;
-
- if ((ff_pkt->flags & FO_COMPRESS) && ff_pkt->Compress_algo == COMPRESS_LZO1X) {
- if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
- cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
- cbuf2 = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header);
- max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
- } else {
- cbuf = (Bytef *)jcr->compress_buf;
- cbuf2 = (Bytef *)jcr->compress_buf + sizeof(comp_stream_header);
- max_compress_len = jcr->compress_buf_size; /* set max length */
- }
- ch.magic = COMPRESS_LZO1X;
- ch.version = COMP_HEAD_VERSION;
- wbuf = jcr->compress_buf; /* compressed output here */
- cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
+ if (!setup_compression(bctx)) {
+ goto err;
}
- #endif
-#else
- const uint32_t max_compress_len = 0;
-#endif
- if (ff_pkt->flags & FO_ENCRYPT) {
- if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
- Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse or offset data not supported.\n"));
- goto err;
- }
- /** Allocate the cipher context */
- if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
- &cipher_block_size)) == NULL) {
- /* Shouldn't happen! */
- Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
- goto err;
- }
-
- /**
- * Grow the crypto buffer, if necessary.
- * crypto_cipher_update() will buffer up to (cipher_block_size - 1).
- * We grow crypto_buf to the maximum number of blocks that
- * could be returned for the given read buffer size.
- * (Using the larger of either rsize or max_compress_len)
- */
- jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
- (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
- cipher_block_size - 1) / cipher_block_size * cipher_block_size);
-
- wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
+ if (bctx.ff_pkt->flags & FO_ENCRYPT && !crypto_allocate_ctx(bctx)) {
+ return false;
}
/**
* <file-index> <stream> <expected stream length>
*/
if (!sd->fsend("%ld %d %lld", jcr->JobFiles, stream,
- (int64_t)ff_pkt->statp.st_size)) {
+ (int64_t)bctx.ff_pkt->statp.st_size)) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
* Make space at beginning of buffer for fileAddr because this
* same buffer will be used for writing if compression is off.
*/
- if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
- rbuf += OFFSET_FADDR_SIZE;
- rsize -= OFFSET_FADDR_SIZE;
-#ifdef HAVE_FREEBSD_OS
+ if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
+ bctx.rbuf += OFFSET_FADDR_SIZE;
+ bctx.rsize -= OFFSET_FADDR_SIZE;
+#if defined(HAVE_FREEBSD_OS) || defined(__FreeBSD_kernel__)
/**
* To read FreeBSD partitions, the read size must be
* a multiple of 512.
*/
- rsize = (rsize/512) * 512;
+ bctx.rsize = (bctx.rsize/512) * 512;
#endif
}
/** a RAW device read on win32 only works if the buffer is a multiple of 512 */
#ifdef HAVE_WIN32
- if (S_ISBLK(ff_pkt->statp.st_mode))
- rsize = (rsize/512) * 512;
-#endif
-
- /**
- * Read the file data
- */
- while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
-
- /** Check for sparse blocks */
- if (ff_pkt->flags & FO_SPARSE) {
- ser_declare;
- bool allZeros = false;
- if ((sd->msglen == rsize &&
- fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
- ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
- (uint64_t)ff_pkt->statp.st_size == 0)) {
- allZeros = is_buf_zero(rbuf, rsize);
- }
- if (!allZeros) {
- /** Put file address as first data in buffer */
- ser_begin(wbuf, OFFSET_FADDR_SIZE);
- ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
- }
- fileAddr += sd->msglen; /* update file address */
- /** Skip block of all zeros */
- if (allZeros) {
- continue; /* skip block of zeros */
- }
- } else if (ff_pkt->flags & FO_OFFSETS) {
- ser_declare;
- ser_begin(wbuf, OFFSET_FADDR_SIZE);
- ser_uint64(ff_pkt->bfd.offset); /* store offset in begin of buffer */
- }
-
- jcr->ReadBytes += sd->msglen; /* count bytes read */
-
- /** Uncompressed cipher input length */
- cipher_input_len = sd->msglen;
-
- /** Update checksum if requested */
- if (digest) {
- crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
- }
-
- /** Update signing digest if requested */
- if (signing_digest) {
- crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
- }
-
-#ifdef HAVE_LIBZ
- /** Do compression if turned on */
- if (ff_pkt->flags & FO_COMPRESS && ff_pkt->Compress_algo == COMPRESS_GZIP && jcr->pZLIB_compress_workset) {
- Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
-
- ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
- ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
- ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
- ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
-
- if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
- Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
- jcr->setJobStatus(JS_ErrorTerminated);
- goto err;
- }
- compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
- /** reset zlib stream to be able to begin from scratch again */
- if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
- Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
- jcr->setJobStatus(JS_ErrorTerminated);
- goto err;
- }
-
- Dmsg2(400, "GZIP compressed len=%d uncompressed len=%d\n", compress_len,
- sd->msglen);
-
- sd->msglen = compress_len; /* set compressed length */
- cipher_input_len = compress_len;
+ if (S_ISBLK(bctx.ff_pkt->statp.st_mode)) {
+ bctx.rsize = (bctx.rsize/512) * 512;
+ }
+ Dmsg1(200, "Fattrs=0X%x\n", bctx.ff_pkt->bfd.fattrs);
+ if (bctx.ff_pkt->bfd.fattrs & FILE_ATTRIBUTE_ENCRYPTED) {
+ if (!p_ReadEncryptedFileRaw) {
+ Jmsg0(bctx.jcr, M_FATAL, 0, _("Windows Encrypted data not supported on this OS.\n"));
+ goto err;
}
-#endif
-#ifdef HAVE_LZO
- /** Do compression if turned on */
- if (ff_pkt->flags & FO_COMPRESS && ff_pkt->Compress_algo == COMPRESS_LZO1X && jcr->LZO_compress_workset) {
- lzo_uint len; /* TODO: See with the latest patch how to handle lzo_uint with 64bit */
-
- ser_declare;
- ser_begin(cbuf, sizeof(comp_stream_header));
-
- Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
-
- lzores = lzo1x_1_compress((const unsigned char*)rbuf, sd->msglen, cbuf2,
- &len, jcr->LZO_compress_workset);
- compress_len = len;
- if (lzores == LZO_E_OK && compress_len <= max_compress_len) {
- /* complete header */
- ser_uint32(COMPRESS_LZO1X);
- ser_uint32(compress_len);
- ser_uint16(ch.level);
- ser_uint16(ch.version);
- } else {
- /** this should NEVER happen */
- Jmsg(jcr, M_FATAL, 0, _("Compression LZO error: %d\n"), lzores);
- jcr->setJobStatus(JS_ErrorTerminated);
- goto err;
- }
-
- Dmsg2(400, "LZO compressed len=%d uncompressed len=%d\n", compress_len,
- sd->msglen);
-
- compress_len += sizeof(comp_stream_header); /* add size of header */
- sd->msglen = compress_len; /* set compressed length */
- cipher_input_len = compress_len;
+ /* This single call reads all EFS data delivers it to a callback */
+ if (p_ReadEncryptedFileRaw((PFE_EXPORT_FUNC)read_efs_data_cb, &bctx,
+ bctx.ff_pkt->bfd.pvContext) != 0) {
+ goto err;
}
+ /* All read, so skip to finish sending */
+ goto finish_sending;
+ }
+ /* Fall through to standard bread() loop */
#endif
- /**
- * Note, here we prepend the current record length to the beginning
- * of the encrypted data. This is because both sparse and compression
- * restore handling want records returned to them with exactly the
- * same number of bytes that were processed in the backup handling.
- * That is, both are block filters rather than a stream. When doing
- * compression, the compression routines may buffer data, so that for
- * any one record compressed, when it is decompressed the same size
- * will not be obtained. Of course, the buffered data eventually comes
- * out in subsequent crypto_cipher_update() calls or at least
- * when crypto_cipher_finalize() is called. Unfortunately, this
- * "feature" of encryption enormously complicates the restore code.
- */
- if (ff_pkt->flags & FO_ENCRYPT) {
- uint32_t initial_len = 0;
- ser_declare;
-
- if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
- cipher_input_len += OFFSET_FADDR_SIZE;
- }
-
- /** Encrypt the length of the input block */
- uint8_t packet_len[sizeof(uint32_t)];
-
- ser_begin(packet_len, sizeof(uint32_t));
- ser_uint32(cipher_input_len); /* store data len in begin of buffer */
- Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
-
- if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
- (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
- /** Encryption failed. Shouldn't happen. */
- Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
- goto err;
- }
-
- /** Encrypt the input block */
- if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
- (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
- if ((initial_len + encrypted_len) == 0) {
- /** No full block of data available, read more data */
- continue;
- }
- Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
- sd->msglen);
- sd->msglen = initial_len + encrypted_len; /* set encrypted length */
- } else {
- /** Encryption failed. Shouldn't happen. */
- Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
- goto err;
- }
- }
-
- /* Send the buffer to the Storage daemon */
- if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
- sd->msglen += OFFSET_FADDR_SIZE; /* include fileAddr in size */
- }
- sd->msg = wbuf; /* set correct write buffer */
- if (!sd->send()) {
- if (!jcr->is_job_canceled()) {
- Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
- sd->bstrerror());
- }
+ /*
+ * Normal read the file data in a loop and send it to SD
+ */
+ while ((sd->msglen=(uint32_t)bread(&bctx.ff_pkt->bfd, bctx.rbuf, bctx.rsize)) > 0) {
+ if (!process_and_send_data(bctx)) {
goto err;
}
- Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
- /* #endif */
- jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
- sd->msg = msgsave; /* restore read buffer */
-
} /* end while read file data */
+ goto finish_sending;
+finish_sending:
if (sd->msglen < 0) { /* error */
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
- ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
+ bctx.ff_pkt->fname, be.bstrerror(bctx.ff_pkt->bfd.berrno));
if (jcr->JobErrors++ > 1000) { /* insanity check */
Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
}
- } else if (ff_pkt->flags & FO_ENCRYPT) {
+ } else if (bctx.ff_pkt->flags & FO_ENCRYPT) {
/**
* For encryption, we must call finalize to push out any
* buffered data.
*/
- if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
- &encrypted_len)) {
+ if (!crypto_cipher_finalize(bctx.cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
+ &bctx.encrypted_len)) {
/* Padding failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
goto err;
}
/** Note, on SSL pre-0.9.7, there is always some output */
- if (encrypted_len > 0) {
- sd->msglen = encrypted_len; /* set encrypted length */
- sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
+ if (bctx.encrypted_len > 0) {
+ sd->msglen = bctx.encrypted_len; /* set encrypted length */
+ sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
if (!sd->send()) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
- sd->msg = msgsave; /* restore bnet buffer */
+ sd->msg = bctx.msgsave; /* restore bnet buffer */
}
}
+
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
}
/** Free the cipher context */
- if (cipher_ctx) {
- crypto_cipher_free(cipher_ctx);
+ if (bctx.cipher_ctx) {
+ crypto_cipher_free(bctx.cipher_ctx);
}
return 1;
err:
/** Free the cipher context */
- if (cipher_ctx) {
- crypto_cipher_free(cipher_ctx);
+ if (bctx.cipher_ctx) {
+ crypto_cipher_free(bctx.cipher_ctx);
}
- sd->msg = msgsave; /* restore bnet buffer */
+ sd->msg = bctx.msgsave; /* restore bnet buffer */
sd->msglen = 0;
return 0;
}
-bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
+
+/*
+ * Apply processing (sparse, compression, encryption, and
+ * send to the SD.
+ */
+bool process_and_send_data(bctx_t &bctx)
{
- BSOCK *sd = jcr->store_bsock;
+ BSOCK *sd = bctx.sd;
+ JCR *jcr = bctx.jcr;
+
+ /** Check for sparse blocks */
+ if (bctx.ff_pkt->flags & FO_SPARSE) {
+ ser_declare;
+ bool allZeros = false;
+ if ((sd->msglen == bctx.rsize &&
+ bctx.fileAddr+sd->msglen < (uint64_t)bctx.ff_pkt->statp.st_size) ||
+ ((bctx.ff_pkt->type == FT_RAW || bctx.ff_pkt->type == FT_FIFO) &&
+ (uint64_t)bctx.ff_pkt->statp.st_size == 0)) {
+ allZeros = is_buf_zero(bctx.rbuf, bctx.rsize);
+ }
+ if (!allZeros) {
+ /** Put file address as first data in buffer */
+ ser_begin(bctx.wbuf, OFFSET_FADDR_SIZE);
+ ser_uint64(bctx.fileAddr); /* store fileAddr in begin of buffer */
+ }
+ bctx.fileAddr += sd->msglen; /* update file address */
+ /** Skip block of all zeros */
+ if (allZeros) {
+ return true; /* skip block of zeros */
+ }
+ } else if (bctx.ff_pkt->flags & FO_OFFSETS) {
+ ser_declare;
+ ser_begin(bctx.wbuf, OFFSET_FADDR_SIZE);
+ ser_uint64(bctx.ff_pkt->bfd.offset); /* store offset in begin of buffer */
+ }
+
+ jcr->ReadBytes += sd->msglen; /* count bytes read */
+
+ /** Uncompressed cipher input length */
+ bctx.cipher_input_len = sd->msglen;
+
+ /** Update checksum if requested */
+ if (bctx.digest) {
+ crypto_digest_update(bctx.digest, (uint8_t *)bctx.rbuf, sd->msglen);
+ }
+
+ /** Update signing digest if requested */
+ if (bctx.signing_digest) {
+ crypto_digest_update(bctx.signing_digest, (uint8_t *)bctx.rbuf, sd->msglen);
+ }
+
+ if (have_libz && !do_libz_compression(bctx)) {
+ goto err;
+ }
+
+ if (have_lzo && !do_lzo_compression(bctx)) {
+ goto err;
+ }
+
+ /**
+ * Note, here we prepend the current record length to the beginning
+ * of the encrypted data. This is because both sparse and compression
+ * restore handling want records returned to them with exactly the
+ * same number of bytes that were processed in the backup handling.
+ * That is, both are block filters rather than a stream. When doing
+ * compression, the compression routines may buffer data, so that for
+ * any one record compressed, when it is decompressed the same size
+ * will not be obtained. Of course, the buffered data eventually comes
+ * out in subsequent crypto_cipher_update() calls or at least
+ * when crypto_cipher_finalize() is called. Unfortunately, this
+ * "feature" of encryption enormously complicates the restore code.
+ */
+ if (bctx.ff_pkt->flags & FO_ENCRYPT) {
+ uint32_t initial_len = 0;
+ ser_declare;
+
+ if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
+ bctx.cipher_input_len += OFFSET_FADDR_SIZE;
+ }
+
+ /** Encrypt the length of the input block */
+ uint8_t packet_len[sizeof(uint32_t)];
+
+ ser_begin(packet_len, sizeof(uint32_t));
+ ser_uint32(bctx.cipher_input_len); /* store data len in begin of buffer */
+ Dmsg1(20, "Encrypt len=%d\n", bctx.cipher_input_len);
+
+ if (!crypto_cipher_update(bctx.cipher_ctx, packet_len, sizeof(packet_len),
+ (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
+ /** Encryption failed. Shouldn't happen. */
+ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
+ goto err;
+ }
+
+ /** Encrypt the input block */
+ if (crypto_cipher_update(bctx.cipher_ctx, bctx.cipher_input, bctx.cipher_input_len,
+ (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &bctx.encrypted_len)) {
+ if ((initial_len + bctx.encrypted_len) == 0) {
+ /** No full block of data available, read more data */
+ return true;
+ }
+ Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", bctx.encrypted_len,
+ sd->msglen);
+ sd->msglen = initial_len + bctx.encrypted_len; /* set encrypted length */
+ } else {
+ /** Encryption failed. Shouldn't happen. */
+ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
+ goto err;
+ }
+ }
+
+ /* Send the buffer to the Storage daemon */
+ if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
+ sd->msglen += OFFSET_FADDR_SIZE; /* include fileAddr in size */
+ }
+ sd->msg = bctx.wbuf; /* set correct write buffer */
+ if (!sd->send()) {
+ if (!jcr->is_job_canceled()) {
+ Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
+ sd->bstrerror());
+ }
+ goto err;
+ }
+ Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
+ /* #endif */
+ jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
+ sd->msg = bctx.msgsave; /* restore read buffer */
+ return true;
+
+err:
+ return false;
+}
+
+bool encode_and_send_attributes(bctx_t &bctx)
+{
+ BSOCK *sd = bctx.jcr->store_bsock;
+ JCR *jcr = bctx.jcr;
+ FF_PKT *ff_pkt = bctx.ff_pkt;
char attribs[MAXSTRING];
char attribsExBuf[MAXSTRING];
char *attribsEx = NULL;
int comp_len;
bool stat;
int hangup = get_hangup();
+ int blowup = get_blowup();
#ifdef FD_NO_SEND_TEST
return true;
#endif
Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
/** Find what data stream we will use, then encode the attributes */
- if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
+ if ((bctx.data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
/* This should not happen */
Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
return false;
}
- encode_stat(attribs, &ff_pkt->statp, sizeof(ff_pkt->statp), ff_pkt->LinkFI, data_stream);
+ encode_stat(attribs, &ff_pkt->statp, sizeof(ff_pkt->statp), ff_pkt->LinkFI, bctx.data_stream);
/** Now possibly extend the attributes */
if (IS_FT_OBJECT(ff_pkt->type)) {
pm_strcpy(jcr->last_fname, ff_pkt->fname);
jcr->unlock();
+ /* Display the information about the current file if requested */
+ if (is_message_type_set(jcr, M_SAVED)) {
+ ATTR attr;
+ memcpy(&attr.statp, &ff_pkt->statp, sizeof(struct stat));
+ attr.type = ff_pkt->type;
+ attr.ofname = (POOLMEM *)ff_pkt->fname;
+ attr.olname = (POOLMEM *)ff_pkt->link;
+ print_ls_output(jcr, &attr, M_SAVED);
+ }
+
/* Debug code: check if we must hangup */
- if (hangup && (jcr->JobFiles > (uint32_t)hangup)) {
+ if (hangup > 0 && (jcr->JobFiles > (uint32_t)hangup)) {
+ jcr->setJobStatus(JS_Incomplete);
Jmsg1(jcr, M_FATAL, 0, "Debug hangup requested after %d files.\n", hangup);
set_hangup(0);
return false;
}
+ if (blowup > 0 && (jcr->JobFiles > (uint32_t)blowup)) {
+ Jmsg1(jcr, M_ABORT, 0, "Debug blowup requested after %d files.\n", blowup);
+ return false;
+ }
+
/**
* Send Attributes header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
- if (!jcr->is_canceled()) {
- Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
- sd->bstrerror());
+ if (!jcr->is_canceled() && !jcr->is_incomplete()) {
+ Jmsg2(jcr, M_FATAL, 0, _("Network send error to SD. Data=%s ERR=%s\n"),
+ sd->msg, sd->bstrerror());
}
return false;
}
return stat;
}
-/**
+/*
+ * Setup bctx for doing compression
+ */
+static bool setup_compression(bctx_t &bctx)
+{
+ JCR *jcr = bctx.jcr;
+
+#if defined(HAVE_LIBZ) || defined(HAVE_LZO)
+ bctx.compress_len = 0;
+ bctx.max_compress_len = 0;
+ bctx.cbuf = NULL;
+ #ifdef HAVE_LIBZ
+ int zstat;
+
+ if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_GZIP) {
+ if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
+ bctx.cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
+ bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
+ } else {
+ bctx.cbuf = (Bytef *)jcr->compress_buf;
+ bctx.max_compress_len = jcr->compress_buf_size; /* set max length */
+ }
+ bctx.wbuf = jcr->compress_buf; /* compressed output here */
+ bctx.cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
+
+ /**
+ * Only change zlib parameters if there is no pending operation.
+ * This should never happen as deflatereset is called after each
+ * deflate.
+ */
+
+ if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
+ /** set gzip compression level - must be done per file */
+ if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
+ bctx.ff_pkt->Compress_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
+ Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
+ jcr->setJobStatus(JS_ErrorTerminated);
+ return false;
+ }
+ }
+ }
+ #endif
+ #ifdef HAVE_LZO
+ memset(&bctx.ch, 0, sizeof(comp_stream_header));
+ bctx.cbuf2 = NULL;
+
+ if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_LZO1X) {
+ if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
+ bctx.cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
+ bctx.cbuf2 = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header);
+ bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
+ } else {
+ bctx.cbuf = (Bytef *)jcr->compress_buf;
+ bctx.cbuf2 = (Bytef *)jcr->compress_buf + sizeof(comp_stream_header);
+ bctx.max_compress_len = jcr->compress_buf_size; /* set max length */
+ }
+ bctx.ch.magic = COMPRESS_LZO1X;
+ bctx.ch.version = COMP_HEAD_VERSION;
+ bctx.wbuf = jcr->compress_buf; /* compressed output here */
+ bctx.cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
+ }
+ #endif
+#else
+ bctx.max_compress_len = 0;
+#endif
+ return true;
+}
+
+/*
+ * Send MacOS resource fork to SD
+ */
+#ifdef HAVE_DARWIN_OS
+static bool send_resource_fork(bctx_t &bctx)
+{
+ FF_PKT *ff_pkt = bctx.ff_pkt;
+ JCR *jcr = bctx.jcr;
+ BSOCK *sd = bctx.sd;
+ int stat;
+
+ /** Regular files can have resource forks and Finder Info */
+ if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
+ ff_pkt->flags & FO_HFSPLUS)) {
+ if (ff_pkt->hfsinfo.rsrclength > 0) {
+ int flags;
+ int rsrc_stream;
+ if (bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
+ ff_pkt->ff_errno = errno;
+ berrno be;
+ Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
+ ff_pkt->fname, be.bstrerror());
+ jcr->JobErrors++;
+ if (is_bopen(&ff_pkt->bfd)) {
+ bclose(&ff_pkt->bfd);
+ }
+ return true;
+ }
+ flags = ff_pkt->flags;
+ ff_pkt->flags &= ~(FO_COMPRESS|FO_SPARSE|FO_OFFSETS);
+ if (flags & FO_ENCRYPT) {
+ rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
+ } else {
+ rsrc_stream = STREAM_MACOS_FORK_DATA;
+ }
+ stat = send_data(bctx, rsrc_stream);
+ ff_pkt->flags = flags;
+ bclose(&ff_pkt->bfd);
+ if (!stat) {
+ return false;
+ }
+ }
+
+ Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
+ sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
+ Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
+ pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
+ sd->msglen = 32;
+ if (bctx.digest) {
+ crypto_digest_update(bctx.digest, (uint8_t *)sd->msg, sd->msglen);
+ }
+ if (bctx.signing_digest) {
+ crypto_digest_update(bctx.signing_digest, (uint8_t *)sd->msg, sd->msglen);
+ }
+ sd->send();
+ sd->signal(BNET_EOD);
+ }
+ return true;
+}
+#endif
+
+static bool do_libz_compression(bctx_t &bctx)
+{
+#ifdef HAVE_LIBZ
+ JCR *jcr = bctx.jcr;
+ BSOCK *sd = bctx.sd;
+ int zstat;
+
+ /** Do compression if turned on */
+ if (bctx.ff_pkt->flags & FO_COMPRESS && bctx.ff_pkt->Compress_algo == COMPRESS_GZIP && jcr->pZLIB_compress_workset) {
+ Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", bctx.cbuf, bctx.rbuf, sd->msglen);
+
+ ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)bctx.rbuf;
+ ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
+ ((z_stream*)jcr->pZLIB_compress_workset)->next_out = bctx.cbuf;
+ ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = bctx.max_compress_len;
+
+ if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
+ Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
+ jcr->setJobStatus(JS_ErrorTerminated);
+ return false;
+ }
+ bctx.compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
+ /** reset zlib stream to be able to begin from scratch again */
+ if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
+ Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
+ jcr->setJobStatus(JS_ErrorTerminated);
+ return false;
+ }
+
+ Dmsg2(400, "GZIP compressed len=%d uncompressed len=%d\n", bctx.compress_len,
+ sd->msglen);
+
+ sd->msglen = bctx.compress_len; /* set compressed length */
+ bctx.cipher_input_len = bctx.compress_len;
+ }
+#endif
+ return true;
+}
+
+static bool do_lzo_compression(bctx_t &bctx)
+{
+#ifdef HAVE_LZO
+ JCR *jcr = bctx.jcr;
+ BSOCK *sd = bctx.sd;
+ int lzores;
+
+ /** Do compression if turned on */
+ if (bctx.ff_pkt->flags & FO_COMPRESS && bctx.ff_pkt->Compress_algo == COMPRESS_LZO1X && jcr->LZO_compress_workset) {
+ lzo_uint len; /* TODO: See with the latest patch how to handle lzo_uint with 64bit */
+
+ ser_declare;
+ ser_begin(bctx.cbuf, sizeof(comp_stream_header));
+
+ Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", bctx.cbuf, bctx.rbuf, sd->msglen);
+
+ lzores = lzo1x_1_compress((const unsigned char*)bctx.rbuf, sd->msglen, bctx.cbuf2,
+ &len, jcr->LZO_compress_workset);
+ bctx.compress_len = len;
+ if (lzores == LZO_E_OK && bctx.compress_len <= bctx.max_compress_len) {
+ /* complete header */
+ ser_uint32(COMPRESS_LZO1X);
+ ser_uint32(bctx.compress_len);
+ ser_uint16(bctx.ch.level);
+ ser_uint16(bctx.ch.version);
+ } else {
+ /** this should NEVER happen */
+ Jmsg(jcr, M_FATAL, 0, _("Compression LZO error: %d\n"), lzores);
+ jcr->setJobStatus(JS_ErrorTerminated);
+ return false;
+ }
+
+ Dmsg2(400, "LZO compressed len=%d uncompressed len=%d\n", bctx.compress_len,
+ sd->msglen);
+
+ bctx.compress_len += sizeof(comp_stream_header); /* add size of header */
+ sd->msglen = bctx.compress_len; /* set compressed length */
+ bctx.cipher_input_len = bctx.compress_len;
+ }
+#endif
+ return true;
+}
+
+/*
+ * Do in place strip of path
+ */
+static bool do_snap_strip(FF_PKT *ff)
+{
+ /* if the string starts with the snapshot path name, we can replace
+ * by the volume name. The volume_path is smaller than the snapshot_path
+ * snapshot_path = volume_path + /.snapshots/job-xxxx
+ */
+ ASSERT(strlen(ff->snapshot_path) > strlen(ff->volume_path));
+ int sp_first = strlen(ff->snapshot_path); /* point after snapshot_path in fname */
+ if (strncmp(ff->fname, ff->snapshot_path, sp_first) == 0) {
+ int last = pm_strcpy(ff->snap_fname, ff->volume_path);
+ last = MAX(last - 1, 0);
+
+ if (ff->snap_fname[last] == '/') {
+ if (ff->fname[sp_first] == '/') { /* compare with the first character of the string (sp_first not sp_first-1) */
+ ff->snap_fname[last] = 0;
+ }
+ } else {
+ if (ff->fname[sp_first] != '/') {
+ pm_strcat(ff->snap_fname, "/");
+ }
+ }
+
+ pm_strcat(ff->snap_fname, ff->fname + sp_first);
+ ASSERT(strlen(ff->fname) > strlen(ff->snap_fname));
+ strcpy(ff->fname, ff->snap_fname);
+ Dmsg2(DT_SNAPSHOT|20, "%s -> %s\n", ff->fname_save, ff->fname);
+ }
+ if (strncmp(ff->link, ff->snapshot_path, sp_first) == 0) {
+ int last = pm_strcpy(ff->snap_fname, ff->volume_path);
+ last = MAX(last - 1, 0);
+
+ if (ff->snap_fname[last] == '/') {
+ if (ff->link[sp_first] == '/') { /* compare with the first character of the string (sp_first not sp_first-1) */
+ ff->snap_fname[last] = 0;
+ }
+ } else {
+ if (ff->link[sp_first] != '/') {
+ pm_strcat(ff->snap_fname, "/");
+ }
+ }
+
+ pm_strcat(ff->snap_fname, ff->link + sp_first);
+ ASSERT(strlen(ff->link) > strlen(ff->snap_fname));
+ strcpy(ff->link, ff->snap_fname);
+ Dmsg2(DT_SNAPSHOT|20, "%s -> %s\n", ff->link_save, ff->link);
+ }
+
+ return true;
+}
+
+/*
* Do in place strip of path
*/
static bool do_strip(int count, char *in)
* for dealing with snapshots, by removing the snapshot directory, or
* in handling vendor migrations where files have been restored with
* a vendor product into a subdirectory.
+ *
+ * When we are using snapshots, we might need to convert the path
+ * back to the original one using the strip_snap_path option.
*/
void strip_path(FF_PKT *ff_pkt)
{
- if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
+ if (!ff_pkt->strip_snap_path &&
+ (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0))
+ {
Dmsg1(200, "No strip for %s\n", ff_pkt->fname);
return;
}
+ /* shared part between strip and snapshot */
if (!ff_pkt->fname_save) {
ff_pkt->fname_save = get_pool_memory(PM_FNAME);
ff_pkt->link_save = get_pool_memory(PM_FNAME);
+ *ff_pkt->link_save = 0;
}
pm_strcpy(ff_pkt->fname_save, ff_pkt->fname);
if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
Dsm_check(200);
}
+ if (ff_pkt->strip_snap_path) {
+ if (!do_snap_strip(ff_pkt)) {
+ Dmsg1(0, "Something wrong with do_snap_strip(%s)\n", ff_pkt->fname);
+ unstrip_path(ff_pkt);
+ goto rtn;
+ }
+ }
+
+ /* See if we want also to strip the path */
+ if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
+ goto rtn;
+ }
+
/**
* Strip path. If it doesn't succeed put it back. If
* it does, and there is a different link string,
}
rtn:
- Dmsg3(100, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
+ Dmsg3(10, "fname=%s stripped=%s link=%s\n", ff_pkt->fname_save, ff_pkt->fname,
ff_pkt->link);
}
void unstrip_path(FF_PKT *ff_pkt)
{
- if (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0) {
+ if (!ff_pkt->strip_snap_path &&
+ (!(ff_pkt->flags & FO_STRIPPATH) || ff_pkt->strip_path <= 0))
+ {
return;
}
+
strcpy(ff_pkt->fname, ff_pkt->fname_save);
if (ff_pkt->type != FT_LNK && ff_pkt->fname != ff_pkt->link) {
- Dmsg2(500, "strcpy link=%s link_save=%s\n", ff_pkt->link,
+ Dmsg2(10, "strcpy link=%s link_save=%s\n", ff_pkt->link,
ff_pkt->link_save);
strcpy(ff_pkt->link, ff_pkt->link_save);
- Dmsg2(500, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
+ Dmsg2(10, "strcpy link=%d link_save=%d\n", strlen(ff_pkt->link),
strlen(ff_pkt->link_save));
Dsm_check(200);
}
#if defined(WIN32_VSS)
/* STOP VSS ON WIN32 */
/* tell vss to close the backup session */
- if (jcr->VSS) {
+ if (jcr->Snapshot) {
if (g_pVSSClient->CloseBackup()) {
/* inform user about writer states */
for (int i=0; i<(int)g_pVSSClient->GetWriterCount(); i++) {
projects / bacula / bacula / blobdiff