enable or disable this feature. The configuration option could
specify the maximum number of threads in the file daemon.
- If the theads could spool the data to separate spool files
+ If the threads could spool the data to separate spool files
the restore process will not be much slower.
Why: Multiple concurrent backups of a large fileserver with many
/* Size of File Address stored in STREAM_SPARSE_DATA. Do NOT change! */
#define SPARSE_FADDR_SIZE (sizeof(uint64_t))
+/* Size of crypto length stored at head of crypto buffer. Do NOT change! */
+#define CRYPTO_LEN_SIZE ((int)sizeof(uint32_t))
+
/* This is for dumb compilers/libraries like Solaris. Linux GCC
* does it correctly, so it might be worthwhile
if (jcr->JobStatus != JS_Terminated) {
return false;
}
+
migration_cleanup(jcr, jcr->JobStatus);
if (mig_jcr) {
UAContext *ua = new_ua_context(jcr);
mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
set_jcr_job_status(mig_jcr, TermCode);
-
-
update_job_end_record(mig_jcr);
/* Update final items to set them to the previous job's values */
break;
}
} else {
- term_msg = _("%s -- no files to migrate");
+ msg_type = M_ERROR; /* Generate error message */
+ term_msg = _("*** %s Error ***");
}
bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
if (!get_cmd(ua, "$ ")) {
break;
}
- parse_ua_args(ua);
+ parse_args_only(ua->cmd, &ua->args, &ua->argc, ua->argk, ua->argv, MAX_CMD_ARGS);
if (ua->argc == 0) {
bsendmsg(tree->ua, _("Illegal command. Enter \"done\" to exit.\n"));
continue;
TREE_NODE *node;
char cwd[2000];
+
if (ua->argc != 2) {
+ bsendmsg(ua, _("Too many arguments. Try using double quotes.\n"));
return 1;
}
- strip_leading_space(ua->argk[1]);
node = tree_cwd(ua->argk[1], tree->root, tree->node);
if (!node) {
/* Try once more if Win32 drive -- make absolute */
* Read the file data
*/
while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
- int sparseBlock = 0;
+ bool sparseBlock = false;
/* Check for sparse blocks */
if (ff_pkt->flags & FO_SPARSE) {
ser_begin(wbuf, SPARSE_FADDR_SIZE);
ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
}
+ fileAddr += sd->msglen; /* update file address */
+ if (sparseBlock) {
+ continue; /* skip block of zeros */
+ }
}
jcr->ReadBytes += sd->msglen; /* count bytes read */
- fileAddr += sd->msglen;
/* Uncompressed cipher input length */
cipher_input_len = sd->msglen;
#ifdef HAVE_LIBZ
/* Do compression if turned on */
- if (!sparseBlock && (ff_pkt->flags & FO_GZIP) && jcr->pZLIB_compress_workset) {
+ if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
cipher_input_len = compress_len;
}
#endif
-
- if (!sparseBlock && (ff_pkt->flags & FO_ENCRYPT)) {
+ /*
+ * Note, here we prepend the current record length to the beginning
+ * of the encrypted data. This is because both sparse and compression
+ * restore handling want records returned to them with exactly the
+ * same number of bytes that were processed in the backup handling.
+ * That is, both are block filters rather than a stream. When doing
+ * compression, the compression routines may buffer data, so that for
+ * any one record compressed, when it is decompressed the same size
+ * will not be obtained. Of course, the buffered data eventually comes
+ * out in subsequent crypto_cipher_update() calls or at least
+ * when crypto_cipher_finalize() is called. Unfortunately, this
+ * "feature" of encryption enormously complicates the restore code.
+ */
+ if (ff_pkt->flags & FO_ENCRYPT) {
uint32_t initial_len = 0;
ser_declare;
uint8_t packet_len[sizeof(uint32_t)];
ser_begin(packet_len, sizeof(uint32_t));
- ser_uint32(cipher_input_len); /* store fileAddr in begin of buffer */
+ ser_uint32(cipher_input_len); /* store data len in begin of buffer */
+ Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
(u_int8_t *)jcr->crypto_buf, &initial_len)) {
}
/* Send the buffer to the Storage daemon */
- if (!sparseBlock) {
- if (ff_pkt->flags & FO_SPARSE) {
- sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
- }
- sd->msg = wbuf; /* set correct write buffer */
- if (!bnet_send(sd)) {
- Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
- bnet_strerror(sd));
- goto err;
- }
+ if (ff_pkt->flags & FO_SPARSE) {
+ sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
+ }
+ sd->msg = wbuf; /* set correct write buffer */
+ if (!bnet_send(sd)) {
+ Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
+ bnet_strerror(sd));
+ goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
/* #endif */
} /* end while read file data */
- /* Send any remaining encrypted data + padding */
- if (sd->msglen >= 0) {
- if (ff_pkt->flags & FO_ENCRYPT) {
- if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf,
- &encrypted_len)) {
- /* Padding failed. Shouldn't happen. */
- Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
- goto err;
- }
-
- if (encrypted_len > 0) {
- sd->msglen = encrypted_len; /* set encrypted length */
-
- sd->msg = jcr->crypto_buf; /* set correct write buffer */
- if (!bnet_send(sd)) {
- Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
- bnet_strerror(sd));
- goto err;
- }
- Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
- jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
- sd->msg = msgsave; /* restore bnet buffer */
- }
- }
- } else {
+ if (sd->msglen < 0) { /* error */
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
if (jcr->Errors++ > 1000) { /* insanity check */
Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
}
+ } else if (ff_pkt->flags & FO_ENCRYPT) {
+ /*
+ * For encryption, we must call finalize to push out any
+ * buffered data.
+ */
+ if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto_buf,
+ &encrypted_len)) {
+ /* Padding failed. Shouldn't happen. */
+ Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
+ goto err;
+ }
+
+ /* Note, on SSL pre-0.9.7, there is always some output */
+ if (encrypted_len > 0) {
+ sd->msglen = encrypted_len; /* set encrypted length */
+ sd->msg = jcr->crypto_buf; /* set correct write buffer */
+ if (!bnet_send(sd)) {
+ Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
+ bnet_strerror(sd));
+ goto err;
+ }
+ Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
+ jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
+ sd->msg = msgsave; /* restore bnet buffer */
+ }
}
if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
#endif
}
+static void unser_crypto_size(JCR *jcr)
+{
+ unser_declare;
+ if (jcr->crypto_size == 0 && jcr->crypto_count >= CRYPTO_LEN_SIZE) {
+ unser_begin(&jcr->crypto_buf[0], CRYPTO_LEN_SIZE);
+ unser_uint32(jcr->crypto_size);
+ jcr->crypto_size += CRYPTO_LEN_SIZE;
+ }
+}
+
bool store_data(JCR *jcr, BFILE *bfd, char *data, const int32_t length, bool win32_decomp)
{
if (win32_decomp) {
if (flags & FO_ENCRYPT) {
ASSERT(cipher);
- unser_declare;
while (jcr->crypto_size > 0 && jcr->crypto_count > 0 && wsize > 0) {
uint32_t chunk_size = 16;
wsize -= chunk_size;
if (jcr->crypto_count >= jcr->crypto_size) {
-
- char *packet = &jcr->crypto_buf[4]; /* Decrypted, possibly decompressed output here. */
- uint32_t packet_size = jcr->crypto_size - 4;
+ char *packet = &jcr->crypto_buf[CRYPTO_LEN_SIZE]; /* Decrypted, possibly decompressed output here. */
+ uint32_t packet_size = jcr->crypto_size - CRYPTO_LEN_SIZE;
if (flags & FO_GZIP) {
if (!decompress_data(jcr, &packet, &packet_size)) {
jcr->crypto_count += decrypted_len;
- if (jcr->crypto_size == 0 && jcr->crypto_count >= 4) {
- unser_begin(&jcr->crypto_buf[0], sizeof(uint32_t));
- unser_uint32(jcr->crypto_size);
- jcr->crypto_size += 4;
- }
+ unser_crypto_size(jcr);
+ wsize = jcr->crypto_size - CRYPTO_LEN_SIZE;
+ Dmsg1(10, "Decrypt size=%d\n", wsize);
+ wbuf = &jcr->crypto_buf[CRYPTO_LEN_SIZE]; /* Decrypted, possibly decompressed output here. */
if (jcr->crypto_size == 0 || jcr->crypto_count < jcr->crypto_size) {
return 0;
}
- wsize = jcr->crypto_size - 4;
- wbuf = &jcr->crypto_buf[4]; /* Decrypted, possibly decompressed output here. */
}
if (flags & FO_SPARSE) {
char ec1[50]; /* Buffer printing huge values */
/* Write out the remaining block and free the cipher context */
- jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf, jcr->crypto_count + cipher_block_size);
+ jcr->crypto_buf = check_pool_memory_size(jcr->crypto_buf, jcr->crypto_count +
+ cipher_block_size);
- if (!crypto_cipher_finalize(cipher, (uint8_t *)&jcr->crypto_buf[jcr->crypto_count], &decrypted_len)) {
+ if (!crypto_cipher_finalize(cipher, (uint8_t *)&jcr->crypto_buf[jcr->crypto_count],
+ &decrypted_len)) {
/* Writing out the final, buffered block failed. Shouldn't happen. */
Jmsg1(jcr, M_FATAL, 0, _("Decryption error for %s\n"), jcr->last_fname);
}
jcr->crypto_count += decrypted_len;
- if (jcr->crypto_size == 0) {
- ASSERT(jcr->crypto_count >= 4);
- jcr->crypto_size = ntohl(*(uint32_t *)&jcr->crypto_buf[0]) + 4;
- }
+ unser_crypto_size(jcr);
+ wsize = jcr->crypto_size - CRYPTO_LEN_SIZE;
+ Dmsg1(10, "Unser size=%d\n", wsize);
+ wbuf = &jcr->crypto_buf[CRYPTO_LEN_SIZE]; /* Decrypted, possibly decompressed output here. */
ASSERT(jcr->crypto_count == jcr->crypto_size);
- wbuf = &jcr->crypto_buf[4];
- wsize = jcr->crypto_size - 4;
-
if (flags & FO_GZIP) {
decompress_data(jcr, &wbuf, &wsize);
} else {
* Returns: true on success
* false on failure
*/
-bool crypto_digest_finalize (DIGEST *digest, uint8_t *dest, uint32_t *length) {
+bool crypto_digest_finalize (DIGEST *digest, uint8_t *dest, uint32_t *length)
+{
if (!EVP_DigestFinal(&digest->ctx, dest, (unsigned int *)length)) {
return false;
} else {
char *next_arg(char **s);
int parse_args(POOLMEM *cmd, POOLMEM **args, int *argc,
char **argk, char **argv, int max_args);
+int parse_args_only(POOLMEM *cmd, POOLMEM **args, int *argc,
+ char **argk, char **argv, int max_args);
void split_path_and_filename(const char *fname, POOLMEM **path,
int *pnl, POOLMEM **file, int *fnl);
int bsscanf(const char *buf, const char *fmt, ...);
/* util.c */
-int is_buf_zero (char *buf, int len);
+bool is_buf_zero (char *buf, int len);
void lcase (char *str);
void bash_spaces (char *str);
void bash_spaces (POOL_MEM &pm);
/*
* Return next argument from command line. Note, this
- * routine is destructive.
+ * routine is destructive because it stored 0 at the end
+ * of each argument.
+ * Called with pointer to pointer to command line. This
+ * pointer is updated to point to the remainder of the
+ * command line.
+ *
+ * Returns pointer to next argument -- don't store the result
+ * in the pointer you passed as an argument ...
+ * The next argument is terminated by a space unless within
+ * quotes. Double quote characters (unless preceded by a \) are
+ * stripped.
+ *
*/
char *next_arg(char **s)
{
}
Dmsg1(900, "Next arg=%s\n", p);
for (n = q = p; *p ; ) {
- if (*p == '\\') {
- p++;
+ if (*p == '\\') { /* slash? */
+ p++; /* yes, skip it */
if (*p) {
*q++ = *p++;
} else {
continue;
}
if (*p == '"') { /* start or end of quote */
- if (in_quote) {
- p++; /* skip quote */
- in_quote = false;
- continue;
- }
- in_quote = true;
p++;
+ in_quote = !in_quote; /* change state */
continue;
}
- if (!in_quote && B_ISSPACE(*p)) { /* end of field */
+ if (!in_quote && B_ISSPACE(*p)) { /* end of field */
p++;
break;
}
/*
* This routine parses the input command line.
* It makes a copy in args, then builds an
- * argc, argv like list where
+ * argc, argk, argv list where:
*
* argc = count of arguments
* argk[i] = argument keyword (part preceding =)
* argk[2] = arg3
* argv[2] =
*/
-
int parse_args(POOLMEM *cmd, POOLMEM **args, int *argc,
char **argk, char **argv, int max_args)
{
- char *p, *q, *n;
+ char *p;
+
+ parse_args_only(cmd, args, argc, argk, argv, max_args);
- pm_strcpy(args, cmd);
- strip_trailing_junk(*args);
- p = *args;
- *argc = 0;
- /* Pick up all arguments */
- while (*argc < max_args) {
- n = next_arg(&p);
- if (*n) {
- argk[*argc] = n;
- argv[(*argc)++] = NULL;
- } else {
- break;
- }
- }
/* Separate keyword and value */
for (int i=0; i < *argc; i++) {
p = strchr(argk[i], '=');
if (p) {
*p++ = 0; /* terminate keyword and point to value */
- /* Unquote quoted values */
- if (*p == '"') {
- for (n = q = ++p; *p && *p != '"'; ) {
- if (*p == '\\') {
- p++;
- }
- *q++ = *p++;
- }
- *q = 0; /* terminate string */
- p = n; /* point to string */
- }
if (strlen(p) > MAX_NAME_LENGTH-1) {
p[MAX_NAME_LENGTH-1] = 0; /* truncate to max len */
}
}
argv[i] = p; /* save ptr to value or NULL */
}
-#ifdef xxxx
+#ifdef xxx_debug
for (int i=0; i < *argc; i++) {
Pmsg3(000, "Arg %d: kw=%s val=%s\n", i, argk[i], argv[i]?argv[i]:"NULL");
}
return 1;
}
+
+/*
+ * This routine parses the input command line.
+ * It makes a copy in args, then builds an
+ * argc, argk, but no argv (values).
+ * This routine is useful for scanning command lines where the data
+ * is a filename and no keywords are expected. If we scan a filename
+ * for keywords, any = in the filename will be interpreted as the
+ * end of a keyword, and this is not good.
+ *
+ * argc = count of arguments
+ * argk[i] = argument keyword (part preceding =)
+ * argv[i] = NULL
+ *
+ * example: arg1 arg2=abc arg3=
+ *
+ * argc = c
+ * argk[0] = arg1
+ * argv[0] = NULL
+ * argk[1] = arg2=abc
+ * argv[1] = NULL
+ * argk[2] = arg3
+ * argv[2] =
+ */
+int parse_args_only(POOLMEM *cmd, POOLMEM **args, int *argc,
+ char **argk, char **argv, int max_args)
+{
+ char *p, *n;
+
+ pm_strcpy(args, cmd);
+ strip_trailing_junk(*args);
+ p = *args;
+ *argc = 0;
+ /* Pick up all arguments */
+ while (*argc < max_args) {
+ n = next_arg(&p);
+ if (*n) {
+ argk[*argc] = n;
+ argv[(*argc)++] = NULL;
+ } else {
+ break;
+ }
+ }
+ return 1;
+}
+
+
/*
* Given a full filename, split it into its path
* and filename parts. They are returned in pool memory
*/
/* Return true of buffer has all zero bytes */
-int is_buf_zero(char *buf, int len)
+bool is_buf_zero(char *buf, int len)
{
uint64_t *ip;
char *p;
int i, len64, done, rem;
if (buf[0] != 0) {
- return 0;
+ return false;
}
ip = (uint64_t *)buf;
/* Optimize by checking uint64_t for zero */
len64 = len / sizeof(uint64_t);
for (i=0; i < len64; i++) {
if (ip[i] != 0) {
- return 0;
+ return false;
}
}
done = len64 * sizeof(uint64_t); /* bytes already checked */
rem = len - done;
for (i = 0; i < rem; i++) {
if (p[i] != 0) {
- return 0;
+ return false;
}
}
- return 1;
+ return true;
}
}
return NULL;
}
-
#undef VERSION
#define VERSION "1.39.29"
-#define BDATE "01 December 2006"
-#define LSMDATE "01Dec06"
+#define BDATE "02 December 2006"
+#define LSMDATE "02Dec06"
#define PROG_COPYRIGHT "Copyright (C) %d-2006 Free Software Foundation Europe e.V.\n"
#define BYEAR "2006" /* year for copyright messages in progs */
Technical notes on version 1.39
General:
+03Dec06
+kes Move unserial code in restore.c to a subroutine. Add a bit of debug
+ code.
+kes Rework a bit of code in backup.c to handle sparse blocks correctly.
+ The main problem was that signatures were being generated on blocks
+ of zeros, which is unnecessary.
+02Dec06
+kes Fix scanner (next_arg) to handle leading double quote correctly.
+kes Modify cd command in restore tree to look at full argument without
+ keywords. This fixes bug #716.
01Dec06
kes Do not update Migrated Job type if migration does not terminate
normally. This fixes bug #719.
projects / bacula / bacula / commitdiff