/*
- * Append code for Storage daemon
- * Kern Sibbald, May MM
- *
- * Version $Id$
- */
-/*
- Copyright (C) 2000-2006 Kern Sibbald
+ Bacula(R) - The Network Backup Solution
+
+ Copyright (C) 2000-2015 Kern Sibbald
+ Copyright (C) 2000-2014 Free Software Foundation Europe e.V.
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License
- version 2 as amended with additional clauses defined in the
- file LICENSE in the main source directory.
+ The original author of Bacula is Kern Sibbald, with contributions
+ from many others, a complete list can be found in the file AUTHORS.
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- the file LICENSE for additional details.
+ You may use this file and others of this release according to the
+ license defined in the LICENSE file, which includes the Affero General
+ Public License, v3.0 ("AGPLv3") and some additional permissions and
+ terms pursuant to its AGPLv3 Section 7.
+ This notice must be preserved when any source code is
+ conveyed and/or propagated.
+
+ Bacula(R) is a registered trademark of Kern Sibbald.
+*/
+/*
+ * Append code for Storage daemon
+ * Kern Sibbald, May MM
+ *
*/
#include "bacula.h"
/* Responses sent to the File daemon */
static char OK_data[] = "3000 OK data\n";
+static char OK_append[] = "3000 OK append data\n";
/* Forward referenced functions */
+
+/*
+ * Check if we can mark this job incomplete
+ *
+ */
+void possible_incomplete_job(JCR *jcr, int32_t last_file_index)
+{
+ /*
+ * Note, here we decide if it is worthwhile to restart
+ * the Job at this point. For the moment, if at least
+ * 10 Files have been seen.
+ * We must be sure that the saved files are safe.
+ * Using this function when their is as comm line problem is probably safe,
+ * it is inappropriate to use it for a any failure that could
+ * involve corrupted data.
+ */
+ if (jcr->spool_attributes && last_file_index > 10) {
+ jcr->setJobStatus(JS_Incomplete);
+ }
+}
/*
- * Append Data sent from File daemon
+ * Append Data sent from Client (FD/SD)
*
*/
bool do_append_data(JCR *jcr)
{
int32_t n;
int32_t file_index, stream, last_file_index;
- BSOCK *ds;
- BSOCK *fd_sock = jcr->file_bsock;
+ uint64_t stream_len;
+ BSOCK *fd = jcr->file_bsock;
bool ok = true;
DEV_RECORD rec;
char buf1[100], buf2[100];
DCR *dcr = jcr->dcr;
DEVICE *dev;
char ec[50];
+ POOLMEM *eblock = NULL;
+ POOL_MEM errmsg(PM_EMSG);
-
- if (!dcr) {
- Jmsg0(jcr, M_FATAL, 0, _("DCR is NULL!!!\n"));
+ if (!dcr) {
+ pm_strcpy(jcr->errmsg, _("DCR is NULL!!!\n"));
+ Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
- }
+ }
dev = dcr->dev;
- if (!dev) {
- Jmsg0(jcr, M_FATAL, 0, _("DEVICE is NULL!!!\n"));
+ if (!dev) {
+ pm_strcpy(jcr->errmsg, _("DEVICE is NULL!!!\n"));
+ Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
- }
+ }
- Dmsg1(100, "Start append data. res=%d\n", dev->reserved_device);
+ Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved());
memset(&rec, 0, sizeof(rec));
- ds = fd_sock;
-
- if (!bnet_set_buffer_size(ds, dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
- set_jcr_job_status(jcr, JS_ErrorTerminated);
- Jmsg0(jcr, M_FATAL, 0, _("Unable to set network buffer size.\n"));
+ if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
+ jcr->setJobStatus(JS_ErrorTerminated);
+ pm_strcpy(jcr->errmsg, _("Unable to set network buffer size.\n"));
+ Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
if (!acquire_device_for_append(dcr)) {
- set_jcr_job_status(jcr, JS_ErrorTerminated);
- jcr->dcr = NULL;
+ jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
- set_jcr_job_status(jcr, JS_Running);
- dir_send_job_status(jcr);
+ jcr->sendJobStatus(JS_Running);
+ //ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
begin_attribute_spool(jcr);
Dmsg0(100, "Just after acquire_device_for_append\n");
+ //ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
if (!write_session_label(dcr, SOS_LABEL)) {
Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"),
dev->bstrerror());
- set_jcr_job_status(jcr, JS_ErrorTerminated);
+ jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
+
+ //ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/* Tell File daemon to send data */
- if (!bnet_fsend(fd_sock, OK_data)) {
+ if (!fd->fsend(OK_data)) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"),
- be.strerror(fd_sock->b_errno));
+ be.bstrerror(fd->b_errno));
ok = false;
}
* - Multiple records of data
* - EOD record
*
- * The Stream header is just used to sychronize things, and
+ * The Stream header is just used to synchronize things, and
* none of the stream header is written to tape.
* The Multiple records of data, contain first the Attributes,
* then after another stream header, the file data, then
*/
dcr->VolFirstIndex = dcr->VolLastIndex = 0;
jcr->run_time = time(NULL); /* start counting time for rates */
- for (last_file_index = 0; ok && !job_canceled(jcr); ) {
+
+ GetMsg *qfd;
+
+ qfd = New(GetMsg(jcr, fd, NULL, GETMSG_MAX_MSG_SIZE));
+ qfd->start_read_sock();
+
+ for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) {
/* Read Stream header from the File daemon.
* The stream header consists of the following:
* file_index (sequential Bacula file index, base 1)
* stream (Bacula number to distinguish parts of data)
- * info (Info for Storage daemon -- compressed, encryped, ...)
- * info is not currently used, so is read, but ignored!
+ * stream_len (Expected length of this stream. This
+ * will be the size backed up if the file does not
+ * grow during the backup.
*/
- if ((n=bget_msg(ds)) <= 0) {
- if (n == BNET_SIGNAL && ds->msglen == BNET_EOD) {
+ n = qfd->bget_msg(NULL);
+ if (n <= 0) {
+ if (n == BNET_SIGNAL && qfd->msglen == BNET_EOD) {
+ Dmsg0(200, "Got EOD on reading header.\n");
break; /* end of data */
}
- Jmsg1(jcr, M_FATAL, 0, _("Error reading data header from FD. ERR=%s\n"),
- bnet_strerror(ds));
+ Jmsg3(jcr, M_FATAL, 0, _("Error reading data header from FD. n=%d msglen=%d ERR=%s\n"),
+ n, qfd->msglen, fd->bstrerror());
+ // ASX TODO the fd->bstrerror() can be related to the wrong error, I should Queue the error too
+ possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
}
- /*
- * This hand scanning is a bit more complicated than a simple
- * sscanf, but it allows us to handle any size integer up to
- * int64_t without worrying about whether %d, %ld, %lld, or %q
- * is the correct format for each different architecture.
- * It is a real pity that sscanf() is not portable.
- */
- char *p = ds->msg;
- while (B_ISSPACE(*p)) {
- p++;
- }
- file_index = (int32_t)str_to_int64(p);
- while (B_ISDIGIT(*p)) {
- p++;
- }
- if (!B_ISSPACE(*p) || !B_ISDIGIT(*(p+1))) {
- Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), ds->msg);
+ if (sscanf(qfd->msg, "%ld %ld %lld", &file_index, &stream, &stream_len) != 3) {
+ // TODO ASX already done in bufmsg, should reuse the values
+ char buf[256];
+ Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), asciidump(qfd->msg, qfd->msglen, buf, sizeof(buf)));
ok = false;
+ possible_incomplete_job(jcr, last_file_index);
break;
}
- stream = (int32_t)str_to_int64(p);
- Dmsg2(890, "<filed: Header FilInx=%d stream=%d\n", file_index, stream);
+ Dmsg3(890, "<filed: Header FilInx=%d stream=%d stream_len=%lld\n",
+ file_index, stream, stream_len);
- if (!(file_index > 0 && (file_index == last_file_index ||
- file_index == last_file_index + 1))) {
- Jmsg0(jcr, M_FATAL, 0, _("File index from FD not positive or sequential\n"));
- ok = false;
- break;
+ /*
+ * We make sure the file_index is advancing sequentially.
+ * An incomplete job can start the file_index at any number.
+ * otherwise, it must start at 1.
+ */
+ if (jcr->rerunning && file_index > 0 && last_file_index == 0) {
+ goto fi_checked;
+ }
+ Dmsg2(400, "file_index=%d last_file_index=%d\n", file_index, last_file_index);
+ if (file_index > 0 && (file_index == last_file_index ||
+ file_index == last_file_index + 1)) {
+ goto fi_checked;
}
+ Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or last_FI=%d\n"),
+ file_index, last_file_index);
+ possible_incomplete_job(jcr, last_file_index);
+ ok = false;
+ break;
+
+fi_checked:
if (file_index != last_file_index) {
jcr->JobFiles = file_index;
last_file_index = file_index;
/* Read data stream from the File daemon.
* The data stream is just raw bytes
*/
- while ((n=bget_msg(ds)) > 0 && !job_canceled(jcr)) {
+ while ((n=qfd->bget_msg(NULL)) > 0 && !jcr->is_job_canceled()) {
+
rec.VolSessionId = jcr->VolSessionId;
rec.VolSessionTime = jcr->VolSessionTime;
rec.FileIndex = file_index;
rec.Stream = stream;
- rec.data_len = ds->msglen;
- rec.data = ds->msg; /* use message buffer */
+ rec.StreamLen = stream_len;
+ rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */
+ rec.data_len = qfd->msglen;
+ rec.data = qfd->msg; /* use message buffer */
Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n",
- rec.FileIndex, rec.VolSessionId,
+ rec.FileIndex, rec.VolSessionId,
stream_to_ascii(buf1, rec.Stream,rec.FileIndex),
rec.data_len);
-
- while (!write_record_to_block(dcr->block, &rec)) {
- Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", rec.data_len,
- rec.remainder);
- if (!write_block_to_device(dcr)) {
- Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
- dev->print_name(), dev->bstrerror());
- ok = false;
- break;
- }
- }
+ ok = dcr->write_record(&rec);
if (!ok) {
- Dmsg0(400, "Not OK\n");
+ Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
+ dcr->dev->print_name(), dcr->dev->bstrerror());
break;
}
jcr->JobBytes += rec.data_len; /* increment bytes this job */
+ jcr->JobBytes += qfd->bmsg->jobbytes; // if the block as been downloaded, count it
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId,
stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len);
- /* Send attributes and digest to Director for Catalog */
- if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX ||
- crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
- if (!jcr->no_attributes) {
- if (are_attributes_spooled(jcr)) {
- jcr->dir_bsock->spool = true;
- }
- Dmsg0(850, "Send attributes to dir.\n");
- if (!dir_update_file_attributes(dcr, &rec)) {
- jcr->dir_bsock->spool = false;
- Jmsg(jcr, M_FATAL, 0, _("Error updating file attributes. ERR=%s\n"),
- bnet_strerror(jcr->dir_bsock));
- ok = false;
- break;
- }
- jcr->dir_bsock->spool = false;
- }
- }
+ send_attrs_to_dir(jcr, &rec);
Dmsg0(650, "Enter bnet_get\n");
}
- Dmsg1(650, "End read loop with FD. Stat=%d\n", n);
-
- if (is_bnet_error(ds)) {
- Dmsg1(350, "Network read error from FD. ERR=%s\n", bnet_strerror(ds));
- Jmsg1(jcr, M_FATAL, 0, _("Network error on data channel. ERR=%s\n"),
- bnet_strerror(ds));
+ Dmsg2(650, "End read loop with FD. JobFiles=%d Stat=%d\n", jcr->JobFiles, n);
+
+ if (fd->is_error()) {
+ if (!jcr->is_job_canceled()) {
+ Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror());
+ Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"),
+ fd->bstrerror());
+ possible_incomplete_job(jcr, last_file_index);
+ }
ok = false;
break;
}
}
- time_t job_elapsed = time(NULL) - jcr->run_time;
+ qfd->wait_read_sock();
+ free_GetMsg(qfd);
- if (job_elapsed <= 0) {
- job_elapsed = 1;
+ if (eblock != NULL) {
+ free_pool_memory(eblock);
}
- Jmsg(dcr->jcr, M_INFO, 0, _("Job write elapsed time = %02d:%02d:%02d, Transfer rate = %s bytes/second\n"),
- job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
- edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
-
/* Create Job status for end of session label */
- set_jcr_job_status(jcr, ok?JS_Terminated:JS_ErrorTerminated);
+ jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated);
+
+ if (ok) {
+ /* Terminate connection with Client */
+ fd->fsend(OK_append);
+ do_client_commands(jcr); /* finish dialog with Client */
+ } else {
+ fd->fsend("3999 Failed append\n");
+ }
Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus);
/*
- * If !OK, check if we can still write. This may not be the case
+ * Check if we can still write. This may not be the case
* if we are at the end of the tape or we got a fatal I/O error.
*/
if (ok || dev->can_write()) {
if (!write_session_label(dcr, EOS_LABEL)) {
- Jmsg1(jcr, M_FATAL, 0, _("Error writting end session label. ERR=%s\n"),
- dev->bstrerror());
- set_jcr_job_status(jcr, JS_ErrorTerminated);
+ /* Print only if ok and not cancelled to avoid spurious messages */
+ if (ok && !jcr->is_job_canceled()) {
+ Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"),
+ dev->bstrerror());
+ possible_incomplete_job(jcr, last_file_index);
+ }
+ jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
- if (dev->VolCatInfo.VolCatName[0] == 0) {
- Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
- }
- Dmsg0(90, "back from write_end_session_label()\n");
/* Flush out final partial block of this session */
- if (!write_block_to_device(dcr)) {
- Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
- dev->print_name(), dev->bstrerror());
- Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n"));
+ if (!dcr->write_final_block_to_device()) {
+ /* Print only if ok and not cancelled to avoid spurious messages */
+ if (ok && !jcr->is_job_canceled()) {
+ Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
+ dev->print_name(), dev->bstrerror());
+ Dmsg0(100, _("Set ok=FALSE after write_final_block_to_device.\n"));
+ possible_incomplete_job(jcr, last_file_index);
+ }
+ jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
}
- if (dev->VolCatInfo.VolCatName[0] == 0) {
- Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
- }
-
- if (!ok) {
+ flush_jobmedia_queue(jcr);
+ if (!ok && !jcr->is_JobStatus(JS_Incomplete)) {
discard_data_spool(dcr);
} else {
+ /* Note: if commit is OK, the device will remain blocked */
commit_data_spool(dcr);
}
- if (ok) {
- ok = dvd_close_job(dcr); /* do DVD cleanup if any */
+ /*
+ * Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
+ * and the subsequent Jmsg() editing will break
+ */
+ int32_t job_elapsed = time(NULL) - jcr->run_time;
+
+ if (job_elapsed <= 0) {
+ job_elapsed = 1;
}
-
- /* Release the device -- and send final Vol info to DIR */
+
+ Jmsg(dcr->jcr, M_INFO, 0, _("Elapsed time=%02d:%02d:%02d, Transfer rate=%s Bytes/second\n"),
+ job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
+ edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
+
+ /*
+ * Release the device -- and send final Vol info to DIR
+ * and unlock it.
+ */
release_device(dcr);
- if (!ok || job_canceled(jcr)) {
+ if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
- dir_send_job_status(jcr); /* update director */
+ jcr->sendJobStatus(); /* update director */
Dmsg1(100, "return from do_append_data() ok=%d\n", ok);
return ok;
}
+
+
+/* Send attributes and digest to Director for Catalog */
+bool send_attrs_to_dir(JCR *jcr, DEV_RECORD *rec)
+{
+ if (rec->maskedStream == STREAM_UNIX_ATTRIBUTES ||
+ rec->maskedStream == STREAM_UNIX_ATTRIBUTES_EX ||
+ rec->maskedStream == STREAM_RESTORE_OBJECT ||
+ crypto_digest_stream_type(rec->maskedStream) != CRYPTO_DIGEST_NONE) {
+ if (!jcr->no_attributes) {
+ BSOCK *dir = jcr->dir_bsock;
+ if (are_attributes_spooled(jcr)) {
+ dir->set_spooling();
+ }
+ Dmsg1(850, "Send attributes to dir. FI=%d\n", rec->FileIndex);
+ if (!dir_update_file_attributes(jcr->dcr, rec)) {
+ Jmsg(jcr, M_FATAL, 0, _("Error updating file attributes. ERR=%s\n"),
+ dir->bstrerror());
+ dir->clear_spooling();
+ return false;
+ }
+ dir->clear_spooling();
+ }
+ }
+ return true;
+}
projects / bacula / bacula / blobdiff