2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2016 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
21 * block_util.c -- tape block utility functions
23 * Kern Sibbald, split from block.c March MMXII
31 static const int dbgel = 160;
33 #ifdef DEBUG_BLOCK_CHECKSUM
34 static const bool debug_block_checksum = true;
36 static const bool debug_block_checksum = false;
39 #ifdef NO_TAPE_WRITE_TEST
40 static const bool no_tape_write_test = true;
42 static const bool no_tape_write_test = false;
46 * Dump the block header, then walk through
47 * the block printing out the record headers.
49 void dump_block(DEV_BLOCK *b, const char *msg)
53 char Id[BLKHDR_ID_LENGTH+1];
54 uint32_t CheckSum, BlockCheckSum;
55 uint32_t block_len, reclen;
57 uint32_t VolSessionId, VolSessionTime, data_len;
61 char buf1[100], buf2[100];
63 if ((debug_level & ~DT_ALL) < 250) {
66 unser_begin(b->buf, BLKHDR1_LENGTH);
67 unser_uint32(CheckSum);
68 unser_uint32(block_len);
69 unser_uint32(BlockNumber);
70 unser_bytes(Id, BLKHDR_ID_LENGTH);
71 ASSERT(unser_length(b->buf) == BLKHDR1_LENGTH);
72 Id[BLKHDR_ID_LENGTH] = 0;
74 unser_uint32(VolSessionId);
75 unser_uint32(VolSessionTime);
79 VolSessionId = VolSessionTime = 0;
84 if (block_len > 4000000 || block_len < BLKHDR_CS_LENGTH) {
85 Dmsg4(20, "!!!Dump block %s %p blocksize too %s %lu\n",
87 (block_len < BLKHDR_CS_LENGTH)?"small":"big",
92 BlockCheckSum = bcrc32((uint8_t *)b->buf+BLKHDR_CS_LENGTH,
93 block_len-BLKHDR_CS_LENGTH);
94 Pmsg6(000, _("Dump block %s %p: size=%d BlkNum=%d\n"
95 " Hdrcksum=%x cksum=%x\n"),
96 msg, b, block_len, BlockNumber, CheckSum, BlockCheckSum);
99 unser_begin(p, WRITE_RECHDR_LENGTH);
100 if (rhl == RECHDR1_LENGTH) {
101 unser_uint32(VolSessionId);
102 unser_uint32(VolSessionTime);
104 unser_int32(FileIndex);
106 unser_uint32(data_len);
109 Pmsg6(000, _(" Rec: VId=%u VT=%u FI=%s Strm=%s len=%d reclen=%d\n"),
110 VolSessionId, VolSessionTime, FI_to_ascii(buf1, FileIndex),
111 stream_to_ascii(buf2, Stream, FileIndex), data_len, reclen);
116 * Create a new block structure.
117 * We pass device so that the block can inherit the
118 * min and max block sizes.
120 DEV_BLOCK *new_block(DEVICE *dev)
122 DEV_BLOCK *block = (DEV_BLOCK *)get_memory(sizeof(DEV_BLOCK));
125 memset(block, 0, sizeof(DEV_BLOCK));
127 /* If the user has specified a max_block_size, use it as the default */
128 if (dev->max_block_size == 0) {
129 len = DEFAULT_BLOCK_SIZE;
131 len = dev->max_block_size;
135 * Round to multiple of block size + ensure that
136 * the data length is a multiple of the block size
138 block->buf_len = len;
139 block->buf = get_memory(block->buf_len);
140 block->rechdr_items = 0;
142 block->BlockVer = BLOCK_VER; /* default write version */
143 Dmsg2(150, "New block len=%d block=%p\n", len, block);
149 * Duplicate an existing block (eblock)
151 DEV_BLOCK *dup_block(DEV_BLOCK *eblock)
153 DEV_BLOCK *block = (DEV_BLOCK *)get_memory(sizeof(DEV_BLOCK));
154 int buf_len = sizeof_pool_memory(eblock->buf);
156 memcpy(block, eblock, sizeof(DEV_BLOCK));
157 block->buf = get_memory(buf_len);
158 memcpy(block->buf, eblock->buf, buf_len);
164 * Only the first block checksum error was reported.
165 * If there are more, report it now.
167 void print_block_read_errors(JCR *jcr, DEV_BLOCK *block)
169 if (block->read_errors > 1) {
170 Jmsg(jcr, M_ERROR, 0, _("%d block read errors not printed.\n"),
176 void DCR::free_blocks()
186 void free_block(DEV_BLOCK *block)
188 if (block && block->buf) {
189 Dmsg1(999, "free_block buffer %x\n", block->buf);
190 free_memory(block->buf);
191 Dmsg1(999, "free_block block %x\n", block);
192 free_memory((POOLMEM *)block);
196 bool is_block_empty(DEV_BLOCK *block)
198 return block->binbuf <= WRITE_BLKHDR_LENGTH;
201 /* Empty the block -- for writing */
202 void empty_block(DEV_BLOCK *block)
204 block->binbuf = WRITE_BLKHDR_LENGTH;
205 Dmsg3(200, "empty len=%d block=%p set binbuf=%d\n",
206 block->buf_len, block, block->binbuf);
207 block->bufp = block->buf + block->binbuf;
208 block->buf[0] = 0; /* clear for debugging */
209 block->bufp[0] = 0; /* clear for debugging */
211 block->write_failed = false;
212 block->block_read = false;
213 block->needs_write = false;
214 block->FirstIndex = block->LastIndex = 0;
218 * Create block header just before write. The space
219 * in the buffer should have already been reserved by
222 uint32_t ser_block_header(DEV_BLOCK *block, bool do_checksum)
225 uint32_t block_len = block->binbuf;
228 Dmsg1(160, "block_header: block_len=%d\n", block_len);
229 ser_begin(block->buf, BLKHDR2_LENGTH);
230 ser_uint32(block->CheckSum);
231 ser_uint32(block_len);
232 ser_uint32(block->BlockNumber);
233 ser_bytes(WRITE_BLKHDR_ID, BLKHDR_ID_LENGTH);
234 if (BLOCK_VER >= 2) {
235 ser_uint32(block->VolSessionId);
236 ser_uint32(block->VolSessionTime);
239 /* Checksum whole block except for the checksum */
241 block->CheckSum = bcrc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH,
242 block_len-BLKHDR_CS_LENGTH);
244 Dmsg1(160, "ser_block_header: checksum=%x\n", block->CheckSum);
245 ser_begin(block->buf, BLKHDR2_LENGTH);
246 ser_uint32(block->CheckSum); /* now add checksum to block header */
247 return block->CheckSum;
251 * Unserialize the block header for reading block.
252 * This includes setting all the buffer pointers correctly.
254 * Returns: false on failure (not a block)
257 bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block)
260 char Id[BLKHDR_ID_LENGTH+1];
261 uint32_t BlockCheckSum;
264 uint32_t BlockNumber;
267 if (block->no_header) {
270 unser_begin(block->buf, BLKHDR_LENGTH);
271 unser_uint32(block->CheckSum);
272 unser_uint32(block_len);
273 unser_uint32(BlockNumber);
274 unser_bytes(Id, BLKHDR_ID_LENGTH);
275 ASSERT(unser_length(block->buf) == BLKHDR1_LENGTH);
276 Id[BLKHDR_ID_LENGTH] = 0;
279 bhl = BLKHDR1_LENGTH;
281 block->bufp = block->buf + bhl;
282 //Dmsg3(100, "Block=%p buf=%p bufp=%p\n", block, block->buf, block->bufp);
283 if (strncmp(Id, BLKHDR1_ID, BLKHDR_ID_LENGTH) != 0) {
284 dev->dev_errno = EIO;
285 Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Wanted ID: \"%s\", got \"%s\". Buffer discarded.\n"),
286 dev->file, dev->block_num, BLKHDR1_ID, Id);
287 if (block->read_errors == 0 || verbose >= 2) {
288 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
290 block->read_errors++;
293 } else if (Id[3] == '2') {
294 unser_uint32(block->VolSessionId);
295 unser_uint32(block->VolSessionTime);
296 bhl = BLKHDR2_LENGTH;
298 block->bufp = block->buf + bhl;
299 if (strncmp(Id, BLKHDR2_ID, BLKHDR_ID_LENGTH) != 0) {
300 dev->dev_errno = EIO;
301 Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Wanted ID: \"%s\", got \"%s\". Buffer discarded.\n"),
302 dev->file, dev->block_num, BLKHDR2_ID, Id);
303 if (block->read_errors == 0 || verbose >= 2) {
304 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
306 block->read_errors++;
310 dev->dev_errno = EIO;
311 Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Wanted ID: \"%s\", got \"%s\". Buffer discarded.\n"),
312 dev->file, dev->block_num, BLKHDR2_ID, Id);
313 Dmsg1(50, "%s", dev->errmsg);
314 if (block->read_errors == 0 || verbose >= 2) {
315 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
317 block->read_errors++;
318 unser_uint32(block->VolSessionId);
319 unser_uint32(block->VolSessionTime);
324 if (block_len > MAX_BLOCK_LENGTH) {
325 dev->dev_errno = EIO;
326 Mmsg3(dev->errmsg, _("Volume data error at %u:%u! Block length %u is insane (too large), probably due to a bad archive.\n"),
327 dev->file, dev->block_num, block_len);
328 if (block->read_errors == 0 || verbose >= 2) {
329 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
331 block->read_errors++;
335 Dmsg1(390, "unser_block_header block_len=%d\n", block_len);
336 /* Find end of block or end of buffer whichever is smaller */
337 if (block_len > block->read_len) {
338 block_end = block->read_len;
340 block_end = block_len;
342 block->binbuf = block_end - bhl;
343 Dmsg2(200, "set block=%p binbuf=%d\n", block, block->binbuf);
344 block->block_len = block_len;
345 block->BlockNumber = BlockNumber;
346 Dmsg3(390, "Read binbuf = %d %d block_len=%d\n", block->binbuf,
348 if (block_len <= block->read_len && dev->do_checksum()) {
349 BlockCheckSum = bcrc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH,
350 block_len-BLKHDR_CS_LENGTH);
351 if (BlockCheckSum != block->CheckSum) {
352 dev->dev_errno = EIO;
353 Mmsg6(dev->errmsg, _("Volume data error at %u:%u!\n"
354 "Block checksum mismatch in block=%u len=%d: calc=%x blk=%x\n"),
355 dev->file, dev->block_num, (unsigned)BlockNumber,
356 block_len, BlockCheckSum, block->CheckSum);
357 if (block->read_errors == 0 || verbose >= 2) {
358 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
359 dump_block(block, "with checksum error");
361 block->read_errors++;
371 * Calculate how many bytes to write and then clear to end
374 uint32_t get_len_and_clear_block(DEV_BLOCK *block, DEVICE *dev, uint32_t &pad)
378 * Clear to the end of the buffer if it is not full,
379 * and on tape devices, apply min and fixed blocking.
381 wlen = block->binbuf;
382 if (wlen != block->buf_len) {
383 Dmsg2(250, "binbuf=%d buf_len=%d\n", block->binbuf, block->buf_len);
385 /* Adjust write size to min/max for tapes only */
386 if (dev->is_tape()) {
387 /* check for fixed block size */
388 if (dev->min_block_size == dev->max_block_size) {
389 wlen = block->buf_len; /* fixed block size already rounded */
390 /* Check for min block size */
391 } else if (wlen < dev->min_block_size) {
392 wlen = ((dev->min_block_size + TAPE_BSIZE - 1) / TAPE_BSIZE) * TAPE_BSIZE;
393 /* Ensure size is rounded */
395 wlen = ((wlen + TAPE_BSIZE - 1) / TAPE_BSIZE) * TAPE_BSIZE;
398 ASSERT(wlen <= block->buf_len);
399 /* Clear from end of data to end of block */
400 if (wlen-block->binbuf > 0) {
401 memset(block->bufp, 0, wlen-block->binbuf); /* clear garbage */
403 pad = wlen - block->binbuf; /* padding or zeros written */
404 Dmsg4(150, "Zero end blk: cleared=%d buf_len=%d wlen=%d binbuf=%d\n",
405 pad, block->buf_len, wlen, block->binbuf);
410 return wlen; /* bytes to write */
414 * Determine if user defined volume size has been
415 * reached, and if so, return true, otherwise
418 bool user_volume_size_reached(DCR *dcr, bool quiet)
420 bool hit_max1, hit_max2;
421 uint64_t size, max_size;
422 DEVICE *dev = dcr->dev;
427 size = dev->VolCatInfo.VolCatBytes + dcr->block->binbuf;
428 /* Limit maximum Volume size to value specified by user */
429 hit_max1 = (dev->max_volume_size > 0) && (size >= dev->max_volume_size);
430 hit_max2 = (dev->VolCatInfo.VolCatMaxBytes > 0) &&
431 (size >= dev->VolCatInfo.VolCatMaxBytes);
433 max_size = dev->max_volume_size;
435 max_size = dev->VolCatInfo.VolCatMaxBytes;
437 if (hit_max1 || hit_max2) {
439 Jmsg(dcr->jcr, M_INFO, 0, _("User defined maximum volume size %s will be exceeded on device %s.\n"
440 " Marking Volume \"%s\" as Full.\n"),
441 edit_uint64_with_commas(max_size, ed1), dev->print_name(),
442 dev->getVolCatName());
444 Dmsg4(100, "Maximum volume size %s exceeded Vol=%s device=%s.\n"
445 "Marking Volume \"%s\" as Full.\n",
446 edit_uint64_with_commas(max_size, ed1), dev->getVolCatName(),
447 dev->print_name(), dev->getVolCatName());
450 Dmsg1(dbgel, "Return from user_volume_size_reached=%d\n", rtn);
456 void reread_last_block(DCR *dcr)
458 #define CHECK_LAST_BLOCK
459 #ifdef CHECK_LAST_BLOCK
461 DEVICE *dev = dcr->dev;
463 DEV_BLOCK *block = dcr->block;
465 * If the device is a tape and it supports backspace record,
466 * we backspace over one or two eof marks depending on
467 * how many we just wrote, then over the last record,
468 * then re-read it and verify that the block number is
471 if (dev->is_tape() && dev->has_cap(CAP_BSR)) {
472 /* Now back up over what we wrote and read the last block */
476 Jmsg(jcr, M_ERROR, 0, _("Backspace file at EOT failed. ERR=%s\n"),
477 be.bstrerror(dev->dev_errno));
479 if (ok && dev->has_cap(CAP_TWOEOF) && !dev->bsf(1)) {
482 Jmsg(jcr, M_ERROR, 0, _("Backspace file at EOT failed. ERR=%s\n"),
483 be.bstrerror(dev->dev_errno));
485 /* Backspace over record */
486 if (ok && !dev->bsr(1)) {
489 Jmsg(jcr, M_ERROR, 0, _("Backspace record at EOT failed. ERR=%s\n"),
490 be.bstrerror(dev->dev_errno));
492 * On FreeBSD systems, if the user got here, it is likely that his/her
493 * tape drive is "frozen". The correct thing to do is a
494 * rewind(), but if we do that, higher levels in cleaning up, will
495 * most likely write the EOS record over the beginning of the
496 * tape. The rewind *is* done later in mount.c when another
497 * tape is requested. Note, the clrerror() call in bsr()
498 * calls ioctl(MTCERRSTAT), which *should* fix the problem.
502 DEV_BLOCK *lblock = new_block(dev);
503 /* Note, this can destroy dev->errmsg */
505 if (!dcr->read_block_from_dev(NO_BLOCK_NUMBER_CHECK)) {
506 Jmsg(jcr, M_ERROR, 0, _("Re-read last block at EOT failed. ERR=%s"),
510 * If we wrote block and the block numbers don't agree
511 * we have a possible problem.
513 if (lblock->BlockNumber != dev->LastBlock) {
514 if (dev->LastBlock > (lblock->BlockNumber + 1)) {
515 Jmsg(jcr, M_FATAL, 0, _(
516 "Re-read of last block: block numbers differ by more than one.\n"
517 "Probable tape misconfiguration and data loss. Read block=%u Want block=%u.\n"),
518 lblock->BlockNumber, dev->LastBlock);
520 Jmsg(jcr, M_ERROR, 0, _(
521 "Re-read of last block OK, but block numbers differ. Read block=%u Want block=%u.\n"),
522 lblock->BlockNumber, dev->LastBlock);
525 Jmsg(jcr, M_INFO, 0, _("Re-read of last block succeeded.\n"));
536 * If this routine is called, we do our bookkeeping and
537 * then assure that the volume will not be written any
540 bool terminate_writing_volume(DCR *dcr)
542 DEVICE *dev = dcr->dev;
547 if (dev->is_ateot()) {
548 return ok; /* already been here return now */
551 /* Create a JobMedia record to indicated end of medium */
552 dev->VolCatInfo.VolCatFiles = dev->get_file();
553 if (!dir_create_jobmedia_record(dcr)) {
554 Dmsg0(50, "Error from create JobMedia\n");
555 dev->dev_errno = EIO;
556 Mmsg2(dev->errmsg, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"),
557 dev->getVolCatName(), dcr->jcr->Job);
558 Jmsg(dcr->jcr, M_FATAL, 0, "%s", dev->errmsg);
561 flush_jobmedia_queue(dcr->jcr);
562 dcr->block->write_failed = true;
563 if (dev->can_append() && !dev->weof(1)) { /* end the tape */
564 dev->VolCatInfo.VolCatErrors++;
565 Jmsg(dcr->jcr, M_ERROR, 0, _("Error writing final EOF to tape. This Volume may not be readable.\n"
568 Dmsg0(50, "Error writing final EOF to volume.\n");
571 ok = write_ansi_ibm_labels(dcr, ANSI_EOV_LABEL, dev->VolHdr.VolumeName);
574 Dmsg2(100, "Set VolCatStatus Full size=%lld vol=%s\n",
575 dev->VolCatInfo.VolCatBytes, dev->VolCatInfo.VolCatName);
577 /* If still in append mode mark volume Full */
578 if (bstrcmp(dev->VolCatInfo.VolCatStatus, "Append")) {
579 dev->setVolCatStatus("Full");
582 if (!dir_update_volume_info(dcr, false, true)) {
583 Mmsg(dev->errmsg, _("Error sending Volume info to Director.\n"));
585 Dmsg0(50, "Error updating volume info.\n");
587 Dmsg2(150, "dir_update_volume_info vol=%s to terminate writing -- %s\n",
588 dev->getVolCatName(), ok?"OK":"ERROR");
590 dev->notify_newvol_in_attached_dcrs(NULL);
592 /* Set new file/block parameters for current dcr */
593 set_new_file_parameters(dcr);
595 if (ok && dev->has_cap(CAP_TWOEOF) && dev->can_append() && !dev->weof(1)) { /* end the tape */
596 dev->VolCatInfo.VolCatErrors++;
597 /* This may not be fatal since we already wrote an EOF */
598 Jmsg(dcr->jcr, M_ERROR, 0, "%s", dev->errmsg);
599 Dmsg0(50, "Writing second EOF failed.\n");
602 dev->set_ateot(); /* no more writing this tape */
603 Dmsg2(150, "Leave terminate_writing_volume=%s -- %s\n",
604 dev->getVolCatName(), ok?"OK":"ERROR");
610 * If a new volume has been mounted since our last write
611 * Create a JobMedia record for the previous volume written,
612 * and set new parameters to write this volume
613 * The same applies for if we are in a new file.
615 bool check_for_newvol_or_newfile(DCR *dcr)
619 if (dcr->NewVol || dcr->NewFile) {
620 if (job_canceled(jcr)) {
621 Dmsg0(100, "Canceled\n");
624 /* Create a jobmedia record for this job */
625 if (!dir_create_jobmedia_record(dcr)) {
626 dcr->dev->dev_errno = EIO;
627 Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"),
628 dcr->getVolCatName(), jcr->Job);
629 set_new_volume_parameters(dcr);
630 Dmsg0(100, "cannot create media record\n");
634 flush_jobmedia_queue(jcr);
635 /* Note, setting a new volume also handles any pending new file */
636 set_new_volume_parameters(dcr);
638 set_new_file_parameters(dcr);
645 * Do bookkeeping when a new file is created on a Volume. This is
646 * also done for disk files to generate the jobmedia records for
649 bool do_new_file_bookkeeping(DCR *dcr)
651 DEVICE *dev = dcr->dev;
654 /* Create a JobMedia record so restore can seek */
655 if (!dir_create_jobmedia_record(dcr)) {
656 Dmsg0(40, "Error from create_job_media.\n");
657 dev->dev_errno = EIO;
658 Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"),
659 dcr->getVolCatName(), jcr->Job);
660 Dmsg0(40, "Call terminate_writing_volume\n");
661 terminate_writing_volume(dcr);
662 dev->dev_errno = EIO;
665 dev->VolCatInfo.VolCatFiles = dev->get_file();
666 if (!dir_update_volume_info(dcr, false, false)) {
667 Dmsg0(50, "Error from update_vol_info.\n");
668 Dmsg0(40, "Call terminate_writing_volume\n");
669 terminate_writing_volume(dcr);
670 dev->dev_errno = EIO;
673 Dmsg0(100, "dir_update_volume_info max file size -- OK\n");
675 dev->notify_newfile_in_attached_dcrs();
677 /* Set new file/block parameters for current dcr */
678 set_new_file_parameters(dcr);