2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2016 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
21 * block.c -- tape block handling functions
23 * Kern Sibbald, March MMI
24 * added BB02 format October MMII
32 #ifdef DEBUG_BLOCK_CHECKSUM
33 static const bool debug_block_checksum = true;
35 static const bool debug_block_checksum = false;
38 #ifdef NO_TAPE_WRITE_TEST
39 static const bool no_tape_write_test = true;
41 static const bool no_tape_write_test = false;
45 bool do_new_file_bookkeeping(DCR *dcr);
46 //bool do_dvd_size_checks(DCR *dcr);
47 void reread_last_block(DCR *dcr);
48 uint32_t get_len_and_clear_block(DEV_BLOCK *block, DEVICE *dev, uint32_t &pad);
49 uint32_t ser_block_header(DEV_BLOCK *block, bool do_checksum);
50 bool unser_block_header(JCR *jcr, DEVICE *dev, DEV_BLOCK *block);
53 * Write a block to the device, with locking and unlocking
55 * Returns: true on success
59 bool DCR::write_block_to_device(bool final)
65 Dmsg0(200, "Write to spool\n");
66 ok = write_block_to_spool_file(dcr);
70 if (!is_dev_locked()) { /* device already locked? */
71 /* note, do not change this to dcr->rLock */
72 dev->rLock(false); /* no, lock it */
75 if (!check_for_newvol_or_newfile(dcr)) {
77 goto bail_out; /* fatal error */
80 Dmsg1(500, "Write block to dev=%p\n", dcr->dev);
81 if (!write_block_to_dev()) {
82 Dmsg1(40, "*** Failed write_block_to_dev block=%p\n", block);
83 if (job_canceled(jcr) || jcr->getJobType() == JT_SYSTEM) {
85 Dmsg2(40, "cancel=%d or SYSTEM=%d\n", job_canceled(jcr),
86 jcr->getJobType() == JT_SYSTEM);
88 ok = fixup_device_block_write_error(dcr);
91 if (ok && final && !dir_create_jobmedia_record(dcr)) {
92 Jmsg(jcr, M_FATAL, 0, _("Error writing final JobMedia record to catalog.\n"));
96 if (!dcr->is_dev_locked()) { /* did we lock dev above? */
97 /* note, do not change this to dcr->dunlock */
98 dev->Unlock(); /* unlock it now */
104 * Write a block to the device
106 * Returns: true on success or EOT
107 * false on hard error
109 bool DCR::write_block_to_dev()
112 uint32_t wlen; /* length to write */
116 uint32_t pad; /* padding or zeros written */
118 if (no_tape_write_test) {
122 if (job_canceled(jcr)) {
126 Dmsg3(200, "fd=%d bufp-buf=%d binbuf=%d\n", dev->fd(),
127 block->bufp-block->buf, block->binbuf);
128 ASSERT2(block->binbuf == ((uint32_t)(block->bufp - block->buf)), "binbuf badly set");
130 if (is_block_empty(block)) { /* Does block have data in it? */
131 Dmsg0(150, "return write_block_to_dev no data to write\n");
135 dump_block(block, "before write");
136 if (dev->at_weot()) {
137 Dmsg0(50, "==== FATAL: At EOM with ST_WEOT.\n");
138 dev->dev_errno = ENOSPC;
139 Jmsg1(jcr, M_FATAL, 0, _("Cannot write block. Device at EOM. dev=%s\n"), dev->print_name());
142 if (!dev->can_append()) {
143 dev->dev_errno = EIO;
144 Jmsg1(jcr, M_FATAL, 0, _("Attempt to write on read-only Volume. dev=%s\n"), dev->print_name());
145 Dmsg1(50, "Attempt to write on read-only Volume. dev=%s\n", dev->print_name());
149 if (!dev->is_open()) {
150 Jmsg1(jcr, M_FATAL, 0, _("Attempt to write on closed device=%s\n"), dev->print_name());
151 Dmsg1(50, "Attempt to write on closed device=%s\n", dev->print_name());
155 wlen = get_len_and_clear_block(block, dev, pad);
156 block->block_len = wlen;
158 checksum = ser_block_header(block, dev->do_checksum());
160 if (user_volume_size_reached(dcr, true)) {
161 Dmsg0(40, "Calling terminate_writing_volume\n");
162 terminate_writing_volume(dcr);
163 reread_last_block(dcr); /* Only used on tapes */
164 dev->dev_errno = ENOSPC;
169 * Limit maximum File size on volume to user specified value.
170 * In practical terms, this means to put an EOF mark on
171 * a tape after every X bytes. This effectively determines
172 * how many index records we have (JobMedia). If you set
173 * max_file_size too small, it will cause a lot of shoe-shine
174 * on very fast modern tape (LTO-3 and above).
176 if ((dev->max_file_size > 0) &&
177 (dev->file_size+block->binbuf) >= dev->max_file_size) {
178 dev->file_size = 0; /* reset file size */
180 if (!dev->weof(1)) { /* write eof */
181 Dmsg0(50, "WEOF error in max file size.\n");
182 Jmsg(jcr, M_FATAL, 0, _("Unable to write EOF. ERR=%s\n"),
184 Dmsg0(40, "Calling terminate_writing_volume\n");
185 terminate_writing_volume(dcr);
186 dev->dev_errno = ENOSPC;
189 if (!write_ansi_ibm_labels(dcr, ANSI_EOF_LABEL, dev->VolHdr.VolumeName)) {
193 if (!do_new_file_bookkeeping(dcr)) {
194 /* Error message already sent */
199 dev->updateVolCatWrites(1);
201 #ifdef DEBUG_BLOCK_ZEROING
202 uint32_t *bp = (uint32_t *)block->buf;
203 if (bp[0] == 0 && bp[1] == 0 && bp[2] == 0 && block->buf[12] == 0) {
204 Jmsg0(jcr, M_ABORT, 0, _("Write block header zeroed.\n"));
209 * Do write here, make a somewhat feeble attempt to recover from
210 * I/O errors, or from the OS telling us it is busy.
216 if (retry > 0 && stat == -1 && errno == EBUSY) {
218 Dmsg4(100, "===== write retry=%d stat=%d errno=%d: ERR=%s\n",
219 retry, stat, errno, be.bstrerror());
220 bmicrosleep(5, 0); /* pause a bit if busy or lots of errors */
223 stat = dev->write(block->buf, (size_t)wlen);
224 Dmsg4(110, "Write() BlockAddr=%lld NextAddr=%lld Vol=%s wlen=%d\n",
225 block->BlockAddr, dev->lseek(dcr, 0, SEEK_CUR),
226 dev->VolHdr.VolumeName, wlen);
227 } while (stat == -1 && (errno == EBUSY || errno == EIO) && retry++ < 3);
229 if (debug_block_checksum) {
230 uint32_t achecksum = ser_block_header(block, dev->do_checksum());
231 if (checksum != achecksum) {
232 Jmsg2(jcr, M_ERROR, 0, _("Block checksum changed during write: before=%u after=%u\n"),
233 checksum, achecksum);
234 dump_block(block, "with checksum error");
238 #ifdef DEBUG_BLOCK_ZEROING
239 if (bp[0] == 0 && bp[1] == 0 && bp[2] == 0 && block->buf[12] == 0) {
240 Jmsg0(jcr, M_ABORT, 0, _("Write block header zeroed.\n"));
244 if (stat != (ssize_t)wlen) {
245 /* Some devices simply report EIO when the volume is full.
246 * With a little more thought we may be able to check
247 * capacity and distinguish real errors and EOT
248 * conditions. In any case, we probably want to
249 * simulate an End of Medium.
253 dev->clrerror(-1); /* saves errno in dev->dev_errno */
254 if (dev->dev_errno == 0) {
255 dev->dev_errno = ENOSPC; /* out of space */
257 if (dev->dev_errno != ENOSPC) {
258 dev->VolCatInfo.VolCatErrors++;
259 Jmsg4(jcr, M_ERROR, 0, _("Write error at %u:%u on device %s. ERR=%s.\n"),
260 dev->file, dev->block_num, dev->print_name(), be.bstrerror());
263 dev->dev_errno = ENOSPC; /* out of space */
265 if (dev->dev_errno == ENOSPC) {
266 dev->update_freespace();
267 if (dev->is_freespace_ok() && dev->free_space < dev->min_free_space) {
269 Jmsg(jcr, M_WARNING, 0, _("Out of freespace caused End of Volume \"%s\" at %u:%u on device %s. Write of %u bytes got %d.\n"),
270 dev->getVolCatName(),
271 dev->file, dev->block_num, dev->print_name(), wlen, stat);
273 dev->clear_nospace();
274 Jmsg(jcr, M_INFO, 0, _("End of Volume \"%s\" at %u:%u on device %s. Write of %u bytes got %d.\n"),
275 dev->getVolCatName(),
276 dev->file, dev->block_num, dev->print_name(), wlen, stat);
279 if (chk_dbglvl(100)) {
281 Dmsg7(90, "==== Write error. fd=%d size=%u rtn=%d dev_blk=%d blk_blk=%d errno=%d: ERR=%s\n",
282 dev->fd(), wlen, stat, dev->block_num, block->BlockNumber,
283 dev->dev_errno, be.bstrerror(dev->dev_errno));
286 Dmsg0(40, "Calling terminate_writing_volume\n");
287 ok = terminate_writing_volume(dcr);
289 reread_last_block(dcr);
294 /* We successfully wrote the block, now do housekeeping */
295 Dmsg2(1300, "VolCatBytes=%lld newVolCatBytes=%lld\n", dev->VolCatInfo.VolCatBytes,
296 (dev->VolCatInfo.VolCatBytes+wlen));
297 dev->updateVolCatBytes(wlen);
298 dev->updateVolCatBlocks(1);
299 dev->EndBlock = dev->block_num;
300 dev->EndFile = dev->file;
301 dev->LastBlock = block->BlockNumber;
302 block->BlockNumber++;
304 /* Update dcr values */
305 if (dev->is_tape()) {
306 if (dcr->EndFile != dev->EndFile || dev->EndBlock > dcr->EndBlock) {
307 dcr->EndBlock = dev->EndBlock;
308 dcr->EndFile = dev->EndFile;
312 /* Save address of block just written */
313 uint64_t addr = dev->file_addr + wlen - 1;
314 if (dcr->EndFile == (uint32_t)(addr >> 32) &&
315 (uint32_t)addr < dcr->EndBlock) {
316 Pmsg4(000, "Possible incorrect EndBlock oldEndBlock=%u newEndBlock=%u oldEndFile=%u newEndFile=%u\n",
317 dcr->EndBlock, (uint32_t)addr, dcr->EndFile, (uint32_t)(addr >> 32));
319 dcr->EndBlock = (uint32_t)addr;
320 dcr->EndFile = (uint32_t)(addr >> 32);
322 dev->block_num = (uint32_t)addr;
323 dev->file = (uint32_t)(addr >> 32);
324 block->BlockAddr = dev->file_addr + wlen;
325 Dmsg3(150, "Set block->BlockAddr=%lld wlen=%d block=%x\n",
326 block->BlockAddr, wlen, block);
328 dcr->VolMediaId = dev->VolCatInfo.VolMediaId;
329 Dmsg3(150, "VolFirstIndex=%d blockFirstIndex=%d Vol=%s\n",
330 dcr->VolFirstIndex, block->FirstIndex, dcr->VolumeName);
331 if (dcr->VolFirstIndex == 0 && block->FirstIndex > 0) {
332 dcr->VolFirstIndex = block->FirstIndex;
334 if (block->LastIndex > (int32_t)dcr->VolLastIndex) {
335 dcr->VolLastIndex = block->LastIndex;
337 dcr->WroteVol = true;
338 dev->file_addr += wlen; /* update file address */
339 dev->file_size += wlen;
340 dev->part_size += wlen;
341 dev->setVolCatInfo(false); /* Needs update */
343 Dmsg2(1300, "write_block: wrote block %d bytes=%d\n", dev->block_num, wlen);
350 * Read block with locking
353 bool DCR::read_block_from_device(bool check_block_numbers)
357 Dmsg0(250, "Enter read_block_from_device\n");
359 ok = read_block_from_dev(check_block_numbers);
361 Dmsg0(250, "Leave read_block_from_device\n");
365 static void set_block_position(DCR *dcr, DEVICE *dev, DEV_BLOCK *block)
367 if (dev->is_tape()) {
368 block->Block = dev->block_num;
369 block->File = dev->file;
371 block->Block = (uint32_t)dev->file_addr;
372 block->File = (uint32_t)(dev->file_addr >> 32);
378 * Read the next block into the block structure and unserialize
379 * the block header. For a file, the block may be partially
380 * or completely in the current buffer.
382 bool DCR::read_block_from_dev(bool check_block_numbers)
389 if (job_canceled(jcr)) {
390 Mmsg(dev->errmsg, _("Job failed or canceled.\n"));
396 Mmsg(dev->errmsg, _("Attempt to read past end of tape or file.\n"));
401 Dmsg1(250, "Full read in read_block_from_device() len=%d\n", block->buf_len);
403 if (!dev->is_open()) {
404 Mmsg4(dev->errmsg, _("Attempt to read closed device: fd=%d at file:blk %u:%u on device %s\n"),
405 dev->fd(), dev->file, dev->block_num, dev->print_name());
406 Jmsg(dcr->jcr, M_FATAL, 0, "%s", dev->errmsg);
407 Pmsg1(000, "%s", dev->errmsg);
412 set_block_position(dcr, dev, block);
416 dev->dev_errno = EIO;
417 Mmsg1(dev->errmsg, _("Block buffer size looping problem on device %s\n"),
419 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
424 dump_block(block, "before read");
427 /* Check for DVD part file end */
428 if (dev->at_eof() && dev->is_dvd() && dev->num_dvd_parts > 0 &&
429 dev->part <= dev->num_dvd_parts) {
430 Dmsg0(400, "Call dvd_open_next_part\n");
431 if (dvd_open_next_part(dcr) < 0) {
432 Mmsg3(dev->errmsg, _("Unable to open device part=%d %s: ERR=%s\n"),
433 dev->part, dev->print_name(), dev->bstrerror());
434 Jmsg(dcr->jcr, M_FATAL, 0, "%s", dev->errmsg);
435 dev->dev_errno = EIO;
439 #endif /* xxxBUILD_DVD */
445 boffset_t pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); /* get curr pos */
447 if ((retry > 0 && stat == -1 && errno == EBUSY)) {
449 Dmsg4(100, "===== read retry=%d stat=%d errno=%d: ERR=%s\n",
450 retry, stat, errno, be.bstrerror());
451 bmicrosleep(10, 0); /* pause a bit if busy or lots of errors */
454 stat = dev->read(block->buf, (size_t)block->buf_len);
456 } while (stat == -1 && (errno == EBUSY || errno == EINTR || errno == EIO) && retry++ < 3);
457 Dmsg3(110, "Read() vol=%s nbytes=%d addr=%lld\n",
458 dev->VolHdr.VolumeName, stat, pos);
462 Dmsg2(90, "Read device fd=%d got: ERR=%s\n", dev->fd(), be.bstrerror());
464 if (dev->file == 0 && dev->block_num == 0) { /* Attempt to read label */
465 Mmsg(dev->errmsg, _("The Volume=%s on device=%s appears to be unlabeled.\n"),
466 "", dev->VolCatInfo.VolCatName, dev->print_name());
468 Mmsg5(dev->errmsg, _("Read error on fd=%d at file:blk %u:%u on device %s. ERR=%s.\n"),
469 dev->fd(), dev->file, dev->block_num, dev->print_name(), be.bstrerror());
471 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
472 if (dev->at_eof()) { /* EOF just seen? */
473 dev->set_eot(); /* yes, error => EOT */
477 if (stat == 0) { /* Got EOF ! */
478 if (dev->file == 0 && dev->block_num == 0) { /* Attempt to read label */
479 Mmsg3(dev->errmsg, _("The %sVolume=%s on device=%s appears to be unlabeled.\n"),
480 "", dev->VolCatInfo.VolCatName, dev->print_name());
482 Mmsg4(dev->errmsg, _("Read zero %sbytes Vol=%s at %lld on device %s.\n"),
483 "", dev->VolCatInfo.VolCatName, pos, dev->print_name());
487 Dmsg1(100, "%s", dev->errmsg);
488 if (dev->at_eof()) { /* EOF already read? */
489 dev->set_eot(); /* yes, 2 EOFs => EOT */
493 Dmsg2(150, "==== Read zero bytes. vol=%s at %lld\n", dev->VolCatInfo.VolCatName, pos);
494 return false; /* return eof */
497 /* Continue here for successful read */
499 block->read_len = stat; /* save length read */
500 if (block->read_len == 80 &&
501 (dcr->VolCatInfo.LabelType != B_BACULA_LABEL ||
502 dcr->device->label_type != B_BACULA_LABEL)) {
503 /* ***FIXME*** should check label */
504 Dmsg2(100, "Ignore 80 byte ANSI label at %u:%u\n", dev->file, dev->block_num);
506 goto reread; /* skip ANSI/IBM label */
509 if (block->read_len < BLKHDR2_LENGTH) {
510 dev->dev_errno = EIO;
511 Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Very short block of %d bytes on device %s discarded.\n"),
512 dev->file, dev->block_num, block->read_len, dev->print_name());
513 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
514 dev->set_short_block();
515 block->read_len = block->binbuf = 0;
516 Dmsg2(50, "set block=%p binbuf=%d\n", block, block->binbuf);
517 return false; /* return error */
520 //BlockNumber = block->BlockNumber + 1;
521 if (!unser_block_header(jcr, dev, block)) {
523 dev->file_addr += block->read_len;
524 dev->file_size += block->read_len;
531 * If the block is bigger than the buffer, we reposition for
532 * re-reading the block, allocate a buffer of the correct size,
535 Dmsg2(150, "block_len=%d buf_len=%d\n", block->block_len, block->buf_len);
536 if (block->block_len > block->buf_len) {
537 dev->dev_errno = EIO;
538 Mmsg2(dev->errmsg, _("Block length %u is greater than buffer %u. Attempting recovery.\n"),
539 block->block_len, block->buf_len);
540 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
541 Pmsg1(000, "%s", dev->errmsg);
542 /* Attempt to reposition to re-read the block */
543 if (dev->is_tape()) {
544 Dmsg0(250, "BSR for reread; block too big for buffer.\n");
546 Mmsg(dev->errmsg, "%s", dev->bstrerror());
547 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
552 Dmsg0(250, "Seek to beginning of block for reread.\n");
553 boffset_t pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); /* get curr pos */
554 pos -= block->read_len;
555 dev->lseek(dcr, pos, SEEK_SET);
556 dev->file_addr = pos;
558 Mmsg1(dev->errmsg, _("Setting block buffer size to %u bytes.\n"), block->block_len);
559 Jmsg(jcr, M_INFO, 0, "%s", dev->errmsg);
560 Pmsg1(000, "%s", dev->errmsg);
561 /* Set new block length */
562 dev->max_block_size = block->block_len;
563 block->buf_len = block->block_len;
564 free_memory(block->buf);
565 block->buf = get_memory(block->buf_len);
568 goto reread; /* re-read block with correct block size */
571 if (block->block_len > block->read_len) {
572 dev->dev_errno = EIO;
573 Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Short block of %d bytes on device %s discarded.\n"),
574 dev->file, dev->block_num, block->read_len, dev->print_name());
575 Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
576 dev->set_short_block();
577 block->read_len = block->binbuf = 0;
578 return false; /* return error */
581 dev->clear_short_block();
583 dev->updateVolCatReads(1);
584 dev->updateVolCatReadBytes(block->read_len);
586 dev->EndBlock = dev->block_num;
587 dev->EndFile = dev->file;
590 /* Update dcr values */
591 if (dev->is_tape()) {
592 if (dcr->EndFile != dev->EndFile || dev->EndBlock > dcr->EndBlock) {
593 dcr->EndBlock = dev->EndBlock;
594 dcr->EndFile = dev->EndFile;
597 /* We need to take care about a short block in EndBlock/File
600 uint32_t len = MIN(block->read_len, block->block_len);
601 uint64_t addr = dev->file_addr + len - 1;
602 if ((uint32_t)(addr >> 32) != dcr->EndFile || (uint32_t)addr > dcr->EndBlock) {
603 dcr->EndBlock = (uint32_t)addr;
604 dcr->EndFile = (uint32_t)(addr >> 32);
606 dev->block_num = dev->EndBlock = (uint32_t)addr;
607 dev->file = dev->EndFile = (uint32_t)(addr >> 32);
609 dcr->VolMediaId = dev->VolCatInfo.VolMediaId;
610 dev->file_addr += block->read_len;
611 dev->file_size += block->read_len;
614 * If we read a short block on disk,
615 * seek to beginning of next block. This saves us
616 * from shuffling blocks around in the buffer. Take a
617 * look at this from an efficiency stand point later, but
618 * it should only happen once at the end of each job.
620 * I've been lseek()ing negative relative to SEEK_CUR for 30
621 * years now. However, it seems that with the new off_t definition,
622 * it is not possible to seek negative amounts, so we use two
623 * lseek(). One to get the position, then the second to do an
624 * absolute positioning -- so much for efficiency. KES Sep 02.
626 Dmsg0(250, "At end of read block\n");
627 if (block->read_len > block->block_len && !dev->is_tape()) {
629 boffset_t pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); /* get curr pos */
630 Dmsg1(250, "Current lseek pos=%s\n", edit_int64(pos, ed1));
631 pos -= (block->read_len - block->block_len);
632 dev->lseek(dcr, pos, SEEK_SET);
633 Dmsg3(250, "Did lseek pos=%s blk_size=%d rdlen=%d\n",
634 edit_int64(pos, ed1), block->block_len,
636 dev->file_addr = pos;
637 dev->file_size = pos;
639 Dmsg3(150, "Exit read_block read_len=%d block_len=%d binbuf=%d\n",
640 block->read_len, block->block_len, block->binbuf);
641 block->block_read = true;