2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
31 * Kern Sibbald, March 2004
39 /* Forward referenced subroutines */
40 static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name);
41 static bool open_data_spool_file(DCR *dcr);
42 static bool close_data_spool_file(DCR *dcr);
43 static bool despool_data(DCR *dcr, bool commit);
44 static int read_block_from_spool_file(DCR *dcr);
45 static bool open_attr_spool_file(JCR *jcr, BSOCK *bs);
46 static bool close_attr_spool_file(JCR *jcr, BSOCK *bs);
47 static bool write_spool_header(DCR *dcr);
48 static bool write_spool_data(DCR *dcr);
50 struct spool_stats_t {
51 uint32_t data_jobs; /* current jobs spooling data */
53 uint32_t total_data_jobs; /* total jobs to have spooled data */
54 uint32_t total_attr_jobs;
55 int64_t max_data_size; /* max data size */
56 int64_t max_attr_size;
57 int64_t data_size; /* current data size (all jobs running) */
61 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
62 spool_stats_t spool_stats;
65 * Header for data spool record */
67 int32_t FirstIndex; /* FirstIndex for buffer */
68 int32_t LastIndex; /* LastIndex for buffer */
69 uint32_t len; /* length of next buffer */
78 void list_spool_stats(void sendit(const char *msg, int len, void *sarg), void *arg)
80 char ed1[30], ed2[30];
81 POOL_MEM msg(PM_MESSAGE);
84 len = Mmsg(msg, _("Spooling statistics:\n"));
86 if (spool_stats.data_jobs || spool_stats.max_data_size) {
87 len = Mmsg(msg, _("Data spooling: %u active jobs, %s bytes; %u total jobs, %s max bytes/job.\n"),
88 spool_stats.data_jobs, edit_uint64_with_commas(spool_stats.data_size, ed1),
89 spool_stats.total_data_jobs,
90 edit_uint64_with_commas(spool_stats.max_data_size, ed2));
92 sendit(msg.c_str(), len, arg);
94 if (spool_stats.attr_jobs || spool_stats.max_attr_size) {
95 len = Mmsg(msg, _("Attr spooling: %u active jobs, %s bytes; %u total jobs, %s max bytes.\n"),
96 spool_stats.attr_jobs, edit_uint64_with_commas(spool_stats.attr_size, ed1),
97 spool_stats.total_attr_jobs,
98 edit_uint64_with_commas(spool_stats.max_attr_size, ed2));
100 sendit(msg.c_str(), len, arg);
104 bool begin_data_spool(DCR *dcr)
107 if (!dcr->dev->is_dvd() && dcr->jcr->spool_data) {
108 Dmsg0(100, "Turning on data spooling\n");
109 dcr->spool_data = true;
110 stat = open_data_spool_file(dcr);
112 dcr->spooling = true;
113 Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data ...\n"));
115 spool_stats.data_jobs++;
122 bool discard_data_spool(DCR *dcr)
125 Dmsg0(100, "Data spooling discarded\n");
126 return close_data_spool_file(dcr);
131 bool commit_data_spool(DCR *dcr)
136 Dmsg0(100, "Committing spooled data\n");
137 stat = despool_data(dcr, true /*commit*/);
139 Dmsg1(100, _("Bad return from despool WroteVol=%d\n"), dcr->WroteVol);
140 close_data_spool_file(dcr);
143 return close_data_spool_file(dcr);
148 static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name)
151 if (dcr->dev->device->spool_directory) {
152 dir = dcr->dev->device->spool_directory;
154 dir = working_directory;
156 Mmsg(name, "%s/%s.data.%u.%s.%s.spool", dir, my_name, dcr->jcr->JobId,
157 dcr->jcr->Job, dcr->device->hdr.name);
161 static bool open_data_spool_file(DCR *dcr)
163 POOLMEM *name = get_pool_memory(PM_MESSAGE);
166 make_unique_data_spool_filename(dcr, &name);
167 if ((spool_fd = open(name, O_CREAT|O_TRUNC|O_RDWR|O_BINARY, 0640)) >= 0) {
168 dcr->spool_fd = spool_fd;
169 dcr->jcr->spool_attributes = true;
172 Jmsg(dcr->jcr, M_FATAL, 0, _("Open data spool file %s failed: ERR=%s\n"), name,
174 free_pool_memory(name);
177 Dmsg1(100, "Created spool file: %s\n", name);
178 free_pool_memory(name);
182 static bool close_data_spool_file(DCR *dcr)
184 POOLMEM *name = get_pool_memory(PM_MESSAGE);
187 spool_stats.data_jobs--;
188 spool_stats.total_data_jobs++;
189 if (spool_stats.data_size < dcr->job_spool_size) {
190 spool_stats.data_size = 0;
192 spool_stats.data_size -= dcr->job_spool_size;
194 dcr->job_spool_size = 0;
197 make_unique_data_spool_filename(dcr, &name);
198 close(dcr->spool_fd);
200 dcr->spooling = false;
202 Dmsg1(100, "Deleted spool file: %s\n", name);
203 free_pool_memory(name);
207 static const char *spool_name = "*spool*";
210 * NB! This routine locks the device, but if committing will
211 * not unlock it. If not committing, it will be unlocked.
213 static bool despool_data(DCR *dcr, bool commit)
223 Dmsg0(100, "Despooling data\n");
224 if (jcr->dcr->job_spool_size == 0) {
225 Jmsg(jcr, M_WARNING, 0, _("Despooling zero bytes. Your disk is probably FULL!\n"));
229 * Commit means that the job is done, so we commit, otherwise, we
230 * are despooling because of user spool size max or some error
231 * (e.g. filesystem full).
234 Jmsg(jcr, M_INFO, 0, _("Committing spooled data to Volume \"%s\". Despooling %s bytes ...\n"),
235 jcr->dcr->VolumeName,
236 edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
237 set_jcr_job_status(jcr, JS_DataCommitting);
239 Jmsg(jcr, M_INFO, 0, _("Writing spooled data to Volume. Despooling %s bytes ...\n"),
240 edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
241 set_jcr_job_status(jcr, JS_DataDespooling);
243 set_jcr_job_status(jcr, JS_DataDespooling);
244 dir_send_job_status(jcr);
245 dcr->despool_wait = true;
246 dcr->spooling = false;
248 * We work with device blocked, but not locked so that
249 * other threads -- e.g. reservations can lock the device
252 dcr->dblock(BST_DESPOOLING);
253 dcr->despool_wait = false;
254 dcr->despooling = true;
257 * This is really quite kludgy and should be fixed some time.
258 * We create a dev structure to read from the spool file
261 rdev = (DEVICE *)malloc(sizeof(DEVICE));
262 memset(rdev, 0, sizeof(DEVICE));
263 rdev->dev_name = get_memory(strlen(spool_name)+1);
264 bstrncpy(rdev->dev_name, spool_name, sizeof(rdev->dev_name));
265 rdev->errmsg = get_pool_memory(PM_EMSG);
267 rdev->max_block_size = dcr->dev->max_block_size;
268 rdev->min_block_size = dcr->dev->min_block_size;
269 rdev->device = dcr->dev->device;
270 rdcr = new_dcr(jcr, NULL, rdev);
271 rdcr->spool_fd = dcr->spool_fd;
272 block = dcr->block; /* save block */
273 dcr->block = rdcr->block; /* make read and write block the same */
275 Dmsg1(800, "read/write block size = %d\n", block->buf_len);
276 lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
278 #if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
279 posix_fadvise(rdcr->spool_fd, 0, 0, POSIX_FADV_WILLNEED);
282 /* Add run time, to get current wait time */
283 int32_t despool_start = time(NULL) - jcr->run_time;
285 set_new_file_parameters(dcr);
288 if (job_canceled(jcr)) {
292 stat = read_block_from_spool_file(rdcr);
293 if (stat == RB_EOT) {
295 } else if (stat == RB_ERROR) {
299 ok = write_block_to_device(dcr);
301 Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
302 dcr->dev->print_name(), dcr->dev->bstrerror());
303 Dmsg2(000, "Fatal append error on device %s: ERR=%s\n",
304 dcr->dev->print_name(), dcr->dev->bstrerror());
306 Dmsg3(800, "Write block ok=%d FI=%d LI=%d\n", ok, block->FirstIndex, block->LastIndex);
309 if (!dir_create_jobmedia_record(dcr)) {
310 Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"),
311 dcr->VolCatInfo.VolCatName, jcr->Job);
313 /* Set new file/block parameters for current dcr */
314 set_new_file_parameters(dcr);
317 * Subtracting run_time give us elapsed time - wait_time since
318 * we started despooling. Note, don't use time_t as it is 32 or 64
319 * bits depending on the OS and doesn't edit with %d
321 int32_t despool_elapsed = time(NULL) - despool_start - jcr->run_time;
323 if (despool_elapsed <= 0) {
327 Jmsg(dcr->jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"),
328 despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60,
329 edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1));
331 dcr->block = block; /* reset block */
333 lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
334 if (ftruncate(rdcr->spool_fd, 0) != 0) {
336 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
338 /* Note, try continuing despite ftruncate problem */
342 if (spool_stats.data_size < dcr->job_spool_size) {
343 spool_stats.data_size = 0;
345 spool_stats.data_size -= dcr->job_spool_size;
348 P(dcr->dev->spool_mutex);
349 dcr->dev->spool_size -= dcr->job_spool_size;
350 dcr->job_spool_size = 0; /* zap size in input dcr */
351 V(dcr->dev->spool_mutex);
352 free_memory(rdev->dev_name);
353 free_pool_memory(rdev->errmsg);
354 /* Be careful to NULL the jcr and free rdev after free_dcr() */
359 dcr->spooling = true; /* turn on spooling again */
360 dcr->despooling = false;
363 * We are done, so unblock the device, but if we have done a
364 * commit, leave it locked so that the job cleanup does not
365 * need to wait to release the device (no re-acquire of the lock).
368 unblock_device(dcr->dev);
369 /* If doing a commit, leave the device locked -- unlocked in release_device() */
373 set_jcr_job_status(jcr, JS_Running);
374 dir_send_job_status(jcr);
379 * Read a block from the spool file
381 * Returns RB_OK on success
382 * RB_EOT when file done
385 static int read_block_from_spool_file(DCR *dcr)
390 DEV_BLOCK *block = dcr->block;
393 stat = read(dcr->spool_fd, (char *)&hdr, (size_t)rlen);
395 Dmsg0(100, "EOT on spool read.\n");
397 } else if (stat != (ssize_t)rlen) {
400 Jmsg(dcr->jcr, M_FATAL, 0, _("Spool header read error. ERR=%s\n"),
403 Pmsg2(000, _("Spool read error. Wanted %u bytes, got %d\n"), rlen, stat);
404 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool header read error. Wanted %u bytes, got %d\n"), rlen, stat);
409 if (rlen > block->buf_len) {
410 Pmsg2(000, _("Spool block too big. Max %u bytes, got %u\n"), block->buf_len, rlen);
411 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool block too big. Max %u bytes, got %u\n"), block->buf_len, rlen);
414 stat = read(dcr->spool_fd, (char *)block->buf, (size_t)rlen);
415 if (stat != (ssize_t)rlen) {
416 Pmsg2(000, _("Spool data read error. Wanted %u bytes, got %d\n"), rlen, stat);
417 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool data read error. Wanted %u bytes, got %d\n"), rlen, stat);
420 /* Setup write pointers */
421 block->binbuf = rlen;
422 block->bufp = block->buf + block->binbuf;
423 block->FirstIndex = hdr.FirstIndex;
424 block->LastIndex = hdr.LastIndex;
425 block->VolSessionId = dcr->jcr->VolSessionId;
426 block->VolSessionTime = dcr->jcr->VolSessionTime;
427 Dmsg2(800, "Read block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex);
432 * Write a block to the spool file
434 * Returns: true on success or EOT
435 * false on hard error
437 bool write_block_to_spool_file(DCR *dcr)
439 uint32_t wlen, hlen; /* length to write */
440 bool despool = false;
441 DEV_BLOCK *block = dcr->block;
443 if (job_canceled(dcr->jcr)) {
446 ASSERT(block->binbuf == ((uint32_t) (block->bufp - block->buf)));
447 if (block->binbuf <= WRITE_BLKHDR_LENGTH) { /* Does block have data in it? */
451 hlen = sizeof(spool_hdr);
452 wlen = block->binbuf;
453 P(dcr->dev->spool_mutex);
454 dcr->job_spool_size += hlen + wlen;
455 dcr->dev->spool_size += hlen + wlen;
456 if ((dcr->max_job_spool_size > 0 && dcr->job_spool_size >= dcr->max_job_spool_size) ||
457 (dcr->dev->max_spool_size > 0 && dcr->dev->spool_size >= dcr->dev->max_spool_size)) {
460 V(dcr->dev->spool_mutex);
462 spool_stats.data_size += hlen + wlen;
463 if (spool_stats.data_size > spool_stats.max_data_size) {
464 spool_stats.max_data_size = spool_stats.data_size;
469 char ec1[30], ec2[30], ec3[30], ec4[30];
470 Dmsg4(100, "Despool in write_block_to_spool_file max_size=%s size=%s "
471 "max_job_size=%s job_size=%s\n",
472 edit_uint64_with_commas(dcr->max_job_spool_size, ec1),
473 edit_uint64_with_commas(dcr->job_spool_size, ec2),
474 edit_uint64_with_commas(dcr->dev->max_spool_size, ec3),
475 edit_uint64_with_commas(dcr->dev->spool_size, ec4));
477 Jmsg(dcr->jcr, M_INFO, 0, _("User specified spool size reached.\n"));
478 if (!despool_data(dcr, false)) {
479 Pmsg0(000, _("Bad return from despool in write_block.\n"));
482 /* Despooling cleared these variables so reset them */
483 P(dcr->dev->spool_mutex);
484 dcr->job_spool_size += hlen + wlen;
485 dcr->dev->spool_size += hlen + wlen;
486 V(dcr->dev->spool_mutex);
487 Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data again ...\n"));
491 if (!write_spool_header(dcr)) {
494 if (!write_spool_data(dcr)) {
498 Dmsg2(800, "Wrote block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex);
503 static bool write_spool_header(DCR *dcr)
507 DEV_BLOCK *block = dcr->block;
509 hdr.FirstIndex = block->FirstIndex;
510 hdr.LastIndex = block->LastIndex;
511 hdr.len = block->binbuf;
514 for (int retry=0; retry<=1; retry++) {
515 stat = write(dcr->spool_fd, (char*)&hdr, sizeof(hdr));
518 Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing header to spool file. ERR=%s\n"),
521 if (stat != (ssize_t)sizeof(hdr)) {
522 Jmsg(dcr->jcr, M_ERROR, 0, _("Error writing header to spool file."
523 " Disk probably full. Attempting recovery. Wanted to write=%d got=%d\n"),
524 (int)stat, (int)sizeof(hdr));
525 /* If we wrote something, truncate it, then despool */
527 #if defined(HAVE_WIN32)
528 boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR);
530 boffset_t pos = lseek(dcr->spool_fd, 0, SEEK_CUR);
532 if (ftruncate(dcr->spool_fd, pos - stat) != 0) {
534 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
536 /* Note, try continuing despite ftruncate problem */
539 if (!despool_data(dcr, false)) {
540 Jmsg(dcr->jcr, M_FATAL, 0, _("Fatal despooling error."));
543 continue; /* try again */
547 Jmsg(dcr->jcr, M_FATAL, 0, _("Retrying after header spooling error failed.\n"));
551 static bool write_spool_data(DCR *dcr)
554 DEV_BLOCK *block = dcr->block;
557 for (int retry=0; retry<=1; retry++) {
558 stat = write(dcr->spool_fd, block->buf, (size_t)block->binbuf);
561 Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing data to spool file. ERR=%s\n"),
564 if (stat != (ssize_t)block->binbuf) {
566 * If we wrote something, truncate it and the header, then despool
569 #if defined(HAVE_WIN32)
570 boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR);
572 boffset_t pos = lseek(dcr->spool_fd, 0, SEEK_CUR);
574 if (ftruncate(dcr->spool_fd, pos - stat - sizeof(spool_hdr)) != 0) {
576 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
578 /* Note, try continuing despite ftruncate problem */
581 if (!despool_data(dcr, false)) {
582 Jmsg(dcr->jcr, M_FATAL, 0, _("Fatal despooling error."));
585 if (!write_spool_header(dcr)) {
588 continue; /* try again */
592 Jmsg(dcr->jcr, M_FATAL, 0, _("Retrying after data spooling error failed.\n"));
598 bool are_attributes_spooled(JCR *jcr)
600 return jcr->spool_attributes && jcr->dir_bsock->m_spool_fd;
604 * Create spool file for attributes.
605 * This is done by "attaching" to the bsock, and when
606 * it is called, the output is written to a file.
607 * The actual spooling is turned on and off in
608 * append.c only during writing of the attributes.
610 bool begin_attribute_spool(JCR *jcr)
612 if (!jcr->no_attributes && jcr->spool_attributes) {
613 return open_attr_spool_file(jcr, jcr->dir_bsock);
618 bool discard_attribute_spool(JCR *jcr)
620 if (are_attributes_spooled(jcr)) {
621 return close_attr_spool_file(jcr, jcr->dir_bsock);
626 static void update_attr_spool_size(ssize_t size)
630 if ((spool_stats.attr_size - size) > 0) {
631 spool_stats.attr_size -= size;
633 spool_stats.attr_size = 0;
639 static void make_unique_spool_filename(JCR *jcr, POOLMEM **name, int fd)
641 Mmsg(name, "%s/%s.attr.%s.%d.spool", working_directory, my_name,
646 * Tell Director where to find the attributes spool file
647 * Note, if we are not on the same machine, the Director will
648 * return an error, and the higher level routine will transmit
649 * the data record by record -- using bsock->despool().
651 static bool blast_attr_spool_file(JCR *jcr, boffset_t size)
653 /* send full spool file name */
654 POOLMEM *name = get_pool_memory(PM_MESSAGE);
655 make_unique_spool_filename(jcr, &name, jcr->dir_bsock->m_fd);
657 jcr->dir_bsock->fsend("BlastAttr Job=%s File=%s\n", jcr->Job, name);
658 free_pool_memory(name);
660 if (jcr->dir_bsock->recv() <= 0) {
661 Jmsg(jcr, M_FATAL, 0, _("Network error on BlastAttributes.\n"));
665 if (!bstrcmp(jcr->dir_bsock->msg, "1000 OK BlastAttr\n")) {
671 bool commit_attribute_spool(JCR *jcr)
677 Dmsg1(100, "Commit attributes at %s\n", bstrftimes(tbuf, sizeof(tbuf),
678 (utime_t)time(NULL)));
679 if (are_attributes_spooled(jcr)) {
680 if (fseeko(jcr->dir_bsock->m_spool_fd, 0, SEEK_END) != 0) {
682 Jmsg(jcr, M_FATAL, 0, _("Fseek on attributes file failed: ERR=%s\n"),
686 size = ftello(jcr->dir_bsock->m_spool_fd);
689 Jmsg(jcr, M_FATAL, 0, _("Fseek on attributes file failed: ERR=%s\n"),
694 if (spool_stats.attr_size + size > spool_stats.max_attr_size) {
695 spool_stats.max_attr_size = spool_stats.attr_size + size;
697 spool_stats.attr_size += size;
699 set_jcr_job_status(jcr, JS_AttrDespooling);
700 dir_send_job_status(jcr);
701 Jmsg(jcr, M_INFO, 0, _("Sending spooled attrs to the Director. Despooling %s bytes ...\n"),
702 edit_uint64_with_commas(size, ec1));
704 if (!blast_attr_spool_file(jcr, size)) {
705 /* Can't read spool file from director side,
706 * send content over network.
708 jcr->dir_bsock->despool(update_attr_spool_size, size);
710 return close_attr_spool_file(jcr, jcr->dir_bsock);
715 close_attr_spool_file(jcr, jcr->dir_bsock);
719 static bool open_attr_spool_file(JCR *jcr, BSOCK *bs)
721 POOLMEM *name = get_pool_memory(PM_MESSAGE);
723 make_unique_spool_filename(jcr, &name, bs->m_fd);
724 bs->m_spool_fd = fopen(name, "w+b");
725 if (!bs->m_spool_fd) {
727 Jmsg(jcr, M_FATAL, 0, _("fopen attr spool file %s failed: ERR=%s\n"), name,
729 free_pool_memory(name);
733 spool_stats.attr_jobs++;
735 free_pool_memory(name);
739 static bool close_attr_spool_file(JCR *jcr, BSOCK *bs)
745 Dmsg1(100, "Close attr spool file at %s\n", bstrftimes(tbuf, sizeof(tbuf),
746 (utime_t)time(NULL)));
747 if (!bs->m_spool_fd) {
750 name = get_pool_memory(PM_MESSAGE);
752 spool_stats.attr_jobs--;
753 spool_stats.total_attr_jobs++;
755 make_unique_spool_filename(jcr, &name, bs->m_fd);
756 fclose(bs->m_spool_fd);
758 free_pool_memory(name);
759 bs->m_spool_fd = NULL;
760 bs->clear_spooling();