2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation plus additions
11 that are listed in the file LICENSE.
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
31 * Kern Sibbald, March 2004
39 /* Forward referenced subroutines */
40 static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name);
41 static bool open_data_spool_file(DCR *dcr);
42 static bool close_data_spool_file(DCR *dcr);
43 static bool despool_data(DCR *dcr, bool commit);
44 static int read_block_from_spool_file(DCR *dcr);
45 static bool open_attr_spool_file(JCR *jcr, BSOCK *bs);
46 static bool close_attr_spool_file(JCR *jcr, BSOCK *bs);
47 static bool write_spool_header(DCR *dcr);
48 static bool write_spool_data(DCR *dcr);
50 struct spool_stats_t {
51 uint32_t data_jobs; /* current jobs spooling data */
53 uint32_t total_data_jobs; /* total jobs to have spooled data */
54 uint32_t total_attr_jobs;
55 int64_t max_data_size; /* max data size */
56 int64_t max_attr_size;
57 int64_t data_size; /* current data size (all jobs running) */
61 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
62 spool_stats_t spool_stats;
65 * Header for data spool record */
67 int32_t FirstIndex; /* FirstIndex for buffer */
68 int32_t LastIndex; /* LastIndex for buffer */
69 uint32_t len; /* length of next buffer */
78 void list_spool_stats(void sendit(const char *msg, int len, void *sarg), void *arg)
80 char ed1[30], ed2[30];
81 POOL_MEM msg(PM_MESSAGE);
84 if (spool_stats.data_jobs || spool_stats.max_data_size) {
85 len = Mmsg(msg, _("Data spooling: %u active jobs, %s bytes; %u total jobs, %s max bytes/job.\n"),
86 spool_stats.data_jobs, edit_uint64_with_commas(spool_stats.data_size, ed1),
87 spool_stats.total_data_jobs,
88 edit_uint64_with_commas(spool_stats.max_data_size, ed2));
90 sendit(msg.c_str(), len, arg);
92 if (spool_stats.attr_jobs || spool_stats.max_attr_size) {
93 len = Mmsg(msg, _("Attr spooling: %u active jobs, %s bytes; %u total jobs, %s max bytes.\n"),
94 spool_stats.attr_jobs, edit_uint64_with_commas(spool_stats.attr_size, ed1),
95 spool_stats.total_attr_jobs,
96 edit_uint64_with_commas(spool_stats.max_attr_size, ed2));
98 sendit(msg.c_str(), len, arg);
102 bool begin_data_spool(DCR *dcr)
105 if (!dcr->dev->is_dvd() && dcr->jcr->spool_data) {
106 Dmsg0(100, "Turning on data spooling\n");
107 dcr->spool_data = true;
108 stat = open_data_spool_file(dcr);
110 dcr->spooling = true;
111 Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data ...\n"));
113 spool_stats.data_jobs++;
120 bool discard_data_spool(DCR *dcr)
123 Dmsg0(100, "Data spooling discarded\n");
124 return close_data_spool_file(dcr);
129 bool commit_data_spool(DCR *dcr)
134 Dmsg0(100, "Committing spooled data\n");
135 stat = despool_data(dcr, true /*commit*/);
137 Dmsg1(100, _("Bad return from despool WroteVol=%d\n"), dcr->WroteVol);
138 close_data_spool_file(dcr);
141 return close_data_spool_file(dcr);
146 static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name)
149 if (dcr->dev->device->spool_directory) {
150 dir = dcr->dev->device->spool_directory;
152 dir = working_directory;
154 Mmsg(name, "%s/%s.data.%u.%s.%s.spool", dir, my_name, dcr->jcr->JobId,
155 dcr->jcr->Job, dcr->device->hdr.name);
159 static bool open_data_spool_file(DCR *dcr)
161 POOLMEM *name = get_pool_memory(PM_MESSAGE);
164 make_unique_data_spool_filename(dcr, &name);
165 if ((spool_fd = open(name, O_CREAT|O_TRUNC|O_RDWR|O_BINARY, 0640)) >= 0) {
166 dcr->spool_fd = spool_fd;
167 dcr->jcr->spool_attributes = true;
170 Jmsg(dcr->jcr, M_FATAL, 0, _("Open data spool file %s failed: ERR=%s\n"), name,
172 free_pool_memory(name);
175 Dmsg1(100, "Created spool file: %s\n", name);
176 free_pool_memory(name);
180 static bool close_data_spool_file(DCR *dcr)
182 POOLMEM *name = get_pool_memory(PM_MESSAGE);
185 spool_stats.data_jobs--;
186 spool_stats.total_data_jobs++;
187 if (spool_stats.data_size < dcr->job_spool_size) {
188 spool_stats.data_size = 0;
190 spool_stats.data_size -= dcr->job_spool_size;
192 dcr->job_spool_size = 0;
195 make_unique_data_spool_filename(dcr, &name);
196 close(dcr->spool_fd);
198 dcr->spooling = false;
200 Dmsg1(100, "Deleted spool file: %s\n", name);
201 free_pool_memory(name);
205 static const char *spool_name = "*spool*";
208 * NB! This routine locks the device, but if committing will
209 * not unlock it. If not committing, it will be unlocked.
211 static bool despool_data(DCR *dcr, bool commit)
221 Dmsg0(100, "Despooling data\n");
223 * Commit means that the job is done, so we commit, otherwise, we
224 * are despooling because of user spool size max or some error
225 * (e.g. filesystem full).
228 Jmsg(jcr, M_INFO, 0, _("Committing spooled data to Volume \"%s\". Despooling %s bytes ...\n"),
229 jcr->dcr->VolumeName,
230 edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
232 Jmsg(jcr, M_INFO, 0, _("Writing spooled data to Volume. Despooling %s bytes ...\n"),
233 edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
235 dcr->despool_wait = true;
236 dcr->spooling = false;
238 dcr->despool_wait = false;
239 dcr->despooling = true;
240 dcr->dev_locked = true;
243 * This is really quite kludgy and should be fixed some time.
244 * We create a dev structure to read from the spool file
247 rdev = (DEVICE *)malloc(sizeof(DEVICE));
248 memset(rdev, 0, sizeof(DEVICE));
249 rdev->dev_name = get_memory(strlen(spool_name)+1);
250 bstrncpy(rdev->dev_name, spool_name, sizeof(rdev->dev_name));
251 rdev->errmsg = get_pool_memory(PM_EMSG);
253 rdev->max_block_size = dcr->dev->max_block_size;
254 rdev->min_block_size = dcr->dev->min_block_size;
255 rdev->device = dcr->dev->device;
256 rdcr = new_dcr(jcr, rdev);
257 rdcr->spool_fd = dcr->spool_fd;
258 rdcr->jcr = jcr; /* set a valid jcr */
259 block = dcr->block; /* save block */
260 dcr->block = rdcr->block; /* make read and write block the same */
262 Dmsg1(800, "read/write block size = %d\n", block->buf_len);
263 lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
265 #if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
266 posix_fadvise(rdcr->spool_fd, 0, 0, POSIX_FADV_WILLNEED);
269 /* Add run time, to get current wait time */
270 time_t despool_start = time(NULL) - jcr->run_time;
273 if (job_canceled(jcr)) {
277 stat = read_block_from_spool_file(rdcr);
278 if (stat == RB_EOT) {
280 } else if (stat == RB_ERROR) {
284 ok = write_block_to_device(dcr);
286 Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
287 dcr->dev->print_name(), dcr->dev->bstrerror());
289 Dmsg3(800, "Write block ok=%d FI=%d LI=%d\n", ok, block->FirstIndex, block->LastIndex);
292 /* Subtracting run_time give us elapsed time - wait_time since we started despooling */
293 time_t despool_elapsed = time(NULL) - despool_start - jcr->run_time;
295 if (despool_elapsed <= 0) {
299 Jmsg(dcr->jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s bytes/second\n"),
300 despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60,
301 edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1));
303 dcr->block = block; /* reset block */
305 lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
306 if (ftruncate(rdcr->spool_fd, 0) != 0) {
308 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
310 /* Note, try continuing despite ftruncate problem */
314 if (spool_stats.data_size < dcr->job_spool_size) {
315 spool_stats.data_size = 0;
317 spool_stats.data_size -= dcr->job_spool_size;
320 P(dcr->dev->spool_mutex);
321 dcr->dev->spool_size -= dcr->job_spool_size;
322 dcr->job_spool_size = 0; /* zap size in input dcr */
323 V(dcr->dev->spool_mutex);
324 free_memory(rdev->dev_name);
325 free_pool_memory(rdev->errmsg);
326 /* Be careful to NULL the jcr and free rdev after free_dcr() */
331 dcr->spooling = true; /* turn on spooling again */
332 dcr->despooling = false;
333 /* If doing a commit, leave the device locked -- unlocked in release_device() */
335 dcr->dev_locked = false;
342 * Read a block from the spool file
344 * Returns RB_OK on success
345 * RB_EOT when file done
348 static int read_block_from_spool_file(DCR *dcr)
353 DEV_BLOCK *block = dcr->block;
356 stat = read(dcr->spool_fd, (char *)&hdr, (size_t)rlen);
358 Dmsg0(100, "EOT on spool read.\n");
360 } else if (stat != (ssize_t)rlen) {
363 Jmsg(dcr->jcr, M_FATAL, 0, _("Spool header read error. ERR=%s\n"),
366 Pmsg2(000, _("Spool read error. Wanted %u bytes, got %d\n"), rlen, stat);
367 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool header read error. Wanted %u bytes, got %d\n"), rlen, stat);
372 if (rlen > block->buf_len) {
373 Pmsg2(000, _("Spool block too big. Max %u bytes, got %u\n"), block->buf_len, rlen);
374 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool block too big. Max %u bytes, got %u\n"), block->buf_len, rlen);
377 stat = read(dcr->spool_fd, (char *)block->buf, (size_t)rlen);
378 if (stat != (ssize_t)rlen) {
379 Pmsg2(000, _("Spool data read error. Wanted %u bytes, got %d\n"), rlen, stat);
380 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool data read error. Wanted %u bytes, got %d\n"), rlen, stat);
383 /* Setup write pointers */
384 block->binbuf = rlen;
385 block->bufp = block->buf + block->binbuf;
386 block->FirstIndex = hdr.FirstIndex;
387 block->LastIndex = hdr.LastIndex;
388 block->VolSessionId = dcr->jcr->VolSessionId;
389 block->VolSessionTime = dcr->jcr->VolSessionTime;
390 Dmsg2(800, "Read block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex);
395 * Write a block to the spool file
397 * Returns: true on success or EOT
398 * false on hard error
400 bool write_block_to_spool_file(DCR *dcr)
402 uint32_t wlen, hlen; /* length to write */
403 bool despool = false;
404 DEV_BLOCK *block = dcr->block;
406 ASSERT(block->binbuf == ((uint32_t) (block->bufp - block->buf)));
407 if (block->binbuf <= WRITE_BLKHDR_LENGTH) { /* Does block have data in it? */
411 hlen = sizeof(spool_hdr);
412 wlen = block->binbuf;
413 P(dcr->dev->spool_mutex);
414 dcr->job_spool_size += hlen + wlen;
415 dcr->dev->spool_size += hlen + wlen;
416 if ((dcr->max_job_spool_size > 0 && dcr->job_spool_size >= dcr->max_job_spool_size) ||
417 (dcr->dev->max_spool_size > 0 && dcr->dev->spool_size >= dcr->dev->max_spool_size)) {
420 V(dcr->dev->spool_mutex);
422 spool_stats.data_size += hlen + wlen;
423 if (spool_stats.data_size > spool_stats.max_data_size) {
424 spool_stats.max_data_size = spool_stats.data_size;
429 char ec1[30], ec2[30], ec3[30], ec4[30];
430 Dmsg4(100, "Despool in write_block_to_spool_file max_size=%s size=%s "
431 "max_job_size=%s job_size=%s\n",
432 edit_uint64_with_commas(dcr->max_job_spool_size, ec1),
433 edit_uint64_with_commas(dcr->job_spool_size, ec2),
434 edit_uint64_with_commas(dcr->dev->max_spool_size, ec3),
435 edit_uint64_with_commas(dcr->dev->spool_size, ec4));
437 Jmsg(dcr->jcr, M_INFO, 0, _("User specified spool size reached.\n"));
438 if (!despool_data(dcr, false)) {
439 Pmsg0(000, _("Bad return from despool in write_block.\n"));
442 /* Despooling cleared these variables so reset them */
443 P(dcr->dev->spool_mutex);
444 dcr->job_spool_size += hlen + wlen;
445 dcr->dev->spool_size += hlen + wlen;
446 V(dcr->dev->spool_mutex);
447 Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data again ...\n"));
451 if (!write_spool_header(dcr)) {
454 if (!write_spool_data(dcr)) {
458 Dmsg2(800, "Wrote block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex);
463 static bool write_spool_header(DCR *dcr)
467 DEV_BLOCK *block = dcr->block;
469 hdr.FirstIndex = block->FirstIndex;
470 hdr.LastIndex = block->LastIndex;
471 hdr.len = block->binbuf;
474 for (int retry=0; retry<=1; retry++) {
475 stat = write(dcr->spool_fd, (char*)&hdr, sizeof(hdr));
478 Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing header to spool file. ERR=%s\n"),
481 if (stat != (ssize_t)sizeof(hdr)) {
482 /* If we wrote something, truncate it, then despool */
484 #if defined(HAVE_WIN32)
485 boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR);
487 boffset_t pos = lseek(dcr->spool_fd, (off_t)0, SEEK_CUR);
489 if (ftruncate(dcr->spool_fd, pos - stat) != 0) {
491 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
493 /* Note, try continuing despite ftruncate problem */
496 if (!despool_data(dcr, false)) {
497 Jmsg(dcr->jcr, M_FATAL, 0, _("Fatal despooling error."));
500 continue; /* try again */
504 Jmsg(dcr->jcr, M_FATAL, 0, _("Retrying after header spooling error failed.\n"));
508 static bool write_spool_data(DCR *dcr)
511 DEV_BLOCK *block = dcr->block;
514 for (int retry=0; retry<=1; retry++) {
515 stat = write(dcr->spool_fd, block->buf, (size_t)block->binbuf);
518 Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing data to spool file. ERR=%s\n"),
521 if (stat != (ssize_t)block->binbuf) {
523 * If we wrote something, truncate it and the header, then despool
526 #if defined(HAVE_WIN32)
527 boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR);
529 boffset_t pos = lseek(dcr->spool_fd, (off_t)0, SEEK_CUR);
531 if (ftruncate(dcr->spool_fd, pos - stat - sizeof(spool_hdr)) != 0) {
533 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
535 /* Note, try continuing despite ftruncate problem */
538 if (!despool_data(dcr, false)) {
539 Jmsg(dcr->jcr, M_FATAL, 0, _("Fatal despooling error."));
542 if (!write_spool_header(dcr)) {
545 continue; /* try again */
549 Jmsg(dcr->jcr, M_FATAL, 0, _("Retrying after data spooling error failed.\n"));
555 bool are_attributes_spooled(JCR *jcr)
557 return jcr->spool_attributes && jcr->dir_bsock->spool_fd;
561 * Create spool file for attributes.
562 * This is done by "attaching" to the bsock, and when
563 * it is called, the output is written to a file.
564 * The actual spooling is turned on and off in
565 * append.c only during writing of the attributes.
567 bool begin_attribute_spool(JCR *jcr)
569 if (!jcr->no_attributes && jcr->spool_attributes) {
570 return open_attr_spool_file(jcr, jcr->dir_bsock);
575 bool discard_attribute_spool(JCR *jcr)
577 if (are_attributes_spooled(jcr)) {
578 return close_attr_spool_file(jcr, jcr->dir_bsock);
583 static void update_attr_spool_size(ssize_t size)
587 if ((spool_stats.attr_size - size) > 0) {
588 spool_stats.attr_size -= size;
590 spool_stats.attr_size = 0;
596 bool commit_attribute_spool(JCR *jcr)
601 if (are_attributes_spooled(jcr)) {
602 if (fseeko(jcr->dir_bsock->spool_fd, 0, SEEK_END) != 0) {
604 Jmsg(jcr, M_FATAL, 0, _("Fseek on attributes file failed: ERR=%s\n"),
608 size = ftello(jcr->dir_bsock->spool_fd);
611 Jmsg(jcr, M_FATAL, 0, _("Fseek on attributes file failed: ERR=%s\n"),
616 if (spool_stats.attr_size + size > spool_stats.max_attr_size) {
617 spool_stats.max_attr_size = spool_stats.attr_size + size;
619 spool_stats.attr_size += size;
621 Jmsg(jcr, M_INFO, 0, _("Sending spooled attrs to the Director. Despooling %s bytes ...\n"),
622 edit_uint64_with_commas(size, ec1));
623 jcr->dir_bsock->despool(update_attr_spool_size, size);
624 return close_attr_spool_file(jcr, jcr->dir_bsock);
629 close_attr_spool_file(jcr, jcr->dir_bsock);
633 static void make_unique_spool_filename(JCR *jcr, POOLMEM **name, int fd)
635 Mmsg(name, "%s/%s.attr.%s.%d.spool", working_directory, my_name,
640 bool open_attr_spool_file(JCR *jcr, BSOCK *bs)
642 POOLMEM *name = get_pool_memory(PM_MESSAGE);
644 make_unique_spool_filename(jcr, &name, bs->fd);
645 bs->spool_fd = fopen(name, "w+b");
648 Jmsg(jcr, M_FATAL, 0, _("fopen attr spool file %s failed: ERR=%s\n"), name,
650 free_pool_memory(name);
654 spool_stats.attr_jobs++;
656 free_pool_memory(name);
660 bool close_attr_spool_file(JCR *jcr, BSOCK *bs)
667 name = get_pool_memory(PM_MESSAGE);
669 spool_stats.attr_jobs--;
670 spool_stats.total_attr_jobs++;
672 make_unique_spool_filename(jcr, &name, bs->fd);
673 fclose(bs->spool_fd);
675 free_pool_memory(name);