2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation plus additions
11 that are listed in the file LICENSE.
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
31 * Kern Sibbald, March 2004
39 /* Forward referenced subroutines */
40 static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name);
41 static bool open_data_spool_file(DCR *dcr);
42 static bool close_data_spool_file(DCR *dcr);
43 static bool despool_data(DCR *dcr, bool commit);
44 static int read_block_from_spool_file(DCR *dcr);
45 static bool open_attr_spool_file(JCR *jcr, BSOCK *bs);
46 static bool close_attr_spool_file(JCR *jcr, BSOCK *bs);
47 static bool write_spool_header(DCR *dcr);
48 static bool write_spool_data(DCR *dcr);
50 struct spool_stats_t {
51 uint32_t data_jobs; /* current jobs spooling data */
53 uint32_t total_data_jobs; /* total jobs to have spooled data */
54 uint32_t total_attr_jobs;
55 int64_t max_data_size; /* max data size */
56 int64_t max_attr_size;
57 int64_t data_size; /* current data size (all jobs running) */
61 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
62 spool_stats_t spool_stats;
65 * Header for data spool record */
67 int32_t FirstIndex; /* FirstIndex for buffer */
68 int32_t LastIndex; /* LastIndex for buffer */
69 uint32_t len; /* length of next buffer */
78 void list_spool_stats(void sendit(const char *msg, int len, void *sarg), void *arg)
80 char ed1[30], ed2[30];
81 POOL_MEM msg(PM_MESSAGE);
84 if (spool_stats.data_jobs || spool_stats.max_data_size) {
85 len = Mmsg(msg, _("Data spooling: %u active jobs, %s bytes; %u total jobs, %s max bytes/job.\n"),
86 spool_stats.data_jobs, edit_uint64_with_commas(spool_stats.data_size, ed1),
87 spool_stats.total_data_jobs,
88 edit_uint64_with_commas(spool_stats.max_data_size, ed2));
90 sendit(msg.c_str(), len, arg);
92 if (spool_stats.attr_jobs || spool_stats.max_attr_size) {
93 len = Mmsg(msg, _("Attr spooling: %u active jobs, %s bytes; %u total jobs, %s max bytes.\n"),
94 spool_stats.attr_jobs, edit_uint64_with_commas(spool_stats.attr_size, ed1),
95 spool_stats.total_attr_jobs,
96 edit_uint64_with_commas(spool_stats.max_attr_size, ed2));
98 sendit(msg.c_str(), len, arg);
102 bool begin_data_spool(DCR *dcr)
105 if (!dcr->dev->is_dvd() && dcr->jcr->spool_data) {
106 Dmsg0(100, "Turning on data spooling\n");
107 dcr->spool_data = true;
108 stat = open_data_spool_file(dcr);
110 dcr->spooling = true;
111 Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data ...\n"));
113 spool_stats.data_jobs++;
120 bool discard_data_spool(DCR *dcr)
123 Dmsg0(100, "Data spooling discarded\n");
124 return close_data_spool_file(dcr);
129 bool commit_data_spool(DCR *dcr)
134 Dmsg0(100, "Committing spooled data\n");
135 stat = despool_data(dcr, true /*commit*/);
137 Dmsg1(100, _("Bad return from despool WroteVol=%d\n"), dcr->WroteVol);
138 close_data_spool_file(dcr);
141 return close_data_spool_file(dcr);
146 static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name)
149 if (dcr->dev->device->spool_directory) {
150 dir = dcr->dev->device->spool_directory;
152 dir = working_directory;
154 Mmsg(name, "%s/%s.data.%u.%s.%s.spool", dir, my_name, dcr->jcr->JobId,
155 dcr->jcr->Job, dcr->device->hdr.name);
159 static bool open_data_spool_file(DCR *dcr)
161 POOLMEM *name = get_pool_memory(PM_MESSAGE);
164 make_unique_data_spool_filename(dcr, &name);
165 if ((spool_fd = open(name, O_CREAT|O_TRUNC|O_RDWR|O_BINARY, 0640)) >= 0) {
166 dcr->spool_fd = spool_fd;
167 dcr->jcr->spool_attributes = true;
170 Jmsg(dcr->jcr, M_FATAL, 0, _("Open data spool file %s failed: ERR=%s\n"), name,
172 free_pool_memory(name);
175 Dmsg1(100, "Created spool file: %s\n", name);
176 free_pool_memory(name);
180 static bool close_data_spool_file(DCR *dcr)
182 POOLMEM *name = get_pool_memory(PM_MESSAGE);
185 spool_stats.data_jobs--;
186 spool_stats.total_data_jobs++;
187 if (spool_stats.data_size < dcr->job_spool_size) {
188 spool_stats.data_size = 0;
190 spool_stats.data_size -= dcr->job_spool_size;
192 dcr->job_spool_size = 0;
195 make_unique_data_spool_filename(dcr, &name);
196 close(dcr->spool_fd);
198 dcr->spooling = false;
200 Dmsg1(100, "Deleted spool file: %s\n", name);
201 free_pool_memory(name);
205 static const char *spool_name = "*spool*";
208 * NB! This routine locks the device, but if committing will
209 * not unlock it. If not committing, it will be unlocked.
211 static bool despool_data(DCR *dcr, bool commit)
221 Dmsg0(100, "Despooling data\n");
223 * Commit means that the job is done, so we commit, otherwise, we
224 * are despooling because of user spool size max or some error
225 * (e.g. filesystem full).
228 Jmsg(jcr, M_INFO, 0, _("Committing spooled data to Volume \"%s\". Despooling %s bytes ...\n"),
229 jcr->dcr->VolumeName,
230 edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
232 Jmsg(jcr, M_INFO, 0, _("Writing spooled data to Volume. Despooling %s bytes ...\n"),
233 edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
235 dcr->despool_wait = true;
236 dcr->spooling = false;
238 dcr->despool_wait = false;
239 dcr->despooling = true;
240 dcr->dev_locked = true;
243 * This is really quite kludgy and should be fixed some time.
244 * We create a dev structure to read from the spool file
247 rdev = (DEVICE *)malloc(sizeof(DEVICE));
248 memset(rdev, 0, sizeof(DEVICE));
249 rdev->dev_name = get_memory(strlen(spool_name)+1);
250 bstrncpy(rdev->dev_name, spool_name, sizeof(rdev->dev_name));
251 rdev->errmsg = get_pool_memory(PM_EMSG);
253 rdev->max_block_size = dcr->dev->max_block_size;
254 rdev->min_block_size = dcr->dev->min_block_size;
255 rdev->device = dcr->dev->device;
256 rdcr = new_dcr(jcr, rdev);
257 rdcr->spool_fd = dcr->spool_fd;
258 rdcr->jcr = jcr; /* set a valid jcr */
259 block = dcr->block; /* save block */
260 dcr->block = rdcr->block; /* make read and write block the same */
262 Dmsg1(800, "read/write block size = %d\n", block->buf_len);
263 lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
265 /* Add run time, to get current wait time */
266 time_t despool_start = time(NULL) - jcr->run_time;
269 if (job_canceled(jcr)) {
273 stat = read_block_from_spool_file(rdcr);
274 if (stat == RB_EOT) {
276 } else if (stat == RB_ERROR) {
280 ok = write_block_to_device(dcr);
282 Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
283 dcr->dev->print_name(), dcr->dev->bstrerror());
285 Dmsg3(800, "Write block ok=%d FI=%d LI=%d\n", ok, block->FirstIndex, block->LastIndex);
288 /* Subtracting run_time give us elapsed time - wait_time since we started despooling */
289 time_t despool_elapsed = time(NULL) - despool_start - jcr->run_time;
291 if (despool_elapsed <= 0) {
295 Jmsg(dcr->jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s bytes/second\n"),
296 despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60,
297 edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1));
299 dcr->block = block; /* reset block */
301 lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
302 if (ftruncate(rdcr->spool_fd, 0) != 0) {
304 Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
306 Pmsg1(000, _("Bad return from ftruncate. ERR=%s\n"), be.strerror());
311 if (spool_stats.data_size < dcr->job_spool_size) {
312 spool_stats.data_size = 0;
314 spool_stats.data_size -= dcr->job_spool_size;
317 P(dcr->dev->spool_mutex);
318 dcr->dev->spool_size -= dcr->job_spool_size;
319 dcr->job_spool_size = 0; /* zap size in input dcr */
320 V(dcr->dev->spool_mutex);
321 free_memory(rdev->dev_name);
322 free_pool_memory(rdev->errmsg);
323 /* Be careful to NULL the jcr and free rdev after free_dcr() */
328 dcr->spooling = true; /* turn on spooling again */
329 dcr->despooling = false;
330 /* If doing a commit, leave the device locked -- unlocked in release_device() */
332 dcr->dev_locked = false;
339 * Read a block from the spool file
341 * Returns RB_OK on success
342 * RB_EOT when file done
345 static int read_block_from_spool_file(DCR *dcr)
350 DEV_BLOCK *block = dcr->block;
353 stat = read(dcr->spool_fd, (char *)&hdr, (size_t)rlen);
355 Dmsg0(100, "EOT on spool read.\n");
357 } else if (stat != (ssize_t)rlen) {
360 Jmsg(dcr->jcr, M_FATAL, 0, _("Spool header read error. ERR=%s\n"),
363 Pmsg2(000, _("Spool read error. Wanted %u bytes, got %d\n"), rlen, stat);
364 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool header read error. Wanted %u bytes, got %d\n"), rlen, stat);
369 if (rlen > block->buf_len) {
370 Pmsg2(000, _("Spool block too big. Max %u bytes, got %u\n"), block->buf_len, rlen);
371 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool block too big. Max %u bytes, got %u\n"), block->buf_len, rlen);
374 stat = read(dcr->spool_fd, (char *)block->buf, (size_t)rlen);
375 if (stat != (ssize_t)rlen) {
376 Pmsg2(000, _("Spool data read error. Wanted %u bytes, got %d\n"), rlen, stat);
377 Jmsg2(dcr->jcr, M_FATAL, 0, _("Spool data read error. Wanted %u bytes, got %d\n"), rlen, stat);
380 /* Setup write pointers */
381 block->binbuf = rlen;
382 block->bufp = block->buf + block->binbuf;
383 block->FirstIndex = hdr.FirstIndex;
384 block->LastIndex = hdr.LastIndex;
385 block->VolSessionId = dcr->jcr->VolSessionId;
386 block->VolSessionTime = dcr->jcr->VolSessionTime;
387 Dmsg2(800, "Read block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex);
392 * Write a block to the spool file
394 * Returns: true on success or EOT
395 * false on hard error
397 bool write_block_to_spool_file(DCR *dcr)
399 uint32_t wlen, hlen; /* length to write */
400 bool despool = false;
401 DEV_BLOCK *block = dcr->block;
403 ASSERT(block->binbuf == ((uint32_t) (block->bufp - block->buf)));
404 if (block->binbuf <= WRITE_BLKHDR_LENGTH) { /* Does block have data in it? */
408 hlen = sizeof(spool_hdr);
409 wlen = block->binbuf;
410 P(dcr->dev->spool_mutex);
411 dcr->job_spool_size += hlen + wlen;
412 dcr->dev->spool_size += hlen + wlen;
413 if ((dcr->max_job_spool_size > 0 && dcr->job_spool_size >= dcr->max_job_spool_size) ||
414 (dcr->dev->max_spool_size > 0 && dcr->dev->spool_size >= dcr->dev->max_spool_size)) {
417 V(dcr->dev->spool_mutex);
419 spool_stats.data_size += hlen + wlen;
420 if (spool_stats.data_size > spool_stats.max_data_size) {
421 spool_stats.max_data_size = spool_stats.data_size;
426 char ec1[30], ec2[30], ec3[30], ec4[30];
427 Dmsg4(100, "Despool in write_block_to_spool_file max_size=%s size=%s "
428 "max_job_size=%s job_size=%s\n",
429 edit_uint64_with_commas(dcr->max_job_spool_size, ec1),
430 edit_uint64_with_commas(dcr->job_spool_size, ec2),
431 edit_uint64_with_commas(dcr->dev->max_spool_size, ec3),
432 edit_uint64_with_commas(dcr->dev->spool_size, ec4));
434 Jmsg(dcr->jcr, M_INFO, 0, _("User specified spool size reached.\n"));
435 if (!despool_data(dcr, false)) {
436 Pmsg0(000, _("Bad return from despool in write_block.\n"));
439 /* Despooling cleared these variables so reset them */
440 P(dcr->dev->spool_mutex);
441 dcr->job_spool_size += hlen + wlen;
442 dcr->dev->spool_size += hlen + wlen;
443 V(dcr->dev->spool_mutex);
444 Jmsg(dcr->jcr, M_INFO, 0, _("Spooling data again ...\n"));
448 if (!write_spool_header(dcr)) {
451 if (!write_spool_data(dcr)) {
455 Dmsg2(800, "Wrote block FI=%d LI=%d\n", block->FirstIndex, block->LastIndex);
460 static bool write_spool_header(DCR *dcr)
464 DEV_BLOCK *block = dcr->block;
466 hdr.FirstIndex = block->FirstIndex;
467 hdr.LastIndex = block->LastIndex;
468 hdr.len = block->binbuf;
471 for (int retry=0; retry<=1; retry++) {
472 stat = write(dcr->spool_fd, (char*)&hdr, sizeof(hdr));
475 Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing header to spool file. ERR=%s\n"),
478 if (stat != (ssize_t)sizeof(hdr)) {
479 /* If we wrote something, truncate it, then despool */
481 #if defined(HAVE_WIN32)
482 boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR);
484 boffset_t pos = lseek(dcr->spool_fd, (off_t)0, SEEK_CUR);
486 if (ftruncate(dcr->spool_fd, pos - stat) != 0) {
488 Jmsg(dcr->jcr, M_FATAL, 0, _("Ftruncate spool file failed: ERR=%s\n"),
493 if (!despool_data(dcr, false)) {
494 Jmsg(dcr->jcr, M_FATAL, 0, _("Fatal despooling error."));
497 continue; /* try again */
501 Jmsg(dcr->jcr, M_FATAL, 0, _("Retrying after header spooling error failed.\n"));
505 static bool write_spool_data(DCR *dcr)
508 DEV_BLOCK *block = dcr->block;
511 for (int retry=0; retry<=1; retry++) {
512 stat = write(dcr->spool_fd, block->buf, (size_t)block->binbuf);
515 Jmsg(dcr->jcr, M_FATAL, 0, _("Error writing data to spool file. ERR=%s\n"),
518 if (stat != (ssize_t)block->binbuf) {
520 * If we wrote something, truncate it and the header, then despool
523 #if defined(HAVE_WIN32)
524 boffset_t pos = _lseeki64(dcr->spool_fd, (__int64)0, SEEK_CUR);
526 boffset_t pos = lseek(dcr->spool_fd, (off_t)0, SEEK_CUR);
528 if (ftruncate(dcr->spool_fd, pos - stat - sizeof(spool_hdr)) != 0) {
530 Jmsg(dcr->jcr, M_FATAL, 0, _("Ftruncate spool file failed: ERR=%s\n"),
535 if (!despool_data(dcr, false)) {
536 Jmsg(dcr->jcr, M_FATAL, 0, _("Fatal despooling error."));
539 if (!write_spool_header(dcr)) {
542 continue; /* try again */
546 Jmsg(dcr->jcr, M_FATAL, 0, _("Retrying after data spooling error failed.\n"));
552 bool are_attributes_spooled(JCR *jcr)
554 return jcr->spool_attributes && jcr->dir_bsock->spool_fd;
558 * Create spool file for attributes.
559 * This is done by "attaching" to the bsock, and when
560 * it is called, the output is written to a file.
561 * The actual spooling is turned on and off in
562 * append.c only during writing of the attributes.
564 bool begin_attribute_spool(JCR *jcr)
566 if (!jcr->no_attributes && jcr->spool_attributes) {
567 return open_attr_spool_file(jcr, jcr->dir_bsock);
572 bool discard_attribute_spool(JCR *jcr)
574 if (are_attributes_spooled(jcr)) {
575 return close_attr_spool_file(jcr, jcr->dir_bsock);
580 static void update_attr_spool_size(ssize_t size)
584 if ((spool_stats.attr_size - size) > 0) {
585 spool_stats.attr_size -= size;
587 spool_stats.attr_size = 0;
593 bool commit_attribute_spool(JCR *jcr)
598 if (are_attributes_spooled(jcr)) {
599 if (fseeko(jcr->dir_bsock->spool_fd, 0, SEEK_END) != 0) {
601 Jmsg(jcr, M_FATAL, 0, _("Fseek on attributes file failed: ERR=%s\n"),
605 size = ftello(jcr->dir_bsock->spool_fd);
608 Jmsg(jcr, M_FATAL, 0, _("Fseek on attributes file failed: ERR=%s\n"),
613 if (spool_stats.attr_size + size > spool_stats.max_attr_size) {
614 spool_stats.max_attr_size = spool_stats.attr_size + size;
616 spool_stats.attr_size += size;
618 Jmsg(jcr, M_INFO, 0, _("Sending spooled attrs to the Director. Despooling %s bytes ...\n"),
619 edit_uint64_with_commas(size, ec1));
620 jcr->dir_bsock->despool(update_attr_spool_size, size);
621 return close_attr_spool_file(jcr, jcr->dir_bsock);
626 close_attr_spool_file(jcr, jcr->dir_bsock);
630 static void make_unique_spool_filename(JCR *jcr, POOLMEM **name, int fd)
632 Mmsg(name, "%s/%s.attr.%s.%d.spool", working_directory, my_name,
637 bool open_attr_spool_file(JCR *jcr, BSOCK *bs)
639 POOLMEM *name = get_pool_memory(PM_MESSAGE);
641 make_unique_spool_filename(jcr, &name, bs->fd);
642 bs->spool_fd = fopen(name, "w+b");
645 Jmsg(jcr, M_FATAL, 0, _("fopen attr spool file %s failed: ERR=%s\n"), name,
647 free_pool_memory(name);
651 spool_stats.attr_jobs++;
653 free_pool_memory(name);
657 bool close_attr_spool_file(JCR *jcr, BSOCK *bs)
664 name = get_pool_memory(PM_MESSAGE);
666 spool_stats.attr_jobs--;
667 spool_stats.total_attr_jobs++;
669 make_unique_spool_filename(jcr, &name, bs->fd);
670 fclose(bs->spool_fd);
672 free_pool_memory(name);