2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2005 Kern Sibbald
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 version 2 as amended with additional clauses defined in the
16 file LICENSE in the main source directory.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 the file LICENSE for additional details.
28 /* Forward referenced functions */
29 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
30 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum);
31 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
32 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
35 * Find all the requested files and send them
36 * to the Storage daemon.
38 * Note, we normally carry on a one-way
39 * conversation from this point on with the SD, simply blasting
40 * data to him. To properly know what is going on, we
41 * also run a "heartbeat" monitor which reads the socket and
42 * reacts accordingly (at the moment it has nothing to do
43 * except echo the heartbeat to the Director).
46 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
51 sd = jcr->store_bsock;
53 set_jcr_job_status(jcr, JS_Running);
55 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
58 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
62 buf_size = client->max_network_buffer_size;
64 buf_size = 0; /* use default */
66 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
67 set_jcr_job_status(jcr, JS_ErrorTerminated);
68 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 jcr->acl_text = get_pool_memory(PM_MESSAGE);
90 /* Subroutine save_file() is called for each file */
91 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
92 ok = false; /* error */
93 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
97 free_pool_memory(jcr->acl_text);
99 stop_heartbeat_monitor(jcr);
101 bnet_sig(sd, BNET_EOD); /* end of sending data */
107 if (jcr->compress_buf) {
108 free_pool_memory(jcr->compress_buf);
109 jcr->compress_buf = NULL;
111 Dmsg1(100, "end blast_data ok=%d\n", ok);
116 * Called here by find() for each file included.
117 * This is a callback. The original is find_files() above.
119 * Send the file and its data to the Storage daemon.
123 * -1 to ignore file/directory (not used here)
125 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
127 int stat, data_stream;
128 struct CHKSUM chksum;
130 JCR *jcr = (JCR *)vjcr;
132 if (job_canceled(jcr)) {
136 sd = jcr->store_bsock;
137 jcr->num_files_examined++; /* bump total file count */
139 switch (ff_pkt->type) {
140 case FT_LNKSAVED: /* Hard linked, file already saved */
141 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
144 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
147 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
150 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
153 return 1; /* not used */
155 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"),
157 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
160 /* Suppress message for /dev filesystems */
161 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
162 Jmsg(jcr, M_INFO, 1, _(" Filesystem change prohibited. Will not descend into %s\n"),
165 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
168 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"),
170 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
173 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
176 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
179 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
182 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
186 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
187 be.strerror(ff_pkt->ff_errno));
193 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
194 be.strerror(ff_pkt->ff_errno));
200 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
201 be.strerror(ff_pkt->ff_errno));
207 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
210 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
214 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
215 be.strerror(ff_pkt->ff_errno));
220 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
225 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
229 * Setup for signature handling.
230 * Then initialise the file descriptor we use for data and other streams.
232 chksum_init(&chksum, ff_pkt->flags);
235 if (ff_pkt->flags & FO_PORTABLE) {
236 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
238 if (ff_pkt->reader) {
239 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
240 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
246 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
251 * Open any file with data that we intend to save, then save it.
253 * Note, if is_win32_backup, we must open the Directory so that
254 * the BackupRead will save its permissions and ownership streams.
256 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
257 ff_pkt->statp.st_size > 0) ||
258 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
259 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
261 if (ff_pkt->type == FT_FIFO) {
262 tid = start_thread_timer(pthread_self(), 60);
266 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
267 ff_pkt->ff_errno = errno;
269 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
273 stop_thread_timer(tid);
279 stop_thread_timer(tid);
282 stat = send_data(jcr, data_stream, ff_pkt, &chksum);
283 bclose(&ff_pkt->bfd);
289 #ifdef HAVE_DARWIN_OS
290 /* Regular files can have resource forks and Finder Info */
291 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
292 ff_pkt->flags & FO_HFSPLUS)) {
293 if (ff_pkt->hfsinfo.rsrclength > 0) {
295 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
296 ff_pkt->ff_errno = errno;
298 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
301 if (is_bopen(&ff_pkt->bfd)) {
302 bclose(&ff_pkt->bfd);
306 flags = ff_pkt->flags;
307 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
308 stat = send_data(jcr, STREAM_MACOS_FORK_DATA, ff_pkt, &chksum);
309 ff_pkt->flags = flags;
310 bclose(&ff_pkt->bfd);
316 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
317 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
318 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
319 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
321 chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen);
323 bnet_sig(sd, BNET_EOD);
327 if (ff_pkt->flags & FO_ACL) {
328 /* Read access ACLs for files, dirs and links */
329 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
332 /* Directories can have default ACLs too */
333 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
334 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
340 /* Terminate any signature and send it to Storage daemon and the Director */
341 if (chksum.updated) {
343 chksum_final(&chksum);
344 if (chksum.type == CHKSUM_MD5) {
345 stream = STREAM_MD5_SIGNATURE;
346 } else if (chksum.type == CHKSUM_SHA1) {
347 stream = STREAM_SHA1_SIGNATURE;
349 Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i.\n"), chksum.type);
352 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream);
353 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
354 memcpy(sd->msg, chksum.signature, chksum.length);
355 sd->msglen = chksum.length;
357 bnet_sig(sd, BNET_EOD); /* end of checksum */
365 * Send data read from an already open file descriptor.
367 * We return 1 on sucess and 0 on errors.
370 * We use ff_pkt->statp.st_size when FO_SPARSE.
371 * Currently this is not a problem as the only other stream, resource forks,
372 * are not handled as sparse files.
374 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum)
376 BSOCK *sd = jcr->store_bsock;
377 uint64_t fileAddr = 0; /* file address */
379 int rsize = jcr->buf_size; /* read buffer size */
381 #ifdef FD_NO_SEND_TEST
386 rbuf = sd->msg; /* read buffer */
387 wbuf = sd->msg; /* write buffer */
390 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
394 uLong compress_len, max_compress_len = 0;
395 const Bytef *cbuf = NULL;
397 if (ff_pkt->flags & FO_GZIP) {
398 if (ff_pkt->flags & FO_SPARSE) {
399 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
400 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
402 cbuf = (Bytef *)jcr->compress_buf;
403 max_compress_len = jcr->compress_buf_size; /* set max length */
405 wbuf = jcr->compress_buf; /* compressed output here */
410 * Send Data header to Storage daemon
411 * <file-index> <stream> <info>
413 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
414 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
418 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
421 * Make space at beginning of buffer for fileAddr because this
422 * same buffer will be used for writing if compression if off.
424 if (ff_pkt->flags & FO_SPARSE) {
425 rbuf += SPARSE_FADDR_SIZE;
426 rsize -= SPARSE_FADDR_SIZE;
427 #ifdef HAVE_FREEBSD_OS
429 * To read FreeBSD partitions, the read size must be
432 rsize = (rsize/512) * 512;
436 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
438 if (S_ISBLK(ff_pkt->statp.st_mode))
439 rsize = (rsize/512) * 512;
445 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
448 /* Check for sparse blocks */
449 if (ff_pkt->flags & FO_SPARSE) {
451 if (sd->msglen == rsize &&
452 fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size ||
453 ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
454 (uint64_t)ff_pkt->statp.st_size == 0)) {
455 sparseBlock = is_buf_zero(rbuf, rsize);
458 ser_begin(wbuf, SPARSE_FADDR_SIZE);
459 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
462 jcr->ReadBytes += sd->msglen; /* count bytes read */
463 fileAddr += sd->msglen;
465 /* Update checksum if requested */
466 chksum_update(chksum, (unsigned char *)rbuf, sd->msglen);
469 /* Do compression if turned on */
470 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
472 compress_len = max_compress_len;
473 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
475 /* NOTE! This call modifies compress_len !!! */
476 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
477 (const Bytef *)rbuf, (uLong)sd->msglen,
478 ff_pkt->GZIP_level)) != Z_OK) {
479 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
482 set_jcr_job_status(jcr, JS_ErrorTerminated);
485 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
486 compress_len, sd->msglen);
488 sd->msglen = compress_len; /* set compressed length */
492 /* Send the buffer to the Storage daemon */
494 if (ff_pkt->flags & FO_SPARSE) {
495 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
497 sd->msg = wbuf; /* set correct write buffer */
498 if (!bnet_send(sd)) {
499 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
501 sd->msg = msgsave; /* restore bnet buffer */
506 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
508 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
509 sd->msg = msgsave; /* restore read buffer */
511 } /* end while read file data */
514 if (sd->msglen < 0) {
516 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
517 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
518 if (jcr->Errors++ > 1000) { /* insanity check */
519 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
524 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
525 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
534 * Read and send an ACL for the last encountered file.
536 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
539 BSOCK *sd = jcr->store_bsock;
542 #ifdef FD_NO_SEND_TEST
546 len = bacl_get(jcr, acltype);
548 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
552 return true; /* no ACL */
556 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
557 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
562 /* Send the buffer to the storage deamon */
563 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
565 sd->msg = jcr->acl_text;
566 sd->msglen = len + 1;
567 if (!bnet_send(sd)) {
570 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
575 jcr->JobBytes += sd->msglen;
577 if (!bnet_sig(sd, BNET_EOD)) {
578 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
583 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
588 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
590 BSOCK *sd = jcr->store_bsock;
591 char attribs[MAXSTRING];
592 char attribsEx[MAXSTRING];
595 #ifdef FD_NO_SEND_TEST
599 /* Find what data stream we will use, then encode the attributes */
600 data_stream = select_data_stream(ff_pkt);
601 encode_stat(attribs, ff_pkt, data_stream);
603 /* Now possibly extend the attributes */
604 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
606 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
609 jcr->JobFiles++; /* increment number of files sent */
610 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
611 pm_strcpy(jcr->last_fname, ff_pkt->fname);
615 * Send Attributes header to Storage daemon
616 * <file-index> <stream> <info>
618 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
619 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
623 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
626 * Send file attributes to Storage daemon
629 * Filename (full path)
631 * Link name (if type==FT_LNK or FT_LNKSAVED)
632 * Encoded extended-attributes (for Win32)
634 * For a directory, link is the same as fname, but with trailing
635 * slash. For a linked file, link is the link.
637 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
638 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
639 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
640 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
642 } else if (ff_pkt->type == FT_DIREND) {
643 /* Here link is the canonical filename (i.e. with trailing slash) */
644 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
645 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
647 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
648 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
651 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
653 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
657 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */