2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2005 Kern Sibbald
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 version 2 as amended with additional clauses defined in the
16 file LICENSE in the main source directory.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 the file LICENSE for additional details.
28 /* Forward referenced functions */
29 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
30 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum);
31 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
32 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
35 * Find all the requested files and send them
36 * to the Storage daemon.
38 * Note, we normally carry on a one-way
39 * conversation from this point on with the SD, simply blasting
40 * data to him. To properly know what is going on, we
41 * also run a "heartbeat" monitor which reads the socket and
42 * reacts accordingly (at the moment it has nothing to do
43 * except echo the heartbeat to the Director).
46 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
51 sd = jcr->store_bsock;
53 set_jcr_job_status(jcr, JS_Running);
55 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
58 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
62 buf_size = client->max_network_buffer_size;
64 buf_size = 0; /* use default */
66 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
67 set_jcr_job_status(jcr, JS_ErrorTerminated);
68 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 jcr->acl_text = get_pool_memory(PM_MESSAGE);
90 /* Subroutine save_file() is called for each file */
91 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
92 ok = false; /* error */
93 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
97 free_pool_memory(jcr->acl_text);
99 bnet_sig(sd, BNET_EOD); /* end of sending data */
101 stop_heartbeat_monitor(jcr);
107 if (jcr->compress_buf) {
108 free_pool_memory(jcr->compress_buf);
109 jcr->compress_buf = NULL;
111 Dmsg1(100, "end blast_data ok=%d\n", ok);
116 * Called here by find() for each file included.
117 * This is a callback. The original is find_files() above.
119 * Send the file and its data to the Storage daemon.
123 * -1 to ignore file/directory (not used here)
125 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
127 int stat, data_stream;
128 struct CHKSUM chksum;
130 JCR *jcr = (JCR *)vjcr;
132 if (job_canceled(jcr)) {
136 sd = jcr->store_bsock;
137 jcr->num_files_examined++; /* bump total file count */
139 switch (ff_pkt->type) {
140 case FT_LNKSAVED: /* Hard linked, file already saved */
141 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
144 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
147 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
150 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
153 return 1; /* not used */
155 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"),
157 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
160 /* Suppress message for /dev filesystems */
161 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
162 Jmsg(jcr, M_INFO, 1, _(" Filesystem change prohibited. Will not descend into %s\n"),
165 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
168 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"),
170 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
173 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
176 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
179 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
182 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
186 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
187 be.strerror(ff_pkt->ff_errno));
193 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
194 be.strerror(ff_pkt->ff_errno));
200 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
201 be.strerror(ff_pkt->ff_errno));
207 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
210 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
214 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
215 be.strerror(ff_pkt->ff_errno));
220 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
225 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
229 * Setup for signature handling.
230 * Then initialise the file descriptor we use for data and other streams.
232 chksum_init(&chksum, ff_pkt->flags);
235 if (ff_pkt->flags & FO_PORTABLE) {
236 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
238 if (ff_pkt->reader) {
239 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
240 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
246 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
251 * Open any file with data that we intend to save, then save it.
253 * Note, if is_win32_backup, we must open the Directory so that
254 * the BackupRead will save its permissions and ownership streams.
256 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
257 ff_pkt->statp.st_size > 0) ||
258 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
259 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
261 if (ff_pkt->type == FT_FIFO) {
262 tid = start_thread_timer(pthread_self(), 60);
266 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
267 ff_pkt->ff_errno = errno;
269 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
273 stop_thread_timer(tid);
279 stop_thread_timer(tid);
282 stat = send_data(jcr, data_stream, ff_pkt, &chksum);
283 bclose(&ff_pkt->bfd);
289 #ifdef HAVE_DARWIN_OS
290 /* Regular files can have resource forks and Finder Info */
291 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
292 ff_pkt->flags & FO_HFSPLUS)) {
293 if (ff_pkt->hfsinfo.rsrclength > 0) {
295 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
296 ff_pkt->ff_errno = errno;
298 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
301 if (is_bopen(&ff_pkt->bfd)) {
302 bclose(&ff_pkt->bfd);
306 flags = ff_pkt->flags;
307 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
308 stat = send_data(jcr, STREAM_MACOS_FORK_DATA, ff_pkt, &chksum);
309 ff_pkt->flags = flags;
310 bclose(&ff_pkt->bfd);
316 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
317 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
318 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
319 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
321 chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen);
323 bnet_sig(sd, BNET_EOD);
327 if (ff_pkt->flags & FO_ACL) {
328 /* Read access ACLs for files, dirs and links */
329 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
332 /* Directories can have default ACLs too */
333 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
334 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
340 /* Terminate any signature and send it to Storage daemon and the Director */
341 if (chksum.updated) {
343 chksum_final(&chksum);
344 if (chksum.type == CHKSUM_MD5) {
345 stream = STREAM_MD5_SIGNATURE;
346 } else if (chksum.type == CHKSUM_SHA1) {
347 stream = STREAM_SHA1_SIGNATURE;
349 Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i.\n"), chksum.type);
352 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream);
353 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
354 memcpy(sd->msg, chksum.signature, chksum.length);
355 sd->msglen = chksum.length;
357 bnet_sig(sd, BNET_EOD); /* end of checksum */
365 * Send data read from an already open file descriptor.
367 * We return 1 on sucess and 0 on errors.
370 * We use ff_pkt->statp.st_size when FO_SPARSE.
371 * Currently this is not a problem as the only other stream, resource forks,
372 * are not handled as sparse files.
374 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum)
376 BSOCK *sd = jcr->store_bsock;
377 uint64_t fileAddr = 0; /* file address */
379 int rsize = jcr->buf_size; /* read buffer size */
381 #ifdef FD_NO_SEND_TEST
386 rbuf = sd->msg; /* read buffer */
387 wbuf = sd->msg; /* write buffer */
390 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
394 uLong compress_len, max_compress_len = 0;
395 const Bytef *cbuf = NULL;
397 if (ff_pkt->flags & FO_GZIP) {
398 if (ff_pkt->flags & FO_SPARSE) {
399 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
400 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
402 cbuf = (Bytef *)jcr->compress_buf;
403 max_compress_len = jcr->compress_buf_size; /* set max length */
405 wbuf = jcr->compress_buf; /* compressed output here */
410 * Send Data header to Storage daemon
411 * <file-index> <stream> <info>
413 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
414 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
418 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
421 * Make space at beginning of buffer for fileAddr because this
422 * same buffer will be used for writing if compression if off.
424 if (ff_pkt->flags & FO_SPARSE) {
425 rbuf += SPARSE_FADDR_SIZE;
426 rsize -= SPARSE_FADDR_SIZE;
427 #ifdef HAVE_FREEBSD_OS
429 * To read FreeBSD partitions, the read size must be
432 rsize = (rsize/512) * 512;
436 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
438 if (S_ISBLK(ff_pkt->statp.st_mode))
439 rsize = (rsize/512) * 512;
445 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
448 /* Check for sparse blocks */
449 if (ff_pkt->flags & FO_SPARSE) {
451 if (sd->msglen == rsize &&
452 (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) {
453 sparseBlock = is_buf_zero(rbuf, rsize);
456 ser_begin(wbuf, SPARSE_FADDR_SIZE);
457 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
460 jcr->ReadBytes += sd->msglen; /* count bytes read */
461 fileAddr += sd->msglen;
463 /* Update checksum if requested */
464 chksum_update(chksum, (unsigned char *)rbuf, sd->msglen);
467 /* Do compression if turned on */
468 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
470 compress_len = max_compress_len;
471 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
473 /* NOTE! This call modifies compress_len !!! */
474 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
475 (const Bytef *)rbuf, (uLong)sd->msglen,
476 ff_pkt->GZIP_level)) != Z_OK) {
477 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
480 set_jcr_job_status(jcr, JS_ErrorTerminated);
483 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
484 compress_len, sd->msglen);
486 sd->msglen = compress_len; /* set compressed length */
490 /* Send the buffer to the Storage daemon */
492 if (ff_pkt->flags & FO_SPARSE) {
493 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
495 sd->msg = wbuf; /* set correct write buffer */
496 if (!bnet_send(sd)) {
497 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
499 sd->msg = msgsave; /* restore bnet buffer */
504 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
506 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
507 sd->msg = msgsave; /* restore read buffer */
509 } /* end while read file data */
512 if (sd->msglen < 0) {
514 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
515 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
516 if (jcr->Errors++ > 1000) { /* insanity check */
517 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
522 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
523 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
532 * Read and send an ACL for the last encountered file.
534 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
537 BSOCK *sd = jcr->store_bsock;
540 #ifdef FD_NO_SEND_TEST
544 len = bacl_get(jcr, acltype);
546 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
550 return true; /* no ACL */
554 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
555 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
560 /* Send the buffer to the storage deamon */
561 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
563 sd->msg = jcr->acl_text;
564 sd->msglen = len + 1;
565 if (!bnet_send(sd)) {
568 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
573 jcr->JobBytes += sd->msglen;
575 if (!bnet_sig(sd, BNET_EOD)) {
576 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
581 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
586 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
588 BSOCK *sd = jcr->store_bsock;
589 char attribs[MAXSTRING];
590 char attribsEx[MAXSTRING];
593 #ifdef FD_NO_SEND_TEST
597 /* Find what data stream we will use, then encode the attributes */
598 data_stream = select_data_stream(ff_pkt);
599 encode_stat(attribs, ff_pkt, data_stream);
601 /* Now possibly extend the attributes */
602 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
604 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
607 jcr->JobFiles++; /* increment number of files sent */
608 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
609 pm_strcpy(jcr->last_fname, ff_pkt->fname);
613 * Send Attributes header to Storage daemon
614 * <file-index> <stream> <info>
616 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
617 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
621 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
624 * Send file attributes to Storage daemon
627 * Filename (full path)
629 * Link name (if type==FT_LNK or FT_LNKSAVED)
630 * Encoded extended-attributes (for Win32)
632 * For a directory, link is the same as fname, but with trailing
633 * slash. For a linked file, link is the link.
635 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
636 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
637 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
638 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
640 } else if (ff_pkt->type == FT_DIREND) {
641 /* Here link is the canonical filename (i.e. with trailing slash) */
642 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
643 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
645 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
646 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
649 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
651 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
655 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */