2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2005 Kern Sibbald
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 version 2 as amended with additional clauses defined in the
16 file LICENSE in the main source directory.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 the file LICENSE for additional details.
28 /* Forward referenced functions */
29 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
30 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum);
31 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
32 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
35 * Find all the requested files and send them
36 * to the Storage daemon.
38 * Note, we normally carry on a one-way
39 * conversation from this point on with the SD, simply blasting
40 * data to him. To properly know what is going on, we
41 * also run a "heartbeat" monitor which reads the socket and
42 * reacts accordingly (at the moment it has nothing to do
43 * except echo the heartbeat to the Director).
46 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
51 sd = jcr->store_bsock;
53 set_jcr_job_status(jcr, JS_Running);
55 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
58 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
62 buf_size = client->max_network_buffer_size;
64 buf_size = 0; /* use default */
66 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
67 set_jcr_job_status(jcr, JS_ErrorTerminated);
68 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 jcr->acl_text = get_pool_memory(PM_MESSAGE);
90 /* Subroutine save_file() is called for each file */
91 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
92 ok = false; /* error */
93 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
97 free_pool_memory(jcr->acl_text);
99 bnet_sig(sd, BNET_EOD); /* end of sending data */
101 stop_heartbeat_monitor(jcr);
107 if (jcr->compress_buf) {
108 free_pool_memory(jcr->compress_buf);
109 jcr->compress_buf = NULL;
111 Dmsg1(100, "end blast_data ok=%d\n", ok);
116 * Called here by find() for each file included.
117 * This is a callback. The original is find_files() above.
119 * Send the file and its data to the Storage daemon.
123 * -1 to ignore file/directory (not used here)
125 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
127 int stat, data_stream;
128 struct CHKSUM chksum;
130 JCR *jcr = (JCR *)vjcr;
132 if (job_canceled(jcr)) {
136 sd = jcr->store_bsock;
137 jcr->num_files_examined++; /* bump total file count */
139 switch (ff_pkt->type) {
140 case FT_LNKSAVED: /* Hard linked, file already saved */
141 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
144 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
147 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
150 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
153 return 1; /* not used */
155 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"),
157 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
160 /* Suppress message for /dev filesystems */
161 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
162 Jmsg(jcr, M_INFO, 1, _(" Filesystem change prohibited. Will not descend into %s\n"),
165 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
168 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"),
170 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
173 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
176 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
179 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
182 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
186 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
187 be.strerror(ff_pkt->ff_errno));
193 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
194 be.strerror(ff_pkt->ff_errno));
200 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
201 be.strerror(ff_pkt->ff_errno));
207 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
210 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
214 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
215 be.strerror(ff_pkt->ff_errno));
220 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
225 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
229 * Setup for signature handling.
230 * Then initialise the file descriptor we use for data and other streams.
232 chksum_init(&chksum, ff_pkt->flags);
235 if (ff_pkt->flags & FO_PORTABLE) {
236 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
238 if (ff_pkt->reader) {
239 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
240 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
246 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
251 * Open any file with data that we intend to save, then save it.
253 * Note, if is_win32_backup, we must open the Directory so that
254 * the BackupRead will save its permissions and ownership streams.
256 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
257 ff_pkt->statp.st_size > 0) ||
258 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
259 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
261 if (ff_pkt->type == FT_FIFO) {
262 tid = start_thread_timer(pthread_self(), 60);
266 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
267 ff_pkt->ff_errno = errno;
269 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
273 stop_thread_timer(tid);
279 stop_thread_timer(tid);
282 stat = send_data(jcr, data_stream, ff_pkt, &chksum);
283 bclose(&ff_pkt->bfd);
289 #ifdef HAVE_DARWIN_OS
290 /* Regular files can have resource forks and Finder Info */
291 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
292 ff_pkt->flags & FO_HFSPLUS)) {
293 if (ff_pkt->hfsinfo.rsrclength > 0) {
295 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
296 ff_pkt->ff_errno = errno;
298 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
301 if (is_bopen(&ff_pkt->bfd)) {
302 bclose(&ff_pkt->bfd);
306 flags = ff_pkt->flags;
307 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
308 stat = send_data(jcr, STREAM_MACOS_FORK_DATA, ff_pkt, &chksum);
309 ff_pkt->flags = flags;
310 bclose(&ff_pkt->bfd);
316 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
317 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
318 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
319 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
321 chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen);
323 bnet_sig(sd, BNET_EOD);
327 if (ff_pkt->flags & FO_ACL) {
328 /* Read access ACLs for files, dirs and links */
329 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
332 /* Directories can have default ACLs too */
333 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
334 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
340 /* Terminate any signature and send it to Storage daemon and the Director */
341 if (chksum.updated) {
343 chksum_final(&chksum);
344 if (chksum.type == CHKSUM_MD5) {
345 stream = STREAM_MD5_SIGNATURE;
346 } else if (chksum.type == CHKSUM_SHA1) {
347 stream = STREAM_SHA1_SIGNATURE;
349 Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i.\n"), chksum.type);
352 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream);
353 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
354 memcpy(sd->msg, chksum.signature, chksum.length);
355 sd->msglen = chksum.length;
357 bnet_sig(sd, BNET_EOD); /* end of checksum */
365 * Send data read from an already open file descriptor.
367 * We return 1 on sucess and 0 on errors.
370 * We use ff_pkt->statp.st_size when FO_SPARSE.
371 * Currently this is not a problem as the only other stream, resource forks,
372 * are not handled as sparse files.
374 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum)
376 BSOCK *sd = jcr->store_bsock;
377 uint64_t fileAddr = 0; /* file address */
379 int rsize = jcr->buf_size; /* read buffer size */
383 rbuf = sd->msg; /* read buffer */
384 wbuf = sd->msg; /* write buffer */
387 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
391 uLong compress_len, max_compress_len = 0;
392 const Bytef *cbuf = NULL;
394 if (ff_pkt->flags & FO_GZIP) {
395 if (ff_pkt->flags & FO_SPARSE) {
396 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
397 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
399 cbuf = (Bytef *)jcr->compress_buf;
400 max_compress_len = jcr->compress_buf_size; /* set max length */
402 wbuf = jcr->compress_buf; /* compressed output here */
407 * Send Data header to Storage daemon
408 * <file-index> <stream> <info>
410 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
411 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
415 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
418 * Make space at beginning of buffer for fileAddr because this
419 * same buffer will be used for writing if compression if off.
421 if (ff_pkt->flags & FO_SPARSE) {
422 rbuf += SPARSE_FADDR_SIZE;
423 rsize -= SPARSE_FADDR_SIZE;
424 #ifdef HAVE_FREEBSD_OS
426 * To read FreeBSD partitions, the read size must be
429 rsize = (rsize/512) * 512;
433 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
435 if (S_ISBLK(ff_pkt->statp.st_mode))
436 rsize = (rsize/512) * 512;
442 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
445 /* Check for sparse blocks */
446 if (ff_pkt->flags & FO_SPARSE) {
448 if (sd->msglen == rsize &&
449 (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) {
450 sparseBlock = is_buf_zero(rbuf, rsize);
453 ser_begin(wbuf, SPARSE_FADDR_SIZE);
454 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
457 jcr->ReadBytes += sd->msglen; /* count bytes read */
458 fileAddr += sd->msglen;
460 /* Update checksum if requested */
461 chksum_update(chksum, (unsigned char *)rbuf, sd->msglen);
464 /* Do compression if turned on */
465 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
467 compress_len = max_compress_len;
468 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
470 /* NOTE! This call modifies compress_len !!! */
471 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
472 (const Bytef *)rbuf, (uLong)sd->msglen,
473 ff_pkt->GZIP_level)) != Z_OK) {
474 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
477 set_jcr_job_status(jcr, JS_ErrorTerminated);
480 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
481 compress_len, sd->msglen);
483 sd->msglen = compress_len; /* set compressed length */
487 /* Send the buffer to the Storage daemon */
489 if (ff_pkt->flags & FO_SPARSE) {
490 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
492 sd->msg = wbuf; /* set correct write buffer */
493 if (!bnet_send(sd)) {
494 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
496 sd->msg = msgsave; /* restore bnet buffer */
501 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
503 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
504 sd->msg = msgsave; /* restore read buffer */
506 } /* end while read file data */
509 if (sd->msglen < 0) {
511 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
512 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
513 if (jcr->Errors++ > 1000) { /* insanity check */
514 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
519 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
520 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
529 * Read and send an ACL for the last encountered file.
531 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
534 BSOCK *sd = jcr->store_bsock;
538 len = bacl_get(jcr, acltype);
540 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
544 return true; /* no ACL */
548 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
549 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
554 /* Send the buffer to the storage deamon */
555 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
557 sd->msg = jcr->acl_text;
558 sd->msglen = len + 1;
559 if (!bnet_send(sd)) {
562 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
567 jcr->JobBytes += sd->msglen;
569 if (!bnet_sig(sd, BNET_EOD)) {
570 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
575 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
580 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
582 BSOCK *sd = jcr->store_bsock;
583 char attribs[MAXSTRING];
584 char attribsEx[MAXSTRING];
587 #ifdef FD_NO_SEND_TEST
591 /* Find what data stream we will use, then encode the attributes */
592 data_stream = select_data_stream(ff_pkt);
593 encode_stat(attribs, ff_pkt, data_stream);
595 /* Now possibly extend the attributes */
596 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
598 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
601 jcr->JobFiles++; /* increment number of files sent */
602 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
603 pm_strcpy(jcr->last_fname, ff_pkt->fname);
607 * Send Attributes header to Storage daemon
608 * <file-index> <stream> <info>
610 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
611 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
615 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
618 * Send file attributes to Storage daemon
621 * Filename (full path)
623 * Link name (if type==FT_LNK or FT_LNKSAVED)
624 * Encoded extended-attributes (for Win32)
626 * For a directory, link is the same as fname, but with trailing
627 * slash. For a linked file, link is the link.
629 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
630 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
631 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
632 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
634 } else if (ff_pkt->type == FT_DIREND) {
635 /* Here link is the canonical filename (i.e. with trailing slash) */
636 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
637 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
639 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
640 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
643 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
645 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
649 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */