2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2005 Kern Sibbald
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 version 2 as amended with additional clauses defined in the
16 file LICENSE in the main source directory.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 the file LICENSE for additional details.
28 /* Forward referenced functions */
29 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
30 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum);
31 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
32 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
35 * Find all the requested files and send them
36 * to the Storage daemon.
38 * Note, we normally carry on a one-way
39 * conversation from this point on with the SD, simply blasting
40 * data to him. To properly know what is going on, we
41 * also run a "heartbeat" monitor which reads the socket and
42 * reacts accordingly (at the moment it has nothing to do
43 * except echo the heartbeat to the Director).
46 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
51 sd = jcr->store_bsock;
53 set_jcr_job_status(jcr, JS_Running);
55 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
58 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
62 buf_size = client->max_network_buffer_size;
64 buf_size = 0; /* use default */
66 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
67 set_jcr_job_status(jcr, JS_ErrorTerminated);
68 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 jcr->acl_text = get_pool_memory(PM_MESSAGE);
90 /* Subroutine save_file() is called for each file */
91 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
92 ok = false; /* error */
93 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
97 free_pool_memory(jcr->acl_text);
99 bnet_sig(sd, BNET_EOD); /* end of sending data */
101 stop_heartbeat_monitor(jcr);
107 if (jcr->compress_buf) {
108 free_pool_memory(jcr->compress_buf);
109 jcr->compress_buf = NULL;
111 Dmsg1(100, "end blast_data ok=%d\n", ok);
116 * Called here by find() for each file included.
117 * This is a callback. The original is find_files() above.
119 * Send the file and its data to the Storage daemon.
123 * -1 to ignore file/directory (not used here)
125 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
127 int stat, data_stream;
128 struct CHKSUM chksum;
130 JCR *jcr = (JCR *)vjcr;
132 if (job_canceled(jcr)) {
136 sd = jcr->store_bsock;
137 jcr->num_files_examined++; /* bump total file count */
139 switch (ff_pkt->type) {
140 case FT_LNKSAVED: /* Hard linked, file already saved */
141 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
144 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
147 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
150 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
153 return 1; /* not used */
158 if (ff_pkt->type == FT_NORECURSE) {
159 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"),
161 } else if (ff_pkt->type == FT_NOFSCHG) {
162 Jmsg(jcr, M_INFO, 1, _(" File system change prohibited. Will not descend into %s\n"),
164 } else if (ff_pkt->type == FT_INVALIDFS) {
165 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"),
168 ff_pkt->type = FT_DIREND; /* value is used below */
169 Dmsg1(130, "FT_DIR saving: %s\n", ff_pkt->link);
172 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
175 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
178 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
182 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
183 be.strerror(ff_pkt->ff_errno));
189 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
190 be.strerror(ff_pkt->ff_errno));
196 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
197 be.strerror(ff_pkt->ff_errno));
203 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
206 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
210 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
211 be.strerror(ff_pkt->ff_errno));
216 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
221 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
223 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
228 * Setup for signature handling.
229 * Then initialise the file descriptor we use for data and other streams.
231 chksum_init(&chksum, ff_pkt->flags);
234 if (ff_pkt->flags & FO_PORTABLE) {
235 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
237 if (ff_pkt->reader) {
238 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
239 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
246 * Open any file with data that we intend to save, then save it.
248 * Note, if is_win32_backup, we must open the Directory so that
249 * the BackupRead will save its permissions and ownership streams.
251 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
252 ff_pkt->statp.st_size > 0) ||
253 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
254 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
256 if (ff_pkt->type == FT_FIFO) {
257 tid = start_thread_timer(pthread_self(), 60);
261 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
262 ff_pkt->ff_errno = errno;
264 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
268 stop_thread_timer(tid);
274 stop_thread_timer(tid);
277 stat = send_data(jcr, data_stream, ff_pkt, &chksum);
278 bclose(&ff_pkt->bfd);
284 #ifdef HAVE_DARWIN_OS
285 /* Regular files can have resource forks and Finder Info */
286 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
287 ff_pkt->flags & FO_HFSPLUS)) {
288 if (ff_pkt->hfsinfo.rsrclength > 0) {
290 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
291 ff_pkt->ff_errno = errno;
293 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
296 if (is_bopen(&ff_pkt->bfd)) {
297 bclose(&ff_pkt->bfd);
301 flags = ff_pkt->flags;
302 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
303 stat = send_data(jcr, STREAM_MACOS_FORK_DATA, ff_pkt, &chksum);
304 ff_pkt->flags = flags;
305 bclose(&ff_pkt->bfd);
311 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
312 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
313 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
314 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
316 chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen);
318 bnet_sig(sd, BNET_EOD);
322 if (ff_pkt->flags & FO_ACL) {
323 /* Read access ACLs for files, dirs and links */
324 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
327 /* Directories can have default ACLs too */
328 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
329 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
335 /* Terminate any signature and send it to Storage daemon and the Director */
336 if (chksum.updated) {
338 chksum_final(&chksum);
339 if (chksum.type == CHKSUM_MD5) {
340 stream = STREAM_MD5_SIGNATURE;
341 } else if (chksum.type == CHKSUM_SHA1) {
342 stream = STREAM_SHA1_SIGNATURE;
344 Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i.\n"), chksum.type);
347 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream);
348 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
349 memcpy(sd->msg, chksum.signature, chksum.length);
350 sd->msglen = chksum.length;
352 bnet_sig(sd, BNET_EOD); /* end of checksum */
360 * Send data read from an already open file descriptor.
362 * We return 1 on sucess and 0 on errors.
365 * We use ff_pkt->statp.st_size when FO_SPARSE.
366 * Currently this is not a problem as the only other stream, resource forks,
367 * are not handled as sparse files.
369 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum)
371 BSOCK *sd = jcr->store_bsock;
372 uint64_t fileAddr = 0; /* file address */
374 int rsize = jcr->buf_size; /* read buffer size */
378 rbuf = sd->msg; /* read buffer */
379 wbuf = sd->msg; /* write buffer */
382 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
386 uLong compress_len, max_compress_len = 0;
387 const Bytef *cbuf = NULL;
389 if (ff_pkt->flags & FO_GZIP) {
390 if (ff_pkt->flags & FO_SPARSE) {
391 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
392 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
394 cbuf = (Bytef *)jcr->compress_buf;
395 max_compress_len = jcr->compress_buf_size; /* set max length */
397 wbuf = jcr->compress_buf; /* compressed output here */
402 * Send Data header to Storage daemon
403 * <file-index> <stream> <info>
405 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
406 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
410 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
413 * Make space at beginning of buffer for fileAddr because this
414 * same buffer will be used for writing if compression if off.
416 if (ff_pkt->flags & FO_SPARSE) {
417 rbuf += SPARSE_FADDR_SIZE;
418 rsize -= SPARSE_FADDR_SIZE;
419 #ifdef HAVE_FREEBSD_OS
421 * To read FreeBSD partitions, the read size must be
424 rsize = (rsize/512) * 512;
428 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
430 if (S_ISBLK(ff_pkt->statp.st_mode))
431 rsize = (rsize/512) * 512;
437 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
440 /* Check for sparse blocks */
441 if (ff_pkt->flags & FO_SPARSE) {
443 if (sd->msglen == rsize &&
444 (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) {
445 sparseBlock = is_buf_zero(rbuf, rsize);
448 ser_begin(wbuf, SPARSE_FADDR_SIZE);
449 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
452 jcr->ReadBytes += sd->msglen; /* count bytes read */
453 fileAddr += sd->msglen;
455 /* Update checksum if requested */
456 chksum_update(chksum, (unsigned char *)rbuf, sd->msglen);
459 /* Do compression if turned on */
460 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
462 compress_len = max_compress_len;
463 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
465 /* NOTE! This call modifies compress_len !!! */
466 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
467 (const Bytef *)rbuf, (uLong)sd->msglen,
468 ff_pkt->GZIP_level)) != Z_OK) {
469 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
472 set_jcr_job_status(jcr, JS_ErrorTerminated);
475 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
476 compress_len, sd->msglen);
478 sd->msglen = compress_len; /* set compressed length */
482 /* Send the buffer to the Storage daemon */
484 if (ff_pkt->flags & FO_SPARSE) {
485 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
487 sd->msg = wbuf; /* set correct write buffer */
488 if (!bnet_send(sd)) {
489 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
491 sd->msg = msgsave; /* restore bnet buffer */
496 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
498 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
499 sd->msg = msgsave; /* restore read buffer */
501 } /* end while read file data */
504 if (sd->msglen < 0) {
506 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
507 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
508 if (jcr->Errors++ > 1000) { /* insanity check */
509 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
514 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
515 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
524 * Read and send an ACL for the last encountered file.
526 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
529 BSOCK *sd = jcr->store_bsock;
533 len = bacl_get(jcr, acltype);
535 Jmsg1(jcr, M_WARNING, 0, "Error reading ACL of %s\n", jcr->last_fname);
539 return true; /* no ACL */
543 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
544 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
549 /* Send the buffer to the storage deamon */
550 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
552 sd->msg = jcr->acl_text;
553 sd->msglen = len + 1;
554 if (!bnet_send(sd)) {
557 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
562 jcr->JobBytes += sd->msglen;
564 if (!bnet_sig(sd, BNET_EOD)) {
565 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
570 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
575 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
577 BSOCK *sd = jcr->store_bsock;
578 char attribs[MAXSTRING];
579 char attribsEx[MAXSTRING];
582 #ifdef FD_NO_SEND_TEST
586 /* Find what data stream we will use, then encode the attributes */
587 data_stream = select_data_stream(ff_pkt);
588 encode_stat(attribs, ff_pkt, data_stream);
590 /* Now possibly extend the attributes */
591 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
593 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
596 jcr->JobFiles++; /* increment number of files sent */
597 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
598 pm_strcpy(jcr->last_fname, ff_pkt->fname);
602 * Send Attributes header to Storage daemon
603 * <file-index> <stream> <info>
605 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
606 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
610 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
613 * Send file attributes to Storage daemon
616 * Filename (full path)
618 * Link name (if type==FT_LNK or FT_LNKSAVED)
619 * Encoded extended-attributes (for Win32)
621 * For a directory, link is the same as fname, but with trailing
622 * slash. For a linked file, link is the link.
624 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
625 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
626 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
627 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
629 } else if (ff_pkt->type == FT_DIREND) {
630 /* Here link is the canonical filename (i.e. with trailing slash) */
631 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
632 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
634 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
635 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
638 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
640 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
644 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */