2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2005 Kern Sibbald
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 version 2 as amended with additional clauses defined in the
16 file LICENSE in the main source directory.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 the file LICENSE for additional details.
28 /* Forward referenced functions */
29 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
30 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum);
31 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
32 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
35 * Find all the requested files and send them
36 * to the Storage daemon.
38 * Note, we normally carry on a one-way
39 * conversation from this point on with the SD, simply blasting
40 * data to him. To properly know what is going on, we
41 * also run a "heartbeat" monitor which reads the socket and
42 * reacts accordingly (at the moment it has nothing to do
43 * except echo the heartbeat to the Director).
46 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
51 sd = jcr->store_bsock;
53 set_jcr_job_status(jcr, JS_Running);
55 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
58 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
62 buf_size = client->max_network_buffer_size;
64 buf_size = 0; /* use default */
66 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
67 set_jcr_job_status(jcr, JS_ErrorTerminated);
68 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 jcr->acl_text = get_pool_memory(PM_MESSAGE);
90 /* Subroutine save_file() is called for each file */
91 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
92 ok = false; /* error */
93 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
97 free_pool_memory(jcr->acl_text);
99 bnet_sig(sd, BNET_EOD); /* end of sending data */
101 stop_heartbeat_monitor(jcr);
107 if (jcr->compress_buf) {
108 free_pool_memory(jcr->compress_buf);
109 jcr->compress_buf = NULL;
111 Dmsg1(100, "end blast_data ok=%d\n", ok);
116 * Called here by find() for each file included.
117 * This is a callback. The original is find_files() above.
119 * Send the file and its data to the Storage daemon.
123 * -1 to ignore file/directory (not used here)
125 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
127 int stat, data_stream;
128 struct CHKSUM chksum;
130 JCR *jcr = (JCR *)vjcr;
132 if (job_canceled(jcr)) {
136 sd = jcr->store_bsock;
137 jcr->num_files_examined++; /* bump total file count */
139 switch (ff_pkt->type) {
140 case FT_LNKSAVED: /* Hard linked, file already saved */
141 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
144 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
147 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
150 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
153 return 1; /* not used */
155 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"),
157 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
160 /* Suppress message for /dev filesystems */
161 if (strncmp(ff_pkt->fname, "/dev/", 5) != 0) {
162 Jmsg(jcr, M_INFO, 1, _(" Filesystem change prohibited. Will not descend into %s\n"),
165 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
168 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"),
170 ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
173 Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
176 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
179 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
182 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
186 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
187 be.strerror(ff_pkt->ff_errno));
193 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
194 be.strerror(ff_pkt->ff_errno));
200 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
201 be.strerror(ff_pkt->ff_errno));
207 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
210 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
214 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
215 be.strerror(ff_pkt->ff_errno));
220 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
225 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
227 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
232 * Setup for signature handling.
233 * Then initialise the file descriptor we use for data and other streams.
235 chksum_init(&chksum, ff_pkt->flags);
238 if (ff_pkt->flags & FO_PORTABLE) {
239 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
241 if (ff_pkt->reader) {
242 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
243 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
250 * Open any file with data that we intend to save, then save it.
252 * Note, if is_win32_backup, we must open the Directory so that
253 * the BackupRead will save its permissions and ownership streams.
255 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
256 ff_pkt->statp.st_size > 0) ||
257 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
258 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
260 if (ff_pkt->type == FT_FIFO) {
261 tid = start_thread_timer(pthread_self(), 60);
265 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
266 ff_pkt->ff_errno = errno;
268 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
272 stop_thread_timer(tid);
278 stop_thread_timer(tid);
281 stat = send_data(jcr, data_stream, ff_pkt, &chksum);
282 bclose(&ff_pkt->bfd);
288 #ifdef HAVE_DARWIN_OS
289 /* Regular files can have resource forks and Finder Info */
290 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
291 ff_pkt->flags & FO_HFSPLUS)) {
292 if (ff_pkt->hfsinfo.rsrclength > 0) {
294 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
295 ff_pkt->ff_errno = errno;
297 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
300 if (is_bopen(&ff_pkt->bfd)) {
301 bclose(&ff_pkt->bfd);
305 flags = ff_pkt->flags;
306 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
307 stat = send_data(jcr, STREAM_MACOS_FORK_DATA, ff_pkt, &chksum);
308 ff_pkt->flags = flags;
309 bclose(&ff_pkt->bfd);
315 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
316 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
317 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
318 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
320 chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen);
322 bnet_sig(sd, BNET_EOD);
326 if (ff_pkt->flags & FO_ACL) {
327 /* Read access ACLs for files, dirs and links */
328 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
331 /* Directories can have default ACLs too */
332 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
333 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
339 /* Terminate any signature and send it to Storage daemon and the Director */
340 if (chksum.updated) {
342 chksum_final(&chksum);
343 if (chksum.type == CHKSUM_MD5) {
344 stream = STREAM_MD5_SIGNATURE;
345 } else if (chksum.type == CHKSUM_SHA1) {
346 stream = STREAM_SHA1_SIGNATURE;
348 Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i.\n"), chksum.type);
351 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream);
352 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
353 memcpy(sd->msg, chksum.signature, chksum.length);
354 sd->msglen = chksum.length;
356 bnet_sig(sd, BNET_EOD); /* end of checksum */
364 * Send data read from an already open file descriptor.
366 * We return 1 on sucess and 0 on errors.
369 * We use ff_pkt->statp.st_size when FO_SPARSE.
370 * Currently this is not a problem as the only other stream, resource forks,
371 * are not handled as sparse files.
373 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum)
375 BSOCK *sd = jcr->store_bsock;
376 uint64_t fileAddr = 0; /* file address */
378 int rsize = jcr->buf_size; /* read buffer size */
382 rbuf = sd->msg; /* read buffer */
383 wbuf = sd->msg; /* write buffer */
386 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
390 uLong compress_len, max_compress_len = 0;
391 const Bytef *cbuf = NULL;
393 if (ff_pkt->flags & FO_GZIP) {
394 if (ff_pkt->flags & FO_SPARSE) {
395 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
396 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
398 cbuf = (Bytef *)jcr->compress_buf;
399 max_compress_len = jcr->compress_buf_size; /* set max length */
401 wbuf = jcr->compress_buf; /* compressed output here */
406 * Send Data header to Storage daemon
407 * <file-index> <stream> <info>
409 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
410 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
414 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
417 * Make space at beginning of buffer for fileAddr because this
418 * same buffer will be used for writing if compression if off.
420 if (ff_pkt->flags & FO_SPARSE) {
421 rbuf += SPARSE_FADDR_SIZE;
422 rsize -= SPARSE_FADDR_SIZE;
423 #ifdef HAVE_FREEBSD_OS
425 * To read FreeBSD partitions, the read size must be
428 rsize = (rsize/512) * 512;
432 /* a RAW device read on win32 only works if the buffer is a multiple of 512 */
434 if (S_ISBLK(ff_pkt->statp.st_mode))
435 rsize = (rsize/512) * 512;
441 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
444 /* Check for sparse blocks */
445 if (ff_pkt->flags & FO_SPARSE) {
447 if (sd->msglen == rsize &&
448 (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) {
449 sparseBlock = is_buf_zero(rbuf, rsize);
452 ser_begin(wbuf, SPARSE_FADDR_SIZE);
453 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
456 jcr->ReadBytes += sd->msglen; /* count bytes read */
457 fileAddr += sd->msglen;
459 /* Update checksum if requested */
460 chksum_update(chksum, (unsigned char *)rbuf, sd->msglen);
463 /* Do compression if turned on */
464 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
466 compress_len = max_compress_len;
467 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
469 /* NOTE! This call modifies compress_len !!! */
470 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
471 (const Bytef *)rbuf, (uLong)sd->msglen,
472 ff_pkt->GZIP_level)) != Z_OK) {
473 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
476 set_jcr_job_status(jcr, JS_ErrorTerminated);
479 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
480 compress_len, sd->msglen);
482 sd->msglen = compress_len; /* set compressed length */
486 /* Send the buffer to the Storage daemon */
488 if (ff_pkt->flags & FO_SPARSE) {
489 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
491 sd->msg = wbuf; /* set correct write buffer */
492 if (!bnet_send(sd)) {
493 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
495 sd->msg = msgsave; /* restore bnet buffer */
500 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
502 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
503 sd->msg = msgsave; /* restore read buffer */
505 } /* end while read file data */
508 if (sd->msglen < 0) {
510 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
511 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
512 if (jcr->Errors++ > 1000) { /* insanity check */
513 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
518 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
519 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
528 * Read and send an ACL for the last encountered file.
530 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
533 BSOCK *sd = jcr->store_bsock;
537 len = bacl_get(jcr, acltype);
539 Jmsg1(jcr, M_WARNING, 0, _("Error reading ACL of %s\n"), jcr->last_fname);
543 return true; /* no ACL */
547 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
548 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
553 /* Send the buffer to the storage deamon */
554 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
556 sd->msg = jcr->acl_text;
557 sd->msglen = len + 1;
558 if (!bnet_send(sd)) {
561 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
566 jcr->JobBytes += sd->msglen;
568 if (!bnet_sig(sd, BNET_EOD)) {
569 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
574 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
579 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
581 BSOCK *sd = jcr->store_bsock;
582 char attribs[MAXSTRING];
583 char attribsEx[MAXSTRING];
586 #ifdef FD_NO_SEND_TEST
590 /* Find what data stream we will use, then encode the attributes */
591 data_stream = select_data_stream(ff_pkt);
592 encode_stat(attribs, ff_pkt, data_stream);
594 /* Now possibly extend the attributes */
595 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
597 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
600 jcr->JobFiles++; /* increment number of files sent */
601 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
602 pm_strcpy(jcr->last_fname, ff_pkt->fname);
606 * Send Attributes header to Storage daemon
607 * <file-index> <stream> <info>
609 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
610 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
614 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
617 * Send file attributes to Storage daemon
620 * Filename (full path)
622 * Link name (if type==FT_LNK or FT_LNKSAVED)
623 * Encoded extended-attributes (for Win32)
625 * For a directory, link is the same as fname, but with trailing
626 * slash. For a linked file, link is the link.
628 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
629 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
630 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
631 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
633 } else if (ff_pkt->type == FT_DIREND) {
634 /* Here link is the canonical filename (i.e. with trailing slash) */
635 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
636 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
638 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
639 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
642 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
644 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
648 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */