2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2005 Kern Sibbald
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License
15 version 2 as amended with additional clauses defined in the
16 file LICENSE in the main source directory.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 the file LICENSE for additional details.
28 /* Forward referenced functions */
29 static int save_file(FF_PKT *ff_pkt, void *pkt, bool top_level);
30 static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum);
31 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream);
32 static bool read_and_send_acl(JCR *jcr, int acltype, int stream);
35 * Find all the requested files and send them
36 * to the Storage daemon.
38 * Note, we normally carry on a one-way
39 * conversation from this point on with the SD, simply blasting
40 * data to him. To properly know what is going on, we
41 * also run a "heartbeat" monitor which reads the socket and
42 * reacts accordingly (at the moment it has nothing to do
43 * except echo the heartbeat to the Director).
46 bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
51 sd = jcr->store_bsock;
53 set_jcr_job_status(jcr, JS_Running);
55 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
58 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
62 buf_size = client->max_network_buffer_size;
64 buf_size = 0; /* use default */
66 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
67 set_jcr_job_status(jcr, JS_ErrorTerminated);
68 Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 jcr->acl_text = get_pool_memory(PM_MESSAGE);
90 /* Subroutine save_file() is called for each file */
91 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
92 ok = false; /* error */
93 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 // Jmsg(jcr, M_FATAL, 0, _("Find files error.\n"));
97 free_pool_memory(jcr->acl_text);
99 bnet_sig(sd, BNET_EOD); /* end of sending data */
101 stop_heartbeat_monitor(jcr);
107 if (jcr->compress_buf) {
108 free_pool_memory(jcr->compress_buf);
109 jcr->compress_buf = NULL;
111 Dmsg1(100, "end blast_data ok=%d\n", ok);
116 * Called here by find() for each file included.
117 * This is a callback. The original is find_files() above.
119 * Send the file and its data to the Storage daemon.
123 * -1 to ignore file/directory (not used here)
125 static int save_file(FF_PKT *ff_pkt, void *vjcr, bool top_level)
127 int stat, data_stream;
128 struct CHKSUM chksum;
130 JCR *jcr = (JCR *)vjcr;
132 if (job_canceled(jcr)) {
136 sd = jcr->store_bsock;
137 jcr->num_files_examined++; /* bump total file count */
139 switch (ff_pkt->type) {
140 case FT_LNKSAVED: /* Hard linked, file already saved */
141 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
144 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
147 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
150 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
153 return 1; /* not used */
158 if (ff_pkt->type == FT_NORECURSE) {
159 Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"),
161 } else if (ff_pkt->type == FT_NOFSCHG) {
162 Jmsg(jcr, M_INFO, 1, _(" File system change prohibited. Will not descend into %s\n"),
164 } else if (ff_pkt->type == FT_INVALIDFS) {
165 Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"),
168 ff_pkt->type = FT_DIREND; /* value is used below */
169 Dmsg1(130, "FT_DIR saving: %s\n", ff_pkt->link);
172 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
175 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
178 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
182 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
183 be.strerror(ff_pkt->ff_errno));
189 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
190 be.strerror(ff_pkt->ff_errno));
196 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
197 be.strerror(ff_pkt->ff_errno));
203 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
206 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
210 Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
211 be.strerror(ff_pkt->ff_errno));
216 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
221 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
223 if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
228 * Setup for signature handling.
229 * Then initialise the file descriptor we use for data and other streams.
231 chksum_init(&chksum, ff_pkt->flags);
234 if (ff_pkt->flags & FO_PORTABLE) {
235 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
237 if (ff_pkt->reader) {
238 if (!set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr)) {
239 Jmsg(jcr, M_FATAL, 0, _("Python reader program \"%s\" not found.\n"),
246 * Open any file with data that we intend to save, then save it.
248 * Note, if is_win32_backup, we must open the Directory so that
249 * the BackupRead will save its permissions and ownership streams.
251 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
252 ff_pkt->statp.st_size > 0) ||
253 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
254 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
256 if (ff_pkt->type == FT_FIFO) {
257 tid = start_thread_timer(pthread_self(), 60);
261 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
262 ff_pkt->ff_errno = errno;
264 Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
268 stop_thread_timer(tid);
274 stop_thread_timer(tid);
277 stat = send_data(jcr, data_stream, ff_pkt, &chksum);
278 bclose(&ff_pkt->bfd);
284 #ifdef HAVE_DARWIN_OS
285 /* Regular files can have resource forks and Finder Info */
286 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
287 ff_pkt->flags & FO_HFSPLUS)) {
288 if (ff_pkt->hfsinfo.rsrclength > 0) {
290 if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
291 ff_pkt->ff_errno = errno;
293 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname,
296 if (is_bopen(&ff_pkt->bfd)) {
297 bclose(&ff_pkt->bfd);
301 flags = ff_pkt->flags;
302 ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
303 stat = send_data(jcr, STREAM_MACOS_FORK_DATA, ff_pkt, &chksum);
304 ff_pkt->flags = flags;
305 bclose(&ff_pkt->bfd);
311 Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
312 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
313 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
314 memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
316 chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen);
318 bnet_sig(sd, BNET_EOD);
322 if (ff_pkt->flags & FO_ACL) {
323 /* Read access ACLs for files, dirs and links */
324 if (!read_and_send_acl(jcr, BACL_TYPE_ACCESS, STREAM_UNIX_ATTRIBUTES_ACCESS_ACL)) {
327 /* Directories can have default ACLs too */
328 if (ff_pkt->type == FT_DIREND && (BACL_CAP & BACL_CAP_DEFAULTS_DIR)) {
329 if (!read_and_send_acl(jcr, BACL_TYPE_DEFAULT, STREAM_UNIX_ATTRIBUTES_DEFAULT_ACL)) {
335 /* Terminate any signature and send it to Storage daemon and the Director */
336 if (chksum.updated) {
338 chksum_final(&chksum);
339 if (chksum.type == CHKSUM_MD5) {
340 stream = STREAM_MD5_SIGNATURE;
341 } else if (chksum.type == CHKSUM_SHA1) {
342 stream = STREAM_SHA1_SIGNATURE;
344 Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i.\n"), chksum.type);
347 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream);
348 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
349 memcpy(sd->msg, chksum.signature, chksum.length);
350 sd->msglen = chksum.length;
352 bnet_sig(sd, BNET_EOD); /* end of checksum */
360 * Send data read from an already open file descriptor.
362 * We return 1 on sucess and 0 on errors.
365 * We use ff_pkt->statp.st_size when FO_SPARSE.
366 * Currently this is not a problem as the only other stream, resource forks,
367 * are not handled as sparse files.
369 int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, struct CHKSUM *chksum)
371 BSOCK *sd = jcr->store_bsock;
372 uint64_t fileAddr = 0; /* file address */
374 int rsize = jcr->buf_size; /* read buffer size */
378 rbuf = sd->msg; /* read buffer */
379 wbuf = sd->msg; /* write buffer */
382 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
386 uLong compress_len, max_compress_len = 0;
387 const Bytef *cbuf = NULL;
389 if (ff_pkt->flags & FO_GZIP) {
390 if (ff_pkt->flags & FO_SPARSE) {
391 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
392 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
394 cbuf = (Bytef *)jcr->compress_buf;
395 max_compress_len = jcr->compress_buf_size; /* set max length */
397 wbuf = jcr->compress_buf; /* compressed output here */
402 * Send Data header to Storage daemon
403 * <file-index> <stream> <info>
405 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
406 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
410 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
413 * Make space at beginning of buffer for fileAddr because this
414 * same buffer will be used for writing if compression if off.
416 if (ff_pkt->flags & FO_SPARSE) {
417 rbuf += SPARSE_FADDR_SIZE;
418 rsize -= SPARSE_FADDR_SIZE;
419 #ifdef HAVE_FREEBSD_OS
421 * To read FreeBSD partitions, the read size must be
424 rsize = (rsize/512) * 512;
431 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
434 /* Check for sparse blocks */
435 if (ff_pkt->flags & FO_SPARSE) {
437 if (sd->msglen == rsize &&
438 (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) {
439 sparseBlock = is_buf_zero(rbuf, rsize);
442 ser_begin(wbuf, SPARSE_FADDR_SIZE);
443 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
446 jcr->ReadBytes += sd->msglen; /* count bytes read */
447 fileAddr += sd->msglen;
449 /* Update checksum if requested */
450 chksum_update(chksum, (unsigned char *)rbuf, sd->msglen);
453 /* Do compression if turned on */
454 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
456 compress_len = max_compress_len;
457 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
459 /* NOTE! This call modifies compress_len !!! */
460 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
461 (const Bytef *)rbuf, (uLong)sd->msglen,
462 ff_pkt->GZIP_level)) != Z_OK) {
463 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
466 set_jcr_job_status(jcr, JS_ErrorTerminated);
469 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
470 compress_len, sd->msglen);
472 sd->msglen = compress_len; /* set compressed length */
476 /* Send the buffer to the Storage daemon */
478 if (ff_pkt->flags & FO_SPARSE) {
479 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
481 sd->msg = wbuf; /* set correct write buffer */
482 if (!bnet_send(sd)) {
483 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
485 sd->msg = msgsave; /* restore bnet buffer */
490 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
492 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
493 sd->msg = msgsave; /* restore read buffer */
495 } /* end while read file data */
498 if (sd->msglen < 0) {
500 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
501 ff_pkt->fname, be.strerror(ff_pkt->bfd.berrno));
502 if (jcr->Errors++ > 1000) { /* insanity check */
503 Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
508 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
509 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
518 * Read and send an ACL for the last encountered file.
520 static bool read_and_send_acl(JCR *jcr, int acltype, int stream)
523 BSOCK *sd = jcr->store_bsock;
527 len = bacl_get(jcr, acltype);
529 Jmsg1(jcr, M_WARNING, 0, "Error reading ACL of %s\n", jcr->last_fname);
533 return true; /* no ACL */
537 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) {
538 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
543 /* Send the buffer to the storage deamon */
544 Dmsg2(400, "Backing up ACL type 0x%2x <%s>\n", acltype, jcr->acl_text);
546 sd->msg = jcr->acl_text;
547 sd->msglen = len + 1;
548 if (!bnet_send(sd)) {
551 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
556 jcr->JobBytes += sd->msglen;
558 if (!bnet_sig(sd, BNET_EOD)) {
559 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
564 Dmsg1(200, "ACL of file: %s successfully backed up!\n", jcr->last_fname);
569 static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
571 BSOCK *sd = jcr->store_bsock;
572 char attribs[MAXSTRING];
573 char attribsEx[MAXSTRING];
576 #ifdef FD_NO_SEND_TEST
580 /* Find what data stream we will use, then encode the attributes */
581 data_stream = select_data_stream(ff_pkt);
582 encode_stat(attribs, ff_pkt, data_stream);
584 /* Now possibly extend the attributes */
585 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
587 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
590 jcr->JobFiles++; /* increment number of files sent */
591 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
592 pm_strcpy(jcr->last_fname, ff_pkt->fname);
596 * Send Attributes header to Storage daemon
597 * <file-index> <stream> <info>
599 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
600 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
604 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
607 * Send file attributes to Storage daemon
610 * Filename (full path)
612 * Link name (if type==FT_LNK or FT_LNKSAVED)
613 * Encoded extended-attributes (for Win32)
615 * For a directory, link is the same as fname, but with trailing
616 * slash. For a linked file, link is the link.
618 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
619 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
620 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
621 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
623 } else if (ff_pkt->type == FT_DIREND) {
624 /* Here link is the canonical filename (i.e. with trailing slash) */
625 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
626 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
628 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
629 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
632 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
634 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
638 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */