2 * Bacula File Daemon backup.c send file attributes and data
3 * to the Storage daemon.
5 * Kern Sibbald, March MM
11 Copyright (C) 2000-2003 Kern Sibbald and John Walker
13 This program is free software; you can redistribute it and/or
14 modify it under the terms of the GNU General Public License as
15 published by the Free Software Foundation; either version 2 of
16 the License, or (at your option) any later version.
18 This program is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21 General Public License for more details.
23 You should have received a copy of the GNU General Public
24 License along with this program; if not, write to the Free
25 Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
33 static int save_file(FF_PKT *ff_pkt, void *pkt);
36 * Find all the requested files and send them
37 * to the Storage daemon.
39 * Note, we normally carry on a one-way
40 * conversation from this point on with the SD, simply blasting
41 * data to him. To properly know what is going on, we
42 * also run a "heartbeat" monitor which reads the socket and
43 * reacts accordingly (at the moment it has nothing to do
44 * except echo the heartbeat to the Director).
47 int blast_data_to_storage_daemon(JCR *jcr, char *addr)
52 sd = jcr->store_bsock;
54 set_jcr_job_status(jcr, JS_Running);
56 Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->fd);
59 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
63 buf_size = client->max_network_buffer_size;
65 buf_size = 0; /* use default */
67 if (!bnet_set_buffer_size(sd, buf_size, BNET_SETBUF_WRITE)) {
68 set_jcr_job_status(jcr, JS_ErrorTerminated);
72 jcr->buf_size = sd->msglen;
73 /* Adjust for compression so that output buffer is
74 * 12 bytes + 0.1% larger than input buffer plus 18 bytes.
75 * This gives a bit extra plus room for the sparse addr if any.
76 * Note, we adjust the read size to be smaller so that the
77 * same output buffer can be used without growing it.
79 jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
80 jcr->compress_buf = get_memory(jcr->compress_buf_size);
82 Dmsg1(300, "set_find_options ff=%p\n", jcr->ff);
83 set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
84 Dmsg0(300, "start find files\n");
86 start_heartbeat_monitor(jcr);
88 /* Subroutine save_file() is called for each file */
89 if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
91 set_jcr_job_status(jcr, JS_ErrorTerminated);
94 stop_heartbeat_monitor(jcr);
96 bnet_sig(sd, BNET_EOD); /* end data connection */
102 if (jcr->compress_buf) {
103 free_pool_memory(jcr->compress_buf);
104 jcr->compress_buf = NULL;
106 Dmsg1(300, "end blast_data stat=%d\n", stat);
111 * Called here by find() for each file included.
113 * *****FIXME***** add FSMs File System Modules
115 * Send the file and its data to the Storage daemon.
117 static int save_file(FF_PKT *ff_pkt, void *vjcr)
119 char attribs[MAXSTRING];
120 char attribsEx[MAXSTRING];
121 int stat, attr_stream, data_stream;
122 struct MD5Context md5c;
123 struct SHA1Context sha1c;
126 unsigned char signature[30]; /* large enough for either signature */
128 JCR *jcr = (JCR *)vjcr;
131 if (job_canceled(jcr)) {
135 sd = jcr->store_bsock;
136 jcr->num_files_examined++; /* bump total file count */
138 switch (ff_pkt->type) {
139 case FT_LNKSAVED: /* Hard linked, file already saved */
140 Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
143 Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
146 Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
149 Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
152 return 1; /* not used */
154 Dmsg1(130, "FT_DIR saving: %s\n", ff_pkt->link);
157 Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
160 Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
163 Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
166 Jmsg(jcr, M_NOTSAVED, -1, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname,
167 strerror(ff_pkt->ff_errno));
171 Jmsg(jcr, M_NOTSAVED, -1, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname,
172 strerror(ff_pkt->ff_errno));
176 Jmsg(jcr, M_NOTSAVED, -1, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname,
177 strerror(ff_pkt->ff_errno));
182 Jmsg(jcr, M_SKIPPED, -1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
185 Jmsg(jcr, M_NOTSAVED, -1, _(" Archive file not saved: %s\n"), ff_pkt->fname);
188 Jmsg(jcr, M_SKIPPED, -1, _(" Recursion turned off. Directory skipped: %s\n"),
192 Jmsg(jcr, M_SKIPPED, -1, _(" File system change prohibited. Directory skipped. %s\n"),
196 Jmsg(jcr, M_NOTSAVED, -1, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname,
197 strerror(ff_pkt->ff_errno));
201 Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"), ff_pkt->type, ff_pkt->fname);
207 if (ff_pkt->flags & FO_PORTABLE) {
208 set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
212 * Open any file with data that we intend to save.
213 * Note, if is_win32_backup, we must open the Directory so that
214 * the BackupRead will save its permissions and ownership streams.
216 if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
217 ff_pkt->statp.st_size > 0) ||
218 ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
219 (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
221 if (ff_pkt->type == FT_FIFO) {
222 tid = start_thread_timer(pthread_self(), 60);
226 if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
227 ff_pkt->ff_errno = errno;
228 Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname,
229 berror(&ff_pkt->bfd));
231 stop_thread_timer(tid);
234 stop_thread_timer(tid);
237 Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
239 /* Find what data stream we will use, then encode the attributes */
240 data_stream = select_data_stream(ff_pkt);
241 encode_stat(attribs, ff_pkt, data_stream);
243 /* Now possibly extend the attributes */
244 attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
246 Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
249 jcr->JobFiles++; /* increment number of files sent */
250 ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
251 pm_strcpy(&jcr->last_fname, ff_pkt->fname);
255 * Send Attributes header to Storage daemon
256 * <file-index> <stream> <info>
258 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) {
259 if (is_bopen(&ff_pkt->bfd)) {
260 bclose(&ff_pkt->bfd);
262 set_jcr_job_status(jcr, JS_ErrorTerminated);
265 Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
268 * Send file attributes to Storage daemon
271 * Filename (full path)
273 * Link name (if type==FT_LNK or FT_LNKSAVED)
274 * Encoded extended-attributes (for Win32)
276 * For a directory, link is the same as fname, but with trailing
277 * slash. For a linked file, link is the link.
279 if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
280 Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
281 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
282 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
284 } else if (ff_pkt->type == FT_DIREND) {
285 /* Here link is the canonical filename (i.e. with trailing slash) */
286 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
287 ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
289 stat = bnet_fsend(sd, "%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
290 ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
293 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
295 if (is_bopen(&ff_pkt->bfd)) {
296 bclose(&ff_pkt->bfd);
298 set_jcr_job_status(jcr, JS_ErrorTerminated);
301 bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */
304 * If the file has data, read it and send to the Storage daemon
307 if (is_bopen(&ff_pkt->bfd)) {
308 uint64_t fileAddr = 0; /* file address */
310 int rsize = jcr->buf_size; /* read buffer size */
313 rbuf = sd->msg; /* read buffer */
314 wbuf = sd->msg; /* write buffer */
317 Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
321 uLong compress_len, max_compress_len = 0;
322 const Bytef *cbuf = NULL;
324 if (ff_pkt->flags & FO_GZIP) {
325 if (ff_pkt->flags & FO_SPARSE) {
326 cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
327 max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
329 cbuf = (Bytef *)jcr->compress_buf;
330 max_compress_len = jcr->compress_buf_size; /* set max length */
332 wbuf = jcr->compress_buf; /* compressed output here */
337 * Send Data header to Storage daemon
338 * <file-index> <stream> <info>
340 if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, data_stream)) {
341 bclose(&ff_pkt->bfd);
342 set_jcr_job_status(jcr, JS_ErrorTerminated);
345 Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
347 if (ff_pkt->flags & FO_MD5) {
349 } else if (ff_pkt->flags & FO_SHA1) {
354 * Make space at beginning of buffer for fileAddr because this
355 * same buffer will be used for writing if compression if off.
357 if (ff_pkt->flags & FO_SPARSE) {
358 rbuf += SPARSE_FADDR_SIZE;
359 rsize -= SPARSE_FADDR_SIZE;
360 #ifdef HAVE_FREEBSD_OS
362 * To read FreeBSD partitions, the read size must be
365 rsize = (rsize/512) * 512;
372 while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
375 /* Check for sparse blocks */
376 if (ff_pkt->flags & FO_SPARSE) {
378 if (sd->msglen == rsize &&
379 (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) {
380 sparseBlock = is_buf_zero(rbuf, rsize);
383 ser_begin(wbuf, SPARSE_FADDR_SIZE);
384 ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
387 jcr->ReadBytes += sd->msglen; /* count bytes read */
388 fileAddr += sd->msglen;
390 /* Update MD5 if requested */
391 if (ff_pkt->flags & FO_MD5) {
392 MD5Update(&md5c, (unsigned char *)rbuf, sd->msglen);
394 } else if (ff_pkt->flags & FO_SHA1) {
395 SHA1Update(&sha1c, (unsigned char *)rbuf, sd->msglen);
400 /* Do compression if turned on */
401 if (!sparseBlock && ff_pkt->flags & FO_GZIP) {
403 compress_len = max_compress_len;
404 Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len,
406 /* NOTE! This call modifies compress_len !!! */
407 if ((zstat=compress2((Bytef *)cbuf, &compress_len,
408 (const Bytef *)rbuf, (uLong)sd->msglen,
409 ff_pkt->GZIP_level)) != Z_OK) {
410 Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat);
413 bclose(&ff_pkt->bfd);
414 set_jcr_job_status(jcr, JS_ErrorTerminated);
417 Dmsg2(400, "compressed len=%d uncompressed len=%d\n",
418 compress_len, sd->msglen);
420 sd->msglen = compress_len; /* set compressed length */
424 /* Send the buffer to the Storage daemon */
426 if (ff_pkt->flags & FO_SPARSE) {
427 sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
429 sd->msg = wbuf; /* set correct write buffer */
430 if (!bnet_send(sd)) {
431 sd->msg = msgsave; /* restore bnet buffer */
433 bclose(&ff_pkt->bfd);
434 set_jcr_job_status(jcr, JS_ErrorTerminated);
438 Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
440 jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */
441 sd->msg = msgsave; /* restore read buffer */
443 } /* end while read file data */
446 if (sd->msglen < 0) {
447 Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
448 ff_pkt->fname, berror(&ff_pkt->bfd));
451 bclose(&ff_pkt->bfd); /* close file */
452 if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */
453 set_jcr_job_status(jcr, JS_ErrorTerminated);
458 /* Terminate any MD5 signature and send it to Storage daemon and the Director */
459 if (gotMD5 && ff_pkt->flags & FO_MD5) {
460 MD5Final(signature, &md5c);
461 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_MD5_SIGNATURE);
462 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
463 memcpy(sd->msg, signature, 16);
466 bnet_sig(sd, BNET_EOD); /* end of MD5 */
469 } else if (gotSHA1 && ff_pkt->flags & FO_SHA1) {
470 /* Terminate any SHA1 signature and send it to Storage daemon and the Director */
471 SHA1Final(&sha1c, signature);
472 bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_SHA1_SIGNATURE);
473 Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
474 memcpy(sd->msg, signature, 20);
477 bnet_sig(sd, BNET_EOD); /* end of SHA1 */