From 92ae1d20125eea44af20cd5e07f1ca96e3a0b474 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sun, 12 Dec 2004 19:52:41 +0000 Subject: [PATCH] Integrated Preben 'Peppe' Guldberg three cleanup patches (backup, chksum, and verify) git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@1760 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/filed/backup.c | 605 ++++++++++++++++---------------------- bacula/src/filed/chksum.h | 58 ++++ bacula/src/filed/verify.c | 296 ++++++++----------- 3 files changed, 430 insertions(+), 529 deletions(-) diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index e6dafd5482..8ede73770b 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -8,7 +8,7 @@ * */ /* - Copyright (C) 2000-2004 Kern Sibbald and John Walker + Copyright (C) 2000-2004 Kern Sibbald This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -36,6 +36,7 @@ #endif static int save_file(FF_PKT *ff_pkt, void *pkt); +static int send_data(int stream, FF_PKT *ff_pkt, BSOCK *sd, JCR *jcr, struct CHKSUM *chksum); /* * Find all the requested files and send them @@ -130,15 +131,9 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) char attribs[MAXSTRING]; char attribsEx[MAXSTRING]; int stat, attr_stream, data_stream; - bool hfsplus = false; - struct MD5Context md5c; - struct SHA1Context sha1c; - int gotMD5 = 0; - int gotSHA1 = 0; - unsigned char signature[30]; /* large enough for either signature */ + struct CHKSUM chksum; BSOCK *sd; JCR *jcr = (JCR *)vjcr; - POOLMEM *msgsave; if (job_canceled(jcr)) { return 0; @@ -167,16 +162,16 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) case FT_INVALIDFS: case FT_DIREND: if (ff_pkt->type == FT_NORECURSE) { - Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"), + Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend into %s\n"), ff_pkt->fname); } else if (ff_pkt->type == FT_NOFSCHG) { - Jmsg(jcr, M_INFO, 1, _(" File system change prohibited. Will not descend into %s\n"), + Jmsg(jcr, M_INFO, 1, _(" File system change prohibited. Will not descend into %s\n"), ff_pkt->fname); } else if (ff_pkt->type == FT_INVALIDFS) { - Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"), + Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend into %s\n"), ff_pkt->fname); } - ff_pkt->type = FT_DIREND; /* value is used below */ + ff_pkt->type = FT_DIREND; /* value is used below */ Dmsg1(130, "FT_DIR saving: %s\n", ff_pkt->link); break; case FT_SPEC: @@ -233,70 +228,6 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) return 1; } - binit(&ff_pkt->bfd); - if (ff_pkt->flags & FO_PORTABLE) { - set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */ - } - if (ff_pkt->reader) { - set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr); - } - - /* - * Open any file with data that we intend to save. - * Note, if is_win32_backup, we must open the Directory so that - * the BackupRead will save its permissions and ownership streams. - */ - if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && - ff_pkt->statp.st_size > 0) || - ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO || - (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) { - btimer_t *tid; - if (ff_pkt->type == FT_FIFO) { - tid = start_thread_timer(pthread_self(), 60); - } else { - tid = NULL; - } - if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { - ff_pkt->ff_errno = errno; - berrno be; - Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname, - be.strerror()); - jcr->Errors++; - if (tid) { - stop_thread_timer(tid); - tid = NULL; - } - return 1; - } - if (tid) { - stop_thread_timer(tid); - tid = NULL; - } - } - - binit(&ff_pkt->rsrc_bfd); -#ifdef HAVE_DARWIN_OS - /* Open resource fork if necessary */ - if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && - ff_pkt->flags & FO_HFSPLUS)) { - /* Remember Finder Info, whether we have data or fork, or not */ - hfsplus = true; - if (ff_pkt->hfsinfo.rsrclength > 0) { - if (!bopen_rsrc(&ff_pkt->rsrc_bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { - ff_pkt->ff_errno = errno; - berrno be; - Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname, - be.strerror()); - jcr->Errors++; - if (is_bopen(&ff_pkt->bfd)) { - bclose(&ff_pkt->bfd); - } - return 1; - } - } - } -#endif - Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname); /* Find what data stream we will use, then encode the attributes */ @@ -320,12 +251,6 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) */ if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, attr_stream)) { berrno be; - if (is_bopen(&ff_pkt->bfd)) { - bclose(&ff_pkt->bfd); - } - if (is_bopen(&ff_pkt->rsrc_bfd)) { - bclose(&ff_pkt->rsrc_bfd); - } Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), bnet_strerror(sd)); return 0; @@ -361,291 +286,107 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg); if (!stat) { berrno be; - if (is_bopen(&ff_pkt->bfd)) { - bclose(&ff_pkt->bfd); - } - if (is_bopen(&ff_pkt->rsrc_bfd)) { - bclose(&ff_pkt->rsrc_bfd); - } Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), bnet_strerror(sd)); return 0; } bnet_sig(sd, BNET_EOD); /* indicate end of attributes data */ - if (is_bopen(&ff_pkt->bfd) || hfsplus) { - if (ff_pkt->flags & FO_MD5) { - MD5Init(&md5c); - } else if (ff_pkt->flags & FO_SHA1) { - SHA1Init(&sha1c); - } + /* + * Setup for signature handling. + * Then initialise the file descriptor we use for data and other streams. + */ + chksum_init(&chksum, ff_pkt->flags); + + binit(&ff_pkt->bfd); + if (ff_pkt->flags & FO_PORTABLE) { + set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */ + } + if (ff_pkt->reader) { + set_prog(&ff_pkt->bfd, ff_pkt->reader, jcr); } /* - * If the file has data, read it and send to the Storage daemon + * Open any file with data that we intend to save, then save it. * + * Note, if is_win32_backup, we must open the Directory so that + * the BackupRead will save its permissions and ownership streams. */ - if (is_bopen(&ff_pkt->bfd)) { - uint64_t fileAddr = 0; /* file address */ - char *rbuf, *wbuf; - int rsize = jcr->buf_size; /* read buffer size */ - - msgsave = sd->msg; - rbuf = sd->msg; /* read buffer */ - wbuf = sd->msg; /* write buffer */ - - - Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type); - - -#ifdef HAVE_LIBZ - uLong compress_len, max_compress_len = 0; - const Bytef *cbuf = NULL; - - if (ff_pkt->flags & FO_GZIP) { - if (ff_pkt->flags & FO_SPARSE) { - cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE; - max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE; - } else { - cbuf = (Bytef *)jcr->compress_buf; - max_compress_len = jcr->compress_buf_size; /* set max length */ - } - wbuf = jcr->compress_buf; /* compressed output here */ + if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && + ff_pkt->statp.st_size > 0) || + ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO || + (!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) { + btimer_t *tid; + if (ff_pkt->type == FT_FIFO) { + tid = start_thread_timer(pthread_self(), 60); + } else { + tid = NULL; } -#endif - - /* - * Send Data header to Storage daemon - * - */ - if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, data_stream)) { + if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { + ff_pkt->ff_errno = errno; berrno be; - bclose(&ff_pkt->bfd); - if (is_bopen(&ff_pkt->rsrc_bfd)) { - bclose(&ff_pkt->rsrc_bfd); + Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open %s: ERR=%s.\n"), ff_pkt->fname, + be.strerror()); + jcr->Errors++; + if (tid) { + stop_thread_timer(tid); + tid = NULL; } - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - bnet_strerror(sd)); - return 0; - } - Dmsg1(300, ">stored: datahdr %s\n", sd->msg); - - /* - * Make space at beginning of buffer for fileAddr because this - * same buffer will be used for writing if compression if off. - */ - if (ff_pkt->flags & FO_SPARSE) { - rbuf += SPARSE_FADDR_SIZE; - rsize -= SPARSE_FADDR_SIZE; -#ifdef HAVE_FREEBSD_OS - /* - * To read FreeBSD partitions, the read size must be - * a multiple of 512. - */ - rsize = (rsize/512) * 512; -#endif + return 1; } - - /* - * Read the file data - */ - while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) { - int sparseBlock = 0; - - /* Check for sparse blocks */ - if (ff_pkt->flags & FO_SPARSE) { - ser_declare; - if (sd->msglen == rsize && - (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) { - sparseBlock = is_buf_zero(rbuf, rsize); - } - - ser_begin(wbuf, SPARSE_FADDR_SIZE); - ser_uint64(fileAddr); /* store fileAddr in begin of buffer */ - } - - jcr->ReadBytes += sd->msglen; /* count bytes read */ - fileAddr += sd->msglen; - - /* Update MD5 if requested */ - if (ff_pkt->flags & FO_MD5) { - MD5Update(&md5c, (unsigned char *)rbuf, sd->msglen); - gotMD5 = 1; - } else if (ff_pkt->flags & FO_SHA1) { - SHA1Update(&sha1c, (unsigned char *)rbuf, sd->msglen); - gotSHA1 = 1; - } - -#ifdef HAVE_LIBZ - /* Do compression if turned on */ - if (!sparseBlock && ff_pkt->flags & FO_GZIP) { - int zstat; - compress_len = max_compress_len; - Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len, - rbuf, sd->msglen); - /* NOTE! This call modifies compress_len !!! */ - if ((zstat=compress2((Bytef *)cbuf, &compress_len, - (const Bytef *)rbuf, (uLong)sd->msglen, - ff_pkt->GZIP_level)) != Z_OK) { - Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat); - sd->msg = msgsave; - sd->msglen = 0; - bclose(&ff_pkt->bfd); - if (is_bopen(&ff_pkt->rsrc_bfd)) { - bclose(&ff_pkt->rsrc_bfd); - } - set_jcr_job_status(jcr, JS_ErrorTerminated); - return 0; - } - Dmsg2(400, "compressed len=%d uncompressed len=%d\n", - compress_len, sd->msglen); - - sd->msglen = compress_len; /* set compressed length */ - } -#endif - - /* Send the buffer to the Storage daemon */ - if (!sparseBlock) { - if (ff_pkt->flags & FO_SPARSE) { - sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */ - } - sd->msg = wbuf; /* set correct write buffer */ - if (!bnet_send(sd)) { - berrno be; - Jmsg2(jcr, M_FATAL, 0, _("Network send error %d to SD. ERR=%s\n"), - sd->msglen, bnet_strerror(sd)); - sd->msg = msgsave; /* restore bnet buffer */ - sd->msglen = 0; - bclose(&ff_pkt->bfd); - if (is_bopen(&ff_pkt->rsrc_bfd)) { - bclose(&ff_pkt->rsrc_bfd); - } - return 0; - } - } - Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); - /* #endif */ - jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */ - sd->msg = msgsave; /* restore read buffer */ - - } /* end while read file data */ - - - if (sd->msglen < 0) { - berrno be; - be.set_errno(ff_pkt->bfd.berrno); - Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), - ff_pkt->fname, be.strerror()); + if (tid) { + stop_thread_timer(tid); + tid = NULL; } - - bclose(&ff_pkt->bfd); /* close file */ - if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */ - berrno be; - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - bnet_strerror(sd)); - if (is_bopen(&ff_pkt->rsrc_bfd)) { - bclose(&ff_pkt->rsrc_bfd); - } + stat = send_data(data_stream, ff_pkt, sd, jcr, &chksum); + bclose(&ff_pkt->bfd); + if (!stat) { return 0; } } #ifdef HAVE_DARWIN_OS - /* - * If resource fork has data, read it and send to the Storage daemon - * - */ - if (is_bopen(&ff_pkt->rsrc_bfd)) { - uint64_t fileAddr = 0; /* file address */ - int rsize = jcr->buf_size; /* read buffer size */ - - Dmsg1(300, "Saving resource fork for \"%s\"", ff_pkt->fname); - - /* - * Send Data header to Storage daemon - * - */ - if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_MACOS_FORK_DATA)) { - berrno be; - bclose(&ff_pkt->rsrc_bfd); - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - bnet_strerror(sd)); - return 0; - } - Dmsg1(300, ">stored: datahdr %s\n", sd->msg); - - /* - * Read the resource fork - */ - while ((sd->msglen=(uint32_t)bread(&ff_pkt->rsrc_bfd, sd->msg, rsize)) > 0) { - jcr->ReadBytes += sd->msglen; /* count bytes read */ - fileAddr += sd->msglen; - - /* Update MD5 if requested */ - if (ff_pkt->flags & FO_MD5) { - MD5Update(&md5c, (unsigned char *)sd->msg, sd->msglen); - gotMD5 = 1; - } else if (ff_pkt->flags & FO_SHA1) { - SHA1Update(&sha1c, (unsigned char *)sd->msg, sd->msglen); - gotSHA1 = 1; - } - - /* Send the buffer to the Storage daemon */ - if (!bnet_send(sd)) { + /* Open resource fork if necessary and save content */ + if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && + ff_pkt->flags & FO_HFSPLUS)) { + if (ff_pkt->hfsinfo.rsrclength > 0) { + int flags; + if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { + ff_pkt->ff_errno = errno; berrno be; - Jmsg2(jcr, M_FATAL, 0, _("Network send error %d to SD. ERR=%s\n"), - sd->msglen, bnet_strerror(sd)); - sd->msglen = 0; - bclose(&ff_pkt->rsrc_bfd); + Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s.\n"), ff_pkt->fname, + be.strerror()); + jcr->Errors++; + if (is_bopen(&ff_pkt->bfd)) { + bclose(&ff_pkt->bfd); + } + return 1; + } + flags = ff_pkt->flags; + ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE); + stat = send_data(STREAM_MACOS_FORK_DATA, ff_pkt, sd, jcr, &chksum); + ff_pkt->flags = flags; + bclose(&ff_pkt->bfd); + if (!stat) { return 0; } - Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); - /* #endif */ - jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */ - } /* end while read file data */ - - if (sd->msglen < 0) { - berrno be; - be.set_errno(ff_pkt->rsrc_bfd.berrno); - Jmsg(jcr, M_ERROR, 0, _("Read error on resource fork of %s. ERR=%s\n"), - ff_pkt->fname, be.strerror()); } - bclose(&ff_pkt->rsrc_bfd); /* close resource fork */ - if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of resource fork */ - berrno be; - Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), - bnet_strerror(sd)); - return 0; - } - } -#endif - -#ifdef HAVE_DARWIN_OS - /* Handle Finder Info */ - if (S_ISREG(ff_pkt->statp.st_mode) && ff_pkt->flags & FO_HFSPLUS) { Dmsg1(300, "Saving Finder Info for \"%s\"", ff_pkt->fname); bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES); Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32); sd->msglen = 32; - /* Update MD5 if requested */ - if (ff_pkt->flags & FO_MD5) { - MD5Update(&md5c, (unsigned char *)sd->msg, sd->msglen); - gotMD5 = 1; - } else if (ff_pkt->flags & FO_SHA1) { - SHA1Update(&sha1c, (unsigned char *)sd->msg, sd->msglen); - gotSHA1 = 1; - } + chksum_update(&chksum, (unsigned char *)sd->msg, sd->msglen); bnet_send(sd); bnet_sig(sd, BNET_EOD); } #endif - #ifdef HAVE_ACL /* ACL stream */ if (ff_pkt->flags & FO_ACL) { + POOLMEM *msgsave; char *acl_text; /* Read ACLs for files, dirs and links */ if (ff_pkt->type == FT_DIREND) { @@ -672,23 +413,23 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) acl_text = acl_to_any_text(myAcl, NULL, ',', TEXT_ABBREVIATE); acl_free(myAcl); } - + /* If there is an ACL, send it to the Storage daemon */ if (acl_text) { sd = jcr->store_bsock; pm_strcpy(&jcr->last_fname, ff_pkt->fname); - + /* - * Send ACL header - * - */ + * Send ACL header + * + */ if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_UNIX_ATTRIBUTES_ACL)) { berrno be; Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), bnet_strerror(sd)); return 0; } - + /* Send the buffer to the storage deamon */ msgsave = sd->msg; sd->msg = acl_text; @@ -714,27 +455,183 @@ static int save_file(FF_PKT *ff_pkt, void *vjcr) } #endif - /* Terminate any MD5 signature and send it to Storage daemon and the Director */ - if (gotMD5 && ff_pkt->flags & FO_MD5) { - MD5Final(signature, &md5c); - bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_MD5_SIGNATURE); - Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); - memcpy(sd->msg, signature, 16); - sd->msglen = 16; - bnet_send(sd); - bnet_sig(sd, BNET_EOD); /* end of MD5 */ - gotMD5 = 0; + /* Terminate any signature and send it to Storage daemon and the Director */ + if (chksum.updated) { + int stream = 0; + chksum_final(&chksum); + if (chksum.type == CHKSUM_MD5) { + stream = STREAM_MD5_SIGNATURE; + } else if (chksum.type == CHKSUM_SHA1) { + stream = STREAM_SHA1_SIGNATURE; + } else { + Jmsg1(jcr, M_WARNING, 0, _("Unknown signature type %i."), chksum.type); + } + if (stream != 0) { + bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream); + Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); + memcpy(sd->msg, chksum.signature, chksum.length); + sd->msglen = chksum.length; + bnet_send(sd); + bnet_sig(sd, BNET_EOD); /* end of checksum */ + } + } - } else if (gotSHA1 && ff_pkt->flags & FO_SHA1) { - /* Terminate any SHA1 signature and send it to Storage daemon and the Director */ - SHA1Final(&sha1c, signature); - bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, STREAM_SHA1_SIGNATURE); - Dmsg1(300, "bfiled>stored:header %s\n", sd->msg); - memcpy(sd->msg, signature, 20); - sd->msglen = 20; - bnet_send(sd); - bnet_sig(sd, BNET_EOD); /* end of SHA1 */ - gotMD5 = 0; + return 1; +} + +/* + * Send data read from an already open file descriptor. + * + * We return 1 on sucess and 0 on errors. + * + * ***FIXME*** + * We use ff_pkt->statp.st_size when FO_SPARSE. + * Currently this is not a problem as the only other stream, resource forks, + * are not handled as sparse files. + */ +int send_data(int stream, FF_PKT *ff_pkt, BSOCK *sd, JCR *jcr, struct CHKSUM *chksum) +{ + uint64_t fileAddr = 0; /* file address */ + char *rbuf, *wbuf; + int rsize = jcr->buf_size; /* read buffer size */ + POOLMEM *msgsave; + + msgsave = sd->msg; + rbuf = sd->msg; /* read buffer */ + wbuf = sd->msg; /* write buffer */ + + + Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type); + + +#ifdef HAVE_LIBZ + uLong compress_len, max_compress_len = 0; + const Bytef *cbuf = NULL; + + if (ff_pkt->flags & FO_GZIP) { + if (ff_pkt->flags & FO_SPARSE) { + cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE; + max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE; + } else { + cbuf = (Bytef *)jcr->compress_buf; + max_compress_len = jcr->compress_buf_size; /* set max length */ + } + wbuf = jcr->compress_buf; /* compressed output here */ } +#endif + + /* + * Send Data header to Storage daemon + * + */ + if (!bnet_fsend(sd, "%ld %d 0", jcr->JobFiles, stream)) { + berrno be; + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + bnet_strerror(sd)); + return 0; + } + Dmsg1(300, ">stored: datahdr %s\n", sd->msg); + + /* + * Make space at beginning of buffer for fileAddr because this + * same buffer will be used for writing if compression if off. + */ + if (ff_pkt->flags & FO_SPARSE) { + rbuf += SPARSE_FADDR_SIZE; + rsize -= SPARSE_FADDR_SIZE; +#ifdef HAVE_FREEBSD_OS + /* + * To read FreeBSD partitions, the read size must be + * a multiple of 512. + */ + rsize = (rsize/512) * 512; +#endif + } + + /* + * Read the file data + */ + while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) { + int sparseBlock = 0; + + /* Check for sparse blocks */ + if (ff_pkt->flags & FO_SPARSE) { + ser_declare; + if (sd->msglen == rsize && + (fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size)) { + sparseBlock = is_buf_zero(rbuf, rsize); + } + + ser_begin(wbuf, SPARSE_FADDR_SIZE); + ser_uint64(fileAddr); /* store fileAddr in begin of buffer */ + } + + jcr->ReadBytes += sd->msglen; /* count bytes read */ + fileAddr += sd->msglen; + + /* Update checksum if requested */ + chksum_update(chksum, (unsigned char *)rbuf, sd->msglen); + +#ifdef HAVE_LIBZ + /* Do compression if turned on */ + if (!sparseBlock && ff_pkt->flags & FO_GZIP) { + int zstat; + compress_len = max_compress_len; + Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len, + rbuf, sd->msglen); + /* NOTE! This call modifies compress_len !!! */ + if ((zstat=compress2((Bytef *)cbuf, &compress_len, + (const Bytef *)rbuf, (uLong)sd->msglen, + ff_pkt->GZIP_level)) != Z_OK) { + Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat); + sd->msg = msgsave; + sd->msglen = 0; + set_jcr_job_status(jcr, JS_ErrorTerminated); + return 0; + } + Dmsg2(400, "compressed len=%d uncompressed len=%d\n", + compress_len, sd->msglen); + + sd->msglen = compress_len; /* set compressed length */ + } +#endif + + /* Send the buffer to the Storage daemon */ + if (!sparseBlock) { + if (ff_pkt->flags & FO_SPARSE) { + sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */ + } + sd->msg = wbuf; /* set correct write buffer */ + if (!bnet_send(sd)) { + berrno be; + Jmsg2(jcr, M_FATAL, 0, _("Network send error %d to SD. ERR=%s\n"), + sd->msglen, bnet_strerror(sd)); + sd->msg = msgsave; /* restore bnet buffer */ + sd->msglen = 0; + return 0; + } + } + Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); + /* #endif */ + jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed */ + sd->msg = msgsave; /* restore read buffer */ + + } /* end while read file data */ + + + if (sd->msglen < 0) { + berrno be; + be.set_errno(ff_pkt->bfd.berrno); + Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), + ff_pkt->fname, be.strerror()); + } + + if (!bnet_sig(sd, BNET_EOD)) { /* indicate end of file data */ + berrno be; + Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), + bnet_strerror(sd)); + return 0; + } + return 1; } diff --git a/bacula/src/filed/chksum.h b/bacula/src/filed/chksum.h index 7d0d45bf1a..125edce567 100644 --- a/bacula/src/filed/chksum.h +++ b/bacula/src/filed/chksum.h @@ -2,6 +2,64 @@ * General routines for handling the various checksum supported. */ +/* + Copyright (C) 2000-2004 Kern Sibbald and John Walker + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public + License along with this program; if not, write to the Free + Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, + MA 02111-1307, USA. + + */ + +#ifndef _CHKSUM_H_ +#define _CHKSUM_H_ + +#include "bacula.h" + +/* + * Link these to findlib options. Doing so allows for simpler handling of + * signatures in the callers. + * If multiple signatures are specified, the order in chksum_init() matters. + * Still, spell out our own names in case we want to change the approach. + */ +#define CHKSUM_NONE 0 +#define CHKSUM_MD5 FO_MD5 +#define CHKSUM_SHA1 FO_SHA1 + +union chksumContext { + MD5Context md5; + SHA1Context sha1; +}; + +struct CHKSUM { + int type; /* One of CHKSUM_* above */ + char name[5]; /* Big enough for NONE, MD5, SHA1, etc. */ + bool updated; /* True if updated by chksum_update() */ + chksumContext context; /* Context for the algorithm at hand */ + int length; /* Length of signature */ + unsigned char signature[30]; /* Large enough for either signature */ +}; + +int chksum_init(CHKSUM *chksum, int flags); +int chksum_update(CHKSUM *chksum, void *buf, unsigned len); +int chksum_final(CHKSUM *chksum); + +#endif +/* + * General routines for handling the various checksum supported. + */ + /* Copyright (C) 2004 Kern Sibbald diff --git a/bacula/src/filed/verify.c b/bacula/src/filed/verify.c index 4e125f439f..35b28e6ab1 100644 --- a/bacula/src/filed/verify.c +++ b/bacula/src/filed/verify.c @@ -7,7 +7,7 @@ * */ /* - Copyright (C) 2000-2004 Kern Sibbald and John Walker + Copyright (C) 2000-2004 Kern Sibbald This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -30,6 +30,7 @@ #include "filed.h" static int verify_file(FF_PKT *ff_pkt, void *my_pkt); +static int read_chksum(BFILE *bfd, CHKSUM *chksum, JCR *jcr, char *fname); /* * Find all the requested files and send attributes @@ -66,14 +67,9 @@ static int verify_file(FF_PKT *ff_pkt, void *pkt) { char attribs[MAXSTRING]; char attribsEx[MAXSTRING]; - int64_t n; int stat; BFILE bfd; - BFILE rsrc_bfd; - bool hfsplus = false; - struct MD5Context md5c; - struct SHA1Context sha1c; - unsigned char signature[25]; /* large enough for either */ + struct CHKSUM chksum; BSOCK *dir; JCR *jcr = (JCR *)pkt; @@ -158,187 +154,137 @@ static int verify_file(FF_PKT *ff_pkt, void *pkt) return 1; } - binit(&bfd); + /* Encode attributes and possibly extend them */ + encode_stat(attribs, ff_pkt, 0); + encode_attribsEx(jcr, attribsEx, ff_pkt); + + P(jcr->mutex); + jcr->JobFiles++; /* increment number of files sent */ + pm_strcpy(jcr->last_fname, ff_pkt->fname); + V(jcr->mutex); + /* + * Send file attributes to Director + * File_index + * Stream + * Verify Options + * Filename (full path) + * Encoded attributes + * Link name (if type==FT_LNK) + * For a directory, link is the same as fname, but with trailing + * slash. For a linked file, link is the link. + */ + /* Send file attributes to Director (note different format than for Storage) */ + Dmsg2(400, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, ff_pkt->fname); + if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) { + stat = bnet_fsend(dir, "%d %d %s %s%c%s%c%s%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, + 0, attribs, 0, ff_pkt->link, 0); + } else if (ff_pkt->type == FT_DIREND) { + /* Here link is the canonical filename (i.e. with trailing slash) */ + stat = bnet_fsend(dir,"%d %d %s %s%c%s%c%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link, + 0, attribs, 0, 0); + } else { + stat = bnet_fsend(dir,"%d %d %s %s%c%s%c%c", jcr->JobFiles, + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, + 0, attribs, 0, 0); + } + Dmsg2(20, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg); + if (!stat) { + Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), bnet_strerror(dir)); + return 0; + } + + /* + * The remainder of the function is all about getting the checksum. + * First we initialise, then we read files, other streams and Finder Info. + */ if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && - ff_pkt->statp.st_size > 0) || - ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) { - if ((bopen(&bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0)) < 0) { - ff_pkt->ff_errno = errno; - berrno be; - be.set_errno(bfd.berrno); - Jmsg(jcr, M_NOTSAVED, 1, _(" Cannot open %s: ERR=%s.\n"), - ff_pkt->fname, be.strerror()); - jcr->Errors++; - return 1; + ff_pkt->flags & (FO_MD5|FO_SHA1))) { + chksum_init(&chksum, ff_pkt->flags); + binit(&bfd); + + if (ff_pkt->statp.st_size > 0 || ff_pkt->type == FT_RAW + || ff_pkt->type == FT_FIFO) { + if ((bopen(&bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0)) < 0) { + ff_pkt->ff_errno = errno; + berrno be; + be.set_errno(bfd.berrno); + Jmsg(jcr, M_NOTSAVED, 1, _(" Cannot open %s: ERR=%s.\n"), + ff_pkt->fname, be.strerror()); + jcr->Errors++; + return 1; + } + read_chksum(&bfd, &chksum, jcr, ff_pkt->fname); + bclose(&bfd); } - } - binit(&rsrc_bfd); /* we check if this is open below */ #ifdef HAVE_DARWIN_OS - /* Open resource fork if necessary */ - if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) && - ff_pkt->flags & FO_HFSPLUS)) { - /* Remember Finder Info, whether we have data or fork, or not */ - hfsplus = true; - if (ff_pkt->hfsinfo.rsrclength > 0) { - if (bopen_rsrc(&rsrc_bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { - ff_pkt->ff_errno = errno; - berrno be; + /* Open resource fork if necessary */ + if (ff_pkt->flags & FO_HFSPLUS && ff_pkt->hfsinfo.rsrclength > 0) { + if (bopen_rsrc(&bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) { + ff_pkt->ff_errno = errno; + berrno be; Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for %s: ERR=%s\n"), ff_pkt->fname, be.strerror()); - jcr->Errors++; - if (is_bopen(&ff_pkt->bfd)) { - bclose(&ff_pkt->bfd); - } - return 1; - } + jcr->Errors++; + if (is_bopen(&ff_pkt->bfd)) { + bclose(&ff_pkt->bfd); + } + return 1; + } + read_chksum(&bfd, &chksum, jcr, ff_pkt->fname); + bclose(&bfd); + } + if (ff_pkt->flags & FO_HFSPLUS) { + chksum_update(&chksum, ((unsigned char *)ff_pkt->hfsinfo.fndrinfo), 32); } - } #endif - encode_stat(attribs, ff_pkt, 0); - - /* Now possibly extend the attributes */ - encode_attribsEx(jcr, attribsEx, ff_pkt); - - P(jcr->mutex); - jcr->JobFiles++; /* increment number of files sent */ - pm_strcpy(jcr->last_fname, ff_pkt->fname); - V(jcr->mutex); + /* compute MD5 or SHA1 hash */ + if (chksum.updated) { + char chksumbuf[40]; /* 24 should do */ + int stream = 0; - /* - * Send file attributes to Director - * File_index - * Stream - * Verify Options - * Filename (full path) - * Encoded attributes - * Link name (if type==FT_LNK) - * For a directory, link is the same as fname, but with trailing - * slash. For a linked file, link is the link. - */ - /* Send file attributes to Director (note different format than for Storage) */ - Dmsg2(400, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, ff_pkt->fname); - if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) { - stat = bnet_fsend(dir, "%d %d %s %s%c%s%c%s%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, - 0, attribs, 0, ff_pkt->link, 0); - } else if (ff_pkt->type == FT_DIREND) { - /* Here link is the canonical filename (i.e. with trailing slash) */ - stat = bnet_fsend(dir,"%d %d %s %s%c%s%c%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link, - 0, attribs, 0, 0); - } else { - stat = bnet_fsend(dir,"%d %d %s %s%c%s%c%c", jcr->JobFiles, - STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname, - 0, attribs, 0, 0); - } - Dmsg2(20, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg); - if (!stat) { - Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), bnet_strerror(dir)); - if (is_bopen(&bfd)) { - bclose(&bfd); - } - if (is_bopen(&rsrc_bfd)) { - bclose(&rsrc_bfd); - } - return 0; + chksum_final(&chksum); + if (chksum.type == CHKSUM_MD5) { + stream = STREAM_MD5_SIGNATURE; + } else if (chksum.type == CHKSUM_SHA1) { + stream = STREAM_SHA1_SIGNATURE; + } + bin_to_base64(chksumbuf, (char *)chksum.signature, chksum.length); + Dmsg3(400, "send inx=%d %s=%s\n", jcr->JobFiles, chksum.name, chksumbuf); + bnet_fsend(dir, "%d %d %s *%s-%d*", jcr->JobFiles, stream, chksumbuf, + chksum.name, jcr->JobFiles); + Dmsg3(20, "bfiled>bdird: %s len=%d: msg=%s\n", chksum.name, + dir->msglen, dir->msg); } + } - /* compute MD5 or SHA1 hash */ - if ((is_bopen(&bfd) || hfsplus) && ff_pkt->flags & FO_MD5) { - char MD5buf[40]; /* 24 should do */ - MD5Init(&md5c); - if (is_bopen(&bfd)) { - while ((n=bread(&bfd, jcr->big_buf, jcr->buf_size)) > 0) { - MD5Update(&md5c, ((unsigned char *)jcr->big_buf), (int)n); - jcr->JobBytes += n; - jcr->ReadBytes += n; - } - if (n < 0) { - berrno be; - be.set_errno(bfd.berrno); - Jmsg(jcr, M_ERROR, 1, _("Error reading file %s: ERR=%s\n"), - ff_pkt->fname, be.strerror()); - jcr->Errors++; - } - } -#ifdef HAVE_DARWIN_OS - if (is_bopen(&rsrc_bfd)) { - while ((n=bread(&rsrc_bfd, jcr->big_buf, jcr->buf_size)) > 0) { - MD5Update(&md5c, ((unsigned char *)jcr->big_buf), (int)n); - jcr->JobBytes += n; - jcr->ReadBytes += n; - } - if (n < 0) { - berrno be; - be.set_errno(rsrc_bfd.berrno); - Jmsg(jcr, M_ERROR, 1, _("Error reading file %s: ERR=%s\n"), - ff_pkt->fname, be.strerror()); - jcr->Errors++; - } - } - if (ff_pkt->flags & FO_HFSPLUS) { - MD5Update(&md5c, ((unsigned char *)ff_pkt->hfsinfo.fndrinfo), 32); - } -#endif - MD5Final(signature, &md5c); + return 1; +} - bin_to_base64(MD5buf, (char *)signature, 16); /* encode 16 bytes */ - Dmsg2(400, "send inx=%d MD5=%s\n", jcr->JobFiles, MD5buf); - bnet_fsend(dir, "%d %d %s *MD5-%d*", jcr->JobFiles, STREAM_MD5_SIGNATURE, MD5buf, - jcr->JobFiles); - Dmsg2(20, "bfiled>bdird: MD5 len=%d: msg=%s\n", dir->msglen, dir->msg); - } else if ((is_bopen(&bfd) || hfsplus) && ff_pkt->flags & FO_SHA1) { - char SHA1buf[40]; /* 24 should do */ - SHA1Init(&sha1c); - if (is_bopen(&bfd)) { - while ((n=bread(&bfd, jcr->big_buf, jcr->buf_size)) > 0) { - SHA1Update(&sha1c, ((unsigned char *)jcr->big_buf), (int)n); - jcr->JobBytes += n; - jcr->ReadBytes += n; - } - if (n < 0) { - berrno be; - be.set_errno(bfd.berrno); - Jmsg(jcr, M_ERROR, 1, _("Error reading file %s: ERR=%s\n"), - ff_pkt->fname, be.strerror()); - jcr->Errors++; - } - } -#ifdef HAVE_DARWIN_OS - if (is_bopen(&rsrc_bfd)) { - while ((n=bread(&rsrc_bfd, jcr->big_buf, jcr->buf_size)) > 0) { - SHA1Update(&sha1c, ((unsigned char *)jcr->big_buf), (int)n); - jcr->JobBytes += n; - jcr->ReadBytes += n; - } - if (n < 0) { - berrno be; - be.set_errno(rsrc_bfd.berrno); - Jmsg(jcr, M_ERROR, 1, _("Error reading file %s: ERR=%s\n"), - ff_pkt->fname, be.strerror()); - jcr->Errors++; - } - } - if (ff_pkt->flags & FO_HFSPLUS) { - SHA1Update(&sha1c, ((unsigned char *)ff_pkt->hfsinfo.fndrinfo), 32); - } -#endif - SHA1Final(&sha1c, signature); +/* + * Read checksum of bfd, updating chksum + * In case of errors we need the job control record and file name. + */ +int read_chksum(BFILE *bfd, CHKSUM *chksum, JCR *jcr, char *fname) +{ + int64_t n; - bin_to_base64(SHA1buf, (char *)signature, 20); /* encode 20 bytes */ - Dmsg2(400, "send inx=%d SHA1=%s\n", jcr->JobFiles, SHA1buf); - bnet_fsend(dir, "%d %d %s *SHA1-%d*", jcr->JobFiles, STREAM_SHA1_SIGNATURE, - SHA1buf, jcr->JobFiles); - Dmsg2(20, "bfiled>bdird: SHA1 len=%d: msg=%s\n", dir->msglen, dir->msg); - } - if (is_bopen(&bfd)) { - bclose(&bfd); - } - if (is_bopen(&rsrc_bfd)) { - bclose(&rsrc_bfd); - } - return 1; + while ((n=bread(bfd, jcr->big_buf, jcr->buf_size)) > 0) { + chksum_update(chksum, ((unsigned char *)jcr->big_buf), (int)n); + jcr->JobBytes += n; + jcr->ReadBytes += n; + } + if (n < 0) { + berrno be; + be.set_errno(bfd->berrno); + Jmsg(jcr, M_ERROR, 1, _("Error reading file %s: ERR=%s\n"), + fname, be.strerror()); + jcr->Errors++; + return -1; } + return 0; +} -- 2.39.5