From bcfc3499e3ed90d43bef43e4a60edabecb4b236f Mon Sep 17 00:00:00 2001 From: Laurent Papier Date: Wed, 18 May 2011 23:25:30 +0200 Subject: [PATCH] Add LZO compression support in bacula-fd. Add new generic compressed streams This should make easier to add additional compression scheme Rename a few define, add an element containing compression algorithm in some data structures. --- bacula/autoconf/acconfig.h | 3 + bacula/autoconf/configure.in | 14 ++ bacula/src/ch.h | 58 ++++++ bacula/src/dird/fd_cmds.c | 2 +- bacula/src/dird/inc_conf.c | 1 + bacula/src/filed/Makefile.in | 5 +- bacula/src/filed/backup.c | 97 +++++++++- bacula/src/filed/filed.h | 4 + bacula/src/filed/job.c | 16 +- bacula/src/filed/restore.c | 189 +++++++++++++++----- bacula/src/filed/restore.h | 1 + bacula/src/fileopts.h | 2 +- bacula/src/findlib/attribs.c | 78 +++++--- bacula/src/findlib/bfile.c | 42 +++++ bacula/src/findlib/find.c | 6 +- bacula/src/findlib/find.h | 7 +- bacula/src/findlib/match.c | 25 ++- bacula/src/jcr.h | 1 + bacula/src/stored/Makefile.in | 3 +- bacula/src/stored/bextract.c | 97 ++++++++++ bacula/src/stored/bscan.c | 5 + bacula/src/stored/record.c | 20 +++ bacula/src/stored/stored.h | 4 + bacula/src/streams.h | 9 + bacula/src/tools/testfind.c | 18 +- regress/all-disk-tests | 3 + regress/scripts/new-test-bacula-dir.conf.in | 51 ++++++ regress/tests/lzo-encrypt-test | 47 +++++ regress/tests/lzo-test | 52 ++++++ regress/tests/sparse-lzo-test | 42 +++++ 30 files changed, 805 insertions(+), 97 deletions(-) create mode 100644 bacula/src/ch.h create mode 100755 regress/tests/lzo-encrypt-test create mode 100755 regress/tests/lzo-test create mode 100755 regress/tests/sparse-lzo-test diff --git a/bacula/autoconf/acconfig.h b/bacula/autoconf/acconfig.h index de62f6c49e..d655307d64 100644 --- a/bacula/autoconf/acconfig.h +++ b/bacula/autoconf/acconfig.h @@ -114,6 +114,9 @@ /* Define if you have zlib */ #undef HAVE_LIBZ +/* Define if you have lzo lib */ +#undef HAVE_LZO + /* Define if you have libacl */ #undef HAVE_ACL diff --git a/bacula/autoconf/configure.in b/bacula/autoconf/configure.in index f7387bbd91..525a0e2a55 100644 --- a/bacula/autoconf/configure.in +++ b/bacula/autoconf/configure.in @@ -2495,6 +2495,19 @@ if test x$ZLIBS = x-lz; then fi AC_SUBST(ZLIBS) +dnl +dnl Check for lzo +dnl +AC_CHECK_HEADERS(lzo/lzoconf.h) +AC_CHECK_HEADERS(lzo/lzo1x.h) +AC_CHECK_LIB(lzo2, lzo1x_1_compress, [LZOLIBS="-llzo2"]) +have_lzo=no +if test x$LZOLIBS = x-llzo2; then + AC_DEFINE(HAVE_LZO) + have_lzo=yes +fi +AC_SUBST(LZOLIBS) + dnl dnl Check for ACL support and libraries dnl @@ -3468,6 +3481,7 @@ Configuration on `date`: TLS support: ${support_tls} Encryption support: ${support_crypto} ZLIB support: ${have_zlib} + LZO support: ${have_lzo} enable-smartalloc: ${support_smartalloc} enable-lockmgr: ${support_lockmgr} bat support: ${support_bat} diff --git a/bacula/src/ch.h b/bacula/src/ch.h new file mode 100644 index 0000000000..168cfd0be0 --- /dev/null +++ b/bacula/src/ch.h @@ -0,0 +1,58 @@ +/* + Bacula® - The Network Backup Solution + + Copyright (C) 2000-2008 Free Software Foundation Europe e.V. + + The main author of Bacula is Kern Sibbald, with contributions from + many others, a complete list can be found in the file AUTHORS. + This program is Free Software; you can redistribute it and/or + modify it under the terms of version three of the GNU Affero General Public + License as published by the Free Software Foundation and included + in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + + Bacula® is a registered trademark of Kern Sibbald. + The licensor of Bacula is the Free Software Foundation Europe + (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, + Switzerland, email:ftf@fsfeurope.org. +*/ +/** + * Compressed stream header struct + * + * Laurent Papier + * + */ + +#ifndef __CH_H +#define __CH_H 1 + +/* + * Compression algorithm signature. 4 letters as a 32bits integer + */ +#define COMPRESS_NONE 0x4e4f4e45 /* used for incompressible block */ +#define COMPRESS_GZIP 0x475a4950 +#define COMPRESS_LZO1X 0x4c5a4f58 + +/* + * Compression header version + */ +#define COMP_HEAD_VERSION 0x1 + +/* Compressed data stream header */ +typedef struct { + uint32_t magic; /* compression algo used in this compressed data stream */ + uint16_t level; /* compression level used */ + uint16_t version; /* for futur evolution */ + uint32_t size; /* compressed size of the original data */ +} comp_stream_header; + +#endif /* __CH_H */ diff --git a/bacula/src/dird/fd_cmds.c b/bacula/src/dird/fd_cmds.c index 2c6914d9fb..85e6055343 100644 --- a/bacula/src/dird/fd_cmds.c +++ b/bacula/src/dird/fd_cmds.c @@ -379,7 +379,7 @@ static bool send_fileset(JCR *jcr) bool done=false; /* print warning only if compression enabled in FS */ int j = 0; for (k=0; fo->opts[k]!='\0'; k++) { - /* Z compress option is followed by the single-digit compress level */ + /* Z compress option is followed by the single-digit compress level or 'o' */ if (fo->opts[k]=='Z') { done=true; k++; /* skip option and level */ diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c index ccd8258221..de91e7a862 100644 --- a/bacula/src/dird/inc_conf.c +++ b/bacula/src/dird/inc_conf.c @@ -228,6 +228,7 @@ static struct s_fs_opt FS_options[] = { {"gzip7", INC_KW_COMPRESSION, "Z7"}, {"gzip8", INC_KW_COMPRESSION, "Z8"}, {"gzip9", INC_KW_COMPRESSION, "Z9"}, + {"lzo", INC_KW_COMPRESSION, "Zo"}, {"blowfish", INC_KW_ENCRYPTION, "B"}, /* ***FIXME*** not implemented */ {"3des", INC_KW_ENCRYPTION, "3"}, /* ***FIXME*** not implemented */ {"yes", INC_KW_ONEFS, "0"}, diff --git a/bacula/src/filed/Makefile.in b/bacula/src/filed/Makefile.in index 8f0f10d024..c67a2e0320 100644 --- a/bacula/src/filed/Makefile.in +++ b/bacula/src/filed/Makefile.in @@ -41,6 +41,7 @@ EXTRAOBJS = @OBJLIST@ CAP_LIBS = @CAP_LIBS@ FDLIBS = @FDLIBS@ # extra libs for File daemon ZLIBS = @ZLIBS@ +LZOLIBS = @LZOLIBS@ # extra items for linking on Win32 WIN32OBJS = win32/winmain.o win32/winlib.a win32/winres.res @@ -93,12 +94,12 @@ win32: $(WIN32OBJS) bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbacpy$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@ @echo "Linking $@ ..." $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(SVROBJS) \ - $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \ + $(WIN32LIBS) $(FDLIBS) $(ZLIBS) $(LZOLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \ $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) static-bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind.a ../lib/libbacpy$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@ $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -static -L../lib -L../findlib -o $@ $(SVROBJS) \ - $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \ + $(WIN32LIBS) $(FDLIBS) $(ZLIBS) $(LZOLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \ $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) strip $@ diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 6ac8721c6d..09de90ea8c 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -35,6 +35,7 @@ #include "bacula.h" #include "filed.h" +#include "ch.h" #ifdef HAVE_DARWIN_OS const bool have_darwin_os = true; @@ -110,12 +111,22 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) * Note, we adjust the read size to be smaller so that the * same output buffer can be used without growing it. * + * For LZO1X compression the recommended value is : + * output_block_size = input_block_size + (input_block_size / 16) + 64 + 3 + sizeof(comp_stream_header) + * * The zlib compression workset is initialized here to minimize * the "per file" load. The jcr member is only set, if the init * was successful. + * + * For the same reason, lzo compression is initialized here. */ +#ifdef HAVE_LZO + jcr->compress_buf_size = MAX(jcr->buf_size + (jcr->buf_size / 16) + 67 + sizeof(comp_stream_header), jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30); + jcr->compress_buf = get_memory(jcr->compress_buf_size); +#else jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30; jcr->compress_buf = get_memory(jcr->compress_buf_size); +#endif #ifdef HAVE_LIBZ z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream)); @@ -133,6 +144,17 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) } #endif +#ifdef HAVE_LZO + lzo_voidp pLzoMem = (lzo_voidp) malloc(LZO1X_1_MEM_COMPRESS); + if (pLzoMem) { + if (lzo_init() == LZO_E_OK) { + jcr->LZO_compress_workset = pLzoMem; + } else { + free (pLzoMem); + } + } +#endif + if (!crypto_session_start(jcr)) { return false; } @@ -207,6 +229,11 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) free (jcr->pZLIB_compress_workset); jcr->pZLIB_compress_workset = NULL; } + if (jcr->LZO_compress_workset) { + free (jcr->LZO_compress_workset); + jcr->LZO_compress_workset = NULL; + } + crypto_session_end(jcr); @@ -604,7 +631,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) goto good_rtn; } flags = ff_pkt->flags; - ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE|FO_OFFSETS); + ff_pkt->flags &= ~(FO_COMPRESS|FO_SPARSE|FO_OFFSETS); if (flags & FO_ENCRYPT) { rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA; } else { @@ -821,13 +848,14 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type); -#ifdef HAVE_LIBZ +#if defined(HAVE_LIBZ) || defined(HAVE_LZO) uLong compress_len = 0; uLong max_compress_len = 0; const Bytef *cbuf = NULL; + #ifdef HAVE_LIBZ int zstat; - if (ff_pkt->flags & FO_GZIP) { + if ((ff_pkt->flags & FO_COMPRESS) && ff_pkt->Compress_algo == COMPRESS_GZIP) { if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) { cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE; max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE; @@ -847,13 +875,38 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) { /** set gzip compression level - must be done per file */ if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset, - ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) { + ff_pkt->Compress_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); jcr->setJobStatus(JS_ErrorTerminated); goto err; } } } + #endif + #ifdef HAVE_LZO + Bytef *cbuf2; + int lzores; + comp_stream_header ch; + + memset(&ch, 0, sizeof(comp_stream_header)); + cbuf2 = NULL; + + if ((ff_pkt->flags & FO_COMPRESS) && ff_pkt->Compress_algo == COMPRESS_LZO1X) { + if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) { + cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE; + cbuf2 = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header); + max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE; + } else { + cbuf = (Bytef *)jcr->compress_buf; + cbuf2 = (Bytef *)jcr->compress_buf + sizeof(comp_stream_header); + max_compress_len = jcr->compress_buf_size; /* set max length */ + } + ch.magic = COMPRESS_LZO1X; + ch.version = COMP_HEAD_VERSION; + wbuf = jcr->compress_buf; /* compressed output here */ + cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ + } + #endif #else const uint32_t max_compress_len = 0; #endif @@ -968,7 +1021,7 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, #ifdef HAVE_LIBZ /** Do compression if turned on */ - if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) { + if (ff_pkt->flags & FO_COMPRESS && ff_pkt->Compress_algo == COMPRESS_GZIP && jcr->pZLIB_compress_workset) { Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen); ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf; @@ -989,13 +1042,45 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, goto err; } - Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len, + Dmsg2(400, "GZIP compressed len=%d uncompressed len=%d\n", compress_len, sd->msglen); sd->msglen = compress_len; /* set compressed length */ cipher_input_len = compress_len; } #endif +#ifdef HAVE_LZO + /** Do compression if turned on */ + if (ff_pkt->flags & FO_COMPRESS && ff_pkt->Compress_algo == COMPRESS_LZO1X && jcr->LZO_compress_workset) { + ser_declare; + ser_begin(cbuf, sizeof(comp_stream_header)); + + Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen); + + lzores = lzo1x_1_compress((const unsigned char*)rbuf, sd->msglen, cbuf2, &compress_len, jcr->LZO_compress_workset); + if (lzores == LZO_E_OK && compress_len <= max_compress_len) + { + /* complete header */ + ser_uint32(COMPRESS_LZO1X); + ser_uint32(compress_len); + ser_uint16(ch.level); + ser_uint16(ch.version); + } else { + /** this should NEVER happen */ + Jmsg(jcr, M_FATAL, 0, _("Compression LZO error: %d\n"), lzores); + jcr->setJobStatus(JS_ErrorTerminated); + goto err; + } + + Dmsg2(400, "LZO compressed len=%d uncompressed len=%d\n", compress_len, + sd->msglen); + + compress_len += sizeof(comp_stream_header); /* add size of header */ + sd->msglen = compress_len; /* set compressed length */ + cipher_input_len = compress_len; + } +#endif + /** * Note, here we prepend the current record length to the beginning * of the encrypted data. This is because both sparse and compression diff --git a/bacula/src/filed/filed.h b/bacula/src/filed/filed.h index f6c785f596..71d26d69ec 100644 --- a/bacula/src/filed/filed.h +++ b/bacula/src/filed/filed.h @@ -76,6 +76,10 @@ typedef enum { #else #define uLongf uint32_t #endif +#ifdef HAVE_LZO +#include +#include +#endif extern CLIENT *me; /* "Global" Client resource */ diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index 92018aac66..417a3931de 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -34,6 +34,7 @@ #include "bacula.h" #include "filed.h" +#include "ch.h" #if defined(WIN32_VSS) #include "vss.h" @@ -1385,9 +1386,18 @@ static int set_options(findFOPTS *fo, const char *opts) case 'W': fo->flags |= FO_ENHANCEDWILD; break; - case 'Z': /* gzip compression */ - fo->flags |= FO_GZIP; - fo->GZIP_level = *++p - '0'; + case 'Z': /* compression */ + p++; /* skip Z */ + if (*p >= '0' && *p <= '9') { + fo->flags |= FO_COMPRESS; + fo->Compress_algo = COMPRESS_GZIP; + fo->Compress_level = *p - '0'; + } + else if (*p == 'o') { + fo->flags |= FO_COMPRESS; + fo->Compress_algo = COMPRESS_LZO1X; + fo->Compress_level = 1; /* not used with LZO */ + } break; case 'K': fo->flags |= FO_NOATIME; diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c index ed4d7243ff..7c4b028578 100644 --- a/bacula/src/filed/restore.c +++ b/bacula/src/filed/restore.c @@ -34,6 +34,7 @@ #include "bacula.h" #include "filed.h" +#include "ch.h" #include "restore.h" #ifdef HAVE_DARWIN_OS @@ -81,6 +82,12 @@ const bool have_libz = true; #else const bool have_libz = false; #endif +#ifdef HAVE_LZO +const bool have_lzo = true; +#else +const bool have_lzo = false; +#endif + static void deallocate_cipher(r_ctx &rctx); static void deallocate_fork_cipher(r_ctx &rctx); @@ -90,8 +97,8 @@ static void close_previous_stream(r_ctx &rctx); static bool verify_signature(JCR *jcr, r_ctx &rctx); int32_t extract_data(JCR *jcr, BFILE *bfd, POOLMEM *buf, int32_t buflen, - uint64_t *addr, int flags, RESTORE_CIPHER_CTX *cipher_ctx); -bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, + uint64_t *addr, int flags, int32_t stream, RESTORE_CIPHER_CTX *cipher_ctx); +bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, int32_t stream, RESTORE_CIPHER_CTX *cipher_ctx); /* @@ -197,12 +204,20 @@ void do_restore(JCR *jcr) * St Bernard code goes here if implemented -- see end of file */ - if (have_libz) { + /* use the same buffer size to decompress both gzip and lzo */ + if (have_libz || have_lzo) { uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100; jcr->compress_buf = get_memory(compress_buf_size); jcr->compress_buf_size = compress_buf_size; } +#ifdef HAVE_LZO + if (lzo_init() != LZO_E_OK) { + Jmsg(jcr, M_FATAL, 0, _("LZO init failed\n")); + goto bail_out; + } +#endif + if (have_crypto) { rctx.cipher_ctx.buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE); if (have_darwin_os) { @@ -499,10 +514,15 @@ void do_restore(JCR *jcr) case STREAM_GZIP_DATA: case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: case STREAM_ENCRYPTED_FILE_DATA: case STREAM_ENCRYPTED_WIN32_DATA: case STREAM_ENCRYPTED_FILE_GZIP_DATA: case STREAM_ENCRYPTED_WIN32_GZIP_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: /* * Force an expected, consistent stream type here */ @@ -512,8 +532,9 @@ void do_restore(JCR *jcr) || rctx.prev_stream == STREAM_ENCRYPTED_SESSION_DATA)) { rctx.flags = 0; - if (rctx.stream == STREAM_SPARSE_DATA || - rctx.stream == STREAM_SPARSE_GZIP_DATA) { + if (rctx.stream == STREAM_SPARSE_DATA + || rctx.stream == STREAM_SPARSE_COMPRESSED_DATA + || rctx.stream == STREAM_SPARSE_GZIP_DATA) { rctx.flags |= FO_SPARSE; } @@ -521,13 +542,21 @@ void do_restore(JCR *jcr) || rctx.stream == STREAM_SPARSE_GZIP_DATA || rctx.stream == STREAM_WIN32_GZIP_DATA || rctx.stream == STREAM_ENCRYPTED_FILE_GZIP_DATA + || rctx.stream == STREAM_COMPRESSED_DATA + || rctx.stream == STREAM_SPARSE_COMPRESSED_DATA + || rctx.stream == STREAM_WIN32_COMPRESSED_DATA + || rctx.stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA + || rctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA || rctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) { - rctx.flags |= FO_GZIP; + rctx.flags |= FO_COMPRESS; + rctx.comp_stream = rctx.stream; } if (rctx.stream == STREAM_ENCRYPTED_FILE_DATA || rctx.stream == STREAM_ENCRYPTED_FILE_GZIP_DATA || rctx.stream == STREAM_ENCRYPTED_WIN32_DATA + || rctx.stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA + || rctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA || rctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) { /* * Set up a decryption context @@ -561,7 +590,7 @@ void do_restore(JCR *jcr) } if (extract_data(jcr, &rctx.bfd, sd->msg, sd->msglen, &rctx.fileAddr, - rctx.flags, &rctx.cipher_ctx) < 0) { + rctx.flags, rctx.stream, &rctx.cipher_ctx) < 0) { rctx.extract = false; bclose(&rctx.bfd); continue; @@ -616,7 +645,7 @@ void do_restore(JCR *jcr) } if (extract_data(jcr, &rctx.forkbfd, sd->msg, sd->msglen, &rctx.fork_addr, rctx.fork_flags, - &rctx.fork_cipher_ctx) < 0) { + rctx.stream, &rctx.fork_cipher_ctx) < 0) { rctx.extract = false; bclose(&rctx.forkbfd); continue; @@ -1078,44 +1107,112 @@ bool sparse_data(JCR *jcr, BFILE *bfd, uint64_t *addr, char **data, uint32_t *le return true; } -bool decompress_data(JCR *jcr, char **data, uint32_t *length) +bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length) { -#ifdef HAVE_LIBZ - uLong compress_len; - int stat; char ec1[50]; /* Buffer printing huge values */ - /* - * NOTE! We only use uLong and Byte because they are - * needed by the zlib routines, they should not otherwise - * be used in Bacula. - */ - compress_len = jcr->compress_buf_size; - Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length); - while ((stat=uncompress((Byte *)jcr->compress_buf, &compress_len, - (const Byte *)*data, (uLong)*length)) == Z_BUF_ERROR) + Dmsg1(200, "Stream found in decompress_data(): %d\n", stream); + if(stream == STREAM_COMPRESSED_DATA || stream == STREAM_SPARSE_COMPRESSED_DATA || stream == STREAM_WIN32_COMPRESSED_DATA + || stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA || stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA) { - /* - * The buffer size is too small, try with a bigger one + uint32_t comp_magic, comp_len; + uint16_t comp_level, comp_version; +#ifdef HAVE_LZO + lzo_uint compress_len; + const unsigned char *cbuf; + int r, real_compress_len; +#endif + + /* read compress header */ + unser_declare; + unser_begin(*data, sizeof(comp_stream_header)); + unser_uint32(comp_magic); + unser_uint32(comp_len); + unser_uint16(comp_level); + unser_uint16(comp_version); + Dmsg4(200, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len, + comp_level, comp_version); + + /* version check */ + if (comp_version != COMP_HEAD_VERSION) { + Qmsg(jcr, M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version); + return false; + } + /* size check */ + if (comp_len + sizeof(comp_stream_header) != *length) { + Qmsg(jcr, M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"), + comp_len, *length); + return false; + } + switch(comp_magic) { +#ifdef HAVE_LZO + case COMPRESS_LZO1X: + compress_len = jcr->compress_buf_size; + cbuf = (const unsigned char*)*data + sizeof(comp_stream_header); + real_compress_len = *length - sizeof(comp_stream_header); + Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length); + while ((r=lzo1x_decompress_safe(cbuf, real_compress_len, + (unsigned char *)jcr->compress_buf, &compress_len, NULL)) == LZO_E_OUTPUT_OVERRUN) + { + /* + * The buffer size is too small, try with a bigger one + */ + compress_len = jcr->compress_buf_size = jcr->compress_buf_size + (jcr->compress_buf_size >> 1); + Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length); + jcr->compress_buf = check_pool_memory_size(jcr->compress_buf, + compress_len); + } + if (r != LZO_E_OK) { + Qmsg(jcr, M_ERROR, 0, _("LZO uncompression error on file %s. ERR=%d\n"), + jcr->last_fname, r); + return false; + } + *data = jcr->compress_buf; + *length = compress_len; + Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1)); + return true; +#endif + default: + Qmsg(jcr, M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic); + return false; + } + } else { +#ifdef HAVE_LIBZ + uLong compress_len; + int stat; + + /* + * NOTE! We only use uLong and Byte because they are + * needed by the zlib routines, they should not otherwise + * be used in Bacula. */ - compress_len = jcr->compress_buf_size = jcr->compress_buf_size + (jcr->compress_buf_size >> 1); + compress_len = jcr->compress_buf_size; Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length); - jcr->compress_buf = check_pool_memory_size(jcr->compress_buf, - compress_len); - } - if (stat != Z_OK) { - Qmsg(jcr, M_ERROR, 0, _("Uncompression error on file %s. ERR=%s\n"), - jcr->last_fname, zlib_strerror(stat)); - return false; - } - *data = jcr->compress_buf; - *length = compress_len; - Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1)); - return true; + while ((stat=uncompress((Byte *)jcr->compress_buf, &compress_len, + (const Byte *)*data, (uLong)*length)) == Z_BUF_ERROR) + { + /* + * The buffer size is too small, try with a bigger one + */ + compress_len = jcr->compress_buf_size = jcr->compress_buf_size + (jcr->compress_buf_size >> 1); + Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length); + jcr->compress_buf = check_pool_memory_size(jcr->compress_buf, + compress_len); + } + if (stat != Z_OK) { + Qmsg(jcr, M_ERROR, 0, _("Uncompression error on file %s. ERR=%s\n"), + jcr->last_fname, zlib_strerror(stat)); + return false; + } + *data = jcr->compress_buf; + *length = compress_len; + Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1)); + return true; #else - Qmsg(jcr, M_ERROR, 0, _("GZIP data stream found, but GZIP not configured!\n")); - return false; + Qmsg(jcr, M_ERROR, 0, _("GZIP data stream found, but GZIP not configured!\n")); + return false; #endif + } } static void unser_crypto_packet_len(RESTORE_CIPHER_CTX *ctx) @@ -1157,7 +1254,7 @@ bool store_data(JCR *jcr, BFILE *bfd, char *data, const int32_t length, bool win * Return value is the number of bytes written, or -1 on errors. */ int32_t extract_data(JCR *jcr, BFILE *bfd, POOLMEM *buf, int32_t buflen, - uint64_t *addr, int flags, RESTORE_CIPHER_CTX *cipher_ctx) + uint64_t *addr, int flags, int32_t stream, RESTORE_CIPHER_CTX *cipher_ctx) { char *wbuf; /* write buffer */ uint32_t wsize; /* write size */ @@ -1242,8 +1339,8 @@ int32_t extract_data(JCR *jcr, BFILE *bfd, POOLMEM *buf, int32_t buflen, } } - if (flags & FO_GZIP) { - if (!decompress_data(jcr, &wbuf, &wsize)) { + if (flags & FO_COMPRESS) { + if (!decompress_data(jcr, stream, &wbuf, &wsize)) { goto bail_out; } } @@ -1332,7 +1429,7 @@ static void close_previous_stream(r_ctx &rctx) * writing it to bfd. * Return value is true on success, false on failure. */ -bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, +bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, int32_t stream, RESTORE_CIPHER_CTX *cipher_ctx) { uint32_t decrypted_len = 0; @@ -1383,8 +1480,8 @@ again: } } - if (flags & FO_GZIP) { - if (!decompress_data(jcr, &wbuf, &wsize)) { + if (flags & FO_COMPRESS) { + if (!decompress_data(jcr, stream, &wbuf, &wsize)) { return false; } } @@ -1430,7 +1527,7 @@ static void deallocate_cipher(r_ctx &rctx) * Flush and deallocate previous stream's cipher context */ if (rctx.cipher_ctx.cipher) { - flush_cipher(rctx.jcr, &rctx.bfd, &rctx.fileAddr, rctx.flags, &rctx.cipher_ctx); + flush_cipher(rctx.jcr, &rctx.bfd, &rctx.fileAddr, rctx.flags, rctx.comp_stream, &rctx.cipher_ctx); crypto_cipher_free(rctx.cipher_ctx.cipher); rctx.cipher_ctx.cipher = NULL; } @@ -1443,7 +1540,7 @@ static void deallocate_fork_cipher(r_ctx &rctx) * Flush and deallocate previous stream's fork cipher context */ if (rctx.fork_cipher_ctx.cipher) { - flush_cipher(rctx.jcr, &rctx.forkbfd, &rctx.fork_addr, rctx.fork_flags, &rctx.fork_cipher_ctx); + flush_cipher(rctx.jcr, &rctx.forkbfd, &rctx.fork_addr, rctx.fork_flags, rctx.comp_stream, &rctx.fork_cipher_ctx); crypto_cipher_free(rctx.fork_cipher_ctx.cipher); rctx.fork_cipher_ctx.cipher = NULL; } diff --git a/bacula/src/filed/restore.h b/bacula/src/filed/restore.h index f68af2c6d5..ffb0897d1d 100644 --- a/bacula/src/filed/restore.h +++ b/bacula/src/filed/restore.h @@ -43,6 +43,7 @@ struct r_ctx { int32_t stream; /* stream less new bits */ int32_t prev_stream; /* previous stream */ int32_t full_stream; /* full stream including new bits */ + int32_t comp_stream; /* last compressed stream found. needed only to restore encrypted compressed backup */ BFILE bfd; /* File content */ uint64_t fileAddr; /* file write address */ uint32_t size; /* Size of file */ diff --git a/bacula/src/fileopts.h b/bacula/src/fileopts.h index 0de2ad8545..4767a5d227 100644 --- a/bacula/src/fileopts.h +++ b/bacula/src/fileopts.h @@ -43,7 +43,7 @@ */ #define FO_PORTABLE_DATA (1<<0) /* Data is portable */ #define FO_MD5 (1<<1) /* Do MD5 checksum */ -#define FO_GZIP (1<<2) /* Do Zlib compression */ +#define FO_COMPRESS (1<<2) /* Do compression */ #define FO_NO_RECURSION (1<<3) /* no recursion in directories */ #define FO_MULTIFS (1<<4) /* multiple file systems */ #define FO_SPARSE (1<<5) /* do sparse file checking */ diff --git a/bacula/src/findlib/attribs.c b/bacula/src/findlib/attribs.c index 419689fd8e..0d6e7c2f69 100644 --- a/bacula/src/findlib/attribs.c +++ b/bacula/src/findlib/attribs.c @@ -36,6 +36,7 @@ #include "bacula.h" #include "find.h" +#include "ch.h" static uid_t my_uid = 1; static gid_t my_gid = 1; @@ -103,32 +104,58 @@ int select_data_stream(FF_PKT *ff_pkt) /** Compression is not supported for Mac fork data */ if (stream == STREAM_MACOS_FORK_DATA) { - ff_pkt->flags &= ~FO_GZIP; + ff_pkt->flags &= ~FO_COMPRESS; } /** * Handle compression and encryption options */ -#ifdef HAVE_LIBZ - if (ff_pkt->flags & FO_GZIP) { - switch (stream) { - case STREAM_WIN32_DATA: - stream = STREAM_WIN32_GZIP_DATA; - break; - case STREAM_SPARSE_DATA: - stream = STREAM_SPARSE_GZIP_DATA; - break; - case STREAM_FILE_DATA: - stream = STREAM_GZIP_DATA; - break; - default: - /** - * All stream types that do not support gzip should clear out - * FO_GZIP above, and this code block should be unreachable. - */ - ASSERT(!(ff_pkt->flags & FO_GZIP)); - return STREAM_NONE; - } +#if defined(HAVE_LIBZ) || defined(HAVE_LZO) + if (ff_pkt->flags & FO_COMPRESS) { + #ifdef HAVE_LIBZ + if(ff_pkt->Compress_algo == COMPRESS_GZIP) { + switch (stream) { + case STREAM_WIN32_DATA: + stream = STREAM_WIN32_GZIP_DATA; + break; + case STREAM_SPARSE_DATA: + stream = STREAM_SPARSE_GZIP_DATA; + break; + case STREAM_FILE_DATA: + stream = STREAM_GZIP_DATA; + break; + default: + /** + * All stream types that do not support compression should clear out + * FO_COMPRESS above, and this code block should be unreachable. + */ + ASSERT(!(ff_pkt->flags & FO_COMPRESS)); + return STREAM_NONE; + } + } + #endif + #ifdef HAVE_LZO + if(ff_pkt->Compress_algo == COMPRESS_LZO1X) { + switch (stream) { + case STREAM_WIN32_DATA: + stream = STREAM_WIN32_COMPRESSED_DATA; + break; + case STREAM_SPARSE_DATA: + stream = STREAM_SPARSE_COMPRESSED_DATA; + break; + case STREAM_FILE_DATA: + stream = STREAM_COMPRESSED_DATA; + break; + default: + /** + * All stream types that do not support compression should clear out + * FO_COMPRESS above, and this code block should be unreachable. + */ + ASSERT(!(ff_pkt->flags & FO_COMPRESS)); + return STREAM_NONE; + } + } + #endif } #endif #ifdef HAVE_CRYPTO @@ -140,12 +167,18 @@ int select_data_stream(FF_PKT *ff_pkt) case STREAM_WIN32_GZIP_DATA: stream = STREAM_ENCRYPTED_WIN32_GZIP_DATA; break; + case STREAM_WIN32_COMPRESSED_DATA: + stream = STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA; + break; case STREAM_FILE_DATA: stream = STREAM_ENCRYPTED_FILE_DATA; break; case STREAM_GZIP_DATA: stream = STREAM_ENCRYPTED_FILE_GZIP_DATA; break; + case STREAM_COMPRESSED_DATA: + stream = STREAM_ENCRYPTED_FILE_COMPRESSED_DATA; + break; default: /* All stream types that do not support encryption should clear out * FO_ENCRYPT above, and this code block should be unreachable. */ @@ -423,7 +456,8 @@ bool set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd) return true; } if (attr->data_stream == STREAM_WIN32_DATA || - attr->data_stream == STREAM_WIN32_GZIP_DATA) { + attr->data_stream == STREAM_WIN32_GZIP_DATA || + attr->data_stream == STREAM_WIN32_COMPRESSED_DATA) { if (is_bopen(ofd)) { bclose(ofd); } diff --git a/bacula/src/findlib/bfile.c b/bacula/src/findlib/bfile.c index b67408d762..fbcac7d562 100644 --- a/bacula/src/findlib/bfile.c +++ b/bacula/src/findlib/bfile.c @@ -79,8 +79,10 @@ bool is_win32_stream(int stream) switch (stream) { case STREAM_WIN32_DATA: case STREAM_WIN32_GZIP_DATA: + case STREAM_WIN32_COMPRESSED_DATA: case STREAM_ENCRYPTED_WIN32_DATA: case STREAM_ENCRYPTED_WIN32_GZIP_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: return true; } return false; @@ -99,12 +101,16 @@ const char *stream_to_ascii(int stream) return _("MD5 digest"); case STREAM_GZIP_DATA: return _("GZIP data"); + case STREAM_COMPRESSED_DATA: + return _("Compressed data"); case STREAM_UNIX_ATTRIBUTES_EX: return _("Extended attributes"); case STREAM_SPARSE_DATA: return _("Sparse data"); case STREAM_SPARSE_GZIP_DATA: return _("GZIP sparse data"); + case STREAM_SPARSE_COMPRESSED_DATA: + return _("Compressed sparse data"); case STREAM_PROGRAM_NAMES: return _("Program names"); case STREAM_PROGRAM_DATA: @@ -115,6 +121,8 @@ const char *stream_to_ascii(int stream) return _("Win32 data"); case STREAM_WIN32_GZIP_DATA: return _("Win32 GZIP data"); + case STREAM_WIN32_COMPRESSED_DATA: + return _("Win32 compressed data"); case STREAM_MACOS_FORK_DATA: return _("MacOS Fork data"); case STREAM_HFSPLUS_ATTRIBUTES: @@ -137,8 +145,12 @@ const char *stream_to_ascii(int stream) return _("Encrypted session data"); case STREAM_ENCRYPTED_FILE_GZIP_DATA: return _("Encrypted GZIP data"); + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + return _("Encrypted compressed data"); case STREAM_ENCRYPTED_WIN32_GZIP_DATA: return _("Encrypted Win32 GZIP data"); + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: + return _("Encrypted Win32 Compressed data"); case STREAM_ENCRYPTED_MACOS_FORK_DATA: return _("Encrypted MacOS fork data"); case STREAM_ACL_AIX_TEXT: @@ -421,6 +433,13 @@ bool is_restore_stream_supported(int stream) case STREAM_GZIP_DATA: case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: +#endif +#ifndef HAVE_LZO + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: #endif case STREAM_MACOS_FORK_DATA: case STREAM_HFSPLUS_ATTRIBUTES: @@ -432,6 +451,13 @@ bool is_restore_stream_supported(int stream) case STREAM_GZIP_DATA: case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: +#endif +#ifdef HAVE_LZO + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: #endif case STREAM_WIN32_DATA: case STREAM_UNIX_ATTRIBUTES: @@ -452,6 +478,8 @@ bool is_restore_stream_supported(int stream) case STREAM_ENCRYPTED_FILE_GZIP_DATA: case STREAM_ENCRYPTED_WIN32_DATA: case STREAM_ENCRYPTED_WIN32_GZIP_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: #endif case 0: /* compatibility with old tapes */ return true; @@ -862,6 +890,13 @@ bool is_restore_stream_supported(int stream) case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: #endif +#ifndef HAVE_LZO + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: +#endif #ifndef HAVE_DARWIN_OS case STREAM_MACOS_FORK_DATA: case STREAM_HFSPLUS_ATTRIBUTES: @@ -873,6 +908,13 @@ bool is_restore_stream_supported(int stream) case STREAM_GZIP_DATA: case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: +#endif +#ifdef HAVE_LZO + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: #endif case STREAM_WIN32_DATA: case STREAM_UNIX_ATTRIBUTES: diff --git a/bacula/src/findlib/find.c b/bacula/src/findlib/find.c index 77ac196dab..3098f615c1 100644 --- a/bacula/src/findlib/find.c +++ b/bacula/src/findlib/find.c @@ -192,7 +192,8 @@ find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, bool to for (j=0; jopts_list.size(); j++) { findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j); ff->flags |= fo->flags; - ff->GZIP_level = fo->GZIP_level; + ff->Compress_algo = fo->Compress_algo; + ff->Compress_level = fo->Compress_level; ff->strip_path = fo->strip_path; ff->fstypes = fo->fstype; ff->drivetypes = fo->drivetype; @@ -300,7 +301,8 @@ static bool accept_file(FF_PKT *ff) for (j = 0; j < incexe->opts_list.size(); j++) { findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j); ff->flags = fo->flags; - ff->GZIP_level = fo->GZIP_level; + ff->Compress_algo = fo->Compress_algo; + ff->Compress_level = fo->Compress_level; ff->fstypes = fo->fstype; ff->drivetypes = fo->drivetype; diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h index 804eeaec2b..cda9e9b016 100644 --- a/bacula/src/findlib/find.h +++ b/bacula/src/findlib/find.h @@ -77,6 +77,7 @@ int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); struct s_included_file { struct s_included_file *next; uint32_t options; /* backup options */ + uint32_t algo; /* compression algorithm. 4 letters stored as an interger */ int level; /* compression level */ int len; /* length of fname */ int pattern; /* set if wild card pattern */ @@ -108,7 +109,8 @@ enum { /* File options structure */ struct findFOPTS { uint32_t flags; /* options in bits */ - int GZIP_level; /* GZIP level */ + uint32_t Compress_algo; /* compression algorithm. 4 letters stored as an interger */ + int Compress_level; /* compression level */ int strip_path; /* strip path count */ char VerifyOpts[MAX_FOPTS]; /* verify options */ char AccurateOpts[MAX_FOPTS]; /* accurate mode options */ @@ -196,7 +198,8 @@ struct FF_PKT { /* Values set by accept_file while processing Options */ uint32_t flags; /* backup options */ - int GZIP_level; /* compression level */ + uint32_t Compress_algo; /* compression algorithm. 4 letters stored as an interger */ + int Compress_level; /* compression level */ int strip_path; /* strip path count */ bool cmd_plugin; /* set if we have a command plugin */ alist fstypes; /* allowed file system types */ diff --git a/bacula/src/findlib/match.c b/bacula/src/findlib/match.c index 3e84e33526..beffad50bc 100644 --- a/bacula/src/findlib/match.c +++ b/bacula/src/findlib/match.c @@ -42,6 +42,7 @@ #include "bacula.h" #include "find.h" +#include "ch.h" #include @@ -185,10 +186,19 @@ void add_fname_to_include_list(FF_PKT *ff, int prefixed, const char *fname) case 'A': inc->options |= FO_ACL; break; - case 'Z': /* gzip compression */ - inc->options |= FO_GZIP; - inc->level = *++rp - '0'; - Dmsg1(200, "Compression level=%d\n", inc->level); + case 'Z': /* compression */ + rp++; /* skip Z */ + if (*rp >= '0' && *rp <= '9') { + inc->options |= FO_COMPRESS; + inc->algo = COMPRESS_GZIP; + inc->level = *rp - '0'; + } + else if (*rp == 'o') { + inc->options |= FO_COMPRESS; + inc->algo = COMPRESS_LZO1X; + inc->level = 1; /* not used with LZO */ + } + Dmsg2(200, "Compression alg=%d level=%d\n", inc->algo, inc->level); break; case 'K': inc->options |= FO_NOATIME; @@ -246,8 +256,8 @@ void add_fname_to_include_list(FF_PKT *ff, int prefixed, const char *fname) { } next->next = inc; } - Dmsg3(100, "add_fname_to_include prefix=%d gzip=%d fname=%s\n", - prefixed, !!(inc->options & FO_GZIP), inc->fname); + Dmsg4(100, "add_fname_to_include prefix=%d compres=%d alg= %d fname=%s\n", + prefixed, !!(inc->options & FO_COMPRESS), inc->algo, inc->fname); } /* @@ -302,7 +312,8 @@ struct s_included_file *get_next_included_file(FF_PKT *ff, struct s_included_fil */ if (inc) { ff->flags = inc->options; - ff->GZIP_level = inc->level; + ff->Compress_algo = inc->algo; + ff->Compress_level = inc->level; } return inc; } diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index f29d6fd172..cf880a342d 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -382,6 +382,7 @@ public: POOLMEM *compress_buf; /* Compression buffer */ int32_t compress_buf_size; /* Length of compression buffer */ void *pZLIB_compress_workset; /* zlib compression session data */ + void *LZO_compress_workset; /* lzo compression session data */ int32_t replace; /* Replace options */ int32_t buf_size; /* length of buffer */ FF_PKT *ff; /* Find Files packet */ diff --git a/bacula/src/stored/Makefile.in b/bacula/src/stored/Makefile.in index 849217c5ac..e3af805855 100644 --- a/bacula/src/stored/Makefile.in +++ b/bacula/src/stored/Makefile.in @@ -74,6 +74,7 @@ EXTRAOBJS = @OBJLIST@ CAP_LIBS = @CAP_LIBS@ ZLIBS=@ZLIBS@ +LZOLIBS = @LZOLIBS@ .SUFFIXES: .c .o @@ -128,7 +129,7 @@ bextract.o: bextract.c bextract: Makefile $(BEXTOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @echo "Compiling $<" - $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) \ + $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZOLIBS) \ -lbacfind -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bscan.o: bscan.c diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c index 2e2a44919d..d28f0f0a6a 100644 --- a/bacula/src/stored/bextract.c +++ b/bacula/src/stored/bextract.c @@ -35,6 +35,7 @@ #include "bacula.h" #include "stored.h" +#include "ch.h" #include "findlib/find.h" extern bool parse_sd_config(CONFIG *config, const char *configfile, int exit_code); @@ -465,6 +466,102 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) #endif break; + /* Compressed data stream */ + case STREAM_COMPRESSED_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: + case STREAM_WIN32_COMPRESSED_DATA: + if (extract) { + uint32_t comp_magic, comp_len; + uint16_t comp_level, comp_version; +#ifdef HAVE_LZO + lzo_uint compress_len; + const unsigned char *cbuf; + int r, real_compress_len; +#endif + + if (rec->maskedStream == STREAM_SPARSE_COMPRESSED_DATA) { + ser_declare; + uint64_t faddr; + char ec1[50]; + wbuf = rec->data + OFFSET_FADDR_SIZE; + wsize = rec->data_len - OFFSET_FADDR_SIZE; + ser_begin(rec->data, OFFSET_FADDR_SIZE); + unser_uint64(faddr); + if (fileAddr != faddr) { + fileAddr = faddr; + if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { + berrno be; + Emsg3(M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"), + edit_uint64(fileAddr, ec1), attr->ofname, be.bstrerror()); + extract = false; + return true; + } + } + } else { + wbuf = rec->data; + wsize = rec->data_len; + } + + /* read compress header */ + unser_declare; + unser_begin(wbuf, sizeof(comp_stream_header)); + unser_uint32(comp_magic); + unser_uint32(comp_len); + unser_uint16(comp_level); + unser_uint16(comp_version); + Dmsg4(200, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len, + comp_level, comp_version); + + /* version check */ + if (comp_version != COMP_HEAD_VERSION) { + Emsg1(M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version); + return false; + } + /* size check */ + if (comp_len + sizeof(comp_stream_header) != wsize) { + Emsg2(M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"), + comp_len, wsize); + return false; + } + + switch(comp_magic) { +#ifdef HAVE_LZO + case COMPRESS_LZO1X: + compress_len = compress_buf_size; + cbuf = (const unsigned char*) wbuf + sizeof(comp_stream_header); + real_compress_len = wsize - sizeof(comp_stream_header); + Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, wsize); + while ((r=lzo1x_decompress_safe(cbuf, real_compress_len, + (unsigned char *)compress_buf, &compress_len, NULL)) == LZO_E_OUTPUT_OVERRUN) + { + + /* The buffer size is too small, try with a bigger one */ + compress_len = 2 * compress_len; + compress_buf = check_pool_memory_size(compress_buf, + compress_len); + } + if (r != LZO_E_OK) { + Emsg1(M_ERROR, 0, _("LZO uncompression error. ERR=%d\n"), r); + extract = false; + return true; + } + break; +#endif + default: + Emsg1(M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic); + extract = false; + return true; + } + + Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total); + store_data(&bfd, compress_buf, compress_len); + total += compress_len; + fileAddr += compress_len; + Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, + compress_len); + } + break; + case STREAM_MD5_DIGEST: case STREAM_SHA1_DIGEST: case STREAM_SHA256_DIGEST: diff --git a/bacula/src/stored/bscan.c b/bacula/src/stored/bscan.c index e987feb0b4..648dd9d370 100644 --- a/bacula/src/stored/bscan.c +++ b/bacula/src/stored/bscan.c @@ -734,8 +734,11 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) break; case STREAM_GZIP_DATA: + case STREAM_COMPRESSED_DATA: case STREAM_ENCRYPTED_FILE_GZIP_DATA: + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: case STREAM_ENCRYPTED_WIN32_GZIP_DATA: + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: /* No correct, we should (decrypt and) expand it done using JCR */ @@ -744,12 +747,14 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) break; case STREAM_SPARSE_GZIP_DATA: + case STREAM_SPARSE_COMPRESSED_DATA: mjcr->JobBytes += rec->data_len - sizeof(uint64_t); /* No correct, we should expand it */ free_jcr(mjcr); /* done using JCR */ break; /* Win32 GZIP stream */ case STREAM_WIN32_GZIP_DATA: + case STREAM_WIN32_COMPRESSED_DATA: mjcr->JobBytes += rec->data_len; free_jcr(mjcr); /* done using JCR */ break; diff --git a/bacula/src/stored/record.c b/bacula/src/stored/record.c index 430e2690af..32f09a9100 100644 --- a/bacula/src/stored/record.c +++ b/bacula/src/stored/record.c @@ -109,12 +109,16 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "contWIN32-DATA"; case STREAM_WIN32_GZIP_DATA: return "contWIN32-GZIP"; + case STREAM_WIN32_COMPRESSED_DATA: + return "contWIN32-COMPRESSED"; case STREAM_MD5_DIGEST: return "contMD5"; case STREAM_SHA1_DIGEST: return "contSHA1"; case STREAM_GZIP_DATA: return "contGZIP"; + case STREAM_COMPRESSED_DATA: + return "contCOMPRESSED"; case STREAM_UNIX_ATTRIBUTES_EX: return "contUNIX-ATTR-EX"; case STREAM_RESTORE_OBJECT: @@ -123,6 +127,8 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "contSPARSE-DATA"; case STREAM_SPARSE_GZIP_DATA: return "contSPARSE-GZIP"; + case STREAM_SPARSE_COMPRESSED_DATA: + return "contSPARSE-COMPRESSED"; case STREAM_PROGRAM_NAMES: return "contPROG-NAMES"; case STREAM_PROGRAM_DATA: @@ -143,10 +149,14 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "contENCRYPTED-FILE"; case STREAM_ENCRYPTED_FILE_GZIP_DATA: return "contENCRYPTED-GZIP"; + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + return "contENCRYPTED-COMPRESSED"; case STREAM_ENCRYPTED_WIN32_DATA: return "contENCRYPTED-WIN32-DATA"; case STREAM_ENCRYPTED_WIN32_GZIP_DATA: return "contENCRYPTED-WIN32-GZIP"; + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: + return "contENCRYPTED-WIN32-COMPRESSED"; case STREAM_ENCRYPTED_MACOS_FORK_DATA: return "contENCRYPTED-MACOS-RSRC"; case STREAM_PLUGIN_NAME: @@ -167,12 +177,16 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "WIN32-DATA"; case STREAM_WIN32_GZIP_DATA: return "WIN32-GZIP"; + case STREAM_WIN32_COMPRESSED_DATA: + return "WIN32-COMPRESSED"; case STREAM_MD5_DIGEST: return "MD5"; case STREAM_SHA1_DIGEST: return "SHA1"; case STREAM_GZIP_DATA: return "GZIP"; + case STREAM_COMPRESSED_DATA: + return "COMPRESSED"; case STREAM_UNIX_ATTRIBUTES_EX: return "UNIX-ATTR-EX"; case STREAM_RESTORE_OBJECT: @@ -181,6 +195,8 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "SPARSE-DATA"; case STREAM_SPARSE_GZIP_DATA: return "SPARSE-GZIP"; + case STREAM_SPARSE_COMPRESSED_DATA: + return "SPARSE-COMPRESSED"; case STREAM_PROGRAM_NAMES: return "PROG-NAMES"; case STREAM_PROGRAM_DATA: @@ -203,10 +219,14 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "ENCRYPTED-FILE"; case STREAM_ENCRYPTED_FILE_GZIP_DATA: return "ENCRYPTED-GZIP"; + case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: + return "ENCRYPTED-COMPRESSED"; case STREAM_ENCRYPTED_WIN32_DATA: return "ENCRYPTED-WIN32-DATA"; case STREAM_ENCRYPTED_WIN32_GZIP_DATA: return "ENCRYPTED-WIN32-GZIP"; + case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: + return "ENCRYPTED-WIN32-COMPRESSED"; case STREAM_ENCRYPTED_MACOS_FORK_DATA: return "ENCRYPTED-MACOS-RSRC"; diff --git a/bacula/src/stored/stored.h b/bacula/src/stored/stored.h index 3e1f967752..a15a36a73e 100644 --- a/bacula/src/stored/stored.h +++ b/bacula/src/stored/stored.h @@ -72,6 +72,10 @@ const int sd_dbglvl = 300; #else #define uLongf uint32_t #endif +#ifdef HAVE_LZO +#include +#include +#endif #ifdef HAVE_FNMATCH #include #else diff --git a/bacula/src/streams.h b/bacula/src/streams.h index 0a1bbfce5b..1fc8c829cb 100644 --- a/bacula/src/streams.h +++ b/bacula/src/streams.h @@ -97,6 +97,15 @@ #define STREAM_PLUGIN_NAME 26 /* Plugin "file" string */ #define STREAM_PLUGIN_DATA 27 /* Plugin specific data */ #define STREAM_RESTORE_OBJECT 28 /* Plugin restore object */ +/* Non GZip compressed streams. Those streams can handle arbitrary compression algorithm data + * as an additional header is stored at the beginning of the stream. + * see stream_compressed_header definition for more details. + */ +#define STREAM_COMPRESSED_DATA 29 /* Compressed file data */ +#define STREAM_SPARSE_COMPRESSED_DATA 30 /* Sparse compressed data stream */ +#define STREAM_WIN32_COMPRESSED_DATA 31 /* Compressed Win32 BackupRead data */ +#define STREAM_ENCRYPTED_FILE_COMPRESSED_DATA 32 /* Encrypted, compressed data */ +#define STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA 33 /* Encrypted, compressed Win32 BackupRead data */ /** * Additional Stream definitions. Once defined these must NEVER diff --git a/bacula/src/tools/testfind.c b/bacula/src/tools/testfind.c index eb8f10c14c..15e4ac67f5 100644 --- a/bacula/src/tools/testfind.c +++ b/bacula/src/tools/testfind.c @@ -35,6 +35,7 @@ #include "bacula.h" #include "dird/dird.h" #include "findlib/find.h" +#include "ch.h" #if defined(HAVE_WIN32) #define isatty(fd) (fd==0) @@ -629,10 +630,19 @@ static void set_options(findFOPTS *fo, const char *opts) case 'W': fo->flags |= FO_ENHANCEDWILD; break; - case 'Z': /* gzip compression */ - fo->flags |= FO_GZIP; - fo->GZIP_level = *++p - '0'; - Dmsg1(200, "Compression level=%d\n", fo->GZIP_level); + case 'Z': /* compression */ + p++; /* skip Z */ + if (*p >= '0' && *p <= '9') { + fo->flags |= FO_COMPRESS; + fo->Compress_algo = COMPRESS_GZIP; + fo->Compress_level = *p - '0'; + } + else if (*p == 'o') { + fo->flags |= FO_COMPRESS; + fo->Compress_algo = COMPRESS_LZO1X; + fo->Compress_level = 1; /* not used with LZO */ + } + Dmsg2(200, "Compression alg=%d level=%d\n", fo->Compress_algo, fo->Compress_level); break; case 'X': fo->flags |= FO_XATTR; diff --git a/regress/all-disk-tests b/regress/all-disk-tests index 5662bc7f11..8de4024362 100755 --- a/regress/all-disk-tests +++ b/regress/all-disk-tests @@ -21,7 +21,9 @@ nice tests/bscan-test nice tests/bsr-opt-test nice tests/comment-test nice tests/compressed-test +nice tests/lzo-test nice tests/compress-encrypt-test +nice tests/lzo-encrypt-test nice tests/concurrent-jobs-test nice tests/copy-job-test nice tests/copy-jobspan-test @@ -64,6 +66,7 @@ nice tests/maxvol2-test nice tests/messages-test nice tests/next-vol-test nice tests/sparse-compressed-test +nice tests/sparse-lzo-test nice tests/sparse-test nice tests/strip-test nice tests/two-jobs-test diff --git a/regress/scripts/new-test-bacula-dir.conf.in b/regress/scripts/new-test-bacula-dir.conf.in index 02e107c192..534324cca3 100644 --- a/regress/scripts/new-test-bacula-dir.conf.in +++ b/regress/scripts/new-test-bacula-dir.conf.in @@ -131,6 +131,34 @@ Job { Maximum Concurrent Jobs = 10 } +Job { + Name = "LZOTest" + Type = Backup + Client=@hostname@-fd + FileSet="LZOSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes +} + +Job { + Name = "SparseLZOTest" + Type = Backup + Client=@hostname@-fd + FileSet="SparseLZOSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes + Maximum Concurrent Jobs = 10 +} + Job { Name = "FIFOTest" Type = Backup @@ -238,6 +266,17 @@ FileSet { } } +FileSet { + Name = "LZOSet" + Include { + Options { + signature=MD5 + compression=LZO + } + File = <@tmpdir@/file-list + } +} + FileSet { Name = "FIFOSet" Include { @@ -262,6 +301,18 @@ FileSet { } } +FileSet { + Name = "SparseLZOSet" + Include { + Options { + signature=MD5 + compression=LZO + sparse=yes + } + File = <@tmpdir@/file-list + } +} + FileSet { Name = "MonsterFileSet" Include { diff --git a/regress/tests/lzo-encrypt-test b/regress/tests/lzo-encrypt-test new file mode 100755 index 0000000000..d2414718ba --- /dev/null +++ b/regress/tests/lzo-encrypt-test @@ -0,0 +1,47 @@ +#!/bin/sh +# +# Run a simple backup with encryption and compression of the Bacula build directory +# then verify the signatures. +# +TestName="lzo-encrypt-test" +JobName=LZOTest +. scripts/functions + +scripts/cleanup +scripts/copy-crypto-confs +echo "${cwd}/build" >${cwd}/tmp/file-list + +start_test + +cat <${cwd}/tmp/bconcmds +@$out /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=TestVolume001 +run job=$JobName yes +wait +messages +list volumes +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +@# setdebug level=0 fd +restore where=${cwd}/tmp/bacula-restores storage=File +5 +mark * +done +yes +wait +messages +quit +END_OF_DATA + +run_bacula +sleep 2 +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +end_test diff --git a/regress/tests/lzo-test b/regress/tests/lzo-test new file mode 100755 index 0000000000..96a8dab552 --- /dev/null +++ b/regress/tests/lzo-test @@ -0,0 +1,52 @@ +#!/bin/sh +# +# Run a simple backup of the Bacula build directory using the compressed option +# then restore it. +# +TestName="lzo-test" +JobName=lzo +. scripts/functions + +scripts/cleanup +scripts/copy-test-confs +echo "${cwd}/build" >${cwd}/tmp/file-list + +start_test + +cat <${cwd}/tmp/bconcmds +@$out /dev/null +messages +@$out ${cwd}/tmp/log1.out +status all +status all +messages +label storage=File volume=TestVolume001 +run job=LZOTest storage=File yes +wait +messages +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +restore where=${cwd}/tmp/bacula-restores select storage=File +unmark * +mark * +done +yes +wait +messages +quit +END_OF_DATA + +run_bacula +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs +check_restore_diff +grep " Software Compression" ${cwd}/tmp/log1.out | grep "%" 2>&1 1>/dev/null +if [ $? != 0 ] ; then + echo " !!!!! No compression !!!!!" + bstat=1 +fi +end_test diff --git a/regress/tests/sparse-lzo-test b/regress/tests/sparse-lzo-test new file mode 100755 index 0000000000..343f0936db --- /dev/null +++ b/regress/tests/sparse-lzo-test @@ -0,0 +1,42 @@ +#!/bin/sh +# +# Run a simple backup of the Bacula build directory using the Sparse option +# then restore it. +# +TestName="sparse-lzo-test" +JobName=Sparse-lzo +. scripts/functions + +cwd=`pwd` +scripts/cleanup +scripts/copy-test-confs +echo "${cwd}/build" >${cwd}/tmp/file-list + +start_test + +cat >${cwd}/tmp/bconcmds <