Add LZO compression support in bacula-fd.

Add new generic compressed streams This should make easier to
add additional compression scheme

Rename a few define, add an element containing compression algorithm
in some data structures.

diff --git a/bacula/autoconf/acconfig.h b/bacula/autoconf/acconfig.h
index de62f6c49e9742e631ccc6fc6c134c81332fc30e..d655307d64e990b1423b59155ae7d925808efb3d 100644 (file)
--- a/bacula/autoconf/acconfig.h
+++ b/bacula/autoconf/acconfig.h
@@ -114,6 +114,9 @@
 /* Define if you have zlib */
 #undef HAVE_LIBZ
 
+/* Define if you have lzo lib */
+#undef HAVE_LZO
+
 /* Define if you have libacl */
 #undef HAVE_ACL
 

diff --git a/bacula/autoconf/configure.in b/bacula/autoconf/configure.in
index f7387bbd9170e46159db035d8b71075d8130a070..525a0e2a552c89dfb41a0c256bd19d09a93cce01 100644 (file)
--- a/bacula/autoconf/configure.in
+++ b/bacula/autoconf/configure.in
@@ -2495,6 +2495,19 @@ if test x$ZLIBS = x-lz; then
 fi
 AC_SUBST(ZLIBS)
 
+dnl
+dnl Check for lzo
+dnl
+AC_CHECK_HEADERS(lzo/lzoconf.h)
+AC_CHECK_HEADERS(lzo/lzo1x.h)
+AC_CHECK_LIB(lzo2, lzo1x_1_compress, [LZOLIBS="-llzo2"])
+have_lzo=no
+if test x$LZOLIBS = x-llzo2; then
+   AC_DEFINE(HAVE_LZO)
+   have_lzo=yes
+fi
+AC_SUBST(LZOLIBS)
+
 dnl
 dnl Check for ACL support and libraries
 dnl
@@ -3468,6 +3481,7 @@ Configuration on `date`:
    TLS support:            ${support_tls}
    Encryption support:     ${support_crypto} 
    ZLIB support:           ${have_zlib}
+   LZO support:            ${have_lzo}
    enable-smartalloc:      ${support_smartalloc} 
    enable-lockmgr:         ${support_lockmgr}
    bat support:            ${support_bat}

diff --git a/bacula/src/ch.h b/bacula/src/ch.h
new file mode 100644 (file)
index 0000000..168cfd0
--- /dev/null
+++ b/bacula/src/ch.h
@@ -0,0 +1,58 @@
+/*
+   Bacula® - The Network Backup Solution
+
+   Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
+
+   The main author of Bacula is Kern Sibbald, with contributions from
+   many others, a complete list can be found in the file AUTHORS.
+   This program is Free Software; you can redistribute it and/or
+   modify it under the terms of version three of the GNU Affero General Public
+   License as published by the Free Software Foundation and included
+   in the file LICENSE.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+   02110-1301, USA.
+
+   Bacula® is a registered trademark of Kern Sibbald.
+   The licensor of Bacula is the Free Software Foundation Europe
+   (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
+   Switzerland, email:ftf@fsfeurope.org.
+*/
+/**
+ * Compressed stream header struct
+ *
+ *  Laurent Papier
+ *
+ */
+
+#ifndef __CH_H
+#define __CH_H 1
+
+/*
+ * Compression algorithm signature. 4 letters as a 32bits integer
+ */
+#define COMPRESS_NONE  0x4e4f4e45  /* used for incompressible block */
+#define COMPRESS_GZIP  0x475a4950
+#define COMPRESS_LZO1X 0x4c5a4f58
+
+/*
+ * Compression header version
+ */
+#define COMP_HEAD_VERSION 0x1
+
+/* Compressed data stream header */
+typedef struct {
+   uint32_t magic;      /* compression algo used in this compressed data stream */
+   uint16_t level;      /* compression level used */
+   uint16_t version;    /* for futur evolution */
+   uint32_t size;       /* compressed size of the original data */
+} comp_stream_header;
+
+#endif /* __CH_H */

diff --git a/bacula/src/dird/fd_cmds.c b/bacula/src/dird/fd_cmds.c
index 2c6914d9fb37e51e0abd62776a1de52e89f731fc..85e6055343cdd02c13641086f726196c6054a03f 100644 (file)
--- a/bacula/src/dird/fd_cmds.c
+++ b/bacula/src/dird/fd_cmds.c
@@ -379,7 +379,7 @@ static bool send_fileset(JCR *jcr)
                bool done=false;         /* print warning only if compression enabled in FS */ 
                int j = 0;
                for (k=0; fo->opts[k]!='\0'; k++) {                   
-                 /* Z compress option is followed by the single-digit compress level */
+                 /* Z compress option is followed by the single-digit compress level or 'o' */
                  if (fo->opts[k]=='Z') {
                     done=true;
                     k++;                /* skip option and level */

diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c
index ccd8258221f97729f2b536a19181e09d4e86398b..de91e7a862a0baebb1073dbe0a7f8c8a793d0912 100644 (file)
--- a/bacula/src/dird/inc_conf.c
+++ b/bacula/src/dird/inc_conf.c
@@ -228,6 +228,7 @@ static struct s_fs_opt FS_options[] = {
    {"gzip7",    INC_KW_COMPRESSION,  "Z7"},
    {"gzip8",    INC_KW_COMPRESSION,  "Z8"},
    {"gzip9",    INC_KW_COMPRESSION,  "Z9"},
+   {"lzo",      INC_KW_COMPRESSION,  "Zo"},
    {"blowfish", INC_KW_ENCRYPTION,    "B"},   /* ***FIXME*** not implemented */
    {"3des",     INC_KW_ENCRYPTION,    "3"},   /* ***FIXME*** not implemented */
    {"yes",      INC_KW_ONEFS,         "0"},

diff --git a/bacula/src/filed/Makefile.in b/bacula/src/filed/Makefile.in
index 8f0f10d024ba574fb05c03a4fd8b08add776c4b7..c67a2e032045ed954c0a3a5ea56f73aace08dd45 100644 (file)
--- a/bacula/src/filed/Makefile.in
+++ b/bacula/src/filed/Makefile.in
@@ -41,6 +41,7 @@ EXTRAOBJS = @OBJLIST@
 CAP_LIBS = @CAP_LIBS@
 FDLIBS = @FDLIBS@                # extra libs for File daemon
 ZLIBS = @ZLIBS@
+LZOLIBS = @LZOLIBS@
 
 # extra items for linking on Win32
 WIN32OBJS = win32/winmain.o win32/winlib.a win32/winres.res
@@ -93,12 +94,12 @@ win32:      $(WIN32OBJS)
 bacula-fd:  Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbacpy$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@
        @echo "Linking $@ ..."
        $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(SVROBJS) \
-         $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \
+         $(WIN32LIBS) $(FDLIBS) $(ZLIBS) $(LZOLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \
          $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS)
 
 static-bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind.a ../lib/libbacpy$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@
        $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -static -L../lib -L../findlib -o $@ $(SVROBJS) \
-          $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \
+          $(WIN32LIBS) $(FDLIBS) $(ZLIBS) $(LZOLIBS) -lbacfind -lbacpy -lbaccfg -lbac -lm $(PYTHON_LIBS) $(LIBS) \
           $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS)
        strip $@
 

diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c
index 6ac8721c6d7f390dd0d99a692b7230e2b0571f7e..09de90ea8c9568e1aa204977f305a7c354a21f44 100644 (file)
--- a/bacula/src/filed/backup.c
+++ b/bacula/src/filed/backup.c
@@ -35,6 +35,7 @@
 
 #include "bacula.h"
 #include "filed.h"
+#include "ch.h"
 
 #ifdef HAVE_DARWIN_OS
 const bool have_darwin_os = true;
@@ -110,12 +111,22 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
     *  Note, we adjust the read size to be smaller so that the
     *  same output buffer can be used without growing it.
     *
+    *  For LZO1X compression the recommended value is :
+    *                  output_block_size = input_block_size + (input_block_size / 16) + 64 + 3 + sizeof(comp_stream_header)
+    *
     * The zlib compression workset is initialized here to minimize
     *  the "per file" load. The jcr member is only set, if the init 
     *  was successful.
+    *
+    *  For the same reason, lzo compression is initialized here.
     */
+#ifdef HAVE_LZO
+   jcr->compress_buf_size = MAX(jcr->buf_size + (jcr->buf_size / 16) + 67 + sizeof(comp_stream_header), jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30);
+   jcr->compress_buf = get_memory(jcr->compress_buf_size);
+#else
    jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
    jcr->compress_buf = get_memory(jcr->compress_buf_size);
+#endif
    
 #ifdef HAVE_LIBZ
    z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));  
@@ -133,6 +144,17 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
    }
 #endif
 
+#ifdef HAVE_LZO
+   lzo_voidp pLzoMem = (lzo_voidp) malloc(LZO1X_1_MEM_COMPRESS);
+   if (pLzoMem) {
+      if (lzo_init() == LZO_E_OK) {
+         jcr->LZO_compress_workset = pLzoMem;
+      } else {
+         free (pLzoMem);
+      }
+   }
+#endif
+
    if (!crypto_session_start(jcr)) {
       return false;
    }
@@ -207,6 +229,11 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
       free (jcr->pZLIB_compress_workset);
       jcr->pZLIB_compress_workset = NULL;
    }
+   if (jcr->LZO_compress_workset) {
+      free (jcr->LZO_compress_workset);
+      jcr->LZO_compress_workset = NULL;
+   }
+
    crypto_session_end(jcr);
 
 
@@ -604,7 +631,7 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
                goto good_rtn;
             }
             flags = ff_pkt->flags;
-            ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE|FO_OFFSETS);
+            ff_pkt->flags &= ~(FO_COMPRESS|FO_SPARSE|FO_OFFSETS);
             if (flags & FO_ENCRYPT) {
                rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
             } else {
@@ -821,13 +848,14 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
 
    Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
 
-#ifdef HAVE_LIBZ
+#if defined(HAVE_LIBZ) || defined(HAVE_LZO)
    uLong compress_len = 0;
    uLong max_compress_len = 0;
    const Bytef *cbuf = NULL;
+ #ifdef HAVE_LIBZ
    int zstat;
 
-   if (ff_pkt->flags & FO_GZIP) {
+   if ((ff_pkt->flags & FO_COMPRESS) && ff_pkt->Compress_algo == COMPRESS_GZIP) {
       if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
          cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
          max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
@@ -847,13 +875,38 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
       if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
          /** set gzip compression level - must be done per file */
          if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset, 
-              ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
+              ff_pkt->Compress_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
             Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
             jcr->setJobStatus(JS_ErrorTerminated);
             goto err;
          }
       }
    }
+ #endif
+ #ifdef HAVE_LZO
+   Bytef *cbuf2;
+   int lzores;
+   comp_stream_header ch;
+
+   memset(&ch, 0, sizeof(comp_stream_header));
+   cbuf2 = NULL;
+
+   if ((ff_pkt->flags & FO_COMPRESS) && ff_pkt->Compress_algo == COMPRESS_LZO1X) {
+      if ((ff_pkt->flags & FO_SPARSE) || (ff_pkt->flags & FO_OFFSETS)) {
+         cbuf = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE;
+         cbuf2 = (Bytef *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header);
+         max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
+      } else {
+         cbuf = (Bytef *)jcr->compress_buf;
+         cbuf2 = (Bytef *)jcr->compress_buf + sizeof(comp_stream_header);
+         max_compress_len = jcr->compress_buf_size; /* set max length */
+      }
+      ch.magic = COMPRESS_LZO1X;
+      ch.version = COMP_HEAD_VERSION;
+      wbuf = jcr->compress_buf;    /* compressed output here */
+      cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
+   }
+ #endif
 #else
    const uint32_t max_compress_len = 0;
 #endif
@@ -968,7 +1021,7 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
 
 #ifdef HAVE_LIBZ
       /** Do compression if turned on */
-      if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
+      if (ff_pkt->flags & FO_COMPRESS && ff_pkt->Compress_algo == COMPRESS_GZIP && jcr->pZLIB_compress_workset) {
          Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
          
          ((z_stream*)jcr->pZLIB_compress_workset)->next_in   = (Bytef *)rbuf;
@@ -989,13 +1042,45 @@ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
             goto err;
          }
 
-         Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len, 
+         Dmsg2(400, "GZIP compressed len=%d uncompressed len=%d\n", compress_len, 
                sd->msglen);
 
          sd->msglen = compress_len;      /* set compressed length */
          cipher_input_len = compress_len;
       }
 #endif
+#ifdef HAVE_LZO
+      /** Do compression if turned on */
+      if (ff_pkt->flags & FO_COMPRESS && ff_pkt->Compress_algo == COMPRESS_LZO1X && jcr->LZO_compress_workset) {
+         ser_declare;
+         ser_begin(cbuf, sizeof(comp_stream_header));
+
+         Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
+
+         lzores = lzo1x_1_compress((const unsigned char*)rbuf, sd->msglen, cbuf2, &compress_len, jcr->LZO_compress_workset);
+         if (lzores == LZO_E_OK && compress_len <= max_compress_len)
+         {
+            /* complete header */
+            ser_uint32(COMPRESS_LZO1X);
+            ser_uint32(compress_len);
+            ser_uint16(ch.level);
+            ser_uint16(ch.version);
+         } else {
+            /** this should NEVER happen */
+            Jmsg(jcr, M_FATAL, 0, _("Compression LZO error: %d\n"), lzores);
+            jcr->setJobStatus(JS_ErrorTerminated);
+            goto err;
+         }
+
+         Dmsg2(400, "LZO compressed len=%d uncompressed len=%d\n", compress_len, 
+               sd->msglen);
+
+         compress_len += sizeof(comp_stream_header); /* add size of header */
+         sd->msglen = compress_len;      /* set compressed length */
+         cipher_input_len = compress_len;
+      }
+#endif
+
       /**
        * Note, here we prepend the current record length to the beginning
        *  of the encrypted data. This is because both sparse and compression

diff --git a/bacula/src/filed/filed.h b/bacula/src/filed/filed.h
index f6c785f59695bd9c15422480433bc9f42de0f622..71d26d69ec63bd1a290c247df14ce1ffb67591fc 100644 (file)
--- a/bacula/src/filed/filed.h
+++ b/bacula/src/filed/filed.h
@@ -76,6 +76,10 @@ typedef enum {
 #else
 #define uLongf uint32_t
 #endif
+#ifdef HAVE_LZO
+#include <lzo/lzoconf.h>
+#include <lzo/lzo1x.h>
+#endif
 
 extern CLIENT *me;                    /* "Global" Client resource */
 

diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c
index 92018aac66caeac672c9ac96869783704cfcf492..417a3931de57c7114bd1a41137c190e6412ee84f 100644 (file)
--- a/bacula/src/filed/job.c
+++ b/bacula/src/filed/job.c
@@ -34,6 +34,7 @@
 
 #include "bacula.h"
 #include "filed.h"
+#include "ch.h"
 
 #if defined(WIN32_VSS)
 #include "vss.h"
@@ -1385,9 +1386,18 @@ static int set_options(findFOPTS *fo, const char *opts)
       case 'W':
          fo->flags |= FO_ENHANCEDWILD;
          break;
-      case 'Z':                 /* gzip compression */
-         fo->flags |= FO_GZIP;
-         fo->GZIP_level = *++p - '0';
+      case 'Z':                 /* compression */
+         p++;                   /* skip Z */
+         if (*p >= '0' && *p <= '9') {
+            fo->flags |= FO_COMPRESS;
+            fo->Compress_algo = COMPRESS_GZIP;
+            fo->Compress_level = *p - '0';
+         }
+         else if (*p == 'o') {
+            fo->flags |= FO_COMPRESS;
+            fo->Compress_algo = COMPRESS_LZO1X;
+            fo->Compress_level = 1; /* not used with LZO */
+         }
          break;
       case 'K':
          fo->flags |= FO_NOATIME;

diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c
index ed4d7243ff37d1300259a61220ab505346d1200a..7c4b028578aca0d7a5f6f5fb36e15286d22aa479 100644 (file)
--- a/bacula/src/filed/restore.c
+++ b/bacula/src/filed/restore.c
@@ -34,6 +34,7 @@
 
 #include "bacula.h"
 #include "filed.h"
+#include "ch.h"
 #include "restore.h"
 
 #ifdef HAVE_DARWIN_OS
@@ -81,6 +82,12 @@ const bool have_libz = true;
 #else
 const bool have_libz = false;
 #endif
+#ifdef HAVE_LZO
+const bool have_lzo = true;
+#else
+const bool have_lzo = false;
+#endif
+
 
 static void deallocate_cipher(r_ctx &rctx);
 static void deallocate_fork_cipher(r_ctx &rctx);
@@ -90,8 +97,8 @@ static void close_previous_stream(r_ctx &rctx);
 
 static bool verify_signature(JCR *jcr, r_ctx &rctx);
 int32_t extract_data(JCR *jcr, BFILE *bfd, POOLMEM *buf, int32_t buflen,
-                     uint64_t *addr, int flags, RESTORE_CIPHER_CTX *cipher_ctx);
-bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, 
+                     uint64_t *addr, int flags, int32_t stream, RESTORE_CIPHER_CTX *cipher_ctx);
+bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, int32_t stream,
                   RESTORE_CIPHER_CTX *cipher_ctx);
 
 /*
@@ -197,12 +204,20 @@ void do_restore(JCR *jcr)
     * St Bernard code goes here if implemented -- see end of file
     */
 
-   if (have_libz) {
+   /* use the same buffer size to decompress both gzip and lzo */
+   if (have_libz || have_lzo) {
       uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100;
       jcr->compress_buf = get_memory(compress_buf_size);
       jcr->compress_buf_size = compress_buf_size;
    }
 
+#ifdef HAVE_LZO
+   if (lzo_init() != LZO_E_OK) {
+      Jmsg(jcr, M_FATAL, 0, _("LZO init failed\n"));
+      goto bail_out;
+   }
+#endif
+
    if (have_crypto) {
       rctx.cipher_ctx.buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
       if (have_darwin_os) {
@@ -499,10 +514,15 @@ void do_restore(JCR *jcr)
       case STREAM_GZIP_DATA:
       case STREAM_SPARSE_GZIP_DATA:
       case STREAM_WIN32_GZIP_DATA:
+      case STREAM_COMPRESSED_DATA:
+      case STREAM_SPARSE_COMPRESSED_DATA:
+      case STREAM_WIN32_COMPRESSED_DATA:
       case STREAM_ENCRYPTED_FILE_DATA:
       case STREAM_ENCRYPTED_WIN32_DATA:
       case STREAM_ENCRYPTED_FILE_GZIP_DATA:
       case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
+      case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+      case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
          /*
           * Force an expected, consistent stream type here
           */
@@ -512,8 +532,9 @@ void do_restore(JCR *jcr)
                          || rctx.prev_stream == STREAM_ENCRYPTED_SESSION_DATA)) {
             rctx.flags = 0;
 
-            if (rctx.stream == STREAM_SPARSE_DATA || 
-                rctx.stream == STREAM_SPARSE_GZIP_DATA) {
+            if (rctx.stream == STREAM_SPARSE_DATA
+                  || rctx.stream == STREAM_SPARSE_COMPRESSED_DATA
+                  || rctx.stream == STREAM_SPARSE_GZIP_DATA) {
                rctx.flags |= FO_SPARSE;
             }
 
@@ -521,13 +542,21 @@ void do_restore(JCR *jcr)
                   || rctx.stream == STREAM_SPARSE_GZIP_DATA
                   || rctx.stream == STREAM_WIN32_GZIP_DATA
                   || rctx.stream == STREAM_ENCRYPTED_FILE_GZIP_DATA
+                  || rctx.stream == STREAM_COMPRESSED_DATA
+                  || rctx.stream == STREAM_SPARSE_COMPRESSED_DATA
+                  || rctx.stream == STREAM_WIN32_COMPRESSED_DATA
+                  || rctx.stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA
+                  || rctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA
                   || rctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) {
-               rctx.flags |= FO_GZIP;
+               rctx.flags |= FO_COMPRESS;
+               rctx.comp_stream = rctx.stream;
             }
 
             if (rctx.stream == STREAM_ENCRYPTED_FILE_DATA
                   || rctx.stream == STREAM_ENCRYPTED_FILE_GZIP_DATA
                   || rctx.stream == STREAM_ENCRYPTED_WIN32_DATA
+                  || rctx.stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA
+                  || rctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA
                   || rctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) {               
                /*
                 * Set up a decryption context
@@ -561,7 +590,7 @@ void do_restore(JCR *jcr)
             }
 
             if (extract_data(jcr, &rctx.bfd, sd->msg, sd->msglen, &rctx.fileAddr,
-                             rctx.flags, &rctx.cipher_ctx) < 0) {
+                             rctx.flags, rctx.stream, &rctx.cipher_ctx) < 0) {
                rctx.extract = false;
                bclose(&rctx.bfd);
                continue;
@@ -616,7 +645,7 @@ void do_restore(JCR *jcr)
                }
 
                if (extract_data(jcr, &rctx.forkbfd, sd->msg, sd->msglen, &rctx.fork_addr, rctx.fork_flags,
-                                &rctx.fork_cipher_ctx) < 0) {
+                                rctx.stream, &rctx.fork_cipher_ctx) < 0) {
                   rctx.extract = false;
                   bclose(&rctx.forkbfd);
                   continue;
@@ -1078,44 +1107,112 @@ bool sparse_data(JCR *jcr, BFILE *bfd, uint64_t *addr, char **data, uint32_t *le
       return true;
 }
 
-bool decompress_data(JCR *jcr, char **data, uint32_t *length)
+bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length)
 {
-#ifdef HAVE_LIBZ
-   uLong compress_len;
-   int stat;
    char ec1[50]; /* Buffer printing huge values */
 
-   /* 
-    * NOTE! We only use uLong and Byte because they are
-    * needed by the zlib routines, they should not otherwise
-    * be used in Bacula.
-    */
-   compress_len = jcr->compress_buf_size;
-   Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length);
-   while ((stat=uncompress((Byte *)jcr->compress_buf, &compress_len,
-                           (const Byte *)*data, (uLong)*length)) == Z_BUF_ERROR)
+   Dmsg1(200, "Stream found in decompress_data(): %d\n", stream);
+   if(stream == STREAM_COMPRESSED_DATA || stream == STREAM_SPARSE_COMPRESSED_DATA || stream == STREAM_WIN32_COMPRESSED_DATA
+       || stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA || stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA)
    {
-      /*
-       * The buffer size is too small, try with a bigger one
+      uint32_t comp_magic, comp_len;
+      uint16_t comp_level, comp_version;
+#ifdef HAVE_LZO
+      lzo_uint compress_len;
+      const unsigned char *cbuf;
+      int r, real_compress_len;
+#endif
+
+      /* read compress header */
+      unser_declare;
+      unser_begin(*data, sizeof(comp_stream_header));
+      unser_uint32(comp_magic);
+      unser_uint32(comp_len);
+      unser_uint16(comp_level);
+      unser_uint16(comp_version);
+      Dmsg4(200, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len,
+                              comp_level, comp_version);
+
+      /* version check */
+      if (comp_version != COMP_HEAD_VERSION) {
+         Qmsg(jcr, M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version);
+         return false;
+      }
+      /* size check */
+      if (comp_len + sizeof(comp_stream_header) != *length) {
+         Qmsg(jcr, M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"),
+              comp_len, *length);
+         return false;
+      }
+      switch(comp_magic) {
+#ifdef HAVE_LZO
+         case COMPRESS_LZO1X:
+            compress_len = jcr->compress_buf_size;
+            cbuf = (const unsigned char*)*data + sizeof(comp_stream_header);
+            real_compress_len = *length - sizeof(comp_stream_header);
+            Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length);
+            while ((r=lzo1x_decompress_safe(cbuf, real_compress_len,
+                                            (unsigned char *)jcr->compress_buf, &compress_len, NULL)) == LZO_E_OUTPUT_OVERRUN)
+            {
+               /*
+                * The buffer size is too small, try with a bigger one
+                */
+               compress_len = jcr->compress_buf_size = jcr->compress_buf_size + (jcr->compress_buf_size >> 1);
+               Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length);
+               jcr->compress_buf = check_pool_memory_size(jcr->compress_buf,
+                                                    compress_len);
+            }
+            if (r != LZO_E_OK) {
+               Qmsg(jcr, M_ERROR, 0, _("LZO uncompression error on file %s. ERR=%d\n"),
+                    jcr->last_fname, r);
+               return false;
+            }
+            *data = jcr->compress_buf;
+            *length = compress_len;
+            Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1));
+            return true;
+#endif
+         default:
+            Qmsg(jcr, M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic);
+            return false;
+      }
+    } else {
+#ifdef HAVE_LIBZ
+      uLong compress_len;
+      int stat;
+
+      /* 
+       * NOTE! We only use uLong and Byte because they are
+       * needed by the zlib routines, they should not otherwise
+       * be used in Bacula.
        */
-      compress_len = jcr->compress_buf_size = jcr->compress_buf_size + (jcr->compress_buf_size >> 1);
+      compress_len = jcr->compress_buf_size;
       Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length);
-      jcr->compress_buf = check_pool_memory_size(jcr->compress_buf,
-                                                 compress_len);
-   }
-   if (stat != Z_OK) {
-      Qmsg(jcr, M_ERROR, 0, _("Uncompression error on file %s. ERR=%s\n"),
-           jcr->last_fname, zlib_strerror(stat));
-      return false;
-   }
-   *data = jcr->compress_buf;
-   *length = compress_len;
-   Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1));
-   return true;
+      while ((stat=uncompress((Byte *)jcr->compress_buf, &compress_len,
+                              (const Byte *)*data, (uLong)*length)) == Z_BUF_ERROR)
+      {
+         /*
+          * The buffer size is too small, try with a bigger one
+          */
+         compress_len = jcr->compress_buf_size = jcr->compress_buf_size + (jcr->compress_buf_size >> 1);
+         Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length);
+         jcr->compress_buf = check_pool_memory_size(jcr->compress_buf,
+                                                    compress_len);
+      }
+      if (stat != Z_OK) {
+         Qmsg(jcr, M_ERROR, 0, _("Uncompression error on file %s. ERR=%s\n"),
+              jcr->last_fname, zlib_strerror(stat));
+         return false;
+      }
+      *data = jcr->compress_buf;
+      *length = compress_len;
+      Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1));
+      return true;
 #else
-   Qmsg(jcr, M_ERROR, 0, _("GZIP data stream found, but GZIP not configured!\n"));
-   return false;
+      Qmsg(jcr, M_ERROR, 0, _("GZIP data stream found, but GZIP not configured!\n"));
+      return false;
 #endif
+   }
 }
 
 static void unser_crypto_packet_len(RESTORE_CIPHER_CTX *ctx)
@@ -1157,7 +1254,7 @@ bool store_data(JCR *jcr, BFILE *bfd, char *data, const int32_t length, bool win
  * Return value is the number of bytes written, or -1 on errors.
  */
 int32_t extract_data(JCR *jcr, BFILE *bfd, POOLMEM *buf, int32_t buflen,
-                     uint64_t *addr, int flags, RESTORE_CIPHER_CTX *cipher_ctx)
+                     uint64_t *addr, int flags, int32_t stream, RESTORE_CIPHER_CTX *cipher_ctx)
 {
    char *wbuf;                 /* write buffer */
    uint32_t wsize;             /* write size */
@@ -1242,8 +1339,8 @@ int32_t extract_data(JCR *jcr, BFILE *bfd, POOLMEM *buf, int32_t buflen,
       }
    }
 
-   if (flags & FO_GZIP) {
-      if (!decompress_data(jcr, &wbuf, &wsize)) {
+   if (flags & FO_COMPRESS) {
+      if (!decompress_data(jcr, stream, &wbuf, &wsize)) {
          goto bail_out;
       }
    }
@@ -1332,7 +1429,7 @@ static void close_previous_stream(r_ctx &rctx)
  * writing it to bfd.
  * Return value is true on success, false on failure.
  */
-bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags,
+bool flush_cipher(JCR *jcr, BFILE *bfd, uint64_t *addr, int flags, int32_t stream,
                   RESTORE_CIPHER_CTX *cipher_ctx)
 {
    uint32_t decrypted_len = 0;
@@ -1383,8 +1480,8 @@ again:
       }
    }
 
-   if (flags & FO_GZIP) {
-      if (!decompress_data(jcr, &wbuf, &wsize)) {
+   if (flags & FO_COMPRESS) {
+      if (!decompress_data(jcr, stream, &wbuf, &wsize)) {
          return false;
       }
    }
@@ -1430,7 +1527,7 @@ static void deallocate_cipher(r_ctx &rctx)
     * Flush and deallocate previous stream's cipher context
     */
    if (rctx.cipher_ctx.cipher) {
-      flush_cipher(rctx.jcr, &rctx.bfd, &rctx.fileAddr, rctx.flags, &rctx.cipher_ctx);
+      flush_cipher(rctx.jcr, &rctx.bfd, &rctx.fileAddr, rctx.flags, rctx.comp_stream, &rctx.cipher_ctx);
       crypto_cipher_free(rctx.cipher_ctx.cipher);
       rctx.cipher_ctx.cipher = NULL;
    }
@@ -1443,7 +1540,7 @@ static void deallocate_fork_cipher(r_ctx &rctx)
     * Flush and deallocate previous stream's fork cipher context
     */
    if (rctx.fork_cipher_ctx.cipher) {
-      flush_cipher(rctx.jcr, &rctx.forkbfd, &rctx.fork_addr, rctx.fork_flags, &rctx.fork_cipher_ctx);
+      flush_cipher(rctx.jcr, &rctx.forkbfd, &rctx.fork_addr, rctx.fork_flags, rctx.comp_stream, &rctx.fork_cipher_ctx);
       crypto_cipher_free(rctx.fork_cipher_ctx.cipher);
       rctx.fork_cipher_ctx.cipher = NULL;
    }

diff --git a/bacula/src/filed/restore.h b/bacula/src/filed/restore.h
index f68af2c6d5c89139de5b0f170ea012262028745b..ffb0897d1dcc01627efe13a6e298b5f1d5029c72 100644 (file)
--- a/bacula/src/filed/restore.h
+++ b/bacula/src/filed/restore.h
@@ -43,6 +43,7 @@ struct r_ctx {
    int32_t stream;                     /* stream less new bits */
    int32_t prev_stream;                /* previous stream */
    int32_t full_stream;                /* full stream including new bits */
+   int32_t comp_stream;                /* last compressed stream found. needed only to restore encrypted compressed backup */
    BFILE bfd;                          /* File content */
    uint64_t fileAddr;                  /* file write address */
    uint32_t size;                      /* Size of file */

diff --git a/bacula/src/fileopts.h b/bacula/src/fileopts.h
index 0de2ad8545f104876bc418372878ec326fac2247..4767a5d22776ff8b3a91251ccfbbc1c4fedc0541 100644 (file)
--- a/bacula/src/fileopts.h
+++ b/bacula/src/fileopts.h
@@ -43,7 +43,7 @@
  */
 #define FO_PORTABLE_DATA (1<<0)       /* Data is portable */
 #define FO_MD5           (1<<1)       /* Do MD5 checksum */
-#define FO_GZIP          (1<<2)       /* Do Zlib compression */
+#define FO_COMPRESS      (1<<2)       /* Do compression */
 #define FO_NO_RECURSION  (1<<3)       /* no recursion in directories */
 #define FO_MULTIFS       (1<<4)       /* multiple file systems */
 #define FO_SPARSE        (1<<5)       /* do sparse file checking */

diff --git a/bacula/src/findlib/attribs.c b/bacula/src/findlib/attribs.c
index 419689fd8ef027220e1952d1e76f5734ec852be5..0d6e7c2f69a6efd83b3e65a8824e53785fdcbacd 100644 (file)
--- a/bacula/src/findlib/attribs.c
+++ b/bacula/src/findlib/attribs.c
@@ -36,6 +36,7 @@
 
 #include "bacula.h"
 #include "find.h"
+#include "ch.h"
 
 static uid_t my_uid = 1;
 static gid_t my_gid = 1;                        
@@ -103,32 +104,58 @@ int select_data_stream(FF_PKT *ff_pkt)
 
    /** Compression is not supported for Mac fork data */
    if (stream == STREAM_MACOS_FORK_DATA) {
-      ff_pkt->flags &= ~FO_GZIP;
+      ff_pkt->flags &= ~FO_COMPRESS;
    }
 
    /**
     * Handle compression and encryption options
     */
-#ifdef HAVE_LIBZ
-   if (ff_pkt->flags & FO_GZIP) {
-      switch (stream) {
-      case STREAM_WIN32_DATA:
-         stream = STREAM_WIN32_GZIP_DATA;
-         break;
-      case STREAM_SPARSE_DATA:
-         stream = STREAM_SPARSE_GZIP_DATA;
-         break;
-      case STREAM_FILE_DATA:
-         stream = STREAM_GZIP_DATA;
-         break;
-      default:
-         /**
-          * All stream types that do not support gzip should clear out
-          * FO_GZIP above, and this code block should be unreachable.
-          */
-         ASSERT(!(ff_pkt->flags & FO_GZIP));
-         return STREAM_NONE;
-      }
+#if defined(HAVE_LIBZ) || defined(HAVE_LZO)
+   if (ff_pkt->flags & FO_COMPRESS) {
+      #ifdef HAVE_LIBZ
+         if(ff_pkt->Compress_algo == COMPRESS_GZIP) {
+            switch (stream) {
+            case STREAM_WIN32_DATA:
+                  stream = STREAM_WIN32_GZIP_DATA;
+               break;
+            case STREAM_SPARSE_DATA:
+                  stream = STREAM_SPARSE_GZIP_DATA;
+               break;
+            case STREAM_FILE_DATA:
+                  stream = STREAM_GZIP_DATA;
+               break;
+            default:
+               /**
+                * All stream types that do not support compression should clear out
+                * FO_COMPRESS above, and this code block should be unreachable.
+                */
+               ASSERT(!(ff_pkt->flags & FO_COMPRESS));
+               return STREAM_NONE;
+            }
+         }
+      #endif
+      #ifdef HAVE_LZO
+         if(ff_pkt->Compress_algo == COMPRESS_LZO1X) {
+            switch (stream) {
+            case STREAM_WIN32_DATA:
+                  stream = STREAM_WIN32_COMPRESSED_DATA;
+               break;
+            case STREAM_SPARSE_DATA:
+                  stream = STREAM_SPARSE_COMPRESSED_DATA;
+               break;
+            case STREAM_FILE_DATA:
+                  stream = STREAM_COMPRESSED_DATA;
+               break;
+            default:
+               /**
+                * All stream types that do not support compression should clear out
+                * FO_COMPRESS above, and this code block should be unreachable.
+                */
+               ASSERT(!(ff_pkt->flags & FO_COMPRESS));
+               return STREAM_NONE;
+            }
+         }
+      #endif
    }
 #endif
 #ifdef HAVE_CRYPTO
@@ -140,12 +167,18 @@ int select_data_stream(FF_PKT *ff_pkt)
       case STREAM_WIN32_GZIP_DATA:
          stream = STREAM_ENCRYPTED_WIN32_GZIP_DATA;
          break;
+      case STREAM_WIN32_COMPRESSED_DATA:
+         stream = STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA;
+         break;
       case STREAM_FILE_DATA:
          stream = STREAM_ENCRYPTED_FILE_DATA;
          break;
       case STREAM_GZIP_DATA:
          stream = STREAM_ENCRYPTED_FILE_GZIP_DATA;
          break;
+      case STREAM_COMPRESSED_DATA:
+         stream = STREAM_ENCRYPTED_FILE_COMPRESSED_DATA;
+         break;
       default:
          /* All stream types that do not support encryption should clear out
           * FO_ENCRYPT above, and this code block should be unreachable. */
@@ -423,7 +456,8 @@ bool set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd)
        return true;
    }
    if (attr->data_stream == STREAM_WIN32_DATA ||
-       attr->data_stream == STREAM_WIN32_GZIP_DATA) {
+       attr->data_stream == STREAM_WIN32_GZIP_DATA ||
+       attr->data_stream == STREAM_WIN32_COMPRESSED_DATA) {
       if (is_bopen(ofd)) {
          bclose(ofd);
       }

diff --git a/bacula/src/findlib/bfile.c b/bacula/src/findlib/bfile.c
index b67408d76225949d594d1756e715c350dc699953..fbcac7d562684d56fb7d1079fb21c879d5f7c003 100644 (file)
--- a/bacula/src/findlib/bfile.c
+++ b/bacula/src/findlib/bfile.c
@@ -79,8 +79,10 @@ bool is_win32_stream(int stream)
    switch (stream) {
    case STREAM_WIN32_DATA:
    case STREAM_WIN32_GZIP_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
    case STREAM_ENCRYPTED_WIN32_DATA:
    case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
       return true;
    }
    return false;
@@ -99,12 +101,16 @@ const char *stream_to_ascii(int stream)
       return _("MD5 digest");
    case STREAM_GZIP_DATA:
       return _("GZIP data");
+   case STREAM_COMPRESSED_DATA:
+      return _("Compressed data");
    case STREAM_UNIX_ATTRIBUTES_EX:
       return _("Extended attributes");
    case STREAM_SPARSE_DATA:
       return _("Sparse data");
    case STREAM_SPARSE_GZIP_DATA:
       return _("GZIP sparse data");
+   case STREAM_SPARSE_COMPRESSED_DATA:
+      return _("Compressed sparse data");
    case STREAM_PROGRAM_NAMES:
       return _("Program names");
    case STREAM_PROGRAM_DATA:
@@ -115,6 +121,8 @@ const char *stream_to_ascii(int stream)
       return _("Win32 data");
    case STREAM_WIN32_GZIP_DATA:
       return _("Win32 GZIP data");
+   case STREAM_WIN32_COMPRESSED_DATA:
+      return _("Win32 compressed data");
    case STREAM_MACOS_FORK_DATA:
       return _("MacOS Fork data");
    case STREAM_HFSPLUS_ATTRIBUTES:
@@ -137,8 +145,12 @@ const char *stream_to_ascii(int stream)
       return _("Encrypted session data");
    case STREAM_ENCRYPTED_FILE_GZIP_DATA:
       return _("Encrypted GZIP data");
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+      return _("Encrypted compressed data");
    case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
       return _("Encrypted Win32 GZIP data");
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
+      return _("Encrypted Win32 Compressed data");
    case STREAM_ENCRYPTED_MACOS_FORK_DATA:
       return _("Encrypted MacOS fork data");
    case STREAM_ACL_AIX_TEXT:
@@ -421,6 +433,13 @@ bool is_restore_stream_supported(int stream)
    case STREAM_GZIP_DATA:
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:
+#endif
+#ifndef HAVE_LZO
+   case STREAM_COMPRESSED_DATA:
+   case STREAM_SPARSE_COMPRESSED_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
 #endif
    case STREAM_MACOS_FORK_DATA:
    case STREAM_HFSPLUS_ATTRIBUTES:
@@ -432,6 +451,13 @@ bool is_restore_stream_supported(int stream)
    case STREAM_GZIP_DATA:
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:
+#endif
+#ifdef HAVE_LZO
+   case STREAM_COMPRESSED_DATA:
+   case STREAM_SPARSE_COMPRESSED_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
 #endif
    case STREAM_WIN32_DATA:
    case STREAM_UNIX_ATTRIBUTES:
@@ -452,6 +478,8 @@ bool is_restore_stream_supported(int stream)
    case STREAM_ENCRYPTED_FILE_GZIP_DATA:
    case STREAM_ENCRYPTED_WIN32_DATA:
    case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
 #endif
    case 0:                            /* compatibility with old tapes */
       return true;
@@ -862,6 +890,13 @@ bool is_restore_stream_supported(int stream)
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:    
 #endif
+#ifndef HAVE_LZO
+   case STREAM_COMPRESSED_DATA:
+   case STREAM_SPARSE_COMPRESSED_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
+#endif
 #ifndef HAVE_DARWIN_OS
    case STREAM_MACOS_FORK_DATA:
    case STREAM_HFSPLUS_ATTRIBUTES:
@@ -873,6 +908,13 @@ bool is_restore_stream_supported(int stream)
    case STREAM_GZIP_DATA:
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:    
+#endif
+#ifdef HAVE_LZO
+   case STREAM_COMPRESSED_DATA:
+   case STREAM_SPARSE_COMPRESSED_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
 #endif
    case STREAM_WIN32_DATA:
    case STREAM_UNIX_ATTRIBUTES:

diff --git a/bacula/src/findlib/find.c b/bacula/src/findlib/find.c
index 77ac196dabf57b33a94e31ff276227f6cf98666c..3098f615c1e0a8a7bd793c9c357bef0398267921 100644 (file)
--- a/bacula/src/findlib/find.c
+++ b/bacula/src/findlib/find.c
@@ -192,7 +192,8 @@ find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, bool to
          for (j=0; j<incexe->opts_list.size(); j++) {
             findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j);
             ff->flags |= fo->flags;
-            ff->GZIP_level = fo->GZIP_level;
+            ff->Compress_algo = fo->Compress_algo;
+            ff->Compress_level = fo->Compress_level;
             ff->strip_path = fo->strip_path;
             ff->fstypes = fo->fstype;
             ff->drivetypes = fo->drivetype;
@@ -300,7 +301,8 @@ static bool accept_file(FF_PKT *ff)
    for (j = 0; j < incexe->opts_list.size(); j++) {
       findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j);
       ff->flags = fo->flags;
-      ff->GZIP_level = fo->GZIP_level;
+      ff->Compress_algo = fo->Compress_algo;
+      ff->Compress_level = fo->Compress_level;
       ff->fstypes = fo->fstype;
       ff->drivetypes = fo->drivetype;
 

diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h
index 804eeaec2b236c558a6943ff391e5374e7a245d9..cda9e9b016ab58be990e8ede36e69047b637aa62 100644 (file)
--- a/bacula/src/findlib/find.h
+++ b/bacula/src/findlib/find.h
@@ -77,6 +77,7 @@ int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result);
 struct s_included_file {
    struct s_included_file *next;
    uint32_t options;                  /* backup options */
+   uint32_t algo;                     /* compression algorithm. 4 letters stored as an interger */
    int level;                         /* compression level */
    int len;                           /* length of fname */
    int pattern;                       /* set if wild card pattern */
@@ -108,7 +109,8 @@ enum {
 /* File options structure */
 struct findFOPTS {
    uint32_t flags;                    /* options in bits */
-   int GZIP_level;                    /* GZIP level */
+   uint32_t Compress_algo;            /* compression algorithm. 4 letters stored as an interger */
+   int Compress_level;                /* compression level */
    int strip_path;                    /* strip path count */
    char VerifyOpts[MAX_FOPTS];        /* verify options */
    char AccurateOpts[MAX_FOPTS];      /* accurate mode options */
@@ -196,7 +198,8 @@ struct FF_PKT {
 
    /* Values set by accept_file while processing Options */
    uint32_t flags;                    /* backup options */
-   int GZIP_level;                    /* compression level */
+   uint32_t Compress_algo;            /* compression algorithm. 4 letters stored as an interger */
+   int Compress_level;                /* compression level */
    int strip_path;                    /* strip path count */
    bool cmd_plugin;                   /* set if we have a command plugin */
    alist fstypes;                     /* allowed file system types */

diff --git a/bacula/src/findlib/match.c b/bacula/src/findlib/match.c
index 3e84e33526a63a2047f5110ceae0e490765f3151..beffad50bc01ec3259fd9150c4120ae12a0844ae 100644 (file)
--- a/bacula/src/findlib/match.c
+++ b/bacula/src/findlib/match.c
@@ -42,6 +42,7 @@
 
 #include "bacula.h"
 #include "find.h"
+#include "ch.h"
 
 #include <sys/types.h>
 
@@ -185,10 +186,19 @@ void add_fname_to_include_list(FF_PKT *ff, int prefixed, const char *fname)
          case 'A':
             inc->options |= FO_ACL;
             break;
-         case 'Z':                 /* gzip compression */
-            inc->options |= FO_GZIP;
-            inc->level = *++rp - '0';
-            Dmsg1(200, "Compression level=%d\n", inc->level);
+         case 'Z':                 /* compression */
+            rp++;                   /* skip Z */
+            if (*rp >= '0' && *rp <= '9') {
+               inc->options |= FO_COMPRESS;
+               inc->algo = COMPRESS_GZIP;
+               inc->level = *rp - '0';
+            }
+            else if (*rp == 'o') {
+               inc->options |= FO_COMPRESS;
+               inc->algo = COMPRESS_LZO1X;
+               inc->level = 1; /* not used with LZO */
+            }
+            Dmsg2(200, "Compression alg=%d level=%d\n", inc->algo, inc->level);
             break;
          case 'K':
             inc->options |= FO_NOATIME;
@@ -246,8 +256,8 @@ void add_fname_to_include_list(FF_PKT *ff, int prefixed, const char *fname)
          { }
       next->next = inc;
    }
-   Dmsg3(100, "add_fname_to_include prefix=%d gzip=%d fname=%s\n", 
-         prefixed, !!(inc->options & FO_GZIP), inc->fname);
+   Dmsg4(100, "add_fname_to_include prefix=%d compres=%d alg= %d fname=%s\n", 
+         prefixed, !!(inc->options & FO_COMPRESS), inc->algo, inc->fname);
 }
 
 /*
@@ -302,7 +312,8 @@ struct s_included_file *get_next_included_file(FF_PKT *ff, struct s_included_fil
     */
    if (inc) {
       ff->flags = inc->options;
-      ff->GZIP_level = inc->level;
+      ff->Compress_algo = inc->algo;
+      ff->Compress_level = inc->level;
    }
    return inc;
 }

diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h
index f29d6fd1726440ec6474c94dc5212bc4b7dc9c08..cf880a342d883f1dee6462ae56b811f1a8f52114 100644 (file)
--- a/bacula/src/jcr.h
+++ b/bacula/src/jcr.h
@@ -382,6 +382,7 @@ public:
    POOLMEM *compress_buf;             /* Compression buffer */
    int32_t compress_buf_size;         /* Length of compression buffer */
    void *pZLIB_compress_workset;      /* zlib compression session data */
+   void *LZO_compress_workset;        /* lzo compression session data */
    int32_t replace;                   /* Replace options */
    int32_t buf_size;                  /* length of buffer */
    FF_PKT *ff;                        /* Find Files packet */

diff --git a/bacula/src/stored/Makefile.in b/bacula/src/stored/Makefile.in
index 849217c5acb91cb73588cab0b778b5aebcc0c1f4..e3af8058558ee552dc8984f59e90d11e13ab6dd5 100644 (file)
--- a/bacula/src/stored/Makefile.in
+++ b/bacula/src/stored/Makefile.in
@@ -74,6 +74,7 @@ EXTRAOBJS = @OBJLIST@
 
 CAP_LIBS = @CAP_LIBS@
 ZLIBS=@ZLIBS@
+LZOLIBS = @LZOLIBS@
 
 
 .SUFFIXES:     .c .o
@@ -128,7 +129,7 @@ bextract.o: bextract.c
 
 bextract: Makefile $(BEXTOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE)
        @echo "Compiling $<"
-       $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) \
+       $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZOLIBS) \
           -lbacfind -lbaccfg -lbac -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS)
 
 bscan.o: bscan.c

diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c
index 2e2a44919d59924ef1c6d3546e0dec3a02541215..d28f0f0a6aa7905c418dd8f38cd114f95677b22e 100644 (file)
--- a/bacula/src/stored/bextract.c
+++ b/bacula/src/stored/bextract.c
@@ -35,6 +35,7 @@
 
 #include "bacula.h"
 #include "stored.h"
+#include "ch.h"
 #include "findlib/find.h"
 
 extern bool parse_sd_config(CONFIG *config, const char *configfile, int exit_code);
@@ -465,6 +466,102 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
 #endif
       break;
 
+   /* Compressed data stream */
+   case STREAM_COMPRESSED_DATA:
+   case STREAM_SPARSE_COMPRESSED_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
+      if (extract) {
+         uint32_t comp_magic, comp_len;
+         uint16_t comp_level, comp_version;
+#ifdef HAVE_LZO
+         lzo_uint compress_len;
+         const unsigned char *cbuf;
+         int r, real_compress_len;
+#endif
+
+         if (rec->maskedStream == STREAM_SPARSE_COMPRESSED_DATA) {
+            ser_declare;
+            uint64_t faddr;
+            char ec1[50];
+            wbuf = rec->data + OFFSET_FADDR_SIZE;
+            wsize = rec->data_len - OFFSET_FADDR_SIZE;
+            ser_begin(rec->data, OFFSET_FADDR_SIZE);
+            unser_uint64(faddr);
+            if (fileAddr != faddr) {
+               fileAddr = faddr;
+               if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) {
+                  berrno be;
+                  Emsg3(M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"),
+                     edit_uint64(fileAddr, ec1), attr->ofname, be.bstrerror());
+                  extract = false;
+                  return true;
+               }
+            }
+         } else {
+            wbuf = rec->data;
+            wsize = rec->data_len;
+         }
+
+         /* read compress header */
+         unser_declare;
+         unser_begin(wbuf, sizeof(comp_stream_header));
+         unser_uint32(comp_magic);
+         unser_uint32(comp_len);
+         unser_uint16(comp_level);
+         unser_uint16(comp_version);
+         Dmsg4(200, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len,
+                                 comp_level, comp_version);
+
+         /* version check */
+         if (comp_version != COMP_HEAD_VERSION) {
+            Emsg1(M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version);
+            return false;
+         }
+         /* size check */
+         if (comp_len + sizeof(comp_stream_header) != wsize) {
+            Emsg2(M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"),
+                 comp_len, wsize);
+            return false;
+         }
+
+          switch(comp_magic) {
+#ifdef HAVE_LZO
+            case COMPRESS_LZO1X:
+               compress_len = compress_buf_size;
+               cbuf = (const unsigned char*) wbuf + sizeof(comp_stream_header);
+               real_compress_len = wsize - sizeof(comp_stream_header);
+               Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, wsize);
+               while ((r=lzo1x_decompress_safe(cbuf, real_compress_len,
+                                               (unsigned char *)compress_buf, &compress_len, NULL)) == LZO_E_OUTPUT_OVERRUN)
+               {
+
+                  /* The buffer size is too small, try with a bigger one */
+                  compress_len = 2 * compress_len;
+                  compress_buf = check_pool_memory_size(compress_buf,
+                                                  compress_len);
+               }
+               if (r != LZO_E_OK) {
+                  Emsg1(M_ERROR, 0, _("LZO uncompression error. ERR=%d\n"), r);
+                  extract = false;
+                  return true;
+               }
+               break;
+#endif
+            default:
+               Emsg1(M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic);
+               extract = false;
+               return true;
+         }
+
+         Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total);
+         store_data(&bfd, compress_buf, compress_len);
+         total += compress_len;
+         fileAddr += compress_len;
+         Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len,
+            compress_len);
+      }
+      break;
+
    case STREAM_MD5_DIGEST:
    case STREAM_SHA1_DIGEST:
    case STREAM_SHA256_DIGEST:

diff --git a/bacula/src/stored/bscan.c b/bacula/src/stored/bscan.c
index e987feb0b49021140bd13cfb1cf706628741a731..648dd9d37077979733cde2fd48406acfaafdd408 100644 (file)
--- a/bacula/src/stored/bscan.c
+++ b/bacula/src/stored/bscan.c
@@ -734,8 +734,11 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
       break;
 
    case STREAM_GZIP_DATA:
+   case STREAM_COMPRESSED_DATA:
    case STREAM_ENCRYPTED_FILE_GZIP_DATA:
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
    case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
       /* No correct, we should (decrypt and) expand it 
          done using JCR 
       */
@@ -744,12 +747,14 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
       break;
 
    case STREAM_SPARSE_GZIP_DATA:
+   case STREAM_SPARSE_COMPRESSED_DATA:
       mjcr->JobBytes += rec->data_len - sizeof(uint64_t); /* No correct, we should expand it */
       free_jcr(mjcr);                 /* done using JCR */
       break;
 
    /* Win32 GZIP stream */
    case STREAM_WIN32_GZIP_DATA:
+   case STREAM_WIN32_COMPRESSED_DATA:
       mjcr->JobBytes += rec->data_len;
       free_jcr(mjcr);                 /* done using JCR */
       break;

diff --git a/bacula/src/stored/record.c b/bacula/src/stored/record.c
index 430e2690afbeb02fae8df75d7688ff1af96036b3..32f09a91004914e389692fcc1764d09429ad36a8 100644 (file)
--- a/bacula/src/stored/record.c
+++ b/bacula/src/stored/record.c
@@ -109,12 +109,16 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
          return "contWIN32-DATA";
       case STREAM_WIN32_GZIP_DATA:
          return "contWIN32-GZIP";
+      case STREAM_WIN32_COMPRESSED_DATA:
+         return "contWIN32-COMPRESSED";
       case STREAM_MD5_DIGEST:
          return "contMD5";
       case STREAM_SHA1_DIGEST:
          return "contSHA1";
       case STREAM_GZIP_DATA:
          return "contGZIP";
+      case STREAM_COMPRESSED_DATA:
+         return "contCOMPRESSED";
       case STREAM_UNIX_ATTRIBUTES_EX:
          return "contUNIX-ATTR-EX";
       case STREAM_RESTORE_OBJECT:
@@ -123,6 +127,8 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
          return "contSPARSE-DATA";
       case STREAM_SPARSE_GZIP_DATA:
          return "contSPARSE-GZIP";
+      case STREAM_SPARSE_COMPRESSED_DATA:
+         return "contSPARSE-COMPRESSED";
       case STREAM_PROGRAM_NAMES:
          return "contPROG-NAMES";
       case STREAM_PROGRAM_DATA:
@@ -143,10 +149,14 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
          return "contENCRYPTED-FILE";
       case STREAM_ENCRYPTED_FILE_GZIP_DATA:
          return "contENCRYPTED-GZIP";
+      case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+         return "contENCRYPTED-COMPRESSED";
       case STREAM_ENCRYPTED_WIN32_DATA:
          return "contENCRYPTED-WIN32-DATA";
       case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
          return "contENCRYPTED-WIN32-GZIP";
+      case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
+         return "contENCRYPTED-WIN32-COMPRESSED";
       case STREAM_ENCRYPTED_MACOS_FORK_DATA:
          return "contENCRYPTED-MACOS-RSRC";
       case STREAM_PLUGIN_NAME:
@@ -167,12 +177,16 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
       return "WIN32-DATA";
    case STREAM_WIN32_GZIP_DATA:
       return "WIN32-GZIP";
+   case STREAM_WIN32_COMPRESSED_DATA:
+      return "WIN32-COMPRESSED";
    case STREAM_MD5_DIGEST:
       return "MD5";
    case STREAM_SHA1_DIGEST:
       return "SHA1";
    case STREAM_GZIP_DATA:
       return "GZIP";
+   case STREAM_COMPRESSED_DATA:
+      return "COMPRESSED";
    case STREAM_UNIX_ATTRIBUTES_EX:
       return "UNIX-ATTR-EX";
    case STREAM_RESTORE_OBJECT:
@@ -181,6 +195,8 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
       return "SPARSE-DATA";
    case STREAM_SPARSE_GZIP_DATA:
       return "SPARSE-GZIP";
+   case STREAM_SPARSE_COMPRESSED_DATA:
+      return "SPARSE-COMPRESSED";
    case STREAM_PROGRAM_NAMES:
       return "PROG-NAMES";
    case STREAM_PROGRAM_DATA:
@@ -203,10 +219,14 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
       return "ENCRYPTED-FILE";
    case STREAM_ENCRYPTED_FILE_GZIP_DATA:
       return "ENCRYPTED-GZIP";
+   case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
+      return "ENCRYPTED-COMPRESSED";
    case STREAM_ENCRYPTED_WIN32_DATA:
       return "ENCRYPTED-WIN32-DATA";
    case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
       return "ENCRYPTED-WIN32-GZIP";
+   case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
+      return "ENCRYPTED-WIN32-COMPRESSED";
    case STREAM_ENCRYPTED_MACOS_FORK_DATA:
       return "ENCRYPTED-MACOS-RSRC";
 

diff --git a/bacula/src/stored/stored.h b/bacula/src/stored/stored.h
index 3e1f967752290314f81c1ba7fc67b548b78ea6ed..a15a36a73ee2e154fc6bb6a24c69d54d766e547a 100644 (file)
--- a/bacula/src/stored/stored.h
+++ b/bacula/src/stored/stored.h
@@ -72,6 +72,10 @@ const int sd_dbglvl = 300;
 #else
 #define uLongf uint32_t
 #endif
+#ifdef HAVE_LZO
+#include <lzo/lzoconf.h>
+#include <lzo/lzo1x.h>
+#endif
 #ifdef HAVE_FNMATCH
 #include <fnmatch.h>
 #else

diff --git a/bacula/src/streams.h b/bacula/src/streams.h
index 0a1bbfce5ba0ab686e9686bb841eede35bb17143..1fc8c829cb66297545a78fc69bea880cf8f4be3c 100644 (file)
--- a/bacula/src/streams.h
+++ b/bacula/src/streams.h
@@ -97,6 +97,15 @@
 #define STREAM_PLUGIN_NAME                 26    /* Plugin "file" string */
 #define STREAM_PLUGIN_DATA                 27    /* Plugin specific data */
 #define STREAM_RESTORE_OBJECT              28    /* Plugin restore object */
+/* Non GZip compressed streams. Those streams can handle arbitrary compression algorithm data
+ * as an additional header is stored at the beginning of the stream.
+ * see stream_compressed_header definition for more details.
+ */
+#define STREAM_COMPRESSED_DATA                 29    /* Compressed file data */
+#define STREAM_SPARSE_COMPRESSED_DATA          30    /* Sparse compressed data stream */
+#define STREAM_WIN32_COMPRESSED_DATA           31    /* Compressed Win32 BackupRead data */
+#define STREAM_ENCRYPTED_FILE_COMPRESSED_DATA  32    /* Encrypted, compressed data */
+#define STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA 33    /* Encrypted, compressed Win32 BackupRead data */
 
 /**
  * Additional Stream definitions. Once defined these must NEVER

diff --git a/bacula/src/tools/testfind.c b/bacula/src/tools/testfind.c
index eb8f10c14cf56c9a327bead79ee493bee6f1f922..15e4ac67f5114593a0dbefce1be2dc963e0f189b 100644 (file)
--- a/bacula/src/tools/testfind.c
+++ b/bacula/src/tools/testfind.c
@@ -35,6 +35,7 @@
 #include "bacula.h"
 #include "dird/dird.h"
 #include "findlib/find.h"
+#include "ch.h"
 
 #if defined(HAVE_WIN32)
 #define isatty(fd) (fd==0)
@@ -629,10 +630,19 @@ static void set_options(findFOPTS *fo, const char *opts)
       case 'W':
          fo->flags |= FO_ENHANCEDWILD;
          break;
-      case 'Z':                 /* gzip compression */
-         fo->flags |= FO_GZIP;
-         fo->GZIP_level = *++p - '0';
-         Dmsg1(200, "Compression level=%d\n", fo->GZIP_level);
+      case 'Z':                 /* compression */
+         p++;                   /* skip Z */
+         if (*p >= '0' && *p <= '9') {
+            fo->flags |= FO_COMPRESS;
+            fo->Compress_algo = COMPRESS_GZIP;
+            fo->Compress_level = *p - '0';
+         }
+         else if (*p == 'o') {
+            fo->flags |= FO_COMPRESS;
+            fo->Compress_algo = COMPRESS_LZO1X;
+            fo->Compress_level = 1; /* not used with LZO */
+         }
+         Dmsg2(200, "Compression alg=%d level=%d\n", fo->Compress_algo, fo->Compress_level);
          break;
       case 'X':
          fo->flags |= FO_XATTR;

diff --git a/regress/all-disk-tests b/regress/all-disk-tests
index 5662bc7f11d67f8d9a993fdb8736ec5e2a3302cd..8de402436289d485b14f88731eb316b3d7493c66 100755 (executable)
--- a/regress/all-disk-tests
+++ b/regress/all-disk-tests
@@ -21,7 +21,9 @@ nice tests/bscan-test
 nice tests/bsr-opt-test
 nice tests/comment-test
 nice tests/compressed-test
+nice tests/lzo-test
 nice tests/compress-encrypt-test
+nice tests/lzo-encrypt-test
 nice tests/concurrent-jobs-test
 nice tests/copy-job-test
 nice tests/copy-jobspan-test
@@ -64,6 +66,7 @@ nice tests/maxvol2-test
 nice tests/messages-test
 nice tests/next-vol-test
 nice tests/sparse-compressed-test
+nice tests/sparse-lzo-test
 nice tests/sparse-test
 nice tests/strip-test
 nice tests/two-jobs-test

diff --git a/regress/scripts/new-test-bacula-dir.conf.in b/regress/scripts/new-test-bacula-dir.conf.in
index 02e107c192f6bac0afe33534960c0d882730fb8e..534324cca395a4aefd7d4b983ad219a6daa599bd 100644 (file)
--- a/regress/scripts/new-test-bacula-dir.conf.in
+++ b/regress/scripts/new-test-bacula-dir.conf.in
@@ -131,6 +131,34 @@ Job {
   Maximum Concurrent Jobs = 10
 }
 
+Job {
+  Name = "LZOTest"
+  Type = Backup
+  Client=@hostname@-fd
+  FileSet="LZOSet"
+  Storage = File
+  Messages = Standard
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+  Write Bootstrap = "@working_dir@/NightlySave.bsr"
+  Max Run Time = 30min
+  SpoolData=yes
+}
+
+Job {
+  Name = "SparseLZOTest"
+  Type = Backup
+  Client=@hostname@-fd
+  FileSet="SparseLZOSet"
+  Storage = File
+  Messages = Standard
+  Pool = Default
+  Write Bootstrap = "@working_dir@/NightlySave.bsr"
+  Max Run Time = 30min
+  SpoolData=yes
+  Maximum Concurrent Jobs = 10
+}
+
 Job {
   Name = "FIFOTest"
   Type = Backup
@@ -238,6 +266,17 @@ FileSet {
   }
 }
 
+FileSet {
+  Name = "LZOSet"
+  Include {
+    Options {
+      signature=MD5
+      compression=LZO
+    }
+    File = <@tmpdir@/file-list
+  }
+}
+
 FileSet {
   Name = "FIFOSet"
   Include {
@@ -262,6 +301,18 @@ FileSet {
   }
 }
 
+FileSet {
+  Name = "SparseLZOSet"
+  Include {
+    Options {
+      signature=MD5
+      compression=LZO
+      sparse=yes
+    }
+    File = <@tmpdir@/file-list
+  }
+}
+
 FileSet {
   Name = "MonsterFileSet"
   Include {

diff --git a/regress/tests/lzo-encrypt-test b/regress/tests/lzo-encrypt-test
new file mode 100755 (executable)
index 0000000..d241471
--- /dev/null
+++ b/regress/tests/lzo-encrypt-test
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Run a simple backup with encryption and compression of the Bacula build directory
+#   then verify the signatures.
+#
+TestName="lzo-encrypt-test"
+JobName=LZOTest
+. scripts/functions
+
+scripts/cleanup
+scripts/copy-crypto-confs
+echo "${cwd}/build" >${cwd}/tmp/file-list
+
+start_test
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+label storage=File volume=TestVolume001
+run job=$JobName yes
+wait
+messages
+list volumes
+@# 
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out
+@# setdebug level=0 fd
+restore where=${cwd}/tmp/bacula-restores storage=File
+5
+mark *
+done
+yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+sleep 2
+check_for_zombie_jobs storage=File 
+stop_bacula
+
+check_two_logs
+check_restore_diff
+end_test

diff --git a/regress/tests/lzo-test b/regress/tests/lzo-test
new file mode 100755 (executable)
index 0000000..96a8dab
--- /dev/null
+++ b/regress/tests/lzo-test
@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# Run a simple backup of the Bacula build directory using the compressed option
+#   then restore it.
+#
+TestName="lzo-test"
+JobName=lzo
+. scripts/functions
+
+scripts/cleanup
+scripts/copy-test-confs
+echo "${cwd}/build" >${cwd}/tmp/file-list
+
+start_test
+      
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+status all
+status all
+messages
+label storage=File volume=TestVolume001
+run job=LZOTest storage=File yes
+wait
+messages
+@# 
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out
+restore where=${cwd}/tmp/bacula-restores select storage=File
+unmark *
+mark *
+done
+yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+check_for_zombie_jobs storage=File
+stop_bacula
+
+check_two_logs
+check_restore_diff
+grep " Software Compression" ${cwd}/tmp/log1.out | grep "%" 2>&1 1>/dev/null
+if [ $? != 0 ] ; then
+   echo "  !!!!! No compression !!!!!"
+   bstat=1
+fi
+end_test

diff --git a/regress/tests/sparse-lzo-test b/regress/tests/sparse-lzo-test
new file mode 100755 (executable)
index 0000000..343f093
--- /dev/null
+++ b/regress/tests/sparse-lzo-test
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Run a simple backup of the Bacula build directory using the Sparse option
+#   then restore it.
+#
+TestName="sparse-lzo-test"
+JobName=Sparse-lzo
+. scripts/functions
+
+cwd=`pwd`
+scripts/cleanup
+scripts/copy-test-confs
+echo "${cwd}/build" >${cwd}/tmp/file-list
+
+start_test
+
+cat >${cwd}/tmp/bconcmds <<END_OF_DATA
+@$out /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+label storage=File volume=TestVolume001
+run job=SparseLZOTest yes
+wait
+messages
+@# 
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out   
+restore where=${cwd}/tmp/bacula-restores select all storage=File done
+yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+check_for_zombie_jobs storage=File 
+stop_bacula
+
+check_two_logs
+check_restore_diff
+end_test