kes Implement --with-batch-insert in configure and detection of thread

     safe libraries.  It is on by default, but turned off if no thread
     safe library (e.g. SQLite 2) is available.

git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4971 91ce42f0-d328-0410-95d8-f526ca767f89

diff --git a/bacula/autoconf/bacula-macros/db.m4 b/bacula/autoconf/bacula-macros/db.m4
index b22810ea939d9f97efd843cdabb0ea0a832efe66..5907f6c18f84e9a7e3c2638f37ced664496f6579 100644 (file)
--- a/bacula/autoconf/bacula-macros/db.m4
+++ b/bacula/autoconf/bacula-macros/db.m4
@@ -97,6 +97,7 @@ AC_ARG_WITH(mysql,
        AC_DEFINE(HAVE_THREAD_SAFE_MYSQL)
     fi
     SQL_BINDIR=$MYSQL_BINDIR
+    SQL_LIB=$MYSQL_LIBDIR/libmysqlclient_r.a
 
     AC_DEFINE(HAVE_MYSQL)
     AC_MSG_RESULT(yes)
@@ -188,6 +189,7 @@ AC_ARG_WITH(embedded-mysql,
     SQL_INCLUDE=-I$MYSQL_INCDIR
     SQL_LFLAGS="-L$MYSQL_LIBDIR -lmysqld -lz -lm -lcrypt"
     SQL_BINDIR=$MYSQL_BINDIR
+    SQL_LIB=$MYSQL_LIBDIR/libmysqld.a
 
     AC_DEFINE(HAVE_MYSQL)
     AC_DEFINE(HAVE_EMBEDDED_MYSQL)
@@ -272,6 +274,7 @@ AC_ARG_WITH(sqlite,
      SQL_INCLUDE=-I$SQLITE_INCDIR
      SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite"
      SQL_BINDIR=$SQLITE_BINDIR
+     SQL_LIB=$SQLITE_LIBDIR/libsqlite.a
 
      AC_DEFINE(HAVE_SQLITE)
      AC_MSG_RESULT(yes)
@@ -353,6 +356,7 @@ AC_ARG_WITH(sqlite3,
      SQL_INCLUDE=-I$SQLITE_INCDIR
      SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite3"
      SQL_BINDIR=$SQLITE_BINDIR
+     SQL_LIB=$SQLITE_LIBDIR/libsqlite3.a
 
      AC_DEFINE(HAVE_SQLITE3)
      AC_MSG_RESULT(yes)
@@ -451,6 +455,8 @@ AC_ARG_WITH(postgresql,
       SQL_INCLUDE=-I$POSTGRESQL_INCDIR
       SQL_LFLAGS=$POSTGRESQL_LFLAGS
       SQL_BINDIR=$POSTGRESQL_BINDIR
+      SQL_LIB=$POSTGRESQL_LIBDIR/libpq.a
+
       AC_DEFINE(HAVE_POSTGRESQL)
       AC_MSG_RESULT(yes)
       db_found=yes

diff --git a/bacula/autoconf/config.h.in b/bacula/autoconf/config.h.in
index 5806d22ea10639dba9523c99f9ea2998f7cfb34c..fa55809f8b2655c0c6253d0ac2d5bfba8866e25f 100644 (file)
--- a/bacula/autoconf/config.h.in
+++ b/bacula/autoconf/config.h.in
@@ -229,9 +229,6 @@
 /* Define if OPENSSL is available */
 #undef HAVE_OPENSSL
 
-/* Define if the OpenSSL library is export-constrained to 128bit ciphers */
-#undef HAVE_OPENSSL_EXPORT_LIBRARY
-
 /* Define if comm encryption should be enabled */
 #undef HAVE_TLS
 
@@ -282,6 +279,9 @@
 /* Set if Bacula bat Qt4 GUI support enabled */
 #undef HAVE_BAT
 
+/* Set if DB batch insert code enabled */
+#undef HAVE_BATCH_INSERT
+
 /* Define to 1 if you have the MacOS X function CFLocaleCopyCurrent in the
    CoreFoundation framework. */
 #undef HAVE_CFLOCALECOPYCURRENT
@@ -506,6 +506,9 @@
 /* Define if OpenSSL library is available */
 #undef HAVE_OPENSSL
 
+/* Define if the OpenSSL library is export-contrained to 128bit ciphers */
+#undef HAVE_OPENSSL_EXPORT_LIBRARY
+
 /* Define to 1 if you have the `posix_fadvise' function. */
 #undef HAVE_POSIX_FADVISE
 

diff --git a/bacula/autoconf/configure.in b/bacula/autoconf/configure.in
index 700a9f8922293eb5047df5e7ca22d7d9ad577048..2f5a58b74f41f25ce138155dae3532561f427014 100644 (file)
--- a/bacula/autoconf/configure.in
+++ b/bacula/autoconf/configure.in
@@ -1231,9 +1231,6 @@ AC_ARG_WITH(fd_group,
     ]
 )
 
-
-
-
 AC_SUBST(dir_user)
 AC_SUBST(dir_group)
 AC_SUBST(sd_user)
@@ -1259,6 +1256,7 @@ AC_SUBST(SBINPERM)
 # ------------------------------------------------
 # Bacula check for various SQL database engines
 # ------------------------------------------------
+SQL_LIB=
 BA_CHECK_POSTGRESQL_DB
 
 BA_CHECK_MYSQL_DB
@@ -1270,6 +1268,27 @@ BA_CHECK_SQLITE_DB
 AC_SUBST(cats)
 AC_SUBST(DB_TYPE)
 
+# -------------------------------------------
+# enable batch attribute DB insert (default on)
+# -------------------------------------------
+support_batch_insert=no
+A=`nm $SQL_LIB | grep pthread_mutex_lock`
+pkg=$?
+if test $pkg = 0; then
+   support_batch_insert=yes
+   AC_ARG_ENABLE(batch-insert,
+     [ --enable-batch-insert       enable the DB batch insert code [disabled]],
+     [if test x$enableval = xno; then
+       support_batch_insert=no
+      else
+       AC_DEFINE(HAVE_BATCH_INSERT, 1, [Set if DB batch insert code enabled]) 
+       support_batch_insert=yes
+     fi])
+fi
+                                                         
+
+
+
 AC_DEFINE(PROTOTYPES)
 
 dnl# --------------------------------------------------------------------------
@@ -2278,6 +2297,7 @@ Configuration on `date`:
   build-stored:              ${build_stored}
   ACL support:               ${have_acl}
   Python support:            ${support_python} ${PYTHON_LIBS}
+  Batch insert enabled:       ${support_batch_insert}
 
   " > config.out
 

diff --git a/bacula/configure b/bacula/configure
index 00383b154fe84b51e9b2838408eb551c57fd322c..db661d36a4ae78f7cb7da113cee9f875bd8c9f7a 100755 (executable)
--- a/bacula/configure
+++ b/bacula/configure
@@ -1449,6 +1449,7 @@ Optional Features:
 
   --disable-readline      disable readline support disable
 
+       --enable-batch-insert       enable the DB batch insert code disabled
   --disable-largefile     omit support for large files
 
 Optional Packages:
@@ -18743,9 +18744,6 @@ fi
 
 
 
-
-
-
 #
 # allow setting default executable permissions
 #
@@ -18767,6 +18765,7 @@ fi
 # ------------------------------------------------
 # Bacula check for various SQL database engines
 # ------------------------------------------------
+SQL_LIB=
 
 db_found=no
 { echo "$as_me:$LINENO: checking for PostgreSQL support" >&5
@@ -19034,6 +19033,8 @@ fi
       SQL_INCLUDE=-I$POSTGRESQL_INCDIR
       SQL_LFLAGS=$POSTGRESQL_LFLAGS
       SQL_BINDIR=$POSTGRESQL_BINDIR
+      SQL_LIB=$POSTGRESQL_LIBDIR/libpq.a
+
       cat >>confdefs.h <<\_ACEOF
 #define HAVE_POSTGRESQL 1
 _ACEOF
@@ -19168,6 +19169,7 @@ _ACEOF
 
     fi
     SQL_BINDIR=$MYSQL_BINDIR
+    SQL_LIB=$MYSQL_LIBDIR/libmysqlclient_r.a
 
     cat >>confdefs.h <<\_ACEOF
 #define HAVE_MYSQL 1
@@ -19270,6 +19272,7 @@ echo "$as_me: error: Invalid MySQL directory $withval - unable to find mysql.h u
     SQL_INCLUDE=-I$MYSQL_INCDIR
     SQL_LFLAGS="-L$MYSQL_LIBDIR -lmysqld -lz -lm -lcrypt"
     SQL_BINDIR=$MYSQL_BINDIR
+    SQL_LIB=$MYSQL_LIBDIR/libmysqld.a
 
     cat >>confdefs.h <<\_ACEOF
 #define HAVE_MYSQL 1
@@ -19367,6 +19370,7 @@ echo "$as_me: error: Invalid SQLite3 directory $withval - unable to find sqlite3
      SQL_INCLUDE=-I$SQLITE_INCDIR
      SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite3"
      SQL_BINDIR=$SQLITE_BINDIR
+     SQL_LIB=$SQLITE_LIBDIR/libsqlite3.a
 
      cat >>confdefs.h <<\_ACEOF
 #define HAVE_SQLITE3 1
@@ -19462,6 +19466,7 @@ echo "$as_me: error: Invalid SQLite directory $withval - unable to find sqlite.h
      SQL_INCLUDE=-I$SQLITE_INCDIR
      SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite"
      SQL_BINDIR=$SQLITE_BINDIR
+     SQL_LIB=$SQLITE_LIBDIR/libsqlite.a
 
      cat >>confdefs.h <<\_ACEOF
 #define HAVE_SQLITE 1
@@ -19495,6 +19500,33 @@ fi
 
 
 
+# -------------------------------------------
+# enable batch attribute DB insert (default on)
+# -------------------------------------------
+support_batch_insert=no
+A=`nm $SQL_LIB | grep pthread_mutex_lock`
+pkg=$?
+if test $pkg = 0; then
+   support_batch_insert=yes
+   # Check whether --enable-batch-insert was given.
+if test "${enable_batch_insert+set}" = set; then
+  enableval=$enable_batch_insert; if test x$enableval = xno; then
+       support_batch_insert=no
+      else
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_BATCH_INSERT 1
+_ACEOF
+
+       support_batch_insert=yes
+     fi
+fi
+
+fi
+
+
+
+
 cat >>confdefs.h <<\_ACEOF
 #define PROTOTYPES 1
 _ACEOF
@@ -35916,6 +35948,7 @@ Configuration on `date`:
   build-stored:              ${build_stored}
   ACL support:               ${have_acl}
   Python support:            ${support_python} ${PYTHON_LIBS}
+  Batch insert enabled:       ${support_batch_insert}
 
   " > config.out
 

diff --git a/bacula/src/cats/make_sqlite3_tables.in b/bacula/src/cats/make_sqlite3_tables.in
index 813cce2601d114f900803d4aa66ab14b4bcbe433..56340855683e15af8fd8dec85b38b56728bb76fb 100644 (file)
--- a/bacula/src/cats/make_sqlite3_tables.in
+++ b/bacula/src/cats/make_sqlite3_tables.in
@@ -353,8 +353,8 @@ INSERT INTO Status (JobStatus,JobStatusLong) VALUES
 INSERT INTO Version (VersionId) VALUES (10);
 
 
-PRAGMA default_synchronous = OFF;
-PRAGMA default_cache_size = 10000;
+PRAGMA default_cache_size = 100000;
+PRAGMA synchronous = NORMAL;
 
 END-OF-DATA
 

diff --git a/bacula/src/cats/sql.c b/bacula/src/cats/sql.c
index 908a70a394c154dfa00760f22d2488c84a231627..c64eb84a1f90906fe769c7915d1a2f309c567b9b 100644 (file)
--- a/bacula/src/cats/sql.c
+++ b/bacula/src/cats/sql.c
@@ -187,8 +187,8 @@ UpdateDB(const char *file, int line, JCR *jcr, B_DB *mdb, char *cmd)
    mdb->num_rows = sql_affected_rows(mdb);
    if (mdb->num_rows < 1) {
       char ed1[30];
-      m_msg(file, line, &mdb->errmsg, _("Update problem: affected_rows=%s\n"),
-         edit_uint64(mdb->num_rows, ed1));
+      m_msg(file, line, &mdb->errmsg, _("Update failed: affected_rows=%s for %s\n"),
+         edit_uint64(mdb->num_rows, ed1), cmd);
       if (verbose) {
 //       j_msg(file, line, jcr, M_INFO, 0, "%s\n", cmd);
       }

diff --git a/bacula/src/cats/sqlite.c b/bacula/src/cats/sqlite.c
index ac6b2e62e2cc9e1d605262e452d47cdfbb06832a..ab4f5a8c2b6d099aec250566b86a99d4cf51e746 100644 (file)
--- a/bacula/src/cats/sqlite.c
+++ b/bacula/src/cats/sqlite.c
@@ -187,6 +187,9 @@ db_open_database(JCR *jcr, B_DB *mdb)
    } else {
       mdb->sqlite_errmsg = NULL;
    }
+#ifdef SQLITE3_INIT_QUERY
+   db_sql_query(mdb, SQLITE3_INIT_QUERY, NULL, NULL);
+#endif
 
 #else
    mdb->db = sqlite_open(
@@ -254,7 +257,11 @@ db_close_database(JCR *jcr, B_DB *mdb)
 }
 
 void db_thread_cleanup()
-{ }
+{
+#ifdef HAVE_SQLITE3
+   sqlite3_thread_cleanup();
+#endif
+}
 
 /*
  * Return the next unique index (auto-increment) for
@@ -262,37 +269,6 @@ void db_thread_cleanup()
  */
 int db_next_index(JCR *jcr, B_DB *mdb, char *table, char *index)
 {
-#ifdef xxxx
-   SQL_ROW row;
-
-   db_lock(mdb);
-
-   Mmsg(mdb->cmd,
-"SELECT id FROM NextId WHERE TableName=\"%s\"", table);
-   if (!QUERY_DB(jcr, mdb, mdb->cmd)) {
-      Mmsg(mdb->errmsg, _("next_index query error: ERR=%s\n"), sql_strerror(mdb));
-      db_unlock(mdb);
-      return 0;
-   }
-   if ((row = sql_fetch_row(mdb)) == NULL) {
-      Mmsg(mdb->errmsg, _("Error fetching index: ERR=%s\n"), sql_strerror(mdb));
-      db_unlock(mdb);
-      return 0;
-   }
-   bstrncpy(index, row[0], 28);
-   sql_free_result(mdb);
-
-   Mmsg(mdb->cmd,
-"UPDATE NextId SET id=id+1 WHERE TableName=\"%s\"", table);
-   if (!QUERY_DB(jcr, mdb, mdb->cmd)) {
-      Mmsg(mdb->errmsg, _("next_index update error: ERR=%s\n"), sql_strerror(mdb));
-      db_unlock(mdb);
-      return 0;
-   }
-   sql_free_result(mdb);
-
-   db_unlock(mdb);
-#endif
    strcpy(index, "NULL");
    return 1;
 }

diff --git a/bacula/src/version.h b/bacula/src/version.h
index 18a1c472af50353221a5110d63d077e6c64d0b71..2a9154d9e0121d1602fa894d99e3fc8c964099a0 100644 (file)
--- a/bacula/src/version.h
+++ b/bacula/src/version.h
@@ -4,8 +4,8 @@
 
 #undef  VERSION
 #define VERSION "2.1.11"
-#define BDATE   "02 June 2007"
-#define LSMDATE "02Jun07"
+#define BDATE   "04 June 2007"
+#define LSMDATE "04Jun07"
 
 #define PROG_COPYRIGHT "Copyright (C) %d-2007 Free Software Foundation Europe e.V.\n"
 #define BYEAR "2007"       /* year for copyright messages in progs */
@@ -57,6 +57,16 @@
 #define SMCHECK
 #endif
 
+/* 
+ * for fastest speed but you must have a UPS to avoid unwanted shutdowns
+ */
+#define SQLITE3_INIT_QUERY "PRAGMA synchronous = OFF"
+
+/*
+ * for more safety, but is 30 times slower than above
+ */
+//#define SQLITE3_INIT_QUERY "PRAGMA synchronous = NORMAL"
+
 /*
  * This should always be on. It enables data encryption code 
  *  providing it is configured.
@@ -70,13 +80,6 @@
  */
 #define USE_BSNPRINTF 1
 
-/*
- * Turn on the following flag to enable batch attribute inserts
- *  in the catalog.  This gives a large speedup.
- */
-#define HAVE_BATCH_FILE_INSERT 1
-
-
 /* Debug flags not normally turned on */
 
 /* #define TRACE_JCR_CHAIN 1 */

diff --git a/bacula/src/win32/External-mingw32 b/bacula/src/win32/External-mingw32
index 35ed4ea6cd3858427347aae7a76d11dd23a9128f..be0dd50dfc3eac9b19fd74e2b9ab499c156e4d03 100644 (file)
--- a/bacula/src/win32/External-mingw32
+++ b/bacula/src/win32/External-mingw32
@@ -41,3 +41,4 @@ DVD_RW_TOOLS|http://fy.chalmers.se/~appro/linux/DVD+RW/tools/dvd+rw-tools-7.0.ta
 #CDRTOOLS|ftp://ftp.berlios.de/pub/cdrecord/alpha/cdrtools-2.01.01a22.tar.bz2
 MKISOFS|http://fy.chalmers.se/~appro/linux/DVD+RW/tools/win32/mkisofs.exe
 DVD_RW_TOOLS|http://fy.chalmers.se/~appro/linux/DVD+RW/tools/dvd+rw-tools-7.0.tar.gz
+Qt4|ftp://ftp.trolltech.com/qt/source/qt-win-opensource-src-4.3.0.zip

diff --git a/bacula/src/win32/build-depkgs-mingw32 b/bacula/src/win32/build-depkgs-mingw32
index 1453090dd2df5e49eedfd3a31622491f5159ea76..233a351b0155ca953e660af21afd594a499fdeaa 100755 (executable)
--- a/bacula/src/win32/build-depkgs-mingw32
+++ b/bacula/src/win32/build-depkgs-mingw32
@@ -1,4 +1,8 @@
 #!/bin/sh
+#
+#  This file is driven by the parameters that are defined in
+#    the file External-mingw32
+#
 
 usage()
 {
@@ -483,9 +487,15 @@ process_dvd_rw_tools()
                 do_patch dvd+rw-tools.patch
         fi
         echo Building dvd+rw-tools
-       m4 -DOS=XMINGW32 Makefile.m4 | make -f - dvd+rw-tools >make.log 2>&1
+        m4 -DOS=XMINGW32 Makefile.m4 | make -f - dvd+rw-tools >make.log 2>&1
         echo Installing dvd+rw-tools
-       m4 -DOS=XMINGW32 Makefile.m4 | make -f - prefix=${DEPPKG_DIR} install >>make.log 2>&1
+        m4 -DOS=XMINGW32 Makefile.m4 | make -f - prefix=${DEPPKG_DIR} install >>make.log 2>&1
+}
+
+process_qt4()
+{
+   get_source "${URL_Qt4}" "${DIR_Qt4}" "${MKD_Qt4}"
+
 }
 
 if [ "$#" -eq 0 ]
@@ -507,6 +517,7 @@ then
         process_cmd_utils
         process_mkisofs
         process_dvd_rw_tools
+        process_qt4
 else
         for dependency in "$@"
         do

diff --git a/bacula/technotes-2.1 b/bacula/technotes-2.1
index d7c58ee944b0ede99724fdac6b1ced4294870343..fa39585f01a7a92ae8c87a84c773ef851c0cf96b 100644 (file)
--- a/bacula/technotes-2.1
+++ b/bacula/technotes-2.1
@@ -1,6 +1,10 @@
               Technical notes on version 2.1
 
 General:
+03Jun07
+kes  Implement --with-batch-insert in configure and detection of thread
+     safe libraries.  It is on by default, but turned off if no thread
+     safe library (e.g. SQLite 2) is available.
 02Jun07
 kes  Fix Bacula PostgreSQL buffer overruns.
 kes  Do better checking for NULL results returned from PostgreSQL;