From 72d6651cc32251ca5b49c6305c8206c348723c53 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sun, 3 Jun 2007 20:27:52 +0000 Subject: [PATCH] kes Implement --with-batch-insert in configure and detection of thread safe libraries. It is on by default, but turned off if no thread safe library (e.g. SQLite 2) is available. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4971 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/autoconf/bacula-macros/db.m4 | 6 ++++ bacula/autoconf/config.h.in | 9 ++++-- bacula/autoconf/configure.in | 26 +++++++++++++++-- bacula/configure | 39 +++++++++++++++++++++++-- bacula/src/cats/make_sqlite3_tables.in | 4 +-- bacula/src/cats/sql.c | 4 +-- bacula/src/cats/sqlite.c | 40 ++++++-------------------- bacula/src/version.h | 21 ++++++++------ bacula/technotes-2.1 | 4 +++ 9 files changed, 99 insertions(+), 54 deletions(-) diff --git a/bacula/autoconf/bacula-macros/db.m4 b/bacula/autoconf/bacula-macros/db.m4 index b22810ea93..5907f6c18f 100644 --- a/bacula/autoconf/bacula-macros/db.m4 +++ b/bacula/autoconf/bacula-macros/db.m4 @@ -97,6 +97,7 @@ AC_ARG_WITH(mysql, AC_DEFINE(HAVE_THREAD_SAFE_MYSQL) fi SQL_BINDIR=$MYSQL_BINDIR + SQL_LIB=$MYSQL_LIBDIR/libmysqlclient_r.a AC_DEFINE(HAVE_MYSQL) AC_MSG_RESULT(yes) @@ -188,6 +189,7 @@ AC_ARG_WITH(embedded-mysql, SQL_INCLUDE=-I$MYSQL_INCDIR SQL_LFLAGS="-L$MYSQL_LIBDIR -lmysqld -lz -lm -lcrypt" SQL_BINDIR=$MYSQL_BINDIR + SQL_LIB=$MYSQL_LIBDIR/libmysqld.a AC_DEFINE(HAVE_MYSQL) AC_DEFINE(HAVE_EMBEDDED_MYSQL) @@ -272,6 +274,7 @@ AC_ARG_WITH(sqlite, SQL_INCLUDE=-I$SQLITE_INCDIR SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite" SQL_BINDIR=$SQLITE_BINDIR + SQL_LIB=$SQLITE_LIBDIR/libsqlite.a AC_DEFINE(HAVE_SQLITE) AC_MSG_RESULT(yes) @@ -353,6 +356,7 @@ AC_ARG_WITH(sqlite3, SQL_INCLUDE=-I$SQLITE_INCDIR SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite3" SQL_BINDIR=$SQLITE_BINDIR + SQL_LIB=$SQLITE_LIBDIR/libsqlite3.a AC_DEFINE(HAVE_SQLITE3) AC_MSG_RESULT(yes) @@ -451,6 +455,8 @@ AC_ARG_WITH(postgresql, SQL_INCLUDE=-I$POSTGRESQL_INCDIR SQL_LFLAGS=$POSTGRESQL_LFLAGS SQL_BINDIR=$POSTGRESQL_BINDIR + SQL_LIB=$POSTGRESQL_LIBDIR/libpq.a + AC_DEFINE(HAVE_POSTGRESQL) AC_MSG_RESULT(yes) db_found=yes diff --git a/bacula/autoconf/config.h.in b/bacula/autoconf/config.h.in index 5806d22ea1..fa55809f8b 100644 --- a/bacula/autoconf/config.h.in +++ b/bacula/autoconf/config.h.in @@ -229,9 +229,6 @@ /* Define if OPENSSL is available */ #undef HAVE_OPENSSL -/* Define if the OpenSSL library is export-constrained to 128bit ciphers */ -#undef HAVE_OPENSSL_EXPORT_LIBRARY - /* Define if comm encryption should be enabled */ #undef HAVE_TLS @@ -282,6 +279,9 @@ /* Set if Bacula bat Qt4 GUI support enabled */ #undef HAVE_BAT +/* Set if DB batch insert code enabled */ +#undef HAVE_BATCH_INSERT + /* Define to 1 if you have the MacOS X function CFLocaleCopyCurrent in the CoreFoundation framework. */ #undef HAVE_CFLOCALECOPYCURRENT @@ -506,6 +506,9 @@ /* Define if OpenSSL library is available */ #undef HAVE_OPENSSL +/* Define if the OpenSSL library is export-contrained to 128bit ciphers */ +#undef HAVE_OPENSSL_EXPORT_LIBRARY + /* Define to 1 if you have the `posix_fadvise' function. */ #undef HAVE_POSIX_FADVISE diff --git a/bacula/autoconf/configure.in b/bacula/autoconf/configure.in index 700a9f8922..2f5a58b74f 100644 --- a/bacula/autoconf/configure.in +++ b/bacula/autoconf/configure.in @@ -1231,9 +1231,6 @@ AC_ARG_WITH(fd_group, ] ) - - - AC_SUBST(dir_user) AC_SUBST(dir_group) AC_SUBST(sd_user) @@ -1259,6 +1256,7 @@ AC_SUBST(SBINPERM) # ------------------------------------------------ # Bacula check for various SQL database engines # ------------------------------------------------ +SQL_LIB= BA_CHECK_POSTGRESQL_DB BA_CHECK_MYSQL_DB @@ -1270,6 +1268,27 @@ BA_CHECK_SQLITE_DB AC_SUBST(cats) AC_SUBST(DB_TYPE) +# ------------------------------------------- +# enable batch attribute DB insert (default on) +# ------------------------------------------- +support_batch_insert=no +A=`nm $SQL_LIB | grep pthread_mutex_lock` +pkg=$? +if test $pkg = 0; then + support_batch_insert=yes + AC_ARG_ENABLE(batch-insert, + [ --enable-batch-insert enable the DB batch insert code [disabled]], + [if test x$enableval = xno; then + support_batch_insert=no + else + AC_DEFINE(HAVE_BATCH_INSERT, 1, [Set if DB batch insert code enabled]) + support_batch_insert=yes + fi]) +fi + + + + AC_DEFINE(PROTOTYPES) dnl# -------------------------------------------------------------------------- @@ -2278,6 +2297,7 @@ Configuration on `date`: build-stored: ${build_stored} ACL support: ${have_acl} Python support: ${support_python} ${PYTHON_LIBS} + Batch insert enabled: ${support_batch_insert} " > config.out diff --git a/bacula/configure b/bacula/configure index 00383b154f..db661d36a4 100755 --- a/bacula/configure +++ b/bacula/configure @@ -1449,6 +1449,7 @@ Optional Features: --disable-readline disable readline support disable + --enable-batch-insert enable the DB batch insert code disabled --disable-largefile omit support for large files Optional Packages: @@ -18743,9 +18744,6 @@ fi - - - # # allow setting default executable permissions # @@ -18767,6 +18765,7 @@ fi # ------------------------------------------------ # Bacula check for various SQL database engines # ------------------------------------------------ +SQL_LIB= db_found=no { echo "$as_me:$LINENO: checking for PostgreSQL support" >&5 @@ -19034,6 +19033,8 @@ fi SQL_INCLUDE=-I$POSTGRESQL_INCDIR SQL_LFLAGS=$POSTGRESQL_LFLAGS SQL_BINDIR=$POSTGRESQL_BINDIR + SQL_LIB=$POSTGRESQL_LIBDIR/libpq.a + cat >>confdefs.h <<\_ACEOF #define HAVE_POSTGRESQL 1 _ACEOF @@ -19168,6 +19169,7 @@ _ACEOF fi SQL_BINDIR=$MYSQL_BINDIR + SQL_LIB=$MYSQL_LIBDIR/libmysqlclient_r.a cat >>confdefs.h <<\_ACEOF #define HAVE_MYSQL 1 @@ -19270,6 +19272,7 @@ echo "$as_me: error: Invalid MySQL directory $withval - unable to find mysql.h u SQL_INCLUDE=-I$MYSQL_INCDIR SQL_LFLAGS="-L$MYSQL_LIBDIR -lmysqld -lz -lm -lcrypt" SQL_BINDIR=$MYSQL_BINDIR + SQL_LIB=$MYSQL_LIBDIR/libmysqld.a cat >>confdefs.h <<\_ACEOF #define HAVE_MYSQL 1 @@ -19367,6 +19370,7 @@ echo "$as_me: error: Invalid SQLite3 directory $withval - unable to find sqlite3 SQL_INCLUDE=-I$SQLITE_INCDIR SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite3" SQL_BINDIR=$SQLITE_BINDIR + SQL_LIB=$SQLITE_LIBDIR/libsqlite3.a cat >>confdefs.h <<\_ACEOF #define HAVE_SQLITE3 1 @@ -19462,6 +19466,7 @@ echo "$as_me: error: Invalid SQLite directory $withval - unable to find sqlite.h SQL_INCLUDE=-I$SQLITE_INCDIR SQL_LFLAGS="-L$SQLITE_LIBDIR -lsqlite" SQL_BINDIR=$SQLITE_BINDIR + SQL_LIB=$SQLITE_LIBDIR/libsqlite.a cat >>confdefs.h <<\_ACEOF #define HAVE_SQLITE 1 @@ -19495,6 +19500,33 @@ fi +# ------------------------------------------- +# enable batch attribute DB insert (default on) +# ------------------------------------------- +support_batch_insert=no +A=`nm $SQL_LIB | grep pthread_mutex_lock` +pkg=$? +if test $pkg = 0; then + support_batch_insert=yes + # Check whether --enable-batch-insert was given. +if test "${enable_batch_insert+set}" = set; then + enableval=$enable_batch_insert; if test x$enableval = xno; then + support_batch_insert=no + else + +cat >>confdefs.h <<\_ACEOF +#define HAVE_BATCH_INSERT 1 +_ACEOF + + support_batch_insert=yes + fi +fi + +fi + + + + cat >>confdefs.h <<\_ACEOF #define PROTOTYPES 1 _ACEOF @@ -35916,6 +35948,7 @@ Configuration on `date`: build-stored: ${build_stored} ACL support: ${have_acl} Python support: ${support_python} ${PYTHON_LIBS} + Batch insert enabled: ${support_batch_insert} " > config.out diff --git a/bacula/src/cats/make_sqlite3_tables.in b/bacula/src/cats/make_sqlite3_tables.in index 813cce2601..5634085568 100644 --- a/bacula/src/cats/make_sqlite3_tables.in +++ b/bacula/src/cats/make_sqlite3_tables.in @@ -353,8 +353,8 @@ INSERT INTO Status (JobStatus,JobStatusLong) VALUES INSERT INTO Version (VersionId) VALUES (10); -PRAGMA default_synchronous = OFF; -PRAGMA default_cache_size = 10000; +PRAGMA default_cache_size = 100000; +PRAGMA synchronous = NORMAL; END-OF-DATA diff --git a/bacula/src/cats/sql.c b/bacula/src/cats/sql.c index 908a70a394..c64eb84a1f 100644 --- a/bacula/src/cats/sql.c +++ b/bacula/src/cats/sql.c @@ -187,8 +187,8 @@ UpdateDB(const char *file, int line, JCR *jcr, B_DB *mdb, char *cmd) mdb->num_rows = sql_affected_rows(mdb); if (mdb->num_rows < 1) { char ed1[30]; - m_msg(file, line, &mdb->errmsg, _("Update problem: affected_rows=%s\n"), - edit_uint64(mdb->num_rows, ed1)); + m_msg(file, line, &mdb->errmsg, _("Update failed: affected_rows=%s for %s\n"), + edit_uint64(mdb->num_rows, ed1), cmd); if (verbose) { // j_msg(file, line, jcr, M_INFO, 0, "%s\n", cmd); } diff --git a/bacula/src/cats/sqlite.c b/bacula/src/cats/sqlite.c index ac6b2e62e2..ab4f5a8c2b 100644 --- a/bacula/src/cats/sqlite.c +++ b/bacula/src/cats/sqlite.c @@ -187,6 +187,9 @@ db_open_database(JCR *jcr, B_DB *mdb) } else { mdb->sqlite_errmsg = NULL; } +#ifdef SQLITE3_INIT_QUERY + db_sql_query(mdb, SQLITE3_INIT_QUERY, NULL, NULL); +#endif #else mdb->db = sqlite_open( @@ -254,7 +257,11 @@ db_close_database(JCR *jcr, B_DB *mdb) } void db_thread_cleanup() -{ } +{ +#ifdef HAVE_SQLITE3 + sqlite3_thread_cleanup(); +#endif +} /* * Return the next unique index (auto-increment) for @@ -262,37 +269,6 @@ void db_thread_cleanup() */ int db_next_index(JCR *jcr, B_DB *mdb, char *table, char *index) { -#ifdef xxxx - SQL_ROW row; - - db_lock(mdb); - - Mmsg(mdb->cmd, -"SELECT id FROM NextId WHERE TableName=\"%s\"", table); - if (!QUERY_DB(jcr, mdb, mdb->cmd)) { - Mmsg(mdb->errmsg, _("next_index query error: ERR=%s\n"), sql_strerror(mdb)); - db_unlock(mdb); - return 0; - } - if ((row = sql_fetch_row(mdb)) == NULL) { - Mmsg(mdb->errmsg, _("Error fetching index: ERR=%s\n"), sql_strerror(mdb)); - db_unlock(mdb); - return 0; - } - bstrncpy(index, row[0], 28); - sql_free_result(mdb); - - Mmsg(mdb->cmd, -"UPDATE NextId SET id=id+1 WHERE TableName=\"%s\"", table); - if (!QUERY_DB(jcr, mdb, mdb->cmd)) { - Mmsg(mdb->errmsg, _("next_index update error: ERR=%s\n"), sql_strerror(mdb)); - db_unlock(mdb); - return 0; - } - sql_free_result(mdb); - - db_unlock(mdb); -#endif strcpy(index, "NULL"); return 1; } diff --git a/bacula/src/version.h b/bacula/src/version.h index 18a1c472af..2a9154d9e0 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -4,8 +4,8 @@ #undef VERSION #define VERSION "2.1.11" -#define BDATE "02 June 2007" -#define LSMDATE "02Jun07" +#define BDATE "04 June 2007" +#define LSMDATE "04Jun07" #define PROG_COPYRIGHT "Copyright (C) %d-2007 Free Software Foundation Europe e.V.\n" #define BYEAR "2007" /* year for copyright messages in progs */ @@ -57,6 +57,16 @@ #define SMCHECK #endif +/* + * for fastest speed but you must have a UPS to avoid unwanted shutdowns + */ +#define SQLITE3_INIT_QUERY "PRAGMA synchronous = OFF" + +/* + * for more safety, but is 30 times slower than above + */ +//#define SQLITE3_INIT_QUERY "PRAGMA synchronous = NORMAL" + /* * This should always be on. It enables data encryption code * providing it is configured. @@ -70,13 +80,6 @@ */ #define USE_BSNPRINTF 1 -/* - * Turn on the following flag to enable batch attribute inserts - * in the catalog. This gives a large speedup. - */ -#define HAVE_BATCH_FILE_INSERT 1 - - /* Debug flags not normally turned on */ /* #define TRACE_JCR_CHAIN 1 */ diff --git a/bacula/technotes-2.1 b/bacula/technotes-2.1 index d7c58ee944..fa39585f01 100644 --- a/bacula/technotes-2.1 +++ b/bacula/technotes-2.1 @@ -1,6 +1,10 @@ Technical notes on version 2.1 General: +03Jun07 +kes Implement --with-batch-insert in configure and detection of thread + safe libraries. It is on by default, but turned off if no thread + safe library (e.g. SQLite 2) is available. 02Jun07 kes Fix Bacula PostgreSQL buffer overruns. kes Do better checking for NULL results returned from PostgreSQL; -- 2.39.2