Try to use multi-row insert statements for mysql.

As it seems we could speedup the insert of the batch mode
somewhat by using multi-row insert statements. This implements
a very simple routine which batch up 32 inserts into one
query before pushing it to the database. The 32 is arbitrary
and just a test value for now we can always bump or lower it.

diff --git a/bacula/src/cats/bdb_mysql.h b/bacula/src/cats/bdb_mysql.h
index 6868708e4ede5d668965731d038b5f1389cae641..3eddf2477fea3dbf51be8b0dae2a5975a51c927d 100644 (file)
--- a/bacula/src/cats/bdb_mysql.h
+++ b/bacula/src/cats/bdb_mysql.h
@@ -29,6 +29,13 @@
 #ifndef __BDB_MYSQL_H_
 #define __BDB_MYSQL_H_ 1
 
+/*
+ * Number of insert statements to batch-up in batch insert
+ * mode. We use multi-row inserts only in the batch mode
+ * on the private database connection.
+ */
+#define MYSQL_CHANGES_PER_BATCH_INSERT 32
+
 class B_DB_MYSQL: public B_DB_PRIV {
 private:
    MYSQL *m_db_handle;

diff --git a/bacula/src/cats/mysql.c b/bacula/src/cats/mysql.c
index 513e4bb5f8f3ce8ee9beca98f61fd9c0a2d275d9..653973aacd5be93d7133b58465f855dff53ac024 100644 (file)
--- a/bacula/src/cats/mysql.c
+++ b/bacula/src/cats/mysql.c
@@ -588,6 +588,11 @@ bool B_DB_MYSQL::sql_batch_start(JCR *jcr)
                               "DeltaSeq integer)");
    db_unlock(this);
 
+   /*
+    * Keep track of the number of changes in batch mode.
+    */
+   changes = 0;
+
    return retval;
 }
 
@@ -600,6 +605,13 @@ bool B_DB_MYSQL::sql_batch_end(JCR *jcr, const char *error)
 {
    m_status = 0;
 
+   /*
+    * Flush any pending inserts.
+    */
+   if (changes) {
+      return sql_query(cmd);
+   }
+
    return true;
 }
 
@@ -624,12 +636,41 @@ bool B_DB_MYSQL::sql_batch_insert(JCR *jcr, ATTR_DBR *ar)
       digest = ar->Digest;
    }
 
-   Mmsg(cmd, "INSERT INTO batch VALUES "
-        "(%u,%s,'%s','%s','%s','%s',%u)",
-        ar->FileIndex, edit_int64(ar->JobId,ed1), esc_path,
-        esc_name, ar->attr, digest, ar->DeltaSeq);
+   /*
+    * Try to batch up multiple inserts using multi-row inserts.
+    */
+   if (changes == 0) {
+      Mmsg(cmd, "INSERT INTO batch VALUES "
+           "(%u,%s,'%s','%s','%s','%s',%u)",
+           ar->FileIndex, edit_int64(ar->JobId,ed1), esc_path,
+           esc_name, ar->attr, digest, ar->DeltaSeq);
+      changes++;
+   } else {
+      /*
+       * We use the esc_obj for temporary storage otherwise
+       * we keep on copying data.
+       */
+      Mmsg(esc_obj, ",(%u,%s,'%s','%s','%s','%s',%u)",
+           ar->FileIndex, edit_int64(ar->JobId,ed1), esc_path,
+           esc_name, ar->attr, digest, ar->DeltaSeq);
+      pm_strcat(cmd, esc_obj);
+      changes++;
+   }
 
-   return sql_query(cmd);
+   /*
+    * See if we need to flush the query buffer filled
+    * with multi-row inserts.
+    */
+   if ((changes % MYSQL_CHANGES_PER_BATCH_INSERT) == 0) {
+      if (!sql_query(cmd)) {
+         changes = 0;
+         return false;
+      } else {
+         changes = 0;
+      }
+   } else {
+      return true;
+   }
 }
 
 /*