Don't compute accurate list with MD5 if not used

During accurate queries, database engine have to deal with MD5 and
store them in memory or on disk. Just replace MD5 by space in the
query when we don't use them.

diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h
index 4eaf6f3f760a350585480f8a209d74d97e72219c..0a384bfc058bd73f5a8b2c5c4c51d72a24d027bc 100644 (file)
--- a/bacula/src/cats/protos.h
+++ b/bacula/src/cats/protos.h
@@ -104,7 +104,7 @@ bool db_find_failed_job_since(JCR *jcr, B_DB *mdb, JOB_DBR *jr, POOLMEM *stime,
 /* sql_get.c */
 bool db_get_volume_jobids(JCR *jcr, B_DB *mdb, 
                          MEDIA_DBR *mr, db_list_ctx *lst);
-bool db_get_base_file_list(JCR *jcr, B_DB *mdb,
+bool db_get_base_file_list(JCR *jcr, B_DB *mdb, bool use_md5,
                            DB_RESULT_HANDLER *result_handler,void *ctx);
 int db_get_path_record(JCR *jcr, B_DB *mdb);
 bool db_get_pool_record(JCR *jcr, B_DB *db, POOL_DBR *pdbr);
@@ -123,7 +123,7 @@ int db_get_job_volume_parameters(JCR *jcr, B_DB *mdb, JobId_t JobId, VOL_PARAMS
 int db_get_client_record(JCR *jcr, B_DB *mdb, CLIENT_DBR *cdbr);
 int db_get_counter_record(JCR *jcr, B_DB *mdb, COUNTER_DBR *cr);
 bool db_get_query_dbids(JCR *jcr, B_DB *mdb, POOL_MEM &query, dbid_list &ids);
-bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, DB_RESULT_HANDLER *result_handler, void *ctx);
+bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, bool use_md5, DB_RESULT_HANDLER *result_handler, void *ctx);
 bool db_get_base_jobid(JCR *jcr, B_DB *mdb, JOB_DBR *jr, JobId_t *jobid);
 bool db_accurate_get_jobids(JCR *jcr, B_DB *mdb, JOB_DBR *jr, db_list_ctx *jobids);
 bool db_get_used_base_jobids(JCR *jcr, B_DB *mdb, POOLMEM *jobids, db_list_ctx *result);

diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c
index bd440dd350af4ffe9bc25057fede8260e0b403e7..755c76540140c858939fd3c516b58c13c1835400 100644 (file)
--- a/bacula/src/cats/sql_get.c
+++ b/bacula/src/cats/sql_get.c
@@ -1083,6 +1083,15 @@ bool db_get_media_record(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr)
    return ok;
 }
 
+/* Remove all MD5 from a query (can save lot of memory with many files) */
+static void replace_md5(char *q)
+{
+   char *p = q;
+   while ((p = strstr(p, ", MD5"))) {
+      memset(p, ' ', 5 * sizeof(char));
+   }
+}
+
 /**
  * Find the last "accurate" backup state (that can take deleted files in
  * account)
@@ -1094,7 +1103,7 @@ bool db_get_media_record(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr)
  *
  * TODO: See if we can do the SORT only if needed (as an argument)
  */
-bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, 
+bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, bool use_md5,
                       DB_RESULT_HANDLER *result_handler, void *ctx)
 {
    if (!*jobids) {
@@ -1111,7 +1120,7 @@ bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids,
    Mmsg(buf2, select_recent_version_with_basejob[db_type], 
         jobids, jobids, jobids, jobids);
    Mmsg(buf,
-"SELECT Path.Path, Filename.Name, T1.FileIndex, T1.JobId, LStat, MD5, MarkId "
+"SELECT Path.Path, Filename.Name, T1.FileIndex, T1.JobId, LStat, MarkId, MD5 "
  "FROM ( %s ) AS T1 "
  "JOIN Filename ON (Filename.FilenameId = T1.FilenameId) "
  "JOIN Path ON (Path.PathId = T1.PathId) "
@@ -1119,6 +1128,9 @@ bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids,
 "ORDER BY T1.JobTDate, FileIndex ASC",/* Return sorted by JobId, */
                                       /* FileIndex for restore code */ 
         buf2.c_str());
+   if (!use_md5) {
+      replace_md5(buf.c_str());
+   }
    Dmsg1(100, "q=%s\n", buf.c_str());
 #else
    /*  
@@ -1238,16 +1250,19 @@ bail_out:
    return ret;
 }
 
-bool db_get_base_file_list(JCR *jcr, B_DB *mdb,
+bool db_get_base_file_list(JCR *jcr, B_DB *mdb, bool use_md5,
                            DB_RESULT_HANDLER *result_handler, void *ctx)
 {
    POOL_MEM buf(PM_MESSAGE);
          
    Mmsg(buf,
- "SELECT Path, Name, FileIndex, JobId, LStat, MD5, 0 As MarkId "
+ "SELECT Path, Name, FileIndex, JobId, LStat, 0 As MarkId, MD5 "
    "FROM new_basefile%lld ORDER BY JobId, FileIndex ASC",
         (uint64_t) jcr->JobId);
-
+   
+   if (!use_md5) {
+      replace_md5(buf.c_str());
+   }
    return db_sql_query(mdb, buf.c_str(), result_handler, ctx);
 }
 

diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c
index 9467a478b9a0f2b867242f9dac37766821d85809..7bd06a9eff34e863394aa953a032bff4a1b16137 100644 (file)
--- a/bacula/src/dird/backup.c
+++ b/bacula/src/dird/backup.c
@@ -138,6 +138,8 @@ static bool get_base_jobids(JCR *jcr, db_list_ctx *jobids)
 
 /*
  * Foreach files in currrent list, send "/path/fname\0LStat\0MD5\0Delta" to FD
+ *      row[0]=Path, row[1]=Filename, row[2]=FileIndex
+ *      row[3]=JobId row[4]=LStat row[5]=MarkId row[6]=MD5
  */
 static int accurate_list_handler(void *ctx, int num_fields, char **row)
 {
@@ -154,14 +156,14 @@ static int accurate_list_handler(void *ctx, int num_fields, char **row)
    /* sending with checksum */
    if (jcr->use_accurate_chksum 
        && num_fields == 7 
-       && row[5][0] /* skip checksum = '0' */
-       && row[5][1])
+       && row[6][0] /* skip checksum = '0' */
+       && row[6][1])
    { 
       jcr->file_bsock->fsend("%s%s%c%s%c%s%c%s", 
-                             row[0], row[1], 0, row[4], 0, row[5], 0, row[6]); 
+                             row[0], row[1], 0, row[4], 0, row[6], 0, row[5]); 
    } else {
       jcr->file_bsock->fsend("%s%s%c%s%c%c%s", 
-                             row[0], row[1], 0, row[4], 0, 0, row[6]); 
+                             row[0], row[1], 0, row[4], 0, 0, row[5]); 
    }
    return 0;
 }
@@ -290,11 +292,12 @@ bool send_accurate_current_files(JCR *jcr)
    if (jcr->HasBase) {
       jcr->nb_base_files = str_to_int64(nb.list);
       db_create_base_file_list(jcr, jcr->db, jobids.list);
-      db_get_base_file_list(jcr, jcr->db, 
+      db_get_base_file_list(jcr, jcr->db, jcr->use_accurate_chksum,
                             accurate_list_handler, (void *)jcr);
 
    } else {
-      db_get_file_list(jcr, jcr->db_batch, jobids.list, 
+      db_get_file_list(jcr, jcr->db_batch,
+                       jobids.list, jcr->use_accurate_chksum,
                        accurate_list_handler, (void *)jcr);
    } 
 

diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c
index 9970d1790a93b8f0395cf45c304d43e7ce389d6f..1562a0cf8df39192a2eee313cee9675a9044973b 100644 (file)
--- a/bacula/src/dird/ua_restore.c
+++ b/bacula/src/dird/ua_restore.c
@@ -1106,7 +1106,10 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx)
 
 #define new_get_file_list
 #ifdef new_get_file_list
-   if (!db_get_file_list(ua->jcr, ua->db, rx->JobIds, insert_tree_handler, (void *)&tree)) {
+   if (!db_get_file_list(ua->jcr, ua->db, 
+                         rx->JobIds, false /* do not use md5 */, 
+                         insert_tree_handler, (void *)&tree))
+   {
       ua->error_msg("%s", db_strerror(ua->db));
    }
    if (*rx->BaseJobIds) {

diff --git a/bacula/src/dird/ua_tree.c b/bacula/src/dird/ua_tree.c
index 389d86861d810c8e9a92fa19a531b5e2a03779ac..a40acdf973b2456c075ec9557a3f8a3ccd21df25 100644 (file)
--- a/bacula/src/dird/ua_tree.c
+++ b/bacula/src/dird/ua_tree.c
@@ -181,7 +181,7 @@ bool user_select_files_from_tree(TREE_CTX *tree)
  *
  *   See uar_sel_files in sql_cmds.c for query that calls us.
  *      row[0]=Path, row[1]=Filename, row[2]=FileIndex
- *      row[3]=JobId row[4]=LStat
+ *      row[3]=JobId row[4]=LStat row[5]=MarkId
  */
 int insert_tree_handler(void *ctx, int num_fields, char **row)
 {

diff --git a/bacula/src/dird/vbackup.c b/bacula/src/dird/vbackup.c
index 632c416f061c02f35d3fd9ecd51af58e418aba87..0edec869769510c739af71845635737227d49727 100644 (file)
--- a/bacula/src/dird/vbackup.c
+++ b/bacula/src/dird/vbackup.c
@@ -483,7 +483,7 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids)
       return false;
    }
 
-   if (!db_get_file_list(jcr, jcr->db_batch, jobids, 
+   if (!db_get_file_list(jcr, jcr->db_batch, jobids, false /* don't use md5 */,
                          insert_bootstrap_handler, (void *)rx.bsr))
    {
       Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db_batch));

diff --git a/bacula/src/tools/bbatch.c b/bacula/src/tools/bbatch.c
index c7d5cab5576a3dd15ee47113866f3bed8c92cb46..ef877013ec7ffa22b582e9284ef9591dfeb3f60e 100644 (file)
--- a/bacula/src/tools/bbatch.c
+++ b/bacula/src/tools/bbatch.c
@@ -196,7 +196,7 @@ int main (int argc, char *argv[])
       }
 
       start = get_current_btime();
-      db_get_file_list(NULL, db, restore_list, list_handler, &nb_file);
+      db_get_file_list(NULL, db, restore_list, false, list_handler, &nb_file);
       end = get_current_btime();
 
       Pmsg3(0, _("Computing file list for jobid=%s files=%lld secs=%d\n"),