From b933f7d2a50b0ab6ee6089874ad70b0e817e6205 Mon Sep 17 00:00:00 2001 From: Eric Bollengier Date: Fri, 19 Nov 2010 23:08:57 +0100 Subject: [PATCH] Don't compute accurate list with MD5 if not used During accurate queries, database engine have to deal with MD5 and store them in memory or on disk. Just replace MD5 by space in the query when we don't use them. --- bacula/src/cats/protos.h | 4 ++-- bacula/src/cats/sql_get.c | 25 ++++++++++++++++++++----- bacula/src/dird/backup.c | 15 +++++++++------ bacula/src/dird/ua_restore.c | 5 ++++- bacula/src/dird/ua_tree.c | 2 +- bacula/src/dird/vbackup.c | 2 +- bacula/src/tools/bbatch.c | 2 +- 7 files changed, 38 insertions(+), 17 deletions(-) diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h index 4eaf6f3f76..0a384bfc05 100644 --- a/bacula/src/cats/protos.h +++ b/bacula/src/cats/protos.h @@ -104,7 +104,7 @@ bool db_find_failed_job_since(JCR *jcr, B_DB *mdb, JOB_DBR *jr, POOLMEM *stime, /* sql_get.c */ bool db_get_volume_jobids(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr, db_list_ctx *lst); -bool db_get_base_file_list(JCR *jcr, B_DB *mdb, +bool db_get_base_file_list(JCR *jcr, B_DB *mdb, bool use_md5, DB_RESULT_HANDLER *result_handler,void *ctx); int db_get_path_record(JCR *jcr, B_DB *mdb); bool db_get_pool_record(JCR *jcr, B_DB *db, POOL_DBR *pdbr); @@ -123,7 +123,7 @@ int db_get_job_volume_parameters(JCR *jcr, B_DB *mdb, JobId_t JobId, VOL_PARAMS int db_get_client_record(JCR *jcr, B_DB *mdb, CLIENT_DBR *cdbr); int db_get_counter_record(JCR *jcr, B_DB *mdb, COUNTER_DBR *cr); bool db_get_query_dbids(JCR *jcr, B_DB *mdb, POOL_MEM &query, dbid_list &ids); -bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, DB_RESULT_HANDLER *result_handler, void *ctx); +bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, bool use_md5, DB_RESULT_HANDLER *result_handler, void *ctx); bool db_get_base_jobid(JCR *jcr, B_DB *mdb, JOB_DBR *jr, JobId_t *jobid); bool db_accurate_get_jobids(JCR *jcr, B_DB *mdb, JOB_DBR *jr, db_list_ctx *jobids); bool db_get_used_base_jobids(JCR *jcr, B_DB *mdb, POOLMEM *jobids, db_list_ctx *result); diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index bd440dd350..755c765401 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -1083,6 +1083,15 @@ bool db_get_media_record(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr) return ok; } +/* Remove all MD5 from a query (can save lot of memory with many files) */ +static void replace_md5(char *q) +{ + char *p = q; + while ((p = strstr(p, ", MD5"))) { + memset(p, ' ', 5 * sizeof(char)); + } +} + /** * Find the last "accurate" backup state (that can take deleted files in * account) @@ -1094,7 +1103,7 @@ bool db_get_media_record(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr) * * TODO: See if we can do the SORT only if needed (as an argument) */ -bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, +bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, bool use_md5, DB_RESULT_HANDLER *result_handler, void *ctx) { if (!*jobids) { @@ -1111,7 +1120,7 @@ bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, Mmsg(buf2, select_recent_version_with_basejob[db_type], jobids, jobids, jobids, jobids); Mmsg(buf, -"SELECT Path.Path, Filename.Name, T1.FileIndex, T1.JobId, LStat, MD5, MarkId " +"SELECT Path.Path, Filename.Name, T1.FileIndex, T1.JobId, LStat, MarkId, MD5 " "FROM ( %s ) AS T1 " "JOIN Filename ON (Filename.FilenameId = T1.FilenameId) " "JOIN Path ON (Path.PathId = T1.PathId) " @@ -1119,6 +1128,9 @@ bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, "ORDER BY T1.JobTDate, FileIndex ASC",/* Return sorted by JobId, */ /* FileIndex for restore code */ buf2.c_str()); + if (!use_md5) { + replace_md5(buf.c_str()); + } Dmsg1(100, "q=%s\n", buf.c_str()); #else /* @@ -1238,16 +1250,19 @@ bail_out: return ret; } -bool db_get_base_file_list(JCR *jcr, B_DB *mdb, +bool db_get_base_file_list(JCR *jcr, B_DB *mdb, bool use_md5, DB_RESULT_HANDLER *result_handler, void *ctx) { POOL_MEM buf(PM_MESSAGE); Mmsg(buf, - "SELECT Path, Name, FileIndex, JobId, LStat, MD5, 0 As MarkId " + "SELECT Path, Name, FileIndex, JobId, LStat, 0 As MarkId, MD5 " "FROM new_basefile%lld ORDER BY JobId, FileIndex ASC", (uint64_t) jcr->JobId); - + + if (!use_md5) { + replace_md5(buf.c_str()); + } return db_sql_query(mdb, buf.c_str(), result_handler, ctx); } diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 9467a478b9..7bd06a9eff 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -138,6 +138,8 @@ static bool get_base_jobids(JCR *jcr, db_list_ctx *jobids) /* * Foreach files in currrent list, send "/path/fname\0LStat\0MD5\0Delta" to FD + * row[0]=Path, row[1]=Filename, row[2]=FileIndex + * row[3]=JobId row[4]=LStat row[5]=MarkId row[6]=MD5 */ static int accurate_list_handler(void *ctx, int num_fields, char **row) { @@ -154,14 +156,14 @@ static int accurate_list_handler(void *ctx, int num_fields, char **row) /* sending with checksum */ if (jcr->use_accurate_chksum && num_fields == 7 - && row[5][0] /* skip checksum = '0' */ - && row[5][1]) + && row[6][0] /* skip checksum = '0' */ + && row[6][1]) { jcr->file_bsock->fsend("%s%s%c%s%c%s%c%s", - row[0], row[1], 0, row[4], 0, row[5], 0, row[6]); + row[0], row[1], 0, row[4], 0, row[6], 0, row[5]); } else { jcr->file_bsock->fsend("%s%s%c%s%c%c%s", - row[0], row[1], 0, row[4], 0, 0, row[6]); + row[0], row[1], 0, row[4], 0, 0, row[5]); } return 0; } @@ -290,11 +292,12 @@ bool send_accurate_current_files(JCR *jcr) if (jcr->HasBase) { jcr->nb_base_files = str_to_int64(nb.list); db_create_base_file_list(jcr, jcr->db, jobids.list); - db_get_base_file_list(jcr, jcr->db, + db_get_base_file_list(jcr, jcr->db, jcr->use_accurate_chksum, accurate_list_handler, (void *)jcr); } else { - db_get_file_list(jcr, jcr->db_batch, jobids.list, + db_get_file_list(jcr, jcr->db_batch, + jobids.list, jcr->use_accurate_chksum, accurate_list_handler, (void *)jcr); } diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index 9970d1790a..1562a0cf8d 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -1106,7 +1106,10 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) #define new_get_file_list #ifdef new_get_file_list - if (!db_get_file_list(ua->jcr, ua->db, rx->JobIds, insert_tree_handler, (void *)&tree)) { + if (!db_get_file_list(ua->jcr, ua->db, + rx->JobIds, false /* do not use md5 */, + insert_tree_handler, (void *)&tree)) + { ua->error_msg("%s", db_strerror(ua->db)); } if (*rx->BaseJobIds) { diff --git a/bacula/src/dird/ua_tree.c b/bacula/src/dird/ua_tree.c index 389d86861d..a40acdf973 100644 --- a/bacula/src/dird/ua_tree.c +++ b/bacula/src/dird/ua_tree.c @@ -181,7 +181,7 @@ bool user_select_files_from_tree(TREE_CTX *tree) * * See uar_sel_files in sql_cmds.c for query that calls us. * row[0]=Path, row[1]=Filename, row[2]=FileIndex - * row[3]=JobId row[4]=LStat + * row[3]=JobId row[4]=LStat row[5]=MarkId */ int insert_tree_handler(void *ctx, int num_fields, char **row) { diff --git a/bacula/src/dird/vbackup.c b/bacula/src/dird/vbackup.c index 632c416f06..0edec86976 100644 --- a/bacula/src/dird/vbackup.c +++ b/bacula/src/dird/vbackup.c @@ -483,7 +483,7 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids) return false; } - if (!db_get_file_list(jcr, jcr->db_batch, jobids, + if (!db_get_file_list(jcr, jcr->db_batch, jobids, false /* don't use md5 */, insert_bootstrap_handler, (void *)rx.bsr)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db_batch)); diff --git a/bacula/src/tools/bbatch.c b/bacula/src/tools/bbatch.c index c7d5cab557..ef877013ec 100644 --- a/bacula/src/tools/bbatch.c +++ b/bacula/src/tools/bbatch.c @@ -196,7 +196,7 @@ int main (int argc, char *argv[]) } start = get_current_btime(); - db_get_file_list(NULL, db, restore_list, list_handler, &nb_file); + db_get_file_list(NULL, db, restore_list, false, list_handler, &nb_file); end = get_current_btime(); Pmsg3(0, _("Computing file list for jobid=%s files=%lld secs=%d\n"), -- 2.39.5