From 2eb03ee783662022e0504778961659b7ef9701c8 Mon Sep 17 00:00:00 2001 From: Eric Bollengier Date: Tue, 8 Jul 2008 20:05:44 +0000 Subject: [PATCH] ebl Update to use a separate db connexion to compute and send accurate file list. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@7345 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/patches/testing/accurate-batch.patch | 247 ++++++++++++++------ 1 file changed, 170 insertions(+), 77 deletions(-) diff --git a/bacula/patches/testing/accurate-batch.patch b/bacula/patches/testing/accurate-batch.patch index 49650dec49..2d33ff1949 100644 --- a/bacula/patches/testing/accurate-batch.patch +++ b/bacula/patches/testing/accurate-batch.patch @@ -1,35 +1,67 @@ -Index: src/dird/backup.c +Index: src/cats/sql.c =================================================================== ---- src/dird/backup.c (revision 7339) -+++ src/dird/backup.c (working copy) -@@ -133,6 +133,7 @@ - return true; - } - POOLMEM *jobids = get_pool_memory(PM_FNAME); -+ - db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, jobids); +--- src/cats/sql.c (révision 7344) ++++ src/cats/sql.c (copie de travail) +@@ -635,5 +635,43 @@ + return; + } - if (*jobids == 0) { -@@ -150,8 +151,14 @@ - Dmsg2(200, "jobids=%s nb=%s\n", jobids, nb); - jcr->file_bsock->fsend("accurate files=%s\n", nb); ++/* ++ * Open a new connexion to mdb catalog. This function is used ++ * by batch and accurate mode. ++ */ ++bool db_open_batch_connexion(JCR *jcr, B_DB *mdb) ++{ ++ int multi_db=false; -- db_get_file_list(jcr, jcr->db, jobids, accurate_list_handler, (void *)jcr); -+ if (!db_open_batch_connexion(jcr, jcr->db)) { -+ Mmsg1(&jcr->db->errmsg, "Can't start accurate mode"); -+ Jmsg1(jcr, M_FATAL, 0, "%s", jcr->db->errmsg); -+ return false; ++#ifdef HAVE_BATCH_FILE_INSERT ++ multi_db=true; /* we force a new connexion only if batch insert is enabled */ ++#endif ++ ++ if (!jcr->db_batch) { ++ jcr->db_batch = db_init_database(jcr, ++ mdb->db_name, ++ mdb->db_user, ++ mdb->db_password, ++ mdb->db_address, ++ mdb->db_port, ++ mdb->db_socket, ++ multi_db /* multi_db = true when using batch mode */); ++ if (!jcr->db_batch) { ++ Jmsg0(jcr, M_FATAL, 0, "Could not init batch connexion"); ++ return false; ++ } ++ ++ if (!db_open_database(jcr, jcr->db_batch)) { ++ Mmsg2(&jcr->db_batch->errmsg, _("Could not open database \"%s\": ERR=%s\n"), ++ jcr->db_batch->db_name, db_strerror(jcr->db_batch)); ++ Jmsg1(jcr, M_FATAL, 0, "%s", jcr->db_batch->errmsg); ++ return false; ++ } ++ Dmsg3(100, "initdb ref=%d connected=%d db=%p\n", jcr->db_batch->ref_count, ++ jcr->db_batch->connected, jcr->db_batch->db); ++ + } - -+ db_get_file_list(jcr, jcr->db_batch, jobids, accurate_list_handler, (void *)jcr); ++ return true; ++} + - free_pool_memory(jobids); - free_pool_memory(nb); - + #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_SQLITE || HAVE_POSTGRESQL*/ +Index: src/cats/protos.h +=================================================================== +--- src/cats/protos.h (révision 7344) ++++ src/cats/protos.h (copie de travail) +@@ -49,6 +49,7 @@ + int mult_db_connections); + int db_open_database(JCR *jcr, B_DB *db); + void db_close_database(JCR *jcr, B_DB *db); ++bool db_open_batch_connexion(JCR *jcr, B_DB *mdb); + void db_escape_string(JCR *jcr, B_DB *db, char *snew, char *old, int len); + char *db_strerror(B_DB *mdb); + int db_next_index(JCR *jcr, B_DB *mdb, char *table, char *index); Index: src/cats/sql_create.c =================================================================== ---- src/cats/sql_create.c (revision 7339) -+++ src/cats/sql_create.c (working copy) +--- src/cats/sql_create.c (révision 7344) ++++ src/cats/sql_create.c (copie de travail) @@ -758,7 +758,7 @@ { int JobStatus = jcr->JobStatus; @@ -39,61 +71,37 @@ Index: src/cats/sql_create.c Dmsg0(50,"db_create_file_record : no files\n"); return true; } -@@ -831,20 +831,8 @@ - return true; - } +@@ -845,38 +845,19 @@ + Dmsg1(dbglevel, "Fname=%s\n", ar->fname); + Dmsg0(dbglevel, "put_file_into_catalog\n"); --/* -- * Create File record in B_DB -- * -- * In order to reduce database size, we store the File attributes, -- * the FileName, and the Path separately. In principle, there -- * is a single FileName record and a single Path record, no matter -- * how many times it occurs. This is this subroutine, we separate -- * the file and the path and fill temporary tables with this three records. -- */ --bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) -+bool db_open_batch_connexion(JCR *jcr, B_DB *mdb) - { -- Dmsg1(dbglevel, "Fname=%s\n", ar->fname); -- Dmsg0(dbglevel, "put_file_into_catalog\n"); -- - if (!jcr->db_batch) { - Dmsg2(100, "Opendb attr. Stream=%d fname=%s\n", ar->Stream, ar->fname); - jcr->db_batch = db_init_database(jcr, -@@ -868,15 +856,42 @@ - Jmsg1(jcr, M_FATAL, 0, "%s", mdb->errmsg); - return false; - } -- -+ Dmsg3(100, "initdb ref=%d connected=%d db=%p\n", jcr->db_batch->ref_count, -+ jcr->db_batch->connected, jcr->db_batch->db); -+ -+ } -+ return true; -+} -+ -+/* -+ * Create File record in B_DB -+ * -+ * In order to reduce database size, we store the File attributes, -+ * the FileName, and the Path separately. In principle, there -+ * is a single FileName record and a single Path record, no matter -+ * how many times it occurs. This is this subroutine, we separate -+ * the file and the path and fill temporary tables with this three records. -+ */ -+bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) -+{ -+ Dmsg1(dbglevel, "Fname=%s\n", ar->fname); -+ Dmsg0(dbglevel, "put_file_into_catalog\n"); -+ +- if (!jcr->db_batch) { +- Dmsg2(100, "Opendb attr. Stream=%d fname=%s\n", ar->Stream, ar->fname); +- jcr->db_batch = db_init_database(jcr, +- mdb->db_name, +- mdb->db_user, +- mdb->db_password, +- mdb->db_address, +- mdb->db_port, +- mdb->db_socket, +- 1 /* multi_db = true */); +- if (!jcr->db_batch) { +- Mmsg1(&mdb->errmsg, _("Could not init batch database: \"%s\".\n"), +- jcr->db->db_name); +- Jmsg1(jcr, M_FATAL, 0, "%s", mdb->errmsg); +- return false; +- } ++ /* Open the dedicated connexion */ + if (!jcr->batch_started) { -+ + +- if (!db_open_database(jcr, jcr->db_batch)) { +- Mmsg2(&mdb->errmsg, _("Could not open database \"%s\": ERR=%s\n"), +- jcr->db->db_name, db_strerror(jcr->db_batch)); +- Jmsg1(jcr, M_FATAL, 0, "%s", mdb->errmsg); + if (!db_open_batch_connexion(jcr, mdb)) { -+ Mmsg1(&mdb->errmsg, -+ "Can't start batch mode: ERR=%s", db_strerror(jcr->db_batch)); -+ Jmsg1(jcr, M_FATAL, 0, "%s", mdb->errmsg); -+ return false; + return false; +- } +- + } if (!sql_batch_start(jcr, jcr->db_batch)) { Mmsg1(&mdb->errmsg, @@ -107,3 +115,88 @@ Index: src/cats/sql_create.c } B_DB *bdb = jcr->db_batch; +@@ -895,12 +876,12 @@ + + + /* +- if (bdb->changes > 100000) { +- db_write_batch_file_records(jcr); +- bdb->changes = 0; +- sql_batch_start(jcr, bdb); +- } +-*/ ++ * if (bdb->changes > 100000) { ++ * db_write_batch_file_records(jcr); ++ * bdb->changes = 0; ++ * sql_batch_start(jcr, bdb); ++ * } ++ */ + + return sql_batch_insert(jcr, bdb, ar); + } +Index: src/jcr.h +=================================================================== +--- src/jcr.h (révision 7344) ++++ src/jcr.h (copie de travail) +@@ -226,7 +226,8 @@ + bool cached_attribute; /* set if attribute is cached */ + POOLMEM *attr; /* Attribute string from SD */ + B_DB *db; /* database pointer */ +- B_DB *db_batch; /* database pointer for batch insert */ ++ B_DB *db_batch; /* database pointer for batch and accurate */ ++ bool batch_started; /* is batch mode already started ? */ + ATTR_DBR *ar; /* DB attribute record */ + guid_list *id_list; /* User/group id to name list */ + bool accurate; /* true if job is accurate */ + +Index: src/dird/backup.c +=================================================================== +--- src/dird/backup.c (révision 7344) ++++ src/dird/backup.c (copie de travail) +@@ -133,6 +133,7 @@ + return true; + } + POOLMEM *jobids = get_pool_memory(PM_FNAME); ++ + db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, jobids); + + if (*jobids == 0) { +@@ -150,13 +151,19 @@ + Dmsg2(200, "jobids=%s nb=%s\n", jobids, nb); + jcr->file_bsock->fsend("accurate files=%s\n", nb); + +- db_get_file_list(jcr, jcr->db, jobids, accurate_list_handler, (void *)jcr); ++ if (!db_open_batch_connexion(jcr, jcr->db)) { ++ Jmsg0(jcr, M_FATAL, 0, "Can't get dedicate sql connexion"); ++ return false; ++ } + ++ db_get_file_list(jcr, jcr->db_batch, jobids, accurate_list_handler, (void *)jcr); ++ ++ /* TODO: close the batch connexion ? (can be used very soon) */ ++ + free_pool_memory(jobids); + free_pool_memory(nb); + + jcr->file_bsock->signal(BNET_EOD); +- /* TODO: use response() ? */ + + return true; + } +Index: src/dird/job.c +=================================================================== +--- src/dird/job.c (révision 7344) ++++ src/dird/job.c (copie de travail) +@@ -960,10 +960,10 @@ + pthread_cond_destroy(&jcr->term_wait); + jcr->term_wait_inited = false; + } +- if (jcr->db_batch && jcr->db_batch != jcr->db) { ++ if (jcr->db_batch) { + db_close_database(jcr, jcr->db_batch); ++ jcr->db_batch = NULL; + } +- jcr->db_batch = NULL; + if (jcr->db) { + db_close_database(jcr, jcr->db); + jcr->db = NULL; -- 2.39.5