From: Eric Bollengier Date: Fri, 23 Mar 2007 21:56:09 +0000 (+0000) Subject: ebl make mysql working with batch mode X-Git-Tag: Release-7.0.0~6726 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=ab8ca4d45935c5ab0d6e6aadcc2f0594a33e501e;p=bacula%2Fbacula ebl make mysql working with batch mode git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4399 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index 524c7de52d..57cdf34ee6 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -686,14 +686,14 @@ bool db_create_fileset_record(JCR *jcr, B_DB *mdb, FILESET_DBR *fsr) */ bool my_batch_start(JCR *jcr, B_DB *mdb) { - return QUERY_DB(jcr, mdb, + return db_sql_query(mdb, " CREATE TEMPORARY TABLE batch " " (fileindex integer, " " jobid integer, " " path blob, " " name blob, " " lstat tinyblob, " - " md5 tinyblob) "); + " md5 tinyblob) ",NULL, NULL); } /* @@ -722,7 +722,7 @@ bool my_batch_insert(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) ar->FileIndex, edit_int64(ar->JobId,ed1), mdb->path, mdb->fname, ar->attr, digest); - return QUERY_DB(jcr, mdb, mdb->cmd); + return INSERT_DB(jcr, mdb, mdb->cmd); } /* set error to something to abort operation */ @@ -765,52 +765,53 @@ bool db_write_batch_file_records(JCR *jcr) } /* we have to lock tables */ - if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_lock_path_query)) { + if (!db_sql_query(jcr->db_batch, sql_batch_lock_path_query, NULL, NULL)) { Jmsg(jcr, M_FATAL, 0, "Can't lock Path table %s\n", jcr->db_batch->errmsg); return false; } - if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_fill_path_query)) { + if (!db_sql_query(jcr->db_batch, sql_batch_fill_path_query, NULL, NULL)) { Jmsg(jcr, M_FATAL, 0, "Can't fill Path table %s\n",jcr->db_batch->errmsg); - QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query); + db_sql_query(jcr->db_batch, sql_batch_unlock_tables_query, NULL, NULL); return false; } - if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query)) { + if (!db_sql_query(jcr->db_batch, sql_batch_unlock_tables_query,NULL,NULL)) { Jmsg(jcr, M_FATAL, 0, "Can't unlock Path table %s\n", jcr->db_batch->errmsg); return false; } /* we have to lock tables */ - if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_lock_filename_query)) { + if (!db_sql_query(jcr->db_batch,sql_batch_lock_filename_query,NULL, NULL)) { Jmsg(jcr, M_FATAL, 0, "Can't lock Filename table %s\n", jcr->db_batch->errmsg); return false; } - if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_fill_filename_query)) { + if (!db_sql_query(jcr->db_batch,sql_batch_fill_filename_query, NULL,NULL)) { Jmsg(jcr,M_FATAL,0,"Can't fill Filename table %s\n",jcr->db_batch->errmsg); QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query); return false; } - if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query)) { + if (!db_sql_query(jcr->db_batch, sql_batch_unlock_tables_query,NULL,NULL)) { Jmsg(jcr, M_FATAL, 0, "Can't unlock Filename table %s\n", jcr->db_batch->errmsg); return false; } - if (!QUERY_DB(jcr, jcr->db_batch, + if (!db_sql_query(jcr->db_batch, " INSERT INTO File (FileIndex, JobId, PathId, FilenameId, LStat, MD5)" " SELECT batch.FileIndex, batch.JobId, Path.PathId, " " Filename.FilenameId,batch.LStat, batch.MD5 " " FROM batch " " JOIN Path ON (batch.Path = Path.Path) " - " JOIN Filename ON (batch.Name = Filename.Name) ")) + " JOIN Filename ON (batch.Name = Filename.Name) ", + NULL,NULL)) { Jmsg(jcr, M_FATAL, 0, "Can't fill File table %s\n", jcr->db_batch->errmsg); return false; } - QUERY_DB(jcr, jcr->db_batch, "DROP TABLE batch"); + db_sql_query(jcr->db_batch, "DROP TABLE batch", NULL,NULL); return true; } @@ -849,7 +850,8 @@ bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) } if (!sql_batch_start(jcr, jcr->db_batch)) { - Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db_batch)); + Jmsg(jcr, M_FATAL, 0, + "Can't start batch mode %s", db_strerror(jcr->db_batch)); return false; } } @@ -874,6 +876,7 @@ bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) if (jcr->changes > 100000) { db_write_batch_file_records(jcr); jcr->changes = 0; + sql_batch_start(jcr, jcr->db_batch); } */ diff --git a/bacula/src/tools/bbatch.c b/bacula/src/tools/bbatch.c index 116f491f13..793fa948f0 100644 --- a/bacula/src/tools/bbatch.c +++ b/bacula/src/tools/bbatch.c @@ -58,7 +58,7 @@ #include "cats/cats.h" /* Forward referenced functions */ -static void do_batch(); +static void *do_batch(void *); /* Local variables */ @@ -69,8 +69,6 @@ static const char *db_user = "bacula"; static const char *db_password = ""; static const char *db_host = NULL; -static JCR *bjcr; - char *datafile=NULL; static void usage() @@ -91,6 +89,9 @@ PROG_COPYRIGHT exit(1); } +/* number of thread started */ +int nb=0; + int main (int argc, char *argv[]) { int ch; @@ -98,13 +99,15 @@ int main (int argc, char *argv[]) bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); init_stack_dump(); - + + char **files = (char **) malloc (10 * sizeof(char *)); + int i; my_name_is(argc, argv, "bbatch"); init_msg(NULL, NULL); OSDependentInit(); - while ((ch = getopt(argc, argv, "c:d:n:P:Su:vf:w:?")) != -1) { + while ((ch = getopt(argc, argv, "h:c:d:n:P:Su:vf:w:?")) != -1) { switch (ch) { case 'd': /* debug level */ debug_level = atoi(optarg); @@ -137,7 +140,9 @@ int main (int argc, char *argv[]) break; case 'f': - datafile = optarg; + if (nb < 10 ) { + files[nb++] = optarg; + } break; case '?': @@ -154,53 +159,55 @@ int main (int argc, char *argv[]) usage(); } - if (!datafile) { - Pmsg0(0, _("You must specified a data file\n")); - usage(); - } - #ifdef HAVE_BATCH_FILE_INSERT printf("With new Batch mode\n"); #else printf("Without new Batch mode\n"); #endif + i = nb; + while (--i >= 0) { + pthread_t thid; + JCR *bjcr = new_jcr(sizeof(JCR), NULL); + bjcr->bsr = NULL; + bjcr->VolSessionId = 1; + bjcr->VolSessionTime = (uint32_t)time(NULL); + bjcr->NumReadVolumes = 0; + bjcr->NumWriteVolumes = 0; + bjcr->JobId = getpid(); + bjcr->JobType = JT_CONSOLE; + bjcr->JobLevel = L_FULL; + bjcr->JobStatus = JS_Running; + bjcr->where = bstrdup(files[i]); + bjcr->job_name = get_pool_memory(PM_FNAME); + pm_strcpy(bjcr->job_name, "Dummy.Job.Name"); + bjcr->client_name = get_pool_memory(PM_FNAME); + pm_strcpy(bjcr->client_name, "Dummy.Client.Name"); + bstrncpy(bjcr->Job, "bbatch", sizeof(bjcr->Job)); + bjcr->fileset_name = get_pool_memory(PM_FNAME); + pm_strcpy(bjcr->fileset_name, "Dummy.fileset.name"); + bjcr->fileset_md5 = get_pool_memory(PM_FNAME); + pm_strcpy(bjcr->fileset_md5, "Dummy.fileset.md5"); + + if ((db=db_init_database(NULL, db_name, db_user, db_password, + db_host, 0, NULL, 0)) == NULL) { + Emsg0(M_ERROR_TERM, 0, _("Could not init Bacula database\n")); + } + if (!db_open_database(NULL, db)) { + Emsg0(M_ERROR_TERM, 0, db_strerror(db)); + } + Dmsg0(200, "Database opened\n"); + if (verbose) { + Pmsg2(000, _("Using Database: %s, User: %s\n"), db_name, db_user); + } + + bjcr->db = db; - bjcr = new_jcr(sizeof(JCR), NULL); - bjcr->bsr = NULL; - bjcr->VolSessionId = 1; - bjcr->VolSessionTime = (uint32_t)time(NULL); - bjcr->NumReadVolumes = 0; - bjcr->NumWriteVolumes = 0; - bjcr->JobId = getpid(); - bjcr->JobType = JT_CONSOLE; - bjcr->JobLevel = L_FULL; - bjcr->JobStatus = JS_Running; - bjcr->where = bstrdup(""); - bjcr->job_name = get_pool_memory(PM_FNAME); - pm_strcpy(bjcr->job_name, "Dummy.Job.Name"); - bjcr->client_name = get_pool_memory(PM_FNAME); - pm_strcpy(bjcr->client_name, "Dummy.Client.Name"); - bstrncpy(bjcr->Job, "bbatch", sizeof(bjcr->Job)); - bjcr->fileset_name = get_pool_memory(PM_FNAME); - pm_strcpy(bjcr->fileset_name, "Dummy.fileset.name"); - bjcr->fileset_md5 = get_pool_memory(PM_FNAME); - pm_strcpy(bjcr->fileset_md5, "Dummy.fileset.md5"); - - if ((db=db_init_database(NULL, db_name, db_user, db_password, - db_host, 0, NULL, 0)) == NULL) { - Emsg0(M_ERROR_TERM, 0, _("Could not init Bacula database\n")); - } - if (!db_open_database(NULL, db)) { - Emsg0(M_ERROR_TERM, 0, db_strerror(db)); + pthread_create(&thid, NULL, do_batch, bjcr); } - Dmsg0(200, "Database opened\n"); - if (verbose) { - Pmsg2(000, _("Using Database: %s, User: %s\n"), db_name, db_user); - } - - bjcr->db = db; - do_batch(); + while (nb > 0) { + bmicrosleep(1,0); + } return 0; } @@ -231,18 +238,20 @@ static void fill_attr(ATTR_DBR *ar, char *data) break; } index++; - p++; + b = ++p; } } } -static void do_batch() +static void *do_batch(void *jcr) { + JCR *bjcr = (JCR *)jcr; char data[1024]; int lineno = 0; struct ATTR_DBR ar; memset(&ar, 0, sizeof(ar)); btime_t begin = get_current_btime(); + char *datafile = bjcr->where; FILE *fd = fopen(datafile, "r"); if (!fd) { @@ -268,4 +277,7 @@ static void do_batch() printf("Insert time = %llims\n", (end - begin) / 10000); printf("Create %u files at %.2f/s\n", lineno, (lineno / ((float)((end - begin) / 1000000)))); + nb--; + pthread_exit(NULL); + return NULL; }