From 499d3bfccd921205c591665281ce121a4dc76d5e Mon Sep 17 00:00:00 2001 From: Eric Bollengier Date: Wed, 21 Mar 2007 22:46:20 +0000 Subject: [PATCH] ebl make batch mode usable cleanup in postgresql.c rename internal db_batch_... to sql_batch_.. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4379 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/cats/cats.h | 33 ++++++++++++++++--------- bacula/src/cats/postgresql.c | 30 +++++++---------------- bacula/src/cats/protos.h | 6 ++--- bacula/src/cats/sql_create.c | 47 ++++++++++++++++++------------------ bacula/src/version.h | 2 +- 5 files changed, 58 insertions(+), 60 deletions(-) diff --git a/bacula/src/cats/cats.h b/bacula/src/cats/cats.h index 85a1a29103..e098ae5a7d 100644 --- a/bacula/src/cats/cats.h +++ b/bacula/src/cats/cats.h @@ -171,6 +171,10 @@ struct B_DB { #define sql_fetch_field(x) my_sqlite_fetch_field(x) #define sql_num_fields(x) ((x)->ncolumn) #define SQL_ROW char** + +#define sql_batch_start(x,y) my_batch_start(x,y) +#define sql_batch_end(x,y,z) my_batch_end(x,y,z) +#define sql_batch_insert(x,y,z) my_batch_insert(x,y,z) #define sql_batch_lock_path_query my_sqlite_batch_lock_query #define sql_batch_lock_filename_query my_sqlite_batch_lock_query #define sql_batch_unlock_tables_query my_sqlite_batch_unlock_query @@ -297,6 +301,9 @@ struct B_DB { #define sql_field_seek(x, y) my_sqlite_field_seek((x), (y)) #define sql_fetch_field(x) my_sqlite_fetch_field(x) #define sql_num_fields(x) ((x)->ncolumn) +#define sql_batch_start(x,y) my_batch_start(x,y) +#define sql_batch_end(x,y,z) my_batch_end(x,y,z) +#define sql_batch_insert(x,y,z) my_batch_insert(x,y,z) #define SQL_ROW char** #define sql_batch_lock_path_query my_sqlite_batch_lock_query #define sql_batch_lock_filename_query my_sqlite_batch_lock_query @@ -379,16 +386,19 @@ struct B_DB { #define sql_field_seek(x, y) mysql_field_seek((x)->result, (y)) #define sql_fetch_field(x) mysql_fetch_field((x)->result) #define sql_num_fields(x) (int)mysql_num_fields((x)->result) -#define sql_batch_start(x) db_batch_start(x) +#define SQL_ROW MYSQL_ROW +#define SQL_FIELD MYSQL_FIELD + +#define sql_batch_start(x,y) my_batch_start(x,y) +#define sql_batch_end(x,y,z) my_batch_end(x,y,z) +#define sql_batch_insert(x,y,z) my_batch_insert(x,y,z) +#define sql_batch_lock_path_query my_mysql_batch_lock_path_query #define sql_batch_lock_filename_query my_mysql_batch_lock_filename_query #define sql_batch_unlock_tables_query my_mysql_batch_unlock_tables_query #define sql_batch_fill_filename_query my_mysql_batch_fill_filename_query #define sql_batch_fill_path_query my_mysql_batch_fill_path_query -#define SQL_ROW MYSQL_ROW -#define SQL_FIELD MYSQL_FIELD -int my_mysql_batch_start(B_DB *mdb); extern char* my_mysql_batch_lock_path_query; extern char* my_mysql_batch_lock_filename_query; extern char* my_mysql_batch_unlock_tables_query; @@ -467,12 +477,11 @@ void my_postgresql_data_seek (B_DB *mdb, int row); int my_postgresql_currval (B_DB *mdb, char *table_name); void my_postgresql_field_seek (B_DB *mdb, int row); POSTGRESQL_FIELD * my_postgresql_fetch_field(B_DB *mdb); -int my_postgresql_lock_table(B_DB *mdb, const char *table); -int my_postgresql_unlock_table(B_DB *mdb); -int my_postgresql_batch_start(B_DB *mdb); -int my_postgresql_batch_end(B_DB *mdb, const char *error); + +int my_postgresql_batch_start(JCR *jcr, B_DB *mdb); +int my_postgresql_batch_end(JCR *jcr, B_DB *mdb, const char *error); typedef struct ATTR_DBR ATTR_DBR; -int my_postgresql_batch_insert(B_DB *mdb, ATTR_DBR *ar); +int my_postgresql_batch_insert(JCR *jcr, B_DB *mdb, ATTR_DBR *ar); char *my_postgresql_copy_escape(char *dest, char *src, size_t len); extern char* my_pg_batch_lock_path_query; @@ -495,8 +504,10 @@ extern char* my_pg_batch_fill_path_query; #define sql_field_seek(x, y) my_postgresql_field_seek((x), (y)) #define sql_fetch_field(x) my_postgresql_fetch_field(x) #define sql_num_fields(x) ((x)->num_fields) -#define sql_lock_table(x,y) my_postgresql_lock_table(x, y) -#define sql_unlock_table(x,y) my_postgresql_unlock_table(x) + +#define sql_batch_start(x,y) my_postgresql_batch_start(x,y) +#define sql_batch_end(x,y,z) my_postgresql_batch_end(x,y,z) +#define sql_batch_insert(x,y,z) my_postgresql_batch_insert(x,y,z) #define sql_batch_lock_path_query my_pg_batch_lock_path_query #define sql_batch_lock_filename_query my_pg_batch_lock_filename_query #define sql_batch_unlock_tables_query my_pg_batch_unlock_tables_query diff --git a/bacula/src/cats/postgresql.c b/bacula/src/cats/postgresql.c index d573b09f42..13c7b2f38b 100644 --- a/bacula/src/cats/postgresql.c +++ b/bacula/src/cats/postgresql.c @@ -540,19 +540,7 @@ int my_postgresql_currval(B_DB *mdb, char *table_name) return id; } -int my_postgresql_lock_table(B_DB *mdb, const char *table) -{ - my_postgresql_query(mdb, "BEGIN"); - Mmsg(mdb->cmd, "LOCK TABLE %s IN SHARE ROW EXCLUSIVE MODE", table); - return my_postgresql_query(mdb, mdb->cmd); -} - -int my_postgresql_unlock_table(B_DB *mdb) -{ - return my_postgresql_query(mdb, "COMMIT"); -} - -int my_postgresql_batch_start(B_DB *mdb) +int my_postgresql_batch_start(JCR *jcr, B_DB *mdb) { Dmsg0(500, "my_postgresql_batch_start started\n"); @@ -584,10 +572,10 @@ int my_postgresql_batch_start(B_DB *mdb) // how many fields in the set? mdb->num_fields = (int) PQnfields(mdb->result); mdb->num_rows = 0; - mdb->status = 0; + mdb->status = 1; } else { Dmsg0(500, "we failed\n"); - mdb->status = 1; + mdb->status = 0; } Dmsg0(500, "my_postgresql_batch_start finishing\n"); @@ -596,7 +584,7 @@ int my_postgresql_batch_start(B_DB *mdb) } /* set error to something to abort operation */ -int my_postgresql_batch_end(B_DB *mdb, const char *error) +int my_postgresql_batch_end(JCR *jcr, B_DB *mdb, const char *error) { int res; int count=30; @@ -612,12 +600,12 @@ int my_postgresql_batch_end(B_DB *mdb, const char *error) if (res == 1) { Dmsg0(500, "ok\n"); - mdb->status = 0; + mdb->status = 1; } if (res <= 0) { Dmsg0(500, "we failed\n"); - mdb->status = 1; + mdb->status = 0; Mmsg1(&mdb->errmsg, _("error ending batch mode: %s\n"), PQerrorMessage(mdb->db)); } @@ -626,7 +614,7 @@ int my_postgresql_batch_end(B_DB *mdb, const char *error) return mdb->status; } -int my_postgresql_batch_insert(B_DB *mdb, ATTR_DBR *ar) +int my_postgresql_batch_insert(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) { int res; int count=30; @@ -659,12 +647,12 @@ int my_postgresql_batch_insert(B_DB *mdb, ATTR_DBR *ar) if (res == 1) { Dmsg0(500, "ok\n"); mdb->changes++; - mdb->status = 0; + mdb->status = 1; } if (res <= 0) { Dmsg0(500, "we failed\n"); - mdb->status = 1; + mdb->status = 0; Mmsg1(&mdb->errmsg, _("error ending batch mode: %s\n"), PQerrorMessage(mdb->db)); } diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h index ef11adddcb..4ea426458b 100644 --- a/bacula/src/cats/protos.h +++ b/bacula/src/cats/protos.h @@ -68,9 +68,9 @@ bool db_create_device_record(JCR *jcr, B_DB *mdb, DEVICE_DBR *dr); bool db_create_storage_record(JCR *jcr, B_DB *mdb, STORAGE_DBR *sr); bool db_create_mediatype_record(JCR *jcr, B_DB *mdb, MEDIATYPE_DBR *mr); bool db_write_batch_file_records(JCR *jcr); -bool db_batch_start(B_DB *mdb); -bool db_batch_end(B_DB *mdb, const char *error); -bool db_batch_insert(B_DB *mdb, ATTR_DBR *ar); +bool my_batch_start(JCR *jcr, B_DB *mdb); +bool my_batch_end(JCR *jcr, B_DB *mdb, const char *error); +bool my_batch_insert(JCR *jcr, B_DB *mdb, ATTR_DBR *ar); /* delete.c */ int db_delete_pool_record(JCR *jcr, B_DB *db, POOL_DBR *pool_dbr); diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index a689f9f613..524c7de52d 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -668,7 +668,7 @@ bool db_create_fileset_record(JCR *jcr, B_DB *mdb, FILESET_DBR *fsr) * }; */ -/* All db_batch_* functions are used to do bulk batch insert in File/Filename/Path +/* All sql_batch_* functions are used to do bulk batch insert in File/Filename/Path * tables. This code can be activated by adding "#define HAVE_BATCH_FILE_INSERT 1" * in baconfig.h * @@ -684,23 +684,23 @@ bool db_create_fileset_record(JCR *jcr, B_DB *mdb, FILESET_DBR *fsr) * Returns 1 if OK * 0 if failed */ -bool db_batch_start(B_DB *mdb) +bool my_batch_start(JCR *jcr, B_DB *mdb) { - return sql_query(mdb, + return QUERY_DB(jcr, mdb, " CREATE TEMPORARY TABLE batch " " (fileindex integer, " " jobid integer, " " path blob, " " name blob, " " lstat tinyblob, " - " md5 tinyblob) ") == 0; + " md5 tinyblob) "); } /* * Returns 1 if OK * 0 if failed */ -bool db_batch_insert(B_DB *mdb, ATTR_DBR *ar) +bool my_batch_insert(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) { size_t len; char *digest; @@ -722,7 +722,7 @@ bool db_batch_insert(B_DB *mdb, ATTR_DBR *ar) ar->FileIndex, edit_int64(ar->JobId,ed1), mdb->path, mdb->fname, ar->attr, digest); - return sql_query(mdb, mdb->cmd) == 0; + return QUERY_DB(jcr, mdb, mdb->cmd); } /* set error to something to abort operation */ @@ -730,10 +730,10 @@ bool db_batch_insert(B_DB *mdb, ATTR_DBR *ar) * Returns 1 if OK * 0 if failed */ -bool db_batch_end(B_DB *mdb, const char *error) +bool my_batch_end(JCR *jcr, B_DB *mdb, const char *error) { - Dmsg0(50, "db_batch_end started"); + Dmsg0(50, "sql_batch_end started"); if (mdb) { mdb->status = 0; @@ -755,7 +755,7 @@ bool db_write_batch_file_records(JCR *jcr) return true; } - if (!db_batch_end(jcr->db_batch, NULL)) { + if (!sql_batch_end(jcr, jcr->db_batch, NULL)) { Jmsg(jcr, M_FATAL, 0, "Bad batch end %s\n", jcr->db_batch->errmsg); return false; } @@ -765,52 +765,52 @@ bool db_write_batch_file_records(JCR *jcr) } /* we have to lock tables */ - if (sql_query(jcr->db_batch, sql_batch_lock_path_query) != 0) { + if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_lock_path_query)) { Jmsg(jcr, M_FATAL, 0, "Can't lock Path table %s\n", jcr->db_batch->errmsg); return false; } - if (sql_query(jcr->db_batch, sql_batch_fill_path_query) != 0) { + if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_fill_path_query)) { Jmsg(jcr, M_FATAL, 0, "Can't fill Path table %s\n",jcr->db_batch->errmsg); - sql_query(jcr->db_batch, sql_batch_unlock_tables_query); + QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query); return false; } - if (sql_query(jcr->db_batch, sql_batch_unlock_tables_query) !=0) { + if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query)) { Jmsg(jcr, M_FATAL, 0, "Can't unlock Path table %s\n", jcr->db_batch->errmsg); return false; } /* we have to lock tables */ - if (sql_query(jcr->db_batch, sql_batch_lock_filename_query) != 0) { + if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_lock_filename_query)) { Jmsg(jcr, M_FATAL, 0, "Can't lock Filename table %s\n", jcr->db_batch->errmsg); return false; } - if (sql_query(jcr->db_batch, sql_batch_fill_filename_query) != 0) { + if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_fill_filename_query)) { Jmsg(jcr,M_FATAL,0,"Can't fill Filename table %s\n",jcr->db_batch->errmsg); - sql_query(jcr->db_batch, sql_batch_unlock_tables_query); + QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query); return false; } - if (sql_query(jcr->db_batch, sql_batch_unlock_tables_query) != 0) { + if (!QUERY_DB(jcr, jcr->db_batch, sql_batch_unlock_tables_query)) { Jmsg(jcr, M_FATAL, 0, "Can't unlock Filename table %s\n", jcr->db_batch->errmsg); return false; } - if (sql_query(jcr->db_batch, + if (!QUERY_DB(jcr, jcr->db_batch, " INSERT INTO File (FileIndex, JobId, PathId, FilenameId, LStat, MD5)" " SELECT batch.FileIndex, batch.JobId, Path.PathId, " " Filename.FilenameId,batch.LStat, batch.MD5 " " FROM batch " " JOIN Path ON (batch.Path = Path.Path) " - " JOIN Filename ON (batch.Name = Filename.Name) ") != 0) + " JOIN Filename ON (batch.Name = Filename.Name) ")) { Jmsg(jcr, M_FATAL, 0, "Can't fill File table %s\n", jcr->db_batch->errmsg); return false; } - sql_query(jcr->db_batch, "DROP TABLE batch"); + QUERY_DB(jcr, jcr->db_batch, "DROP TABLE batch"); return true; } @@ -848,7 +848,7 @@ bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) return false; } - if (!db_batch_start(jcr->db_batch)) { + if (!sql_batch_start(jcr, jcr->db_batch)) { Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db_batch)); return false; } @@ -872,13 +872,12 @@ bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) /* if (jcr->changes > 100000) { - db_batch_end(bdb, NULL); - db_batch_start(bdb); + db_write_batch_file_records(jcr); jcr->changes = 0; } */ - return db_batch_insert(bdb, ar); + return sql_batch_insert(jcr, bdb, ar); } #else /* ! HAVE_BATCH_FILE_INSERT */ diff --git a/bacula/src/version.h b/bacula/src/version.h index 61e559d825..53ed062551 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -56,7 +56,7 @@ /* Turn on the following flag to enable batch attribute inserts * in the catalog. This gives a large speedup. */ -/* #define HAVE_BATCH_FILE_INSERT 1 */ +#define HAVE_BATCH_FILE_INSERT 1 /* Debug flags not normally turned on */ -- 2.39.5