From d2aaab7dffa967f49c46e913bb40abe1db0953a1 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Thu, 22 Mar 2007 13:10:04 +0000 Subject: [PATCH] kes Implement new prunning code that prunes up to 1000 jobs at the same time. The same technique can be applied to a number of other prune/purge subroutines. kes Add an insanity check when starting a new tape job to ensure that the tape position has not been changed. If so fail the job. kes Fix Win32 build for dlls to add the folllowing: src/lib/bsock.c -- new file with new entry points src/lib/dlist.c -- new entry points src/cats/sql_create.c -- new entry point src/cats/sql_cmds.c -- new DATA item exported git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4383 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/cats/sql_cmds.c | 16 ++++++--- bacula/src/cats/sql_cmds.h | 1 + bacula/src/dird/protos.h | 1 + bacula/src/dird/ua_prune.c | 49 ++++++++++++--------------- bacula/src/dird/ua_purge.c | 34 ++++++++++++++++++- bacula/src/dird/ua_restore.c | 1 + bacula/src/stored/acquire.c | 23 +++++++++++++ bacula/src/win32/README.mingw32 | 10 +++++- bacula/src/win32/cats/bacula_cats.def | 2 ++ bacula/src/win32/dll/Makefile | 1 + bacula/src/win32/dll/bacula.def | 16 +++++++-- bacula/src/win32/libbac/Makefile | 1 + bacula/technotes-2.1 | 12 +++++++ 13 files changed, 131 insertions(+), 36 deletions(-) diff --git a/bacula/src/cats/sql_cmds.c b/bacula/src/cats/sql_cmds.c index 2dc9501480..0222e1544f 100644 --- a/bacula/src/cats/sql_cmds.c +++ b/bacula/src/cats/sql_cmds.c @@ -64,6 +64,14 @@ const char *del_JobMedia = "DELETE FROM JobMedia WHERE JobId=%s"; const char *cnt_JobMedia = "SELECT count(*) FROM JobMedia WHERE MediaId=%s"; const char *sel_JobMedia = "SELECT JobId FROM JobMedia WHERE MediaId=%s"; +/* Count Select JobIds for File deletion */ +const char *count_select_job = + "SELECT count(*) from Job " + "WHERE JobTDate<%s " + "AND ClientId=%s " + "AND PurgedFiles=0"; + + /* Select JobIds for File deletion. */ const char *select_job = "SELECT JobId from Job " @@ -427,9 +435,9 @@ const char *uar_mediatype = * for use when inserting individual files into the tree. */ const char *uar_jobid_fileindex = - "SELECT Job.JobId, File.FileIndex FROM Job,File,Path,Filename,Client " + "SELECT Job.JobId,File.FileIndex FROM Job,File,Path,Filename,Client " "WHERE Job.JobId=File.JobId " - "AND Job.StartTime<'%s' " + "AND Job.StartTime<='%s' " "AND Path.Path='%s' " "AND Filename.Name='%s' " "AND Client.Name='%s' " @@ -439,10 +447,10 @@ const char *uar_jobid_fileindex = "ORDER BY Job.StartTime DESC LIMIT 1"; const char *uar_jobids_fileindex = - "SELECT Job.JobId, File.FileIndex FROM Job,File,Path,Filename,Client " + "SELECT Job.JobId,File.FileIndex FROM Job,File,Path,Filename,Client " "WHERE Job.JobId IN (%s) " "AND Job.JobId=File.JobId " - "AND Job.StartTime<'%s' " + "AND Job.StartTime<='%s' " "AND Path.Path='%s' " "AND Filename.Name='%s' " "AND Client.Name='%s' " diff --git a/bacula/src/cats/sql_cmds.h b/bacula/src/cats/sql_cmds.h index 80d1c16772..566bf93cb3 100644 --- a/bacula/src/cats/sql_cmds.h +++ b/bacula/src/cats/sql_cmds.h @@ -37,6 +37,7 @@ extern const char CATS_IMP_EXP *select_restore_del; extern const char CATS_IMP_EXP *select_admin_del; extern const char CATS_IMP_EXP *select_migrate_del; extern const char CATS_IMP_EXP *select_job; +extern const char CATS_IMP_EXP *count_select_job; extern const char CATS_IMP_EXP *del_File; extern const char CATS_IMP_EXP *cnt_File; extern const char CATS_IMP_EXP *cnt_DelCand; diff --git a/bacula/src/dird/protos.h b/bacula/src/dird/protos.h index 6c4fbbc892..fc910f84a8 100644 --- a/bacula/src/dird/protos.h +++ b/bacula/src/dird/protos.h @@ -267,6 +267,7 @@ int purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr); void purge_files_from_job(UAContext *ua, JobId_t JobId); void purge_job_from_catalog(UAContext *ua, JobId_t JobId); void purge_job_records_from_catalog(UAContext *ua, JobId_t JobId); +void purge_jobs_from_catalog(UAContext *ua, char *jobs); /* ua_run.c */ diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index 89a0f82164..9486317ffb 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -84,17 +84,6 @@ static int count_handler(void *ctx, int num_fields, char **row) } -/* - * Called here to count the number of Jobs to be pruned - */ -static int file_count_handler(void *ctx, int num_fields, char **row) -{ - struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx; - del->tot_ids++; - return 0; -} - - /* * Called here to make in memory list of JobIds to be * deleted and the associated PurgedFiles flag. @@ -217,6 +206,7 @@ int prunecmd(UAContext *ua, const char *cmd) int prune_files(UAContext *ua, CLIENT *client) { struct s_file_del_ctx del; + struct s_count_ctx cnt; POOLMEM *query = get_pool_memory(PM_MESSAGE); int i; utime_t now, period; @@ -236,26 +226,25 @@ int prune_files(UAContext *ua, CLIENT *client) now = (utime_t)time(NULL); /* Select Jobs -- for counting */ - Mmsg(query, select_job, edit_uint64(now - period, ed1), + Mmsg(query, count_select_job, edit_uint64(now - period, ed1), edit_int64(cr.ClientId, ed2)); Dmsg3(050, "select now=%u period=%u sql=%s\n", (uint32_t)now, (uint32_t)period, query); - if (!db_sql_query(ua->db, query, file_count_handler, (void *)&del)) { - if (ua->verbose) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - } + cnt.count = 0; + if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { + bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; } - if (del.tot_ids == 0) { + if (cnt.count == 0) { if (ua->verbose) { bsendmsg(ua, _("No Files found to prune.\n")); } goto bail_out; } - if (del.tot_ids < MAX_DEL_LIST_LEN) { - del.max_ids = del.tot_ids + 1; + if (cnt.count < MAX_DEL_LIST_LEN) { + del.max_ids = cnt.count + 1; } else { del.max_ids = MAX_DEL_LIST_LEN; } @@ -264,6 +253,8 @@ int prune_files(UAContext *ua, CLIENT *client) del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); /* Now process same set but making a delete list */ + Mmsg(query, select_job, edit_uint64(now - period, ed1), + edit_int64(cr.ClientId, ed2)); db_sql_query(ua->db, query, file_delete_handler, (void *)&del); for (i=0; i < del.num_ids; i++) { @@ -417,19 +408,21 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) } /* - * OK, now we have the list of JobId's to be pruned, first check - * if the Files have been purged, if not, purge (delete) them. - * Then delete the Job entry, and finally and JobMedia records. + * OK, now we have the list of JobId's to be pruned, send them + * off to be deleted batched 1000 at a time. */ - for (i=0; i < del.num_ids; i++) { - /* Don't prune current job */ - if (ua->jcr->JobId != del.JobId[i]) { - if (!del.PurgedFiles[i]) { - purge_files_from_job(ua, del.JobId[i]); + + for (i=0; del.num_ids; ) { + for (int j=0; j<1000 && del.num_ids; j++) { + del.num_ids--; + if (ua->jcr->JobId == del.JobId[i]) { + continue; } - purge_job_from_catalog(ua, del.JobId[i]); + pm_strcat(query, ","); + pm_strcpy(query, edit_int64(del.JobId[i++], ed1)); del.num_del++; } + purge_jobs_from_catalog(ua, query); } bsendmsg(ua, _("Pruned %d %s for client %s from catalog.\n"), del.num_del, del.num_del==1?_("Job"):_("Jobs"), client->name()); diff --git a/bacula/src/dird/ua_purge.c b/bacula/src/dird/ua_purge.c index f41156558d..b909fe32f0 100644 --- a/bacula/src/dird/ua_purge.c +++ b/bacula/src/dird/ua_purge.c @@ -452,7 +452,6 @@ void purge_job_records_from_catalog(UAContext *ua, JobId_t JobId) } - /* * Remove File records for a particular Job. */ @@ -475,6 +474,39 @@ void purge_files_from_job(UAContext *ua, JobId_t JobId) db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); } +void purge_jobs_from_catalog(UAContext *ua, char *jobs) +{ + POOL_MEM query(PM_MESSAGE); + + /* Delete (or purge) records associated with the job */ + Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete File sql=%s\n", query.c_str()); + + Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str()); + + Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete Log sql=%s\n", query.c_str()); + + /* Now remove the Job record itself */ + Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete Job sql=%s\n", query.c_str()); + + /* + * Now mark Job as having files purged. This is necessary to + * avoid having too many Jobs to process in future prunings. If + * we don't do this, the number of JobId's in our in memory list + * could grow very large. + */ + Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); +} + + void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr ) {} /* ***FIXME*** implement */ diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index f5c4595adf..6cde0eb41b 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -771,6 +771,7 @@ static bool insert_file_into_findex_list(UAContext *ua, RESTORE_CTX *rx, char *f } if (!rx->found) { ua->error_msg(_("No database record found for: %s\n"), file); +// ua->error_msg("Query=%s\n", rx->query); return true; } return true; diff --git a/bacula/src/stored/acquire.c b/bacula/src/stored/acquire.c index c1b791ee3d..21ae93a3d3 100644 --- a/bacula/src/stored/acquire.c +++ b/bacula/src/stored/acquire.c @@ -378,6 +378,29 @@ DCR *acquire_device_for_append(DCR *dcr) if (dev->num_writers == 0) { memcpy(&dev->VolCatInfo, &dcr->VolCatInfo, sizeof(dev->VolCatInfo)); } + + /* + * Insanity check + * + * Check to see if the tape position as defined by the OS is + * the same as our concept. If it is not, we bail out, because + * it means the user has probably manually rewound the tape. + * Note, we check only if num_writers == 0, but this code will + * also work fine for any number of writers. If num_writers > 0, + * we probably should cancel all jobs using this device, or + * perhaps even abort the SD, or at a minimum, mark the tape + * in error. Another strategy with num_writers == 0, would be + * to rewind the tape and do a new eod() request. + */ + if (dev->is_tape() && dev->num_writers == 0) { + int32_t file = dev->get_os_tape_file(); + if (file >= 0 && file != (int32_t)dev->get_file()) { + Jmsg(jcr, M_FATAL, 0, _("Invalid tape position on volume \"%s\"" + " on device %s. Expected %d, got %d\n"), + dev->VolHdr.VolumeName, dev->print_name(), dev->get_file(), file); + goto get_out; + } + } } } else { /* Not already in append mode, so mount the device */ diff --git a/bacula/src/win32/README.mingw32 b/bacula/src/win32/README.mingw32 index 18bf31de2d..7fc095e39b 100644 --- a/bacula/src/win32/README.mingw32 +++ b/bacula/src/win32/README.mingw32 @@ -143,7 +143,15 @@ use: Replace with the base part of the name of the source code file which contains the new function. Replace with the name of the new function. Remove the leading underscore and place the result -in the .../bacula/src/win32/dll/bacula.def file. +in the file + + .../bacula/src/win32/dll/bacula.def + +If you add a new file, you will need to specify its name in + + .../bacula/src/win32/dll/Makefile +and + .../bacula/src/win32/libbac/Makefile bacula_cats.dll --------------- diff --git a/bacula/src/win32/cats/bacula_cats.def b/bacula/src/win32/cats/bacula_cats.def index 7cf4871f24..86de11ee68 100644 --- a/bacula/src/win32/cats/bacula_cats.def +++ b/bacula/src/win32/cats/bacula_cats.def @@ -32,6 +32,7 @@ _Z24db_create_storage_recordP3JCRP4B_DBP11STORAGE_DBR _Z25db_create_jobmedia_recordP3JCRP4B_DBP12JOBMEDIA_DBR _Z26db_create_mediatype_recordP3JCRP4B_DBP13MEDIATYPE_DBR _Z32db_create_file_attributes_recordP3JCRP4B_DBP8ATTR_DBR +_Z27db_write_batch_file_recordsP3JCR ; bdb_delete.c: ; sql_delete.c: @@ -95,6 +96,7 @@ select_restore_del DATA select_admin_del DATA select_migrate_del DATA select_job DATA +count_select_job DATA del_File DATA cnt_DelCand DATA del_Job DATA diff --git a/bacula/src/win32/dll/Makefile b/bacula/src/win32/dll/Makefile index 1925463f17..c31ee045dc 100644 --- a/bacula/src/win32/dll/Makefile +++ b/bacula/src/win32/dll/Makefile @@ -54,6 +54,7 @@ LIB_OBJS = \ $(OBJDIR)/bnet_server.o \ $(OBJDIR)/bpipe.o \ $(OBJDIR)/bregex.o \ + $(OBJDIR)/bsock.o \ $(OBJDIR)/bsnprintf.o \ $(OBJDIR)/bsys.o \ $(OBJDIR)/btime.o \ diff --git a/bacula/src/win32/dll/bacula.def b/bacula/src/win32/dll/bacula.def index 4304031129..e9c340641a 100644 --- a/bacula/src/win32/dll/bacula.def +++ b/bacula/src/win32/dll/bacula.def @@ -145,7 +145,7 @@ _Z10bnet_closeP5BSOCK _Z10bnet_fsendP5BSOCKPKcz _Z10init_bsockP3JCRiPKcS2_iP8sockaddr _Z10term_bsockP5BSOCK -_Z12bnet_connectP3JCRiiPKcPcS3_ii +_Z12bnet_connectP3JCRixPKcPcS3_ii _Z12is_bnet_stopP5BSOCK _Z13bnet_get_peerP5BSOCKPci _Z13bnet_strerrorP5BSOCK @@ -181,6 +181,15 @@ b_regfree _Z10bvsnprintfPciPKcS_ _Z9bsnprintfPciPKcz +; bsock.c +_ZN5BSOCK4recvEv +_ZN5BSOCK4sendEv +_ZN5BSOCK5closeEv +_ZN5BSOCK5fsendEPKcz +_ZN5BSOCK6signalEi +_ZN5BSOCK7destroyEv + + ; bsys.c: _Z11bmicrosleepll _Z15create_pid_filePcPKci @@ -263,15 +272,18 @@ _Z25crypto_digest_stream_typei _Z12daemon_startv ; dlist.c: +_Z15new_dlistStringPKc +_Z15new_dlistStringPKci _ZN5dlist13binary_insertEPvPFiS0_S0_E _ZN5dlist13binary_searchEPvPFiS0_S0_E _ZN5dlist13insert_beforeEPvS0_ _ZN5dlist22binary_insert_multipleEPvPFiS0_S0_E +_ZN5dlist4nextEPv +_ZN5dlist4prevEPv _ZN5dlist6appendEPv _ZN5dlist6removeEPv _ZN5dlist7destroyEv _ZN5dlist7prependEPv -_ZNK5dlist4nextEPKv ; edit.c: _Z10add_commasPcS_ diff --git a/bacula/src/win32/libbac/Makefile b/bacula/src/win32/libbac/Makefile index 993c22ad6a..5a4b87d6b1 100644 --- a/bacula/src/win32/libbac/Makefile +++ b/bacula/src/win32/libbac/Makefile @@ -38,6 +38,7 @@ LIB_OBJS = \ $(OBJDIR)/bregex.o \ $(OBJDIR)/bshm.o \ $(OBJDIR)/bsnprintf.o \ + $(OBJDIR)/bsock.o \ $(OBJDIR)/bsys.o \ $(OBJDIR)/btime.o \ $(OBJDIR)/btimers.o \ diff --git a/bacula/technotes-2.1 b/bacula/technotes-2.1 index c13f31e45f..27011e7e4b 100644 --- a/bacula/technotes-2.1 +++ b/bacula/technotes-2.1 @@ -1,6 +1,18 @@ Technical notes on version 2.1 General: +22Mar07 +kes Implement new prunning code that prunes up to 1000 jobs at + the same time. The same technique can be applied to a number + of other prune/purge subroutines. +kes Add an insanity check when starting a new tape job to ensure + that the tape position has not been changed. If so fail the + job. +kes Fix Win32 build for dlls to add the folllowing: + src/lib/bsock.c -- new file with new entry points + src/lib/dlist.c -- new entry points + src/cats/sql_create.c -- new entry point + src/cats/sql_cmds.c -- new DATA item exported 21Mar07 ebl batch mode cleanup bscan-test is ok now. kes Implement SD code to check length of disk volume before appending. -- 2.39.5