From: Kern Sibbald Date: Thu, 22 Mar 2007 13:10:04 +0000 (+0000) Subject: kes Implement new prunning code that prunes up to 1000 jobs at X-Git-Tag: Release-7.0.0~6741 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=3fdd044ffe01b33f78f3057c71c3341de496782f;p=bacula%2Fbacula kes Implement new prunning code that prunes up to 1000 jobs at the same time. The same technique can be applied to a number of other prune/purge subroutines. kes Add an insanity check when starting a new tape job to ensure that the tape position has not been changed. If so fail the job. kes Fix Win32 build for dlls to add the folllowing: src/lib/bsock.c -- new file with new entry points src/lib/dlist.c -- new entry points src/cats/sql_create.c -- new entry point src/cats/sql_cmds.c -- new DATA item exported git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4383 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/src/cats/sql_cmds.c b/bacula/src/cats/sql_cmds.c index 2dc9501480..0222e1544f 100644 --- a/bacula/src/cats/sql_cmds.c +++ b/bacula/src/cats/sql_cmds.c @@ -64,6 +64,14 @@ const char *del_JobMedia = "DELETE FROM JobMedia WHERE JobId=%s"; const char *cnt_JobMedia = "SELECT count(*) FROM JobMedia WHERE MediaId=%s"; const char *sel_JobMedia = "SELECT JobId FROM JobMedia WHERE MediaId=%s"; +/* Count Select JobIds for File deletion */ +const char *count_select_job = + "SELECT count(*) from Job " + "WHERE JobTDate<%s " + "AND ClientId=%s " + "AND PurgedFiles=0"; + + /* Select JobIds for File deletion. */ const char *select_job = "SELECT JobId from Job " @@ -427,9 +435,9 @@ const char *uar_mediatype = * for use when inserting individual files into the tree. */ const char *uar_jobid_fileindex = - "SELECT Job.JobId, File.FileIndex FROM Job,File,Path,Filename,Client " + "SELECT Job.JobId,File.FileIndex FROM Job,File,Path,Filename,Client " "WHERE Job.JobId=File.JobId " - "AND Job.StartTime<'%s' " + "AND Job.StartTime<='%s' " "AND Path.Path='%s' " "AND Filename.Name='%s' " "AND Client.Name='%s' " @@ -439,10 +447,10 @@ const char *uar_jobid_fileindex = "ORDER BY Job.StartTime DESC LIMIT 1"; const char *uar_jobids_fileindex = - "SELECT Job.JobId, File.FileIndex FROM Job,File,Path,Filename,Client " + "SELECT Job.JobId,File.FileIndex FROM Job,File,Path,Filename,Client " "WHERE Job.JobId IN (%s) " "AND Job.JobId=File.JobId " - "AND Job.StartTime<'%s' " + "AND Job.StartTime<='%s' " "AND Path.Path='%s' " "AND Filename.Name='%s' " "AND Client.Name='%s' " diff --git a/bacula/src/cats/sql_cmds.h b/bacula/src/cats/sql_cmds.h index 80d1c16772..566bf93cb3 100644 --- a/bacula/src/cats/sql_cmds.h +++ b/bacula/src/cats/sql_cmds.h @@ -37,6 +37,7 @@ extern const char CATS_IMP_EXP *select_restore_del; extern const char CATS_IMP_EXP *select_admin_del; extern const char CATS_IMP_EXP *select_migrate_del; extern const char CATS_IMP_EXP *select_job; +extern const char CATS_IMP_EXP *count_select_job; extern const char CATS_IMP_EXP *del_File; extern const char CATS_IMP_EXP *cnt_File; extern const char CATS_IMP_EXP *cnt_DelCand; diff --git a/bacula/src/dird/protos.h b/bacula/src/dird/protos.h index 6c4fbbc892..fc910f84a8 100644 --- a/bacula/src/dird/protos.h +++ b/bacula/src/dird/protos.h @@ -267,6 +267,7 @@ int purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr); void purge_files_from_job(UAContext *ua, JobId_t JobId); void purge_job_from_catalog(UAContext *ua, JobId_t JobId); void purge_job_records_from_catalog(UAContext *ua, JobId_t JobId); +void purge_jobs_from_catalog(UAContext *ua, char *jobs); /* ua_run.c */ diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index 89a0f82164..9486317ffb 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -84,17 +84,6 @@ static int count_handler(void *ctx, int num_fields, char **row) } -/* - * Called here to count the number of Jobs to be pruned - */ -static int file_count_handler(void *ctx, int num_fields, char **row) -{ - struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx; - del->tot_ids++; - return 0; -} - - /* * Called here to make in memory list of JobIds to be * deleted and the associated PurgedFiles flag. @@ -217,6 +206,7 @@ int prunecmd(UAContext *ua, const char *cmd) int prune_files(UAContext *ua, CLIENT *client) { struct s_file_del_ctx del; + struct s_count_ctx cnt; POOLMEM *query = get_pool_memory(PM_MESSAGE); int i; utime_t now, period; @@ -236,26 +226,25 @@ int prune_files(UAContext *ua, CLIENT *client) now = (utime_t)time(NULL); /* Select Jobs -- for counting */ - Mmsg(query, select_job, edit_uint64(now - period, ed1), + Mmsg(query, count_select_job, edit_uint64(now - period, ed1), edit_int64(cr.ClientId, ed2)); Dmsg3(050, "select now=%u period=%u sql=%s\n", (uint32_t)now, (uint32_t)period, query); - if (!db_sql_query(ua->db, query, file_count_handler, (void *)&del)) { - if (ua->verbose) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - } + cnt.count = 0; + if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { + bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; } - if (del.tot_ids == 0) { + if (cnt.count == 0) { if (ua->verbose) { bsendmsg(ua, _("No Files found to prune.\n")); } goto bail_out; } - if (del.tot_ids < MAX_DEL_LIST_LEN) { - del.max_ids = del.tot_ids + 1; + if (cnt.count < MAX_DEL_LIST_LEN) { + del.max_ids = cnt.count + 1; } else { del.max_ids = MAX_DEL_LIST_LEN; } @@ -264,6 +253,8 @@ int prune_files(UAContext *ua, CLIENT *client) del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); /* Now process same set but making a delete list */ + Mmsg(query, select_job, edit_uint64(now - period, ed1), + edit_int64(cr.ClientId, ed2)); db_sql_query(ua->db, query, file_delete_handler, (void *)&del); for (i=0; i < del.num_ids; i++) { @@ -417,19 +408,21 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) } /* - * OK, now we have the list of JobId's to be pruned, first check - * if the Files have been purged, if not, purge (delete) them. - * Then delete the Job entry, and finally and JobMedia records. + * OK, now we have the list of JobId's to be pruned, send them + * off to be deleted batched 1000 at a time. */ - for (i=0; i < del.num_ids; i++) { - /* Don't prune current job */ - if (ua->jcr->JobId != del.JobId[i]) { - if (!del.PurgedFiles[i]) { - purge_files_from_job(ua, del.JobId[i]); + + for (i=0; del.num_ids; ) { + for (int j=0; j<1000 && del.num_ids; j++) { + del.num_ids--; + if (ua->jcr->JobId == del.JobId[i]) { + continue; } - purge_job_from_catalog(ua, del.JobId[i]); + pm_strcat(query, ","); + pm_strcpy(query, edit_int64(del.JobId[i++], ed1)); del.num_del++; } + purge_jobs_from_catalog(ua, query); } bsendmsg(ua, _("Pruned %d %s for client %s from catalog.\n"), del.num_del, del.num_del==1?_("Job"):_("Jobs"), client->name()); diff --git a/bacula/src/dird/ua_purge.c b/bacula/src/dird/ua_purge.c index f41156558d..b909fe32f0 100644 --- a/bacula/src/dird/ua_purge.c +++ b/bacula/src/dird/ua_purge.c @@ -452,7 +452,6 @@ void purge_job_records_from_catalog(UAContext *ua, JobId_t JobId) } - /* * Remove File records for a particular Job. */ @@ -475,6 +474,39 @@ void purge_files_from_job(UAContext *ua, JobId_t JobId) db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); } +void purge_jobs_from_catalog(UAContext *ua, char *jobs) +{ + POOL_MEM query(PM_MESSAGE); + + /* Delete (or purge) records associated with the job */ + Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete File sql=%s\n", query.c_str()); + + Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str()); + + Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete Log sql=%s\n", query.c_str()); + + /* Now remove the Job record itself */ + Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete Job sql=%s\n", query.c_str()); + + /* + * Now mark Job as having files purged. This is necessary to + * avoid having too many Jobs to process in future prunings. If + * we don't do this, the number of JobId's in our in memory list + * could grow very large. + */ + Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); +} + + void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr ) {} /* ***FIXME*** implement */ diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index f5c4595adf..6cde0eb41b 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -771,6 +771,7 @@ static bool insert_file_into_findex_list(UAContext *ua, RESTORE_CTX *rx, char *f } if (!rx->found) { ua->error_msg(_("No database record found for: %s\n"), file); +// ua->error_msg("Query=%s\n", rx->query); return true; } return true; diff --git a/bacula/src/stored/acquire.c b/bacula/src/stored/acquire.c index c1b791ee3d..21ae93a3d3 100644 --- a/bacula/src/stored/acquire.c +++ b/bacula/src/stored/acquire.c @@ -378,6 +378,29 @@ DCR *acquire_device_for_append(DCR *dcr) if (dev->num_writers == 0) { memcpy(&dev->VolCatInfo, &dcr->VolCatInfo, sizeof(dev->VolCatInfo)); } + + /* + * Insanity check + * + * Check to see if the tape position as defined by the OS is + * the same as our concept. If it is not, we bail out, because + * it means the user has probably manually rewound the tape. + * Note, we check only if num_writers == 0, but this code will + * also work fine for any number of writers. If num_writers > 0, + * we probably should cancel all jobs using this device, or + * perhaps even abort the SD, or at a minimum, mark the tape + * in error. Another strategy with num_writers == 0, would be + * to rewind the tape and do a new eod() request. + */ + if (dev->is_tape() && dev->num_writers == 0) { + int32_t file = dev->get_os_tape_file(); + if (file >= 0 && file != (int32_t)dev->get_file()) { + Jmsg(jcr, M_FATAL, 0, _("Invalid tape position on volume \"%s\"" + " on device %s. Expected %d, got %d\n"), + dev->VolHdr.VolumeName, dev->print_name(), dev->get_file(), file); + goto get_out; + } + } } } else { /* Not already in append mode, so mount the device */ diff --git a/bacula/technotes-2.1 b/bacula/technotes-2.1 index c13f31e45f..27011e7e4b 100644 --- a/bacula/technotes-2.1 +++ b/bacula/technotes-2.1 @@ -1,6 +1,18 @@ Technical notes on version 2.1 General: +22Mar07 +kes Implement new prunning code that prunes up to 1000 jobs at + the same time. The same technique can be applied to a number + of other prune/purge subroutines. +kes Add an insanity check when starting a new tape job to ensure + that the tape position has not been changed. If so fail the + job. +kes Fix Win32 build for dlls to add the folllowing: + src/lib/bsock.c -- new file with new entry points + src/lib/dlist.c -- new entry points + src/cats/sql_create.c -- new entry point + src/cats/sql_cmds.c -- new DATA item exported 21Mar07 ebl batch mode cleanup bscan-test is ok now. kes Implement SD code to check length of disk volume before appending.