X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Fdird%2Fua_prune.c;h=edef66ded05f98d666487ebf1a420cc2353185eb;hb=fcc920dc99e27d72ec49b2efaf822846f7bc1125;hp=f90dd2e06bd26607f353f94c97c07d9d84cff822;hpb=e81b2718330e0f7ae37bc450d8b4e873aef66926;p=bacula%2Fbacula diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index f90dd2e06b..edef66ded0 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -1,12 +1,12 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2002-2007 Free Software Foundation Europe e.V. + Copyright (C) 2002-2009 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public + modify it under the terms of version three of the GNU Affero General Public License as published by the Free Software Foundation and included in the file LICENSE. @@ -15,12 +15,12 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Affero General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - Bacula® is a registered trademark of John Walker. + Bacula® is a registered trademark of Kern Sibbald. The licensor of Bacula is the Free Software Foundation Europe (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. @@ -32,7 +32,6 @@ * * Kern Sibbald, February MMII * - * Version $Id$ */ #include "bacula.h" @@ -41,6 +40,7 @@ /* Imported functions */ /* Forward referenced functions */ +static bool grow_del_list(struct del_ctx *del); /* * Called here to count entries to be deleted @@ -70,16 +70,11 @@ int job_delete_handler(void *ctx, int num_fields, char **row) { struct del_ctx *del = (struct del_ctx *)ctx; - if (del->num_ids == MAX_DEL_LIST_LEN) { + if (!grow_del_list(del)) { return 1; } - if (del->num_ids == del->max_ids) { - del->max_ids = (del->max_ids * 3) / 2; - del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) * del->max_ids); - del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids); - } del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]); -// Dmsg2(60, "row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]); + Dmsg2(60, "job_delete_handler row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]); del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]); return 0; } @@ -88,14 +83,9 @@ int file_delete_handler(void *ctx, int num_fields, char **row) { struct del_ctx *del = (struct del_ctx *)ctx; - if (del->num_ids == MAX_DEL_LIST_LEN) { + if (!grow_del_list(del)) { return 1; } - if (del->num_ids == del->max_ids) { - del->max_ids = (del->max_ids * 3) / 2; - del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) * - del->max_ids); - } del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]); // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]); return 0; @@ -104,21 +94,26 @@ int file_delete_handler(void *ctx, int num_fields, char **row) /* * Prune records from database * - * prune files (from) client=xxx - * prune jobs (from) client=xxx + * prune files (from) client=xxx [pool=yyy] + * prune jobs (from) client=xxx [pool=yyy] * prune volume=xxx + * prune stats */ int prunecmd(UAContext *ua, const char *cmd) { + DIRRES *dir; CLIENT *client; + POOL *pool; POOL_DBR pr; MEDIA_DBR mr; + utime_t retention; int kw; static const char *keywords[] = { NT_("Files"), NT_("Jobs"), NT_("Volume"), + NT_("Stats"), NULL}; if (!open_client_db(ua)) { @@ -127,7 +122,7 @@ int prunecmd(UAContext *ua, const char *cmd) /* First search args */ kw = find_arg_keyword(ua, keywords); - if (kw < 0 || kw > 2) { + if (kw < 0 || kw > 3) { /* no args, so ask user */ kw = do_keyword_prompt(ua, _("Choose item to prune"), keywords); } @@ -135,18 +130,38 @@ int prunecmd(UAContext *ua, const char *cmd) switch (kw) { case 0: /* prune files */ client = get_client_resource(ua); - if (!client || !confirm_retention(ua, &client->FileRetention, "File")) { + if (find_arg_with_value(ua, "pool") >= 0) { + pool = get_pool_resource(ua); + } else { + pool = NULL; + } + /* Pool File Retention takes precedence over client File Retention */ + if (pool && pool->FileRetention > 0) { + if (!confirm_retention(ua, &pool->FileRetention, "File")) { + return false; + } + } else if (!client || !confirm_retention(ua, &client->FileRetention, "File")) { return false; } - prune_files(ua, client); + prune_files(ua, client, pool); return true; case 1: /* prune jobs */ client = get_client_resource(ua); - if (!client || !confirm_retention(ua, &client->JobRetention, "Job")) { + if (find_arg_with_value(ua, "pool") >= 0) { + pool = get_pool_resource(ua); + } else { + pool = NULL; + } + /* Pool Job Retention takes precedence over client Job Retention */ + if (pool && pool->JobRetention > 0) { + if (!confirm_retention(ua, &pool->JobRetention, "Job")) { + return false; + } + } else if (!client || !confirm_retention(ua, &client->JobRetention, "Job")) { return false; } /* ****FIXME**** allow user to select JobType */ - prune_jobs(ua, client, JT_BACKUP); + prune_jobs(ua, client, pool, JT_BACKUP); return 1; case 2: /* prune volume */ if (!select_pool_and_media_dbr(ua, &pr, &mr)) { @@ -162,6 +177,17 @@ int prunecmd(UAContext *ua, const char *cmd) } prune_volume(ua, &mr); return true; + case 3: /* prune stats */ + dir = (DIRRES *)GetNextRes(R_DIRECTOR, NULL); + if (!dir->stats_retention) { + return false; + } + retention = dir->stats_retention; + if (!confirm_retention(ua, &retention, "Statistics")) { + return false; + } + prune_stats(ua, retention); + return true; default: break; } @@ -169,6 +195,26 @@ int prunecmd(UAContext *ua, const char *cmd) return true; } +/* Prune Job stat records from the database. + * + */ +int prune_stats(UAContext *ua, utime_t retention) +{ + char ed1[50]; + POOL_MEM query(PM_MESSAGE); + utime_t now = (utime_t)time(NULL); + + db_lock(ua->db); + Mmsg(query, "DELETE FROM JobHisto WHERE JobTDate < %s", + edit_int64(now - retention, ed1)); + db_sql_query(ua->db, query.c_str(), NULL, NULL); + db_unlock(ua->db); + + ua->info_msg(_("Pruned Jobs from JobHisto catalog.\n")); + + return true; +} + /* * Prune File records from the database. For any Job which * is older than the retention period, we unconditionally delete @@ -179,8 +225,10 @@ int prunecmd(UAContext *ua, const char *cmd) * * This routine assumes you want the pruning to be done. All checking * must be done before calling this routine. + * + * Note: pool can possibly be NULL. */ -int prune_files(UAContext *ua, CLIENT *client) +int prune_files(UAContext *ua, CLIENT *client, POOL *pool) { struct del_ctx del; struct s_count_ctx cnt; @@ -192,18 +240,25 @@ int prune_files(UAContext *ua, CLIENT *client) db_lock(ua->db); memset(&cr, 0, sizeof(cr)); memset(&del, 0, sizeof(del)); - bstrncpy(cr.Name, client->hdr.name, sizeof(cr.Name)); + bstrncpy(cr.Name, client->name(), sizeof(cr.Name)); if (!db_create_client_record(ua->jcr, ua->db, &cr)) { db_unlock(ua->db); return 0; } - period = client->FileRetention; + if (pool && pool->FileRetention > 0) { + period = pool->FileRetention; + } else { + period = client->FileRetention; + } now = (utime_t)time(NULL); - /* Select Jobs -- for counting */ - Mmsg(query, count_select_job, edit_uint64(now - period, ed1), - edit_int64(cr.ClientId, ed2)); +// edit_utime(now-period, ed1, sizeof(ed1)); +// Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s secs.\n"), ed1); + Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs.\n")); + /* Select Jobs -- for counting */ + edit_int64(now - period, ed1); + Mmsg(query, count_select_job, ed1, edit_int64(cr.ClientId, ed2)); Dmsg3(050, "select now=%u period=%u sql=%s\n", (uint32_t)now, (uint32_t)period, query.c_str()); cnt.count = 0; @@ -230,7 +285,7 @@ int prune_files(UAContext *ua, CLIENT *client) del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); /* Now process same set but making a delete list */ - Mmsg(query, select_job, edit_uint64(now - period, ed1), + Mmsg(query, select_job, edit_int64(now - period, ed1), edit_int64(cr.ClientId, ed2)); db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del); @@ -259,19 +314,75 @@ static void drop_temp_tables(UAContext *ua) static bool create_temp_tables(UAContext *ua) { - int i; /* Create temp tables and indicies */ - for (i=0; create_deltabs[i]; i++) { - if (!db_sql_query(ua->db, create_deltabs[i], NULL, (void *)NULL)) { - ua->error_msg("%s", db_strerror(ua->db)); - Dmsg0(050, "create DelTables table failed\n"); - return false; - } + if (!db_sql_query(ua->db, create_deltabs[db_type], NULL, (void *)NULL)) { + ua->error_msg("%s", db_strerror(ua->db)); + Dmsg0(050, "create DelTables table failed\n"); + return false; + } + if (!db_sql_query(ua->db, create_delindex, NULL, (void *)NULL)) { + ua->error_msg("%s", db_strerror(ua->db)); + Dmsg0(050, "create DelInx1 index failed\n"); + return false; + } + return true; +} + +static bool grow_del_list(struct del_ctx *del) +{ + if (del->num_ids == MAX_DEL_LIST_LEN) { + return false; + } + + if (del->num_ids == del->max_ids) { + del->max_ids = (del->max_ids * 3) / 2; + del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) * + del->max_ids); + del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids); } return true; } +struct accurate_check_ctx { + DBId_t ClientId; /* Id of client */ + DBId_t FileSetId; /* Id of FileSet */ +}; + +/* row: Job.Name, FileSet, Client.Name, FileSetId, ClientId, Type */ +static int job_select_handler(void *ctx, int num_fields, char **row) +{ + alist *lst = (alist *)ctx; + struct accurate_check_ctx *res; + ASSERT(num_fields == 6); + + /* If this job doesn't exist anymore in the configuration, delete it */ + if (GetResWithName(R_JOB, row[0]) == NULL) { + return 0; + } + + /* If this fileset doesn't exist anymore in the configuration, delete it */ + if (GetResWithName(R_FILESET, row[1]) == NULL) { + return 0; + } + + /* If this client doesn't exist anymore in the configuration, delete it */ + if (GetResWithName(R_CLIENT, row[2]) == NULL) { + return 0; + } + + /* Don't compute accurate things for Verify jobs */ + if (*row[5] == 'V') { + return 0; + } + res = (struct accurate_check_ctx*) malloc(sizeof(struct accurate_check_ctx)); + res->FileSetId = str_to_int64(row[3]); + res->ClientId = str_to_int64(row[4]); + lst->append(res); + +// Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]); + return 0; +} /* * Pruning Jobs is a bit more complicated than purging Files @@ -285,17 +396,20 @@ static bool create_temp_tables(UAContext *ua) * * For Restore Jobs there are no restrictions. */ -int prune_jobs(UAContext *ua, CLIENT *client, int JobType) +int prune_jobs(UAContext *ua, CLIENT *client, POOL *pool, int JobType) { struct del_ctx del; POOL_MEM query(PM_MESSAGE); utime_t now, period; - CLIENT_DBR cr; + CLIENT_DBR cr ; char ed1[50], ed2[50]; + alist *jobids_check=NULL; + struct accurate_check_ctx *elt; + db_list_ctx jobids, tempids; + JOB_DBR jr; db_lock(ua->db); memset(&cr, 0, sizeof(cr)); - memset(&del, 0, sizeof(del)); bstrncpy(cr.Name, client->name(), sizeof(cr.Name)); if (!db_create_client_record(ua->jcr, ua->db, &cr)) { @@ -303,7 +417,11 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) return 0; } - period = client->JobRetention; + if (pool && pool->JobRetention > 0) { + period = pool->JobRetention; + } else { + period = client->JobRetention; + } now = (utime_t)time(NULL); /* Drop any previous temporary tables still there */ @@ -314,46 +432,114 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) goto bail_out; } + edit_utime(period, ed1, sizeof(ed1)); + Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s.\n"), ed1); + + edit_int64(now - period, ed1); /* Jobs older than ed1 are good candidates */ + edit_int64(cr.ClientId, ed2); + + memset(&del, 0, sizeof(del)); + del.max_ids = 100; + del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); + del.PurgedFiles = (char *)malloc(del.max_ids); + /* * Select all files that are older than the JobRetention period - * and stuff them into the "DeletionCandidates" table. + * and add them into the "DeletionCandidates" table. */ - edit_uint64(now - period, ed1); - Mmsg(query, insert_delcand, (char)JobType, ed1, - edit_int64(cr.ClientId, ed2)); + Mmsg(query, + "INSERT INTO DelCandidates " + "SELECT JobId,PurgedFiles,FileSetId,JobFiles,JobStatus " + "FROM Job " + "WHERE Type IN ('B', 'C', 'M', 'V', 'D', 'R', 'c', 'm', 'g') " + "AND JobTDate<%s AND ClientId=%s", + ed1, ed2); + if (!db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL)) { if (ua->verbose) { ua->error_msg("%s", db_strerror(ua->db)); } - Dmsg0(050, "insert delcand failed\n"); goto bail_out; } - del.max_ids = 100; - del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); - del.PurgedFiles = (char *)malloc(del.max_ids); + /* Now, for the selection, we discard some of them in order to be always + * able to restore files. (ie, last full, last diff, last incrs) + * Note: The DISTINCT could be more useful if we don't get FileSetId + */ + jobids_check = New(alist(10, owned_by_alist)); + Mmsg(query, +"SELECT DISTINCT Job.Name, FileSet, Client.Name, Job.FileSetId, " + "Job.ClientId, Job.Type " + "FROM DelCandidates " + "JOIN Job USING (JobId) " + "JOIN Client USING (ClientId) " + "JOIN FileSet ON (Job.FileSetId = FileSet.FileSetId) " + "WHERE Job.Type IN ('B') " /* Look only Backup jobs */ + "AND Job.JobStatus IN ('T', 'W') " /* Look only useful jobs */ + ); + + /* The job_select_handler will skip jobs or filesets that are no longer + * in the configuration file. Interesting ClientId/FileSetId will be + * added to jobids_check + */ + if (!db_sql_query(ua->db, query.c_str(), job_select_handler, jobids_check)) { + ua->error_msg("%s", db_strerror(ua->db)); + } - /* ed1 = JobTDate */ - edit_int64(cr.ClientId, ed2); - switch (JobType) { - case JT_BACKUP: - Mmsg(query, select_backup_del, ed1, ed2); - break; - case JT_RESTORE: - Mmsg(query, select_restore_del, ed1, ed2); - break; - case JT_VERIFY: - Mmsg(query, select_verify_del, ed1, ed2); - break; - case JT_ADMIN: - Mmsg(query, select_admin_del, ed1, ed2); - break; - case JT_MIGRATE: - Mmsg(query, select_migrate_del, ed1, ed2); - break; + /* For this selection, we exclude current jobs used for restore or + * accurate. This will prevent to prune the last full backup used for + * current backup & restore + */ + memset(&jr, 0, sizeof(jr)); + /* To find useful jobs, we do like an incremental */ + jr.JobLevel = L_INCREMENTAL; + foreach_alist(elt, jobids_check) { + jr.ClientId = elt->ClientId; /* should be always the same */ + jr.FileSetId = elt->FileSetId; + db_accurate_get_jobids(ua->jcr, ua->db, &jr, &tempids); + jobids.cat(tempids); + } + + /* Discard latest Verify level=InitCatalog job + * TODO: can have multiple fileset + */ + Mmsg(query, + "SELECT JobId, JobTDate " + "FROM Job " + "WHERE JobTDate<%s AND ClientId=%s " + "AND Type='V' AND Level='V' " + "ORDER BY JobTDate DESC LIMIT 1", + ed1, ed2); + + if (!db_sql_query(ua->db, query.c_str(), db_list_handler, &jobids)) { + ua->error_msg("%s", db_strerror(ua->db)); } - Dmsg1(150, "Query=%s\n", query.c_str()); + /* If we found jobs to exclude from the DelCandidates list, we should + * also remove BaseJobs that can be linked with them + */ + if (jobids.count > 0) { + Dmsg1(60, "jobids to exclude before basejobs = %s\n", jobids.list); + /* We also need to exclude all basejobs used */ + db_get_used_base_jobids(ua->jcr, ua->db, jobids.list, &jobids); + + /* Removing useful jobs from the DelCandidates list */ + Mmsg(query, "DELETE FROM DelCandidates " + "WHERE JobId IN (%s) " /* JobId used in accurate */ + "AND JobFiles!=0", /* Discard when JobFiles=0 */ + jobids.list); + + if (!db_sql_query(ua->db, query.c_str(), NULL, NULL)) { + ua->error_msg("%s", db_strerror(ua->db)); + goto bail_out; /* Don't continue if the list isn't clean */ + } + Dmsg1(60, "jobids to exclude = %s\n", jobids.list); + } + + /* We use DISTINCT because we can have two times the same job */ + Mmsg(query, + "SELECT DISTINCT DelCandidates.JobId,DelCandidates.PurgedFiles " + "FROM DelCandidates"); if (!db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del)) { ua->error_msg("%s", db_strerror(ua->db)); } @@ -376,6 +562,9 @@ bail_out: if (del.PurgedFiles) { free(del.PurgedFiles); } + if (jobids_check) { + delete jobids_check; + } return 1; } @@ -425,7 +614,6 @@ int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) { POOL_MEM query(PM_MESSAGE); int count = 0; - int i; utime_t now, period; char ed1[50], ed2[50]; @@ -433,18 +621,16 @@ int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) return 0; /* cannot prune Archived volumes */ } - db_lock(ua->db); - /* * Now add to the list of JobIds for Jobs written to this Volume */ edit_int64(mr->MediaId, ed1); period = mr->VolRetention; now = (utime_t)time(NULL); - edit_uint64(now-period, ed2); + edit_int64(now-period, ed2); Mmsg(query, sel_JobMedia, ed1, ed2); - Dmsg3(250, "Now=%d period=%d now-period=%d\n", (int)now, (int)period, - (int)(now-period)); + Dmsg3(250, "Now=%d period=%d now-period=%s\n", (int)now, (int)period, + ed2); Dmsg1(050, "Query=%s\n", query.c_str()); if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)del)) { @@ -454,18 +640,44 @@ int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) Dmsg0(050, "Count failed\n"); goto bail_out; } + count = exclude_running_jobs_from_list(del); + +bail_out: + return count; +} + +/* + * We have a list of jobs to prune or purge. If any of them is + * currently running, we set its JobId to zero which effectively + * excludes it. + * + * Returns the number of jobs that can be prunned or purged. + * + */ +int exclude_running_jobs_from_list(del_ctx *prune_list) +{ + int count = 0; + JCR *jcr; + bool skip; + int i; - for (i=0; i < del->num_ids; i++) { - if (ua->jcr->JobId == del->JobId[i]) { - Dmsg2(150, "skip same job JobId[%d]=%d\n", i, (int)del->JobId[i]); - del->JobId[i] = 0; - continue; + /* Do not prune any job currently running */ + for (i=0; i < prune_list->num_ids; i++) { + skip = false; + foreach_jcr(jcr) { + if (jcr->JobId == prune_list->JobId[i]) { + Dmsg2(050, "skip running job JobId[%d]=%d\n", i, (int)prune_list->JobId[i]); + prune_list->JobId[i] = 0; + skip = true; + break; + } } - Dmsg2(150, "accept JobId[%d]=%d\n", i, (int)del->JobId[i]); + endeach_jcr(jcr); + if (skip) { + continue; /* don't increment count */ + } + Dmsg2(050, "accept JobId[%d]=%d\n", i, (int)prune_list->JobId[i]); count++; } - -bail_out: - db_unlock(ua->db); return count; }