2 Bacula® - The Network Backup Solution
4 Copyright (C) 2002-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- User Agent Database prune Command
31 * Applies retention periods
33 * Kern Sibbald, February MMII
40 /* Imported functions */
42 /* Forward referenced functions */
43 static bool grow_del_list(struct del_ctx *del);
46 * Called here to count entries to be deleted
48 int del_count_handler(void *ctx, int num_fields, char **row)
50 struct s_count_ctx *cnt = (struct s_count_ctx *)ctx;
53 cnt->count = str_to_int64(row[0]);
62 * Called here to make in memory list of JobIds to be
63 * deleted and the associated PurgedFiles flag.
64 * The in memory list will then be transversed
65 * to issue the SQL DELETE commands. Note, the list
66 * is allowed to get to MAX_DEL_LIST_LEN to limit the
67 * maximum malloc'ed memory.
69 int job_delete_handler(void *ctx, int num_fields, char **row)
71 struct del_ctx *del = (struct del_ctx *)ctx;
73 if (!grow_del_list(del)) {
76 del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]);
77 Dmsg2(60, "job_delete_handler row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]);
78 del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]);
82 int file_delete_handler(void *ctx, int num_fields, char **row)
84 struct del_ctx *del = (struct del_ctx *)ctx;
86 if (!grow_del_list(del)) {
89 del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]);
90 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
95 * Prune records from database
97 * prune files (from) client=xxx [pool=yyy]
98 * prune jobs (from) client=xxx [pool=yyy]
102 int prunecmd(UAContext *ua, const char *cmd)
112 static const char *keywords[] = {
119 if (!open_client_db(ua)) {
123 /* First search args */
124 kw = find_arg_keyword(ua, keywords);
125 if (kw < 0 || kw > 3) {
126 /* no args, so ask user */
127 kw = do_keyword_prompt(ua, _("Choose item to prune"), keywords);
131 case 0: /* prune files */
132 client = get_client_resource(ua);
133 if (find_arg_with_value(ua, "pool") >= 0) {
134 pool = get_pool_resource(ua);
138 /* Pool File Retention takes precedence over client File Retention */
139 if (pool && pool->FileRetention > 0) {
140 if (!confirm_retention(ua, &pool->FileRetention, "File")) {
143 } else if (!client || !confirm_retention(ua, &client->FileRetention, "File")) {
146 prune_files(ua, client, pool);
148 case 1: /* prune jobs */
149 client = get_client_resource(ua);
150 if (find_arg_with_value(ua, "pool") >= 0) {
151 pool = get_pool_resource(ua);
155 /* Pool Job Retention takes precedence over client Job Retention */
156 if (pool && pool->JobRetention > 0) {
157 if (!confirm_retention(ua, &pool->JobRetention, "Job")) {
160 } else if (!client || !confirm_retention(ua, &client->JobRetention, "Job")) {
163 /* ****FIXME**** allow user to select JobType */
164 prune_jobs(ua, client, pool, JT_BACKUP);
166 case 2: /* prune volume */
167 if (!select_pool_and_media_dbr(ua, &pr, &mr)) {
170 if (mr.Enabled == 2) {
171 ua->error_msg(_("Cannot prune Volume \"%s\" because it is archived.\n"),
175 if (!confirm_retention(ua, &mr.VolRetention, "Volume")) {
178 prune_volume(ua, &mr);
180 case 3: /* prune stats */
181 dir = (DIRRES *)GetNextRes(R_DIRECTOR, NULL);
182 if (!dir->stats_retention) {
185 retention = dir->stats_retention;
186 if (!confirm_retention(ua, &retention, "Statistics")) {
189 prune_stats(ua, retention);
198 /* Prune Job stat records from the database.
201 int prune_stats(UAContext *ua, utime_t retention)
204 POOL_MEM query(PM_MESSAGE);
205 utime_t now = (utime_t)time(NULL);
208 Mmsg(query, "DELETE FROM JobHisto WHERE JobTDate < %s",
209 edit_int64(now - retention, ed1));
210 db_sql_query(ua->db, query.c_str(), NULL, NULL);
213 ua->info_msg(_("Pruned Jobs from JobHisto catalog.\n"));
219 * Use pool and client specified by user to select jobs to prune
220 * returns add_from string to add in FROM clause
221 * add_where string to add in WHERE clause
223 bool prune_set_filter(UAContext *ua, CLIENT *client, POOL *pool, utime_t period,
224 POOL_MEM *add_from, POOL_MEM *add_where)
227 char ed1[50], ed2[MAX_ESCAPE_NAME_LENGTH];
228 POOL_MEM tmp(PM_MESSAGE);
230 now = (utime_t)time(NULL);
231 edit_int64(now - period, ed1);
232 Dmsg3(150, "now=%lld period=%lld JobTDate=%s\n", now, period, ed1);
233 Mmsg(tmp, " AND JobTDate < %s ", ed1);
234 pm_strcat(*add_where, tmp.c_str());
238 db_escape_string(ua->jcr, ua->db, ed2,
239 client->name(), strlen(client->name()));
240 Mmsg(tmp, " AND Client.Name = '%s' ", ed2);
241 pm_strcat(*add_where, tmp.c_str());
242 pm_strcat(*add_from, " JOIN Client USING (ClientId) ");
246 db_escape_string(ua->jcr, ua->db, ed2,
247 pool->name(), strlen(pool->name()));
248 Mmsg(tmp, " AND Pool.Name = '%s' ", ed2);
249 pm_strcat(*add_where, tmp.c_str());
250 /* Use ON() instead of USING for some old SQLite */
251 pm_strcat(*add_from, " JOIN Pool ON (Job.PoolId = Pool.PoolId) ");
253 Dmsg2(150, "f=%s w=%s\n", add_from->c_str(), add_where->c_str());
259 * Prune File records from the database. For any Job which
260 * is older than the retention period, we unconditionally delete
261 * all File records for that Job. This is simple enough that no
262 * temporary tables are needed. We simply make an in memory list of
263 * the JobIds meeting the prune conditions, then delete all File records
264 * pointing to each of those JobIds.
266 * This routine assumes you want the pruning to be done. All checking
267 * must be done before calling this routine.
269 * Note: client or pool can possibly be NULL (not both).
271 int prune_files(UAContext *ua, CLIENT *client, POOL *pool)
274 struct s_count_ctx cnt;
275 POOL_MEM query(PM_MESSAGE);
276 POOL_MEM sql_where(PM_MESSAGE);
277 POOL_MEM sql_from(PM_MESSAGE);
281 memset(&del, 0, sizeof(del));
283 if (pool && pool->FileRetention > 0) {
284 period = pool->FileRetention;
287 period = client->FileRetention;
289 } else { /* should specify at least pool or client */
294 /* Specify JobTDate and Pool.Name= and/or Client.Name= in the query */
295 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
299 // edit_utime(now-period, ed1, sizeof(ed1));
300 // Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s secs.\n"), ed1);
301 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs.\n"));
302 /* Select Jobs -- for counting */
304 "SELECT COUNT(1) FROM Job %s WHERE PurgedFiles=0 %s",
305 sql_from.c_str(), sql_where.c_str());
306 Dmsg1(050, "select sql=%s\n", query.c_str());
308 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
309 ua->error_msg("%s", db_strerror(ua->db));
310 Dmsg0(050, "Count failed\n");
314 if (cnt.count == 0) {
316 ua->warning_msg(_("No Files found to prune.\n"));
321 if (cnt.count < MAX_DEL_LIST_LEN) {
322 del.max_ids = cnt.count + 1;
324 del.max_ids = MAX_DEL_LIST_LEN;
328 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
330 /* Now process same set but making a delete list */
331 Mmsg(query, "SELECT JobId FROM Job %s WHERE PurgedFiles=0 %s",
332 sql_from.c_str(), sql_where.c_str());
333 Dmsg1(050, "select sql=%s\n", query.c_str());
334 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
336 purge_files_from_job_list(ua, del);
338 edit_uint64_with_commas(del.num_del, ed1);
339 ua->info_msg(_("Pruned Files from %s Jobs for client %s from catalog.\n"),
340 ed1, client->name());
351 static void drop_temp_tables(UAContext *ua)
354 for (i=0; drop_deltabs[i]; i++) {
355 db_sql_query(ua->db, drop_deltabs[i], NULL, (void *)NULL);
359 static bool create_temp_tables(UAContext *ua)
361 /* Create temp tables and indicies */
362 if (!db_sql_query(ua->db, create_deltabs[db_type], NULL, (void *)NULL)) {
363 ua->error_msg("%s", db_strerror(ua->db));
364 Dmsg0(050, "create DelTables table failed\n");
367 if (!db_sql_query(ua->db, create_delindex, NULL, (void *)NULL)) {
368 ua->error_msg("%s", db_strerror(ua->db));
369 Dmsg0(050, "create DelInx1 index failed\n");
375 static bool grow_del_list(struct del_ctx *del)
377 if (del->num_ids == MAX_DEL_LIST_LEN) {
381 if (del->num_ids == del->max_ids) {
382 del->max_ids = (del->max_ids * 3) / 2;
383 del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) *
385 del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids);
390 struct accurate_check_ctx {
391 DBId_t ClientId; /* Id of client */
392 DBId_t FileSetId; /* Id of FileSet */
395 /* row: Job.Name, FileSet, Client.Name, FileSetId, ClientId, Type */
396 static int job_select_handler(void *ctx, int num_fields, char **row)
398 alist *lst = (alist *)ctx;
399 struct accurate_check_ctx *res;
400 ASSERT(num_fields == 6);
402 /* If this job doesn't exist anymore in the configuration, delete it */
403 if (GetResWithName(R_JOB, row[0]) == NULL) {
407 /* If this fileset doesn't exist anymore in the configuration, delete it */
408 if (GetResWithName(R_FILESET, row[1]) == NULL) {
412 /* If this client doesn't exist anymore in the configuration, delete it */
413 if (GetResWithName(R_CLIENT, row[2]) == NULL) {
417 /* Don't compute accurate things for Verify jobs */
418 if (*row[5] == 'V') {
422 res = (struct accurate_check_ctx*) malloc(sizeof(struct accurate_check_ctx));
423 res->FileSetId = str_to_int64(row[3]);
424 res->ClientId = str_to_int64(row[4]);
427 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
432 * Pruning Jobs is a bit more complicated than purging Files
433 * because we delete Job records only if there is a more current
434 * backup of the FileSet. Otherwise, we keep the Job record.
435 * In other words, we never delete the only Job record that
436 * contains a current backup of a FileSet. This prevents the
437 * Volume from being recycled and destroying a current backup.
439 * For Verify Jobs, we do not delete the last InitCatalog.
441 * For Restore Jobs there are no restrictions.
443 int prune_jobs(UAContext *ua, CLIENT *client, POOL *pool, int JobType)
445 POOL_MEM query(PM_MESSAGE);
446 POOL_MEM sql_where(PM_MESSAGE);
447 POOL_MEM sql_from(PM_MESSAGE);
450 alist *jobids_check=NULL;
451 struct accurate_check_ctx *elt;
452 db_list_ctx jobids, tempids;
455 memset(&del, 0, sizeof(del));
457 if (pool && pool->JobRetention > 0) {
458 period = pool->JobRetention;
461 period = client->JobRetention;
463 } else { /* should specify at least pool or client */
468 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
472 /* Drop any previous temporary tables still there */
473 drop_temp_tables(ua);
475 /* Create temp tables and indicies */
476 if (!create_temp_tables(ua)) {
480 edit_utime(period, ed1, sizeof(ed1));
481 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s.\n"), ed1);
484 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
485 del.PurgedFiles = (char *)malloc(del.max_ids);
488 * Select all files that are older than the JobRetention period
489 * and add them into the "DeletionCandidates" table.
492 "INSERT INTO DelCandidates "
493 "SELECT JobId,PurgedFiles,FileSetId,JobFiles,JobStatus "
494 "FROM Job %s " /* JOIN Pool/Client */
495 "WHERE Type IN ('B', 'C', 'M', 'V', 'D', 'R', 'c', 'm', 'g') "
496 " %s ", /* Pool/Client + JobTDate */
497 sql_from.c_str(), sql_where.c_str());
499 Dmsg1(050, "select sql=%s\n", query.c_str());
500 if (!db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL)) {
502 ua->error_msg("%s", db_strerror(ua->db));
507 /* Now, for the selection, we discard some of them in order to be always
508 * able to restore files. (ie, last full, last diff, last incrs)
509 * Note: The DISTINCT could be more useful if we don't get FileSetId
511 jobids_check = New(alist(10, owned_by_alist));
513 "SELECT DISTINCT Job.Name, FileSet, Client.Name, Job.FileSetId, "
514 "Job.ClientId, Job.Type "
515 "FROM DelCandidates "
516 "JOIN Job USING (JobId) "
517 "JOIN Client USING (ClientId) "
518 "JOIN FileSet ON (Job.FileSetId = FileSet.FileSetId) "
519 "WHERE Job.Type IN ('B') " /* Look only Backup jobs */
520 "AND Job.JobStatus IN ('T', 'W') " /* Look only useful jobs */
523 /* The job_select_handler will skip jobs or filesets that are no longer
524 * in the configuration file. Interesting ClientId/FileSetId will be
525 * added to jobids_check
527 if (!db_sql_query(ua->db, query.c_str(), job_select_handler, jobids_check)) {
528 ua->error_msg("%s", db_strerror(ua->db));
531 /* For this selection, we exclude current jobs used for restore or
532 * accurate. This will prevent to prune the last full backup used for
533 * current backup & restore
535 memset(&jr, 0, sizeof(jr));
536 /* To find useful jobs, we do like an incremental */
537 jr.JobLevel = L_INCREMENTAL;
538 foreach_alist(elt, jobids_check) {
539 jr.ClientId = elt->ClientId; /* should be always the same */
540 jr.FileSetId = elt->FileSetId;
541 db_accurate_get_jobids(ua->jcr, ua->db, &jr, &tempids);
545 /* Discard latest Verify level=InitCatalog job
546 * TODO: can have multiple fileset
549 "SELECT JobId, JobTDate "
550 "FROM Job %s " /* JOIN Client/Pool */
551 "WHERE Type='V' AND Level='V' "
552 " %s " /* Pool, JobTDate, Client */
553 "ORDER BY JobTDate DESC LIMIT 1",
554 sql_from.c_str(), sql_where.c_str());
556 if (!db_sql_query(ua->db, query.c_str(), db_list_handler, &jobids)) {
557 ua->error_msg("%s", db_strerror(ua->db));
560 /* If we found jobs to exclude from the DelCandidates list, we should
561 * also remove BaseJobs that can be linked with them
563 if (jobids.count > 0) {
564 Dmsg1(60, "jobids to exclude before basejobs = %s\n", jobids.list);
565 /* We also need to exclude all basejobs used */
566 db_get_used_base_jobids(ua->jcr, ua->db, jobids.list, &jobids);
568 /* Removing useful jobs from the DelCandidates list */
569 Mmsg(query, "DELETE FROM DelCandidates "
570 "WHERE JobId IN (%s) " /* JobId used in accurate */
571 "AND JobFiles!=0", /* Discard when JobFiles=0 */
574 if (!db_sql_query(ua->db, query.c_str(), NULL, NULL)) {
575 ua->error_msg("%s", db_strerror(ua->db));
576 goto bail_out; /* Don't continue if the list isn't clean */
578 Dmsg1(60, "jobids to exclude = %s\n", jobids.list);
581 /* We use DISTINCT because we can have two times the same job */
583 "SELECT DISTINCT DelCandidates.JobId,DelCandidates.PurgedFiles "
584 "FROM DelCandidates");
585 if (!db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del)) {
586 ua->error_msg("%s", db_strerror(ua->db));
589 purge_job_list_from_catalog(ua, del);
591 if (del.num_del > 0) {
592 ua->info_msg(_("Pruned %d %s for client %s from catalog.\n"), del.num_del,
593 del.num_del==1?_("Job"):_("Jobs"), client->name());
594 } else if (ua->verbose) {
595 ua->info_msg(_("No Jobs found to prune.\n"));
599 drop_temp_tables(ua);
604 if (del.PurgedFiles) {
605 free(del.PurgedFiles);
614 * Prune a given Volume
616 bool prune_volume(UAContext *ua, MEDIA_DBR *mr)
618 POOL_MEM query(PM_MESSAGE);
623 if (mr->Enabled == 2) {
624 return false; /* Cannot prune archived volumes */
627 memset(&del, 0, sizeof(del));
629 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
633 /* Prune only Volumes with status "Full", or "Used" */
634 if (strcmp(mr->VolStatus, "Full") == 0 ||
635 strcmp(mr->VolStatus, "Used") == 0) {
636 Dmsg2(050, "get prune list MediaId=%d Volume %s\n", (int)mr->MediaId, mr->VolumeName);
637 count = get_prune_list_for_volume(ua, mr, &del);
638 Dmsg1(050, "Num pruned = %d\n", count);
640 purge_job_list_from_catalog(ua, del);
642 ok = is_volume_purged(ua, mr);
653 * Get prune list for a volume
655 int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del)
657 POOL_MEM query(PM_MESSAGE);
660 char ed1[50], ed2[50];
662 if (mr->Enabled == 2) {
663 return 0; /* cannot prune Archived volumes */
667 * Now add to the list of JobIds for Jobs written to this Volume
669 edit_int64(mr->MediaId, ed1);
670 period = mr->VolRetention;
671 now = (utime_t)time(NULL);
672 edit_int64(now-period, ed2);
673 Mmsg(query, sel_JobMedia, ed1, ed2);
674 Dmsg3(250, "Now=%d period=%d now-period=%s\n", (int)now, (int)period,
677 Dmsg1(050, "Query=%s\n", query.c_str());
678 if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)del)) {
680 ua->error_msg("%s", db_strerror(ua->db));
682 Dmsg0(050, "Count failed\n");
685 count = exclude_running_jobs_from_list(del);
692 * We have a list of jobs to prune or purge. If any of them is
693 * currently running, we set its JobId to zero which effectively
696 * Returns the number of jobs that can be prunned or purged.
699 int exclude_running_jobs_from_list(del_ctx *prune_list)
706 /* Do not prune any job currently running */
707 for (i=0; i < prune_list->num_ids; i++) {
710 if (jcr->JobId == prune_list->JobId[i]) {
711 Dmsg2(050, "skip running job JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);
712 prune_list->JobId[i] = 0;
719 continue; /* don't increment count */
721 Dmsg2(050, "accept JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);