2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2015 Kern Sibbald
5 Copyright (C) 2002-2014 Free Software Foundation Europe e.V.
7 The original author of Bacula is Kern Sibbald, with contributions
8 from many others, a complete list can be found in the file AUTHORS.
10 You may use this file and others of this release according to the
11 license defined in the LICENSE file, which includes the Affero General
12 Public License, v3.0 ("AGPLv3") and some additional permissions and
13 terms pursuant to its AGPLv3 Section 7.
15 This notice must be preserved when any source code is
16 conveyed and/or propagated.
18 Bacula(R) is a registered trademark of Kern Sibbald.
22 * Bacula Director -- User Agent Database prune Command
23 * Applies retention periods
25 * Kern Sibbald, February MMII
32 /* Imported functions */
34 /* Forward referenced functions */
35 static bool grow_del_list(struct del_ctx *del);
36 static bool prune_expired_volumes(UAContext*);
39 * Called here to count entries to be deleted
41 int del_count_handler(void *ctx, int num_fields, char **row)
43 struct s_count_ctx *cnt = (struct s_count_ctx *)ctx;
46 cnt->count = str_to_int64(row[0]);
55 * Called here to make in memory list of JobIds to be
56 * deleted and the associated PurgedFiles flag.
57 * The in memory list will then be transversed
58 * to issue the SQL DELETE commands. Note, the list
59 * is allowed to get to MAX_DEL_LIST_LEN to limit the
60 * maximum malloc'ed memory.
62 int job_delete_handler(void *ctx, int num_fields, char **row)
64 struct del_ctx *del = (struct del_ctx *)ctx;
66 if (!grow_del_list(del)) {
69 del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]);
70 Dmsg2(60, "job_delete_handler row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]);
71 del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]);
75 int file_delete_handler(void *ctx, int num_fields, char **row)
77 struct del_ctx *del = (struct del_ctx *)ctx;
79 if (!grow_del_list(del)) {
82 del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]);
83 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
88 * Prune records from database
90 * prune files (from) client=xxx [pool=yyy]
91 * prune jobs (from) client=xxx [pool=yyy]
95 int prunecmd(UAContext *ua, const char *cmd)
105 static const char *keywords[] = {
113 if (!open_new_client_db(ua)) {
117 /* First search args */
118 kw = find_arg_keyword(ua, keywords);
119 if (kw < 0 || kw > 4) {
120 /* no args, so ask user */
121 kw = do_keyword_prompt(ua, _("Choose item to prune"), keywords);
125 case 0: /* prune files */
126 if (!(client = get_client_resource(ua))) {
129 if (find_arg_with_value(ua, "pool") >= 0) {
130 pool = get_pool_resource(ua);
134 /* Pool File Retention takes precedence over client File Retention */
135 if (pool && pool->FileRetention > 0) {
136 if (!confirm_retention(ua, &pool->FileRetention, "File")) {
139 } else if (!confirm_retention(ua, &client->FileRetention, "File")) {
142 prune_files(ua, client, pool);
145 case 1: /* prune jobs */
146 if (!(client = get_client_resource(ua))) {
149 if (find_arg_with_value(ua, "pool") >= 0) {
150 pool = get_pool_resource(ua);
154 /* Pool Job Retention takes precedence over client Job Retention */
155 if (pool && pool->JobRetention > 0) {
156 if (!confirm_retention(ua, &pool->JobRetention, "Job")) {
159 } else if (!confirm_retention(ua, &client->JobRetention, "Job")) {
162 /* ****FIXME**** allow user to select JobType */
163 prune_jobs(ua, client, pool, JT_BACKUP);
166 case 2: /* prune volume */
168 /* Look for All expired volumes, mostly designed for runscript */
169 if (find_arg(ua, "expired") >= 0) {
170 return prune_expired_volumes(ua);
173 if (!select_pool_and_media_dbr(ua, &pr, &mr)) {
176 if (mr.Enabled == 2) {
177 ua->error_msg(_("Cannot prune Volume \"%s\" because it is archived.\n"),
181 if (!confirm_retention(ua, &mr.VolRetention, "Volume")) {
184 prune_volume(ua, &mr);
186 case 3: /* prune stats */
187 dir = (DIRRES *)GetNextRes(R_DIRECTOR, NULL);
188 if (!dir->stats_retention) {
191 retention = dir->stats_retention;
192 if (!confirm_retention(ua, &retention, "Statistics")) {
195 prune_stats(ua, retention);
197 case 4: /* prune snapshots */
207 /* Prune Job stat records from the database.
210 int prune_stats(UAContext *ua, utime_t retention)
213 POOL_MEM query(PM_MESSAGE);
214 utime_t now = (utime_t)time(NULL);
217 Mmsg(query, "DELETE FROM JobHisto WHERE JobTDate < %s",
218 edit_int64(now - retention, ed1));
219 db_sql_query(ua->db, query.c_str(), NULL, NULL);
222 ua->info_msg(_("Pruned Jobs from JobHisto catalog.\n"));
228 * Use pool and client specified by user to select jobs to prune
229 * returns add_from string to add in FROM clause
230 * add_where string to add in WHERE clause
232 bool prune_set_filter(UAContext *ua, CLIENT *client, POOL *pool, utime_t period,
233 POOL_MEM *add_from, POOL_MEM *add_where)
236 char ed1[50], ed2[MAX_ESCAPE_NAME_LENGTH];
237 POOL_MEM tmp(PM_MESSAGE);
239 now = (utime_t)time(NULL);
240 edit_int64(now - period, ed1);
241 Dmsg3(150, "now=%lld period=%lld JobTDate=%s\n", now, period, ed1);
242 Mmsg(tmp, " AND JobTDate < %s ", ed1);
243 pm_strcat(*add_where, tmp.c_str());
247 db_escape_string(ua->jcr, ua->db, ed2,
248 client->name(), strlen(client->name()));
249 Mmsg(tmp, " AND Client.Name = '%s' ", ed2);
250 pm_strcat(*add_where, tmp.c_str());
251 pm_strcat(*add_from, " JOIN Client USING (ClientId) ");
255 db_escape_string(ua->jcr, ua->db, ed2,
256 pool->name(), strlen(pool->name()));
257 Mmsg(tmp, " AND Pool.Name = '%s' ", ed2);
258 pm_strcat(*add_where, tmp.c_str());
259 /* Use ON() instead of USING for some old SQLite */
260 pm_strcat(*add_from, " JOIN Pool ON (Job.PoolId = Pool.PoolId) ");
262 Dmsg2(150, "f=%s w=%s\n", add_from->c_str(), add_where->c_str());
268 * Prune File records from the database. For any Job which
269 * is older than the retention period, we unconditionally delete
270 * all File records for that Job. This is simple enough that no
271 * temporary tables are needed. We simply make an in memory list of
272 * the JobIds meeting the prune conditions, then delete all File records
273 * pointing to each of those JobIds.
275 * This routine assumes you want the pruning to be done. All checking
276 * must be done before calling this routine.
278 * Note: client or pool can possibly be NULL (not both).
280 int prune_files(UAContext *ua, CLIENT *client, POOL *pool)
283 struct s_count_ctx cnt;
284 POOL_MEM query(PM_MESSAGE);
285 POOL_MEM sql_where(PM_MESSAGE);
286 POOL_MEM sql_from(PM_MESSAGE);
290 memset(&del, 0, sizeof(del));
292 if (pool && pool->FileRetention > 0) {
293 period = pool->FileRetention;
296 period = client->FileRetention;
298 } else { /* should specify at least pool or client */
303 /* Specify JobTDate and Pool.Name= and/or Client.Name= in the query */
304 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
308 // edit_utime(now-period, ed1, sizeof(ed1));
309 // Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s secs.\n"), ed1);
310 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Files.\n"));
311 /* Select Jobs -- for counting */
313 "SELECT COUNT(1) FROM Job %s WHERE PurgedFiles=0 %s",
314 sql_from.c_str(), sql_where.c_str());
315 Dmsg1(100, "select sql=%s\n", query.c_str());
317 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
318 ua->error_msg("%s", db_strerror(ua->db));
319 Dmsg0(100, "Count failed\n");
323 if (cnt.count == 0) {
325 ua->warning_msg(_("No Files found to prune.\n"));
330 if (cnt.count < MAX_DEL_LIST_LEN) {
331 del.max_ids = cnt.count + 1;
333 del.max_ids = MAX_DEL_LIST_LEN;
337 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
339 /* Now process same set but making a delete list */
340 Mmsg(query, "SELECT JobId FROM Job %s WHERE PurgedFiles=0 %s",
341 sql_from.c_str(), sql_where.c_str());
342 Dmsg1(100, "select sql=%s\n", query.c_str());
343 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
345 purge_files_from_job_list(ua, del);
347 edit_uint64_with_commas(del.num_del, ed1);
348 ua->info_msg(_("Pruned Files from %s Jobs for client %s from catalog.\n"),
349 ed1, client->name());
360 static void drop_temp_tables(UAContext *ua)
363 for (i=0; drop_deltabs[i]; i++) {
364 db_sql_query(ua->db, drop_deltabs[i], NULL, (void *)NULL);
368 static bool create_temp_tables(UAContext *ua)
370 /* Create temp tables and indicies */
371 if (!db_sql_query(ua->db, create_deltabs[ua->db->bdb_get_type_index()], NULL, (void *)NULL)) {
372 ua->error_msg("%s", db_strerror(ua->db));
373 Dmsg0(100, "create DelTables table failed\n");
376 if (!db_sql_query(ua->db, create_delindex, NULL, (void *)NULL)) {
377 ua->error_msg("%s", db_strerror(ua->db));
378 Dmsg0(100, "create DelInx1 index failed\n");
384 static bool grow_del_list(struct del_ctx *del)
386 if (del->num_ids == MAX_DEL_LIST_LEN) {
390 if (del->num_ids == del->max_ids) {
391 del->max_ids = (del->max_ids * 3) / 2;
392 del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) *
394 del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids);
399 struct accurate_check_ctx {
400 DBId_t ClientId; /* Id of client */
401 DBId_t FileSetId; /* Id of FileSet */
404 /* row: Job.Name, FileSet, Client.Name, FileSetId, ClientId, Type */
405 static int job_select_handler(void *ctx, int num_fields, char **row)
407 alist *lst = (alist *)ctx;
408 struct accurate_check_ctx *res;
409 ASSERT(num_fields == 6);
411 /* Quick fix for #5507, avoid locking res_head after db_lock() */
414 /* If this job doesn't exist anymore in the configuration, delete it */
415 if (GetResWithName(R_JOB, row[0]) == NULL) {
419 /* If this fileset doesn't exist anymore in the configuration, delete it */
420 if (GetResWithName(R_FILESET, row[1]) == NULL) {
424 /* If this client doesn't exist anymore in the configuration, delete it */
425 if (GetResWithName(R_CLIENT, row[2]) == NULL) {
430 /* Don't compute accurate things for Verify jobs */
431 if (*row[5] == 'V') {
435 res = (struct accurate_check_ctx*) malloc(sizeof(struct accurate_check_ctx));
436 res->FileSetId = str_to_int64(row[3]);
437 res->ClientId = str_to_int64(row[4]);
440 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
445 * Pruning Jobs is a bit more complicated than purging Files
446 * because we delete Job records only if there is a more current
447 * backup of the FileSet. Otherwise, we keep the Job record.
448 * In other words, we never delete the only Job record that
449 * contains a current backup of a FileSet. This prevents the
450 * Volume from being recycled and destroying a current backup.
452 * For Verify Jobs, we do not delete the last InitCatalog.
454 * For Restore Jobs there are no restrictions.
456 int prune_jobs(UAContext *ua, CLIENT *client, POOL *pool, int JobType)
458 POOL_MEM query(PM_MESSAGE);
459 POOL_MEM sql_where(PM_MESSAGE);
460 POOL_MEM sql_from(PM_MESSAGE);
463 alist *jobids_check=NULL;
464 struct accurate_check_ctx *elt;
465 db_list_ctx jobids, tempids;
468 memset(&del, 0, sizeof(del));
470 if (pool && pool->JobRetention > 0) {
471 period = pool->JobRetention;
474 period = client->JobRetention;
476 } else { /* should specify at least pool or client */
481 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
485 /* Drop any previous temporary tables still there */
486 drop_temp_tables(ua);
488 /* Create temp tables and indicies */
489 if (!create_temp_tables(ua)) {
493 edit_utime(period, ed1, sizeof(ed1));
494 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s.\n"), ed1);
497 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
498 del.PurgedFiles = (char *)malloc(del.max_ids);
501 * Select all files that are older than the JobRetention period
502 * and add them into the "DeletionCandidates" table.
505 "INSERT INTO DelCandidates "
506 "SELECT JobId,PurgedFiles,FileSetId,JobFiles,JobStatus "
507 "FROM Job %s " /* JOIN Pool/Client */
508 "WHERE Type IN ('B', 'C', 'M', 'V', 'D', 'R', 'c', 'm', 'g') "
509 " %s ", /* Pool/Client + JobTDate */
510 sql_from.c_str(), sql_where.c_str());
512 Dmsg1(100, "select sql=%s\n", query.c_str());
513 if (!db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL)) {
515 ua->error_msg("%s", db_strerror(ua->db));
520 /* Now, for the selection, we discard some of them in order to be always
521 * able to restore files. (ie, last full, last diff, last incrs)
522 * Note: The DISTINCT could be more useful if we don't get FileSetId
524 jobids_check = New(alist(10, owned_by_alist));
526 "SELECT DISTINCT Job.Name, FileSet, Client.Name, Job.FileSetId, "
527 "Job.ClientId, Job.Type "
528 "FROM DelCandidates "
529 "JOIN Job USING (JobId) "
530 "JOIN Client USING (ClientId) "
531 "JOIN FileSet ON (Job.FileSetId = FileSet.FileSetId) "
532 "WHERE Job.Type IN ('B') " /* Look only Backup jobs */
533 "AND Job.JobStatus IN ('T', 'W') " /* Look only useful jobs */
536 /* The job_select_handler will skip jobs or filesets that are no longer
537 * in the configuration file. Interesting ClientId/FileSetId will be
538 * added to jobids_check (currently disabled in 6.0.7b)
540 if (!db_sql_query(ua->db, query.c_str(), job_select_handler, jobids_check)) {
541 ua->error_msg("%s", db_strerror(ua->db));
544 /* For this selection, we exclude current jobs used for restore or
545 * accurate. This will prevent to prune the last full backup used for
546 * current backup & restore
548 memset(&jr, 0, sizeof(jr));
549 /* To find useful jobs, we do like an incremental */
550 jr.JobLevel = L_INCREMENTAL;
551 foreach_alist(elt, jobids_check) {
552 jr.ClientId = elt->ClientId; /* should be always the same */
553 jr.FileSetId = elt->FileSetId;
554 db_get_accurate_jobids(ua->jcr, ua->db, &jr, &tempids);
558 /* Discard latest Verify level=InitCatalog job
559 * TODO: can have multiple fileset
562 "SELECT JobId, JobTDate "
563 "FROM Job %s " /* JOIN Client/Pool */
564 "WHERE Type='V' AND Level='V' "
565 " %s " /* Pool, JobTDate, Client */
566 "ORDER BY JobTDate DESC LIMIT 1",
567 sql_from.c_str(), sql_where.c_str());
569 if (!db_sql_query(ua->db, query.c_str(), db_list_handler, &jobids)) {
570 ua->error_msg("%s", db_strerror(ua->db));
573 /* If we found jobs to exclude from the DelCandidates list, we should
574 * also remove BaseJobs that can be linked with them
576 if (jobids.count > 0) {
577 Dmsg1(60, "jobids to exclude before basejobs = %s\n", jobids.list);
578 /* We also need to exclude all basejobs used */
579 db_get_used_base_jobids(ua->jcr, ua->db, jobids.list, &jobids);
581 /* Removing useful jobs from the DelCandidates list */
582 Mmsg(query, "DELETE FROM DelCandidates "
583 "WHERE JobId IN (%s) " /* JobId used in accurate */
584 "AND JobFiles!=0", /* Discard when JobFiles=0 */
587 if (!db_sql_query(ua->db, query.c_str(), NULL, NULL)) {
588 ua->error_msg("%s", db_strerror(ua->db));
589 goto bail_out; /* Don't continue if the list isn't clean */
591 Dmsg1(60, "jobids to exclude = %s\n", jobids.list);
594 /* We use DISTINCT because we can have two times the same job */
596 "SELECT DISTINCT DelCandidates.JobId,DelCandidates.PurgedFiles "
597 "FROM DelCandidates");
598 if (!db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del)) {
599 ua->error_msg("%s", db_strerror(ua->db));
602 purge_job_list_from_catalog(ua, del);
604 if (del.num_del > 0) {
605 ua->info_msg(_("Pruned %d %s for client %s from catalog.\n"), del.num_del,
606 del.num_del==1?_("Job"):_("Jobs"), client->name());
607 } else if (ua->verbose) {
608 ua->info_msg(_("No Jobs found to prune.\n"));
612 drop_temp_tables(ua);
617 if (del.PurgedFiles) {
618 free(del.PurgedFiles);
628 * Prune a expired Volumes
630 static bool prune_expired_volumes(UAContext *ua)
633 POOL_MEM query(PM_MESSAGE);
634 POOL_MEM filter(PM_MESSAGE);
641 /* We can restrict to a specific pool */
642 if ((i = find_arg_with_value(ua, "pool")) >= 0) {
644 memset(&pdbr, 0, sizeof(pdbr));
645 bstrncpy(pdbr.Name, ua->argv[i], sizeof(pdbr.Name));
646 if (!db_get_pool_record(ua->jcr, ua->db, &pdbr)) {
647 ua->error_msg("%s", db_strerror(ua->db));
650 Mmsg(query, " AND PoolId = %lld ", (int64_t) pdbr.PoolId);
651 pm_strcat(filter, query.c_str());
654 /* We can restrict by MediaType */
655 if (((i = find_arg_with_value(ua, "mediatype")) >= 0) &&
656 (strlen(ua->argv[i]) <= MAX_NAME_LENGTH))
658 char ed1[MAX_ESCAPE_NAME_LENGTH];
659 db_escape_string(ua->jcr, ua->db, ed1,
660 ua->argv[i], strlen(ua->argv[i]));
661 Mmsg(query, " AND MediaType = '%s' ", ed1);
662 pm_strcat(filter, query.c_str());
666 if ((i = find_arg_with_value(ua, "limit")) >= 0) {
667 if (is_an_integer(ua->argv[i])) {
668 Mmsg(query, " LIMIT %s ", ua->argv[i]);
669 pm_strcat(filter, query.c_str());
671 ua->error_msg(_("Expecting limit argument as integer\n"));
676 lst = New(alist(5, owned_by_alist));
678 Mmsg(query, expired_volumes[db_get_type_index(ua->db)], filter.c_str());
679 db_sql_query(ua->db, query.c_str(), db_string_list_handler, &lst);
681 foreach_alist(val, lst) {
682 memset(&mr, 0, sizeof(mr));
683 bstrncpy(mr.VolumeName, val, sizeof(mr.VolumeName));
684 db_get_media_record(ua->jcr, ua->db, &mr);
685 Mmsg(query, _("Volume \"%s\""), val);
686 Dmsg1(100, "Do prune %s\n", query.c_str());
687 if (confirm_retention(ua, &mr.VolRetention, query.c_str())) {
688 Dmsg1(100, "Call Prune %s\n", query.c_str());
689 prune_volume(ua, &mr);
704 * Prune a given Volume
706 bool prune_volume(UAContext *ua, MEDIA_DBR *mr)
708 POOL_MEM query(PM_MESSAGE);
713 if (mr->Enabled == 2) {
714 return false; /* Cannot prune archived volumes */
717 memset(&del, 0, sizeof(del));
719 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
723 /* Prune only Volumes with status "Full", or "Used" */
724 if (strcmp(mr->VolStatus, "Full") == 0 ||
725 strcmp(mr->VolStatus, "Used") == 0) {
726 Dmsg2(100, "get prune list MediaId=%d Volume %s\n", (int)mr->MediaId, mr->VolumeName);
727 count = get_prune_list_for_volume(ua, mr, &del);
728 Dmsg1(100, "Num pruned = %d\n", count);
730 ua->info_msg(_("Found %d Job(s) associated with the Volume \"%s\" that will be pruned\n"),
731 count, mr->VolumeName);
732 purge_job_list_from_catalog(ua, del);
735 ua->info_msg(_("Found no Job associated with the Volume \"%s\" to prune\n"),
738 ok = is_volume_purged(ua, mr, false);
749 * Get prune list for a volume
751 int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del)
753 POOL_MEM query(PM_MESSAGE);
756 char ed1[50], ed2[50];
758 if (mr->Enabled == 2) {
759 return 0; /* cannot prune Archived volumes */
763 * Now add to the list of JobIds for Jobs written to this Volume
765 edit_int64(mr->MediaId, ed1);
766 period = mr->VolRetention;
767 now = (utime_t)time(NULL);
768 edit_int64(now-period, ed2);
769 Mmsg(query, sel_JobMedia, ed1, ed2);
770 Dmsg3(200, "Now=%d period=%d now-period=%s\n", (int)now, (int)period,
773 Dmsg1(100, "Query=%s\n", query.c_str());
774 if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)del)) {
776 ua->error_msg("%s", db_strerror(ua->db));
778 Dmsg0(100, "Count failed\n");
781 count = exclude_running_jobs_from_list(del);
788 * We have a list of jobs to prune or purge. If any of them is
789 * currently running, we set its JobId to zero which effectively
792 * Returns the number of jobs that can be prunned or purged.
795 int exclude_running_jobs_from_list(del_ctx *prune_list)
802 /* Do not prune any job currently running */
803 for (i=0; i < prune_list->num_ids; i++) {
806 if (jcr->JobId == prune_list->JobId[i]) {
807 Dmsg2(100, "skip running job JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);
808 prune_list->JobId[i] = 0;
815 continue; /* don't increment count */
817 Dmsg2(100, "accept JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);