2 Bacula® - The Network Backup Solution
4 Copyright (C) 2002-2014 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from many
7 others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 Bacula® is a registered trademark of Kern Sibbald.
18 * Bacula Director -- User Agent Database prune Command
19 * Applies retention periods
21 * Kern Sibbald, February MMII
28 /* Imported functions */
30 /* Forward referenced functions */
31 static bool grow_del_list(struct del_ctx *del);
32 static bool prune_expired_volumes(UAContext*);
35 * Called here to count entries to be deleted
37 int del_count_handler(void *ctx, int num_fields, char **row)
39 struct s_count_ctx *cnt = (struct s_count_ctx *)ctx;
42 cnt->count = str_to_int64(row[0]);
51 * Called here to make in memory list of JobIds to be
52 * deleted and the associated PurgedFiles flag.
53 * The in memory list will then be transversed
54 * to issue the SQL DELETE commands. Note, the list
55 * is allowed to get to MAX_DEL_LIST_LEN to limit the
56 * maximum malloc'ed memory.
58 int job_delete_handler(void *ctx, int num_fields, char **row)
60 struct del_ctx *del = (struct del_ctx *)ctx;
62 if (!grow_del_list(del)) {
65 del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]);
66 Dmsg2(60, "job_delete_handler row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]);
67 del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]);
71 int file_delete_handler(void *ctx, int num_fields, char **row)
73 struct del_ctx *del = (struct del_ctx *)ctx;
75 if (!grow_del_list(del)) {
78 del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]);
79 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
84 * Prune records from database
86 * prune files (from) client=xxx [pool=yyy]
87 * prune jobs (from) client=xxx [pool=yyy]
91 int prunecmd(UAContext *ua, const char *cmd)
101 static const char *keywords[] = {
108 if (!open_new_client_db(ua)) {
112 /* First search args */
113 kw = find_arg_keyword(ua, keywords);
114 if (kw < 0 || kw > 3) {
115 /* no args, so ask user */
116 kw = do_keyword_prompt(ua, _("Choose item to prune"), keywords);
120 case 0: /* prune files */
121 if (!(client = get_client_resource(ua))) {
124 if (find_arg_with_value(ua, "pool") >= 0) {
125 pool = get_pool_resource(ua);
129 /* Pool File Retention takes precedence over client File Retention */
130 if (pool && pool->FileRetention > 0) {
131 if (!confirm_retention(ua, &pool->FileRetention, "File")) {
134 } else if (!confirm_retention(ua, &client->FileRetention, "File")) {
137 prune_files(ua, client, pool);
140 case 1: /* prune jobs */
141 if (!(client = get_client_resource(ua))) {
144 if (find_arg_with_value(ua, "pool") >= 0) {
145 pool = get_pool_resource(ua);
149 /* Pool Job Retention takes precedence over client Job Retention */
150 if (pool && pool->JobRetention > 0) {
151 if (!confirm_retention(ua, &pool->JobRetention, "Job")) {
154 } else if (!confirm_retention(ua, &client->JobRetention, "Job")) {
157 /* ****FIXME**** allow user to select JobType */
158 prune_jobs(ua, client, pool, JT_BACKUP);
161 case 2: /* prune volume */
163 /* Look for All expired volumes, mostly designed for runscript */
164 if (find_arg(ua, "expired") >= 0) {
165 return prune_expired_volumes(ua);
168 if (!select_pool_and_media_dbr(ua, &pr, &mr)) {
171 if (mr.Enabled == 2) {
172 ua->error_msg(_("Cannot prune Volume \"%s\" because it is archived.\n"),
176 if (!confirm_retention(ua, &mr.VolRetention, "Volume")) {
179 prune_volume(ua, &mr);
181 case 3: /* prune stats */
182 dir = (DIRRES *)GetNextRes(R_DIRECTOR, NULL);
183 if (!dir->stats_retention) {
186 retention = dir->stats_retention;
187 if (!confirm_retention(ua, &retention, "Statistics")) {
190 prune_stats(ua, retention);
199 /* Prune Job stat records from the database.
202 int prune_stats(UAContext *ua, utime_t retention)
205 POOL_MEM query(PM_MESSAGE);
206 utime_t now = (utime_t)time(NULL);
209 Mmsg(query, "DELETE FROM JobHisto WHERE JobTDate < %s",
210 edit_int64(now - retention, ed1));
211 db_sql_query(ua->db, query.c_str(), NULL, NULL);
214 ua->info_msg(_("Pruned Jobs from JobHisto catalog.\n"));
220 * Use pool and client specified by user to select jobs to prune
221 * returns add_from string to add in FROM clause
222 * add_where string to add in WHERE clause
224 bool prune_set_filter(UAContext *ua, CLIENT *client, POOL *pool, utime_t period,
225 POOL_MEM *add_from, POOL_MEM *add_where)
228 char ed1[50], ed2[MAX_ESCAPE_NAME_LENGTH];
229 POOL_MEM tmp(PM_MESSAGE);
231 now = (utime_t)time(NULL);
232 edit_int64(now - period, ed1);
233 Dmsg3(150, "now=%lld period=%lld JobTDate=%s\n", now, period, ed1);
234 Mmsg(tmp, " AND JobTDate < %s ", ed1);
235 pm_strcat(*add_where, tmp.c_str());
239 db_escape_string(ua->jcr, ua->db, ed2,
240 client->name(), strlen(client->name()));
241 Mmsg(tmp, " AND Client.Name = '%s' ", ed2);
242 pm_strcat(*add_where, tmp.c_str());
243 pm_strcat(*add_from, " JOIN Client USING (ClientId) ");
247 db_escape_string(ua->jcr, ua->db, ed2,
248 pool->name(), strlen(pool->name()));
249 Mmsg(tmp, " AND Pool.Name = '%s' ", ed2);
250 pm_strcat(*add_where, tmp.c_str());
251 /* Use ON() instead of USING for some old SQLite */
252 pm_strcat(*add_from, " JOIN Pool ON (Job.PoolId = Pool.PoolId) ");
254 Dmsg2(150, "f=%s w=%s\n", add_from->c_str(), add_where->c_str());
260 * Prune File records from the database. For any Job which
261 * is older than the retention period, we unconditionally delete
262 * all File records for that Job. This is simple enough that no
263 * temporary tables are needed. We simply make an in memory list of
264 * the JobIds meeting the prune conditions, then delete all File records
265 * pointing to each of those JobIds.
267 * This routine assumes you want the pruning to be done. All checking
268 * must be done before calling this routine.
270 * Note: client or pool can possibly be NULL (not both).
272 int prune_files(UAContext *ua, CLIENT *client, POOL *pool)
275 struct s_count_ctx cnt;
276 POOL_MEM query(PM_MESSAGE);
277 POOL_MEM sql_where(PM_MESSAGE);
278 POOL_MEM sql_from(PM_MESSAGE);
282 memset(&del, 0, sizeof(del));
284 if (pool && pool->FileRetention > 0) {
285 period = pool->FileRetention;
288 period = client->FileRetention;
290 } else { /* should specify at least pool or client */
295 /* Specify JobTDate and Pool.Name= and/or Client.Name= in the query */
296 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
300 // edit_utime(now-period, ed1, sizeof(ed1));
301 // Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s secs.\n"), ed1);
302 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Files.\n"));
303 /* Select Jobs -- for counting */
305 "SELECT COUNT(1) FROM Job %s WHERE PurgedFiles=0 %s",
306 sql_from.c_str(), sql_where.c_str());
307 Dmsg1(100, "select sql=%s\n", query.c_str());
309 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
310 ua->error_msg("%s", db_strerror(ua->db));
311 Dmsg0(100, "Count failed\n");
315 if (cnt.count == 0) {
317 ua->warning_msg(_("No Files found to prune.\n"));
322 if (cnt.count < MAX_DEL_LIST_LEN) {
323 del.max_ids = cnt.count + 1;
325 del.max_ids = MAX_DEL_LIST_LEN;
329 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
331 /* Now process same set but making a delete list */
332 Mmsg(query, "SELECT JobId FROM Job %s WHERE PurgedFiles=0 %s",
333 sql_from.c_str(), sql_where.c_str());
334 Dmsg1(100, "select sql=%s\n", query.c_str());
335 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
337 purge_files_from_job_list(ua, del);
339 edit_uint64_with_commas(del.num_del, ed1);
340 ua->info_msg(_("Pruned Files from %s Jobs for client %s from catalog.\n"),
341 ed1, client->name());
352 static void drop_temp_tables(UAContext *ua)
355 for (i=0; drop_deltabs[i]; i++) {
356 db_sql_query(ua->db, drop_deltabs[i], NULL, (void *)NULL);
360 static bool create_temp_tables(UAContext *ua)
362 /* Create temp tables and indicies */
363 if (!db_sql_query(ua->db, create_deltabs[db_get_type_index(ua->db)], NULL, (void *)NULL)) {
364 ua->error_msg("%s", db_strerror(ua->db));
365 Dmsg0(100, "create DelTables table failed\n");
368 if (!db_sql_query(ua->db, create_delindex, NULL, (void *)NULL)) {
369 ua->error_msg("%s", db_strerror(ua->db));
370 Dmsg0(100, "create DelInx1 index failed\n");
376 static bool grow_del_list(struct del_ctx *del)
378 if (del->num_ids == MAX_DEL_LIST_LEN) {
382 if (del->num_ids == del->max_ids) {
383 del->max_ids = (del->max_ids * 3) / 2;
384 del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) *
386 del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids);
391 struct accurate_check_ctx {
392 DBId_t ClientId; /* Id of client */
393 DBId_t FileSetId; /* Id of FileSet */
396 /* row: Job.Name, FileSet, Client.Name, FileSetId, ClientId, Type */
397 static int job_select_handler(void *ctx, int num_fields, char **row)
399 alist *lst = (alist *)ctx;
400 struct accurate_check_ctx *res;
401 ASSERT(num_fields == 6);
403 /* Quick fix for #5507, avoid locking res_head after db_lock() */
406 /* If this job doesn't exist anymore in the configuration, delete it */
407 if (GetResWithName(R_JOB, row[0]) == NULL) {
411 /* If this fileset doesn't exist anymore in the configuration, delete it */
412 if (GetResWithName(R_FILESET, row[1]) == NULL) {
416 /* If this client doesn't exist anymore in the configuration, delete it */
417 if (GetResWithName(R_CLIENT, row[2]) == NULL) {
422 /* Don't compute accurate things for Verify jobs */
423 if (*row[5] == 'V') {
427 res = (struct accurate_check_ctx*) malloc(sizeof(struct accurate_check_ctx));
428 res->FileSetId = str_to_int64(row[3]);
429 res->ClientId = str_to_int64(row[4]);
432 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
437 * Pruning Jobs is a bit more complicated than purging Files
438 * because we delete Job records only if there is a more current
439 * backup of the FileSet. Otherwise, we keep the Job record.
440 * In other words, we never delete the only Job record that
441 * contains a current backup of a FileSet. This prevents the
442 * Volume from being recycled and destroying a current backup.
444 * For Verify Jobs, we do not delete the last InitCatalog.
446 * For Restore Jobs there are no restrictions.
448 int prune_jobs(UAContext *ua, CLIENT *client, POOL *pool, int JobType)
450 POOL_MEM query(PM_MESSAGE);
451 POOL_MEM sql_where(PM_MESSAGE);
452 POOL_MEM sql_from(PM_MESSAGE);
455 alist *jobids_check=NULL;
456 struct accurate_check_ctx *elt;
457 db_list_ctx jobids, tempids;
460 memset(&del, 0, sizeof(del));
462 if (pool && pool->JobRetention > 0) {
463 period = pool->JobRetention;
466 period = client->JobRetention;
468 } else { /* should specify at least pool or client */
473 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
477 /* Drop any previous temporary tables still there */
478 drop_temp_tables(ua);
480 /* Create temp tables and indicies */
481 if (!create_temp_tables(ua)) {
485 edit_utime(period, ed1, sizeof(ed1));
486 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s.\n"), ed1);
489 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
490 del.PurgedFiles = (char *)malloc(del.max_ids);
493 * Select all files that are older than the JobRetention period
494 * and add them into the "DeletionCandidates" table.
497 "INSERT INTO DelCandidates "
498 "SELECT JobId,PurgedFiles,FileSetId,JobFiles,JobStatus "
499 "FROM Job %s " /* JOIN Pool/Client */
500 "WHERE Type IN ('B', 'C', 'M', 'V', 'D', 'R', 'c', 'm', 'g') "
501 " %s ", /* Pool/Client + JobTDate */
502 sql_from.c_str(), sql_where.c_str());
504 Dmsg1(100, "select sql=%s\n", query.c_str());
505 if (!db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL)) {
507 ua->error_msg("%s", db_strerror(ua->db));
512 /* Now, for the selection, we discard some of them in order to be always
513 * able to restore files. (ie, last full, last diff, last incrs)
514 * Note: The DISTINCT could be more useful if we don't get FileSetId
516 jobids_check = New(alist(10, owned_by_alist));
518 "SELECT DISTINCT Job.Name, FileSet, Client.Name, Job.FileSetId, "
519 "Job.ClientId, Job.Type "
520 "FROM DelCandidates "
521 "JOIN Job USING (JobId) "
522 "JOIN Client USING (ClientId) "
523 "JOIN FileSet ON (Job.FileSetId = FileSet.FileSetId) "
524 "WHERE Job.Type IN ('B') " /* Look only Backup jobs */
525 "AND Job.JobStatus IN ('T', 'W') " /* Look only useful jobs */
528 /* The job_select_handler will skip jobs or filesets that are no longer
529 * in the configuration file. Interesting ClientId/FileSetId will be
530 * added to jobids_check (currently disabled in 6.0.7b)
532 if (!db_sql_query(ua->db, query.c_str(), job_select_handler, jobids_check)) {
533 ua->error_msg("%s", db_strerror(ua->db));
536 /* For this selection, we exclude current jobs used for restore or
537 * accurate. This will prevent to prune the last full backup used for
538 * current backup & restore
540 memset(&jr, 0, sizeof(jr));
541 /* To find useful jobs, we do like an incremental */
542 jr.JobLevel = L_INCREMENTAL;
543 foreach_alist(elt, jobids_check) {
544 jr.ClientId = elt->ClientId; /* should be always the same */
545 jr.FileSetId = elt->FileSetId;
546 db_accurate_get_jobids(ua->jcr, ua->db, &jr, &tempids);
550 /* Discard latest Verify level=InitCatalog job
551 * TODO: can have multiple fileset
554 "SELECT JobId, JobTDate "
555 "FROM Job %s " /* JOIN Client/Pool */
556 "WHERE Type='V' AND Level='V' "
557 " %s " /* Pool, JobTDate, Client */
558 "ORDER BY JobTDate DESC LIMIT 1",
559 sql_from.c_str(), sql_where.c_str());
561 if (!db_sql_query(ua->db, query.c_str(), db_list_handler, &jobids)) {
562 ua->error_msg("%s", db_strerror(ua->db));
565 /* If we found jobs to exclude from the DelCandidates list, we should
566 * also remove BaseJobs that can be linked with them
568 if (jobids.count > 0) {
569 Dmsg1(60, "jobids to exclude before basejobs = %s\n", jobids.list);
570 /* We also need to exclude all basejobs used */
571 db_get_used_base_jobids(ua->jcr, ua->db, jobids.list, &jobids);
573 /* Removing useful jobs from the DelCandidates list */
574 Mmsg(query, "DELETE FROM DelCandidates "
575 "WHERE JobId IN (%s) " /* JobId used in accurate */
576 "AND JobFiles!=0", /* Discard when JobFiles=0 */
579 if (!db_sql_query(ua->db, query.c_str(), NULL, NULL)) {
580 ua->error_msg("%s", db_strerror(ua->db));
581 goto bail_out; /* Don't continue if the list isn't clean */
583 Dmsg1(60, "jobids to exclude = %s\n", jobids.list);
586 /* We use DISTINCT because we can have two times the same job */
588 "SELECT DISTINCT DelCandidates.JobId,DelCandidates.PurgedFiles "
589 "FROM DelCandidates");
590 if (!db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del)) {
591 ua->error_msg("%s", db_strerror(ua->db));
594 purge_job_list_from_catalog(ua, del);
596 if (del.num_del > 0) {
597 ua->info_msg(_("Pruned %d %s for client %s from catalog.\n"), del.num_del,
598 del.num_del==1?_("Job"):_("Jobs"), client->name());
599 } else if (ua->verbose) {
600 ua->info_msg(_("No Jobs found to prune.\n"));
604 drop_temp_tables(ua);
609 if (del.PurgedFiles) {
610 free(del.PurgedFiles);
620 * Prune a expired Volumes
622 static bool prune_expired_volumes(UAContext *ua)
625 POOL_MEM query(PM_MESSAGE);
626 POOL_MEM filter(PM_MESSAGE);
633 /* We can restrict to a specific pool */
634 if ((i = find_arg_with_value(ua, "pool")) >= 0) {
636 memset(&pdbr, 0, sizeof(pdbr));
637 bstrncpy(pdbr.Name, ua->argv[i], sizeof(pdbr.Name));
638 if (!db_get_pool_record(ua->jcr, ua->db, &pdbr)) {
639 ua->error_msg("%s", db_strerror(ua->db));
642 Mmsg(query, " AND PoolId = %lld ", (int64_t) pdbr.PoolId);
643 pm_strcat(filter, query.c_str());
646 /* We can restrict by MediaType */
647 if ((i = find_arg_with_value(ua, "mediatype")) >= 0) {
648 char ed1[MAX_ESCAPE_NAME_LENGTH];
649 db_escape_string(ua->jcr, ua->db, ed1,
650 ua->argv[i], strlen(ua->argv[i]));
651 Mmsg(query, " AND MediaType = '%s' ", ed1);
652 pm_strcat(filter, query.c_str());
656 if ((i = find_arg_with_value(ua, "limit")) >= 0) {
657 if (is_an_integer(ua->argv[i])) {
658 Mmsg(query, " LIMIT %s ", ua->argv[i]);
659 pm_strcat(filter, query.c_str());
661 ua->error_msg(_("Expecting limit argument as integer\n"));
666 lst = New(alist(5, owned_by_alist));
668 Mmsg(query, expired_volumes[db_get_type_index(ua->db)], filter.c_str());
669 db_sql_query(ua->db, query.c_str(), db_string_list_handler, &lst);
671 foreach_alist(val, lst) {
672 memset(&mr, 0, sizeof(mr));
673 bstrncpy(mr.VolumeName, val, sizeof(mr.VolumeName));
674 db_get_media_record(ua->jcr, ua->db, &mr);
675 Mmsg(query, _("Volume \"%s\""), val);
676 if (confirm_retention(ua, &mr.VolRetention, query.c_str())) {
677 prune_volume(ua, &mr);
692 * Prune a given Volume
694 bool prune_volume(UAContext *ua, MEDIA_DBR *mr)
696 POOL_MEM query(PM_MESSAGE);
701 if (mr->Enabled == 2) {
702 return false; /* Cannot prune archived volumes */
705 memset(&del, 0, sizeof(del));
707 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
711 /* Prune only Volumes with status "Full", or "Used" */
712 if (strcmp(mr->VolStatus, "Full") == 0 ||
713 strcmp(mr->VolStatus, "Used") == 0) {
714 Dmsg2(100, "get prune list MediaId=%d Volume %s\n", (int)mr->MediaId, mr->VolumeName);
715 count = get_prune_list_for_volume(ua, mr, &del);
716 Dmsg1(100, "Num pruned = %d\n", count);
718 purge_job_list_from_catalog(ua, del);
720 ok = is_volume_purged(ua, mr);
731 * Get prune list for a volume
733 int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del)
735 POOL_MEM query(PM_MESSAGE);
738 char ed1[50], ed2[50];
740 if (mr->Enabled == 2) {
741 return 0; /* cannot prune Archived volumes */
745 * Now add to the list of JobIds for Jobs written to this Volume
747 edit_int64(mr->MediaId, ed1);
748 period = mr->VolRetention;
749 now = (utime_t)time(NULL);
750 edit_int64(now-period, ed2);
751 Mmsg(query, sel_JobMedia, ed1, ed2);
752 Dmsg3(250, "Now=%d period=%d now-period=%s\n", (int)now, (int)period,
755 Dmsg1(100, "Query=%s\n", query.c_str());
756 if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)del)) {
758 ua->error_msg("%s", db_strerror(ua->db));
760 Dmsg0(100, "Count failed\n");
763 count = exclude_running_jobs_from_list(del);
770 * We have a list of jobs to prune or purge. If any of them is
771 * currently running, we set its JobId to zero which effectively
774 * Returns the number of jobs that can be prunned or purged.
777 int exclude_running_jobs_from_list(del_ctx *prune_list)
784 /* Do not prune any job currently running */
785 for (i=0; i < prune_list->num_ids; i++) {
788 if (jcr->JobId == prune_list->JobId[i]) {
789 Dmsg2(100, "skip running job JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);
790 prune_list->JobId[i] = 0;
797 continue; /* don't increment count */
799 Dmsg2(100, "accept JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);