2 Bacula® - The Network Backup Solution
4 Copyright (C) 2002-2009 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- User Agent Database prune Command
31 * Applies retention periods
33 * Kern Sibbald, February MMII
40 /* Imported functions */
42 /* Forward referenced functions */
43 static bool grow_del_list(struct del_ctx *del);
46 * Called here to count entries to be deleted
48 int del_count_handler(void *ctx, int num_fields, char **row)
50 struct s_count_ctx *cnt = (struct s_count_ctx *)ctx;
53 cnt->count = str_to_int64(row[0]);
62 * Called here to make in memory list of JobIds to be
63 * deleted and the associated PurgedFiles flag.
64 * The in memory list will then be transversed
65 * to issue the SQL DELETE commands. Note, the list
66 * is allowed to get to MAX_DEL_LIST_LEN to limit the
67 * maximum malloc'ed memory.
69 int job_delete_handler(void *ctx, int num_fields, char **row)
71 struct del_ctx *del = (struct del_ctx *)ctx;
73 if (!grow_del_list(del)) {
76 del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]);
77 Dmsg2(60, "job_delete_handler row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]);
78 del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]);
82 int file_delete_handler(void *ctx, int num_fields, char **row)
84 struct del_ctx *del = (struct del_ctx *)ctx;
86 if (!grow_del_list(del)) {
89 del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]);
90 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
95 * Prune records from database
97 * prune files (from) client=xxx [pool=yyy]
98 * prune jobs (from) client=xxx [pool=yyy]
102 int prunecmd(UAContext *ua, const char *cmd)
112 static const char *keywords[] = {
119 if (!open_client_db(ua)) {
123 /* First search args */
124 kw = find_arg_keyword(ua, keywords);
125 if (kw < 0 || kw > 3) {
126 /* no args, so ask user */
127 kw = do_keyword_prompt(ua, _("Choose item to prune"), keywords);
131 case 0: /* prune files */
132 if (!(client = get_client_resource(ua))) {
135 if (find_arg_with_value(ua, "pool") >= 0) {
136 pool = get_pool_resource(ua);
140 /* Pool File Retention takes precedence over client File Retention */
141 if (pool && pool->FileRetention > 0) {
142 if (!confirm_retention(ua, &pool->FileRetention, "File")) {
145 } else if (!confirm_retention(ua, &client->FileRetention, "File")) {
148 prune_files(ua, client, pool);
150 case 1: /* prune jobs */
151 if (!(client = get_client_resource(ua))) {
154 if (find_arg_with_value(ua, "pool") >= 0) {
155 pool = get_pool_resource(ua);
159 /* Pool Job Retention takes precedence over client Job Retention */
160 if (pool && pool->JobRetention > 0) {
161 if (!confirm_retention(ua, &pool->JobRetention, "Job")) {
164 } else if (!confirm_retention(ua, &client->JobRetention, "Job")) {
167 /* ****FIXME**** allow user to select JobType */
168 prune_jobs(ua, client, pool, JT_BACKUP);
170 case 2: /* prune volume */
171 if (!select_pool_and_media_dbr(ua, &pr, &mr)) {
174 if (mr.Enabled == 2) {
175 ua->error_msg(_("Cannot prune Volume \"%s\" because it is archived.\n"),
179 if (!confirm_retention(ua, &mr.VolRetention, "Volume")) {
182 prune_volume(ua, &mr);
184 case 3: /* prune stats */
185 dir = (DIRRES *)GetNextRes(R_DIRECTOR, NULL);
186 if (!dir->stats_retention) {
189 retention = dir->stats_retention;
190 if (!confirm_retention(ua, &retention, "Statistics")) {
193 prune_stats(ua, retention);
202 /* Prune Job stat records from the database.
205 int prune_stats(UAContext *ua, utime_t retention)
208 POOL_MEM query(PM_MESSAGE);
209 utime_t now = (utime_t)time(NULL);
212 Mmsg(query, "DELETE FROM JobHisto WHERE JobTDate < %s",
213 edit_int64(now - retention, ed1));
214 db_sql_query(ua->db, query.c_str(), NULL, NULL);
217 ua->info_msg(_("Pruned Jobs from JobHisto catalog.\n"));
223 * Use pool and client specified by user to select jobs to prune
224 * returns add_from string to add in FROM clause
225 * add_where string to add in WHERE clause
227 bool prune_set_filter(UAContext *ua, CLIENT *client, POOL *pool, utime_t period,
228 POOL_MEM *add_from, POOL_MEM *add_where)
231 char ed1[50], ed2[MAX_ESCAPE_NAME_LENGTH];
232 POOL_MEM tmp(PM_MESSAGE);
234 now = (utime_t)time(NULL);
235 edit_int64(now - period, ed1);
236 Dmsg3(150, "now=%lld period=%lld JobTDate=%s\n", now, period, ed1);
237 Mmsg(tmp, " AND JobTDate < %s ", ed1);
238 pm_strcat(*add_where, tmp.c_str());
242 db_escape_string(ua->jcr, ua->db, ed2,
243 client->name(), strlen(client->name()));
244 Mmsg(tmp, " AND Client.Name = '%s' ", ed2);
245 pm_strcat(*add_where, tmp.c_str());
246 pm_strcat(*add_from, " JOIN Client USING (ClientId) ");
250 db_escape_string(ua->jcr, ua->db, ed2,
251 pool->name(), strlen(pool->name()));
252 Mmsg(tmp, " AND Pool.Name = '%s' ", ed2);
253 pm_strcat(*add_where, tmp.c_str());
254 /* Use ON() instead of USING for some old SQLite */
255 pm_strcat(*add_from, " JOIN Pool ON (Job.PoolId = Pool.PoolId) ");
257 Dmsg2(150, "f=%s w=%s\n", add_from->c_str(), add_where->c_str());
263 * Prune File records from the database. For any Job which
264 * is older than the retention period, we unconditionally delete
265 * all File records for that Job. This is simple enough that no
266 * temporary tables are needed. We simply make an in memory list of
267 * the JobIds meeting the prune conditions, then delete all File records
268 * pointing to each of those JobIds.
270 * This routine assumes you want the pruning to be done. All checking
271 * must be done before calling this routine.
273 * Note: client or pool can possibly be NULL (not both).
275 int prune_files(UAContext *ua, CLIENT *client, POOL *pool)
278 struct s_count_ctx cnt;
279 POOL_MEM query(PM_MESSAGE);
280 POOL_MEM sql_where(PM_MESSAGE);
281 POOL_MEM sql_from(PM_MESSAGE);
285 memset(&del, 0, sizeof(del));
287 if (pool && pool->FileRetention > 0) {
288 period = pool->FileRetention;
291 period = client->FileRetention;
293 } else { /* should specify at least pool or client */
298 /* Specify JobTDate and Pool.Name= and/or Client.Name= in the query */
299 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
303 // edit_utime(now-period, ed1, sizeof(ed1));
304 // Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s secs.\n"), ed1);
305 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Files.\n"));
306 /* Select Jobs -- for counting */
308 "SELECT COUNT(1) FROM Job %s WHERE PurgedFiles=0 %s",
309 sql_from.c_str(), sql_where.c_str());
310 Dmsg1(050, "select sql=%s\n", query.c_str());
312 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
313 ua->error_msg("%s", db_strerror(ua->db));
314 Dmsg0(050, "Count failed\n");
318 if (cnt.count == 0) {
320 ua->warning_msg(_("No Files found to prune.\n"));
325 if (cnt.count < MAX_DEL_LIST_LEN) {
326 del.max_ids = cnt.count + 1;
328 del.max_ids = MAX_DEL_LIST_LEN;
332 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
334 /* Now process same set but making a delete list */
335 Mmsg(query, "SELECT JobId FROM Job %s WHERE PurgedFiles=0 %s",
336 sql_from.c_str(), sql_where.c_str());
337 Dmsg1(050, "select sql=%s\n", query.c_str());
338 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
340 purge_files_from_job_list(ua, del);
342 edit_uint64_with_commas(del.num_del, ed1);
343 ua->info_msg(_("Pruned Files from %s Jobs for client %s from catalog.\n"),
344 ed1, client->name());
355 static void drop_temp_tables(UAContext *ua)
358 for (i=0; drop_deltabs[i]; i++) {
359 db_sql_query(ua->db, drop_deltabs[i], NULL, (void *)NULL);
363 static bool create_temp_tables(UAContext *ua)
365 /* Create temp tables and indicies */
366 if (!db_sql_query(ua->db, create_deltabs[db_get_type_index(ua->db)], NULL, (void *)NULL)) {
367 ua->error_msg("%s", db_strerror(ua->db));
368 Dmsg0(050, "create DelTables table failed\n");
371 if (!db_sql_query(ua->db, create_delindex, NULL, (void *)NULL)) {
372 ua->error_msg("%s", db_strerror(ua->db));
373 Dmsg0(050, "create DelInx1 index failed\n");
379 static bool grow_del_list(struct del_ctx *del)
381 if (del->num_ids == MAX_DEL_LIST_LEN) {
385 if (del->num_ids == del->max_ids) {
386 del->max_ids = (del->max_ids * 3) / 2;
387 del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) *
389 del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids);
394 struct accurate_check_ctx {
395 DBId_t ClientId; /* Id of client */
396 DBId_t FileSetId; /* Id of FileSet */
399 /* row: Job.Name, FileSet, Client.Name, FileSetId, ClientId, Type */
400 static int job_select_handler(void *ctx, int num_fields, char **row)
402 alist *lst = (alist *)ctx;
403 struct accurate_check_ctx *res;
404 ASSERT(num_fields == 6);
406 /* If this job doesn't exist anymore in the configuration, delete it */
407 if (GetResWithName(R_JOB, row[0]) == NULL) {
411 /* If this fileset doesn't exist anymore in the configuration, delete it */
412 if (GetResWithName(R_FILESET, row[1]) == NULL) {
416 /* If this client doesn't exist anymore in the configuration, delete it */
417 if (GetResWithName(R_CLIENT, row[2]) == NULL) {
421 /* Don't compute accurate things for Verify jobs */
422 if (*row[5] == 'V') {
426 res = (struct accurate_check_ctx*) malloc(sizeof(struct accurate_check_ctx));
427 res->FileSetId = str_to_int64(row[3]);
428 res->ClientId = str_to_int64(row[4]);
431 // Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]);
436 * Pruning Jobs is a bit more complicated than purging Files
437 * because we delete Job records only if there is a more current
438 * backup of the FileSet. Otherwise, we keep the Job record.
439 * In other words, we never delete the only Job record that
440 * contains a current backup of a FileSet. This prevents the
441 * Volume from being recycled and destroying a current backup.
443 * For Verify Jobs, we do not delete the last InitCatalog.
445 * For Restore Jobs there are no restrictions.
447 int prune_jobs(UAContext *ua, CLIENT *client, POOL *pool, int JobType)
449 POOL_MEM query(PM_MESSAGE);
450 POOL_MEM sql_where(PM_MESSAGE);
451 POOL_MEM sql_from(PM_MESSAGE);
454 alist *jobids_check=NULL;
455 struct accurate_check_ctx *elt;
456 db_list_ctx jobids, tempids;
459 memset(&del, 0, sizeof(del));
461 if (pool && pool->JobRetention > 0) {
462 period = pool->JobRetention;
465 period = client->JobRetention;
467 } else { /* should specify at least pool or client */
472 if (!prune_set_filter(ua, client, pool, period, &sql_from, &sql_where)) {
476 /* Drop any previous temporary tables still there */
477 drop_temp_tables(ua);
479 /* Create temp tables and indicies */
480 if (!create_temp_tables(ua)) {
484 edit_utime(period, ed1, sizeof(ed1));
485 Jmsg(ua->jcr, M_INFO, 0, _("Begin pruning Jobs older than %s.\n"), ed1);
488 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
489 del.PurgedFiles = (char *)malloc(del.max_ids);
492 * Select all files that are older than the JobRetention period
493 * and add them into the "DeletionCandidates" table.
496 "INSERT INTO DelCandidates "
497 "SELECT JobId,PurgedFiles,FileSetId,JobFiles,JobStatus "
498 "FROM Job %s " /* JOIN Pool/Client */
499 "WHERE Type IN ('B', 'C', 'M', 'V', 'D', 'R', 'c', 'm', 'g') "
500 " %s ", /* Pool/Client + JobTDate */
501 sql_from.c_str(), sql_where.c_str());
503 Dmsg1(050, "select sql=%s\n", query.c_str());
504 if (!db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL)) {
506 ua->error_msg("%s", db_strerror(ua->db));
511 /* Now, for the selection, we discard some of them in order to be always
512 * able to restore files. (ie, last full, last diff, last incrs)
513 * Note: The DISTINCT could be more useful if we don't get FileSetId
515 jobids_check = New(alist(10, owned_by_alist));
517 "SELECT DISTINCT Job.Name, FileSet, Client.Name, Job.FileSetId, "
518 "Job.ClientId, Job.Type "
519 "FROM DelCandidates "
520 "JOIN Job USING (JobId) "
521 "JOIN Client USING (ClientId) "
522 "JOIN FileSet ON (Job.FileSetId = FileSet.FileSetId) "
523 "WHERE Job.Type IN ('B') " /* Look only Backup jobs */
524 "AND Job.JobStatus IN ('T', 'W') " /* Look only useful jobs */
527 /* The job_select_handler will skip jobs or filesets that are no longer
528 * in the configuration file. Interesting ClientId/FileSetId will be
529 * added to jobids_check
531 if (!db_sql_query(ua->db, query.c_str(), job_select_handler, jobids_check)) {
532 ua->error_msg("%s", db_strerror(ua->db));
535 /* For this selection, we exclude current jobs used for restore or
536 * accurate. This will prevent to prune the last full backup used for
537 * current backup & restore
539 memset(&jr, 0, sizeof(jr));
540 /* To find useful jobs, we do like an incremental */
541 jr.JobLevel = L_INCREMENTAL;
542 foreach_alist(elt, jobids_check) {
543 jr.ClientId = elt->ClientId; /* should be always the same */
544 jr.FileSetId = elt->FileSetId;
545 db_accurate_get_jobids(ua->jcr, ua->db, &jr, &tempids);
549 /* Discard latest Verify level=InitCatalog job
550 * TODO: can have multiple fileset
553 "SELECT JobId, JobTDate "
554 "FROM Job %s " /* JOIN Client/Pool */
555 "WHERE Type='V' AND Level='V' "
556 " %s " /* Pool, JobTDate, Client */
557 "ORDER BY JobTDate DESC LIMIT 1",
558 sql_from.c_str(), sql_where.c_str());
560 if (!db_sql_query(ua->db, query.c_str(), db_list_handler, &jobids)) {
561 ua->error_msg("%s", db_strerror(ua->db));
564 /* If we found jobs to exclude from the DelCandidates list, we should
565 * also remove BaseJobs that can be linked with them
567 if (jobids.count > 0) {
568 Dmsg1(60, "jobids to exclude before basejobs = %s\n", jobids.list);
569 /* We also need to exclude all basejobs used */
570 db_get_used_base_jobids(ua->jcr, ua->db, jobids.list, &jobids);
572 /* Removing useful jobs from the DelCandidates list */
573 Mmsg(query, "DELETE FROM DelCandidates "
574 "WHERE JobId IN (%s) " /* JobId used in accurate */
575 "AND JobFiles!=0", /* Discard when JobFiles=0 */
578 if (!db_sql_query(ua->db, query.c_str(), NULL, NULL)) {
579 ua->error_msg("%s", db_strerror(ua->db));
580 goto bail_out; /* Don't continue if the list isn't clean */
582 Dmsg1(60, "jobids to exclude = %s\n", jobids.list);
585 /* We use DISTINCT because we can have two times the same job */
587 "SELECT DISTINCT DelCandidates.JobId,DelCandidates.PurgedFiles "
588 "FROM DelCandidates");
589 if (!db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del)) {
590 ua->error_msg("%s", db_strerror(ua->db));
593 purge_job_list_from_catalog(ua, del);
595 if (del.num_del > 0) {
596 ua->info_msg(_("Pruned %d %s for client %s from catalog.\n"), del.num_del,
597 del.num_del==1?_("Job"):_("Jobs"), client->name());
598 } else if (ua->verbose) {
599 ua->info_msg(_("No Jobs found to prune.\n"));
603 drop_temp_tables(ua);
608 if (del.PurgedFiles) {
609 free(del.PurgedFiles);
618 * Prune a given Volume
620 bool prune_volume(UAContext *ua, MEDIA_DBR *mr)
622 POOL_MEM query(PM_MESSAGE);
627 if (mr->Enabled == 2) {
628 return false; /* Cannot prune archived volumes */
631 memset(&del, 0, sizeof(del));
633 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
637 /* Prune only Volumes with status "Full", or "Used" */
638 if (strcmp(mr->VolStatus, "Full") == 0 ||
639 strcmp(mr->VolStatus, "Used") == 0) {
640 Dmsg2(050, "get prune list MediaId=%d Volume %s\n", (int)mr->MediaId, mr->VolumeName);
641 count = get_prune_list_for_volume(ua, mr, &del);
642 Dmsg1(050, "Num pruned = %d\n", count);
644 purge_job_list_from_catalog(ua, del);
646 ok = is_volume_purged(ua, mr);
657 * Get prune list for a volume
659 int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del)
661 POOL_MEM query(PM_MESSAGE);
664 char ed1[50], ed2[50];
666 if (mr->Enabled == 2) {
667 return 0; /* cannot prune Archived volumes */
671 * Now add to the list of JobIds for Jobs written to this Volume
673 edit_int64(mr->MediaId, ed1);
674 period = mr->VolRetention;
675 now = (utime_t)time(NULL);
676 edit_int64(now-period, ed2);
677 Mmsg(query, sel_JobMedia, ed1, ed2);
678 Dmsg3(250, "Now=%d period=%d now-period=%s\n", (int)now, (int)period,
681 Dmsg1(050, "Query=%s\n", query.c_str());
682 if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)del)) {
684 ua->error_msg("%s", db_strerror(ua->db));
686 Dmsg0(050, "Count failed\n");
689 count = exclude_running_jobs_from_list(del);
696 * We have a list of jobs to prune or purge. If any of them is
697 * currently running, we set its JobId to zero which effectively
700 * Returns the number of jobs that can be prunned or purged.
703 int exclude_running_jobs_from_list(del_ctx *prune_list)
710 /* Do not prune any job currently running */
711 for (i=0; i < prune_list->num_ids; i++) {
714 if (jcr->JobId == prune_list->JobId[i]) {
715 Dmsg2(050, "skip running job JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);
716 prune_list->JobId[i] = 0;
723 continue; /* don't increment count */
725 Dmsg2(050, "accept JobId[%d]=%d\n", i, (int)prune_list->JobId[i]);