3 * Bacula Director -- User Agent Database prune Command
4 * Applies retention periods
6 * Kern Sibbald, February MMII
12 Copyright (C) 2002 Kern Sibbald and John Walker
14 This program is free software; you can redistribute it and/or
15 modify it under the terms of the GNU General Public License as
16 published by the Free Software Foundation; either version 2 of
17 the License, or (at your option) any later version.
19 This program is distributed in the hope that it will be useful,
20 but WITHOUT ANY WARRANTY; without even the implied warranty of
21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22 General Public License for more details.
24 You should have received a copy of the GNU General Public
25 License along with this program; if not, write to the Free
26 Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
35 /* Forward referenced functions */
36 int prune_files(UAContext *ua, CLIENT *client);
37 int prune_jobs(UAContext *ua, CLIENT *client, int JobType);
38 int prune_volume(UAContext *ua, POOL_DBR *pr, MEDIA_DBR *mr);
39 static int mark_media_purged(UAContext *ua, MEDIA_DBR *mr);
42 #define MAX_DEL_LIST_LEN 1000000
45 * Select JobIds for File deletion.
47 static char *select_job =
48 "SELECT JobId from Job "
49 "WHERE JobTDate < %s "
54 * List of SQL commands terminated by NULL for deleting
55 * temporary tables and indicies
57 static char *drop_deltabs[] = {
58 "DROP TABLE DelCandidates",
63 * List of SQL commands to create temp table and indicies
65 static char *create_deltabs[] = {
66 "CREATE TABLE DelCandidates ("
67 "JobId INTEGER UNSIGNED NOT NULL, "
68 "PurgedFiles TINYINT, "
69 "FileSetId INTEGER UNSIGNED)",
70 "CREATE INDEX DelInx1 ON DelCandidates (JobId)",
75 * Fill candidates table with all Files subject to being deleted.
76 * This is used for pruning Jobs (first the files, then the Jobs).
78 static char *insert_delcand =
79 "INSERT INTO DelCandidates "
80 "SELECT JobId, PurgedFiles, FileSetId FROM Job "
81 "WHERE JobTDate < %s "
85 * Select files from the DelCandidates table that have a
86 * more recent backup -- i.e. are not the only backup.
87 * This is the list of files to delete for a Backup Job.
89 static char *select_backup_del =
90 "SELECT DelCandidates.JobId "
91 "FROM Job,DelCandidates "
92 "WHERE Job.JobTDate >= %s "
93 "AND Job.ClientId=%d "
94 "AND Job.JobType='B' "
96 "AND Job.JobStatus='T' "
97 "AND Job.FileSetId=DelCandidates.FileSetId";
100 * Select files from the DelCandidates table that have a
101 * more recent InitCatalog -- i.e. are not the only InitCatalog
102 * This is the list of files to delete for a Verify Job.
104 static char *select_verify_del =
105 "SELECT DelCandidates.JobId "
106 "FROM Job,DelCandidates "
107 "WHERE Job.JobTDate >= %s "
108 "AND Job.ClientId=%d "
109 "AND Job.JobType='V' "
111 "AND Job.JobStatus='T' "
112 "AND Job.FileSetId=DelCandidates.FileSetId";
115 * Select files from the DelCandidates table.
116 * This is the list of files to delete for a Restore Job.
118 static char *select_restore_del =
119 "SELECT DelCandidates.JobId "
120 "FROM Job,DelCandidates "
121 "WHERE Job.JobTDate >= %s "
122 "AND Job.ClientId=%d "
123 "AND Job.JobType='R'";
125 /* In memory list of JobIds */
126 struct s_file_del_ctx {
128 int num_ids; /* ids stored */
129 int max_ids; /* size of array */
130 int num_del; /* number deleted */
131 int tot_ids; /* total to process */
134 struct s_job_del_ctx {
135 JobId_t *JobId; /* array of JobIds */
136 char *PurgedFiles; /* Array of PurgedFile flags */
137 int num_ids; /* ids stored */
138 int max_ids; /* size of array */
139 int num_del; /* number deleted */
140 int tot_ids; /* total to process */
149 * Called here to count entries to be deleted
151 static int count_handler(void *ctx, int num_fields, char **row)
153 struct s_count_ctx *cnt = (struct s_count_ctx *)ctx;
156 cnt->count = atoi(row[0]);
165 * Called here to count the number of Jobs to be pruned
167 static int file_count_handler(void *ctx, int num_fields, char **row)
169 struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx;
176 * Called here to make in memory list of JobIds to be
177 * deleted and the associated PurgedFiles flag.
178 * The in memory list will then be transversed
179 * to issue the SQL DELETE commands. Note, the list
180 * is allowed to get to MAX_DEL_LIST_LEN to limit the
181 * maximum malloc'ed memory.
183 static int job_delete_handler(void *ctx, int num_fields, char **row)
185 struct s_job_del_ctx *del = (struct s_job_del_ctx *)ctx;
187 if (del->num_ids == MAX_DEL_LIST_LEN) {
190 if (del->num_ids == del->max_ids) {
191 del->max_ids = (del->max_ids * 3) / 2;
192 del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) * del->max_ids);
193 del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids);
195 del->JobId[del->num_ids] = (JobId_t)strtod(row[0], NULL);
196 del->PurgedFiles[del->num_ids++] = (char)atoi(row[0]);
200 static int file_delete_handler(void *ctx, int num_fields, char **row)
202 struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx;
204 if (del->num_ids == MAX_DEL_LIST_LEN) {
207 if (del->num_ids == del->max_ids) {
208 del->max_ids = (del->max_ids * 3) / 2;
209 del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) *
212 del->JobId[del->num_ids++] = (JobId_t)strtod(row[0], NULL);
217 * Prune records from database
219 * prune files (from) client=xxx
220 * prune jobs (from) client=xxx
223 int prunecmd(UAContext *ua, char *cmd)
229 static char *keywords[] = {
237 switch (find_arg_keyword(ua, keywords)) {
239 client = select_client_resource(ua);
240 if (!client || !confirm_retention(ua, &client->FileRetention, "File")) {
243 prune_files(ua, client);
246 client = select_client_resource(ua);
247 if (!client || !confirm_retention(ua, &client->JobRetention, "Job")) {
250 /* ****FIXME**** allow user to select JobType */
251 prune_jobs(ua, client, JT_BACKUP);
254 if (!select_pool_and_media_dbr(ua, &pr, &mr)) {
257 if (!confirm_retention(ua, &mr.VolRetention, "Volume")) {
260 prune_volume(ua, &pr, &mr);
265 switch (do_keyword_prompt(ua, _("Choose item to prune"), keywords)) {
267 client = select_client_resource(ua);
268 if (!client || !confirm_retention(ua, &client->FileRetention, "File")) {
271 prune_files(ua, client);
274 client = select_client_resource(ua);
275 if (!client || !confirm_retention(ua, &client->JobRetention, "Job")) {
278 /* ****FIXME**** allow user to select JobType */
279 prune_jobs(ua, client, JT_BACKUP);
282 if (!select_pool_and_media_dbr(ua, &pr, &mr)) {
285 if (!confirm_retention(ua, &mr.VolRetention, "Volume")) {
288 prune_volume(ua, &pr, &mr);
295 * Prune File records from the database. For any Job which
296 * is older than the retention period, we unconditionally delete
297 * all File records for that Job. This is simple enough that no
298 * temporary tables are needed. We simply make an in memory list of
299 * the JobIds meeting the prune conditions, then delete all File records
300 * pointing to each of those JobIds.
302 * This routine assumes you want the pruning to be done. All checking
303 * must be done before calling this routine.
305 int prune_files(UAContext *ua, CLIENT *client)
307 struct s_file_del_ctx del;
308 POOLMEM *query = get_pool_memory(PM_MESSAGE);
312 char ed1[50], ed2[50];
315 memset(&cr, 0, sizeof(cr));
316 memset(&del, 0, sizeof(del));
317 strcpy(cr.Name, client->hdr.name);
318 if (!db_create_client_record(ua->db, &cr)) {
323 period = client->FileRetention;
324 now = (btime_t)time(NULL);
326 /* Select Jobs -- for counting */
327 Mmsg(&query, select_job, edit_uint64(now - period, ed1), cr.ClientId);
328 Dmsg1(050, "select sql=%s\n", query);
329 if (!db_sql_query(ua->db, query, file_count_handler, (void *)&del)) {
331 bsendmsg(ua, "%s", db_strerror(ua->db));
333 Dmsg0(050, "Count failed\n");
337 if (del.tot_ids == 0) {
339 bsendmsg(ua, _("No Files found for client %s to prune from %s catalog.\n"),
340 client->hdr.name, client->catalog->hdr.name);
345 if (del.tot_ids < MAX_DEL_LIST_LEN) {
346 del.max_ids = del.tot_ids + 1;
348 del.max_ids = MAX_DEL_LIST_LEN;
352 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
354 /* Now process same set but making delete list */
355 db_sql_query(ua->db, query, file_delete_handler, (void *)&del);
357 for (i=0; i < del.num_ids; i++) {
358 struct s_count_ctx cnt;
359 Dmsg1(050, "Delete JobId=%d\n", del.JobId[i]);
360 Mmsg(&query, "SELECT count(*) FROM File WHERE JobId=%d", del.JobId[i]);
362 db_sql_query(ua->db, query, count_handler, (void *)&cnt);
363 del.tot_ids += cnt.count;
364 Mmsg(&query, "DELETE FROM File WHERE JobId=%d", del.JobId[i]);
365 db_sql_query(ua->db, query, NULL, (void *)NULL);
367 * Now mark Job as having files purged. This is necessary to
368 * avoid having too many Jobs to process in future prunings. If
369 * we don't do this, the number of JobId's in our in memory list
370 * will grow very large.
372 Mmsg(&query, "UPDATE Job Set PurgedFiles=1 WHERE JobId=%d", del.JobId[i]);
373 db_sql_query(ua->db, query, NULL, (void *)NULL);
374 Dmsg1(050, "Del sql=%s\n", query);
376 edit_uint64_with_commas(del.tot_ids, ed1);
377 edit_uint64_with_commas(del.num_ids, ed2);
378 bsendmsg(ua, _("Pruned %s Files from %s Jobs for client %s from %s catalog.\n"),
379 ed1, ed2, client->hdr.name, client->catalog->hdr.name);
386 free_pool_memory(query);
391 static void drop_temp_tables(UAContext *ua)
394 for (i=0; drop_deltabs[i]; i++) {
395 db_sql_query(ua->db, drop_deltabs[i], NULL, (void *)NULL);
399 static int create_temp_tables(UAContext *ua)
402 /* Create temp tables and indicies */
403 for (i=0; create_deltabs[i]; i++) {
404 if (!db_sql_query(ua->db, create_deltabs[i], NULL, (void *)NULL)) {
405 bsendmsg(ua, "%s", db_strerror(ua->db));
406 Dmsg0(050, "create DelTables table failed\n");
416 * Purging Jobs is a bit more complicated than purging Files
417 * because we delete Job records only if there is a more current
418 * backup of the FileSet. Otherwise, we keep the Job record.
419 * In other words, we never delete the only Job record that
420 * contains a current backup of a FileSet. This prevents the
421 * Volume from being recycled and destroying a current backup.
423 * For Verify Jobs, we do not delete the last InitCatalog.
425 * For Restore Jobs there are no restrictions.
427 int prune_jobs(UAContext *ua, CLIENT *client, int JobType)
429 struct s_job_del_ctx del;
430 struct s_count_ctx cnt;
431 char *query = (char *)get_pool_memory(PM_MESSAGE);
438 memset(&cr, 0, sizeof(cr));
439 memset(&del, 0, sizeof(del));
440 strcpy(cr.Name, client->hdr.name);
441 if (!db_create_client_record(ua->db, &cr)) {
446 period = client->JobRetention;
447 now = (btime_t)time(NULL);
449 /* Drop any previous temporary tables still there */
450 drop_temp_tables(ua);
452 /* Create temp tables and indicies */
453 if (!create_temp_tables(ua)) {
458 * Select all files that are older than the JobRetention period
459 * and stuff them into the "DeletionCandidates" table.
461 edit_uint64(now - period, ed1);
462 Mmsg(&query, insert_delcand, ed1, cr.ClientId);
463 if (!db_sql_query(ua->db, query, NULL, (void *)NULL)) {
465 bsendmsg(ua, "%s", db_strerror(ua->db));
467 Dmsg0(050, "insert delcand failed\n");
471 /* Count Files to be deleted */
472 strcpy(query, "SELECT count(*) FROM DelCandidates");
473 Dmsg1(100, "select sql=%s\n", query);
474 if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) {
476 bsendmsg(ua, "%s", db_strerror(ua->db));
478 Dmsg0(050, "Count failed\n");
482 if (cnt.count == 0) {
484 bsendmsg(ua, _("No Jobs found for client %s to prune from %s catalog.\n"),
485 client->hdr.name, client->catalog->hdr.name);
490 if (cnt.count < MAX_DEL_LIST_LEN) {
491 del.max_ids = cnt.count + 1;
493 del.max_ids = MAX_DEL_LIST_LEN;
495 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
496 del.PurgedFiles = (char *)malloc(del.max_ids);
501 Mmsg(&query, select_backup_del, ed1, cr.ClientId);
504 Mmsg(&query, select_restore_del, ed1, cr.ClientId);
507 Mmsg(&query, select_verify_del, ed1, cr.ClientId);
510 db_sql_query(ua->db, query, job_delete_handler, (void *)&del);
513 * OK, now we have the list of JobId's to be pruned, first check
514 * if the Files have been purged, if not, purge (delete) them.
515 * Then delete the Job entry, and finally and JobMedia records.
517 for (i=0; i < del.num_ids; i++) {
518 Dmsg1(050, "Delete JobId=%d\n", del.JobId[i]);
519 if (!del.PurgedFiles[i]) {
520 Mmsg(&query, "DELETE FROM File WHERE JobId=%d", del.JobId[i]);
521 db_sql_query(ua->db, query, NULL, (void *)NULL);
522 Dmsg1(050, "Del sql=%s\n", query);
525 Mmsg(&query, "DELETE FROM Job WHERE JobId=%d", del.JobId[i]);
526 db_sql_query(ua->db, query, NULL, (void *)NULL);
527 Dmsg1(050, "Del sql=%s\n", query);
529 Mmsg(&query, "DELETE FROM JobMedia WHERE JobId=%d", del.JobId[i]);
530 db_sql_query(ua->db, query, NULL, (void *)NULL);
531 Dmsg1(050, "Del sql=%s\n", query);
533 bsendmsg(ua, _("Pruned %d %s for client %s from %s catalog.\n"), del.num_ids,
534 del.num_ids==1?_("Job"):_("Jobs"), client->hdr.name, client->catalog->hdr.name);
537 drop_temp_tables(ua);
542 if (del.PurgedFiles) {
543 free(del.PurgedFiles);
545 free_pool_memory(query);
550 * Prune a given Volume
552 int prune_volume(UAContext *ua, POOL_DBR *pr, MEDIA_DBR *mr)
554 char *query = (char *)get_pool_memory(PM_MESSAGE);
555 struct s_count_ctx cnt;
556 struct s_file_del_ctx del;
562 memset(&jr, 0, sizeof(jr));
563 memset(&del, 0, sizeof(del));
565 Mmsg(&query, "SELECT count(*) FROM JobMedia WHERE MediaId=%d", mr->MediaId);
566 if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) {
567 bsendmsg(ua, "%s", db_strerror(ua->db));
568 Dmsg0(050, "Count failed\n");
572 if (cnt.count == 0) {
574 bsendmsg(ua, "There are no Jobs associated with Volume %s. It is purged.\n",
577 stat = mark_media_purged(ua, mr);
581 if (cnt.count < MAX_DEL_LIST_LEN) {
582 del.max_ids = cnt.count + 1;
584 del.max_ids = MAX_DEL_LIST_LEN;
587 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
589 /* ***FIXME*** could make this do JobTDate check too */
590 Mmsg(&query, "SELECT JobId FROM JobMedia WHERE MediaId=%d", mr->MediaId);
591 if (!db_sql_query(ua->db, query, file_delete_handler, (void *)&del)) {
593 bsendmsg(ua, "%s", db_strerror(ua->db));
595 Dmsg0(050, "Count failed\n");
599 /* Use Volume Retention to prune Jobs and Files */
600 period = mr->VolRetention;
601 now = (btime_t)time(NULL);
603 Dmsg3(200, "Now=%d period=%d now-period=%d\n", (int)now, (int)period,
605 for (i=0; i < del.num_ids; i++) {
606 jr.JobId = del.JobId[i];
607 if (!db_get_job_record(ua->db, &jr)) {
610 Dmsg2(200, "Looking at %s JobTdate=%d\n", jr.Job, (int)jr.JobTDate);
611 if (jr.JobTDate >= (now - period)) {
614 Dmsg2(200, "Delete JobId=%d Job=%s\n", del.JobId[i], jr.Job);
615 Mmsg(&query, "DELETE FROM File WHERE JobId=%d", del.JobId[i]);
616 db_sql_query(ua->db, query, NULL, (void *)NULL);
617 Mmsg(&query, "DELETE FROM Job WHERE JobId=%d", del.JobId[i]);
618 db_sql_query(ua->db, query, NULL, (void *)NULL);
619 Mmsg(&query, "DELETE FROM JobMedia WHERE JobId=%d", del.JobId[i]);
620 db_sql_query(ua->db, query, NULL, (void *)NULL);
621 Dmsg1(050, "Del sql=%s\n", query);
627 if (ua->verbose && del.num_del != 0) {
628 bsendmsg(ua, _("Pruned %d %s on Volume %s from catalog.\n"), del.num_del,
629 del.num_del == 1 ? "Job" : "Jobs", mr->VolumeName);
632 /* If purged, mark it so */
633 if (del.num_ids == del.num_del) {
634 Dmsg0(200, "Volume is purged.\n");
635 stat = mark_media_purged(ua, mr);
640 free_pool_memory(query);
644 static int mark_media_purged(UAContext *ua, MEDIA_DBR *mr)
646 if (strcmp(mr->VolStatus, "Append") == 0 ||
647 strcmp(mr->VolStatus, "Full") == 0) {
648 strcpy(mr->VolStatus, "Purged");
649 if (!db_update_media_record(ua->db, mr)) {
651 bsendmsg(ua, "%s", db_strerror(ua->db));
657 return strcpy(mr->VolStatus, "Purged") == 0;