2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2015 Kern Sibbald
5 Copyright (C) 2002-2014 Free Software Foundation Europe e.V.
7 The original author of Bacula is Kern Sibbald, with contributions
8 from many others, a complete list can be found in the file AUTHORS.
10 You may use this file and others of this release according to the
11 license defined in the LICENSE file, which includes the Affero General
12 Public License, v3.0 ("AGPLv3") and some additional permissions and
13 terms pursuant to its AGPLv3 Section 7.
15 This notice must be preserved when any source code is
16 conveyed and/or propagated.
18 Bacula(R) is a registered trademark of Kern Sibbald.
22 * Bacula Director -- User Agent Database Purge Command
24 * Purges Files from specific JobIds
26 * Purges Jobs from Volumes
28 * Kern Sibbald, February MMII
35 /* Forward referenced functions */
36 static int purge_files_from_client(UAContext *ua, CLIENT *client);
37 static int purge_jobs_from_client(UAContext *ua, CLIENT *client);
38 int truncate_cmd(UAContext *ua, const char *cmd);
40 static const char *select_jobsfiles_from_client =
41 "SELECT JobId FROM Job "
45 static const char *select_jobs_from_client =
46 "SELECT JobId, PurgedFiles FROM Job "
50 * Purge records from database
52 * Purge Files (from) [Job|JobId|Client|Volume]
53 * Purge Jobs (from) [Client|Volume]
56 * N.B. Not all above is implemented yet.
58 int purge_cmd(UAContext *ua, const char *cmd)
64 memset(&jr, 0, sizeof(jr));
66 static const char *keywords[] = {
72 static const char *files_keywords[] = {
79 static const char *jobs_keywords[] = {
84 /* Special case for the "Action On Purge", this option is working only on
85 * Purged volume, so no jobs or files will be purged.
86 * We are skiping this message if "purge volume action=xxx"
88 if (!(find_arg(ua, "volume") >= 0 && find_arg(ua, "action") >= 0)) {
90 "\nThis command can be DANGEROUS!!!\n\n"
91 "It purges (deletes) all Files from a Job,\n"
92 "JobId, Client or Volume; or it purges (deletes)\n"
93 "all Jobs from a Client or Volume without regard\n"
94 "to retention periods. Normally you should use the\n"
95 "PRUNE command, which respects retention periods.\n"));
98 if (!open_new_client_db(ua)) {
101 switch (find_arg_keyword(ua, keywords)) {
104 switch(find_arg_keyword(ua, files_keywords)) {
107 if (get_job_dbr(ua, &jr)) {
109 edit_int64(jr.JobId, jobid);
110 purge_files_from_jobs(ua, jobid);
114 client = get_client_resource(ua);
116 purge_files_from_client(ua, client);
120 if (select_media_dbr(ua, &mr)) {
121 purge_files_from_volume(ua, &mr);
127 switch(find_arg_keyword(ua, jobs_keywords)) {
129 client = get_client_resource(ua);
131 purge_jobs_from_client(ua, client);
135 if (select_media_dbr(ua, &mr)) {
136 purge_jobs_from_volume(ua, &mr, /*force*/true);
142 /* Perform ActionOnPurge (action=truncate) */
143 if (find_arg(ua, "action") >= 0) {
144 return truncate_cmd(ua, ua->cmd);
147 while ((i=find_arg(ua, NT_("volume"))) >= 0) {
148 if (select_media_dbr(ua, &mr)) {
149 purge_jobs_from_volume(ua, &mr, /*force*/true);
151 *ua->argk[i] = 0; /* zap keyword already seen */
158 switch (do_keyword_prompt(ua, _("Choose item to purge"), keywords)) {
160 client = get_client_resource(ua);
162 purge_files_from_client(ua, client);
166 client = get_client_resource(ua);
168 purge_jobs_from_client(ua, client);
172 if (select_media_dbr(ua, &mr)) {
173 purge_jobs_from_volume(ua, &mr, /*force*/true);
181 * Purge File records from the database. For any Job which
182 * is older than the retention period, we unconditionally delete
183 * all File records for that Job. This is simple enough that no
184 * temporary tables are needed. We simply make an in memory list of
185 * the JobIds meeting the prune conditions, then delete all File records
186 * pointing to each of those JobIds.
188 static int purge_files_from_client(UAContext *ua, CLIENT *client)
191 POOL_MEM query(PM_MESSAGE);
195 memset(&cr, 0, sizeof(cr));
196 bstrncpy(cr.Name, client->name(), sizeof(cr.Name));
197 if (!db_create_client_record(ua->jcr, ua->db, &cr)) {
201 memset(&del, 0, sizeof(del));
203 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
205 ua->info_msg(_("Begin purging files for Client \"%s\"\n"), cr.Name);
207 Mmsg(query, select_jobsfiles_from_client, edit_int64(cr.ClientId, ed1));
208 Dmsg1(050, "select sql=%s\n", query.c_str());
209 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
211 purge_files_from_job_list(ua, del);
213 if (del.num_ids == 0) {
214 ua->warning_msg(_("No Files found for client %s to purge from %s catalog.\n"),
215 client->name(), client->catalog->name());
217 ua->info_msg(_("Files for %d Jobs for client \"%s\" purged from %s catalog.\n"), del.num_ids,
218 client->name(), client->catalog->name());
230 * Purge Job records from the database. For any Job which
231 * is older than the retention period, we unconditionally delete
232 * it and all File records for that Job. This is simple enough that no
233 * temporary tables are needed. We simply make an in memory list of
234 * the JobIds then delete the Job, Files, and JobMedia records in that list.
236 static int purge_jobs_from_client(UAContext *ua, CLIENT *client)
239 POOL_MEM query(PM_MESSAGE);
243 memset(&cr, 0, sizeof(cr));
245 bstrncpy(cr.Name, client->name(), sizeof(cr.Name));
246 if (!db_create_client_record(ua->jcr, ua->db, &cr)) {
250 memset(&del, 0, sizeof(del));
252 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
253 del.PurgedFiles = (char *)malloc(del.max_ids);
255 ua->info_msg(_("Begin purging jobs from Client \"%s\"\n"), cr.Name);
257 Mmsg(query, select_jobs_from_client, edit_int64(cr.ClientId, ed1));
258 Dmsg1(150, "select sql=%s\n", query.c_str());
259 db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del);
261 purge_job_list_from_catalog(ua, del);
263 if (del.num_ids == 0) {
264 ua->warning_msg(_("No Files found for client %s to purge from %s catalog.\n"),
265 client->name(), client->catalog->name());
267 ua->info_msg(_("%d Jobs for client %s purged from %s catalog.\n"), del.num_ids,
268 client->name(), client->catalog->name());
274 if (del.PurgedFiles) {
275 free(del.PurgedFiles);
282 * Remove File records from a list of JobIds
284 void purge_files_from_jobs(UAContext *ua, char *jobs)
286 POOL_MEM query(PM_MESSAGE);
288 Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs);
289 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
290 Dmsg1(050, "Delete File sql=%s\n", query.c_str());
292 Mmsg(query, "DELETE FROM BaseFiles WHERE JobId IN (%s)", jobs);
293 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
294 Dmsg1(050, "Delete BaseFiles sql=%s\n", query.c_str());
296 Mmsg(query, "DELETE FROM PathVisibility WHERE JobId IN (%s)", jobs);
297 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
298 Dmsg1(050, "Delete PathVisibility sql=%s\n", query.c_str());
301 * Now mark Job as having files purged. This is necessary to
302 * avoid having too many Jobs to process in future prunings. If
303 * we don't do this, the number of JobId's in our in memory list
304 * could grow very large.
306 Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs);
307 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
308 Dmsg1(050, "Mark purged sql=%s\n", query.c_str());
312 * Delete jobs (all records) from the catalog in groups of 1000
315 void purge_job_list_from_catalog(UAContext *ua, del_ctx &del)
317 POOL_MEM jobids(PM_MESSAGE);
320 for (int i=0; del.num_ids; ) {
321 Dmsg1(150, "num_ids=%d\n", del.num_ids);
322 pm_strcat(jobids, "");
323 for (int j=0; j<1000 && del.num_ids>0; j++) {
325 if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) {
326 Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]);
330 if (*jobids.c_str() != 0) {
331 pm_strcat(jobids, ",");
333 pm_strcat(jobids, edit_int64(del.JobId[i++], ed1));
334 Dmsg1(150, "Add id=%s\n", ed1);
337 Dmsg1(150, "num_ids=%d\n", del.num_ids);
338 purge_jobs_from_catalog(ua, jobids.c_str());
343 * Delete files from a list of jobs in groups of 1000
346 void purge_files_from_job_list(UAContext *ua, del_ctx &del)
348 POOL_MEM jobids(PM_MESSAGE);
351 * OK, now we have the list of JobId's to be pruned, send them
352 * off to be deleted batched 1000 at a time.
354 for (int i=0; del.num_ids; ) {
355 pm_strcat(jobids, "");
356 for (int j=0; j<1000 && del.num_ids>0; j++) {
358 if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) {
359 Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]);
363 if (*jobids.c_str() != 0) {
364 pm_strcat(jobids, ",");
366 pm_strcat(jobids, edit_int64(del.JobId[i++], ed1));
367 Dmsg1(150, "Add id=%s\n", ed1);
370 purge_files_from_jobs(ua, jobids.c_str());
375 * Change the type of the next copy job to backup.
376 * We need to upgrade the next copy of a normal job,
377 * and also upgrade the next copy when the normal job
378 * already have been purged.
380 * JobId: 1 PriorJobId: 0 (original)
381 * JobId: 2 PriorJobId: 1 (first copy)
382 * JobId: 3 PriorJobId: 1 (second copy)
384 * JobId: 2 PriorJobId: 1 (first copy, now regular backup)
385 * JobId: 3 PriorJobId: 1 (second copy)
387 * => Search through PriorJobId in jobid and
388 * PriorJobId in PriorJobId (jobid)
390 void upgrade_copies(UAContext *ua, char *jobs)
392 POOL_MEM query(PM_MESSAGE);
393 int dbtype = ua->db->bdb_get_type_index();
397 Mmsg(query, uap_upgrade_copies_oldest_job[dbtype], JT_JOB_COPY, jobs, jobs);
398 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
399 Dmsg1(050, "Upgrade copies Log sql=%s\n", query.c_str());
401 /* Now upgrade first copy to Backup */
402 Mmsg(query, "UPDATE Job SET Type='B' " /* JT_JOB_COPY => JT_BACKUP */
403 "WHERE JobId IN ( SELECT JobId FROM cpy_tmp )");
405 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
407 Mmsg(query, "DROP TABLE cpy_tmp");
408 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
414 * Remove all records from catalog for a list of JobIds
416 void purge_jobs_from_catalog(UAContext *ua, char *jobs)
418 POOL_MEM query(PM_MESSAGE);
420 /* Delete (or purge) records associated with the job */
421 purge_files_from_jobs(ua, jobs);
423 Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs);
424 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
425 Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str());
427 Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobs);
428 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
429 Dmsg1(050, "Delete Log sql=%s\n", query.c_str());
431 Mmsg(query, "DELETE FROM RestoreObject WHERE JobId IN (%s)", jobs);
432 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
433 Dmsg1(050, "Delete RestoreObject sql=%s\n", query.c_str());
435 /* The JobId of the Snapshot record is no longer usable
436 * TODO: Migth want to use a copy for the jobid?
438 Mmsg(query, "UPDATE Snapshot SET JobId=0 WHERE JobId IN (%s)", jobs);
439 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
441 upgrade_copies(ua, jobs);
443 /* Now remove the Job record itself */
444 Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs);
445 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
447 Dmsg1(050, "Delete Job sql=%s\n", query.c_str());
450 void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr )
451 {} /* ***FIXME*** implement */
454 * Returns: 1 if Volume purged
455 * 0 if Volume not purged
457 bool purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr, bool force)
459 POOL_MEM query(PM_MESSAGE);
466 stat = strcmp(mr->VolStatus, "Append") == 0 ||
467 strcmp(mr->VolStatus, "Full") == 0 ||
468 strcmp(mr->VolStatus, "Used") == 0 ||
469 strcmp(mr->VolStatus, "Error") == 0;
471 ua->error_msg(_("\nVolume \"%s\" has VolStatus \"%s\" and cannot be purged.\n"
472 "The VolStatus must be: Append, Full, Used, or Error to be purged.\n"),
473 mr->VolumeName, mr->VolStatus);
478 * Check if he wants to purge a single jobid
480 i = find_arg_with_value(ua, "jobid");
481 if (i >= 0 && is_a_number_list(ua->argv[i])) {
482 jobids = ua->argv[i];
487 if (!db_get_volume_jobids(ua->jcr, ua->db, mr, &lst)) {
488 ua->error_msg("%s", db_strerror(ua->db));
489 Dmsg0(050, "Count failed\n");
496 purge_jobs_from_catalog(ua, jobids);
499 ua->info_msg(_("%d File%s on Volume \"%s\" purged from catalog.\n"),
500 lst.count, lst.count<=1?"":"s", mr->VolumeName);
502 purged = is_volume_purged(ua, mr, force);
509 * This routine will check the JobMedia records to see if the
510 * Volume has been purged. If so, it marks it as such and
512 * Returns: true if volume purged
515 * Note, we normally will not purge a volume that has Firstor LastWritten
516 * zero, because it means the volume is most likely being written
517 * however, if the user manually purges using the purge command in
518 * the console, he has been warned, and we go ahead and purge
519 * the volume anyway, if possible).
521 bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr, bool force)
523 POOL_MEM query(PM_MESSAGE);
524 struct s_count_ctx cnt;
528 if (!force && (mr->FirstWritten == 0 || mr->LastWritten == 0)) {
529 goto bail_out; /* not written cannot purge */
532 if (strcmp(mr->VolStatus, "Purged") == 0) {
533 Dmsg1(100, "Volume=%s already purged.\n", mr->VolumeName);
538 /* If purged, mark it so */
540 Mmsg(query, "SELECT 1 FROM JobMedia WHERE MediaId=%s LIMIT 1",
541 edit_int64(mr->MediaId, ed1));
542 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
543 ua->error_msg("%s", db_strerror(ua->db));
544 Dmsg0(050, "Count failed\n");
548 if (cnt.count == 0) {
549 ua->warning_msg(_("There are no more Jobs associated with Volume \"%s\". Marking it purged.\n"),
551 Dmsg1(100, "There are no more Jobs associated with Volume \"%s\". Marking it purged.\n",
553 if (!(purged = mark_media_purged(ua, mr))) {
554 ua->error_msg("%s", db_strerror(ua->db));
562 * Called here to send the appropriate commands to the SD
563 * to do truncate on purge.
565 static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr,
566 char *pool, char *storage,
567 int drive, BSOCK *sd)
570 uint64_t VolBytes = 0;
571 uint64_t VolABytes = 0;
572 uint32_t VolType = 0;
574 /* TODO: Return if not mr->Recyle ? */
579 /* Do it only if action on purge = truncate is set */
580 if (!(mr->ActionOnPurge & ON_PURGE_TRUNCATE)) {
584 * Send the command to truncate the volume after purge. If this feature
585 * is disabled for the specific device, this will be a no-op.
588 /* Protect us from spaces */
589 bash_spaces(mr->VolumeName);
590 bash_spaces(mr->MediaType);
592 bash_spaces(storage);
594 /* Do it by relabeling the Volume, which truncates it */
595 sd->fsend("relabel %s OldName=%s NewName=%s PoolName=%s "
596 "MediaType=%s Slot=%d drive=%d\n",
598 mr->VolumeName, mr->VolumeName,
599 pool, mr->MediaType, mr->Slot, drive);
601 unbash_spaces(mr->VolumeName);
602 unbash_spaces(mr->MediaType);
604 unbash_spaces(storage);
606 /* Send relabel command, and check for valid response */
607 while (sd->recv() >= 0) {
608 ua->send_msg("%s", sd->msg);
609 if (sscanf(sd->msg, "3000 OK label. VolBytes=%llu VolABytes=%lld VolType=%d ",
610 &VolBytes, &VolABytes, &VolType) == 3) {
616 mr->VolBytes = VolBytes;
617 mr->VolABytes = VolABytes;
618 mr->VolType = VolType;
620 set_storageid_in_mr(NULL, mr);
621 if (!db_update_media_record(ua->jcr, ua->db, mr)) {
622 ua->error_msg(_("Can't update volume size in the catalog\n"));
624 ua->send_msg(_("The volume \"%s\" has been truncated\n"), mr->VolumeName);
626 ua->warning_msg(_("Unable to truncate volume \"%s\"\n"), mr->VolumeName);
631 * Implement Bacula bconsole command purge action
632 * purge action=truncate pool= volume= storage= mediatype=
634 * truncate pool= volume= storage= mediatype=
636 * Note, later we might want to rename this action_on_purge_cmd() as
637 * was the original, but only if we add additional actions such as
638 * erase, ... For the moment, we only do a truncate.
641 int truncate_cmd(UAContext *ua, const char *cmd)
643 bool allpools = false;
646 uint32_t *results = NULL;
647 const char *action = "truncate";
654 memset(&pr, 0, sizeof(pr));
656 /* Look at arguments */
657 for (int i=1; i<ua->argc; i++) {
658 if (strcasecmp(ua->argk[i], NT_("allpools")) == 0) {
661 } else if (strcasecmp(ua->argk[i], NT_("volume")) == 0
662 && is_name_valid(ua->argv[i], NULL)) {
663 bstrncpy(mr.VolumeName, ua->argv[i], sizeof(mr.VolumeName));
665 } else if (strcasecmp(ua->argk[i], NT_("mediatype")) == 0
667 bstrncpy(mr.MediaType, ua->argv[i], sizeof(mr.MediaType));
669 } else if (strcasecmp(ua->argk[i], NT_("drive")) == 0 && ua->argv[i]) {
670 drive = atoi(ua->argv[i]);
672 } else if (strcasecmp(ua->argk[i], NT_("action")) == 0
673 && is_name_valid(ua->argv[i], NULL)) {
674 action = ua->argv[i];
679 ua->jcr->wstore = store = get_storage_resource(ua, false);
685 Dmsg0(100, "Can't open db\n");
690 /* force pool selection */
691 pool = get_pool_resource(ua);
693 Dmsg0(100, "Can't get pool resource\n");
696 bstrncpy(pr.Name, pool->name(), sizeof(pr.Name));
697 if (!db_get_pool_record(ua->jcr, ua->db, &pr)) {
698 Dmsg0(100, "Can't get pool record\n");
701 mr.PoolId = pr.PoolId;
705 * Look for all Purged volumes that can be recycled, are enabled and
706 * have more the 10,000 bytes.
711 set_storageid_in_mr(store, &mr);
712 bstrncpy(mr.VolStatus, "Purged", sizeof(mr.VolStatus));
713 if (!db_get_media_ids(ua->jcr, ua->db, &mr, &nb, &results)) {
714 Dmsg0(100, "No results from db_get_media_ids\n");
719 ua->send_msg(_("No Volumes found to perform \"truncate\" command.\n"));
723 if ((sd=open_sd_bsock(ua)) == NULL) {
724 Dmsg0(100, "Can't open connection to sd\n");
729 * Loop over the candidate Volumes and actually truncate them
731 for (int i=0; i < nb; i++) {
733 mr.MediaId = results[i];
734 if (db_get_media_record(ua->jcr, ua->db, &mr)) {
735 /* TODO: ask for drive and change Pool */
736 if (strcasecmp("truncate", action) == 0) {
737 do_truncate_on_purge(ua, &mr, pr.Name, store->dev_name(), drive, sd);
740 Dmsg1(0, "Can't find MediaId=%lld\n", (uint64_t)mr.MediaId);
747 ua->jcr->wstore = NULL;
756 * IF volume status is Append, Full, Used, or Error, mark it Purged
757 * Purged volumes can then be recycled (if enabled).
759 bool mark_media_purged(UAContext *ua, MEDIA_DBR *mr)
762 if (strcmp(mr->VolStatus, "Append") == 0 ||
763 strcmp(mr->VolStatus, "Full") == 0 ||
764 strcmp(mr->VolStatus, "Used") == 0 ||
765 strcmp(mr->VolStatus, "Error") == 0) {
766 bstrncpy(mr->VolStatus, "Purged", sizeof(mr->VolStatus));
767 set_storageid_in_mr(NULL, mr);
768 if (!db_update_media_record(jcr, ua->db, mr)) {
771 pm_strcpy(jcr->VolumeName, mr->VolumeName);
772 generate_plugin_event(jcr, bDirEventVolumePurged);
774 * If the RecyclePool is defined, move the volume there
776 if (mr->RecyclePoolId && mr->RecyclePoolId != mr->PoolId) {
777 POOL_DBR oldpr, newpr;
778 memset(&oldpr, 0, sizeof(POOL_DBR));
779 memset(&newpr, 0, sizeof(POOL_DBR));
780 newpr.PoolId = mr->RecyclePoolId;
781 oldpr.PoolId = mr->PoolId;
782 if ( db_get_pool_numvols(jcr, ua->db, &oldpr)
783 && db_get_pool_numvols(jcr, ua->db, &newpr)) {
784 /* check if destination pool size is ok */
785 if (newpr.MaxVols > 0 && newpr.NumVols >= newpr.MaxVols) {
786 ua->error_msg(_("Unable move recycled Volume in full "
787 "Pool \"%s\" MaxVols=%d\n"),
788 newpr.Name, newpr.MaxVols);
790 } else { /* move media */
791 update_vol_pool(ua, newpr.Name, mr, &oldpr);
794 ua->error_msg("%s", db_strerror(ua->db));
798 /* Send message to Job report, if it is a *real* job */
799 if (jcr && jcr->JobId > 0) {
800 Jmsg(jcr, M_INFO, 0, _("All records pruned from Volume \"%s\"; marking it \"Purged\"\n"),
805 ua->error_msg(_("Cannot purge Volume with VolStatus=%s\n"), mr->VolStatus);
807 return strcmp(mr->VolStatus, "Purged") == 0;