2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
21 * Bacula Director -- User Agent Database Purge Command
23 * Purges Files from specific JobIds
25 * Purges Jobs from Volumes
27 * Kern Sibbald, February MMII
34 /* Forward referenced functions */
35 static int purge_files_from_client(UAContext *ua, CLIENT *client);
36 static int purge_jobs_from_client(UAContext *ua, CLIENT *client);
37 int truncate_cmd(UAContext *ua, const char *cmd);
39 static const char *select_jobsfiles_from_client =
40 "SELECT JobId FROM Job "
44 static const char *select_jobs_from_client =
45 "SELECT JobId, PurgedFiles FROM Job "
49 * Purge records from database
51 * Purge Files (from) [Job|JobId|Client|Volume]
52 * Purge Jobs (from) [Client|Volume]
55 * N.B. Not all above is implemented yet.
57 int purge_cmd(UAContext *ua, const char *cmd)
63 memset(&jr, 0, sizeof(jr));
65 static const char *keywords[] = {
71 static const char *files_keywords[] = {
78 static const char *jobs_keywords[] = {
83 /* Special case for the "Action On Purge", this option is working only on
84 * Purged volume, so no jobs or files will be purged.
85 * We are skipping this message if "purge volume action=xxx"
87 if (!(find_arg(ua, "volume") >= 0 && find_arg(ua, "action") >= 0)) {
89 "\nThis command can be DANGEROUS!!!\n\n"
90 "It purges (deletes) all Files from a Job,\n"
91 "JobId, Client or Volume; or it purges (deletes)\n"
92 "all Jobs from a Client or Volume without regard\n"
93 "to retention periods. Normally you should use the\n"
94 "PRUNE command, which respects retention periods.\n"));
97 if (!open_new_client_db(ua)) {
100 switch (find_arg_keyword(ua, keywords)) {
103 switch(find_arg_keyword(ua, files_keywords)) {
106 if (get_job_dbr(ua, &jr)) {
108 edit_int64(jr.JobId, jobid);
109 purge_files_from_jobs(ua, jobid);
113 /* We restrict the client list to ClientAcl, maybe something to change later */
114 client = get_client_resource(ua, JT_SYSTEM);
116 purge_files_from_client(ua, client);
120 if (select_media_dbr(ua, &mr)) {
121 purge_files_from_volume(ua, &mr);
127 switch(find_arg_keyword(ua, jobs_keywords)) {
129 /* We restrict the client list to ClientAcl, maybe something to change later */
130 client = get_client_resource(ua, JT_SYSTEM);
132 purge_jobs_from_client(ua, client);
136 if (select_media_dbr(ua, &mr)) {
137 purge_jobs_from_volume(ua, &mr, /*force*/true);
143 /* Perform ActionOnPurge (action=truncate) */
144 if (find_arg(ua, "action") >= 0) {
145 return truncate_cmd(ua, ua->cmd);
148 while ((i=find_arg(ua, NT_("volume"))) >= 0) {
149 if (select_media_dbr(ua, &mr)) {
150 purge_jobs_from_volume(ua, &mr, /*force*/true);
152 *ua->argk[i] = 0; /* zap keyword already seen */
159 switch (do_keyword_prompt(ua, _("Choose item to purge"), keywords)) {
161 /* We restrict the client list to ClientAcl, maybe something to change later */
162 client = get_client_resource(ua, JT_SYSTEM);
164 purge_files_from_client(ua, client);
168 /* We restrict the client list to ClientAcl, maybe something to change later */
169 client = get_client_resource(ua, JT_SYSTEM);
171 purge_jobs_from_client(ua, client);
175 if (select_media_dbr(ua, &mr)) {
176 purge_jobs_from_volume(ua, &mr, /*force*/true);
184 * Purge File records from the database. For any Job which
185 * is older than the retention period, we unconditionally delete
186 * all File records for that Job. This is simple enough that no
187 * temporary tables are needed. We simply make an in memory list of
188 * the JobIds meeting the prune conditions, then delete all File records
189 * pointing to each of those JobIds.
191 static int purge_files_from_client(UAContext *ua, CLIENT *client)
194 POOL_MEM query(PM_MESSAGE);
198 memset(&cr, 0, sizeof(cr));
199 bstrncpy(cr.Name, client->name(), sizeof(cr.Name));
200 if (!db_create_client_record(ua->jcr, ua->db, &cr)) {
204 memset(&del, 0, sizeof(del));
206 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
208 ua->info_msg(_("Begin purging files for Client \"%s\"\n"), cr.Name);
210 Mmsg(query, select_jobsfiles_from_client, edit_int64(cr.ClientId, ed1));
211 Dmsg1(050, "select sql=%s\n", query.c_str());
212 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
214 purge_files_from_job_list(ua, del);
216 if (del.num_del == 0) {
217 ua->warning_msg(_("No Files found for client %s to purge from %s catalog.\n"),
218 client->name(), client->catalog->name());
220 ua->info_msg(_("Files for %d Jobs for client \"%s\" purged from %s catalog.\n"), del.num_del,
221 client->name(), client->catalog->name());
233 * Purge Job records from the database. For any Job which
234 * is older than the retention period, we unconditionally delete
235 * it and all File records for that Job. This is simple enough that no
236 * temporary tables are needed. We simply make an in memory list of
237 * the JobIds then delete the Job, Files, and JobMedia records in that list.
239 static int purge_jobs_from_client(UAContext *ua, CLIENT *client)
242 POOL_MEM query(PM_MESSAGE);
246 memset(&cr, 0, sizeof(cr));
248 bstrncpy(cr.Name, client->name(), sizeof(cr.Name));
249 if (!db_create_client_record(ua->jcr, ua->db, &cr)) {
253 memset(&del, 0, sizeof(del));
255 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
256 del.PurgedFiles = (char *)malloc(del.max_ids);
258 ua->info_msg(_("Begin purging jobs from Client \"%s\"\n"), cr.Name);
260 Mmsg(query, select_jobs_from_client, edit_int64(cr.ClientId, ed1));
261 Dmsg1(150, "select sql=%s\n", query.c_str());
262 db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del);
264 purge_job_list_from_catalog(ua, del);
266 if (del.num_del == 0) {
267 ua->warning_msg(_("No Jobs found for client %s to purge from %s catalog.\n"),
268 client->name(), client->catalog->name());
270 ua->info_msg(_("%d Jobs for client %s purged from %s catalog.\n"), del.num_del,
271 client->name(), client->catalog->name());
277 if (del.PurgedFiles) {
278 free(del.PurgedFiles);
285 * Remove File records from a list of JobIds
287 void purge_files_from_jobs(UAContext *ua, char *jobs)
289 POOL_MEM query(PM_MESSAGE);
291 Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs);
292 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
293 Dmsg1(050, "Delete File sql=%s\n", query.c_str());
295 Mmsg(query, "DELETE FROM BaseFiles WHERE JobId IN (%s)", jobs);
296 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
297 Dmsg1(050, "Delete BaseFiles sql=%s\n", query.c_str());
299 Mmsg(query, "DELETE FROM PathVisibility WHERE JobId IN (%s)", jobs);
300 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
301 Dmsg1(050, "Delete PathVisibility sql=%s\n", query.c_str());
304 * Now mark Job as having files purged. This is necessary to
305 * avoid having too many Jobs to process in future prunings. If
306 * we don't do this, the number of JobId's in our in memory list
307 * could grow very large.
309 Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs);
310 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
311 Dmsg1(050, "Mark purged sql=%s\n", query.c_str());
315 * Delete jobs (all records) from the catalog in groups of 1000
318 void purge_job_list_from_catalog(UAContext *ua, del_ctx &del)
320 POOL_MEM jobids(PM_MESSAGE);
323 for (int i=0; del.num_ids; ) {
324 Dmsg1(150, "num_ids=%d\n", del.num_ids);
325 pm_strcat(jobids, "");
326 for (int j=0; j<1000 && del.num_ids>0; j++) {
328 if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) {
329 Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]);
333 if (*jobids.c_str() != 0) {
334 pm_strcat(jobids, ",");
336 pm_strcat(jobids, edit_int64(del.JobId[i++], ed1));
337 Dmsg1(150, "Add id=%s\n", ed1);
340 Dmsg1(150, "num_ids=%d\n", del.num_ids);
341 purge_jobs_from_catalog(ua, jobids.c_str());
346 * Delete files from a list of jobs in groups of 1000
349 void purge_files_from_job_list(UAContext *ua, del_ctx &del)
351 POOL_MEM jobids(PM_MESSAGE);
354 * OK, now we have the list of JobId's to be pruned, send them
355 * off to be deleted batched 1000 at a time.
357 for (int i=0; del.num_ids; ) {
358 pm_strcat(jobids, "");
359 for (int j=0; j<1000 && del.num_ids>0; j++) {
361 if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) {
362 Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]);
366 if (*jobids.c_str() != 0) {
367 pm_strcat(jobids, ",");
369 pm_strcat(jobids, edit_int64(del.JobId[i++], ed1));
370 Dmsg1(150, "Add id=%s\n", ed1);
373 purge_files_from_jobs(ua, jobids.c_str());
378 * Change the type of the next copy job to backup.
379 * We need to upgrade the next copy of a normal job,
380 * and also upgrade the next copy when the normal job
381 * already have been purged.
383 * JobId: 1 PriorJobId: 0 (original)
384 * JobId: 2 PriorJobId: 1 (first copy)
385 * JobId: 3 PriorJobId: 1 (second copy)
387 * JobId: 2 PriorJobId: 1 (first copy, now regular backup)
388 * JobId: 3 PriorJobId: 1 (second copy)
390 * => Search through PriorJobId in jobid and
391 * PriorJobId in PriorJobId (jobid)
393 void upgrade_copies(UAContext *ua, char *jobs)
395 POOL_MEM query(PM_MESSAGE);
396 int dbtype = ua->db->bdb_get_type_index();
400 Mmsg(query, uap_upgrade_copies_oldest_job[dbtype], JT_JOB_COPY, jobs, jobs);
401 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
402 Dmsg1(050, "Upgrade copies Log sql=%s\n", query.c_str());
404 /* Now upgrade first copy to Backup */
405 Mmsg(query, "UPDATE Job SET Type='B' " /* JT_JOB_COPY => JT_BACKUP */
406 "WHERE JobId IN ( SELECT JobId FROM cpy_tmp )");
408 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
410 Mmsg(query, "DROP TABLE cpy_tmp");
411 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
417 * Remove all records from catalog for a list of JobIds
419 void purge_jobs_from_catalog(UAContext *ua, char *jobs)
421 POOL_MEM query(PM_MESSAGE);
423 /* Delete (or purge) records associated with the job */
424 purge_files_from_jobs(ua, jobs);
426 Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs);
427 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
428 Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str());
430 Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobs);
431 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
432 Dmsg1(050, "Delete Log sql=%s\n", query.c_str());
434 Mmsg(query, "DELETE FROM RestoreObject WHERE JobId IN (%s)", jobs);
435 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
436 Dmsg1(050, "Delete RestoreObject sql=%s\n", query.c_str());
438 /* The JobId of the Snapshot record is no longer usable
439 * TODO: Migth want to use a copy for the jobid?
441 Mmsg(query, "UPDATE Snapshot SET JobId=0 WHERE JobId IN (%s)", jobs);
442 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
444 upgrade_copies(ua, jobs);
446 /* Now remove the Job record itself */
447 Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs);
448 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
450 Dmsg1(050, "Delete Job sql=%s\n", query.c_str());
453 void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr )
454 {} /* ***FIXME*** implement */
457 * Returns: 1 if Volume purged
458 * 0 if Volume not purged
460 bool purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr, bool force)
462 POOL_MEM query(PM_MESSAGE);
463 db_list_ctx lst_all, lst;
469 stat = strcmp(mr->VolStatus, "Append") == 0 ||
470 strcmp(mr->VolStatus, "Full") == 0 ||
471 strcmp(mr->VolStatus, "Used") == 0 ||
472 strcmp(mr->VolStatus, "Error") == 0;
474 ua->error_msg(_("\nVolume \"%s\" has VolStatus \"%s\" and cannot be purged.\n"
475 "The VolStatus must be: Append, Full, Used, or Error to be purged.\n"),
476 mr->VolumeName, mr->VolStatus);
481 * Check if he wants to purge a single jobid
483 i = find_arg_with_value(ua, "jobid");
484 if (i >= 0 && is_a_number_list(ua->argv[i])) {
485 jobids = ua->argv[i];
492 if (!db_get_volume_jobids(ua->jcr, ua->db, mr, &lst_all)) {
493 ua->error_msg("%s", db_strerror(ua->db));
494 Dmsg0(050, "Count failed\n");
498 if (lst_all.count > 0) {
499 Mmsg(query, "SELECT JobId FROM Job WHERE JobId IN (%s) AND JobStatus NOT IN ('R', 'C')",
501 if (!db_sql_query(ua->db, query.c_str(), db_list_handler, &lst)) {
502 ua->error_msg("%s", db_strerror(ua->db));
510 purge_jobs_from_catalog(ua, jobids);
511 ua->info_msg(_("%d Job%s on Volume \"%s\" purged from catalog.\n"),
512 lst.count, lst.count<=1?"":"s", mr->VolumeName);
514 purged = is_volume_purged(ua, mr, force);
521 * This routine will check the JobMedia records to see if the
522 * Volume has been purged. If so, it marks it as such and
524 * Returns: true if volume purged
527 * Note, we normally will not purge a volume that has Firstor LastWritten
528 * zero, because it means the volume is most likely being written
529 * however, if the user manually purges using the purge command in
530 * the console, he has been warned, and we go ahead and purge
531 * the volume anyway, if possible).
533 bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr, bool force)
535 POOL_MEM query(PM_MESSAGE);
536 struct s_count_ctx cnt;
540 if (!force && (mr->FirstWritten == 0 || mr->LastWritten == 0)) {
541 goto bail_out; /* not written cannot purge */
544 if (strcmp(mr->VolStatus, "Purged") == 0) {
545 Dmsg1(100, "Volume=%s already purged.\n", mr->VolumeName);
550 /* If purged, mark it so */
552 Mmsg(query, "SELECT 1 FROM JobMedia WHERE MediaId=%s LIMIT 1",
553 edit_int64(mr->MediaId, ed1));
554 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
555 ua->error_msg("%s", db_strerror(ua->db));
556 Dmsg0(050, "Count failed\n");
560 if (cnt.count == 0) {
561 ua->warning_msg(_("There are no more Jobs associated with Volume \"%s\". Marking it purged.\n"),
563 Dmsg1(100, "There are no more Jobs associated with Volume \"%s\". Marking it purged.\n",
565 if (!(purged = mark_media_purged(ua, mr))) {
566 ua->error_msg("%s", db_strerror(ua->db));
574 * Called here to send the appropriate commands to the SD
575 * to do truncate on purge.
577 static void truncate_volume(UAContext *ua, MEDIA_DBR *mr,
578 char *pool, char *storage,
579 int drive, BSOCK *sd)
582 uint64_t VolBytes = 0;
583 uint64_t VolABytes = 0;
584 uint32_t VolType = 0;
590 /* Do it only if action on purge = truncate is set */
591 if (!(mr->ActionOnPurge & ON_PURGE_TRUNCATE)) {
592 ua->error_msg(_("\nThe option \"Action On Purge = Truncate\" was not defined in the Pool resource.\n"
593 "Unable to truncate volume \"%s\"\n"), mr->VolumeName);
598 * Send the command to truncate the volume after purge. If this feature
599 * is disabled for the specific device, this will be a no-op.
602 /* Protect us from spaces */
603 bash_spaces(mr->VolumeName);
604 bash_spaces(mr->MediaType);
606 bash_spaces(storage);
608 /* Do it by relabeling the Volume, which truncates it */
609 sd->fsend("relabel %s OldName=%s NewName=%s PoolName=%s "
610 "MediaType=%s Slot=%d drive=%d\n",
612 mr->VolumeName, mr->VolumeName,
613 pool, mr->MediaType, mr->Slot, drive);
615 unbash_spaces(mr->VolumeName);
616 unbash_spaces(mr->MediaType);
618 unbash_spaces(storage);
620 /* Check for valid response. With cloud volumes, the upload of the part.1 can
621 * generate a dir_update_volume_info() message that is handled by bget_dirmsg()
623 while (bget_dirmsg(sd) >= 0) {
624 ua->send_msg("%s", sd->msg);
625 if (sscanf(sd->msg, "3000 OK label. VolBytes=%llu VolABytes=%lld VolType=%d ",
626 &VolBytes, &VolABytes, &VolType) == 3) {
629 mr->VolBytes = VolBytes;
630 mr->VolABytes = VolABytes;
631 mr->VolType = VolType;
634 mr->VolCloudParts = 0;
635 mr->LastPartBytes = VolBytes;
637 set_storageid_in_mr(NULL, mr);
638 if (!db_update_media_record(ua->jcr, ua->db, mr)) {
639 ua->error_msg(_("Can't update volume size in the catalog\n"));
641 ua->send_msg(_("The volume \"%s\" has been truncated\n"), mr->VolumeName);
645 ua->warning_msg(_("Unable to truncate volume \"%s\"\n"), mr->VolumeName);
650 * Implement Bacula bconsole command purge action
651 * purge action=truncate pool= volume= storage= mediatype=
653 * truncate [cache] pool= volume= storage= mediatype=
655 * If the keyword "cache: is present, then we use the truncate
656 * command rather than relabel so that the driver can decide
657 * whether or not it wants to truncate. Note: only the
658 * Cloud driver permits truncating the cache.
660 * Note, later we might want to rename this action_on_purge_cmd() as
661 * was the original, but only if we add additional actions such as
662 * erase, ... For the moment, we only do a truncate.
665 int truncate_cmd(UAContext *ua, const char *cmd)
669 uint32_t *results = NULL;
670 const char *action = "truncate";
674 char storage[MAX_NAME_LENGTH];
676 if (find_arg(ua, "cache") > 0) {
677 return cloud_volumes_cmd(ua, cmd, "truncate cache");
680 memset(&pr, 0, sizeof(pr));
683 * Look for all Purged volumes that can be recycled, are enabled and
684 * have more than 1,000 bytes (i.e. actually have data).
689 bstrncpy(mr.VolStatus, "Purged", sizeof(mr.VolStatus));
690 /* Get list of volumes to truncate */
691 if (!scan_storage_cmd(ua, cmd, true, /* allfrompool */
692 &drive, &mr, &pr, &action, storage, &nb, &results)) {
696 if ((sd=open_sd_bsock(ua)) == NULL) {
697 Dmsg0(100, "Can't open connection to sd\n");
702 * Loop over the candidate Volumes and actually truncate them
704 for (int i=0; i < nb; i++) {
706 mr.MediaId = results[i];
707 if (db_get_media_record(ua->jcr, ua->db, &mr)) {
708 if (strcasecmp(mr.VolStatus, "Purged") != 0) {
709 ua->send_msg(_("Truncate Volume \"%s\" skipped. Status is \"%s\", but must be \"Purged\".\n"),
710 mr.VolumeName, mr.VolStatus);
714 STORE *store = (STORE*)GetResWithName(R_STORAGE, storage);
715 drive = get_storage_drive(ua, store);
718 /* Must select Pool if not already done */
719 if (pr.PoolId == 0) {
720 pr.PoolId = mr.PoolId;
721 if (!db_get_pool_record(ua->jcr, ua->db, &pr)) {
722 goto bail_out; /* free allocated memory */
725 if (strcasecmp("truncate", action) == 0) {
726 truncate_volume(ua, &mr, pr.Name, storage,
730 Dmsg1(0, "Can't find MediaId=%lu\n", mr.MediaId);
737 ua->jcr->wstore = NULL;
746 * IF volume status is Append, Full, Used, or Error, mark it Purged
747 * Purged volumes can then be recycled (if enabled).
749 bool mark_media_purged(UAContext *ua, MEDIA_DBR *mr)
752 if (strcmp(mr->VolStatus, "Append") == 0 ||
753 strcmp(mr->VolStatus, "Full") == 0 ||
754 strcmp(mr->VolStatus, "Used") == 0 ||
755 strcmp(mr->VolStatus, "Error") == 0) {
756 bstrncpy(mr->VolStatus, "Purged", sizeof(mr->VolStatus));
757 set_storageid_in_mr(NULL, mr);
758 if (!db_update_media_record(jcr, ua->db, mr)) {
761 pm_strcpy(jcr->VolumeName, mr->VolumeName);
762 generate_plugin_event(jcr, bDirEventVolumePurged);
764 * If the RecyclePool is defined, move the volume there
766 if (mr->RecyclePoolId && mr->RecyclePoolId != mr->PoolId) {
767 POOL_DBR oldpr, newpr;
768 memset(&oldpr, 0, sizeof(POOL_DBR));
769 memset(&newpr, 0, sizeof(POOL_DBR));
770 newpr.PoolId = mr->RecyclePoolId;
771 oldpr.PoolId = mr->PoolId;
772 if ( db_get_pool_numvols(jcr, ua->db, &oldpr)
773 && db_get_pool_numvols(jcr, ua->db, &newpr)) {
774 /* check if destination pool size is ok */
775 if (newpr.MaxVols > 0 && newpr.NumVols >= newpr.MaxVols) {
776 ua->error_msg(_("Unable move recycled Volume in full "
777 "Pool \"%s\" MaxVols=%d\n"),
778 newpr.Name, newpr.MaxVols);
780 } else { /* move media */
781 update_vol_pool(ua, newpr.Name, mr, &oldpr);
784 ua->error_msg("%s", db_strerror(ua->db));
788 /* Send message to Job report, if it is a *real* job */
789 if (jcr && jcr->JobId > 0) {
790 Jmsg(jcr, M_INFO, 0, _("All records pruned from Volume \"%s\"; marking it \"Purged\"\n"),
795 ua->error_msg(_("Cannot purge Volume with VolStatus=%s\n"), mr->VolStatus);
797 return strcmp(mr->VolStatus, "Purged") == 0;