2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
21 * Bacula Director -- User Agent Database Purge Command
23 * Purges Files from specific JobIds
25 * Purges Jobs from Volumes
27 * Kern Sibbald, February MMII
34 /* Forward referenced functions */
35 static int purge_files_from_client(UAContext *ua, CLIENT *client);
36 static int purge_jobs_from_client(UAContext *ua, CLIENT *client);
37 int truncate_cmd(UAContext *ua, const char *cmd);
39 static const char *select_jobsfiles_from_client =
40 "SELECT JobId FROM Job "
44 static const char *select_jobs_from_client =
45 "SELECT JobId, PurgedFiles FROM Job "
49 * Purge records from database
51 * Purge Files (from) [Job|JobId|Client|Volume]
52 * Purge Jobs (from) [Client|Volume]
55 * N.B. Not all above is implemented yet.
57 int purge_cmd(UAContext *ua, const char *cmd)
63 memset(&jr, 0, sizeof(jr));
65 static const char *keywords[] = {
71 static const char *files_keywords[] = {
78 static const char *jobs_keywords[] = {
83 /* Special case for the "Action On Purge", this option is working only on
84 * Purged volume, so no jobs or files will be purged.
85 * We are skipping this message if "purge volume action=xxx"
87 if (!(find_arg(ua, "volume") >= 0 && find_arg(ua, "action") >= 0)) {
89 "\nThis command can be DANGEROUS!!!\n\n"
90 "It purges (deletes) all Files from a Job,\n"
91 "JobId, Client or Volume; or it purges (deletes)\n"
92 "all Jobs from a Client or Volume without regard\n"
93 "to retention periods. Normally you should use the\n"
94 "PRUNE command, which respects retention periods.\n"));
97 if (!open_new_client_db(ua)) {
100 switch (find_arg_keyword(ua, keywords)) {
103 switch(find_arg_keyword(ua, files_keywords)) {
106 if (get_job_dbr(ua, &jr)) {
108 edit_int64(jr.JobId, jobid);
109 purge_files_from_jobs(ua, jobid);
113 /* We restrict the client list to ClientAcl, maybe something to change later */
114 client = get_client_resource(ua, JT_SYSTEM);
116 purge_files_from_client(ua, client);
120 if (select_media_dbr(ua, &mr)) {
121 purge_files_from_volume(ua, &mr);
127 switch(find_arg_keyword(ua, jobs_keywords)) {
129 /* We restrict the client list to ClientAcl, maybe something to change later */
130 client = get_client_resource(ua, JT_SYSTEM);
132 purge_jobs_from_client(ua, client);
136 if (select_media_dbr(ua, &mr)) {
137 purge_jobs_from_volume(ua, &mr, /*force*/true);
143 /* Perform ActionOnPurge (action=truncate) */
144 if (find_arg(ua, "action") >= 0) {
145 return truncate_cmd(ua, ua->cmd);
148 while ((i=find_arg(ua, NT_("volume"))) >= 0) {
149 if (select_media_dbr(ua, &mr)) {
150 purge_jobs_from_volume(ua, &mr, /*force*/true);
152 *ua->argk[i] = 0; /* zap keyword already seen */
159 switch (do_keyword_prompt(ua, _("Choose item to purge"), keywords)) {
161 /* We restrict the client list to ClientAcl, maybe something to change later */
162 client = get_client_resource(ua, JT_SYSTEM);
164 purge_files_from_client(ua, client);
168 /* We restrict the client list to ClientAcl, maybe something to change later */
169 client = get_client_resource(ua, JT_SYSTEM);
171 purge_jobs_from_client(ua, client);
175 if (select_media_dbr(ua, &mr)) {
176 purge_jobs_from_volume(ua, &mr, /*force*/true);
184 * Purge File records from the database. For any Job which
185 * is older than the retention period, we unconditionally delete
186 * all File records for that Job. This is simple enough that no
187 * temporary tables are needed. We simply make an in memory list of
188 * the JobIds meeting the prune conditions, then delete all File records
189 * pointing to each of those JobIds.
191 static int purge_files_from_client(UAContext *ua, CLIENT *client)
194 POOL_MEM query(PM_MESSAGE);
198 memset(&cr, 0, sizeof(cr));
199 bstrncpy(cr.Name, client->name(), sizeof(cr.Name));
200 if (!db_create_client_record(ua->jcr, ua->db, &cr)) {
204 memset(&del, 0, sizeof(del));
206 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
208 ua->info_msg(_("Begin purging files for Client \"%s\"\n"), cr.Name);
210 Mmsg(query, select_jobsfiles_from_client, edit_int64(cr.ClientId, ed1));
211 Dmsg1(050, "select sql=%s\n", query.c_str());
212 db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del);
214 purge_files_from_job_list(ua, del);
216 if (del.num_del == 0) {
217 ua->warning_msg(_("No Files found for client %s to purge from %s catalog.\n"),
218 client->name(), client->catalog->name());
220 ua->info_msg(_("Files for %d Jobs for client \"%s\" purged from %s catalog.\n"), del.num_del,
221 client->name(), client->catalog->name());
233 * Purge Job records from the database. For any Job which
234 * is older than the retention period, we unconditionally delete
235 * it and all File records for that Job. This is simple enough that no
236 * temporary tables are needed. We simply make an in memory list of
237 * the JobIds then delete the Job, Files, and JobMedia records in that list.
239 static int purge_jobs_from_client(UAContext *ua, CLIENT *client)
242 POOL_MEM query(PM_MESSAGE);
246 memset(&cr, 0, sizeof(cr));
248 bstrncpy(cr.Name, client->name(), sizeof(cr.Name));
249 if (!db_create_client_record(ua->jcr, ua->db, &cr)) {
253 memset(&del, 0, sizeof(del));
255 del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids);
256 del.PurgedFiles = (char *)malloc(del.max_ids);
258 ua->info_msg(_("Begin purging jobs from Client \"%s\"\n"), cr.Name);
260 Mmsg(query, select_jobs_from_client, edit_int64(cr.ClientId, ed1));
261 Dmsg1(150, "select sql=%s\n", query.c_str());
262 db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del);
264 purge_job_list_from_catalog(ua, del);
266 if (del.num_del == 0) {
267 ua->warning_msg(_("No Jobs found for client %s to purge from %s catalog.\n"),
268 client->name(), client->catalog->name());
270 ua->info_msg(_("%d Jobs for client %s purged from %s catalog.\n"), del.num_del,
271 client->name(), client->catalog->name());
277 if (del.PurgedFiles) {
278 free(del.PurgedFiles);
285 * Remove File records from a list of JobIds
287 void purge_files_from_jobs(UAContext *ua, char *jobs)
289 POOL_MEM query(PM_MESSAGE);
291 Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs);
292 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
293 Dmsg1(050, "Delete File sql=%s\n", query.c_str());
295 Mmsg(query, "DELETE FROM BaseFiles WHERE JobId IN (%s)", jobs);
296 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
297 Dmsg1(050, "Delete BaseFiles sql=%s\n", query.c_str());
299 Mmsg(query, "DELETE FROM PathVisibility WHERE JobId IN (%s)", jobs);
300 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
301 Dmsg1(050, "Delete PathVisibility sql=%s\n", query.c_str());
304 * Now mark Job as having files purged. This is necessary to
305 * avoid having too many Jobs to process in future prunings. If
306 * we don't do this, the number of JobId's in our in memory list
307 * could grow very large.
309 Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs);
310 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
311 Dmsg1(050, "Mark purged sql=%s\n", query.c_str());
315 * Delete jobs (all records) from the catalog in groups of 1000
318 void purge_job_list_from_catalog(UAContext *ua, del_ctx &del)
320 POOL_MEM jobids(PM_MESSAGE);
323 for (int i=0; del.num_ids; ) {
324 Dmsg1(150, "num_ids=%d\n", del.num_ids);
325 pm_strcat(jobids, "");
326 for (int j=0; j<1000 && del.num_ids>0; j++) {
328 if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) {
329 Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]);
333 if (*jobids.c_str() != 0) {
334 pm_strcat(jobids, ",");
336 pm_strcat(jobids, edit_int64(del.JobId[i++], ed1));
337 Dmsg1(150, "Add id=%s\n", ed1);
340 Dmsg1(150, "num_ids=%d\n", del.num_ids);
341 purge_jobs_from_catalog(ua, jobids.c_str());
346 * Delete files from a list of jobs in groups of 1000
349 void purge_files_from_job_list(UAContext *ua, del_ctx &del)
351 POOL_MEM jobids(PM_MESSAGE);
354 * OK, now we have the list of JobId's to be pruned, send them
355 * off to be deleted batched 1000 at a time.
357 for (int i=0; del.num_ids; ) {
358 pm_strcat(jobids, "");
359 for (int j=0; j<1000 && del.num_ids>0; j++) {
361 if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) {
362 Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]);
366 if (*jobids.c_str() != 0) {
367 pm_strcat(jobids, ",");
369 pm_strcat(jobids, edit_int64(del.JobId[i++], ed1));
370 Dmsg1(150, "Add id=%s\n", ed1);
373 purge_files_from_jobs(ua, jobids.c_str());
378 * Change the type of the next copy job to backup.
379 * We need to upgrade the next copy of a normal job,
380 * and also upgrade the next copy when the normal job
381 * already have been purged.
383 * JobId: 1 PriorJobId: 0 (original)
384 * JobId: 2 PriorJobId: 1 (first copy)
385 * JobId: 3 PriorJobId: 1 (second copy)
387 * JobId: 2 PriorJobId: 1 (first copy, now regular backup)
388 * JobId: 3 PriorJobId: 1 (second copy)
390 * => Search through PriorJobId in jobid and
391 * PriorJobId in PriorJobId (jobid)
393 void upgrade_copies(UAContext *ua, char *jobs)
395 POOL_MEM query(PM_MESSAGE);
396 int dbtype = ua->db->bdb_get_type_index();
400 Mmsg(query, uap_upgrade_copies_oldest_job[dbtype], JT_JOB_COPY, jobs, jobs);
401 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
402 Dmsg1(050, "Upgrade copies Log sql=%s\n", query.c_str());
404 /* Now upgrade first copy to Backup */
405 Mmsg(query, "UPDATE Job SET Type='B' " /* JT_JOB_COPY => JT_BACKUP */
406 "WHERE JobId IN ( SELECT JobId FROM cpy_tmp )");
408 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
410 Mmsg(query, "DROP TABLE cpy_tmp");
411 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
417 * Remove all records from catalog for a list of JobIds
419 void purge_jobs_from_catalog(UAContext *ua, char *jobs)
421 POOL_MEM query(PM_MESSAGE);
423 /* Delete (or purge) records associated with the job */
424 purge_files_from_jobs(ua, jobs);
426 Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs);
427 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
428 Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str());
430 Mmsg(query, "DELETE FROM Log WHERE JobId IN (%s)", jobs);
431 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
432 Dmsg1(050, "Delete Log sql=%s\n", query.c_str());
434 Mmsg(query, "DELETE FROM RestoreObject WHERE JobId IN (%s)", jobs);
435 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
436 Dmsg1(050, "Delete RestoreObject sql=%s\n", query.c_str());
438 /* The JobId of the Snapshot record is no longer usable
439 * TODO: Migth want to use a copy for the jobid?
441 Mmsg(query, "UPDATE Snapshot SET JobId=0 WHERE JobId IN (%s)", jobs);
442 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
444 upgrade_copies(ua, jobs);
446 /* Now remove the Job record itself */
447 Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs);
448 db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
450 Dmsg1(050, "Delete Job sql=%s\n", query.c_str());
453 void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr )
454 {} /* ***FIXME*** implement */
457 * Returns: 1 if Volume purged
458 * 0 if Volume not purged
460 bool purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr, bool force)
462 POOL_MEM query(PM_MESSAGE);
469 stat = strcmp(mr->VolStatus, "Append") == 0 ||
470 strcmp(mr->VolStatus, "Full") == 0 ||
471 strcmp(mr->VolStatus, "Used") == 0 ||
472 strcmp(mr->VolStatus, "Error") == 0;
474 ua->error_msg(_("\nVolume \"%s\" has VolStatus \"%s\" and cannot be purged.\n"
475 "The VolStatus must be: Append, Full, Used, or Error to be purged.\n"),
476 mr->VolumeName, mr->VolStatus);
481 * Check if he wants to purge a single jobid
483 i = find_arg_with_value(ua, "jobid");
484 if (i >= 0 && is_a_number_list(ua->argv[i])) {
485 jobids = ua->argv[i];
490 if (!db_get_volume_jobids(ua->jcr, ua->db, mr, &lst)) {
491 ua->error_msg("%s", db_strerror(ua->db));
492 Dmsg0(050, "Count failed\n");
499 purge_jobs_from_catalog(ua, jobids);
502 ua->info_msg(_("%d Job%s on Volume \"%s\" purged from catalog.\n"),
503 lst.count, lst.count<=1?"":"s", mr->VolumeName);
505 purged = is_volume_purged(ua, mr, force);
512 * This routine will check the JobMedia records to see if the
513 * Volume has been purged. If so, it marks it as such and
515 * Returns: true if volume purged
518 * Note, we normally will not purge a volume that has Firstor LastWritten
519 * zero, because it means the volume is most likely being written
520 * however, if the user manually purges using the purge command in
521 * the console, he has been warned, and we go ahead and purge
522 * the volume anyway, if possible).
524 bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr, bool force)
526 POOL_MEM query(PM_MESSAGE);
527 struct s_count_ctx cnt;
531 if (!force && (mr->FirstWritten == 0 || mr->LastWritten == 0)) {
532 goto bail_out; /* not written cannot purge */
535 if (strcmp(mr->VolStatus, "Purged") == 0) {
536 Dmsg1(100, "Volume=%s already purged.\n", mr->VolumeName);
541 /* If purged, mark it so */
543 Mmsg(query, "SELECT 1 FROM JobMedia WHERE MediaId=%s LIMIT 1",
544 edit_int64(mr->MediaId, ed1));
545 if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) {
546 ua->error_msg("%s", db_strerror(ua->db));
547 Dmsg0(050, "Count failed\n");
551 if (cnt.count == 0) {
552 ua->warning_msg(_("There are no more Jobs associated with Volume \"%s\". Marking it purged.\n"),
554 Dmsg1(100, "There are no more Jobs associated with Volume \"%s\". Marking it purged.\n",
556 if (!(purged = mark_media_purged(ua, mr))) {
557 ua->error_msg("%s", db_strerror(ua->db));
565 * Called here to send the appropriate commands to the SD
566 * to do truncate on purge.
568 static void truncate_volume(UAContext *ua, MEDIA_DBR *mr,
569 char *pool, char *storage,
570 int drive, BSOCK *sd)
573 uint64_t VolBytes = 0;
574 uint64_t VolABytes = 0;
575 uint32_t VolType = 0;
581 /* Do it only if action on purge = truncate is set */
582 if (!(mr->ActionOnPurge & ON_PURGE_TRUNCATE)) {
583 ua->error_msg(_("\nThe option \"Action On Purge = Truncate\" was not defined in the Pool resource.\n"
584 "Unable to truncate volume \"%s\"\n"), mr->VolumeName);
589 * Send the command to truncate the volume after purge. If this feature
590 * is disabled for the specific device, this will be a no-op.
593 /* Protect us from spaces */
594 bash_spaces(mr->VolumeName);
595 bash_spaces(mr->MediaType);
597 bash_spaces(storage);
599 /* Do it by relabeling the Volume, which truncates it */
600 sd->fsend("relabel %s OldName=%s NewName=%s PoolName=%s "
601 "MediaType=%s Slot=%d drive=%d\n",
603 mr->VolumeName, mr->VolumeName,
604 pool, mr->MediaType, mr->Slot, drive);
606 unbash_spaces(mr->VolumeName);
607 unbash_spaces(mr->MediaType);
609 unbash_spaces(storage);
611 /* Check for valid response. With cloud volumes, the upload of the part.1 can
612 * generate a dir_update_volume_info() message that is handled by bget_dirmsg()
614 while (bget_dirmsg(sd) >= 0) {
615 ua->send_msg("%s", sd->msg);
616 if (sscanf(sd->msg, "3000 OK label. VolBytes=%llu VolABytes=%lld VolType=%d ",
617 &VolBytes, &VolABytes, &VolType) == 3) {
620 mr->VolBytes = VolBytes;
621 mr->VolABytes = VolABytes;
622 mr->VolType = VolType;
625 mr->VolCloudParts = 0;
626 mr->LastPartBytes = VolBytes;
628 set_storageid_in_mr(NULL, mr);
629 if (!db_update_media_record(ua->jcr, ua->db, mr)) {
630 ua->error_msg(_("Can't update volume size in the catalog\n"));
632 ua->send_msg(_("The volume \"%s\" has been truncated\n"), mr->VolumeName);
636 ua->warning_msg(_("Unable to truncate volume \"%s\"\n"), mr->VolumeName);
641 * Implement Bacula bconsole command purge action
642 * purge action=truncate pool= volume= storage= mediatype=
644 * truncate [cache] pool= volume= storage= mediatype=
646 * If the keyword "cache: is present, then we use the truncate
647 * command rather than relabel so that the driver can decide
648 * whether or not it wants to truncate. Note: only the
649 * Cloud driver permits truncating the cache.
651 * Note, later we might want to rename this action_on_purge_cmd() as
652 * was the original, but only if we add additional actions such as
653 * erase, ... For the moment, we only do a truncate.
656 int truncate_cmd(UAContext *ua, const char *cmd)
660 uint32_t *results = NULL;
661 const char *action = "truncate";
665 char storage[MAX_NAME_LENGTH];
667 if (find_arg(ua, "cache") > 0) {
668 return cloud_volumes_cmd(ua, cmd, "truncate cache");
671 memset(&pr, 0, sizeof(pr));
674 * Look for all Purged volumes that can be recycled, are enabled and
675 * have more the 10,000 bytes.
680 bstrncpy(mr.VolStatus, "Purged", sizeof(mr.VolStatus));
682 if (!scan_storage_cmd(ua, cmd, true, /* allfrompool */
683 &drive, &mr, &pr, &action, storage, &nb, &results)) {
687 if ((sd=open_sd_bsock(ua)) == NULL) {
688 Dmsg0(100, "Can't open connection to sd\n");
693 * Loop over the candidate Volumes and actually truncate them
695 for (int i=0; i < nb; i++) {
697 mr.MediaId = results[i];
698 if (db_get_media_record(ua->jcr, ua->db, &mr)) {
700 STORE *store = (STORE*)GetResWithName(R_STORAGE, storage);
701 drive = get_storage_drive(ua, store);
704 /* Must select Pool if not already done */
705 if (pr.PoolId == 0) {
706 pr.PoolId = mr.PoolId;
707 if (!db_get_pool_record(ua->jcr, ua->db, &pr)) {
711 if (strcasecmp("truncate", action) == 0) {
712 truncate_volume(ua, &mr, pr.Name, storage,
716 Dmsg1(0, "Can't find MediaId=%lu\n", mr.MediaId);
723 ua->jcr->wstore = NULL;
732 * IF volume status is Append, Full, Used, or Error, mark it Purged
733 * Purged volumes can then be recycled (if enabled).
735 bool mark_media_purged(UAContext *ua, MEDIA_DBR *mr)
738 if (strcmp(mr->VolStatus, "Append") == 0 ||
739 strcmp(mr->VolStatus, "Full") == 0 ||
740 strcmp(mr->VolStatus, "Used") == 0 ||
741 strcmp(mr->VolStatus, "Error") == 0) {
742 bstrncpy(mr->VolStatus, "Purged", sizeof(mr->VolStatus));
743 set_storageid_in_mr(NULL, mr);
744 if (!db_update_media_record(jcr, ua->db, mr)) {
747 pm_strcpy(jcr->VolumeName, mr->VolumeName);
748 generate_plugin_event(jcr, bDirEventVolumePurged);
750 * If the RecyclePool is defined, move the volume there
752 if (mr->RecyclePoolId && mr->RecyclePoolId != mr->PoolId) {
753 POOL_DBR oldpr, newpr;
754 memset(&oldpr, 0, sizeof(POOL_DBR));
755 memset(&newpr, 0, sizeof(POOL_DBR));
756 newpr.PoolId = mr->RecyclePoolId;
757 oldpr.PoolId = mr->PoolId;
758 if ( db_get_pool_numvols(jcr, ua->db, &oldpr)
759 && db_get_pool_numvols(jcr, ua->db, &newpr)) {
760 /* check if destination pool size is ok */
761 if (newpr.MaxVols > 0 && newpr.NumVols >= newpr.MaxVols) {
762 ua->error_msg(_("Unable move recycled Volume in full "
763 "Pool \"%s\" MaxVols=%d\n"),
764 newpr.Name, newpr.MaxVols);
766 } else { /* move media */
767 update_vol_pool(ua, newpr.Name, mr, &oldpr);
770 ua->error_msg("%s", db_strerror(ua->db));
774 /* Send message to Job report, if it is a *real* job */
775 if (jcr && jcr->JobId > 0) {
776 Jmsg(jcr, M_INFO, 0, _("All records pruned from Volume \"%s\"; marking it \"Purged\"\n"),
781 ua->error_msg(_("Cannot purge Volume with VolStatus=%s\n"), mr->VolStatus);
783 return strcmp(mr->VolStatus, "Purged") == 0;