2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
20 * Bacula Director -- vbackup.c -- responsible for doing virtual
21 * backup jobs or in other words, consolidation or synthetic
22 * backups. No connection to the Client is made.
24 * Kern Sibbald, July MMVIII
26 * Basic tasks done here:
27 * Open DB and create records for this job.
28 * Figure out what Jobs to copy.
29 * Open Message Channel with Storage daemon to tell him a
30 * job will be starting.
31 * Connect to the storage daemon and run the job.
38 static const int dbglevel = 10;
40 static bool create_bootstrap_file(JCR *jcr, char *jobids);
41 void vbackup_cleanup(JCR *jcr, int TermCode);
44 * Called here before the job is run to do the job
47 bool do_vbackup_init(JCR *jcr)
49 if (!get_or_create_fileset_record(jcr)) {
50 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
54 apply_pool_overrides(jcr);
56 if (!allow_duplicate_job(jcr)) {
61 * If the read pool has not been allocated yet due to the job
62 * being upgraded to a virtual full then allocate it now
64 if (!jcr->rpool_source) {
65 jcr->rpool_source = get_pool_memory(PM_MESSAGE);
67 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
68 if (jcr->jr.PoolId == 0) {
69 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
70 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
74 * Note, at this point, pool is the pool for this job. We
75 * transfer it to rpool (read pool), and a bit later,
76 * pool will be changed to point to the write pool,
77 * which comes from pool->NextPool.
79 jcr->rpool = jcr->pool; /* save read pool */
80 pm_strcpy(jcr->rpool_source, jcr->pool_source);
82 /* If pool storage specified, use it for virtual full */
83 copy_rstorage(jcr, jcr->pool->storage, _("Pool resource"));
85 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
87 jcr->start_time = time(NULL);
88 jcr->jr.StartTime = jcr->start_time;
89 jcr->jr.JobLevel = L_FULL; /* we want this to appear as a Full backup */
90 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
91 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
94 if (!apply_wstorage_overrides(jcr, jcr->pool)) {
98 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
104 * Do a virtual backup, which consolidates all previous backups into
105 * a sort of synthetic Full.
107 * Returns: false on failure
110 bool do_vbackup(JCR *jcr)
112 char level_computed = L_FULL;
120 Dmsg2(100, "rstorage=%p wstorage=%p\n", jcr->rstorage, jcr->wstorage);
121 Dmsg2(100, "Read store=%s, write store=%s\n",
122 ((STORE *)jcr->rstorage->first())->name(),
123 ((STORE *)jcr->wstorage->first())->name());
125 jcr->wasVirtualFull = true; /* remember where we came from */
127 /* Print Job Start message */
128 Jmsg(jcr, M_INFO, 0, _("Start Virtual Backup JobId %s, Job=%s\n"),
129 edit_uint64(jcr->JobId, ed1), jcr->Job);
130 if (!jcr->accurate) {
131 Jmsg(jcr, M_WARNING, 0,
132 _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n"));
135 if (jcr->JobIds && *jcr->JobIds) {
138 POOL_MEM query(PM_MESSAGE);
140 memset(&jr, 0, sizeof(jr));
142 if (is_an_integer(jcr->JobIds)) {
143 /* Single JobId, so start the accurate code based on this id */
145 jr.JobId = str_to_int64(jcr->JobIds);
146 if (!db_get_job_record(jcr, jcr->db, &jr)) {
147 Jmsg(jcr, M_ERROR, 0,
148 _("Unable to get Job record for JobId=%s: ERR=%s\n"),
149 jcr->JobIds, db_strerror(jcr->db));
152 Jmsg(jcr, M_INFO,0,_("Selecting jobs to build the Full state at %s\n"),
155 jr.JobLevel = L_INCREMENTAL; /* Take Full+Diff+Incr */
156 db_get_accurate_jobids(jcr, jcr->db, &jr, &jobids);
158 } else if (sel.set_string(jcr->JobIds, true)) {
159 /* Found alljobid keyword */
160 if (jcr->use_all_JobIds) {
161 jobids.count = sel.size();
162 pm_strcpy(jobids.list, sel.get_expanded_list());
164 /* Need to apply some filter on the job name */
167 "SELECT JobId FROM Job "
168 "WHERE Job.Name = '%s' "
169 "AND Job.JobId IN (%s) "
170 "ORDER BY JobTDate ASC",
172 sel.get_expanded_list());
174 db_sql_query(jcr->db, query.c_str(), db_list_handler, &jobids);
177 if (jobids.count == 0) {
178 Jmsg(jcr, M_FATAL, 0, _("No valid Jobs found from user selection.\n"));
182 Jmsg(jcr, M_INFO, 0, _("Using user supplied JobIds=%s\n"),
187 "SELECT Level FROM Job "
188 "WHERE Job.JobId IN (%s) "
192 /* Will produce something like F,D,I or F,I */
193 db_sql_query(jcr->db, query.c_str(), db_list_handler, &status);
195 /* If no full found in the list, we build a "virtualdiff" or
198 if (strchr(status.list, L_FULL) == NULL) {
199 if (strchr(status.list, L_DIFFERENTIAL)) {
200 level_computed = L_DIFFERENTIAL;
201 Jmsg(jcr, M_INFO, 0, _("No previous Full found in list, "
202 "using Differential level\n"));
205 level_computed = L_INCREMENTAL;
206 Jmsg(jcr, M_INFO, 0, _("No previous Full found in list, "
207 "using Incremental level\n"));
212 } else { /* No argument provided */
213 jcr->jr.JobLevel = L_VIRTUAL_FULL;
214 db_get_accurate_jobids(jcr, jcr->db, &jcr->jr, &jobids);
215 Dmsg1(10, "Accurate jobids=%s\n", jobids.list);
218 if (jobids.count == 0) {
219 Jmsg(jcr, M_FATAL, 0, _("No previous Jobs found.\n"));
222 jobids.count -= jcr->job->BackupsToKeep;
223 if (jobids.count <= 0) {
224 Jmsg(jcr, M_WARNING, 0, _("Insufficient Backups to Keep.\n"));
227 if (jobids.count == 1) {
228 Jmsg(jcr, M_WARNING, 0, _("Only one Job found. Consolidation not needed.\n"));
232 /* Remove number of JobIds we want to keep */
233 for (int i=0; i < (int)jcr->job->BackupsToKeep; i++) {
234 p = strrchr(jobids.list, ','); /* find last jobid */
242 /* Full by default, or might be Incr/Diff when jobid= is used */
243 jcr->jr.JobLevel = level_computed;
245 Jmsg(jcr, M_INFO, 0, "Consolidating JobIds=%s\n", jobids.list);
248 * Now we find the last job that ran and store it's info in
249 * the previous_jr record. We will set our times to the
250 * values from that job so that anything changed after that
251 * time will be picked up on the next backup.
253 p = strrchr(jobids.list, ','); /* find last jobid */
259 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
260 jcr->previous_jr.JobId = str_to_int64(p);
261 Dmsg1(10, "Previous JobId=%s\n", p);
262 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
263 Jmsg(jcr, M_FATAL, 0, _("Error getting Job record for previous Job: ERR=%s"),
264 db_strerror(jcr->db));
268 if (!create_bootstrap_file(jcr, jobids.list)) {
269 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
274 * Open a message channel connection with the Storage
275 * daemon. This is to let him know that our client
276 * will be contacting him for a backup session.
279 Dmsg0(110, "Open connection with storage daemon\n");
280 jcr->setJobStatus(JS_WaitSD);
282 * Start conversation with Storage daemon
284 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
287 sd = jcr->store_bsock;
290 * Now start a job with the Storage daemon
292 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage, /*send_bsr*/true)) {
295 Dmsg0(100, "Storage daemon connection OK\n");
298 * We re-update the job start record so that the start
299 * time is set after the run before job. This avoids
300 * that any files created by the run before job will
301 * be saved twice. They will be backed up in the current
302 * job, but not in the next one unless they are changed.
303 * Without this, they will be backed up in this job and
304 * in the next job run because in that case, their date
305 * is after the start of this run.
307 jcr->start_time = time(NULL);
308 jcr->jr.StartTime = jcr->start_time;
309 jcr->jr.JobTDate = jcr->start_time;
310 jcr->setJobStatus(JS_Running);
312 /* Update job start record */
313 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
314 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
318 /* Declare the job started to start the MaxRunTime check */
319 jcr->setJobStarted();
322 * Start the job prior to starting the message thread below
323 * to avoid two threads from using the BSOCK structure at
326 if (!sd->fsend("run")) {
331 * Now start a Storage daemon message thread
333 if (!start_storage_daemon_message_thread(jcr)) {
337 jcr->setJobStatus(JS_Running);
339 /* Pickup Job termination data */
340 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors */
341 wait_for_storage_daemon_termination(jcr);
342 jcr->setJobStatus(jcr->SDJobStatus);
343 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
344 if (jcr->JobStatus != JS_Terminated) {
347 if (jcr->job->DeleteConsolidatedJobs) {
348 ua = new_ua_context(jcr);
349 purge_jobs_from_catalog(ua, jobids.list);
351 Jmsg(jcr, M_INFO, 0, _("Deleted consolidated JobIds=%s\n"), jobids.list);
354 vbackup_cleanup(jcr, jcr->JobStatus);
360 * Release resources allocated during backup.
362 void vbackup_cleanup(JCR *jcr, int TermCode)
364 char sdt[50], edt[50], schedt[50];
365 char ec1[30], ec3[30], ec4[30], compress[50];
366 char ec7[30], ec8[30], elapsed[50];
367 char term_code[100], sd_term_msg[100];
368 const char *term_msg;
369 int msg_type = M_INFO;
372 double kbps, compression;
374 POOL_MEM query(PM_MESSAGE);
376 Dmsg2(100, "Enter vbackup_cleanup %d %c\n", TermCode, TermCode);
377 memset(&cr, 0, sizeof(cr));
379 jcr->setJobLevel(L_FULL); /* we want this to appear as a Full backup */
380 jcr->jr.JobLevel = L_FULL; /* we want this to appear as a Full backup */
381 jcr->JobFiles = jcr->SDJobFiles;
382 jcr->JobBytes = jcr->SDJobBytes;
383 update_job_end(jcr, TermCode);
385 /* Update final items to set them to the previous job's values */
386 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
387 "JobTDate=%s WHERE JobId=%s",
388 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
389 edit_uint64(jcr->previous_jr.JobTDate, ec1),
390 edit_uint64(jcr->JobId, ec3));
391 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
393 /* Get the fully updated job record */
394 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
395 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
396 db_strerror(jcr->db));
397 jcr->setJobStatus(JS_ErrorTerminated);
400 bstrncpy(cr.Name, jcr->client->name(), sizeof(cr.Name));
401 if (!db_get_client_record(jcr, jcr->db, &cr)) {
402 Jmsg(jcr, M_WARNING, 0, _("Error getting Client record for Job report: ERR=%s"),
403 db_strerror(jcr->db));
406 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
407 if (!db_get_media_record(jcr, jcr->db, &mr)) {
408 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
409 mr.VolumeName, db_strerror(jcr->db));
410 jcr->setJobStatus(JS_ErrorTerminated);
413 update_bootstrap_file(jcr);
415 switch (jcr->JobStatus) {
417 if (jcr->JobErrors || jcr->SDErrors) {
418 term_msg = _("Backup OK -- with warnings");
420 term_msg = _("Backup OK");
424 case JS_ErrorTerminated:
425 term_msg = _("*** Backup Error ***");
426 msg_type = M_ERROR; /* Generate error message */
427 if (jcr->store_bsock) {
428 jcr->store_bsock->signal(BNET_TERMINATE);
429 if (jcr->SD_msg_chan_started) {
430 pthread_cancel(jcr->SD_msg_chan);
435 term_msg = _("Backup Canceled");
436 if (jcr->store_bsock) {
437 jcr->store_bsock->signal(BNET_TERMINATE);
438 if (jcr->SD_msg_chan_started) {
439 pthread_cancel(jcr->SD_msg_chan);
444 term_msg = term_code;
445 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
448 bstrftimes(schedt, sizeof(schedt), jcr->jr.SchedTime);
449 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
450 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
451 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
455 kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime);
456 if (!db_get_job_volume_names(jcr, jcr->db, jcr->jr.JobId, &jcr->VolumeName)) {
458 * Note, if the job has erred, most likely it did not write any
459 * tape, so suppress this "error" message since in that case
460 * it is normal. Or look at it the other way, only for a
461 * normal exit should we complain about this error.
463 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
464 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db));
466 jcr->VolumeName[0] = 0; /* none */
469 if (jcr->ReadBytes == 0) {
470 bstrncpy(compress, "None", sizeof(compress));
472 compression = (double)100 - 100.0 * ((double)jcr->JobBytes / (double)jcr->ReadBytes);
473 if (compression < 0.5) {
474 bstrncpy(compress, "None", sizeof(compress));
476 bsnprintf(compress, sizeof(compress), "%.1f %%", compression);
479 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
481 Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n"
482 " Build OS: %s %s %s\n"
485 " Backup Level: Virtual Full\n"
486 " Client: \"%s\" %s\n"
487 " FileSet: \"%s\" %s\n"
488 " Pool: \"%s\" (From %s)\n"
489 " Catalog: \"%s\" (From %s)\n"
490 " Storage: \"%s\" (From %s)\n"
491 " Scheduled time: %s\n"
494 " Elapsed time: %s\n"
496 " SD Files Written: %s\n"
497 " SD Bytes Written: %s (%sB)\n"
499 " Volume name(s): %s\n"
500 " Volume Session Id: %d\n"
501 " Volume Session Time: %d\n"
502 " Last Volume Bytes: %s (%sB)\n"
504 " SD termination status: %s\n"
505 " Termination: %s\n\n"),
506 BACULA, my_name, VERSION, LSMDATE,
507 HOST_OS, DISTNAME, DISTVER,
510 jcr->client->name(), cr.Uname,
511 jcr->fileset->name(), jcr->FSCreateTime,
512 jcr->pool->name(), jcr->pool_source,
513 jcr->catalog->name(), jcr->catalog_source,
514 jcr->wstore->name(), jcr->wstore_source,
518 edit_utime(RunTime, elapsed, sizeof(elapsed)),
520 edit_uint64_with_commas(jcr->jr.JobFiles, ec1),
521 edit_uint64_with_commas(jcr->jr.JobBytes, ec3),
522 edit_uint64_with_suffix(jcr->jr.JobBytes, ec4),
527 edit_uint64_with_commas(mr.VolBytes, ec7),
528 edit_uint64_with_suffix(mr.VolBytes, ec8),
533 Dmsg0(100, "Leave vbackup_cleanup()\n");
537 * This callback routine is responsible for inserting the
538 * items it gets into the bootstrap structure. For each JobId selected
539 * this routine is called once for each file. We do not allow
540 * duplicate filenames, but instead keep the info from the most
541 * recent file entered (i.e. the JobIds are assumed to be sorted)
543 * See uar_sel_files in sql_cmds.c for query that calls us.
544 * row[0]=Path, row[1]=Filename, row[2]=FileIndex
545 * row[3]=JobId row[4]=LStat
547 int insert_bootstrap_handler(void *ctx, int num_fields, char **row)
551 rblist *bsr_list = (rblist *)ctx;
553 JobId = str_to_int64(row[3]);
554 FileIndex = str_to_int64(row[2]);
555 add_findex(bsr_list, JobId, FileIndex);
560 static bool create_bootstrap_file(JCR *jcr, char *jobids)
566 memset(&rx, 0, sizeof(rx));
567 rx.bsr_list = New(rblist(bsr, &bsr->link));
568 ua = new_ua_context(jcr);
571 #define new_get_file_list
572 #ifdef new_get_file_list
573 if (!db_open_batch_connexion(jcr, jcr->db)) {
574 Jmsg0(jcr, M_FATAL, 0, "Can't get batch sql connexion");
578 if (!db_get_file_list(jcr, jcr->db_batch, jobids, false /* don't use md5 */,
579 true /* use delta */,
580 insert_bootstrap_handler, (void *)rx.bsr_list))
582 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db_batch));
586 JobId_t JobId, last_JobId = 0;
587 rx.query = get_pool_memory(PM_MESSAGE);
588 for (p=rx.JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) {
591 if (JobId == last_JobId) {
592 continue; /* eliminate duplicate JobIds */
596 * Find files for this JobId and insert them in the tree
598 Mmsg(rx.query, uar_sel_files, edit_int64(JobId, ed1));
599 Dmsg1(100, "uar_sel_files=%s\n", rx.query);
600 if (!db_sql_query(ua->db, rx.query, insert_bootstrap_handler, (void *)rx.bsr_list)) {
601 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(ua->db));
603 free_pool_memory(rx.query);
608 complete_bsr(ua, rx.bsr_list);
609 jcr->ExpectedFiles = write_bsr_file(ua, rx);
610 if (chk_dbglvl(10)) {
611 Pmsg1(000, "Found %d files to consolidate.\n", jcr->ExpectedFiles);
613 Jmsg(jcr, M_INFO, 0, _("Found %d files to consolidate into Virtual Full.\n"),
616 free_bsr(rx.bsr_list);
617 return jcr->ExpectedFiles==0?false:true;